import { v7 as uuidV7 } from 'uuid'; import type { RunAttemptRecord, RunRecord, RunStatus } from '../domain/run'; import { RUN_TRANSITIONS, isTerminalRunAttemptStatus, isTerminalRunStatus, } from '../domain/runStateMachine'; import type { LegacyRunningInstanceEvidence, LegacyShadowStartupAttempt, LegacyShadowStartupCandidate, LegacyShadowStartupCursor, LegacyShadowStartupRecoverySource, } from '../ports/legacyShadowStartupRecovery'; import { MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE } from '../ports/legacyShadowStartupRecovery'; import type { RunRepository } from '../ports/runRepository'; import type { LegacyShadowRunWriter } from './legacyShadowRunWriter'; import { RunCommandService } from './runCommandService'; export const MAX_LEGACY_SHADOW_STARTUP_PAGES = 64; export interface LegacyShadowStartupClock { now(): number; } export interface LegacyShadowStartupReconcileSummary { scanned: number; completed: number; cancelled: number; abandoned: number; markedLost: number; repaired: number; pending: number; ambiguous: number; skipped: number; failed: number; truncated: boolean; nextCursor?: LegacyShadowStartupCursor; } export type LegacyShadowStartupStopReason = | 'complete' | 'page_limit' | 'cursor_stalled'; export interface LegacyShadowStartupSummary extends Omit< LegacyShadowStartupReconcileSummary, 'truncated' | 'nextCursor' > { pages: number; stopReason: LegacyShadowStartupStopReason; remaining: boolean; nextCursor?: LegacyShadowStartupCursor; } type EvidenceSelection = | { status: 'matched'; evidence: LegacyRunningInstanceEvidence } | { status: 'none' | 'ambiguous' }; function sameCursor( left: LegacyShadowStartupCursor | undefined, right: LegacyShadowStartupCursor, ): boolean { return ( left !== undefined && left.createdAtMs === right.createdAtMs && left.runId === right.runId ); } function selectEvidence( attempt: Pick, evidence: readonly LegacyRunningInstanceEvidence[], ): EvidenceSelection { const byLog = attempt.logArtifactId === undefined ? [] : evidence.filter( (candidate) => candidate.logArtifactId === attempt.logArtifactId, ); const byPid = attempt.pid === undefined ? [] : evidence.filter((candidate) => candidate.pid === attempt.pid); if (attempt.logArtifactId !== undefined && attempt.pid !== undefined) { const byPidSet = new Set(byPid); const intersection = byLog.filter((candidate) => byPidSet.has(candidate)); if (intersection.length === 1) { return { status: 'matched', evidence: intersection[0] }; } if ( intersection.length > 1 || byLog.length > 1 || byPid.length > 1 || (byLog.length > 0 && byPid.length > 0) ) { return { status: 'ambiguous' }; } if (byLog.length === 1) { return { status: 'matched', evidence: byLog[0] }; } if (byPid.length === 1) { return { status: 'matched', evidence: byPid[0] }; } return { status: 'none' }; } if (attempt.logArtifactId !== undefined) { if (byLog.length === 1) { return { status: 'matched', evidence: byLog[0] }; } return { status: byLog.length > 1 ? 'ambiguous' : 'none' }; } if (attempt.pid !== undefined) { if (byPid.length === 1) { return { status: 'matched', evidence: byPid[0] }; } return { status: byPid.length > 1 ? 'ambiguous' : 'none' }; } if (evidence.length === 1) { return { status: 'matched', evidence: evidence[0] }; } return { status: evidence.length > 1 ? 'ambiguous' : 'none' }; } function terminalRunTarget(attempt: RunAttemptRecord): RunStatus | undefined { if (attempt.status === 'succeeded') return 'succeeded'; if (attempt.status === 'failed') return 'failed'; if (attempt.status === 'cancelled') return 'cancelled'; if (attempt.status === 'timed_out') return 'timed_out'; if (attempt.status === 'lost') return 'lost'; return undefined; } /** Reconciles one bounded page after Legacy startup normalization. */ export class LegacyShadowStartupReconciler { private readonly commands: RunCommandService; private readonly clock: LegacyShadowStartupClock; constructor( private readonly repository: RunRepository, private readonly source: LegacyShadowStartupRecoverySource, private readonly writer: Pick< LegacyShadowRunWriter, 'exited' | 'cancelled' >, options: { clock?: LegacyShadowStartupClock; createEventId?: () => string; } = {}, ) { this.clock = options.clock ?? { now: Date.now }; this.commands = new RunCommandService( repository, options.createEventId ?? uuidV7, ); } async reconcileBatch(options: { origins: readonly RunRecord['executionOrigin'][]; cursor?: LegacyShadowStartupCursor; limit?: number; }): Promise { const enabledOrigins = [...new Set(options.origins)]; const page = await this.source.listCandidates({ origins: enabledOrigins, ...(options.cursor === undefined ? {} : { cursor: options.cursor }), ...(options.limit === undefined ? {} : { limit: options.limit }), }); const summary: LegacyShadowStartupReconcileSummary = { scanned: page.candidates.length, completed: 0, cancelled: 0, abandoned: 0, markedLost: 0, repaired: 0, pending: 0, ambiguous: 0, skipped: 0, failed: 0, truncated: page.truncated, ...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }), }; const origins = new Set(enabledOrigins); for (const candidate of page.candidates) { try { await this.reconcileCandidate(candidate, origins, summary); } catch { summary.failed += 1; } } return summary; } private async reconcileCandidate( candidate: LegacyShadowStartupCandidate, origins: ReadonlySet, summary: LegacyShadowStartupReconcileSummary, ): Promise { const run = await this.repository.findRunById(candidate.runId); if ( !run || run.executionOwner !== 'legacy' || !origins.has(run.executionOrigin) || isTerminalRunStatus(run.status) || run.status === 'lost' ) { summary.skipped += 1; return; } if (candidate.activeAttemptCount === 0) { await this.repairTerminalAttempt(run, summary); return; } if (candidate.activeAttemptCount !== 1 || !candidate.attempt) { summary.ambiguous += 1; return; } const attempt = await this.repository.findAttemptById( candidate.attempt.attemptId, ); if ( !attempt || attempt.runId !== run.id || isTerminalRunAttemptStatus(attempt.status) ) { summary.skipped += 1; return; } const evidence = candidate.legacyCronId === undefined ? { evidence: [], truncated: false } : await this.source.listRunningInstanceEvidence({ legacyCronId: candidate.legacyCronId, }); if (evidence.truncated) { summary.ambiguous += 1; return; } const selected = selectEvidence(attempt, evidence.evidence); if (selected.status === 'ambiguous') { summary.ambiguous += 1; return; } if ( selected.status === 'matched' && selected.evidence.finishedAtMs !== undefined ) { const atMs = this.atOrAfter(run, attempt, selected.evidence.finishedAtMs); if (selected.evidence.outcome === 'stopped') { await this.writer.cancelled( { runId: run.id, attemptId: attempt.id }, { atMs, reason: 'reconcile' }, ); summary.cancelled += 1; return; } if ( selected.evidence.outcome === 'succeeded' || selected.evidence.outcome === 'failed' ) { const succeeded = selected.evidence.outcome === 'succeeded'; await this.writer.exited( { runId: run.id, attemptId: attempt.id }, { atMs, exitCode: succeeded ? 0 : selected.evidence.exitCode === undefined || selected.evidence.exitCode === 0 ? 1 : selected.evidence.exitCode, }, ); summary.completed += 1; return; } } if (run.executionOrigin === 'scheduled_system') { summary.pending += 1; return; } if (run.status === 'queued' && attempt.status === 'claimed') { await this.abandon(run, attempt); summary.abandoned += 1; return; } if ( (run.status === 'dispatching' || run.status === 'running') && ['claimed', 'starting', 'running'].includes(attempt.status) ) { await this.markLost(run, attempt); summary.markedLost += 1; return; } summary.ambiguous += 1; } private async repairTerminalAttempt( run: RunRecord, summary: LegacyShadowStartupReconcileSummary, ): Promise { const attempt = await this.repository.findLatestAttemptByRunId(run.id); const target = attempt ? terminalRunTarget(attempt) : undefined; if ( !attempt || target === undefined || !RUN_TRANSITIONS[run.status].includes(target) ) { summary.ambiguous += 1; return; } await this.commands.transitionRun({ runId: run.id, to: target, expectedVersion: run.version, atMs: this.atOrAfter(run, attempt), ...(attempt.errorCode === undefined ? {} : { errorCode: attempt.errorCode }), ...(attempt.errorSummary === undefined ? {} : { errorSummary: attempt.errorSummary }), actor: { type: 'reconciler' }, dedupeKey: `legacy-startup-repair:${attempt.id}:${target}`, }); summary.repaired += 1; } private async abandon( run: RunRecord, attempt: RunAttemptRecord, ): Promise { const atMs = this.atOrAfter(run, attempt); const attemptResult = await this.commands.transitionRunAttempt({ runId: run.id, attemptId: attempt.id, to: 'cancelled', expectedRunVersion: run.version, atMs, errorCode: 'LEGACY_RECONCILE_ACCEPTANCE_ABANDONED', errorSummary: 'Legacy owner restarted before spawn was observed', actor: { type: 'reconciler' }, dedupeKey: `legacy-startup-abandoned-attempt:${attempt.id}`, }); await this.commands.transitionRun({ runId: run.id, to: 'cancelled', expectedVersion: attemptResult.run.version, atMs, errorCode: 'LEGACY_RECONCILE_ACCEPTANCE_ABANDONED', errorSummary: 'Legacy owner restarted before spawn was observed', actor: { type: 'reconciler' }, dedupeKey: `legacy-startup-abandoned-run:${attempt.id}`, }); } private async markLost( run: RunRecord, attempt: RunAttemptRecord, ): Promise { const atMs = this.atOrAfter(run, attempt); const attemptResult = await this.commands.transitionRunAttempt({ runId: run.id, attemptId: attempt.id, to: 'lost', expectedRunVersion: run.version, atMs, errorCode: 'LEGACY_RECONCILE_OWNER_LOST', errorSummary: 'Legacy worker restarted without terminal evidence', actor: { type: 'reconciler' }, dedupeKey: `legacy-startup-lost-attempt:${attempt.id}`, }); await this.commands.transitionRun({ runId: run.id, to: 'lost', expectedVersion: attemptResult.run.version, atMs, errorCode: 'LEGACY_RECONCILE_OWNER_LOST', errorSummary: 'Legacy worker restarted without terminal evidence', actor: { type: 'reconciler' }, dedupeKey: `legacy-startup-lost-run:${attempt.id}`, }); } private atOrAfter( run: RunRecord, attempt: RunAttemptRecord, evidenceAtMs?: number, ): number { return Math.max( this.clock.now(), run.createdAtMs, run.startedAtMs ?? 0, attempt.createdAtMs, attempt.startedAtMs ?? 0, evidenceAtMs ?? 0, ); } } /** Runs a complete but Profile-bounded, timer-free startup pass. */ export class LegacyShadowStartupSupervisor { constructor( private readonly reconciler: Pick< LegacyShadowStartupReconciler, 'reconcileBatch' >, ) {} async run(options: { origins: readonly RunRecord['executionOrigin'][]; cursor?: LegacyShadowStartupCursor; pageSize?: number; maxPages?: number; }): Promise { const pageSize = options.pageSize ?? 8; const maxPages = options.maxPages ?? 1; if ( !Number.isSafeInteger(pageSize) || pageSize < 1 || pageSize > MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE ) { throw new RangeError( 'pageSize must be between 1 and MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE', ); } if ( !Number.isSafeInteger(maxPages) || maxPages < 1 || maxPages > MAX_LEGACY_SHADOW_STARTUP_PAGES ) { throw new RangeError( 'maxPages must be between 1 and MAX_LEGACY_SHADOW_STARTUP_PAGES', ); } const total: LegacyShadowStartupSummary = { pages: 0, scanned: 0, completed: 0, cancelled: 0, abandoned: 0, markedLost: 0, repaired: 0, pending: 0, ambiguous: 0, skipped: 0, failed: 0, stopReason: 'complete', remaining: false, }; let cursor = options.cursor; for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) { const page = await this.reconciler.reconcileBatch({ origins: options.origins, ...(cursor === undefined ? {} : { cursor }), limit: pageSize, }); total.pages += 1; total.scanned += page.scanned; total.completed += page.completed; total.cancelled += page.cancelled; total.abandoned += page.abandoned; total.markedLost += page.markedLost; total.repaired += page.repaired; total.pending += page.pending; total.ambiguous += page.ambiguous; total.skipped += page.skipped; total.failed += page.failed; if (!page.truncated) return total; if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) { total.stopReason = 'cursor_stalled'; total.remaining = true; return total; } cursor = page.nextCursor; if (pageNumber === maxPages - 1) { total.stopReason = 'page_limit'; total.remaining = true; total.nextCursor = cursor; return total; } } return total; } }