import { v7 as uuidV7 } from 'uuid'; import type { RunAttemptRecord, RunEventRecord, RunRecord, } from '../domain/run'; import { assertRunRetryPolicyRecord, runRetryDelayMs, type RunRetryPolicyRecord, } from '../domain/runRetryPolicy'; import { reserveRunEvent, transitionRun, type RunDomainEventDraft, } from '../domain/runStateMachine'; import { RunVersionConflictError } from '../domain/stateMachineErrors'; import type { RunRepository, RunRepositoryTransaction, } from '../ports/runRepository'; export type RunLostRetryStatus = | 'scheduled' | 'requeued' | 'cancelled' | 'failed_disabled' | 'failed_unsafe' | 'failed_exhausted' | 'not_due' | 'not_eligible' | 'not_found'; export interface RunLostRetryResult { status: RunLostRetryStatus; run?: RunRecord; attempt?: RunAttemptRecord; policy?: RunRetryPolicyRecord; events?: readonly RunEventRecord[]; } export class RunLostRetryTargetError extends Error { readonly code = 'RUN_LOST_RETRY_TARGET_INCONSISTENT'; constructor(message: string) { super(`Run lost retry target is inconsistent: ${message}`); this.name = 'RunLostRetryTargetError'; } } export class RunLostRetryService { private readonly clock: { now(): number }; private readonly createId: () => string; constructor( private readonly repository: RunRepository, options: { clock?: { now(): number }; createId?: () => string } = {}, ) { this.clock = options.clock ?? Date; this.createId = options.createId ?? uuidV7; } reconcile(runId: string): Promise { if (!runId) throw new TypeError('runId is required'); const observedAtMs = this.clock.now(); if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) { throw new RangeError('observedAtMs must be a non-negative safe integer'); } return this.repository.transaction((transaction) => this.reconcileInTransaction(transaction, runId, observedAtMs), ); } private async reconcileInTransaction( transaction: RunRepositoryTransaction, runId: string, observedAtMs: number, ): Promise { const run = await transaction.findRunById(runId); if (!run) return { status: 'not_found' }; if ( run.executionOwner !== 'runtime' || (run.status !== 'lost' && run.status !== 'retry_wait') ) { return { status: 'not_eligible', run }; } const [attempt, policy] = await Promise.all([ transaction.findLatestAttemptByRunId(run.id), transaction.findRetryPolicyByRunId(run.id), ]); if (!attempt || attempt.runId !== run.id || attempt.status !== 'lost') { throw new RunLostRetryTargetError('latest Attempt is not lost'); } if (policy) assertRunRetryPolicyRecord(policy); const atMs = Math.max( observedAtMs, run.createdAtMs, run.startedAtMs ?? 0, attempt.createdAtMs, attempt.startedAtMs ?? 0, attempt.finishedAtMs ?? 0, policy?.updatedAtMs ?? 0, ); if (run.cancelRequestedAtMs !== undefined) { return this.finishRun( transaction, run, attempt, policy, atMs, 'cancelled', 'cancelled', 'RUN_CANCELLED_DURING_LOST_RECOVERY', 'Cancellation won before a replacement Attempt was created', ); } if (!policy || !policy.retryOnLost || policy.maxAttempts <= 1) { return this.finishRun( transaction, run, attempt, policy, atMs, 'failed_disabled', 'failed', 'RUN_LOST_RETRY_DISABLED', 'Run was lost and automatic retry was not enabled at admission', ); } if (policy.safety === 'unknown') { return this.finishRun( transaction, run, attempt, policy, atMs, 'failed_unsafe', 'failed', 'RUN_LOST_RETRY_UNSAFE', 'Run was lost but execution safety was not declared', ); } if (attempt.attempt >= policy.maxAttempts) { return this.finishRun( transaction, run, attempt, policy, atMs, 'failed_exhausted', 'failed', 'RUN_LOST_RETRY_EXHAUSTED', 'Run exhausted its admitted automatic retry attempts', ); } if (run.status === 'lost') { return this.schedule(transaction, run, attempt, policy, atMs); } if ( policy.nextAttemptAtMs === undefined || policy.nextAttemptAtMs > observedAtMs ) { return { status: 'not_due', run, attempt, policy }; } return this.requeue(transaction, run, attempt, policy, atMs); } private async schedule( transaction: RunRepositoryTransaction, run: RunRecord, attempt: RunAttemptRecord, policy: RunRetryPolicyRecord, atMs: number, ): Promise { const lostAtMs = Math.max( run.createdAtMs, attempt.createdAtMs, attempt.finishedAtMs ?? 0, ); const nextAttemptAtMs = lostAtMs + runRetryDelayMs(policy, attempt.attempt); if (!Number.isSafeInteger(nextAttemptAtMs)) { throw new RunLostRetryTargetError('next Attempt time overflowed'); } const decision = transitionRun(run, { to: 'retry_wait', expectedVersion: run.version, atMs, errorCode: 'RUN_LOST_RETRY_SCHEDULED', errorSummary: 'A fresh Attempt will be created after the admitted backoff', }); const nextPolicy: RunRetryPolicyRecord = { ...policy, nextAttemptAtMs, version: policy.version + 1, updatedAtMs: atMs, }; if ( !(await transaction.compareAndSetRun(decision.run, run.version)) || !(await transaction.compareAndSetRetryPolicy(nextPolicy, policy.version)) ) { throw new RunVersionConflictError(run.id, run.version, run.version); } const event = this.event( decision.run, decision.event, attempt.id, `run-lost-retry-scheduled:${attempt.id}`, atMs, { attempt: attempt.attempt, max_attempts: policy.maxAttempts, safety: policy.safety, next_attempt_at_ms: nextAttemptAtMs, }, ); await transaction.appendEvent(event); return { status: 'scheduled', run: decision.run, attempt, policy: nextPolicy, events: [event], }; } private async requeue( transaction: RunRepositoryTransaction, run: RunRecord, lostAttempt: RunAttemptRecord, policy: RunRetryPolicyRecord, atMs: number, ): Promise { const attempt: RunAttemptRecord = { id: this.createId(), runId: run.id, attempt: lostAttempt.attempt + 1, status: 'claimed', executorType: lostAttempt.executorType, callbackSequence: 0, createdAtMs: atMs, }; const queued = transitionRun(run, { to: 'queued', expectedVersion: run.version, atMs, }); const nextPolicy: RunRetryPolicyRecord = { ...policy, version: policy.version + 1, updatedAtMs: atMs, }; delete nextPolicy.nextAttemptAtMs; if ( !(await transaction.compareAndSetRun(queued.run, run.version)) || !(await transaction.compareAndSetRetryPolicy(nextPolicy, policy.version)) ) { throw new RunVersionConflictError(run.id, run.version, run.version); } await transaction.insertAttempt(attempt); const queuedEvent = this.event( queued.run, queued.event, attempt.id, `run-lost-retry-queued:${attempt.id}`, atMs, { previous_attempt_id: lostAttempt.id }, ); await transaction.appendEvent(queuedEvent); const claimed = reserveRunEvent(queued.run, queued.run.version); if ( !(await transaction.compareAndSetRun(claimed.run, queued.run.version)) ) { throw new RunVersionConflictError( run.id, queued.run.version, queued.run.version, ); } const claimedEvent = this.event( claimed.run, { sequence: claimed.sequence, type: 'attempt.claimed', payload: { attempt: attempt.attempt, executor_type: attempt.executorType, version: claimed.run.version, }, }, attempt.id, `run-lost-retry-attempt-claimed:${attempt.id}`, atMs, { previous_attempt_id: lostAttempt.id }, ); await transaction.appendEvent(claimedEvent); return { status: 'requeued', run: claimed.run, attempt, policy: nextPolicy, events: [queuedEvent, claimedEvent], }; } private async finishRun( transaction: RunRepositoryTransaction, run: RunRecord, attempt: RunAttemptRecord, policy: RunRetryPolicyRecord | null, atMs: number, status: | 'cancelled' | 'failed_disabled' | 'failed_unsafe' | 'failed_exhausted', to: 'cancelled' | 'failed', errorCode: string, errorSummary: string, ): Promise { const decision = transitionRun(run, { to, expectedVersion: run.version, atMs, errorCode, errorSummary, }); if (!(await transaction.compareAndSetRun(decision.run, run.version))) { throw new RunVersionConflictError(run.id, run.version, run.version); } let nextPolicy = policy ?? undefined; if (policy?.nextAttemptAtMs !== undefined) { nextPolicy = { ...policy, version: policy.version + 1, updatedAtMs: atMs, }; delete nextPolicy.nextAttemptAtMs; if ( !(await transaction.compareAndSetRetryPolicy( nextPolicy, policy.version, )) ) { throw new RunLostRetryTargetError('retry policy update lost its race'); } } const event = this.event( decision.run, decision.event, attempt.id, `run-lost-retry-${status}:${attempt.id}`, atMs, { attempt: attempt.attempt }, ); await transaction.appendEvent(event); return { status, run: decision.run, attempt, ...(nextPolicy === undefined ? {} : { policy: nextPolicy }), events: [event], }; } private event( run: RunRecord, draft: RunDomainEventDraft, attemptId: string, dedupeKey: string, createdAtMs: number, extraPayload: Readonly>, ): RunEventRecord { return { id: this.createId(), runId: run.id, sequence: draft.sequence, type: draft.type, dedupeKey, actorType: 'reconciler', attemptId, payload: { ...draft.payload, ...extraPayload }, createdAtMs, }; } }