require('ts-node/register/transpile-only'); const assert = require('node:assert/strict'); const { test } = require('node:test'); const { MAX_EXECUTOR_HANDLE_LENGTH, MAX_LOG_ARTIFACT_ID_LENGTH, MAX_RUN_ERROR_CODE_LENGTH, MAX_RUN_ERROR_SUMMARY_LENGTH, isTerminalRunAttemptStatus, isTerminalRunStatus, requestRunCancellation, reserveRunEvent, transitionRun, transitionRunAttempt, } = require('../../back/runtime/domain/runStateMachine'); const { InvalidRunAttemptTransitionError, InvalidRunTransitionError, InvalidTransitionMetadataError, InvalidTransitionTimestampError, RunVersionConflictError, } = require('../../back/runtime/domain/stateMachineErrors'); const CREATED_AT_MS = 1_750_000_000_000; function createRun(overrides = {}) { return { id: '019f70c0-0000-7000-8000-000000000001', projectId: 'default', taskId: 'legacy-cron:1', taskRevision: 'revision-1', triggerType: 'manual', executionOrigin: 'manual', executionOwner: 'runtime', status: 'created', version: 0, eventSequence: 0, priority: 0, createdAtMs: CREATED_AT_MS, ...overrides, }; } function createAttempt(runId, overrides = {}) { return { id: '019f70c0-0000-7000-8000-000000000002', runId, attempt: 1, status: 'claimed', executorType: 'legacy_local', callbackSequence: 0, createdAtMs: CREATED_AT_MS + 1, ...overrides, }; } test('drives the successful Run lifecycle while reserving ordered events', () => { const initial = createRun(); const snapshot = structuredClone(initial); const queued = transitionRun(initial, { to: 'queued', expectedVersion: 0, atMs: CREATED_AT_MS + 1, }); const dispatching = transitionRun(queued.run, { to: 'dispatching', expectedVersion: 1, atMs: CREATED_AT_MS + 2, }); const running = transitionRun(dispatching.run, { to: 'running', expectedVersion: 2, atMs: CREATED_AT_MS + 3, }); const succeeded = transitionRun(running.run, { to: 'succeeded', expectedVersion: 3, atMs: CREATED_AT_MS + 4, }); assert.deepEqual(initial, snapshot, 'state decisions must not mutate input'); assert.equal(succeeded.run.status, 'succeeded'); assert.equal(succeeded.run.version, 4); assert.equal(succeeded.run.eventSequence, 4); assert.equal(succeeded.run.queuedAtMs, CREATED_AT_MS + 1); assert.equal(succeeded.run.startedAtMs, CREATED_AT_MS + 3); assert.equal(succeeded.run.finishedAtMs, CREATED_AT_MS + 4); assert.deepEqual( [queued, dispatching, running, succeeded].map((decision) => [ decision.event.sequence, decision.event.type, ]), [ [1, 'run.queued'], [2, 'run.dispatching'], [3, 'run.running'], [4, 'run.succeeded'], ], ); assert.deepEqual(succeeded.event.payload, { from_status: 'running', to_status: 'succeeded', version: 4, }); }); test('supports retry and lost recovery paths and clears stale errors on queueing', () => { const running = createRun({ status: 'running', version: 3, eventSequence: 3, queuedAtMs: CREATED_AT_MS + 1, startedAtMs: CREATED_AT_MS + 3, }); const retryWait = transitionRun(running, { to: 'retry_wait', expectedVersion: 3, atMs: CREATED_AT_MS + 4, errorCode: 'EXECUTOR_BUSY', errorSummary: 'worker capacity is exhausted', }); const requeued = transitionRun(retryWait.run, { to: 'queued', expectedVersion: 4, atMs: CREATED_AT_MS + 10, }); assert.equal(retryWait.run.errorCode, 'EXECUTOR_BUSY'); assert.equal(retryWait.run.errorSummary, 'worker capacity is exhausted'); assert.equal(requeued.run.queuedAtMs, CREATED_AT_MS + 10); assert.equal(requeued.run.errorCode, undefined); assert.equal(requeued.run.errorSummary, undefined); const lost = createRun({ status: 'lost', version: 8, eventSequence: 8, startedAtMs: CREATED_AT_MS + 2, errorCode: 'LEASE_EXPIRED', errorSummary: 'worker lease expired', }); const recovered = transitionRun(lost, { to: 'queued', expectedVersion: 8, atMs: CREATED_AT_MS + 20, }); assert.equal(recovered.run.status, 'queued'); assert.equal(recovered.run.errorCode, undefined); assert.equal(recovered.run.errorSummary, undefined); }); test('rejects illegal Run transitions, stale versions, timestamps, and metadata', () => { const terminal = createRun({ status: 'succeeded', version: 4, eventSequence: 4, startedAtMs: CREATED_AT_MS + 2, finishedAtMs: CREATED_AT_MS + 3, }); assert.throws( () => transitionRun(terminal, { to: 'queued', expectedVersion: 4, atMs: CREATED_AT_MS + 4, }), InvalidRunTransitionError, ); assert.throws( () => transitionRun(createRun({ status: 'running' }), { to: 'created', expectedVersion: 0, atMs: CREATED_AT_MS + 1, }), InvalidRunTransitionError, ); assert.throws( () => transitionRun(createRun(), { to: 'queued', expectedVersion: 7, atMs: CREATED_AT_MS + 1, }), (error) => { assert.ok(error instanceof RunVersionConflictError); assert.equal(error.code, 'RUN_VERSION_CONFLICT'); assert.equal(error.expectedVersion, 7); assert.equal(error.actualVersion, 0); return true; }, ); assert.throws( () => transitionRun(createRun(), { to: 'queued', expectedVersion: 0, atMs: CREATED_AT_MS - 1, }), InvalidTransitionTimestampError, ); assert.throws( () => transitionRun( createRun({ status: 'running', startedAtMs: CREATED_AT_MS + 10, }), { to: 'succeeded', expectedVersion: 0, atMs: CREATED_AT_MS + 9, }, ), InvalidTransitionTimestampError, ); assert.throws( () => transitionRun(createRun({ status: 'running' }), { to: 'succeeded', expectedVersion: 0, atMs: CREATED_AT_MS + 1, errorCode: 'NOT_ALLOWED', }), InvalidTransitionMetadataError, ); assert.throws( () => transitionRun(createRun({ status: 'running' }), { to: 'failed', expectedVersion: 0, atMs: CREATED_AT_MS + 1, errorCode: 'x'.repeat(MAX_RUN_ERROR_CODE_LENGTH + 1), }), InvalidTransitionMetadataError, ); assert.throws( () => transitionRun(createRun({ status: 'running' }), { to: 'failed', expectedVersion: 0, atMs: CREATED_AT_MS + 1, errorSummary: 'x'.repeat(MAX_RUN_ERROR_SUMMARY_LENGTH + 1), }), InvalidTransitionMetadataError, ); }); test('drives Attempt state without mutating inputs and reserves Run events', () => { let run = createRun({ status: 'dispatching', version: 2, eventSequence: 2, }); let attempt = createAttempt(run.id); const initialRun = structuredClone(run); const initialAttempt = structuredClone(attempt); const starting = transitionRunAttempt(run, attempt, { to: 'starting', expectedRunVersion: 2, atMs: CREATED_AT_MS + 2, }); run = starting.run; attempt = starting.attempt; const running = transitionRunAttempt(run, attempt, { to: 'running', expectedRunVersion: 3, atMs: CREATED_AT_MS + 3, }); run = running.run; attempt = running.attempt; const failed = transitionRunAttempt(run, attempt, { to: 'failed', expectedRunVersion: 4, atMs: CREATED_AT_MS + 4, exitCode: 1, errorCode: 'EXIT_NON_ZERO', errorSummary: 'process exited with code 1', }); assert.deepEqual( initialRun, createRun({ status: 'dispatching', version: 2, eventSequence: 2, }), ); assert.deepEqual(initialAttempt, createAttempt(initialRun.id)); assert.equal(failed.run.status, 'dispatching'); assert.equal(failed.run.version, 5); assert.equal(failed.run.eventSequence, 5); assert.equal(failed.attempt.status, 'failed'); assert.equal(failed.attempt.startedAtMs, CREATED_AT_MS + 3); assert.equal(failed.attempt.finishedAtMs, CREATED_AT_MS + 4); assert.equal(failed.attempt.exitCode, 1); assert.equal(failed.attempt.errorCode, 'EXIT_NON_ZERO'); assert.deepEqual( [starting, running, failed].map((decision) => [ decision.event.sequence, decision.event.type, ]), [ [3, 'attempt.starting'], [4, 'attempt.running'], [5, 'attempt.failed'], ], ); assert.deepEqual(failed.event.payload, { attempt_id: initialAttempt.id, attempt: 1, from_status: 'running', to_status: 'failed', version: 5, exit_code: 1, error_code: 'EXIT_NON_ZERO', }); }); test('rejects invalid Attempt transitions and inconsistent aggregate metadata', () => { const run = createRun({ status: 'dispatching' }); const attempt = createAttempt(run.id); assert.throws( () => transitionRunAttempt(run, attempt, { to: 'succeeded', expectedRunVersion: 0, atMs: CREATED_AT_MS + 2, }), InvalidRunAttemptTransitionError, ); assert.throws( () => transitionRunAttempt(run, createAttempt(run.id, { status: 'failed' }), { to: 'succeeded', expectedRunVersion: 0, atMs: CREATED_AT_MS + 2, }), InvalidRunAttemptTransitionError, ); assert.throws( () => transitionRunAttempt(run, createAttempt('another-run'), { to: 'starting', expectedRunVersion: 0, atMs: CREATED_AT_MS + 2, }), InvalidTransitionMetadataError, ); assert.throws( () => transitionRunAttempt(run, attempt, { to: 'starting', expectedRunVersion: 0, atMs: CREATED_AT_MS + 2, exitCode: 0, }), InvalidTransitionMetadataError, ); assert.throws( () => transitionRunAttempt(createRun({ status: 'succeeded' }), attempt, { to: 'starting', expectedRunVersion: 0, atMs: CREATED_AT_MS + 2, }), InvalidTransitionMetadataError, ); assert.throws( () => transitionRunAttempt(run, attempt, { to: 'starting', expectedRunVersion: 1, atMs: CREATED_AT_MS + 2, }), RunVersionConflictError, ); }); test('accepts bounded execution metadata only while an Attempt starts or runs', () => { const run = createRun({ status: 'dispatching' }); const attempt = createAttempt(run.id); const starting = transitionRunAttempt(run, attempt, { to: 'starting', expectedRunVersion: 0, atMs: CREATED_AT_MS + 2, executorHandle: 'legacy-local:321', pid: 321, logArtifactId: 'legacy-log-1', deadlineAtMs: CREATED_AT_MS + 30_000, }); assert.equal(starting.attempt.executorHandle, 'legacy-local:321'); assert.equal(starting.attempt.pid, 321); assert.equal(starting.attempt.logArtifactId, 'legacy-log-1'); assert.equal(starting.attempt.deadlineAtMs, CREATED_AT_MS + 30_000); assert.equal(starting.event.payload.deadline_at_ms, CREATED_AT_MS + 30_000); for (const command of [ { executorHandle: '' }, { executorHandle: 'x'.repeat(MAX_EXECUTOR_HANDLE_LENGTH + 1) }, { pid: 0 }, { pid: Number.MAX_SAFE_INTEGER + 1 }, { logArtifactId: '' }, { logArtifactId: 'x'.repeat(MAX_LOG_ARTIFACT_ID_LENGTH + 1) }, { deadlineAtMs: CREATED_AT_MS + 2 }, { deadlineAtMs: Number.MAX_SAFE_INTEGER + 1 }, ]) { assert.throws( () => transitionRunAttempt(run, attempt, { to: 'starting', expectedRunVersion: 0, atMs: CREATED_AT_MS + 2, ...command, }), InvalidTransitionMetadataError, ); } assert.throws( () => transitionRunAttempt( run, createAttempt(run.id, { status: 'running', startedAtMs: CREATED_AT_MS + 2, }), { to: 'succeeded', expectedRunVersion: 0, atMs: CREATED_AT_MS + 3, executorHandle: 'late-handle', }, ), InvalidTransitionMetadataError, ); assert.throws( () => transitionRunAttempt( run, createAttempt(run.id, { deadlineAtMs: CREATED_AT_MS + 20_000, }), { to: 'starting', expectedRunVersion: 0, atMs: CREATED_AT_MS + 2, deadlineAtMs: CREATED_AT_MS + 30_000, }, ), /cannot replace/, ); assert.throws( () => transitionRunAttempt( run, createAttempt(run.id, { status: 'starting', startedAtMs: CREATED_AT_MS + 2, }), { to: 'running', expectedRunVersion: 0, atMs: CREATED_AT_MS + 3, deadlineAtMs: CREATED_AT_MS + 30_000, }, ), /only allowed when an Attempt starts/, ); }); test('persists one immutable callback token hash and advances callback sequence once', () => { const currentRun = createRun({ status: 'dispatching', version: 3, eventSequence: 3, }); const currentAttempt = createAttempt(currentRun.id, { status: 'claimed' }); const tokenHash = 'a'.repeat(64); const starting = transitionRunAttempt(currentRun, currentAttempt, { to: 'starting', expectedRunVersion: 3, atMs: CREATED_AT_MS + 10, callbackTokenHash: tokenHash, }); assert.equal(starting.attempt.callbackTokenHash, tokenHash); const running = transitionRunAttempt( starting.run, { ...starting.attempt, status: 'running', startedAtMs: CREATED_AT_MS + 11, }, { to: 'succeeded', expectedRunVersion: starting.run.version, atMs: CREATED_AT_MS + 20, callbackSequence: 1, exitCode: 0, }, ); assert.equal(running.attempt.callbackSequence, 1); assert.equal(running.event.payload.callback_sequence, 1); assert.throws( () => transitionRunAttempt(currentRun, currentAttempt, { to: 'starting', expectedRunVersion: 3, atMs: CREATED_AT_MS + 10, callbackTokenHash: 'not-a-hash', }), InvalidTransitionMetadataError, ); assert.throws( () => transitionRunAttempt( starting.run, { ...starting.attempt, status: 'running', startedAtMs: CREATED_AT_MS + 11, }, { to: 'failed', expectedRunVersion: starting.run.version, atMs: CREATED_AT_MS + 20, callbackSequence: 2, }, ), InvalidTransitionMetadataError, ); }); test('reserves standalone aggregate events without changing status or input', () => { const run = createRun({ status: 'waiting_approval', version: 7, eventSequence: 11, }); const snapshot = structuredClone(run); const reserved = reserveRunEvent(run, 7); assert.deepEqual(run, snapshot); assert.equal(reserved.sequence, 12); assert.equal(reserved.run.status, 'waiting_approval'); assert.equal(reserved.run.version, 8); assert.equal(reserved.run.eventSequence, 12); }); test('reserves one durable cancellation request without changing Run status', () => { const run = createRun({ status: 'running', version: 3, eventSequence: 3, startedAtMs: CREATED_AT_MS + 3, }); const snapshot = structuredClone(run); const accepted = requestRunCancellation(run, { expectedVersion: 3, atMs: CREATED_AT_MS + 4, reason: 'user', }); assert.equal(accepted.status, 'accepted'); assert.deepEqual(run, snapshot); assert.equal(accepted.run.status, 'running'); assert.equal(accepted.run.version, 4); assert.equal(accepted.run.eventSequence, 4); assert.equal(accepted.run.cancelRequestedAtMs, CREATED_AT_MS + 4); assert.equal(accepted.run.cancelReason, 'user'); assert.deepEqual(accepted.event, { sequence: 4, type: 'run.cancel_requested', payload: { status: 'running', reason: 'user', requested_at_ms: CREATED_AT_MS + 4, version: 4, }, }); const duplicate = requestRunCancellation(accepted.run, { expectedVersion: 4, atMs: CREATED_AT_MS + 5, reason: 'policy', }); assert.equal(duplicate.status, 'already_requested'); assert.equal(duplicate.run.version, 4); assert.equal(duplicate.run.cancelReason, 'user'); }); test('does not reserve cancellation events for terminal Runs or invalid input', () => { const terminal = createRun({ status: 'succeeded', version: 4, eventSequence: 4, startedAtMs: CREATED_AT_MS + 2, finishedAtMs: CREATED_AT_MS + 3, }); const decision = requestRunCancellation(terminal, { expectedVersion: 4, atMs: CREATED_AT_MS + 4, reason: 'user', }); assert.equal(decision.status, 'already_terminal'); assert.equal(decision.run.version, 4); assert.throws( () => requestRunCancellation(createRun({ status: 'running' }), { expectedVersion: 0, atMs: CREATED_AT_MS - 1, reason: 'user', }), InvalidTransitionTimestampError, ); assert.throws( () => requestRunCancellation(createRun({ status: 'running' }), { expectedVersion: 0, atMs: CREATED_AT_MS + 1, reason: 'unbounded-raw-reason', }), InvalidTransitionMetadataError, ); }); test('reports terminal Run and Attempt statuses explicitly', () => { assert.equal(isTerminalRunStatus('succeeded'), true); assert.equal(isTerminalRunStatus('lost'), false); assert.equal(isTerminalRunAttemptStatus('lost'), true); assert.equal(isTerminalRunAttemptStatus('running'), false); }); test('allows a dispatching Run to time out before execution starts', () => { const dispatching = createRun({ status: 'dispatching', version: 2, eventSequence: 2, queuedAtMs: CREATED_AT_MS + 1, }); const decision = transitionRun(dispatching, { to: 'timed_out', expectedVersion: 2, atMs: CREATED_AT_MS + 3, errorCode: 'EXECUTION_TIMED_OUT', errorSummary: 'Execution timed out before start', }); assert.equal(decision.run.status, 'timed_out'); assert.equal(decision.run.finishedAtMs, CREATED_AT_MS + 3); assert.equal(decision.event.type, 'run.timed_out'); });