import { QueryTypes, Sequelize, Transaction } from 'sequelize'; import { APPROVED_ACTION_DISPATCH_TABLE } from '../../../migrations/0020-approval-requests'; import { APPROVED_ACTION_DISPATCH_EXECUTION_TABLE } from '../../../migrations/0021-approved-action-dispatch-executions'; import { APPROVED_ACTION_RECOVERY_CONTROL_TABLE, APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE, } from '../../../migrations/0022-approved-action-recovery'; import { APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE } from '../../../migrations/0024-approved-action-recovery-authorization'; import { PROJECT_ROLE_BINDING_TABLE, PROJECT_TABLE, } from '../../../migrations/0017-project-policy'; import { ApprovedActionDispatchBindingConflictError, assertApprovedActionLeaseIdentity, assertApprovedActionResultCode, normalizeApprovedActionDispatchExecutionRecord, type ApprovedActionDispatchExecutionRecord, type ApprovedActionDispatchExecutionSnapshot, } from '../../domain/approvedActionDispatchExecution'; import { ApprovedActionRecoveryBindingConflictError, ApprovedActionRecoveryFenceRejectedError, ApprovedActionRecoveryRepositoryError, MAX_APPROVED_ACTION_RECOVERY_FINDINGS, assertApprovedActionEvidenceDigest, assertApprovedActionRecoveryLeaseDuration, assertApprovedActionRecoveryPageSize, normalizeApprovedActionRecoveryControlRecord, normalizeApprovedActionRecoveryCursor, normalizeApprovedActionRecoveryResolutionRecord, type ApprovedActionRecoveryControlRecord, type ApprovedActionRecoveryResolutionRecord, type ApprovedActionRecoverySnapshot, } from '../../domain/approvedActionRecovery'; import { normalizeApprovedActionRecoveryAuthorizationFact, type ApprovedActionRecoveryAuthorizationFact, } from '../../domain/approvedActionRecoveryAuthorization'; import { InvalidApprovalValueError, assertApprovalMutationId, assertApprovalTimestamp, normalizeApprovedActionDispatchRecord, type ApprovedActionDispatchRecord, } from '../../domain/approvalRequest'; import { normalizePolicySubject } from '../../domain/projectPolicy'; import type { ApprovedActionRecoveryRepository, ClaimApprovedActionRecoveryCommand, ClaimApprovedActionRecoveryResult, ListDueApprovedActionRecoveriesQuery, ListDueApprovedActionRecoveriesResult, RecordApprovedActionRecoveryFindingCommand, ResolveApprovedActionRecoveryCommand, ResolveApprovedActionRecoveryResult, } from '../../ports/approvedActionRecoveryRepository'; const RETRY_ATTEMPTS = 5; interface RecoveryRow { dispatch_id: string; approval_request_id: string; approval_request_version: number | string; dispatch_project_id: string; dispatch_state: string; permission: string; action_type: string; action_ref: string; action_digest: string; preview_digest: string; requested_by_type: string; requested_by_id: string; consumed_by_type: string; consumed_by_id: string; dispatch_created_at_ms: number | string; execution_project_id: string; execution_status: string; execution_version: number | string; attempt_count: number | string; max_attempts: number | string; eligible_at_ms: number | string | null; next_attempt_at_ms: number | string | null; execution_lease_owner: string | null; execution_lease_token: string | null; execution_lease_expires_at_ms: number | string | null; started_at_ms: number | string | null; execution_result_mutation_id: string | null; execution_last_result_code: string | null; completed_at_ms: number | string | null; execution_created_at_ms: number | string; execution_updated_at_ms: number | string; recovery_project_id: string; recovery_execution_version: number | string; recovery_status: string; recovery_version: number | string; next_scan_at_ms: number | string | null; recovery_lease_owner: string | null; recovery_lease_token: string | null; recovery_lease_expires_at_ms: number | string | null; finding_count: number | string; last_finding_mutation_id: string | null; last_finding: string | null; recovery_last_result_code: string | null; last_evidence_digest: string | null; recovery_resolution_mutation_id: string | null; recovery_created_at_ms: number | string; recovery_updated_at_ms: number | string; resolution_project_id: string | null; resolution_execution_version: number | string | null; resolution_mutation_id: string | null; resolution_source: string | null; resolution_decision: string | null; resolution_evidence_digest: string | null; resolution_reason_code: string | null; resolved_by_type: string | null; resolved_by_id: string | null; resolved_at_ms: number | string | null; } interface AuthorizationFactRow { dispatch_id: string; project_id: string; mutation_id: string; resolved_by_id: string; authentication_id: string; assurance: string; authenticated_at_ms: number | string; project_version: number | string; binding_version: number | string; authorized_at_ms: number | string; fact_digest: string; } interface PolicyFenceRow { project_version: number | string; binding_version: number | string | null; } const RECOVERY_SELECT = `SELECT dispatch.id AS dispatch_id, dispatch.approval_request_id AS approval_request_id, dispatch.approval_request_version AS approval_request_version, dispatch.project_id AS dispatch_project_id, dispatch.state AS dispatch_state, dispatch.permission AS permission, dispatch.action_type AS action_type, dispatch.action_ref AS action_ref, dispatch.action_digest AS action_digest, dispatch.preview_digest AS preview_digest, dispatch.requested_by_type AS requested_by_type, dispatch.requested_by_id AS requested_by_id, dispatch.consumed_by_type AS consumed_by_type, dispatch.consumed_by_id AS consumed_by_id, dispatch.created_at_ms AS dispatch_created_at_ms, execution.project_id AS execution_project_id, execution.status AS execution_status, execution.version AS execution_version, execution.attempt_count AS attempt_count, execution.max_attempts AS max_attempts, execution.eligible_at_ms AS eligible_at_ms, execution.next_attempt_at_ms AS next_attempt_at_ms, execution.lease_owner AS execution_lease_owner, execution.lease_token AS execution_lease_token, execution.lease_expires_at_ms AS execution_lease_expires_at_ms, execution.started_at_ms AS started_at_ms, execution.result_mutation_id AS execution_result_mutation_id, execution.last_result_code AS execution_last_result_code, execution.completed_at_ms AS completed_at_ms, execution.created_at_ms AS execution_created_at_ms, execution.updated_at_ms AS execution_updated_at_ms, recovery.project_id AS recovery_project_id, recovery.execution_version AS recovery_execution_version, recovery.status AS recovery_status, recovery.version AS recovery_version, recovery.next_scan_at_ms AS next_scan_at_ms, recovery.lease_owner AS recovery_lease_owner, recovery.lease_token AS recovery_lease_token, recovery.lease_expires_at_ms AS recovery_lease_expires_at_ms, recovery.finding_count AS finding_count, recovery.last_finding_mutation_id AS last_finding_mutation_id, recovery.last_finding AS last_finding, recovery.last_result_code AS recovery_last_result_code, recovery.last_evidence_digest AS last_evidence_digest, recovery.resolution_mutation_id AS recovery_resolution_mutation_id, recovery.created_at_ms AS recovery_created_at_ms, recovery.updated_at_ms AS recovery_updated_at_ms, resolution.project_id AS resolution_project_id, resolution.execution_version AS resolution_execution_version, resolution.mutation_id AS resolution_mutation_id, resolution.source AS resolution_source, resolution.decision AS resolution_decision, resolution.evidence_digest AS resolution_evidence_digest, resolution.reason_code AS resolution_reason_code, resolution.resolved_by_type AS resolved_by_type, resolution.resolved_by_id AS resolved_by_id, resolution.resolved_at_ms AS resolved_at_ms FROM "${APPROVED_ACTION_DISPATCH_TABLE}" AS dispatch JOIN "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}" AS execution ON execution.dispatch_id = dispatch.id JOIN "${APPROVED_ACTION_RECOVERY_CONTROL_TABLE}" AS recovery ON recovery.dispatch_id = dispatch.id LEFT JOIN "${APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE}" AS resolution ON resolution.dispatch_id = dispatch.id`; function numberOrNull(value: number | string | null): number | null { return value === null ? null : Number(value); } function rowToAuthorizationFact( row: AuthorizationFactRow, ): Readonly { return normalizeApprovedActionRecoveryAuthorizationFact({ dispatchId: row.dispatch_id, projectId: row.project_id, mutationId: row.mutation_id, resolvedBy: { type: 'user', id: row.resolved_by_id }, authenticationId: row.authentication_id, assurance: row.assurance as ApprovedActionRecoveryAuthorizationFact['assurance'], authenticatedAtMs: Number(row.authenticated_at_ms), projectVersion: Number(row.project_version), bindingVersion: Number(row.binding_version), authorizedAtMs: Number(row.authorized_at_ms), factDigest: row.fact_digest, }); } function rowToRecoverySnapshot( row: RecoveryRow, ): Readonly { try { const dispatch = normalizeApprovedActionDispatchRecord({ id: row.dispatch_id, approvalRequestId: row.approval_request_id, approvalRequestVersion: Number(row.approval_request_version), projectId: row.dispatch_project_id, state: row.dispatch_state as ApprovedActionDispatchRecord['state'], action: { permission: row.permission as ApprovedActionDispatchRecord['action']['permission'], actionType: row.action_type, actionRef: row.action_ref, actionDigest: row.action_digest, previewDigest: row.preview_digest, }, requestedBy: { type: row.requested_by_type as ApprovedActionDispatchRecord['requestedBy']['type'], id: row.requested_by_id, }, consumedBy: { type: row.consumed_by_type as ApprovedActionDispatchRecord['consumedBy']['type'], id: row.consumed_by_id, }, createdAtMs: Number(row.dispatch_created_at_ms), }); const execution = normalizeApprovedActionDispatchExecutionRecord({ dispatchId: row.dispatch_id, projectId: row.execution_project_id, status: row.execution_status as ApprovedActionDispatchExecutionRecord['status'], version: Number(row.execution_version), attemptCount: Number(row.attempt_count), maxAttempts: Number(row.max_attempts), eligibleAtMs: numberOrNull(row.eligible_at_ms), nextAttemptAtMs: numberOrNull(row.next_attempt_at_ms), leaseOwner: row.execution_lease_owner, leaseToken: row.execution_lease_token, leaseExpiresAtMs: numberOrNull(row.execution_lease_expires_at_ms), startedAtMs: numberOrNull(row.started_at_ms), resultMutationId: row.execution_result_mutation_id, lastResultCode: row.execution_last_result_code, completedAtMs: numberOrNull(row.completed_at_ms), createdAtMs: Number(row.execution_created_at_ms), updatedAtMs: Number(row.execution_updated_at_ms), }); const recovery = normalizeApprovedActionRecoveryControlRecord({ dispatchId: row.dispatch_id, projectId: row.recovery_project_id, executionVersion: Number(row.recovery_execution_version), status: row.recovery_status as ApprovedActionRecoveryControlRecord['status'], version: Number(row.recovery_version), nextScanAtMs: numberOrNull(row.next_scan_at_ms), leaseOwner: row.recovery_lease_owner, leaseToken: row.recovery_lease_token, leaseExpiresAtMs: numberOrNull(row.recovery_lease_expires_at_ms), findingCount: Number(row.finding_count), lastFindingMutationId: row.last_finding_mutation_id, lastFinding: row.last_finding as ApprovedActionRecoveryControlRecord['lastFinding'], lastResultCode: row.recovery_last_result_code, lastEvidenceDigest: row.last_evidence_digest, resolutionMutationId: row.recovery_resolution_mutation_id, createdAtMs: Number(row.recovery_created_at_ms), updatedAtMs: Number(row.recovery_updated_at_ms), }); const action: Readonly = Object.freeze({ dispatch, execution }); let resolution: Readonly | null = null; if (row.resolution_mutation_id !== null) { if ( row.resolution_project_id === null || row.resolution_execution_version === null || row.resolution_source === null || row.resolution_decision === null || row.resolution_reason_code === null || row.resolved_at_ms === null ) { throw new ApprovedActionRecoveryBindingConflictError(); } resolution = normalizeApprovedActionRecoveryResolutionRecord({ dispatchId: row.dispatch_id, projectId: row.resolution_project_id, executionVersion: Number(row.resolution_execution_version), mutationId: row.resolution_mutation_id, source: row.resolution_source as ApprovedActionRecoveryResolutionRecord['source'], decision: row.resolution_decision as ApprovedActionRecoveryResolutionRecord['decision'], evidenceDigest: row.resolution_evidence_digest, reasonCode: row.resolution_reason_code, resolvedBy: row.resolved_by_type === null || row.resolved_by_id === null ? null : { type: row.resolved_by_type as 'user', id: row.resolved_by_id, }, resolvedAtMs: Number(row.resolved_at_ms), }); } if ( dispatch.id !== execution.dispatchId || dispatch.id !== recovery.dispatchId || dispatch.projectId !== execution.projectId || dispatch.projectId !== recovery.projectId || recovery.executionVersion !== execution.version ) { throw new ApprovedActionRecoveryBindingConflictError(); } if (recovery.status === 'resolved') { if ( !['succeeded', 'failed', 'blocked'].includes(execution.status) || execution.resultMutationId !== recovery.resolutionMutationId ) { throw new ApprovedActionRecoveryBindingConflictError(); } } else if (execution.status !== 'executing') { throw new ApprovedActionRecoveryBindingConflictError(); } if ( resolution && (resolution.dispatchId !== dispatch.id || resolution.projectId !== dispatch.projectId || resolution.executionVersion + 1 !== execution.version || resolution.mutationId !== recovery.resolutionMutationId || resolution.mutationId !== execution.resultMutationId) ) { throw new ApprovedActionRecoveryBindingConflictError(); } return Object.freeze({ action, recovery, resolution }); } catch (error) { if ( error instanceof ApprovedActionRecoveryBindingConflictError || error instanceof ApprovedActionDispatchBindingConflictError ) { throw error; } throw new ApprovedActionRecoveryRepositoryError(); } } function assertExactKeys(value: object, expected: readonly string[]): void { const keys = Object.keys(value).sort(); const canonical = [...expected].sort(); if ( keys.length !== canonical.length || keys.some((key, index) => key !== canonical[index]) ) { throw new TypeError('Approved action recovery command shape is invalid'); } } function assertCommand( value: unknown, expected: readonly string[], ): asserts value is object { if (!value || typeof value !== 'object' || Array.isArray(value)) { throw new TypeError('Approved action recovery command must be an object'); } assertExactKeys(value, expected); } function assertVersion(value: number): void { if (!Number.isSafeInteger(value) || value < 0 || value >= 2_147_483_647) { throw new RangeError('Approved action recovery version is invalid'); } } function errorCode(error: unknown): string | undefined { if (!error || typeof error !== 'object') return undefined; for (const candidate of [ error, 'original' in error ? error.original : undefined, 'parent' in error ? error.parent : undefined, ]) { if ( candidate && typeof candidate === 'object' && 'code' in candidate && typeof candidate.code === 'string' ) { return candidate.code; } } return undefined; } function retryDelay(attempt: number): Promise { return new Promise((resolve) => setTimeout(resolve, 2 ** attempt)); } function isKnownError(error: unknown): boolean { return ( error instanceof ApprovedActionRecoveryBindingConflictError || error instanceof ApprovedActionRecoveryFenceRejectedError || error instanceof ApprovedActionRecoveryRepositoryError || error instanceof InvalidApprovalValueError || error instanceof RangeError || error instanceof TypeError ); } function terminalStatus( decision: ResolveApprovedActionRecoveryCommand['decision'], ): 'succeeded' | 'failed' | 'blocked' { if (decision === 'confirm_succeeded') return 'succeeded'; if (decision === 'confirm_failed') return 'failed'; return 'blocked'; } export class LegacySequelizeApprovedActionRecoveryRepository implements ApprovedActionRecoveryRepository { constructor(private readonly database: Sequelize) { if (database.getDialect() !== 'sqlite') { throw new TypeError( 'Approved action recovery repository is SQLite-only; cluster-control requires a PostgreSQL adapter', ); } } private async transactionWithRetry( operation: (transaction: Transaction) => Promise, ): Promise { for (let attempt = 0; attempt < RETRY_ATTEMPTS; attempt += 1) { try { return await this.database.transaction( { type: Transaction.TYPES.IMMEDIATE }, operation, ); } catch (error) { if (isKnownError(error)) throw error; if ( errorCode(error) === 'SQLITE_BUSY' && attempt < RETRY_ATTEMPTS - 1 ) { await retryDelay(attempt); continue; } throw new ApprovedActionRecoveryRepositoryError(); } } throw new ApprovedActionRecoveryRepositoryError(); } private async selectSnapshot( dispatchId: string, transaction?: Transaction, ): Promise | null> { const rows = await this.database.query( `${RECOVERY_SELECT} WHERE dispatch.id = :dispatchId LIMIT 2`, { type: QueryTypes.SELECT, replacements: { dispatchId }, ...(transaction ? { transaction } : {}), }, ); if (rows.length === 0) { const executions = await this.database.query<{ status: string; }>( `SELECT status FROM "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}" WHERE dispatch_id = :dispatchId LIMIT 2`, { type: QueryTypes.SELECT, replacements: { dispatchId }, ...(transaction ? { transaction } : {}), }, ); if ( executions.length !== 0 && (executions.length !== 1 || executions[0].status === 'executing') ) { throw new ApprovedActionRecoveryRepositoryError(); } return null; } if (rows.length !== 1) throw new ApprovedActionRecoveryRepositoryError(); return rowToRecoverySnapshot(rows[0]); } private async updateControl( dispatchId: string, expectedExecutionVersion: number, expectedRecoveryVersion: number, expectedStatuses: readonly string[], values: Record, transaction: Transaction, fence?: { owner: string; leaseToken: string }, ): Promise { const assignments = Object.keys(values) .map((field) => `"${field}" = :${field}`) .join(', '); const statusParameters = Object.fromEntries( expectedStatuses.map((status, index) => [ `expectedStatus${index}`, status, ]), ); const statusList = expectedStatuses .map((_, index) => `:expectedStatus${index}`) .join(', '); const [, affected] = await this.database.query( `UPDATE "${APPROVED_ACTION_RECOVERY_CONTROL_TABLE}" SET ${assignments} WHERE dispatch_id = :dispatchId AND execution_version = :expectedExecutionVersion AND version = :expectedRecoveryVersion AND status IN (${statusList}) ${ fence ? 'AND lease_owner = :owner AND lease_token = :leaseToken' : '' }`, { type: QueryTypes.UPDATE, replacements: { ...values, dispatchId, expectedExecutionVersion, expectedRecoveryVersion, ...statusParameters, ...(fence ?? {}), }, transaction, }, ); if (affected !== 1) throw new ApprovedActionRecoveryFenceRejectedError(); } private async selectAuthorizationFact( dispatchId: string, transaction: Transaction, ): Promise | null> { const rows = await this.database.query( `SELECT dispatch_id, project_id, mutation_id, resolved_by_id, authentication_id, assurance, authenticated_at_ms, project_version, binding_version, authorized_at_ms, fact_digest FROM "${APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE}" WHERE dispatch_id = :dispatchId LIMIT 2`, { type: QueryTypes.SELECT, replacements: { dispatchId }, transaction, }, ); if (rows.length > 1) throw new ApprovedActionRecoveryRepositoryError(); return rows.length === 0 ? null : rowToAuthorizationFact(rows[0]); } private async assertAuthorizationFence( fact: Readonly, transaction: Transaction, ): Promise { const rows = await this.database.query( `SELECT project.version AS project_version, (SELECT MAX(binding.version) FROM "${PROJECT_ROLE_BINDING_TABLE}" AS binding WHERE binding.project_id = project.id AND binding.subject_type = 'user' AND binding.subject_id = :subjectId) AS binding_version FROM "${PROJECT_TABLE}" AS project WHERE project.id = :projectId LIMIT 2`, { type: QueryTypes.SELECT, replacements: { projectId: fact.projectId, subjectId: fact.resolvedBy.id, }, transaction, }, ); if ( rows.length !== 1 || Number(rows[0].project_version) !== fact.projectVersion || rows[0].binding_version === null || Number(rows[0].binding_version) !== fact.bindingVersion ) { throw new ApprovedActionRecoveryFenceRejectedError(); } } private async insertAuthorizationFact( fact: Readonly, transaction: Transaction, ): Promise { await this.database.query( `INSERT INTO "${APPROVED_ACTION_RECOVERY_AUTHORIZATION_TABLE}" (dispatch_id, project_id, mutation_id, resolved_by_id, authentication_id, assurance, authenticated_at_ms, project_version, binding_version, authorized_at_ms, fact_digest) VALUES (:dispatchId, :projectId, :mutationId, :resolvedById, :authenticationId, :assurance, :authenticatedAtMs, :projectVersion, :bindingVersion, :authorizedAtMs, :factDigest)`, { replacements: { dispatchId: fact.dispatchId, projectId: fact.projectId, mutationId: fact.mutationId, resolvedById: fact.resolvedBy.id, authenticationId: fact.authenticationId, assurance: fact.assurance, authenticatedAtMs: fact.authenticatedAtMs, projectVersion: fact.projectVersion, bindingVersion: fact.bindingVersion, authorizedAtMs: fact.authorizedAtMs, factDigest: fact.factDigest, }, transaction, }, ); } async findById( dispatchId: string, ): Promise | null> { assertApprovalMutationId(dispatchId); return this.selectSnapshot(dispatchId); } async listDue( query: ListDueApprovedActionRecoveriesQuery, ): Promise { assertCommand( query, query.cursor === undefined ? ['nowMs', 'limit'] : ['nowMs', 'limit', 'cursor'], ); assertApprovalTimestamp('nowMs', query.nowMs); assertApprovedActionRecoveryPageSize(query.limit); const cursor = query.cursor ? normalizeApprovedActionRecoveryCursor(query.cursor) : undefined; const rows = await this.database.query( `${RECOVERY_SELECT} WHERE execution.status = 'executing' AND execution.lease_expires_at_ms IS NOT NULL AND execution.lease_expires_at_ms <= :nowMs AND recovery.status IN ('armed', 'leased') AND recovery.next_scan_at_ms IS NOT NULL AND recovery.next_scan_at_ms <= :nowMs ${ cursor ? `AND (recovery.next_scan_at_ms > :cursorAt OR (recovery.next_scan_at_ms = :cursorAt AND dispatch.id > :cursorId))` : '' } ORDER BY recovery.next_scan_at_ms ASC, dispatch.id ASC LIMIT :rowLimit`, { type: QueryTypes.SELECT, replacements: { nowMs: query.nowMs, rowLimit: query.limit + 1, ...(cursor ? { cursorAt: cursor.nextScanAtMs, cursorId: cursor.dispatchId } : {}), }, }, ); const truncated = rows.length > query.limit; const selected = rows.slice(0, query.limit).map(rowToRecoverySnapshot); const last = selected[selected.length - 1]; return Object.freeze({ recoveries: Object.freeze(selected), truncated, ...(truncated && last ? { nextCursor: Object.freeze({ nextScanAtMs: last.recovery.nextScanAtMs!, dispatchId: last.action.dispatch.id, }), } : {}), }); } async claim( command: ClaimApprovedActionRecoveryCommand, ): Promise { assertCommand(command, [ 'dispatchId', 'owner', 'leaseToken', 'nowMs', 'leaseDurationMs', ]); assertApprovalMutationId(command.dispatchId); assertApprovedActionLeaseIdentity(command.owner); assertApprovedActionLeaseIdentity(command.leaseToken); assertApprovedActionRecoveryLeaseDuration(command.leaseDurationMs); assertApprovalTimestamp('nowMs', command.nowMs); return this.transactionWithRetry(async (transaction) => { const snapshot = await this.selectSnapshot( command.dispatchId, transaction, ); if (!snapshot) return { status: 'not_found' as const }; const current = snapshot.recovery; if (current.status === 'resolved') { return { status: 'resolved' as const, snapshot }; } if (current.status === 'manual_required') { return { status: 'manual_required' as const, snapshot }; } const execution = snapshot.action.execution; if ( execution.status !== 'executing' || execution.leaseExpiresAtMs === null || execution.leaseExpiresAtMs > command.nowMs ) { return { status: 'execution_active' as const, snapshot }; } if ( current.status === 'leased' && current.leaseOwner === command.owner && current.leaseToken === command.leaseToken && current.leaseExpiresAtMs! > command.nowMs ) { return { status: 'claimed' as const, snapshot }; } if ( current.status === 'leased' && current.leaseExpiresAtMs! > command.nowMs ) { return { status: 'leased' as const, snapshot }; } if ( current.nextScanAtMs === null || current.nextScanAtMs > command.nowMs ) { return { status: 'not_due' as const, snapshot }; } const leaseExpiresAtMs = command.nowMs + command.leaseDurationMs; if (!Number.isSafeInteger(leaseExpiresAtMs)) { throw new RangeError( 'Approved action recovery lease expiry is invalid', ); } await this.updateControl( command.dispatchId, current.executionVersion, current.version, [current.status], { status: 'leased', version: current.version + 1, next_scan_at_ms: leaseExpiresAtMs, lease_owner: command.owner, lease_token: command.leaseToken, lease_expires_at_ms: leaseExpiresAtMs, updated_at_ms: command.nowMs, }, transaction, ); return { status: 'claimed' as const, snapshot: (await this.selectSnapshot(command.dispatchId, transaction))!, }; }); } async recordFinding( command: RecordApprovedActionRecoveryFindingCommand, ): Promise> { assertCommand( command, command.evidenceDigest === undefined && command.retryAtMs === undefined ? [ 'dispatchId', 'expectedExecutionVersion', 'expectedRecoveryVersion', 'owner', 'leaseToken', 'findingMutationId', 'finding', 'resultCode', 'observedAtMs', ] : [ 'dispatchId', 'expectedExecutionVersion', 'expectedRecoveryVersion', 'owner', 'leaseToken', 'findingMutationId', 'finding', 'resultCode', ...(command.evidenceDigest === undefined ? [] : ['evidenceDigest']), 'observedAtMs', ...(command.retryAtMs === undefined ? [] : ['retryAtMs']), ], ); assertApprovalMutationId(command.dispatchId); assertVersion(command.expectedExecutionVersion); assertVersion(command.expectedRecoveryVersion); assertApprovalMutationId(command.findingMutationId); assertApprovedActionLeaseIdentity(command.owner); assertApprovedActionLeaseIdentity(command.leaseToken); if ( ![ 'still_running', 'missing', 'conflict', 'unsupported', 'unavailable', ].includes(command.finding) ) { throw new TypeError('Approved action recovery finding is invalid'); } if (command.evidenceDigest !== undefined) { assertApprovedActionEvidenceDigest(command.evidenceDigest); } assertApprovedActionResultCode(command.resultCode); assertApprovalTimestamp('observedAtMs', command.observedAtMs); const retryable = ['still_running', 'missing', 'unavailable'].includes( command.finding, ); if (retryable !== (command.retryAtMs !== undefined)) { throw new TypeError('Approved action recovery retry shape is invalid'); } if (command.retryAtMs !== undefined) { assertApprovalTimestamp('retryAtMs', command.retryAtMs); if (command.retryAtMs <= command.observedAtMs) { throw new RangeError('Approved action recovery retry must be future'); } } return this.transactionWithRetry(async (transaction) => { const snapshot = await this.selectSnapshot( command.dispatchId, transaction, ); if (!snapshot) throw new ApprovedActionRecoveryFenceRejectedError(); const current = snapshot.recovery; const nextStatus = retryable ? 'armed' : 'manual_required'; if ( current.version === command.expectedRecoveryVersion + 1 && current.executionVersion === command.expectedExecutionVersion && current.status === nextStatus && current.lastFindingMutationId === command.findingMutationId && current.lastFinding === command.finding && current.lastResultCode === command.resultCode && current.lastEvidenceDigest === (command.evidenceDigest ?? null) && current.nextScanAtMs === (command.retryAtMs ?? null) && current.updatedAtMs === command.observedAtMs ) { return snapshot; } if ( snapshot.action.execution.status !== 'executing' || snapshot.action.execution.version !== command.expectedExecutionVersion ) { throw new ApprovedActionRecoveryFenceRejectedError(); } await this.updateControl( command.dispatchId, command.expectedExecutionVersion, command.expectedRecoveryVersion, ['leased'], { status: nextStatus, version: command.expectedRecoveryVersion + 1, next_scan_at_ms: command.retryAtMs ?? null, lease_owner: null, lease_token: null, lease_expires_at_ms: null, finding_count: Math.min( MAX_APPROVED_ACTION_RECOVERY_FINDINGS, current.findingCount + 1, ), last_finding_mutation_id: command.findingMutationId, last_finding: command.finding, last_result_code: command.resultCode, last_evidence_digest: command.evidenceDigest ?? null, updated_at_ms: command.observedAtMs, }, transaction, { owner: command.owner, leaseToken: command.leaseToken }, ); return (await this.selectSnapshot(command.dispatchId, transaction))!; }); } async resolve( command: ResolveApprovedActionRecoveryCommand, ): Promise { assertCommand( command, command.source === 'automatic_evidence' ? [ 'dispatchId', 'expectedExecutionVersion', 'expectedRecoveryVersion', 'owner', 'leaseToken', 'mutationId', 'source', 'decision', 'evidenceDigest', 'reasonCode', 'resolvedAtMs', ] : [ 'dispatchId', 'expectedExecutionVersion', 'expectedRecoveryVersion', 'mutationId', 'source', 'decision', ...(command.evidenceDigest === undefined ? [] : ['evidenceDigest']), 'reasonCode', 'resolvedBy', 'resolvedAtMs', 'authorizationFact', ], ); assertApprovalMutationId(command.dispatchId); assertVersion(command.expectedExecutionVersion); assertVersion(command.expectedRecoveryVersion); assertApprovalMutationId(command.mutationId); assertApprovedActionResultCode(command.reasonCode); if (command.source === 'automatic_evidence') { assertApprovedActionLeaseIdentity(command.owner); assertApprovedActionLeaseIdentity(command.leaseToken); } if (command.evidenceDigest !== undefined) { assertApprovedActionEvidenceDigest(command.evidenceDigest); } assertApprovalTimestamp('resolvedAtMs', command.resolvedAtMs); const authorizationFact = command.source === 'human' ? normalizeApprovedActionRecoveryAuthorizationFact( command.authorizationFact, ) : null; if (command.source === 'human') normalizePolicySubject(command.resolvedBy); return this.transactionWithRetry(async (transaction) => { const snapshot = await this.selectSnapshot( command.dispatchId, transaction, ); if (!snapshot) return { status: 'not_found' as const }; const expectedResolution = normalizeApprovedActionRecoveryResolutionRecord({ dispatchId: command.dispatchId, projectId: snapshot.action.dispatch.projectId, executionVersion: command.expectedExecutionVersion, mutationId: command.mutationId, source: command.source, decision: command.decision, evidenceDigest: command.evidenceDigest ?? null, reasonCode: command.reasonCode, resolvedBy: command.source === 'human' ? command.resolvedBy : null, resolvedAtMs: command.resolvedAtMs, }); if (snapshot.resolution) { if ( JSON.stringify(snapshot.resolution) === JSON.stringify(expectedResolution) ) { if (command.source === 'human') { const storedAuthorization = await this.selectAuthorizationFact( command.dispatchId, transaction, ); if ( !storedAuthorization || JSON.stringify(storedAuthorization) !== JSON.stringify(authorizationFact) ) { return { status: 'already_terminal' as const, snapshot }; } } return { status: 'resolved' as const, snapshot }; } return { status: 'already_terminal' as const, snapshot }; } if ( snapshot.action.execution.status !== 'executing' || snapshot.recovery.status === 'resolved' ) { return { status: 'already_terminal' as const, snapshot }; } if ( snapshot.action.execution.version !== command.expectedExecutionVersion || snapshot.recovery.executionVersion !== command.expectedExecutionVersion || snapshot.recovery.version !== command.expectedRecoveryVersion || snapshot.action.execution.startedAtMs === null || snapshot.action.execution.leaseExpiresAtMs === null || command.resolvedAtMs < snapshot.action.execution.leaseExpiresAtMs || command.resolvedAtMs < snapshot.action.execution.startedAtMs ) { throw new ApprovedActionRecoveryFenceRejectedError(); } if ( command.source === 'automatic_evidence' && (snapshot.recovery.status !== 'leased' || snapshot.recovery.leaseOwner !== command.owner || snapshot.recovery.leaseToken !== command.leaseToken) ) { throw new ApprovedActionRecoveryFenceRejectedError(); } if (command.source === 'human') { if ( authorizationFact === null || authorizationFact.dispatchId !== command.dispatchId || authorizationFact.projectId !== snapshot.action.dispatch.projectId || authorizationFact.mutationId !== command.mutationId || authorizationFact.resolvedBy.type !== 'user' || authorizationFact.resolvedBy.id !== command.resolvedBy.id || authorizationFact.authorizedAtMs !== command.resolvedAtMs ) { throw new ApprovedActionRecoveryFenceRejectedError(); } await this.assertAuthorizationFence(authorizationFact, transaction); } const nextExecutionVersion = command.expectedExecutionVersion + 1; const nextRecoveryVersion = command.expectedRecoveryVersion + 1; const status = terminalStatus(command.decision); const [, executionAffected] = await this.database.query( `UPDATE "${APPROVED_ACTION_DISPATCH_EXECUTION_TABLE}" SET status = :status, version = :nextExecutionVersion, eligible_at_ms = NULL, next_attempt_at_ms = NULL, lease_owner = NULL, lease_token = NULL, lease_expires_at_ms = NULL, result_mutation_id = :mutationId, last_result_code = :reasonCode, completed_at_ms = :resolvedAtMs, updated_at_ms = :resolvedAtMs WHERE dispatch_id = :dispatchId AND status = 'executing' AND version = :expectedExecutionVersion`, { type: QueryTypes.UPDATE, replacements: { dispatchId: command.dispatchId, expectedExecutionVersion: command.expectedExecutionVersion, nextExecutionVersion, mutationId: command.mutationId, reasonCode: command.reasonCode, resolvedAtMs: command.resolvedAtMs, status, }, transaction, }, ); if (executionAffected !== 1) { throw new ApprovedActionRecoveryFenceRejectedError(); } const automatic = command.source === 'automatic_evidence'; await this.updateControl( command.dispatchId, command.expectedExecutionVersion, command.expectedRecoveryVersion, automatic ? ['leased'] : ['armed', 'leased', 'manual_required'], { execution_version: nextExecutionVersion, status: 'resolved', version: nextRecoveryVersion, next_scan_at_ms: null, lease_owner: null, lease_token: null, lease_expires_at_ms: null, ...(automatic ? { finding_count: Math.min( MAX_APPROVED_ACTION_RECOVERY_FINDINGS, snapshot.recovery.findingCount + 1, ), last_finding_mutation_id: command.mutationId, last_finding: command.decision === 'confirm_succeeded' ? 'verified_succeeded' : 'verified_failed', last_result_code: command.reasonCode, last_evidence_digest: command.evidenceDigest, } : {}), resolution_mutation_id: command.mutationId, updated_at_ms: command.resolvedAtMs, }, transaction, automatic ? { owner: command.owner, leaseToken: command.leaseToken } : undefined, ); await this.database.query( `INSERT INTO "${APPROVED_ACTION_RECOVERY_RESOLUTION_TABLE}" (dispatch_id, project_id, execution_version, mutation_id, source, decision, evidence_digest, reason_code, resolved_by_type, resolved_by_id, resolved_at_ms) VALUES (:dispatchId, :projectId, :executionVersion, :mutationId, :source, :decision, :evidenceDigest, :reasonCode, :resolvedByType, :resolvedById, :resolvedAtMs)`, { replacements: { dispatchId: expectedResolution.dispatchId, projectId: expectedResolution.projectId, executionVersion: expectedResolution.executionVersion, mutationId: expectedResolution.mutationId, source: expectedResolution.source, decision: expectedResolution.decision, evidenceDigest: expectedResolution.evidenceDigest, reasonCode: expectedResolution.reasonCode, resolvedByType: expectedResolution.resolvedBy?.type ?? null, resolvedById: expectedResolution.resolvedBy?.id ?? null, resolvedAtMs: expectedResolution.resolvedAtMs, }, transaction, }, ); if (authorizationFact !== null) { await this.insertAuthorizationFact(authorizationFact, transaction); } return { status: 'resolved' as const, snapshot: (await this.selectSnapshot(command.dispatchId, transaction))!, }; }); } }