import type { PostgresClient, PostgresPool } from '@qinglong/runtime-core'; import { InvalidRunManualRetryError, MAX_RUN_MANUAL_RETRY_AUTHENTICATION_AGE_MS, RUN_MANUAL_RETRY_SOURCE_STATUSES, RunManualRetryFenceRejectedError, RunManualRetryNotFoundError, RunManualRetryRateLimitedError, RunManualRetryUnavailableError, normalizeRunManualRetryCommand, normalizeRunManualRetryResult, type RunManualRetryCommand, type RunManualRetryRepository, type RunManualRetryResult, type RunManualRetrySourceStatus, } from '@qinglong/runtime-core/run-manual-retry'; import { POSTGRES_DEFINITION_RETRYABLE_SQL_STATES, POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS, configurePostgresDefinitionTransaction, postgresRequiredBoolean, postgresRequiredInteger, postgresRequiredJsonObject, postgresRequiredString, postgresSqlState, rollbackPostgresDefinitionTransaction, } from '../repository/definitionRepositorySupport'; type Row = Record; export const CLUSTER_RUN_MANUAL_RETRY_RATE_WINDOW_MS = 60_000; export const CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT = 64; const CLUSTER_STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']); const TASK_REVISION_PATTERN = /^qltd:v1:([1-9]\d*):([0-9a-f]{64})$/; interface SourceRun { readonly taskId: string; readonly taskRevision: string; readonly taskName?: string; readonly taskSnapshotRef: string; readonly inputRef?: string; readonly priority: number; } interface ExecutionRevision { readonly contentDigest: string; readonly sourceContentDigest: string; } function unavailable(options?: ErrorOptions): RunManualRetryUnavailableError { return new RunManualRetryUnavailableError(options); } function text(row: Row, key: string): string { return postgresRequiredString(row[key], unavailable); } function integer(row: Row, key: string): number { return postgresRequiredInteger(row[key], unavailable); } function optionalText(row: Row, key: string): string | undefined { if (row[key] === null) return undefined; return text(row, key); } function json(row: Row, key: string): Record { return postgresRequiredJsonObject(row[key], unavailable); } function sourceStatus(value: string): RunManualRetrySourceStatus { if ( !RUN_MANUAL_RETRY_SOURCE_STATUSES.includes( value as RunManualRetrySourceStatus, ) ) { throw new RunManualRetryFenceRejectedError('source_not_terminal'); } return value as RunManualRetrySourceStatus; } function sameFencePayload( value: unknown, command: Readonly, ): boolean { if (!value || typeof value !== 'object' || Array.isArray(value)) return false; const fence = value as Record; return ( fence.project_version === command.policyFence.projectVersion && fence.binding_version === command.policyFence.bindingVersion ); } async function databaseNow(client: PostgresClient): Promise { const result = await client.query(` SELECT floor(extract(epoch FROM statement_timestamp()) * 1000)::bigint AS "nowMs" `); if (result.rows.length !== 1) throw unavailable(); return integer(result.rows[0]!, 'nowMs'); } function confirmStrongAuthentication( command: Readonly, observedAtMs: number, ): void { if ( command.principal.subject.type !== 'user' || !CLUSTER_STRONG_ASSURANCES.has(command.principal.assurance) || command.principal.authenticatedAtMs > observedAtMs || command.principal.expiresAtMs <= observedAtMs || observedAtMs - command.principal.authenticatedAtMs > MAX_RUN_MANUAL_RETRY_AUTHENTICATION_AGE_MS ) { throw new RunManualRetryFenceRejectedError('authentication_changed'); } } async function confirmAuthorization( client: PostgresClient, command: Readonly, ): Promise { const result = await client.query( ` SELECT "ql3"."lock_run_management_policy_fence"( $1::varchar, $2::varchar, $3::varchar, $4::integer, $5::integer ) AS "matches" `, [ command.projectId, command.principal.subject.type, command.principal.subject.id, command.policyFence.projectVersion, command.policyFence.bindingVersion, ], ); if ( result.rows.length !== 1 || !postgresRequiredBoolean(result.rows[0]!.matches, unavailable) ) { throw new RunManualRetryFenceRejectedError('authorization_changed'); } } async function findReplay( client: PostgresClient, command: Readonly, ): Promise { const result = await client.query( ` SELECT run.id AS "runId", run.project_id AS "projectId", run.retry_of_run_id AS "retryOfRunId", run.task_id AS "taskId", run.task_revision AS "taskRevision", run.trigger_type AS "triggerType", run.execution_origin AS "executionOrigin", run.execution_owner AS "executionOwner", run.triggered_by AS "triggeredBy", run.request_id AS "requestId", run.created_at_ms AS "createdAtMs", attempt.id AS "attemptId", attempt.executor_type AS "executorType", created.actor_type AS "createdActorType", created.actor_id AS "createdActorId", created.payload AS "createdPayload", queued.actor_type AS "queuedActorType", queued.actor_id AS "queuedActorId", queued.payload AS "queuedPayload" FROM "ql3"."runs" AS run JOIN "ql3"."run_attempts" AS attempt ON attempt.run_id = run.id AND attempt.attempt = 1 JOIN "ql3"."run_events" AS created ON created.run_id = run.id AND created.sequence = 1 AND created.type = 'run.created' JOIN "ql3"."run_events" AS queued ON queued.run_id = run.id AND queued.sequence = 2 AND queued.type = 'run.queued' WHERE run.project_id = $1 AND run.idempotency_key = $2 `, [command.projectId, `ql3:run-manual-retry:v1:${command.mutationId}`], ); if (result.rows.length > 1) throw unavailable(); return result.rows[0]; } function replayResult( command: Readonly, row: Row, ): Readonly { const created = json(row, 'createdPayload'); const queued = json(row, 'queuedPayload'); if ( text(row, 'projectId') !== command.projectId || text(row, 'retryOfRunId') !== command.sourceRunId || text(row, 'triggerType') !== 'run_manual_retry' || text(row, 'executionOrigin') !== 'manual' || text(row, 'executionOwner') !== 'runtime' || text(row, 'triggeredBy') !== command.principal.subject.id || text(row, 'requestId') !== command.mutationId || text(row, 'executorType') !== 'remote_worker' || text(row, 'createdActorType') !== command.principal.subject.type || text(row, 'createdActorId') !== command.principal.subject.id || text(row, 'queuedActorType') !== command.principal.subject.type || text(row, 'queuedActorId') !== command.principal.subject.id || created.mutation_id !== command.mutationId || created.retry_of_run_id !== command.sourceRunId || created.source_run_status !== command.expectedRunStatus || created.source_run_version !== command.expectedRunVersion || created.inherit_retry_policy !== false || created.authentication_id !== command.principal.authenticationId || created.audit_event_id !== command.auditEventId || typeof created.execution_revision_digest !== 'string' || !/^[0-9a-f]{64}$/.test(created.execution_revision_digest) || !sameFencePayload(created.policy_fence, command) || queued.from_status !== 'created' || queued.to_status !== 'queued' || queued.version !== 2 ) { throw new RunManualRetryFenceRejectedError('mutation_conflict'); } return normalizeRunManualRetryResult({ status: 'existing', projectId: command.projectId, sourceRunId: command.sourceRunId, sourceRunStatus: command.expectedRunStatus, sourceRunVersion: command.expectedRunVersion, runId: text(row, 'runId'), retryOfRunId: text(row, 'retryOfRunId'), taskId: text(row, 'taskId'), taskRevision: text(row, 'taskRevision'), attemptId: text(row, 'attemptId'), runStatus: 'queued', runVersion: 2, eventSequence: 2, executorType: 'remote_worker', executionRevisionDigest: created.execution_revision_digest, createdAtMs: integer(row, 'createdAtMs'), }); } async function findSource( client: PostgresClient, command: Readonly, ): Promise { const result = await client.query( ` SELECT run.project_id AS "projectId", run.task_id AS "taskId", run.task_revision AS "taskRevision", run.task_name AS "taskName", run.task_snapshot_ref AS "taskSnapshotRef", run.parent_run_id AS "parentRunId", run.trigger_type AS "triggerType", run.execution_owner AS "executionOwner", run.input_ref AS "inputRef", run.priority, run.status AS "runStatus", run.version AS "runVersion", attempt.executor_type AS "attemptExecutorType" FROM "ql3"."runs" AS run LEFT JOIN LATERAL ( SELECT executor_type FROM "ql3"."run_attempts" WHERE run_id = run.id ORDER BY attempt DESC LIMIT 1 ) AS attempt ON true WHERE run.id = $1 `, [command.sourceRunId], ); if ( result.rows.length === 0 || result.rows[0]?.projectId !== command.projectId ) { throw new RunManualRetryNotFoundError(); } if (result.rows.length !== 1) throw unavailable(); const row = result.rows[0]!; const status = sourceStatus(text(row, 'runStatus')); if ( status !== command.expectedRunStatus || integer(row, 'runVersion') !== command.expectedRunVersion ) { throw new RunManualRetryFenceRejectedError('source_changed'); } const taskRevision = text(row, 'taskRevision'); const taskSnapshotRef = optionalText(row, 'taskSnapshotRef'); if ( text(row, 'executionOwner') !== 'runtime' || optionalText(row, 'parentRunId') !== undefined || text(row, 'triggerType') === 'plugin_package_workflow' || taskSnapshotRef === undefined || taskSnapshotRef !== taskRevision || optionalText(row, 'attemptExecutorType') !== 'remote_worker' || !TASK_REVISION_PATTERN.test(taskRevision) ) { throw new RunManualRetryFenceRejectedError('source_not_retryable'); } const taskName = optionalText(row, 'taskName'); const inputRef = optionalText(row, 'inputRef'); return Object.freeze({ taskId: text(row, 'taskId'), taskRevision, ...(taskName === undefined ? {} : { taskName }), taskSnapshotRef, ...(inputRef === undefined ? {} : { inputRef }), priority: integer(row, 'priority'), }); } async function confirmTaskAndExecution( client: PostgresClient, projectId: string, source: Readonly, ): Promise { const task = await client.query( ` SELECT revision.enabled FROM "ql3"."task_definitions" AS head JOIN "ql3"."task_definition_revisions" AS revision ON revision.project_id = head.project_id AND revision.task_id = head.task_id AND revision.revision = head.current_revision WHERE head.project_id = $1 AND head.task_id = $2 `, [projectId, source.taskId], ); if ( task.rows.length !== 1 || !postgresRequiredBoolean(task.rows[0]!.enabled, unavailable) ) { throw new RunManualRetryFenceRejectedError('task_disabled'); } const execution = await client.query( ` SELECT source_content_digest AS "sourceContentDigest", content_digest AS "contentDigest" FROM "ql3"."task_execution_revisions" WHERE project_id = $1 AND task_id = $2 AND task_revision = $3 AND executor_type = 'remote_worker' LIMIT 2 `, [projectId, source.taskId, source.taskRevision], ); if (execution.rows.length !== 1) { throw new RunManualRetryFenceRejectedError('source_not_retryable'); } const match = TASK_REVISION_PATTERN.exec(source.taskRevision); const row = execution.rows[0]!; const sourceContentDigest = text(row, 'sourceContentDigest'); const contentDigest = text(row, 'contentDigest'); if ( !match || match[2] !== sourceContentDigest || !/^[0-9a-f]{64}$/.test(contentDigest) ) { throw new RunManualRetryFenceRejectedError('source_not_retryable'); } return Object.freeze({ contentDigest, sourceContentDigest }); } async function consumeRateLimit( client: PostgresClient, command: Readonly, observedAtMs: number, ): Promise { const threshold = Math.max( 0, observedAtMs - CLUSTER_RUN_MANUAL_RETRY_RATE_WINDOW_MS, ); const result = await client.query( ` SELECT created_at_ms AS "createdAtMs" FROM "ql3"."runs" WHERE project_id = $1 AND trigger_type = 'run_manual_retry' AND execution_origin = 'manual' AND triggered_by = $2 AND created_at_ms > $3 ORDER BY created_at_ms DESC, id DESC LIMIT $4 `, [ command.projectId, command.principal.subject.id, threshold, CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT, ], ); if (result.rows.length < CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT) return; const earliestAtMs = integer( result.rows[result.rows.length - 1]!, 'createdAtMs', ); throw new RunManualRetryRateLimitedError( Math.max( 1, earliestAtMs + CLUSTER_RUN_MANUAL_RETRY_RATE_WINDOW_MS - observedAtMs, ), ); } async function insertRetry( client: PostgresClient, command: Readonly, source: Readonly, execution: Readonly, observedAtMs: number, ): Promise { await client.query( ` INSERT INTO "ql3"."runs" ( id, project_id, task_id, task_revision, task_name, task_snapshot_ref, retry_of_run_id, trigger_type, execution_origin, execution_owner, triggered_by, request_id, status, version, event_sequence, priority, idempotency_key, input_ref, created_at_ms, queued_at_ms ) VALUES ( $1, $2, $3, $4, $5, $6, $7, 'run_manual_retry', 'manual', 'runtime', $8, $9, 'queued', 2, 2, $10, $11, $12, $13, $13 ) `, [ command.runId, command.projectId, source.taskId, source.taskRevision, source.taskName ?? null, source.taskSnapshotRef, command.sourceRunId, command.principal.subject.id, command.mutationId, source.priority, `ql3:run-manual-retry:v1:${command.mutationId}`, source.inputRef ?? null, observedAtMs, ], ); await client.query( ` INSERT INTO "ql3"."run_attempts" ( id, run_id, attempt, status, executor_type, callback_sequence, created_at_ms ) VALUES ($1, $2, 1, 'claimed', 'remote_worker', 0, $3) `, [command.attemptId, command.runId, observedAtMs], ); await client.query( ` INSERT INTO "ql3"."run_events" ( id, run_id, sequence, type, dedupe_key, actor_type, actor_id, attempt_id, payload, created_at_ms ) VALUES ($1, $2, 1, 'run.created', $3, $4, $5, $6, $7::jsonb, $8) `, [ command.createdEventId, command.runId, `run-manual-retry-created:${command.mutationId}`, command.principal.subject.type, command.principal.subject.id, command.attemptId, JSON.stringify({ status: 'created', version: 1, execution_owner: 'runtime', executor_type: 'remote_worker', execution_revision_digest: execution.contentDigest, source_content_digest: execution.sourceContentDigest, retry_of_run_id: command.sourceRunId, source_run_status: command.expectedRunStatus, source_run_version: command.expectedRunVersion, inherit_retry_policy: false, mutation_id: command.mutationId, authentication_id: command.principal.authenticationId, audit_event_id: command.auditEventId, policy_fence: { project_version: command.policyFence.projectVersion, binding_version: command.policyFence.bindingVersion, }, }), observedAtMs, ], ); await client.query( ` INSERT INTO "ql3"."run_events" ( id, run_id, sequence, type, dedupe_key, actor_type, actor_id, attempt_id, payload, created_at_ms ) VALUES ($1, $2, 2, 'run.queued', $3, $4, $5, $6, $7::jsonb, $8) `, [ command.queuedEventId, command.runId, `run-manual-retry-queued:${command.mutationId}`, command.principal.subject.type, command.principal.subject.id, command.attemptId, JSON.stringify({ from_status: 'created', to_status: 'queued', version: 2, }), observedAtMs, ], ); await client.query( ` INSERT INTO "ql3"."security_audit_events" ( event_id, request_id, operation_id, project_id, subject_type, subject_id, authentication_id, outcome, reasons, project_version, binding_version, occurred_at_ms ) VALUES ( $1, $2, 'run.retry', $3, $4, $5, $6, 'allowed', $7::jsonb, $8, $9, $10 ) `, [ command.auditEventId, command.requestId, command.projectId, command.principal.subject.type, command.principal.subject.id, command.principal.authenticationId, JSON.stringify(['role_grant', 'strong_authentication']), command.policyFence.projectVersion, command.policyFence.bindingVersion, observedAtMs, ], ); } /** * PostgreSQL authority for one strongly authenticated manual retry. The * Project row lock serializes the existing Run-ledger quota across replicas. */ export class PostgresRunManualRetryRepository implements RunManualRetryRepository { constructor(private readonly pool: PostgresPool) { if (!pool || typeof pool.connect !== 'function') { throw new TypeError('PostgreSQL Run manual retry pool is invalid'); } } async retryRun( value: Readonly, ): Promise> { const command = normalizeRunManualRetryCommand(value); for ( let transactionAttempt = 0; transactionAttempt < POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS; transactionAttempt += 1 ) { let client: PostgresClient; try { client = await this.pool.connect(); } catch (error) { throw unavailable({ cause: error }); } let began = false; try { await configurePostgresDefinitionTransaction(client); began = true; const observedAtMs = await databaseNow(client); confirmStrongAuthentication(command, observedAtMs); await confirmAuthorization(client, command); const replay = await findReplay(client, command); if (replay) { const result = replayResult(command, replay); await client.query('COMMIT'); began = false; return result; } const source = await findSource(client, command); const execution = await confirmTaskAndExecution( client, command.projectId, source, ); await consumeRateLimit(client, command, observedAtMs); await insertRetry(client, command, source, execution, observedAtMs); const result = normalizeRunManualRetryResult({ status: 'accepted', projectId: command.projectId, sourceRunId: command.sourceRunId, sourceRunStatus: command.expectedRunStatus, sourceRunVersion: command.expectedRunVersion, runId: command.runId, retryOfRunId: command.sourceRunId, taskId: source.taskId, taskRevision: source.taskRevision, attemptId: command.attemptId, runStatus: 'queued', runVersion: 2, eventSequence: 2, executorType: 'remote_worker', executionRevisionDigest: execution.contentDigest, createdAtMs: observedAtMs, }); await client.query('COMMIT'); began = false; return result; } catch (error) { if (began) await rollbackPostgresDefinitionTransaction(client); const state = postgresSqlState(error); if ( (state === '23505' || POSTGRES_DEFINITION_RETRYABLE_SQL_STATES.has(state ?? '')) && transactionAttempt + 1 < POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS ) { continue; } if ( error instanceof InvalidRunManualRetryError || error instanceof RunManualRetryNotFoundError || error instanceof RunManualRetryFenceRejectedError || error instanceof RunManualRetryRateLimitedError || error instanceof RunManualRetryUnavailableError ) { throw error; } throw unavailable({ cause: error }); } finally { client.release(); } } throw unavailable(); } }