// PostgreSQL authority for advancing the Plugin Package Workflow frontier. import type { PostgresClient, PostgresPool, RunRecord, } from '@qinglong/runtime-core'; import { InvalidPluginPackageWorkflowFrontierError, MAX_PLUGIN_PACKAGE_WORKFLOW_FRONTIER_PAGE_SIZE, PluginPackageWorkflowFrontierConflictError, PluginPackageWorkflowFrontierUnavailableError, resolvePluginPackageWorkflowFrontier, type PluginPackageWorkflowFrontierAdvanceResult, type PluginPackageWorkflowFrontierCandidate, type PluginPackageWorkflowFrontierCursor, type PluginPackageWorkflowFrontierPage, type PluginPackageWorkflowFrontierRepository, type PluginPackageWorkflowTerminalStatus, } from '@qinglong/runtime-core/plugin-package-workflow-frontier'; import { normalizePluginPackageWorkflowExecutionPlan, type PluginPackageWorkflowExecutionPlan, } from '@qinglong/runtime-core/plugin-package-workflow-execution-plan'; import { normalizeStepRunRecord, type StepRunMutation, type StepRunRecord, } from '@qinglong/runtime-core/step-run'; import { POSTGRES_DEFINITION_RETRYABLE_SQL_STATES, POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS, configurePostgresDefinitionTransaction, postgresRequiredBoolean, postgresRequiredInteger, postgresRequiredJsonObject, postgresRequiredString, postgresSqlState, rollbackPostgresDefinitionTransaction, } from '../../repository/definitionRepositorySupport'; type Row = Record; const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; const DIGEST = /^[0-9a-f]{64}$/; const TERMINAL_RUN_STATUSES = new Set([ 'succeeded', 'failed', 'cancelled', 'timed_out', ]); const RUN_SELECT = ` id, project_id AS "projectId", task_id AS "taskId", task_revision AS "taskRevision", task_name AS "taskName", task_snapshot_ref AS "taskSnapshotRef", legacy_cron_id AS "legacyCronId", parent_run_id AS "parentRunId", retry_of_run_id AS "retryOfRunId", trigger_id AS "triggerId", trigger_type AS "triggerType", execution_origin AS "executionOrigin", execution_owner AS "executionOwner", triggered_by AS "triggeredBy", request_id AS "requestId", scheduled_for_ms AS "scheduledForMs", status, version, event_sequence AS "eventSequence", priority, idempotency_key AS "idempotencyKey", input_ref AS "inputRef", output_ref AS "outputRef", created_at_ms AS "createdAtMs", queued_at_ms AS "queuedAtMs", started_at_ms AS "startedAtMs", finished_at_ms AS "finishedAtMs", cancel_requested_at_ms AS "cancelRequestedAtMs", cancel_reason AS "cancelReason", error_code AS "errorCode", error_summary AS "errorSummary" `.trim(); const STEP_RUN_SELECT = ` id, run_id AS "runId", parent_step_run_id AS "parentStepRunId", step_key AS "stepKey", kind, definition_ref AS "definitionRef", definition_digest AS "definitionDigest", required, status, version, attempt_count AS "attemptCount", input_ref AS "inputRef", output_ref AS "outputRef", approval_request_id AS "approvalRequestId", ready_at_ms AS "readyAtMs", started_at_ms AS "startedAtMs", finished_at_ms AS "finishedAtMs", result_code AS "resultCode", error_summary AS "errorSummary", created_at_ms AS "createdAtMs", updated_at_ms AS "updatedAtMs", last_mutation_id AS "lastMutationId", step_run_digest AS "stepRunDigest", step_run_json AS "stepRunJson" `.trim(); function unavailable( cause?: unknown, ): PluginPackageWorkflowFrontierUnavailableError { return new PluginPackageWorkflowFrontierUnavailableError({ cause: cause instanceof Error ? cause : undefined, }); } function text(row: Row, key: string): string { return postgresRequiredString(row[key], unavailable); } function integer(row: Row, key: string): number { const value = postgresRequiredInteger(row[key], unavailable); if (value < 0) throw unavailable(); return value; } function optionalText(row: Row, key: string): string | undefined { if (row[key] === null || row[key] === undefined) return undefined; return text(row, key); } function optionalInteger(row: Row, key: string): number | undefined { if (row[key] === null || row[key] === undefined) return undefined; return integer(row, key); } function identity(value: unknown, label: string): string { if (typeof value !== 'string' || !IDENTITY.test(value)) { throw new InvalidPluginPackageWorkflowFrontierError( `${label} is invalid`, ); } return value; } function digest(value: unknown): string { if (typeof value !== 'string' || !DIGEST.test(value)) throw unavailable(); return value; } function mapStorageError(error: unknown): Error { if ( error instanceof InvalidPluginPackageWorkflowFrontierError || error instanceof PluginPackageWorkflowFrontierConflictError || error instanceof PluginPackageWorkflowFrontierUnavailableError ) { return error; } return ['23503', '23505', '23514'].includes( postgresSqlState(error) ?? '', ) ? new PluginPackageWorkflowFrontierConflictError() : unavailable(error); } function pageLimit(value: unknown): number { if ( !Number.isInteger(value) || (value as number) < 1 || (value as number) > MAX_PLUGIN_PACKAGE_WORKFLOW_FRONTIER_PAGE_SIZE ) { throw new InvalidPluginPackageWorkflowFrontierError( `page limit must be between 1 and ${MAX_PLUGIN_PACKAGE_WORKFLOW_FRONTIER_PAGE_SIZE}`, ); } return value as number; } function cursor( value: Readonly | undefined, ): Readonly | undefined { if (value === undefined) return undefined; if ( !value || typeof value !== 'object' || Array.isArray(value) || Reflect.ownKeys(value).length !== 2 || !Reflect.has(value, 'admittedAtMs') || !Reflect.has(value, 'planDigest') || !Number.isSafeInteger(value.admittedAtMs) || value.admittedAtMs < 0 || !DIGEST.test(value.planDigest) ) { throw new InvalidPluginPackageWorkflowFrontierError( 'frontier cursor is invalid', ); } return Object.freeze({ admittedAtMs: value.admittedAtMs, planDigest: value.planDigest, }); } function runFromRow(row: Row): Readonly { const run: RunRecord = { id: text(row, 'id'), projectId: text(row, 'projectId'), taskId: text(row, 'taskId'), taskRevision: text(row, 'taskRevision'), triggerType: text(row, 'triggerType'), executionOrigin: text( row, 'executionOrigin', ) as RunRecord['executionOrigin'], executionOwner: text(row, 'executionOwner') as RunRecord['executionOwner'], status: text(row, 'status') as RunRecord['status'], version: integer(row, 'version'), eventSequence: integer(row, 'eventSequence'), priority: integer(row, 'priority'), createdAtMs: integer(row, 'createdAtMs'), }; const optionalTexts = [ ['taskName', 'taskName'], ['taskSnapshotRef', 'taskSnapshotRef'], ['parentRunId', 'parentRunId'], ['retryOfRunId', 'retryOfRunId'], ['triggerId', 'triggerId'], ['triggeredBy', 'triggeredBy'], ['requestId', 'requestId'], ['idempotencyKey', 'idempotencyKey'], ['inputRef', 'inputRef'], ['outputRef', 'outputRef'], ['errorCode', 'errorCode'], ['errorSummary', 'errorSummary'], ] as const; for (const [property, key] of optionalTexts) { const value = optionalText(row, key); if (value !== undefined) { (run as unknown as Record)[property] = value; } } const optionalIntegers = [ ['legacyCronId', 'legacyCronId'], ['scheduledForMs', 'scheduledForMs'], ['queuedAtMs', 'queuedAtMs'], ['startedAtMs', 'startedAtMs'], ['finishedAtMs', 'finishedAtMs'], ['cancelRequestedAtMs', 'cancelRequestedAtMs'], ] as const; for (const [property, key] of optionalIntegers) { const value = optionalInteger(row, key); if (value !== undefined) { (run as unknown as Record)[property] = value; } } const cancelReason = optionalText(row, 'cancelReason'); if (cancelReason !== undefined) { run.cancelReason = cancelReason as NonNullable; } return Object.freeze(run); } function stepRunFromRow(row: Row): Readonly { let stepRun: Readonly; try { stepRun = normalizeStepRunRecord( postgresRequiredJsonObject( row.stepRunJson, unavailable, ) as unknown as StepRunRecord, ); } catch { throw unavailable(); } if ( text(row, 'id') !== stepRun.id || text(row, 'runId') !== stepRun.runId || optionalText(row, 'parentStepRunId') !== (stepRun.parentStepRunId ?? undefined) || text(row, 'stepKey') !== stepRun.stepKey || text(row, 'kind') !== stepRun.kind || text(row, 'definitionRef') !== stepRun.definitionRef || text(row, 'definitionDigest') !== stepRun.definitionDigest || postgresRequiredBoolean(row.required, unavailable) !== stepRun.required || text(row, 'status') !== stepRun.status || integer(row, 'version') !== stepRun.version || integer(row, 'attemptCount') !== stepRun.attemptCount || optionalText(row, 'inputRef') !== (stepRun.inputRef ?? undefined) || optionalText(row, 'outputRef') !== (stepRun.outputRef ?? undefined) || optionalText(row, 'approvalRequestId') !== (stepRun.approvalRequestId ?? undefined) || optionalInteger(row, 'readyAtMs') !== (stepRun.readyAtMs ?? undefined) || optionalInteger(row, 'startedAtMs') !== (stepRun.startedAtMs ?? undefined) || optionalInteger(row, 'finishedAtMs') !== (stepRun.finishedAtMs ?? undefined) || optionalText(row, 'resultCode') !== (stepRun.resultCode ?? undefined) || optionalText(row, 'errorSummary') !== (stepRun.errorSummary ?? undefined) || integer(row, 'createdAtMs') !== stepRun.createdAtMs || integer(row, 'updatedAtMs') !== stepRun.updatedAtMs || text(row, 'lastMutationId') !== stepRun.lastMutationId || text(row, 'stepRunDigest') !== stepRun.stepRunDigest ) { throw unavailable(); } return stepRun; } function assertRunIdentity( run: Readonly, plan: Readonly, ): void { if ( run.id !== plan.runId || run.projectId !== plan.target.projectId || run.taskId !== plan.target.workflowId || run.taskRevision !== plan.target.publicationDigest || run.triggerType !== 'plugin_package_workflow' || run.executionOrigin !== 'system' || run.executionOwner !== 'runtime' || run.requestId !== plan.planId || run.idempotencyKey !== `plugin-package-workflow:${plan.planId}` || run.version !== run.eventSequence ) { throw unavailable(); } } async function updateStepRun( client: PostgresClient, mutation: Readonly, ): Promise { const stepRun = mutation.stepRun; const updated = await client.query( `UPDATE "ql3"."step_runs" SET status = $1, version = $2, attempt_count = $3, output_ref = $4, approval_request_id = $5, ready_at_ms = $6, started_at_ms = $7, finished_at_ms = $8, result_code = $9, error_summary = $10, updated_at_ms = $11, last_mutation_id = $12, step_run_digest = $13, step_run_json = $14::jsonb WHERE id = $15 AND run_id = $16 AND version = $17 AND step_run_digest = $18 AND status = $19`, [ stepRun.status, stepRun.version, stepRun.attemptCount, stepRun.outputRef, stepRun.approvalRequestId, stepRun.readyAtMs, stepRun.startedAtMs, stepRun.finishedAtMs, stepRun.resultCode, stepRun.errorSummary, stepRun.updatedAtMs, stepRun.lastMutationId, stepRun.stepRunDigest, JSON.stringify(stepRun), stepRun.id, stepRun.runId, mutation.expectedStepRunVersion, mutation.expectedStepRunDigest, mutation.previousStatus, ], ); if (updated.rowCount !== 1) { throw new PluginPackageWorkflowFrontierConflictError(); } } async function insertStepMutation( client: PostgresClient, mutation: Readonly, committedAtMs: number, ): Promise { const event = mutation.event; await client.query( `INSERT INTO "ql3"."run_events" ( id, run_id, sequence, type, dedupe_key, actor_type, actor_id, attempt_id, step_run_id, payload, created_at_ms ) VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, $8, $9::jsonb, $10)`, [ event.id, event.runId, event.sequence, event.type, event.dedupeKey ?? null, event.actorType, event.actorId ?? null, mutation.stepRun.id, JSON.stringify(event.payload), event.createdAtMs, ], ); await client.query( `INSERT INTO "ql3"."step_run_mutations" ( mutation_id, mutation_digest, run_id, step_run_id, step_run_digest, event_id, event_sequence, run_version, step_run_json, committed_at_ms ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10)`, [ mutation.mutationId, mutation.mutationDigest, mutation.runId, mutation.stepRun.id, mutation.stepRun.stepRunDigest, event.id, event.sequence, mutation.expectedRunVersion + 1, JSON.stringify(mutation.stepRun), committedAtMs, ], ); } export class PostgresPluginPackageWorkflowFrontierRepository implements PluginPackageWorkflowFrontierRepository { constructor(private readonly pool: PostgresPool) { if ( !pool || typeof pool.query !== 'function' || typeof pool.connect !== 'function' ) { throw unavailable(); } } async listCandidates(queryValue: Readonly<{ limit: number; after?: Readonly; }>): Promise> { if ( !queryValue || typeof queryValue !== 'object' || Array.isArray(queryValue) || !Reflect.has(queryValue, 'limit') || Reflect.ownKeys(queryValue).some( (key) => key !== 'limit' && key !== 'after', ) ) { throw new InvalidPluginPackageWorkflowFrontierError( 'page query is invalid', ); } const limit = pageLimit(queryValue.limit); const after = cursor(queryValue.after); try { const result = await this.pool.query( `SELECT admission.run_id AS "runId", admission.plan_digest AS "planDigest", admission.admitted_at_ms AS "admittedAtMs" FROM "ql3"."plugin_package_workflow_admissions" AS admission JOIN "ql3"."runs" AS run ON run.id = admission.run_id WHERE run.status = 'running' AND run.cancel_requested_at_ms IS NULL AND ( EXISTS ( SELECT 1 FROM "ql3"."plugin_package_workflow_admission_steps" AS step JOIN "ql3"."step_runs" AS current ON current.run_id = step.run_id AND current.id = step.step_run_id WHERE step.plan_digest = admission.plan_digest AND current.status = 'pending' AND ( NOT EXISTS ( SELECT 1 FROM jsonb_array_elements_text(step.needs_json) AS need(value) LEFT JOIN "ql3"."plugin_package_workflow_admission_steps" AS dependency_step ON dependency_step.plan_digest = step.plan_digest AND dependency_step.step_key = need.value LEFT JOIN "ql3"."step_runs" AS dependency ON dependency.run_id = dependency_step.run_id AND dependency.id = dependency_step.step_run_id WHERE dependency.id IS NULL OR dependency.status <> 'succeeded' ) OR EXISTS ( SELECT 1 FROM jsonb_array_elements_text(step.needs_json) AS need(value) JOIN "ql3"."plugin_package_workflow_admission_steps" AS dependency_step ON dependency_step.plan_digest = step.plan_digest AND dependency_step.step_key = need.value JOIN "ql3"."step_runs" AS dependency ON dependency.run_id = dependency_step.run_id AND dependency.id = dependency_step.step_run_id WHERE dependency.status IN ( 'failed', 'skipped', 'cancelled', 'timed_out' ) ) ) ) OR NOT EXISTS ( SELECT 1 FROM "ql3"."plugin_package_workflow_admission_steps" AS step JOIN "ql3"."step_runs" AS current ON current.run_id = step.run_id AND current.id = step.step_run_id WHERE step.plan_digest = admission.plan_digest AND current.status NOT IN ( 'succeeded', 'failed', 'skipped', 'cancelled', 'timed_out' ) ) ) AND ( $1::bigint IS NULL OR admission.admitted_at_ms > $1 OR (admission.admitted_at_ms = $1 AND admission.plan_digest > $2) ) ORDER BY admission.admitted_at_ms, admission.plan_digest LIMIT $3`, [ after?.admittedAtMs ?? null, after?.planDigest ?? '', limit + 1, ], ); const mapped = result.rows.map( (row): Readonly => Object.freeze({ runId: identity(text(row, 'runId'), 'candidate runId'), planDigest: digest(row.planDigest), admittedAtMs: integer(row, 'admittedAtMs'), }), ); const truncated = mapped.length > limit; const candidates = Object.freeze(mapped.slice(0, limit)); const last = candidates.at(-1); return Object.freeze({ candidates, truncated, ...(truncated && last ? { next: Object.freeze({ admittedAtMs: last.admittedAtMs, planDigest: last.planDigest, }), } : {}), }); } catch (error) { throw mapStorageError(error); } } async advance( runIdValue: string, ): Promise> { const runId = identity(runIdValue, 'runId'); for ( let attempt = 0; attempt < POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS; attempt += 1 ) { let client: PostgresClient; try { client = await this.pool.connect(); } catch (error) { throw unavailable(error); } let began = false; try { await configurePostgresDefinitionTransaction(client); began = true; const admission = await client.query( `SELECT plan_digest AS "planDigest", plan_json AS "planJson" FROM "ql3"."plugin_package_workflow_admissions" WHERE run_id = $1 LIMIT 2`, [runId], ); if (admission.rows.length !== 1) { throw new PluginPackageWorkflowFrontierConflictError(); } let plan: Readonly; try { plan = normalizePluginPackageWorkflowExecutionPlan( postgresRequiredJsonObject( admission.rows[0]!.planJson, unavailable, ) as unknown as PluginPackageWorkflowExecutionPlan, ); } catch { throw unavailable(); } if (plan.planDigest !== digest(admission.rows[0]!.planDigest)) { throw unavailable(); } const runRows = await client.query( `SELECT ${RUN_SELECT} FROM "ql3"."runs" WHERE id = $1 LIMIT 2 FOR UPDATE`, [runId], ); if (runRows.rows.length !== 1) { throw new PluginPackageWorkflowFrontierConflictError(); } const run = runFromRow(runRows.rows[0]!); assertRunIdentity(run, plan); const stepRows = await client.query( `SELECT ${STEP_RUN_SELECT} FROM "ql3"."step_runs" WHERE run_id = $1 ORDER BY step_key, id FOR UPDATE`, [runId], ); const stepRuns = stepRows.rows.map(stepRunFromRow); const clock = await client.query( `SELECT floor( extract(epoch FROM transaction_timestamp()) * 1000 )::bigint AS "observedAtMs"`, ); if (clock.rows.length !== 1) throw unavailable(); const observedAtMs = integer(clock.rows[0]!, 'observedAtMs'); const currentStatus = run.status; const resolution = resolvePluginPackageWorkflowFrontier({ plan, run: { ...run, ...(TERMINAL_RUN_STATUSES.has( currentStatus as PluginPackageWorkflowTerminalStatus, ) ? { status: 'running' as const } : {}), }, stepRuns, observedAtMs, }); if ( TERMINAL_RUN_STATUSES.has( currentStatus as PluginPackageWorkflowTerminalStatus, ) ) { if ( resolution.stepMutations.length !== 0 || resolution.terminalStatus !== currentStatus ) { throw unavailable(); } await client.query('COMMIT'); began = false; return Object.freeze({ status: 'settled' as const, runId, planDigest: plan.planDigest, stepMutationCount: 0, readyStepRunIds: Object.freeze([]), terminalStatus: currentStatus as PluginPackageWorkflowTerminalStatus, runVersion: run.version, runEventSequence: run.eventSequence, observedAtMs, }); } if (currentStatus !== 'running') throw unavailable(); const increment = resolution.stepMutations.length + (resolution.terminalTransition === null ? 0 : 1); if (increment === 0) { await client.query('COMMIT'); began = false; return Object.freeze({ status: 'unchanged' as const, runId, planDigest: plan.planDigest, stepMutationCount: 0, readyStepRunIds: resolution.readyStepRunIds, terminalStatus: null, runVersion: run.version, runEventSequence: run.eventSequence, observedAtMs, }); } for (const mutation of resolution.stepMutations) { await updateStepRun(client, mutation); } const terminal = resolution.terminalTransition; const updatedRun = await client.query( `UPDATE "ql3"."runs" SET status = $1, version = version + $2, event_sequence = event_sequence + $2, finished_at_ms = $3, error_code = $4, error_summary = NULL WHERE id = $5 AND status = 'running' AND version = $6 AND event_sequence = $7`, [ terminal?.status ?? 'running', increment, terminal?.finishedAtMs ?? null, terminal?.errorCode ?? null, runId, run.version, run.eventSequence, ], ); if (updatedRun.rowCount !== 1) { throw new PluginPackageWorkflowFrontierConflictError(); } for (const mutation of resolution.stepMutations) { await insertStepMutation(client, mutation, observedAtMs); } if (terminal) { const event = terminal.event; await client.query( `INSERT INTO "ql3"."run_events" ( id, run_id, sequence, type, dedupe_key, actor_type, actor_id, attempt_id, step_run_id, payload, created_at_ms ) VALUES ( $1, $2, $3, $4, $5, $6, $7, NULL, NULL, $8::jsonb, $9 )`, [ event.id, event.runId, event.sequence, event.type, event.dedupeKey ?? null, event.actorType, event.actorId ?? null, JSON.stringify(event.payload), event.createdAtMs, ], ); } await client.query('COMMIT'); began = false; return Object.freeze({ status: terminal ? ('terminal' as const) : ('advanced' as const), runId, planDigest: plan.planDigest, stepMutationCount: resolution.stepMutations.length, readyStepRunIds: terminal ? Object.freeze([]) : resolution.readyStepRunIds, terminalStatus: terminal?.status ?? null, runVersion: run.version + increment, runEventSequence: run.eventSequence + increment, observedAtMs, }); } catch (error) { if (began) await rollbackPostgresDefinitionTransaction(client); const state = postgresSqlState(error); if ( state && POSTGRES_DEFINITION_RETRYABLE_SQL_STATES.has(state) && attempt + 1 < POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS ) { continue; } throw mapStorageError(error); } finally { client.release(); } } throw unavailable(); } }