mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 09:58:46 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
import type { PostgresClient } from '@qinglong/runtime-core';
|
||||
|
||||
/**
|
||||
* Run-owned lock serializing every mutation and attestation that can change or certify one
|
||||
* Attempt's remote-execution authority. Callers must already be inside a
|
||||
* transaction and acquire this before row locks to keep one lock order.
|
||||
*/
|
||||
export async function lockAttemptAuthority(
|
||||
queryable: Pick<PostgresClient, 'query'>,
|
||||
attemptId: string,
|
||||
): Promise<void> {
|
||||
await queryable.query(
|
||||
'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))',
|
||||
[`ql3-attempt-authority:${attemptId}`],
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,929 @@
|
||||
// Owns PostgreSQL persistence for the Run aggregate, Attempts, retry policy, and events.
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunAttemptStatus,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
EXECUTION_ORIGINS,
|
||||
RUN_ATTEMPT_STATUSES,
|
||||
RUN_CANCELLATION_REASONS,
|
||||
RUN_EVENT_ACTOR_TYPES,
|
||||
RUN_STATUSES,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
assertRunRetryPolicyRecord,
|
||||
RUN_RETRY_SAFETIES,
|
||||
type RunRetryPolicyRecord,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
DuplicateIdempotencyKeyError,
|
||||
DuplicateRunAttemptError,
|
||||
DuplicateRunEventError,
|
||||
RunEventPayloadTooLargeError,
|
||||
RunRepositoryBusyError,
|
||||
RunRepositoryConstraintError,
|
||||
RunRepositoryError,
|
||||
RunRepositoryOperationError,
|
||||
} from '@qinglong/runtime-core';
|
||||
import type {
|
||||
RunRepository,
|
||||
RunRepositoryReader,
|
||||
RunRepositoryTransaction,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
normalizeProjectRunListQuery,
|
||||
type ProjectRunListQuery,
|
||||
type ProjectRunListReader,
|
||||
} from '@qinglong/runtime-core/project-run-list';
|
||||
import type {
|
||||
PostgresClient as PostgresRunClient,
|
||||
PostgresPool as PostgresRunPool,
|
||||
PostgresQueryable as PostgresRunQueryable,
|
||||
PostgresQueryResult as PostgresRunQueryResult,
|
||||
} from '@qinglong/runtime-core';
|
||||
|
||||
export type {
|
||||
PostgresClient as PostgresRunClient,
|
||||
PostgresPool as PostgresRunPool,
|
||||
PostgresQueryable as PostgresRunQueryable,
|
||||
PostgresQueryResult as PostgresRunQueryResult,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
MAX_CANCELLATION_RECOVERY_PAGE_SIZE,
|
||||
MAX_RUN_EVENT_PAGE_SIZE,
|
||||
MAX_RUN_EVENT_PAYLOAD_BYTES,
|
||||
} from '@qinglong/runtime-core';
|
||||
import { lockAttemptAuthority } from './attemptAuthorityLock';
|
||||
|
||||
interface ColumnDefinition {
|
||||
readonly column: string;
|
||||
readonly property: string;
|
||||
}
|
||||
|
||||
type QueryRow = Record<string, unknown>;
|
||||
|
||||
const POSTGRES_RUNTIME_STATEMENT_TIMEOUT_MS = 5_000;
|
||||
const POSTGRES_RUNTIME_LOCK_TIMEOUT_MS = 1_000;
|
||||
const POSTGRES_RUNTIME_IDLE_TRANSACTION_TIMEOUT_MS = 10_000;
|
||||
|
||||
const RUN_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
|
||||
{ column: 'id', property: 'id' },
|
||||
{ column: 'project_id', property: 'projectId' },
|
||||
{ column: 'task_id', property: 'taskId' },
|
||||
{ column: 'task_revision', property: 'taskRevision' },
|
||||
{ column: 'task_name', property: 'taskName' },
|
||||
{ column: 'task_snapshot_ref', property: 'taskSnapshotRef' },
|
||||
{ column: 'legacy_cron_id', property: 'legacyCronId' },
|
||||
{ column: 'parent_run_id', property: 'parentRunId' },
|
||||
{ column: 'retry_of_run_id', property: 'retryOfRunId' },
|
||||
{ column: 'trigger_id', property: 'triggerId' },
|
||||
{ column: 'trigger_type', property: 'triggerType' },
|
||||
{ column: 'execution_origin', property: 'executionOrigin' },
|
||||
{ column: 'execution_owner', property: 'executionOwner' },
|
||||
{ column: 'triggered_by', property: 'triggeredBy' },
|
||||
{ column: 'request_id', property: 'requestId' },
|
||||
{ column: 'scheduled_for_ms', property: 'scheduledForMs' },
|
||||
{ column: 'status', property: 'status' },
|
||||
{ column: 'version', property: 'version' },
|
||||
{ column: 'event_sequence', property: 'eventSequence' },
|
||||
{ column: 'priority', property: 'priority' },
|
||||
{ column: 'idempotency_key', property: 'idempotencyKey' },
|
||||
{ column: 'input_ref', property: 'inputRef' },
|
||||
{ column: 'output_ref', property: 'outputRef' },
|
||||
{ column: 'created_at_ms', property: 'createdAtMs' },
|
||||
{ column: 'queued_at_ms', property: 'queuedAtMs' },
|
||||
{ column: 'started_at_ms', property: 'startedAtMs' },
|
||||
{ column: 'finished_at_ms', property: 'finishedAtMs' },
|
||||
{ column: 'cancel_requested_at_ms', property: 'cancelRequestedAtMs' },
|
||||
{ column: 'cancel_reason', property: 'cancelReason' },
|
||||
{ column: 'error_code', property: 'errorCode' },
|
||||
{ column: 'error_summary', property: 'errorSummary' },
|
||||
]);
|
||||
|
||||
const ATTEMPT_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
|
||||
{ column: 'id', property: 'id' },
|
||||
{ column: 'run_id', property: 'runId' },
|
||||
{ column: 'step_run_id', property: 'stepRunId' },
|
||||
{ column: 'attempt', property: 'attempt' },
|
||||
{ column: 'status', property: 'status' },
|
||||
{ column: 'executor_type', property: 'executorType' },
|
||||
{ column: 'worker_id', property: 'workerId' },
|
||||
{ column: 'worker_session_id', property: 'workerSessionId' },
|
||||
{ column: 'worker_generation', property: 'workerGeneration' },
|
||||
{ column: 'executor_handle', property: 'executorHandle' },
|
||||
{ column: 'pid', property: 'pid' },
|
||||
{ column: 'log_artifact_id', property: 'logArtifactId' },
|
||||
{ column: 'lease_token', property: 'leaseToken' },
|
||||
{ column: 'lease_token_digest', property: 'leaseTokenDigest' },
|
||||
{ column: 'lease_generation', property: 'leaseGeneration' },
|
||||
{ column: 'lease_version', property: 'leaseVersion' },
|
||||
{ column: 'lease_expires_at_ms', property: 'leaseExpiresAtMs' },
|
||||
{ column: 'offer_id', property: 'offerId' },
|
||||
{ column: 'deadline_at_ms', property: 'deadlineAtMs' },
|
||||
{ column: 'callback_token_hash', property: 'callbackTokenHash' },
|
||||
{ column: 'callback_sequence', property: 'callbackSequence' },
|
||||
{ column: 'created_at_ms', property: 'createdAtMs' },
|
||||
{ column: 'started_at_ms', property: 'startedAtMs' },
|
||||
{ column: 'finished_at_ms', property: 'finishedAtMs' },
|
||||
{ column: 'exit_code', property: 'exitCode' },
|
||||
{ column: 'error_code', property: 'errorCode' },
|
||||
{ column: 'error_summary', property: 'errorSummary' },
|
||||
]);
|
||||
|
||||
const EVENT_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
|
||||
{ column: 'id', property: 'id' },
|
||||
{ column: 'run_id', property: 'runId' },
|
||||
{ column: 'sequence', property: 'sequence' },
|
||||
{ column: 'type', property: 'type' },
|
||||
{ column: 'dedupe_key', property: 'dedupeKey' },
|
||||
{ column: 'actor_type', property: 'actorType' },
|
||||
{ column: 'actor_id', property: 'actorId' },
|
||||
{ column: 'attempt_id', property: 'attemptId' },
|
||||
{ column: 'step_run_id', property: 'stepRunId' },
|
||||
{ column: 'payload', property: 'payload' },
|
||||
{ column: 'created_at_ms', property: 'createdAtMs' },
|
||||
]);
|
||||
|
||||
const RETRY_POLICY_COLUMNS: readonly ColumnDefinition[] = Object.freeze([
|
||||
{ column: 'run_id', property: 'runId' },
|
||||
{ column: 'max_attempts', property: 'maxAttempts' },
|
||||
{ column: 'retry_on_lost', property: 'retryOnLost' },
|
||||
{ column: 'safety', property: 'safety' },
|
||||
{ column: 'backoff_base_ms', property: 'backoffBaseMs' },
|
||||
{ column: 'backoff_max_ms', property: 'backoffMaxMs' },
|
||||
{ column: 'next_attempt_at_ms', property: 'nextAttemptAtMs' },
|
||||
{ column: 'version', property: 'version' },
|
||||
{ column: 'created_at_ms', property: 'createdAtMs' },
|
||||
{ column: 'updated_at_ms', property: 'updatedAtMs' },
|
||||
]);
|
||||
|
||||
const TERMINAL_RUN_STATUSES = Object.freeze([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
]);
|
||||
|
||||
const BUSY_SQL_STATES = new Set([
|
||||
'08000',
|
||||
'08001',
|
||||
'08003',
|
||||
'08004',
|
||||
'08006',
|
||||
'08007',
|
||||
'08P01',
|
||||
'40001',
|
||||
'40P01',
|
||||
'55P03',
|
||||
'57014',
|
||||
'57P01',
|
||||
'57P02',
|
||||
'57P03',
|
||||
]);
|
||||
|
||||
const RUN_IDEMPOTENCY_CONSTRAINT = 'ql3_runs_project_idempotency_uidx';
|
||||
const ATTEMPT_NUMBER_CONSTRAINT = 'ql3_run_attempts_run_attempt_uidx';
|
||||
const EVENT_SEQUENCE_CONSTRAINT = 'ql3_run_events_run_sequence_uidx';
|
||||
const EVENT_DEDUPE_CONSTRAINT = 'ql3_run_events_run_dedupe_uidx';
|
||||
|
||||
function quoted(identifier: string): string {
|
||||
return `"${identifier}"`;
|
||||
}
|
||||
|
||||
function selectColumns(columns: readonly ColumnDefinition[]): string {
|
||||
return columns
|
||||
.map(({ column, property }) => `${quoted(column)} AS ${quoted(property)}`)
|
||||
.join(', ');
|
||||
}
|
||||
|
||||
function insertSql(
|
||||
tableName: string,
|
||||
columns: readonly ColumnDefinition[],
|
||||
): string {
|
||||
return `INSERT INTO "ql3".${quoted(tableName)} (${columns
|
||||
.map(({ column }) => quoted(column))
|
||||
.join(', ')}) VALUES (${columns
|
||||
.map((_, index) => `$${index + 1}`)
|
||||
.join(', ')})`;
|
||||
}
|
||||
|
||||
function updateSql(
|
||||
tableName: string,
|
||||
columns: readonly ColumnDefinition[],
|
||||
predicate: string,
|
||||
): string {
|
||||
const identityColumn = columns[0];
|
||||
if (!identityColumn) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'PostgreSQL Run repository requires an identity column for updates',
|
||||
);
|
||||
}
|
||||
const mutableColumns = columns.slice(1);
|
||||
return `UPDATE "ql3".${quoted(tableName)} SET ${mutableColumns
|
||||
.map(({ column }, index) => `${quoted(column)} = $${index + 2}`)
|
||||
.join(', ')} WHERE ${predicate} RETURNING ${quoted(identityColumn.column)}`;
|
||||
}
|
||||
|
||||
function writeValues(
|
||||
record: object,
|
||||
columns: readonly ColumnDefinition[],
|
||||
): unknown[] {
|
||||
const values = record as Record<string, unknown>;
|
||||
return columns.map(({ property }) => values[property] ?? null);
|
||||
}
|
||||
|
||||
function requiredString(row: QueryRow, property: string): string {
|
||||
const value = row[property];
|
||||
if (typeof value !== 'string' || value.length === 0) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
`PostgreSQL Run row has an invalid ${property}`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function optionalString(row: QueryRow, property: string): string | undefined {
|
||||
const value = row[property];
|
||||
if (value === null || value === undefined) return undefined;
|
||||
if (typeof value !== 'string') {
|
||||
throw new RunRepositoryConstraintError(
|
||||
`PostgreSQL Run row has an invalid ${property}`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredInteger(row: QueryRow, property: string): number {
|
||||
const value = row[property];
|
||||
if (typeof value === 'number' && Number.isSafeInteger(value)) return value;
|
||||
if (typeof value === 'string' && /^-?(0|[1-9]\d*)$/.test(value)) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isSafeInteger(parsed)) return parsed;
|
||||
}
|
||||
throw new RunRepositoryConstraintError(
|
||||
`PostgreSQL Run row has an invalid ${property}`,
|
||||
);
|
||||
}
|
||||
|
||||
function optionalInteger(row: QueryRow, property: string): number | undefined {
|
||||
if (row[property] === null || row[property] === undefined) return undefined;
|
||||
return requiredInteger(row, property);
|
||||
}
|
||||
|
||||
function requiredBoolean(row: QueryRow, property: string): boolean {
|
||||
const value = row[property];
|
||||
if (typeof value !== 'boolean') {
|
||||
throw new RunRepositoryConstraintError(
|
||||
`PostgreSQL Run row has an invalid ${property}`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredEnum<T extends string>(
|
||||
row: QueryRow,
|
||||
property: string,
|
||||
allowed: readonly T[],
|
||||
): T {
|
||||
const value = requiredString(row, property);
|
||||
if (!allowed.includes(value as T)) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
`PostgreSQL Run row has an unsupported ${property}`,
|
||||
);
|
||||
}
|
||||
return value as T;
|
||||
}
|
||||
|
||||
function assignOptional<T extends object, K extends keyof T>(
|
||||
record: T,
|
||||
key: K,
|
||||
value: T[K] | undefined,
|
||||
): void {
|
||||
if (value !== undefined) record[key] = value;
|
||||
}
|
||||
|
||||
function rowToRun(row: QueryRow): RunRecord {
|
||||
const run: RunRecord = {
|
||||
id: requiredString(row, 'id'),
|
||||
projectId: requiredString(row, 'projectId'),
|
||||
taskId: requiredString(row, 'taskId'),
|
||||
taskRevision: requiredString(row, 'taskRevision'),
|
||||
triggerType: requiredString(row, 'triggerType'),
|
||||
executionOrigin: requiredEnum(row, 'executionOrigin', EXECUTION_ORIGINS),
|
||||
executionOwner: requiredEnum(row, 'executionOwner', [
|
||||
'legacy',
|
||||
'runtime',
|
||||
] as const),
|
||||
status: requiredEnum(row, 'status', RUN_STATUSES),
|
||||
version: requiredInteger(row, 'version'),
|
||||
eventSequence: requiredInteger(row, 'eventSequence'),
|
||||
priority: requiredInteger(row, 'priority'),
|
||||
createdAtMs: requiredInteger(row, 'createdAtMs'),
|
||||
};
|
||||
assignOptional(run, 'taskName', optionalString(row, 'taskName'));
|
||||
assignOptional(
|
||||
run,
|
||||
'taskSnapshotRef',
|
||||
optionalString(row, 'taskSnapshotRef'),
|
||||
);
|
||||
assignOptional(run, 'legacyCronId', optionalInteger(row, 'legacyCronId'));
|
||||
assignOptional(run, 'parentRunId', optionalString(row, 'parentRunId'));
|
||||
assignOptional(run, 'retryOfRunId', optionalString(row, 'retryOfRunId'));
|
||||
assignOptional(run, 'triggerId', optionalString(row, 'triggerId'));
|
||||
assignOptional(run, 'triggeredBy', optionalString(row, 'triggeredBy'));
|
||||
assignOptional(run, 'requestId', optionalString(row, 'requestId'));
|
||||
assignOptional(run, 'scheduledForMs', optionalInteger(row, 'scheduledForMs'));
|
||||
assignOptional(run, 'idempotencyKey', optionalString(row, 'idempotencyKey'));
|
||||
assignOptional(run, 'inputRef', optionalString(row, 'inputRef'));
|
||||
assignOptional(run, 'outputRef', optionalString(row, 'outputRef'));
|
||||
assignOptional(run, 'queuedAtMs', optionalInteger(row, 'queuedAtMs'));
|
||||
assignOptional(run, 'startedAtMs', optionalInteger(row, 'startedAtMs'));
|
||||
assignOptional(run, 'finishedAtMs', optionalInteger(row, 'finishedAtMs'));
|
||||
assignOptional(
|
||||
run,
|
||||
'cancelRequestedAtMs',
|
||||
optionalInteger(row, 'cancelRequestedAtMs'),
|
||||
);
|
||||
if (row.cancelReason !== null && row.cancelReason !== undefined) {
|
||||
run.cancelReason = requiredEnum(
|
||||
row,
|
||||
'cancelReason',
|
||||
RUN_CANCELLATION_REASONS,
|
||||
);
|
||||
}
|
||||
assignOptional(run, 'errorCode', optionalString(row, 'errorCode'));
|
||||
assignOptional(run, 'errorSummary', optionalString(row, 'errorSummary'));
|
||||
return run;
|
||||
}
|
||||
|
||||
function rowToAttempt(row: QueryRow): RunAttemptRecord {
|
||||
const attempt: RunAttemptRecord = {
|
||||
id: requiredString(row, 'id'),
|
||||
runId: requiredString(row, 'runId'),
|
||||
attempt: requiredInteger(row, 'attempt'),
|
||||
status: requiredEnum(row, 'status', RUN_ATTEMPT_STATUSES),
|
||||
executorType: requiredString(row, 'executorType'),
|
||||
callbackSequence: requiredInteger(row, 'callbackSequence'),
|
||||
createdAtMs: requiredInteger(row, 'createdAtMs'),
|
||||
};
|
||||
assignOptional(attempt, 'stepRunId', optionalString(row, 'stepRunId'));
|
||||
assignOptional(attempt, 'workerId', optionalString(row, 'workerId'));
|
||||
assignOptional(
|
||||
attempt,
|
||||
'workerSessionId',
|
||||
optionalString(row, 'workerSessionId'),
|
||||
);
|
||||
assignOptional(
|
||||
attempt,
|
||||
'workerGeneration',
|
||||
optionalInteger(row, 'workerGeneration'),
|
||||
);
|
||||
assignOptional(
|
||||
attempt,
|
||||
'executorHandle',
|
||||
optionalString(row, 'executorHandle'),
|
||||
);
|
||||
assignOptional(attempt, 'pid', optionalInteger(row, 'pid'));
|
||||
assignOptional(
|
||||
attempt,
|
||||
'logArtifactId',
|
||||
optionalString(row, 'logArtifactId'),
|
||||
);
|
||||
assignOptional(attempt, 'leaseToken', optionalString(row, 'leaseToken'));
|
||||
assignOptional(
|
||||
attempt,
|
||||
'leaseTokenDigest',
|
||||
optionalString(row, 'leaseTokenDigest'),
|
||||
);
|
||||
assignOptional(
|
||||
attempt,
|
||||
'leaseGeneration',
|
||||
optionalInteger(row, 'leaseGeneration'),
|
||||
);
|
||||
assignOptional(attempt, 'leaseVersion', optionalInteger(row, 'leaseVersion'));
|
||||
assignOptional(
|
||||
attempt,
|
||||
'leaseExpiresAtMs',
|
||||
optionalInteger(row, 'leaseExpiresAtMs'),
|
||||
);
|
||||
assignOptional(attempt, 'offerId', optionalString(row, 'offerId'));
|
||||
assignOptional(attempt, 'deadlineAtMs', optionalInteger(row, 'deadlineAtMs'));
|
||||
assignOptional(
|
||||
attempt,
|
||||
'callbackTokenHash',
|
||||
optionalString(row, 'callbackTokenHash'),
|
||||
);
|
||||
assignOptional(attempt, 'startedAtMs', optionalInteger(row, 'startedAtMs'));
|
||||
assignOptional(attempt, 'finishedAtMs', optionalInteger(row, 'finishedAtMs'));
|
||||
assignOptional(attempt, 'exitCode', optionalInteger(row, 'exitCode'));
|
||||
assignOptional(attempt, 'errorCode', optionalString(row, 'errorCode'));
|
||||
assignOptional(attempt, 'errorSummary', optionalString(row, 'errorSummary'));
|
||||
return attempt;
|
||||
}
|
||||
|
||||
function normalizePayload(payload: unknown): Readonly<Record<string, unknown>> {
|
||||
let value = payload;
|
||||
if (typeof value === 'string') {
|
||||
try {
|
||||
value = JSON.parse(value);
|
||||
} catch (error) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'PostgreSQL RunEvent payload is invalid JSON',
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'PostgreSQL RunEvent payload is not a JSON object',
|
||||
);
|
||||
}
|
||||
return value as Readonly<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
function rowToEvent(row: QueryRow): RunEventRecord {
|
||||
const event: RunEventRecord = {
|
||||
id: requiredString(row, 'id'),
|
||||
runId: requiredString(row, 'runId'),
|
||||
sequence: requiredInteger(row, 'sequence'),
|
||||
type: requiredString(row, 'type'),
|
||||
actorType: requiredEnum(row, 'actorType', RUN_EVENT_ACTOR_TYPES),
|
||||
payload: normalizePayload(row.payload),
|
||||
createdAtMs: requiredInteger(row, 'createdAtMs'),
|
||||
};
|
||||
assignOptional(event, 'dedupeKey', optionalString(row, 'dedupeKey'));
|
||||
assignOptional(event, 'actorId', optionalString(row, 'actorId'));
|
||||
assignOptional(event, 'attemptId', optionalString(row, 'attemptId'));
|
||||
assignOptional(event, 'stepRunId', optionalString(row, 'stepRunId'));
|
||||
return event;
|
||||
}
|
||||
|
||||
function rowToRetryPolicy(row: QueryRow): RunRetryPolicyRecord {
|
||||
const policy: RunRetryPolicyRecord = {
|
||||
runId: requiredString(row, 'runId'),
|
||||
maxAttempts: requiredInteger(row, 'maxAttempts'),
|
||||
retryOnLost: requiredBoolean(row, 'retryOnLost'),
|
||||
safety: requiredEnum(row, 'safety', RUN_RETRY_SAFETIES),
|
||||
backoffBaseMs: requiredInteger(row, 'backoffBaseMs'),
|
||||
backoffMaxMs: requiredInteger(row, 'backoffMaxMs'),
|
||||
version: requiredInteger(row, 'version'),
|
||||
createdAtMs: requiredInteger(row, 'createdAtMs'),
|
||||
updatedAtMs: requiredInteger(row, 'updatedAtMs'),
|
||||
};
|
||||
assignOptional(
|
||||
policy,
|
||||
'nextAttemptAtMs',
|
||||
optionalInteger(row, 'nextAttemptAtMs'),
|
||||
);
|
||||
assertRunRetryPolicyRecord(policy);
|
||||
return policy;
|
||||
}
|
||||
|
||||
function sqlState(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
const value = (error as { code?: unknown }).code;
|
||||
return typeof value === 'string' ? value : undefined;
|
||||
}
|
||||
|
||||
function constraintName(error: unknown): string | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
const value = (error as { constraint?: unknown }).constraint;
|
||||
return typeof value === 'string' ? value : undefined;
|
||||
}
|
||||
|
||||
function mapPostgresError(error: unknown): RunRepositoryError {
|
||||
if (error instanceof RunRepositoryError) return error;
|
||||
const state = sqlState(error);
|
||||
if (state && BUSY_SQL_STATES.has(state)) {
|
||||
return new RunRepositoryBusyError(error);
|
||||
}
|
||||
if (state?.startsWith('23')) {
|
||||
return new RunRepositoryConstraintError(
|
||||
'PostgreSQL Run repository constraint violation',
|
||||
error,
|
||||
);
|
||||
}
|
||||
return new RunRepositoryOperationError(error);
|
||||
}
|
||||
|
||||
function affectedOneOrNone(result: PostgresRunQueryResult): boolean {
|
||||
const count = result.rowCount ?? result.rows.length;
|
||||
if (count === 0) return false;
|
||||
if (count === 1) return true;
|
||||
throw new RunRepositoryConstraintError(
|
||||
'PostgreSQL compare-and-set affected more than one row',
|
||||
);
|
||||
}
|
||||
|
||||
function assertEventPayloadSize(event: RunEventRecord): void {
|
||||
let serialized: string;
|
||||
try {
|
||||
serialized = JSON.stringify(event.payload);
|
||||
} catch (error) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'RunEvent payload is not JSON serializable',
|
||||
error,
|
||||
);
|
||||
}
|
||||
const bytes = Buffer.byteLength(serialized, 'utf8');
|
||||
if (bytes > MAX_RUN_EVENT_PAYLOAD_BYTES) {
|
||||
throw new RunEventPayloadTooLargeError(bytes, MAX_RUN_EVENT_PAYLOAD_BYTES);
|
||||
}
|
||||
}
|
||||
|
||||
async function queryMapped<TRow extends QueryRow = QueryRow>(
|
||||
queryable: PostgresRunQueryable,
|
||||
text: string,
|
||||
values?: readonly unknown[],
|
||||
): Promise<PostgresRunQueryResult<TRow>> {
|
||||
try {
|
||||
return await queryable.query<TRow>(text, values);
|
||||
} catch (error) {
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
}
|
||||
|
||||
function singleRow<TRow extends QueryRow>(
|
||||
result: PostgresRunQueryResult<TRow>,
|
||||
): TRow | null {
|
||||
const [row] = result.rows;
|
||||
if (!row) return null;
|
||||
if (result.rows.length !== 1) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'PostgreSQL Run repository returned duplicate identity rows',
|
||||
);
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
const RUN_SELECT = selectColumns(RUN_COLUMNS);
|
||||
const ATTEMPT_SELECT = selectColumns(ATTEMPT_COLUMNS);
|
||||
const EVENT_SELECT = selectColumns(EVENT_COLUMNS);
|
||||
const RETRY_POLICY_SELECT = selectColumns(RETRY_POLICY_COLUMNS);
|
||||
|
||||
const INSERT_RUN_SQL = insertSql('runs', RUN_COLUMNS);
|
||||
const INSERT_ATTEMPT_SQL = insertSql('run_attempts', ATTEMPT_COLUMNS);
|
||||
const INSERT_EVENT_SQL = insertSql('run_events', EVENT_COLUMNS);
|
||||
const INSERT_RETRY_POLICY_SQL = insertSql(
|
||||
'run_retry_policies',
|
||||
RETRY_POLICY_COLUMNS,
|
||||
);
|
||||
const UPDATE_RUN_SQL = updateSql(
|
||||
'runs',
|
||||
RUN_COLUMNS,
|
||||
`"id" = $1 AND "version" = $${RUN_COLUMNS.length + 1}`,
|
||||
);
|
||||
const UPDATE_ATTEMPT_SQL = updateSql(
|
||||
'run_attempts',
|
||||
ATTEMPT_COLUMNS,
|
||||
`"id" = $1 AND "status" = $${
|
||||
ATTEMPT_COLUMNS.length + 1
|
||||
} AND "callback_sequence" = $${ATTEMPT_COLUMNS.length + 2}`,
|
||||
);
|
||||
const UPDATE_RETRY_POLICY_SQL = updateSql(
|
||||
'run_retry_policies',
|
||||
RETRY_POLICY_COLUMNS,
|
||||
`"run_id" = $1 AND "version" = $${RETRY_POLICY_COLUMNS.length + 1}`,
|
||||
);
|
||||
|
||||
class PostgresRunReader implements RunRepositoryReader, ProjectRunListReader {
|
||||
constructor(protected readonly queryable: PostgresRunQueryable) {}
|
||||
|
||||
async findRunById(runId: string): Promise<RunRecord | null> {
|
||||
const row = singleRow(
|
||||
await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${RUN_SELECT} FROM "ql3"."runs" WHERE "id" = $1`,
|
||||
[runId],
|
||||
),
|
||||
);
|
||||
return row ? rowToRun(row) : null;
|
||||
}
|
||||
|
||||
async listRunsByProject(
|
||||
value: Readonly<ProjectRunListQuery>,
|
||||
): Promise<readonly RunRecord[]> {
|
||||
const query = normalizeProjectRunListQuery(value);
|
||||
const result = await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${RUN_SELECT} FROM "ql3"."runs"
|
||||
WHERE "project_id" = $1
|
||||
AND (
|
||||
$2::varchar IS NULL
|
||||
OR "created_at_ms" < $3
|
||||
OR ("created_at_ms" = $3 AND "id" < $2)
|
||||
)
|
||||
ORDER BY "created_at_ms" DESC, "id" DESC
|
||||
LIMIT $4`,
|
||||
[
|
||||
query.projectId,
|
||||
query.after?.runId ?? null,
|
||||
query.after?.createdAtMs ?? 0,
|
||||
query.limit,
|
||||
],
|
||||
);
|
||||
return result.rows.map(rowToRun);
|
||||
}
|
||||
|
||||
async findAttemptById(attemptId: string): Promise<RunAttemptRecord | null> {
|
||||
const row = singleRow(
|
||||
await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${ATTEMPT_SELECT} FROM "ql3"."run_attempts" WHERE "id" = $1`,
|
||||
[attemptId],
|
||||
),
|
||||
);
|
||||
return row ? rowToAttempt(row) : null;
|
||||
}
|
||||
|
||||
async findLatestAttemptByRunId(
|
||||
runId: string,
|
||||
): Promise<RunAttemptRecord | null> {
|
||||
const row = singleRow(
|
||||
await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${ATTEMPT_SELECT} FROM "ql3"."run_attempts" WHERE "run_id" = $1 ORDER BY "attempt" DESC, "id" DESC LIMIT 1`,
|
||||
[runId],
|
||||
),
|
||||
);
|
||||
return row ? rowToAttempt(row) : null;
|
||||
}
|
||||
|
||||
async findRetryPolicyByRunId(
|
||||
runId: string,
|
||||
): Promise<RunRetryPolicyRecord | null> {
|
||||
const row = singleRow(
|
||||
await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${RETRY_POLICY_SELECT} FROM "ql3"."run_retry_policies" WHERE "run_id" = $1`,
|
||||
[runId],
|
||||
),
|
||||
);
|
||||
return row ? rowToRetryPolicy(row) : null;
|
||||
}
|
||||
|
||||
async listEvents(
|
||||
runId: string,
|
||||
options: { afterSequence?: number; limit?: number } = {},
|
||||
): Promise<RunEventRecord[]> {
|
||||
const afterSequence = options.afterSequence ?? 0;
|
||||
const limit = options.limit ?? 100;
|
||||
if (!Number.isInteger(afterSequence) || afterSequence < 0) {
|
||||
throw new RangeError('afterSequence must be a non-negative integer');
|
||||
}
|
||||
if (
|
||||
!Number.isInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_RUN_EVENT_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_RUN_EVENT_PAGE_SIZE',
|
||||
);
|
||||
}
|
||||
const result = await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${EVENT_SELECT} FROM "ql3"."run_events" WHERE "run_id" = $1 AND "sequence" > $2 ORDER BY "sequence", "id" LIMIT $3`,
|
||||
[runId, afterSequence, limit],
|
||||
);
|
||||
return result.rows.map(rowToEvent);
|
||||
}
|
||||
|
||||
async listCancellationRequested(
|
||||
options: { beforeMs?: number; limit?: number } = {},
|
||||
): Promise<RunRecord[]> {
|
||||
const beforeMs = options.beforeMs;
|
||||
const limit = options.limit ?? 100;
|
||||
if (
|
||||
beforeMs !== undefined &&
|
||||
(!Number.isSafeInteger(beforeMs) || beforeMs < 0)
|
||||
) {
|
||||
throw new RangeError('beforeMs must be a non-negative safe integer');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_CANCELLATION_RECOVERY_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'limit must be between 1 and MAX_CANCELLATION_RECOVERY_PAGE_SIZE',
|
||||
);
|
||||
}
|
||||
const result = await queryMapped(
|
||||
this.queryable,
|
||||
`SELECT ${RUN_SELECT} FROM "ql3"."runs" WHERE "status" <> ALL($1::text[]) AND "cancel_requested_at_ms" IS NOT NULL AND ($2::bigint IS NULL OR "cancel_requested_at_ms" <= $2) ORDER BY "cancel_requested_at_ms", "id" LIMIT $3`,
|
||||
[TERMINAL_RUN_STATUSES, beforeMs ?? null, limit],
|
||||
);
|
||||
return result.rows.map(rowToRun);
|
||||
}
|
||||
}
|
||||
|
||||
export class PostgresRunTransaction
|
||||
extends PostgresRunReader
|
||||
implements RunRepositoryTransaction
|
||||
{
|
||||
private async assertRunTaskRevisionIsNotQuarantined(
|
||||
run: RunRecord,
|
||||
): Promise<void> {
|
||||
if (run.status !== 'dispatching' && run.status !== 'running') return;
|
||||
const current = await this.findRunById(run.id);
|
||||
if (
|
||||
current &&
|
||||
(current.status === 'dispatching' || current.status === 'running') &&
|
||||
current.taskRevision === run.taskRevision
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const result = await this.queryable.query(
|
||||
`SELECT "ql3"."plugin_package_run_start_allowed"(
|
||||
$1::varchar, $2::varchar, $3::varchar
|
||||
) AS "allowed"`,
|
||||
[run.projectId, run.taskId, run.taskRevision],
|
||||
);
|
||||
if (result.rows.length !== 1 || result.rows[0]?.allowed !== true) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'Run Task revision belongs to a quarantined Package lock',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async insertRun(run: RunRecord): Promise<void> {
|
||||
try {
|
||||
await this.queryable.query(INSERT_RUN_SQL, writeValues(run, RUN_COLUMNS));
|
||||
} catch (error) {
|
||||
if (
|
||||
sqlState(error) === '23505' &&
|
||||
constraintName(error) === RUN_IDEMPOTENCY_CONSTRAINT &&
|
||||
run.idempotencyKey
|
||||
) {
|
||||
throw new DuplicateIdempotencyKeyError(
|
||||
run.projectId,
|
||||
run.idempotencyKey,
|
||||
);
|
||||
}
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async insertAttempt(attempt: RunAttemptRecord): Promise<void> {
|
||||
try {
|
||||
await this.queryable.query(
|
||||
INSERT_ATTEMPT_SQL,
|
||||
writeValues(attempt, ATTEMPT_COLUMNS),
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
sqlState(error) === '23505' &&
|
||||
constraintName(error) === ATTEMPT_NUMBER_CONSTRAINT
|
||||
) {
|
||||
throw new DuplicateRunAttemptError(attempt.runId, attempt.attempt);
|
||||
}
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async insertRetryPolicy(policy: RunRetryPolicyRecord): Promise<void> {
|
||||
assertRunRetryPolicyRecord(policy);
|
||||
try {
|
||||
await this.queryable.query(
|
||||
INSERT_RETRY_POLICY_SQL,
|
||||
writeValues(policy, RETRY_POLICY_COLUMNS),
|
||||
);
|
||||
} catch (error) {
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async compareAndSetRun(
|
||||
run: RunRecord,
|
||||
expectedVersion: number,
|
||||
): Promise<boolean> {
|
||||
if (run.version !== expectedVersion + 1) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'A compare-and-set Run write must increment version exactly once',
|
||||
);
|
||||
}
|
||||
try {
|
||||
await this.assertRunTaskRevisionIsNotQuarantined(run);
|
||||
const result = await queryMapped(this.queryable, UPDATE_RUN_SQL, [
|
||||
...writeValues(run, RUN_COLUMNS),
|
||||
expectedVersion,
|
||||
]);
|
||||
return affectedOneOrNone(result);
|
||||
} catch (error) {
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async compareAndSetAttempt(
|
||||
attempt: RunAttemptRecord,
|
||||
expected: {
|
||||
status: RunAttemptStatus;
|
||||
callbackSequence: number;
|
||||
},
|
||||
): Promise<boolean> {
|
||||
await lockAttemptAuthority(this.queryable, attempt.id);
|
||||
const result = await queryMapped(this.queryable, UPDATE_ATTEMPT_SQL, [
|
||||
...writeValues(attempt, ATTEMPT_COLUMNS),
|
||||
expected.status,
|
||||
expected.callbackSequence,
|
||||
]);
|
||||
return affectedOneOrNone(result);
|
||||
}
|
||||
|
||||
async compareAndSetRetryPolicy(
|
||||
policy: RunRetryPolicyRecord,
|
||||
expectedVersion: number,
|
||||
): Promise<boolean> {
|
||||
if (policy.version !== expectedVersion + 1) {
|
||||
throw new RunRepositoryConstraintError(
|
||||
'A compare-and-set retry policy write must increment version exactly once',
|
||||
);
|
||||
}
|
||||
assertRunRetryPolicyRecord(policy);
|
||||
const result = await queryMapped(this.queryable, UPDATE_RETRY_POLICY_SQL, [
|
||||
...writeValues(policy, RETRY_POLICY_COLUMNS),
|
||||
expectedVersion,
|
||||
]);
|
||||
return affectedOneOrNone(result);
|
||||
}
|
||||
|
||||
async appendEvent(event: RunEventRecord): Promise<void> {
|
||||
assertEventPayloadSize(event);
|
||||
try {
|
||||
await this.queryable.query(
|
||||
INSERT_EVENT_SQL,
|
||||
writeValues(event, EVENT_COLUMNS),
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
sqlState(error) === '23505' &&
|
||||
(constraintName(error) === EVENT_SEQUENCE_CONSTRAINT ||
|
||||
constraintName(error) === EVENT_DEDUPE_CONSTRAINT)
|
||||
) {
|
||||
throw new DuplicateRunEventError(event.runId, event.dedupeKey);
|
||||
}
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Driver-neutral PostgreSQL Run Repository. The cluster-only package owns the
|
||||
* concrete pg.Pool binding; edge/standalone builds never import the driver.
|
||||
*/
|
||||
export class PostgresRunRepository
|
||||
extends PostgresRunReader
|
||||
implements RunRepository
|
||||
{
|
||||
constructor(private readonly pool: PostgresRunPool) {
|
||||
super(pool);
|
||||
}
|
||||
|
||||
async transaction<T>(
|
||||
work: (transaction: RunRepositoryTransaction) => Promise<T>,
|
||||
): Promise<T> {
|
||||
let client: PostgresRunClient;
|
||||
try {
|
||||
client = await this.pool.connect();
|
||||
} catch (error) {
|
||||
throw mapPostgresError(error);
|
||||
}
|
||||
let began = false;
|
||||
let phase: 'begin' | 'work' | 'commit' = 'begin';
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
began = true;
|
||||
await client.query('SET TRANSACTION ISOLATION LEVEL READ COMMITTED');
|
||||
await client.query(`SELECT set_config('statement_timeout', $1, true)`, [
|
||||
`${POSTGRES_RUNTIME_STATEMENT_TIMEOUT_MS}ms`,
|
||||
]);
|
||||
await client.query(`SELECT set_config('lock_timeout', $1, true)`, [
|
||||
`${POSTGRES_RUNTIME_LOCK_TIMEOUT_MS}ms`,
|
||||
]);
|
||||
await client.query(
|
||||
`SELECT set_config('idle_in_transaction_session_timeout', $1, true)`,
|
||||
[`${POSTGRES_RUNTIME_IDLE_TRANSACTION_TIMEOUT_MS}ms`],
|
||||
);
|
||||
phase = 'work';
|
||||
const result = await work(new PostgresRunTransaction(client));
|
||||
phase = 'commit';
|
||||
await client.query('COMMIT');
|
||||
began = false;
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (began) {
|
||||
try {
|
||||
await client.query('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the work/commit failure; release discards broken clients.
|
||||
}
|
||||
}
|
||||
if (phase === 'work') throw error;
|
||||
throw mapPostgresError(error);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,629 @@
|
||||
// Owns PostgreSQL StepRun persistence under the parent Run aggregate authority.
|
||||
import {
|
||||
InvalidStepRunError,
|
||||
MAX_STEP_RUNS_PER_RUN,
|
||||
StepRunFenceConflictError,
|
||||
StepRunMutationConflictError,
|
||||
StepRunRepositoryUnavailableError,
|
||||
StepRunStateConflictError,
|
||||
normalizeListStepRunsQuery,
|
||||
normalizeListStepRunsResult,
|
||||
normalizeStepRunMutation,
|
||||
normalizeStepRunRecord,
|
||||
resolveStepRunMutation,
|
||||
type ApplyStepRunMutationResult,
|
||||
type ListStepRunsQuery,
|
||||
type ListStepRunsResult,
|
||||
type StepRunMutation,
|
||||
type StepRunRecord,
|
||||
type StepRunRepository,
|
||||
} from '@qinglong/runtime-core/step-run';
|
||||
import type {
|
||||
PostgresClient,
|
||||
PostgresPool,
|
||||
PostgresQueryable,
|
||||
} from '@qinglong/runtime-core';
|
||||
import { isDeepStrictEqual } from 'node:util';
|
||||
|
||||
import {
|
||||
POSTGRES_DEFINITION_RETRYABLE_SQL_STATES,
|
||||
POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS,
|
||||
configurePostgresDefinitionTransaction,
|
||||
postgresRequiredBoolean,
|
||||
postgresRequiredInteger,
|
||||
postgresRequiredJsonObject,
|
||||
postgresRequiredString,
|
||||
postgresSqlState,
|
||||
rollbackPostgresDefinitionTransaction,
|
||||
} from '../repository/definitionRepositorySupport';
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
const IDENTITY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const TERMINAL_RUN_STATUSES = new Set([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
]);
|
||||
const STEP_RUN_SELECT = `
|
||||
id,
|
||||
run_id AS "runId",
|
||||
parent_step_run_id AS "parentStepRunId",
|
||||
step_key AS "stepKey",
|
||||
kind,
|
||||
definition_ref AS "definitionRef",
|
||||
definition_digest AS "definitionDigest",
|
||||
required,
|
||||
status,
|
||||
version,
|
||||
attempt_count AS "attemptCount",
|
||||
input_ref AS "inputRef",
|
||||
output_ref AS "outputRef",
|
||||
approval_request_id AS "approvalRequestId",
|
||||
ready_at_ms AS "readyAtMs",
|
||||
started_at_ms AS "startedAtMs",
|
||||
finished_at_ms AS "finishedAtMs",
|
||||
result_code AS "resultCode",
|
||||
error_summary AS "errorSummary",
|
||||
created_at_ms AS "createdAtMs",
|
||||
updated_at_ms AS "updatedAtMs",
|
||||
last_mutation_id AS "lastMutationId",
|
||||
step_run_digest AS "stepRunDigest",
|
||||
step_run_json AS "stepRunJson"
|
||||
`;
|
||||
|
||||
function unavailable(cause?: unknown): StepRunRepositoryUnavailableError {
|
||||
return new StepRunRepositoryUnavailableError({
|
||||
cause: cause instanceof Error ? cause : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function identity(value: unknown, label: string): string {
|
||||
if (typeof value !== 'string' || !IDENTITY_PATTERN.test(value)) {
|
||||
throw new InvalidStepRunError(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredText(row: Row, key: string): string {
|
||||
return postgresRequiredString(row[key], unavailable);
|
||||
}
|
||||
|
||||
function requiredInteger(row: Row, key: string): number {
|
||||
const value = postgresRequiredInteger(row[key], unavailable);
|
||||
if (value < 0) throw unavailable();
|
||||
return value;
|
||||
}
|
||||
|
||||
function nullableText(row: Row, key: string): string | null {
|
||||
const value = row[key];
|
||||
if (value === null) return null;
|
||||
return postgresRequiredString(value, unavailable);
|
||||
}
|
||||
|
||||
function nullableInteger(row: Row, key: string): number | null {
|
||||
const value = row[key];
|
||||
if (value === null) return null;
|
||||
const parsed = postgresRequiredInteger(value, unavailable);
|
||||
if (parsed < 0) throw unavailable();
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function serializedRecordFromRow(row: Row): Readonly<StepRunRecord> {
|
||||
try {
|
||||
return normalizeStepRunRecord(
|
||||
postgresRequiredJsonObject(row.stepRunJson, unavailable) as unknown as
|
||||
StepRunRecord,
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof StepRunRepositoryUnavailableError) throw error;
|
||||
throw unavailable(error);
|
||||
}
|
||||
}
|
||||
|
||||
function recordFromRow(row: Row): Readonly<StepRunRecord> {
|
||||
const record = serializedRecordFromRow(row);
|
||||
const requiredValue = postgresRequiredBoolean(row.required, unavailable);
|
||||
if (
|
||||
record.id !== requiredText(row, 'id') ||
|
||||
record.runId !== requiredText(row, 'runId') ||
|
||||
record.parentStepRunId !== nullableText(row, 'parentStepRunId') ||
|
||||
record.stepKey !== requiredText(row, 'stepKey') ||
|
||||
record.kind !== requiredText(row, 'kind') ||
|
||||
record.definitionRef !== requiredText(row, 'definitionRef') ||
|
||||
record.definitionDigest !== requiredText(row, 'definitionDigest') ||
|
||||
record.required !== requiredValue ||
|
||||
record.status !== requiredText(row, 'status') ||
|
||||
record.version !== requiredInteger(row, 'version') ||
|
||||
record.attemptCount !== requiredInteger(row, 'attemptCount') ||
|
||||
record.inputRef !== nullableText(row, 'inputRef') ||
|
||||
record.outputRef !== nullableText(row, 'outputRef') ||
|
||||
record.approvalRequestId !== nullableText(row, 'approvalRequestId') ||
|
||||
record.readyAtMs !== nullableInteger(row, 'readyAtMs') ||
|
||||
record.startedAtMs !== nullableInteger(row, 'startedAtMs') ||
|
||||
record.finishedAtMs !== nullableInteger(row, 'finishedAtMs') ||
|
||||
record.resultCode !== nullableText(row, 'resultCode') ||
|
||||
record.errorSummary !== nullableText(row, 'errorSummary') ||
|
||||
record.createdAtMs !== requiredInteger(row, 'createdAtMs') ||
|
||||
record.updatedAtMs !== requiredInteger(row, 'updatedAtMs') ||
|
||||
record.lastMutationId !== requiredText(row, 'lastMutationId') ||
|
||||
record.stepRunDigest !== requiredText(row, 'stepRunDigest')
|
||||
) {
|
||||
throw unavailable();
|
||||
}
|
||||
return record;
|
||||
}
|
||||
|
||||
function exactStoredEvent(
|
||||
row: Row,
|
||||
mutation: Readonly<StepRunMutation>,
|
||||
): boolean {
|
||||
const payload = postgresRequiredJsonObject(row.eventPayload, unavailable);
|
||||
const event = mutation.event;
|
||||
return (
|
||||
requiredText(row, 'eventId') === event.id &&
|
||||
requiredText(row, 'eventRunId') === event.runId &&
|
||||
requiredInteger(row, 'storedEventSequence') === event.sequence &&
|
||||
requiredText(row, 'eventType') === event.type &&
|
||||
requiredText(row, 'eventDedupeKey') === event.dedupeKey &&
|
||||
requiredText(row, 'eventActorType') === event.actorType &&
|
||||
(row.eventActorId === null ? undefined : row.eventActorId) ===
|
||||
event.actorId &&
|
||||
requiredText(row, 'eventStepRunId') === event.stepRunId &&
|
||||
requiredInteger(row, 'eventCreatedAtMs') === event.createdAtMs &&
|
||||
isDeepStrictEqual(payload, event.payload)
|
||||
);
|
||||
}
|
||||
|
||||
function storedMutationResult(
|
||||
row: Row,
|
||||
mutation: Readonly<StepRunMutation>,
|
||||
): Readonly<ApplyStepRunMutationResult> {
|
||||
const stepRun = serializedRecordFromRow(row);
|
||||
if (
|
||||
requiredText(row, 'mutationId') !== mutation.mutationId ||
|
||||
requiredText(row, 'mutationDigest') !== mutation.mutationDigest ||
|
||||
requiredText(row, 'storedRunId') !== mutation.runId ||
|
||||
requiredText(row, 'stepRunId') !== mutation.stepRun.id ||
|
||||
requiredText(row, 'storedStepRunDigest') !==
|
||||
mutation.stepRun.stepRunDigest ||
|
||||
JSON.stringify(stepRun) !== JSON.stringify(mutation.stepRun) ||
|
||||
!exactStoredEvent(row, mutation)
|
||||
) {
|
||||
throw new StepRunMutationConflictError();
|
||||
}
|
||||
const runVersion = requiredInteger(row, 'runVersion');
|
||||
const runEventSequence = requiredInteger(row, 'eventSequence');
|
||||
if (
|
||||
runVersion !== mutation.expectedRunVersion + 1 ||
|
||||
runEventSequence !== mutation.expectedRunEventSequence + 1 ||
|
||||
runEventSequence !== mutation.event.sequence
|
||||
) {
|
||||
throw unavailable();
|
||||
}
|
||||
return Object.freeze({
|
||||
status: 'existing',
|
||||
stepRun,
|
||||
runVersion,
|
||||
runEventSequence,
|
||||
});
|
||||
}
|
||||
|
||||
function constraintName(error: unknown): string {
|
||||
if (!error || typeof error !== 'object') return '';
|
||||
const value = (error as { constraint?: unknown }).constraint;
|
||||
return typeof value === 'string' ? value : '';
|
||||
}
|
||||
|
||||
function mapStorageError(error: unknown): Error {
|
||||
if (
|
||||
error instanceof InvalidStepRunError ||
|
||||
error instanceof StepRunFenceConflictError ||
|
||||
error instanceof StepRunMutationConflictError ||
|
||||
error instanceof StepRunRepositoryUnavailableError ||
|
||||
error instanceof StepRunStateConflictError
|
||||
) {
|
||||
return error;
|
||||
}
|
||||
const state = postgresSqlState(error);
|
||||
const constraint = constraintName(error);
|
||||
if (
|
||||
state === '23503' ||
|
||||
state === '23505' ||
|
||||
state === '23514'
|
||||
) {
|
||||
if (
|
||||
constraint === 'ql3_step_runs_parent_fk' ||
|
||||
constraint === 'ql3_step_runs_run_step_uidx' ||
|
||||
constraint === 'ql3_run_attempts_step_run_fk' ||
|
||||
constraint === 'ql3_run_events_step_run_fk'
|
||||
) {
|
||||
return new StepRunStateConflictError();
|
||||
}
|
||||
return new StepRunFenceConflictError();
|
||||
}
|
||||
return unavailable(error);
|
||||
}
|
||||
|
||||
async function findStoredById(
|
||||
queryable: PostgresQueryable,
|
||||
id: string,
|
||||
forUpdate = false,
|
||||
): Promise<Readonly<StepRunRecord> | null> {
|
||||
const result = await queryable.query<Row>(
|
||||
`SELECT ${STEP_RUN_SELECT}
|
||||
FROM "ql3"."step_runs"
|
||||
WHERE id = $1
|
||||
LIMIT 2${forUpdate ? ' FOR UPDATE' : ''}`,
|
||||
[id],
|
||||
);
|
||||
if (result.rows.length > 1) throw unavailable();
|
||||
return result.rows[0] ? recordFromRow(result.rows[0]) : null;
|
||||
}
|
||||
|
||||
async function insertStepRun(
|
||||
client: PostgresClient,
|
||||
stepRun: Readonly<StepRunRecord>,
|
||||
): Promise<void> {
|
||||
await client.query(
|
||||
`INSERT INTO "ql3"."step_runs" (
|
||||
id, run_id, parent_step_run_id, step_key, kind, definition_ref,
|
||||
definition_digest, required, status, version, attempt_count,
|
||||
input_ref, output_ref, approval_request_id, ready_at_ms, started_at_ms,
|
||||
finished_at_ms, result_code, error_summary, created_at_ms,
|
||||
updated_at_ms, last_mutation_id, step_run_digest, step_run_json
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14,
|
||||
$15, $16, $17, $18, $19, $20, $21, $22, $23, $24::jsonb
|
||||
)`,
|
||||
[
|
||||
stepRun.id,
|
||||
stepRun.runId,
|
||||
stepRun.parentStepRunId,
|
||||
stepRun.stepKey,
|
||||
stepRun.kind,
|
||||
stepRun.definitionRef,
|
||||
stepRun.definitionDigest,
|
||||
stepRun.required,
|
||||
stepRun.status,
|
||||
stepRun.version,
|
||||
stepRun.attemptCount,
|
||||
stepRun.inputRef,
|
||||
stepRun.outputRef,
|
||||
stepRun.approvalRequestId,
|
||||
stepRun.readyAtMs,
|
||||
stepRun.startedAtMs,
|
||||
stepRun.finishedAtMs,
|
||||
stepRun.resultCode,
|
||||
stepRun.errorSummary,
|
||||
stepRun.createdAtMs,
|
||||
stepRun.updatedAtMs,
|
||||
stepRun.lastMutationId,
|
||||
stepRun.stepRunDigest,
|
||||
JSON.stringify(stepRun),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
async function updateStepRun(
|
||||
client: PostgresClient,
|
||||
mutation: Readonly<StepRunMutation>,
|
||||
): Promise<void> {
|
||||
const stepRun = mutation.stepRun;
|
||||
const result = await client.query(
|
||||
`UPDATE "ql3"."step_runs"
|
||||
SET status = $1, version = $2, attempt_count = $3,
|
||||
output_ref = $4, approval_request_id = $5, ready_at_ms = $6,
|
||||
started_at_ms = $7, finished_at_ms = $8, result_code = $9,
|
||||
error_summary = $10, updated_at_ms = $11,
|
||||
last_mutation_id = $12, step_run_digest = $13,
|
||||
step_run_json = $14::jsonb
|
||||
WHERE id = $15 AND run_id = $16 AND version = $17
|
||||
AND step_run_digest = $18 AND status = $19`,
|
||||
[
|
||||
stepRun.status,
|
||||
stepRun.version,
|
||||
stepRun.attemptCount,
|
||||
stepRun.outputRef,
|
||||
stepRun.approvalRequestId,
|
||||
stepRun.readyAtMs,
|
||||
stepRun.startedAtMs,
|
||||
stepRun.finishedAtMs,
|
||||
stepRun.resultCode,
|
||||
stepRun.errorSummary,
|
||||
stepRun.updatedAtMs,
|
||||
stepRun.lastMutationId,
|
||||
stepRun.stepRunDigest,
|
||||
JSON.stringify(stepRun),
|
||||
stepRun.id,
|
||||
stepRun.runId,
|
||||
mutation.expectedStepRunVersion,
|
||||
mutation.expectedStepRunDigest,
|
||||
mutation.previousStatus,
|
||||
],
|
||||
);
|
||||
if (result.rowCount !== 1) throw new StepRunFenceConflictError();
|
||||
}
|
||||
|
||||
async function appendRunEvent(
|
||||
client: PostgresClient,
|
||||
mutation: Readonly<StepRunMutation>,
|
||||
): Promise<void> {
|
||||
const event = mutation.event;
|
||||
await client.query(
|
||||
`INSERT INTO "ql3"."run_events" (
|
||||
id, run_id, sequence, type, dedupe_key, actor_type, actor_id,
|
||||
attempt_id, step_run_id, payload, created_at_ms
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, $8, $9::jsonb, $10)`,
|
||||
[
|
||||
event.id,
|
||||
event.runId,
|
||||
event.sequence,
|
||||
event.type,
|
||||
event.dedupeKey,
|
||||
event.actorType,
|
||||
event.actorId ?? null,
|
||||
mutation.stepRun.id,
|
||||
JSON.stringify(event.payload),
|
||||
event.createdAtMs,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
export class PostgresStepRunRepository implements StepRunRepository {
|
||||
constructor(private readonly pool: PostgresPool) {
|
||||
if (
|
||||
!pool ||
|
||||
typeof pool.query !== 'function' ||
|
||||
typeof pool.connect !== 'function'
|
||||
) {
|
||||
throw unavailable();
|
||||
}
|
||||
}
|
||||
|
||||
async findById(idValue: string): Promise<Readonly<StepRunRecord> | null> {
|
||||
const id = identity(idValue, 'StepRun id');
|
||||
try {
|
||||
return await findStoredById(this.pool, id);
|
||||
} catch (error) {
|
||||
throw mapStorageError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async findByRunAndStepKey(
|
||||
runIdValue: string,
|
||||
stepKeyValue: string,
|
||||
): Promise<Readonly<StepRunRecord> | null> {
|
||||
const runId = identity(runIdValue, 'Run id');
|
||||
const stepKey = identity(stepKeyValue, 'step key');
|
||||
try {
|
||||
const result = await this.pool.query<Row>(
|
||||
`SELECT ${STEP_RUN_SELECT}
|
||||
FROM "ql3"."step_runs"
|
||||
WHERE run_id = $1 AND step_key = $2
|
||||
LIMIT 2`,
|
||||
[runId, stepKey],
|
||||
);
|
||||
if (result.rows.length > 1) throw unavailable();
|
||||
return result.rows[0] ? recordFromRow(result.rows[0]) : null;
|
||||
} catch (error) {
|
||||
throw mapStorageError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async listByRun(queryValue: ListStepRunsQuery): Promise<ListStepRunsResult> {
|
||||
const query = normalizeListStepRunsQuery(queryValue);
|
||||
try {
|
||||
const result = await this.pool.query<Row>(
|
||||
`SELECT ${STEP_RUN_SELECT}
|
||||
FROM "ql3"."step_runs"
|
||||
WHERE run_id = $1 AND (
|
||||
$2::varchar IS NULL OR step_key > $3 OR
|
||||
(step_key = $3 AND id > $2)
|
||||
)
|
||||
ORDER BY step_key, id
|
||||
LIMIT $4`,
|
||||
[
|
||||
query.runId,
|
||||
query.after?.id ?? null,
|
||||
query.after?.stepKey ?? '',
|
||||
query.limit + 1,
|
||||
],
|
||||
);
|
||||
const truncated = result.rows.length > query.limit;
|
||||
const stepRuns = result.rows.slice(0, query.limit).map(recordFromRow);
|
||||
const last = stepRuns.at(-1);
|
||||
return normalizeListStepRunsResult(
|
||||
{
|
||||
stepRuns,
|
||||
truncated,
|
||||
...(truncated && last
|
||||
? { next: { stepKey: last.stepKey, id: last.id } }
|
||||
: {}),
|
||||
},
|
||||
query,
|
||||
);
|
||||
} catch (error) {
|
||||
throw mapStorageError(error);
|
||||
}
|
||||
}
|
||||
|
||||
async apply(
|
||||
mutationValue: StepRunMutation,
|
||||
): Promise<Readonly<ApplyStepRunMutationResult>> {
|
||||
const mutation = normalizeStepRunMutation(mutationValue);
|
||||
for (
|
||||
let attempt = 0;
|
||||
attempt < POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS;
|
||||
attempt += 1
|
||||
) {
|
||||
let client: PostgresClient;
|
||||
try {
|
||||
client = await this.pool.connect();
|
||||
} catch (error) {
|
||||
throw unavailable(error);
|
||||
}
|
||||
let began = false;
|
||||
try {
|
||||
await configurePostgresDefinitionTransaction(client);
|
||||
began = true;
|
||||
|
||||
const run = await client.query<Row>(
|
||||
`SELECT status, version, event_sequence AS "eventSequence"
|
||||
FROM "ql3"."runs"
|
||||
WHERE id = $1
|
||||
LIMIT 2
|
||||
FOR UPDATE`,
|
||||
[mutation.runId],
|
||||
);
|
||||
if (run.rows.length !== 1) throw new StepRunFenceConflictError();
|
||||
|
||||
const stored = await client.query<Row>(
|
||||
`SELECT
|
||||
mutation.mutation_id AS "mutationId",
|
||||
mutation.mutation_digest AS "mutationDigest",
|
||||
mutation.run_id AS "storedRunId",
|
||||
mutation.step_run_id AS "stepRunId",
|
||||
mutation.step_run_digest AS "storedStepRunDigest",
|
||||
mutation.event_sequence AS "eventSequence",
|
||||
mutation.run_version AS "runVersion",
|
||||
mutation.step_run_json AS "stepRunJson",
|
||||
event.id AS "eventId",
|
||||
event.run_id AS "eventRunId",
|
||||
event.sequence AS "storedEventSequence",
|
||||
event.type AS "eventType",
|
||||
event.dedupe_key AS "eventDedupeKey",
|
||||
event.actor_type AS "eventActorType",
|
||||
event.actor_id AS "eventActorId",
|
||||
event.step_run_id AS "eventStepRunId",
|
||||
event.payload AS "eventPayload",
|
||||
event.created_at_ms AS "eventCreatedAtMs"
|
||||
FROM "ql3"."step_run_mutations" AS mutation
|
||||
JOIN "ql3"."run_events" AS event
|
||||
ON event.id = mutation.event_id
|
||||
WHERE mutation.mutation_id = $1
|
||||
LIMIT 2`,
|
||||
[mutation.mutationId],
|
||||
);
|
||||
if (stored.rows.length > 1) throw unavailable();
|
||||
if (stored.rows[0]) {
|
||||
const result = storedMutationResult(stored.rows[0], mutation);
|
||||
await client.query('COMMIT');
|
||||
began = false;
|
||||
return result;
|
||||
}
|
||||
|
||||
const runRow = run.rows[0]!;
|
||||
if (
|
||||
requiredInteger(runRow, 'version') !== mutation.expectedRunVersion ||
|
||||
requiredInteger(runRow, 'eventSequence') !==
|
||||
mutation.expectedRunEventSequence
|
||||
) {
|
||||
throw new StepRunFenceConflictError();
|
||||
}
|
||||
if (TERMINAL_RUN_STATUSES.has(requiredText(runRow, 'status'))) {
|
||||
throw new StepRunStateConflictError();
|
||||
}
|
||||
|
||||
const current = await findStoredById(
|
||||
client,
|
||||
mutation.stepRun.id,
|
||||
true,
|
||||
);
|
||||
const resolution = resolveStepRunMutation(current, mutation);
|
||||
if (resolution === 'existing') throw unavailable();
|
||||
|
||||
if (mutation.expectedStepRunVersion === null) {
|
||||
const count = await client.query<Row>(
|
||||
`SELECT COUNT(*) AS count
|
||||
FROM "ql3"."step_runs"
|
||||
WHERE run_id = $1`,
|
||||
[mutation.runId],
|
||||
);
|
||||
if (
|
||||
count.rows.length !== 1 ||
|
||||
requiredInteger(count.rows[0]!, 'count') >= MAX_STEP_RUNS_PER_RUN
|
||||
) {
|
||||
throw new StepRunStateConflictError();
|
||||
}
|
||||
if (mutation.stepRun.parentStepRunId !== null) {
|
||||
const parent = await client.query(
|
||||
`SELECT 1
|
||||
FROM "ql3"."step_runs"
|
||||
WHERE id = $1 AND run_id = $2
|
||||
LIMIT 1`,
|
||||
[mutation.stepRun.parentStepRunId, mutation.runId],
|
||||
);
|
||||
if (parent.rows.length !== 1) {
|
||||
throw new StepRunStateConflictError();
|
||||
}
|
||||
}
|
||||
await insertStepRun(client, mutation.stepRun);
|
||||
} else {
|
||||
await updateStepRun(client, mutation);
|
||||
}
|
||||
|
||||
const updatedRun = await client.query(
|
||||
`UPDATE "ql3"."runs"
|
||||
SET version = version + 1, event_sequence = event_sequence + 1
|
||||
WHERE id = $1 AND version = $2 AND event_sequence = $3`,
|
||||
[
|
||||
mutation.runId,
|
||||
mutation.expectedRunVersion,
|
||||
mutation.expectedRunEventSequence,
|
||||
],
|
||||
);
|
||||
if (updatedRun.rowCount !== 1) {
|
||||
throw new StepRunFenceConflictError();
|
||||
}
|
||||
|
||||
await appendRunEvent(client, mutation);
|
||||
await client.query(
|
||||
`INSERT INTO "ql3"."step_run_mutations" (
|
||||
mutation_id, mutation_digest, run_id, step_run_id,
|
||||
step_run_digest, event_id, event_sequence, run_version,
|
||||
step_run_json, committed_at_ms
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb,
|
||||
floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint
|
||||
)`,
|
||||
[
|
||||
mutation.mutationId,
|
||||
mutation.mutationDigest,
|
||||
mutation.runId,
|
||||
mutation.stepRun.id,
|
||||
mutation.stepRun.stepRunDigest,
|
||||
mutation.event.id,
|
||||
mutation.event.sequence,
|
||||
mutation.expectedRunVersion + 1,
|
||||
JSON.stringify(mutation.stepRun),
|
||||
],
|
||||
);
|
||||
|
||||
await client.query('COMMIT');
|
||||
began = false;
|
||||
return Object.freeze({
|
||||
status: 'applied',
|
||||
stepRun: mutation.stepRun,
|
||||
runVersion: mutation.expectedRunVersion + 1,
|
||||
runEventSequence: mutation.expectedRunEventSequence + 1,
|
||||
});
|
||||
} catch (error) {
|
||||
if (began) await rollbackPostgresDefinitionTransaction(client);
|
||||
const state = postgresSqlState(error);
|
||||
if (
|
||||
state &&
|
||||
POSTGRES_DEFINITION_RETRYABLE_SQL_STATES.has(state) &&
|
||||
attempt + 1 < POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
throw mapStorageError(error);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
throw unavailable();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user