qinglong/packages/ql3-cluster-postgres/src/plugin-package/workflow/pluginPackageWorkflowFrontierRepository.ts

// PostgreSQL authority for advancing the Plugin Package Workflow frontier.
import type {
  PostgresClient,
  PostgresPool,
  RunRecord,
} from '@qinglong/runtime-core';
import {
  InvalidPluginPackageWorkflowFrontierError,
  MAX_PLUGIN_PACKAGE_WORKFLOW_FRONTIER_PAGE_SIZE,
  PluginPackageWorkflowFrontierConflictError,
  PluginPackageWorkflowFrontierUnavailableError,
  resolvePluginPackageWorkflowFrontier,
  type PluginPackageWorkflowFrontierAdvanceResult,
  type PluginPackageWorkflowFrontierCandidate,
  type PluginPackageWorkflowFrontierCursor,
  type PluginPackageWorkflowFrontierPage,
  type PluginPackageWorkflowFrontierRepository,
  type PluginPackageWorkflowTerminalStatus,
} from '@qinglong/runtime-core/plugin-package-workflow-frontier';
import {
  normalizePluginPackageWorkflowExecutionPlan,
  type PluginPackageWorkflowExecutionPlan,
} from '@qinglong/runtime-core/plugin-package-workflow-execution-plan';
import {
  normalizeStepRunRecord,
  type StepRunMutation,
  type StepRunRecord,
} from '@qinglong/runtime-core/step-run';

import {
  POSTGRES_DEFINITION_RETRYABLE_SQL_STATES,
  POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS,
  configurePostgresDefinitionTransaction,
  postgresRequiredBoolean,
  postgresRequiredInteger,
  postgresRequiredJsonObject,
  postgresRequiredString,
  postgresSqlState,
  rollbackPostgresDefinitionTransaction,
} from '../../repository/definitionRepositorySupport';

type Row = Record<string, unknown>;

const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const DIGEST = /^[0-9a-f]{64}$/;
const TERMINAL_RUN_STATUSES = new Set<PluginPackageWorkflowTerminalStatus>([
  'succeeded',
  'failed',
  'cancelled',
  'timed_out',
]);

const RUN_SELECT = `
  id, project_id AS "projectId", task_id AS "taskId",
  task_revision AS "taskRevision", task_name AS "taskName",
  task_snapshot_ref AS "taskSnapshotRef", legacy_cron_id AS "legacyCronId",
  parent_run_id AS "parentRunId", retry_of_run_id AS "retryOfRunId",
  trigger_id AS "triggerId", trigger_type AS "triggerType",
  execution_origin AS "executionOrigin",
  execution_owner AS "executionOwner", triggered_by AS "triggeredBy",
  request_id AS "requestId", scheduled_for_ms AS "scheduledForMs",
  status, version, event_sequence AS "eventSequence", priority,
  idempotency_key AS "idempotencyKey", input_ref AS "inputRef",
  output_ref AS "outputRef", created_at_ms AS "createdAtMs",
  queued_at_ms AS "queuedAtMs", started_at_ms AS "startedAtMs",
  finished_at_ms AS "finishedAtMs",
  cancel_requested_at_ms AS "cancelRequestedAtMs",
  cancel_reason AS "cancelReason", error_code AS "errorCode",
  error_summary AS "errorSummary"
`.trim();

const STEP_RUN_SELECT = `
  id, run_id AS "runId", parent_step_run_id AS "parentStepRunId",
  step_key AS "stepKey", kind, definition_ref AS "definitionRef",
  definition_digest AS "definitionDigest", required, status, version,
  attempt_count AS "attemptCount", input_ref AS "inputRef",
  output_ref AS "outputRef", approval_request_id AS "approvalRequestId",
  ready_at_ms AS "readyAtMs", started_at_ms AS "startedAtMs",
  finished_at_ms AS "finishedAtMs", result_code AS "resultCode",
  error_summary AS "errorSummary", created_at_ms AS "createdAtMs",
  updated_at_ms AS "updatedAtMs", last_mutation_id AS "lastMutationId",
  step_run_digest AS "stepRunDigest", step_run_json AS "stepRunJson"
`.trim();

function unavailable(
  cause?: unknown,
): PluginPackageWorkflowFrontierUnavailableError {
  return new PluginPackageWorkflowFrontierUnavailableError({
    cause: cause instanceof Error ? cause : undefined,
  });
}

function text(row: Row, key: string): string {
  return postgresRequiredString(row[key], unavailable);
}

function integer(row: Row, key: string): number {
  const value = postgresRequiredInteger(row[key], unavailable);
  if (value < 0) throw unavailable();
  return value;
}

function optionalText(row: Row, key: string): string | undefined {
  if (row[key] === null || row[key] === undefined) return undefined;
  return text(row, key);
}

function optionalInteger(row: Row, key: string): number | undefined {
  if (row[key] === null || row[key] === undefined) return undefined;
  return integer(row, key);
}

function identity(value: unknown, label: string): string {
  if (typeof value !== 'string' || !IDENTITY.test(value)) {
    throw new InvalidPluginPackageWorkflowFrontierError(
      `${label} is invalid`,
    );
  }
  return value;
}

function digest(value: unknown): string {
  if (typeof value !== 'string' || !DIGEST.test(value)) throw unavailable();
  return value;
}

function mapStorageError(error: unknown): Error {
  if (
    error instanceof InvalidPluginPackageWorkflowFrontierError ||
    error instanceof PluginPackageWorkflowFrontierConflictError ||
    error instanceof PluginPackageWorkflowFrontierUnavailableError
  ) {
    return error;
  }
  return ['23503', '23505', '23514'].includes(
    postgresSqlState(error) ?? '',
  )
    ? new PluginPackageWorkflowFrontierConflictError()
    : unavailable(error);
}

function pageLimit(value: unknown): number {
  if (
    !Number.isInteger(value) ||
    (value as number) < 1 ||
    (value as number) > MAX_PLUGIN_PACKAGE_WORKFLOW_FRONTIER_PAGE_SIZE
  ) {
    throw new InvalidPluginPackageWorkflowFrontierError(
      `page limit must be between 1 and ${MAX_PLUGIN_PACKAGE_WORKFLOW_FRONTIER_PAGE_SIZE}`,
    );
  }
  return value as number;
}

function cursor(
  value: Readonly<PluginPackageWorkflowFrontierCursor> | undefined,
): Readonly<PluginPackageWorkflowFrontierCursor> | undefined {
  if (value === undefined) return undefined;
  if (
    !value ||
    typeof value !== 'object' ||
    Array.isArray(value) ||
    Reflect.ownKeys(value).length !== 2 ||
    !Reflect.has(value, 'admittedAtMs') ||
    !Reflect.has(value, 'planDigest') ||
    !Number.isSafeInteger(value.admittedAtMs) ||
    value.admittedAtMs < 0 ||
    !DIGEST.test(value.planDigest)
  ) {
    throw new InvalidPluginPackageWorkflowFrontierError(
      'frontier cursor is invalid',
    );
  }
  return Object.freeze({
    admittedAtMs: value.admittedAtMs,
    planDigest: value.planDigest,
  });
}

function runFromRow(row: Row): Readonly<RunRecord> {
  const run: RunRecord = {
    id: text(row, 'id'),
    projectId: text(row, 'projectId'),
    taskId: text(row, 'taskId'),
    taskRevision: text(row, 'taskRevision'),
    triggerType: text(row, 'triggerType'),
    executionOrigin: text(
      row,
      'executionOrigin',
    ) as RunRecord['executionOrigin'],
    executionOwner: text(row, 'executionOwner') as RunRecord['executionOwner'],
    status: text(row, 'status') as RunRecord['status'],
    version: integer(row, 'version'),
    eventSequence: integer(row, 'eventSequence'),
    priority: integer(row, 'priority'),
    createdAtMs: integer(row, 'createdAtMs'),
  };
  const optionalTexts = [
    ['taskName', 'taskName'],
    ['taskSnapshotRef', 'taskSnapshotRef'],
    ['parentRunId', 'parentRunId'],
    ['retryOfRunId', 'retryOfRunId'],
    ['triggerId', 'triggerId'],
    ['triggeredBy', 'triggeredBy'],
    ['requestId', 'requestId'],
    ['idempotencyKey', 'idempotencyKey'],
    ['inputRef', 'inputRef'],
    ['outputRef', 'outputRef'],
    ['errorCode', 'errorCode'],
    ['errorSummary', 'errorSummary'],
  ] as const;
  for (const [property, key] of optionalTexts) {
    const value = optionalText(row, key);
    if (value !== undefined) {
      (run as unknown as Record<string, unknown>)[property] = value;
    }
  }
  const optionalIntegers = [
    ['legacyCronId', 'legacyCronId'],
    ['scheduledForMs', 'scheduledForMs'],
    ['queuedAtMs', 'queuedAtMs'],
    ['startedAtMs', 'startedAtMs'],
    ['finishedAtMs', 'finishedAtMs'],
    ['cancelRequestedAtMs', 'cancelRequestedAtMs'],
  ] as const;
  for (const [property, key] of optionalIntegers) {
    const value = optionalInteger(row, key);
    if (value !== undefined) {
      (run as unknown as Record<string, unknown>)[property] = value;
    }
  }
  const cancelReason = optionalText(row, 'cancelReason');
  if (cancelReason !== undefined) {
    run.cancelReason =
      cancelReason as NonNullable<RunRecord['cancelReason']>;
  }
  return Object.freeze(run);
}

function stepRunFromRow(row: Row): Readonly<StepRunRecord> {
  let stepRun: Readonly<StepRunRecord>;
  try {
    stepRun = normalizeStepRunRecord(
      postgresRequiredJsonObject(
        row.stepRunJson,
        unavailable,
      ) as unknown as StepRunRecord,
    );
  } catch {
    throw unavailable();
  }
  if (
    text(row, 'id') !== stepRun.id ||
    text(row, 'runId') !== stepRun.runId ||
    optionalText(row, 'parentStepRunId') !==
      (stepRun.parentStepRunId ?? undefined) ||
    text(row, 'stepKey') !== stepRun.stepKey ||
    text(row, 'kind') !== stepRun.kind ||
    text(row, 'definitionRef') !== stepRun.definitionRef ||
    text(row, 'definitionDigest') !== stepRun.definitionDigest ||
    postgresRequiredBoolean(row.required, unavailable) !== stepRun.required ||
    text(row, 'status') !== stepRun.status ||
    integer(row, 'version') !== stepRun.version ||
    integer(row, 'attemptCount') !== stepRun.attemptCount ||
    optionalText(row, 'inputRef') !== (stepRun.inputRef ?? undefined) ||
    optionalText(row, 'outputRef') !== (stepRun.outputRef ?? undefined) ||
    optionalText(row, 'approvalRequestId') !==
      (stepRun.approvalRequestId ?? undefined) ||
    optionalInteger(row, 'readyAtMs') !==
      (stepRun.readyAtMs ?? undefined) ||
    optionalInteger(row, 'startedAtMs') !==
      (stepRun.startedAtMs ?? undefined) ||
    optionalInteger(row, 'finishedAtMs') !==
      (stepRun.finishedAtMs ?? undefined) ||
    optionalText(row, 'resultCode') !==
      (stepRun.resultCode ?? undefined) ||
    optionalText(row, 'errorSummary') !==
      (stepRun.errorSummary ?? undefined) ||
    integer(row, 'createdAtMs') !== stepRun.createdAtMs ||
    integer(row, 'updatedAtMs') !== stepRun.updatedAtMs ||
    text(row, 'lastMutationId') !== stepRun.lastMutationId ||
    text(row, 'stepRunDigest') !== stepRun.stepRunDigest
  ) {
    throw unavailable();
  }
  return stepRun;
}

function assertRunIdentity(
  run: Readonly<RunRecord>,
  plan: Readonly<PluginPackageWorkflowExecutionPlan>,
): void {
  if (
    run.id !== plan.runId ||
    run.projectId !== plan.target.projectId ||
    run.taskId !== plan.target.workflowId ||
    run.taskRevision !== plan.target.publicationDigest ||
    run.triggerType !== 'plugin_package_workflow' ||
    run.executionOrigin !== 'system' ||
    run.executionOwner !== 'runtime' ||
    run.requestId !== plan.planId ||
    run.idempotencyKey !== `plugin-package-workflow:${plan.planId}` ||
    run.version !== run.eventSequence
  ) {
    throw unavailable();
  }
}

async function updateStepRun(
  client: PostgresClient,
  mutation: Readonly<StepRunMutation>,
): Promise<void> {
  const stepRun = mutation.stepRun;
  const updated = await client.query(
    `UPDATE "ql3"."step_runs"
     SET status = $1, version = $2, attempt_count = $3, output_ref = $4,
         approval_request_id = $5, ready_at_ms = $6, started_at_ms = $7,
         finished_at_ms = $8, result_code = $9, error_summary = $10,
         updated_at_ms = $11, last_mutation_id = $12,
         step_run_digest = $13, step_run_json = $14::jsonb
     WHERE id = $15 AND run_id = $16 AND version = $17
       AND step_run_digest = $18 AND status = $19`,
    [
      stepRun.status,
      stepRun.version,
      stepRun.attemptCount,
      stepRun.outputRef,
      stepRun.approvalRequestId,
      stepRun.readyAtMs,
      stepRun.startedAtMs,
      stepRun.finishedAtMs,
      stepRun.resultCode,
      stepRun.errorSummary,
      stepRun.updatedAtMs,
      stepRun.lastMutationId,
      stepRun.stepRunDigest,
      JSON.stringify(stepRun),
      stepRun.id,
      stepRun.runId,
      mutation.expectedStepRunVersion,
      mutation.expectedStepRunDigest,
      mutation.previousStatus,
    ],
  );
  if (updated.rowCount !== 1) {
    throw new PluginPackageWorkflowFrontierConflictError();
  }
}

async function insertStepMutation(
  client: PostgresClient,
  mutation: Readonly<StepRunMutation>,
  committedAtMs: number,
): Promise<void> {
  const event = mutation.event;
  await client.query(
    `INSERT INTO "ql3"."run_events" (
       id, run_id, sequence, type, dedupe_key, actor_type, actor_id,
       attempt_id, step_run_id, payload, created_at_ms
     ) VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, $8, $9::jsonb, $10)`,
    [
      event.id,
      event.runId,
      event.sequence,
      event.type,
      event.dedupeKey ?? null,
      event.actorType,
      event.actorId ?? null,
      mutation.stepRun.id,
      JSON.stringify(event.payload),
      event.createdAtMs,
    ],
  );
  await client.query(
    `INSERT INTO "ql3"."step_run_mutations" (
       mutation_id, mutation_digest, run_id, step_run_id, step_run_digest,
       event_id, event_sequence, run_version, step_run_json, committed_at_ms
     ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10)`,
    [
      mutation.mutationId,
      mutation.mutationDigest,
      mutation.runId,
      mutation.stepRun.id,
      mutation.stepRun.stepRunDigest,
      event.id,
      event.sequence,
      mutation.expectedRunVersion + 1,
      JSON.stringify(mutation.stepRun),
      committedAtMs,
    ],
  );
}

export class PostgresPluginPackageWorkflowFrontierRepository
  implements PluginPackageWorkflowFrontierRepository
{
  constructor(private readonly pool: PostgresPool) {
    if (
      !pool ||
      typeof pool.query !== 'function' ||
      typeof pool.connect !== 'function'
    ) {
      throw unavailable();
    }
  }

  async listCandidates(queryValue: Readonly<{
    limit: number;
    after?: Readonly<PluginPackageWorkflowFrontierCursor>;
  }>): Promise<Readonly<PluginPackageWorkflowFrontierPage>> {
    if (
      !queryValue ||
      typeof queryValue !== 'object' ||
      Array.isArray(queryValue) ||
      !Reflect.has(queryValue, 'limit') ||
      Reflect.ownKeys(queryValue).some(
        (key) => key !== 'limit' && key !== 'after',
      )
    ) {
      throw new InvalidPluginPackageWorkflowFrontierError(
        'page query is invalid',
      );
    }
    const limit = pageLimit(queryValue.limit);
    const after = cursor(queryValue.after);
    try {
      const result = await this.pool.query<Row>(
        `SELECT admission.run_id AS "runId",
                admission.plan_digest AS "planDigest",
                admission.admitted_at_ms AS "admittedAtMs"
         FROM "ql3"."plugin_package_workflow_admissions" AS admission
         JOIN "ql3"."runs" AS run ON run.id = admission.run_id
         WHERE run.status = 'running'
           AND run.cancel_requested_at_ms IS NULL
           AND (
             EXISTS (
               SELECT 1
               FROM "ql3"."plugin_package_workflow_admission_steps" AS step
               JOIN "ql3"."step_runs" AS current
                 ON current.run_id = step.run_id
                AND current.id = step.step_run_id
               WHERE step.plan_digest = admission.plan_digest
                 AND current.status = 'pending'
                 AND (
                   NOT EXISTS (
                     SELECT 1
                     FROM jsonb_array_elements_text(step.needs_json)
                       AS need(value)
                     LEFT JOIN
                       "ql3"."plugin_package_workflow_admission_steps"
                       AS dependency_step
                       ON dependency_step.plan_digest = step.plan_digest
                      AND dependency_step.step_key = need.value
                     LEFT JOIN "ql3"."step_runs" AS dependency
                       ON dependency.run_id = dependency_step.run_id
                      AND dependency.id = dependency_step.step_run_id
                     WHERE dependency.id IS NULL
                        OR dependency.status <> 'succeeded'
                   )
                   OR EXISTS (
                     SELECT 1
                     FROM jsonb_array_elements_text(step.needs_json)
                       AS need(value)
                     JOIN "ql3"."plugin_package_workflow_admission_steps"
                       AS dependency_step
                       ON dependency_step.plan_digest = step.plan_digest
                      AND dependency_step.step_key = need.value
                     JOIN "ql3"."step_runs" AS dependency
                       ON dependency.run_id = dependency_step.run_id
                      AND dependency.id = dependency_step.step_run_id
                     WHERE dependency.status IN (
                       'failed', 'skipped', 'cancelled', 'timed_out'
                     )
                   )
                 )
             )
             OR NOT EXISTS (
               SELECT 1
               FROM "ql3"."plugin_package_workflow_admission_steps" AS step
               JOIN "ql3"."step_runs" AS current
                 ON current.run_id = step.run_id
                AND current.id = step.step_run_id
               WHERE step.plan_digest = admission.plan_digest
                 AND current.status NOT IN (
                   'succeeded', 'failed', 'skipped', 'cancelled', 'timed_out'
                 )
             )
           )
           AND (
             $1::bigint IS NULL OR admission.admitted_at_ms > $1 OR
             (admission.admitted_at_ms = $1
              AND admission.plan_digest > $2)
           )
         ORDER BY admission.admitted_at_ms, admission.plan_digest
         LIMIT $3`,
        [
          after?.admittedAtMs ?? null,
          after?.planDigest ?? '',
          limit + 1,
        ],
      );
      const mapped = result.rows.map(
        (row): Readonly<PluginPackageWorkflowFrontierCandidate> =>
          Object.freeze({
            runId: identity(text(row, 'runId'), 'candidate runId'),
            planDigest: digest(row.planDigest),
            admittedAtMs: integer(row, 'admittedAtMs'),
          }),
      );
      const truncated = mapped.length > limit;
      const candidates = Object.freeze(mapped.slice(0, limit));
      const last = candidates.at(-1);
      return Object.freeze({
        candidates,
        truncated,
        ...(truncated && last
          ? {
              next: Object.freeze({
                admittedAtMs: last.admittedAtMs,
                planDigest: last.planDigest,
              }),
            }
          : {}),
      });
    } catch (error) {
      throw mapStorageError(error);
    }
  }

  async advance(
    runIdValue: string,
  ): Promise<Readonly<PluginPackageWorkflowFrontierAdvanceResult>> {
    const runId = identity(runIdValue, 'runId');
    for (
      let attempt = 0;
      attempt < POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS;
      attempt += 1
    ) {
      let client: PostgresClient;
      try {
        client = await this.pool.connect();
      } catch (error) {
        throw unavailable(error);
      }
      let began = false;
      try {
        await configurePostgresDefinitionTransaction(client);
        began = true;
        const admission = await client.query<Row>(
          `SELECT plan_digest AS "planDigest", plan_json AS "planJson"
           FROM "ql3"."plugin_package_workflow_admissions"
           WHERE run_id = $1 LIMIT 2`,
          [runId],
        );
        if (admission.rows.length !== 1) {
          throw new PluginPackageWorkflowFrontierConflictError();
        }
        let plan: Readonly<PluginPackageWorkflowExecutionPlan>;
        try {
          plan = normalizePluginPackageWorkflowExecutionPlan(
            postgresRequiredJsonObject(
              admission.rows[0]!.planJson,
              unavailable,
            ) as unknown as PluginPackageWorkflowExecutionPlan,
          );
        } catch {
          throw unavailable();
        }
        if (plan.planDigest !== digest(admission.rows[0]!.planDigest)) {
          throw unavailable();
        }
        const runRows = await client.query<Row>(
          `SELECT ${RUN_SELECT}
           FROM "ql3"."runs" WHERE id = $1 LIMIT 2 FOR UPDATE`,
          [runId],
        );
        if (runRows.rows.length !== 1) {
          throw new PluginPackageWorkflowFrontierConflictError();
        }
        const run = runFromRow(runRows.rows[0]!);
        assertRunIdentity(run, plan);
        const stepRows = await client.query<Row>(
          `SELECT ${STEP_RUN_SELECT}
           FROM "ql3"."step_runs" WHERE run_id = $1
           ORDER BY step_key, id FOR UPDATE`,
          [runId],
        );
        const stepRuns = stepRows.rows.map(stepRunFromRow);
        const clock = await client.query<Row>(
          `SELECT floor(
             extract(epoch FROM transaction_timestamp()) * 1000
           )::bigint AS "observedAtMs"`,
        );
        if (clock.rows.length !== 1) throw unavailable();
        const observedAtMs = integer(clock.rows[0]!, 'observedAtMs');
        const currentStatus = run.status;
        const resolution = resolvePluginPackageWorkflowFrontier({
          plan,
          run: {
            ...run,
            ...(TERMINAL_RUN_STATUSES.has(
              currentStatus as PluginPackageWorkflowTerminalStatus,
            )
              ? { status: 'running' as const }
              : {}),
          },
          stepRuns,
          observedAtMs,
        });
        if (
          TERMINAL_RUN_STATUSES.has(
            currentStatus as PluginPackageWorkflowTerminalStatus,
          )
        ) {
          if (
            resolution.stepMutations.length !== 0 ||
            resolution.terminalStatus !== currentStatus
          ) {
            throw unavailable();
          }
          await client.query('COMMIT');
          began = false;
          return Object.freeze({
            status: 'settled' as const,
            runId,
            planDigest: plan.planDigest,
            stepMutationCount: 0,
            readyStepRunIds: Object.freeze([]),
            terminalStatus:
              currentStatus as PluginPackageWorkflowTerminalStatus,
            runVersion: run.version,
            runEventSequence: run.eventSequence,
            observedAtMs,
          });
        }
        if (currentStatus !== 'running') throw unavailable();
        const increment =
          resolution.stepMutations.length +
          (resolution.terminalTransition === null ? 0 : 1);
        if (increment === 0) {
          await client.query('COMMIT');
          began = false;
          return Object.freeze({
            status: 'unchanged' as const,
            runId,
            planDigest: plan.planDigest,
            stepMutationCount: 0,
            readyStepRunIds: resolution.readyStepRunIds,
            terminalStatus: null,
            runVersion: run.version,
            runEventSequence: run.eventSequence,
            observedAtMs,
          });
        }
        for (const mutation of resolution.stepMutations) {
          await updateStepRun(client, mutation);
        }
        const terminal = resolution.terminalTransition;
        const updatedRun = await client.query(
          `UPDATE "ql3"."runs"
           SET status = $1, version = version + $2,
               event_sequence = event_sequence + $2,
               finished_at_ms = $3, error_code = $4, error_summary = NULL
           WHERE id = $5 AND status = 'running'
             AND version = $6 AND event_sequence = $7`,
          [
            terminal?.status ?? 'running',
            increment,
            terminal?.finishedAtMs ?? null,
            terminal?.errorCode ?? null,
            runId,
            run.version,
            run.eventSequence,
          ],
        );
        if (updatedRun.rowCount !== 1) {
          throw new PluginPackageWorkflowFrontierConflictError();
        }
        for (const mutation of resolution.stepMutations) {
          await insertStepMutation(client, mutation, observedAtMs);
        }
        if (terminal) {
          const event = terminal.event;
          await client.query(
            `INSERT INTO "ql3"."run_events" (
               id, run_id, sequence, type, dedupe_key, actor_type, actor_id,
               attempt_id, step_run_id, payload, created_at_ms
             ) VALUES (
               $1, $2, $3, $4, $5, $6, $7, NULL, NULL, $8::jsonb, $9
             )`,
            [
              event.id,
              event.runId,
              event.sequence,
              event.type,
              event.dedupeKey ?? null,
              event.actorType,
              event.actorId ?? null,
              JSON.stringify(event.payload),
              event.createdAtMs,
            ],
          );
        }
        await client.query('COMMIT');
        began = false;
        return Object.freeze({
          status: terminal ? ('terminal' as const) : ('advanced' as const),
          runId,
          planDigest: plan.planDigest,
          stepMutationCount: resolution.stepMutations.length,
          readyStepRunIds: terminal
            ? Object.freeze([])
            : resolution.readyStepRunIds,
          terminalStatus: terminal?.status ?? null,
          runVersion: run.version + increment,
          runEventSequence: run.eventSequence + increment,
          observedAtMs,
        });
      } catch (error) {
        if (began) await rollbackPostgresDefinitionTransaction(client);
        const state = postgresSqlState(error);
        if (
          state &&
          POSTGRES_DEFINITION_RETRYABLE_SQL_STATES.has(state) &&
          attempt + 1 < POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS
        ) {
          continue;
        }
        throw mapStorageError(error);
      } finally {
        client.release();
      }
    }
    throw unavailable();
  }
}