feat(ql3): establish 3.0 incubation baseline

packages/ql3-runtime-core/src/run/clusterRunCancellation.ts

@@ -0,0 +1,372 @@
+import {
+  normalizeProjectPolicySubject,
+  type ProjectRole,
+} from '../security/project-policy/projectPolicy';
+import {
+  normalizeSecurityPolicyDecision,
+  type SecurityPolicyFence,
+  type SecuritySubject,
+} from '../security/security';
+import { RUN_STATUSES, type RunStatus } from './run';
+
+export const RUN_CANCELLATION_SCHEMA = 'qinglong/run-cancellation@v1' as const;
+/** @deprecated Use RUN_CANCELLATION_SCHEMA from the profile-neutral export. */
+export const CLUSTER_RUN_CANCELLATION_SCHEMA = RUN_CANCELLATION_SCHEMA;
+export const CLUSTER_RUN_CANCELLATION_STATUSES = [
+  'accepted',
+  'already_requested',
+  'already_terminal',
+] as const;
+
+export type ClusterRunCancellationStatus =
+  (typeof CLUSTER_RUN_CANCELLATION_STATUSES)[number];
+
+export interface ClusterRunCancellationRequestBody {
+  readonly schema: typeof CLUSTER_RUN_CANCELLATION_SCHEMA;
+  readonly mutationId: string;
+}
+
+export interface ClusterRunCancellationWorkflowTarget {
+  readonly packageName: string;
+  readonly workflowId: string;
+}
+
+export interface ClusterRunCancellationCommand {
+  readonly projectId: string;
+  readonly runId: string;
+  readonly mutationId: string;
+  readonly eventId: string;
+  readonly subject: Readonly<SecuritySubject>;
+  readonly policyFence: Readonly<SecurityPolicyFence>;
+  readonly workflowTarget?: Readonly<ClusterRunCancellationWorkflowTarget>;
+}
+
+export interface ClusterRunCancellationResult {
+  readonly status: ClusterRunCancellationStatus;
+  readonly projectId: string;
+  readonly runId: string;
+  readonly runStatus: RunStatus;
+  readonly runVersion: number;
+  readonly eventSequence: number;
+  readonly cancelRequestedAtMs?: number;
+  readonly cancelReason?: 'user' | 'policy' | 'shutdown' | 'reconcile' | 'timeout';
+}
+
+export interface ClusterRunCancellationResponseBody
+  extends ClusterRunCancellationResult {
+  readonly schema: typeof CLUSTER_RUN_CANCELLATION_SCHEMA;
+}
+
+export interface ClusterRunCancellationRepository {
+  requestUserCancellation(
+    command: Readonly<ClusterRunCancellationCommand>,
+  ): Promise<Readonly<ClusterRunCancellationResult>>;
+}
+
+export type ClusterRunCancellationFenceReason =
+  | 'authorization_changed'
+  | 'project_mismatch'
+  | 'state_mismatch';
+
+export class InvalidClusterRunCancellationError extends TypeError {
+  constructor(message: string) {
+    super(`Cluster Run cancellation is invalid: ${message}`);
+    this.name = 'InvalidClusterRunCancellationError';
+  }
+}
+
+export class ClusterRunCancellationNotFoundError extends Error {
+  readonly code = 'CLUSTER_RUN_CANCELLATION_NOT_FOUND';
+  constructor() {
+    super('Cluster Run cancellation target does not exist');
+    this.name = 'ClusterRunCancellationNotFoundError';
+  }
+}
+
+export class ClusterRunCancellationFenceRejectedError extends Error {
+  readonly code = 'CLUSTER_RUN_CANCELLATION_FENCE_REJECTED';
+  constructor(readonly reason: ClusterRunCancellationFenceReason) {
+    super(`Cluster Run cancellation fence rejected: ${reason}`);
+    this.name = 'ClusterRunCancellationFenceRejectedError';
+  }
+}
+
+export class ClusterRunCancellationUnavailableError extends Error {
+  readonly code = 'CLUSTER_RUN_CANCELLATION_UNAVAILABLE';
+  constructor(options?: ErrorOptions) {
+    super('Cluster Run cancellation is unavailable', options);
+    this.name = 'ClusterRunCancellationUnavailableError';
+  }
+}
+
+const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+const EVENT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
+const PACKAGE_NAME_PATTERN =
+  /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
+const WORKFLOW_ID_PATTERN = /^[a-z][a-z0-9-]{0,62}$/;
+const TERMINAL = new Set<RunStatus>([
+  'succeeded',
+  'failed',
+  'cancelled',
+  'timed_out',
+]);
+const CANCEL_REASONS = new Set([
+  'user',
+  'policy',
+  'shutdown',
+  'reconcile',
+  'timeout',
+]);
+
+function exactKeys(value: object, expected: readonly string[]): void {
+  const actual = Object.keys(value).sort();
+  const canonical = [...expected].sort();
+  if (
+    actual.length !== canonical.length ||
+    actual.some((key, index) => key !== canonical[index])
+  ) {
+    throw new InvalidClusterRunCancellationError('shape is invalid');
+  }
+}
+
+function identifier(value: unknown, name: string): string {
+  if (typeof value !== 'string' || !ID_PATTERN.test(value)) {
+    throw new InvalidClusterRunCancellationError(`${name} is invalid`);
+  }
+  return value;
+}
+
+function counter(value: unknown, name: string, minimum: number): number {
+  if (
+    typeof value !== 'number' ||
+    !Number.isSafeInteger(value) ||
+    value < minimum ||
+    value > 2_147_483_647
+  ) {
+    throw new InvalidClusterRunCancellationError(`${name} is invalid`);
+  }
+  return value;
+}
+
+function timestamp(value: unknown, name: string): number {
+  if (
+    typeof value !== 'number' ||
+    !Number.isSafeInteger(value) ||
+    value < 0
+  ) {
+    throw new InvalidClusterRunCancellationError(`${name} is invalid`);
+  }
+  return value;
+}
+
+export function parseClusterRunCancellationRequestBody(
+  value: unknown,
+): Readonly<ClusterRunCancellationRequestBody> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new InvalidClusterRunCancellationError('request body is invalid');
+  }
+  exactKeys(value, ['schema', 'mutationId']);
+  const body = value as Record<string, unknown>;
+  if (body.schema !== CLUSTER_RUN_CANCELLATION_SCHEMA) {
+    throw new InvalidClusterRunCancellationError('schema is invalid');
+  }
+  return Object.freeze({
+    schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
+    mutationId: identifier(body.mutationId, 'mutationId'),
+  });
+}
+
+export function normalizeClusterRunCancellationCommand(
+  value: ClusterRunCancellationCommand,
+): Readonly<ClusterRunCancellationCommand> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new InvalidClusterRunCancellationError('command is invalid');
+  }
+  const hasWorkflowTarget = value.workflowTarget !== undefined;
+  exactKeys(
+    value,
+    [
+      'projectId',
+      'runId',
+      'mutationId',
+      'eventId',
+      'subject',
+      'policyFence',
+      ...(hasWorkflowTarget ? ['workflowTarget'] : []),
+    ],
+  );
+  if (typeof value.eventId !== 'string' || !EVENT_ID_PATTERN.test(value.eventId)) {
+    throw new InvalidClusterRunCancellationError('eventId is invalid');
+  }
+  let subject: Readonly<SecuritySubject>;
+  let fence: Readonly<SecurityPolicyFence> | null;
+  try {
+    subject = normalizeProjectPolicySubject(value.subject);
+    fence = normalizeSecurityPolicyDecision({
+      effect: 'allow',
+      reasons: ['role_grant'],
+      fence: value.policyFence,
+    }).fence;
+  } catch {
+    throw new InvalidClusterRunCancellationError(
+      'authorization authority is invalid',
+    );
+  }
+  if (!fence || fence.bindingVersion === null) {
+    throw new InvalidClusterRunCancellationError(
+      'authorization fence is incomplete',
+    );
+  }
+  let workflowTarget:
+    | Readonly<ClusterRunCancellationWorkflowTarget>
+    | undefined;
+  if (hasWorkflowTarget) {
+    const target = value.workflowTarget;
+    if (!target || typeof target !== 'object' || Array.isArray(target)) {
+      throw new InvalidClusterRunCancellationError(
+        'workflowTarget is invalid',
+      );
+    }
+    exactKeys(target, ['packageName', 'workflowId']);
+    if (
+      typeof target.packageName !== 'string' ||
+      !PACKAGE_NAME_PATTERN.test(target.packageName) ||
+      typeof target.workflowId !== 'string' ||
+      !WORKFLOW_ID_PATTERN.test(target.workflowId)
+    ) {
+      throw new InvalidClusterRunCancellationError(
+        'workflowTarget is invalid',
+      );
+    }
+    workflowTarget = Object.freeze({
+      packageName: target.packageName,
+      workflowId: target.workflowId,
+    });
+  }
+  return Object.freeze({
+    projectId: identifier(value.projectId, 'projectId'),
+    runId: identifier(value.runId, 'runId'),
+    mutationId: identifier(value.mutationId, 'mutationId'),
+    eventId: value.eventId,
+    subject,
+    policyFence: fence,
+    ...(workflowTarget === undefined ? {} : { workflowTarget }),
+  });
+}
+
+export function normalizeClusterRunCancellationResult(
+  value: ClusterRunCancellationResult,
+): Readonly<ClusterRunCancellationResult> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new InvalidClusterRunCancellationError('result is invalid');
+  }
+  const hasCancellation = value.cancelRequestedAtMs !== undefined;
+  exactKeys(
+    value,
+    hasCancellation
+      ? [
+          'status', 'projectId', 'runId', 'runStatus', 'runVersion',
+          'eventSequence', 'cancelRequestedAtMs', 'cancelReason',
+        ]
+      : [
+          'status', 'projectId', 'runId', 'runStatus', 'runVersion',
+          'eventSequence',
+        ],
+  );
+  if (
+    !CLUSTER_RUN_CANCELLATION_STATUSES.includes(value.status) ||
+    !RUN_STATUSES.includes(value.runStatus) ||
+    (value.status === 'accepted' && !hasCancellation) ||
+    (value.status === 'already_requested' && !hasCancellation) ||
+    (value.status === 'already_terminal' && !TERMINAL.has(value.runStatus)) ||
+    (value.status !== 'already_terminal' && TERMINAL.has(value.runStatus))
+  ) {
+    throw new InvalidClusterRunCancellationError('result state is invalid');
+  }
+  let cancelRequestedAtMs: number | undefined;
+  let cancelReason: ClusterRunCancellationResult['cancelReason'];
+  if (hasCancellation) {
+    cancelRequestedAtMs = timestamp(
+      value.cancelRequestedAtMs,
+      'cancelRequestedAtMs',
+    );
+    if (!CANCEL_REASONS.has(value.cancelReason ?? '')) {
+      throw new InvalidClusterRunCancellationError('cancelReason is invalid');
+    }
+    cancelReason = value.cancelReason;
+  } else if (value.cancelReason !== undefined) {
+    throw new InvalidClusterRunCancellationError('cancellation shape is invalid');
+  }
+  return Object.freeze({
+    status: value.status,
+    projectId: identifier(value.projectId, 'projectId'),
+    runId: identifier(value.runId, 'runId'),
+    runStatus: value.runStatus,
+    runVersion: counter(value.runVersion, 'runVersion', 1),
+    eventSequence: counter(value.eventSequence, 'eventSequence', 0),
+    ...(cancelRequestedAtMs === undefined
+      ? {}
+      : { cancelRequestedAtMs, cancelReason: cancelReason! }),
+  });
+}
+
+export function createClusterRunCancellationResponseBody(
+  value: ClusterRunCancellationResult,
+): Readonly<ClusterRunCancellationResponseBody> {
+  return Object.freeze({
+    schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
+    ...normalizeClusterRunCancellationResult(value),
+  });
+}
+
+export function parseClusterRunCancellationResponseBody(
+  value: unknown,
+): Readonly<ClusterRunCancellationResponseBody> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new InvalidClusterRunCancellationError('response body is invalid');
+  }
+  const { schema, ...result } = value as Record<string, unknown>;
+  if (schema !== CLUSTER_RUN_CANCELLATION_SCHEMA) {
+    throw new InvalidClusterRunCancellationError('schema is invalid');
+  }
+  return createClusterRunCancellationResponseBody(
+    result as unknown as ClusterRunCancellationResult,
+  );
+}
+
+export type ClusterRunCancellationAllowedRole = Extract<
+  ProjectRole,
+  'owner' | 'admin' | 'operator'
+>;
+
+// Profile-neutral names are canonical for new Local and Cluster consumers.
+// Cluster-prefixed names remain source-compatible throughout the 3.0 Alpha.
+export const RUN_CANCELLATION_STATUSES = CLUSTER_RUN_CANCELLATION_STATUSES;
+export type RunCancellationStatus = ClusterRunCancellationStatus;
+export type RunCancellationRequestBody = ClusterRunCancellationRequestBody;
+export type RunCancellationWorkflowTarget =
+  ClusterRunCancellationWorkflowTarget;
+export type RunCancellationCommand = ClusterRunCancellationCommand;
+export type RunCancellationResult = ClusterRunCancellationResult;
+export type RunCancellationResponseBody = ClusterRunCancellationResponseBody;
+export type RunCancellationRepository = ClusterRunCancellationRepository;
+export type RunCancellationFenceReason = ClusterRunCancellationFenceReason;
+export type RunCancellationAllowedRole = ClusterRunCancellationAllowedRole;
+export const InvalidRunCancellationError =
+  InvalidClusterRunCancellationError;
+export const RunCancellationNotFoundError =
+  ClusterRunCancellationNotFoundError;
+export const RunCancellationFenceRejectedError =
+  ClusterRunCancellationFenceRejectedError;
+export const RunCancellationUnavailableError =
+  ClusterRunCancellationUnavailableError;
+export const parseRunCancellationRequestBody =
+  parseClusterRunCancellationRequestBody;
+export const normalizeRunCancellationCommand =
+  normalizeClusterRunCancellationCommand;
+export const normalizeRunCancellationResult =
+  normalizeClusterRunCancellationResult;
+export const createRunCancellationResponseBody =
+  createClusterRunCancellationResponseBody;
+export const parseRunCancellationResponseBody =
+  parseClusterRunCancellationResponseBody;

packages/ql3-runtime-core/src/run/clusterRunCancellationConvergence.ts

@@ -0,0 +1,241 @@
+import { MAX_STEP_RUNS_PER_RUN } from './stepRun';
+
+export const MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGE_SIZE = 128;
+export const MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGES_PER_CYCLE = 64;
+
+export interface ClusterRunCancellationConvergencePageCommand {
+  readonly limit: number;
+}
+
+export interface ClusterRunCancellationConvergencePageResult {
+  readonly scanned: number;
+  readonly settledRuns: number;
+  readonly settledAttempts: number;
+  readonly blocked: number;
+  readonly hasMore: boolean;
+}
+
+export interface ClusterRunCancellationConvergenceRepository {
+  convergePage(
+    command: Readonly<ClusterRunCancellationConvergencePageCommand>,
+  ): Promise<Readonly<ClusterRunCancellationConvergencePageResult>>;
+}
+
+export interface ClusterRunCancellationConvergenceCycleResult
+  extends ClusterRunCancellationConvergencePageResult {
+  readonly pages: number;
+  readonly remaining: boolean;
+  readonly stopReason: 'complete' | 'page_limit' | 'blocked';
+}
+
+export interface ClusterRunCancellationConvergenceCoordinatorOptions {
+  readonly pageSize?: number;
+  readonly maxPages?: number;
+}
+
+export class ClusterRunCancellationConvergenceUnavailableError extends Error {
+  readonly code = 'CLUSTER_RUN_CANCELLATION_CONVERGENCE_UNAVAILABLE';
+
+  constructor(options?: ErrorOptions) {
+    super('Cluster Run cancellation convergence is unavailable', options);
+    this.name = 'ClusterRunCancellationConvergenceUnavailableError';
+  }
+}
+
+function boundedInteger(
+  name: string,
+  value: number,
+  minimum: number,
+  maximum: number,
+): number {
+  if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
+    throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
+  }
+  return value;
+}
+
+export function normalizeClusterRunCancellationConvergencePageCommand(
+  value: ClusterRunCancellationConvergencePageCommand,
+): Readonly<ClusterRunCancellationConvergencePageCommand> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new TypeError('Cluster Run cancellation convergence command is invalid');
+  }
+  const keys = Object.keys(value).sort();
+  if (keys.length !== 1 || keys[0] !== 'limit') {
+    throw new TypeError('Cluster Run cancellation convergence command shape is invalid');
+  }
+  const limit = boundedInteger(
+    'Cluster Run cancellation convergence page size',
+    value.limit,
+    1,
+    MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGE_SIZE,
+  );
+  return Object.freeze({ limit });
+}
+
+export function normalizeClusterRunCancellationConvergencePageResult(
+  value: ClusterRunCancellationConvergencePageResult,
+  limit = MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGE_SIZE,
+): Readonly<ClusterRunCancellationConvergencePageResult> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new TypeError('Cluster Run cancellation convergence result is invalid');
+  }
+  const keys = Object.keys(value).sort();
+  if (
+    keys.length !== 5 ||
+    keys.join(',') !== 'blocked,hasMore,scanned,settledAttempts,settledRuns'
+  ) {
+    throw new TypeError('Cluster Run cancellation convergence result shape is invalid');
+  }
+  const maximum = boundedInteger(
+    'Cluster Run cancellation convergence result limit',
+    limit,
+    1,
+    MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGE_SIZE,
+  );
+  const scanned = boundedInteger(
+    'Cluster Run cancellation convergence scanned count',
+    value.scanned,
+    0,
+    maximum,
+  );
+  const settledRuns = boundedInteger(
+    'Cluster Run cancellation convergence settled Run count',
+    value.settledRuns,
+    0,
+    scanned,
+  );
+  const settledAttempts = boundedInteger(
+    'Cluster Run cancellation convergence settled Attempt count',
+    value.settledAttempts,
+    0,
+    scanned * MAX_STEP_RUNS_PER_RUN,
+  );
+  const blocked = boundedInteger(
+    'Cluster Run cancellation convergence blocked count',
+    value.blocked,
+    0,
+    scanned - settledRuns,
+  );
+  if (typeof value.hasMore !== 'boolean') {
+    throw new TypeError('Cluster Run cancellation convergence continuation is invalid');
+  }
+  return Object.freeze({
+    scanned,
+    settledRuns,
+    settledAttempts,
+    blocked,
+    hasMore: value.hasMore,
+  });
+}
+
+/**
+ * Runs a bounded, sequential convergence cycle. It owns no timer or connection;
+ * deployment profiles choose the cadence and the repository owns row locking.
+ */
+export class ClusterRunCancellationConvergenceCoordinator {
+  private readonly pageSize: number;
+  private readonly maxPages: number;
+  private inFlight:
+    | Promise<Readonly<ClusterRunCancellationConvergenceCycleResult>>
+    | undefined;
+
+  constructor(
+    private readonly repository: ClusterRunCancellationConvergenceRepository,
+    options: ClusterRunCancellationConvergenceCoordinatorOptions,
+  ) {
+    if (
+      typeof repository?.convergePage !== 'function' ||
+      !options || typeof options !== 'object' || Array.isArray(options)
+    ) {
+      throw new TypeError('Cluster Run cancellation convergence coordinator is invalid');
+    }
+    this.pageSize = boundedInteger(
+      'Cluster Run cancellation convergence page size',
+      options.pageSize ?? 32,
+      1,
+      MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGE_SIZE,
+    );
+    this.maxPages = boundedInteger(
+      'Cluster Run cancellation convergence page limit',
+      options.maxPages ?? 4,
+      1,
+      MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGES_PER_CYCLE,
+    );
+  }
+
+  reconcile(): Promise<Readonly<ClusterRunCancellationConvergenceCycleResult>> {
+    if (this.inFlight) return this.inFlight;
+    const operation = this.reconcileOnce().finally(() => {
+      if (this.inFlight === operation) this.inFlight = undefined;
+    });
+    this.inFlight = operation;
+    return operation;
+  }
+
+  private async reconcileOnce(): Promise<Readonly<ClusterRunCancellationConvergenceCycleResult>> {
+    let pages = 0;
+    let scanned = 0;
+    let settledRuns = 0;
+    let settledAttempts = 0;
+    let blocked = 0;
+    for (; pages < this.maxPages; pages += 1) {
+      let page: Readonly<ClusterRunCancellationConvergencePageResult>;
+      try {
+        page = normalizeClusterRunCancellationConvergencePageResult(
+          await this.repository.convergePage({
+            limit: this.pageSize,
+          }),
+          this.pageSize,
+        );
+      } catch (error) {
+        if (error instanceof ClusterRunCancellationConvergenceUnavailableError) {
+          throw error;
+        }
+        throw new ClusterRunCancellationConvergenceUnavailableError({ cause: error });
+      }
+      scanned += page.scanned;
+      settledRuns += page.settledRuns;
+      settledAttempts += page.settledAttempts;
+      blocked += page.blocked;
+      const completedPages = pages + 1;
+      if (page.blocked > 0) {
+        return Object.freeze({
+          pages: completedPages,
+          scanned,
+          settledRuns,
+          settledAttempts,
+          blocked,
+          hasMore: page.hasMore,
+          remaining: true,
+          stopReason: 'blocked' as const,
+        });
+      }
+      if (!page.hasMore) {
+        return Object.freeze({
+          pages: completedPages,
+          scanned,
+          settledRuns,
+          settledAttempts,
+          blocked,
+          hasMore: false,
+          remaining: false,
+          stopReason: 'complete' as const,
+        });
+      }
+      if (page.scanned === 0 || page.settledRuns === 0) {
+        throw new ClusterRunCancellationConvergenceUnavailableError();
+      }
+    }
+    return Object.freeze({
+      pages,
+      scanned,
+      settledRuns,
+      settledAttempts,
+      blocked,
+      hasMore: true,
+      remaining: true,
+      stopReason: 'page_limit' as const,
+    });
+  }
+}

packages/ql3-runtime-core/src/run/clusterRunLostRetry.ts

@@ -0,0 +1,513 @@
+import type {
+  RunAttemptRecord,
+  RunEventRecord,
+  RunRecord,
+} from './run';
+import {
+  runRetryDelayMs,
+  type RunRetryPolicyRecord,
+} from './runRetryPolicy';
+
+export const MAX_CLUSTER_RUN_LOST_RETRY_PAGE_SIZE = 64;
+
+export type ClusterRunLostRetryDisposition =
+  | 'scheduled'
+  | 'requeued'
+  | 'failed_disabled'
+  | 'failed_unsafe'
+  | 'failed_exhausted';
+
+export interface ClusterRunLostRetryPageCommand {
+  readonly limit: number;
+}
+
+export interface ClusterRunLostRetryPageResult {
+  readonly scanned: number;
+  readonly scheduled: number;
+  readonly requeued: number;
+  readonly failed: number;
+  readonly raced: number;
+  readonly hasMore: boolean;
+}
+
+export interface ClusterRunLostRetryRepository {
+  reconcilePage(
+    command: Readonly<ClusterRunLostRetryPageCommand>,
+  ): Promise<Readonly<ClusterRunLostRetryPageResult>>;
+}
+
+export interface ClusterRunLostRetryCoordinatorOptions {
+  readonly pageSize?: number;
+}
+
+export interface ClusterRunLostRetryTransitionInput {
+  readonly run: Readonly<RunRecord>;
+  readonly attempt: Readonly<RunAttemptRecord>;
+  readonly policy: Readonly<RunRetryPolicyRecord> | null;
+  readonly observedAtMs: number;
+  readonly runEventId: string;
+  readonly attemptId?: string;
+  readonly attemptEventId?: string;
+}
+
+export interface ClusterRunLostRetryTransition {
+  readonly disposition: ClusterRunLostRetryDisposition;
+  readonly runTransitions: readonly Readonly<RunRecord>[];
+  readonly policy?: Readonly<RunRetryPolicyRecord>;
+  readonly attempt?: Readonly<RunAttemptRecord>;
+  readonly events: readonly Readonly<RunEventRecord>[];
+}
+
+export class ClusterRunLostRetryUnavailableError extends Error {
+  readonly code = 'CLUSTER_RUN_LOST_RETRY_UNAVAILABLE';
+
+  constructor(options?: ErrorOptions) {
+    super('Cluster Run lost retry is unavailable', options);
+    this.name = 'ClusterRunLostRetryUnavailableError';
+  }
+}
+
+export class InvalidClusterRunLostRetryTransitionError extends TypeError {
+  readonly code = 'CLUSTER_RUN_LOST_RETRY_INVALID';
+
+  constructor(message: string) {
+    super(`Cluster Run lost retry is invalid: ${message}`);
+    this.name = 'InvalidClusterRunLostRetryTransitionError';
+  }
+}
+
+function invalid(message: string): never {
+  throw new InvalidClusterRunLostRetryTransitionError(message);
+}
+
+function boundedInteger(
+  name: string,
+  value: number,
+  minimum: number,
+  maximum: number,
+): number {
+  if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
+    throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
+  }
+  return value;
+}
+
+function identifier(name: string, value: string | undefined): string {
+  if (
+    typeof value !== 'string' ||
+    value.length < 1 ||
+    value.length > 128 ||
+    /[\u0000-\u001f\u007f]/.test(value)
+  ) {
+    return invalid(`${name} is invalid`);
+  }
+  return value;
+}
+
+function reserve(
+  current: Readonly<RunRecord>,
+  status: RunRecord['status'],
+  atMs: number,
+  error?: Readonly<{ code: string; summary: string }>,
+): Readonly<RunRecord> {
+  const version = current.version + 1;
+  const eventSequence = current.eventSequence + 1;
+  if (
+    !Number.isSafeInteger(version) ||
+    version < 1 ||
+    !Number.isSafeInteger(eventSequence) ||
+    eventSequence < 1
+  ) {
+    return invalid('Run counter overflowed');
+  }
+  const next: RunRecord = {
+    ...current,
+    status,
+    version,
+    eventSequence,
+  };
+  if (status === 'queued') {
+    next.queuedAtMs = atMs;
+    delete next.errorCode;
+    delete next.errorSummary;
+  } else if (error) {
+    next.errorCode = error.code;
+    next.errorSummary = error.summary;
+  }
+  if (status === 'failed') next.finishedAtMs = atMs;
+  return Object.freeze(next);
+}
+
+function event(
+  id: string,
+  run: Readonly<RunRecord>,
+  attemptId: string,
+  type: string,
+  dedupeKey: string,
+  createdAtMs: number,
+  payload: Readonly<Record<string, unknown>>,
+): Readonly<RunEventRecord> {
+  return Object.freeze({
+    id: identifier('event ID', id),
+    runId: run.id,
+    sequence: run.eventSequence,
+    type,
+    dedupeKey,
+    actorType: 'reconciler',
+    attemptId,
+    payload: Object.freeze({ ...payload, version: run.version }),
+    createdAtMs,
+  });
+}
+
+function transitionTime(
+  input: Readonly<ClusterRunLostRetryTransitionInput>,
+): number {
+  const atMs = input.observedAtMs;
+  if (!Number.isSafeInteger(atMs) || atMs < 0) {
+    return invalid('observation time is invalid');
+  }
+  const lowerBound = Math.max(
+    input.run.createdAtMs,
+    input.run.startedAtMs ?? 0,
+    input.attempt.createdAtMs,
+    input.attempt.startedAtMs ?? 0,
+    input.attempt.finishedAtMs ?? 0,
+    input.policy?.updatedAtMs ?? 0,
+  );
+  if (atMs < lowerBound) return invalid('observation precedes durable state');
+  return atMs;
+}
+
+function finish(
+  input: Readonly<ClusterRunLostRetryTransitionInput>,
+  atMs: number,
+  disposition:
+    | 'failed_disabled'
+    | 'failed_unsafe'
+    | 'failed_exhausted',
+  error: Readonly<{ code: string; summary: string }>,
+): Readonly<ClusterRunLostRetryTransition> {
+  const run = reserve(input.run, 'failed', atMs, error);
+  let policy = input.policy ?? undefined;
+  if (input.policy?.nextAttemptAtMs !== undefined) {
+    const { nextAttemptAtMs: _nextAttemptAtMs, ...withoutNextAttempt } =
+      input.policy;
+    policy = Object.freeze({
+      ...withoutNextAttempt,
+      version: input.policy.version + 1,
+      updatedAtMs: atMs,
+    });
+  }
+  return Object.freeze({
+    disposition,
+    runTransitions: Object.freeze([run]),
+    ...(policy === undefined ? {} : { policy }),
+    events: Object.freeze([
+      event(
+        input.runEventId,
+        run,
+        input.attempt.id,
+        'run.failed',
+        `cluster-lost-retry:${disposition}:${input.attempt.id}`,
+        atMs,
+        {
+          from_status: input.run.status,
+          to_status: 'failed',
+          attempt: input.attempt.attempt,
+          error_code: error.code,
+        },
+      ),
+    ]),
+  });
+}
+
+/**
+ * Pure, profile-neutral lost recovery policy. Storage owns row locking and
+ * persistence; this function can only schedule, create a fresh Attempt, or
+ * terminalize an unsafe/exhausted Run.
+ */
+export function buildClusterRunLostRetryTransition(
+  input: Readonly<ClusterRunLostRetryTransitionInput>,
+): Readonly<ClusterRunLostRetryTransition> {
+  const { run, attempt, policy } = input;
+  if (
+    !run ||
+    !attempt ||
+    run.executionOwner !== 'runtime' ||
+    run.triggerType === 'plugin_package_workflow' ||
+    (run.status !== 'lost' && run.status !== 'retry_wait') ||
+    run.cancelRequestedAtMs !== undefined ||
+    attempt.runId !== run.id ||
+    attempt.status !== 'lost' ||
+    attempt.attempt < 1
+  ) {
+    return invalid('aggregate is not eligible');
+  }
+  const atMs = transitionTime(input);
+  identifier('Run ID', run.id);
+  identifier('Attempt ID', attempt.id);
+  identifier('Run event ID', input.runEventId);
+
+  if (!policy || !policy.retryOnLost || policy.maxAttempts <= 1) {
+    return finish(input, atMs, 'failed_disabled', {
+      code: 'RUN_LOST_RETRY_DISABLED',
+      summary: 'Run was lost and automatic retry was not enabled at admission',
+    });
+  }
+  if (policy.runId !== run.id) return invalid('retry policy Run mismatches');
+  if (policy.safety === 'unknown') {
+    return finish(input, atMs, 'failed_unsafe', {
+      code: 'RUN_LOST_RETRY_UNSAFE',
+      summary: 'Run was lost but execution safety was not declared',
+    });
+  }
+  if (attempt.attempt >= policy.maxAttempts) {
+    return finish(input, atMs, 'failed_exhausted', {
+      code: 'RUN_LOST_RETRY_EXHAUSTED',
+      summary: 'Run exhausted its admitted automatic retry attempts',
+    });
+  }
+
+  if (run.status === 'lost') {
+    const nextAttemptAtMs =
+      Math.max(run.createdAtMs, attempt.createdAtMs, attempt.finishedAtMs ?? 0) +
+      runRetryDelayMs(policy, attempt.attempt);
+    if (!Number.isSafeInteger(nextAttemptAtMs)) {
+      return invalid('next Attempt time overflowed');
+    }
+    const error = {
+      code: 'RUN_LOST_RETRY_SCHEDULED',
+      summary: 'A fresh Attempt will be created after the admitted backoff',
+    };
+    const nextRun = reserve(run, 'retry_wait', atMs, error);
+    const nextPolicy = Object.freeze({
+      ...policy,
+      nextAttemptAtMs,
+      version: policy.version + 1,
+      updatedAtMs: atMs,
+    });
+    return Object.freeze({
+      disposition: 'scheduled',
+      runTransitions: Object.freeze([nextRun]),
+      policy: nextPolicy,
+      events: Object.freeze([
+        event(
+          input.runEventId,
+          nextRun,
+          attempt.id,
+          'run.retry_wait',
+          `cluster-lost-retry:scheduled:${attempt.id}`,
+          atMs,
+          {
+            from_status: 'lost',
+            to_status: 'retry_wait',
+            attempt: attempt.attempt,
+            max_attempts: policy.maxAttempts,
+            safety: policy.safety,
+            next_attempt_at_ms: nextAttemptAtMs,
+            error_code: error.code,
+          },
+        ),
+      ]),
+    });
+  }
+
+  if (
+    policy.nextAttemptAtMs === undefined ||
+    policy.nextAttemptAtMs > atMs
+  ) {
+    return invalid('retry_wait policy is not due');
+  }
+  const nextAttemptId = identifier('replacement Attempt ID', input.attemptId);
+  const nextAttemptEventId = identifier(
+    'Attempt event ID',
+    input.attemptEventId,
+  );
+  const queued = reserve(run, 'queued', atMs);
+  const claimed = reserve(queued, 'queued', atMs);
+  const replacement = Object.freeze({
+    id: nextAttemptId,
+    runId: run.id,
+    attempt: attempt.attempt + 1,
+    status: 'claimed' as const,
+    executorType: attempt.executorType,
+    callbackSequence: 0,
+    createdAtMs: atMs,
+  });
+  const { nextAttemptAtMs: _nextAttemptAtMs, ...withoutNextAttempt } = policy;
+  const nextPolicy = Object.freeze({
+    ...withoutNextAttempt,
+    version: policy.version + 1,
+    updatedAtMs: atMs,
+  });
+  return Object.freeze({
+    disposition: 'requeued',
+    runTransitions: Object.freeze([queued, claimed]),
+    policy: nextPolicy,
+    attempt: replacement,
+    events: Object.freeze([
+      event(
+        input.runEventId,
+        queued,
+        replacement.id,
+        'run.queued',
+        `cluster-lost-retry:queued:${replacement.id}`,
+        atMs,
+        {
+          from_status: 'retry_wait',
+          to_status: 'queued',
+          previous_attempt_id: attempt.id,
+        },
+      ),
+      event(
+        nextAttemptEventId,
+        claimed,
+        replacement.id,
+        'attempt.claimed',
+        `cluster-lost-retry:attempt-claimed:${replacement.id}`,
+        atMs,
+        {
+          attempt: replacement.attempt,
+          executor_type: replacement.executorType,
+          previous_attempt_id: attempt.id,
+        },
+      ),
+    ]),
+  });
+}
+
+export function normalizeClusterRunLostRetryPageCommand(
+  value: ClusterRunLostRetryPageCommand,
+): Readonly<ClusterRunLostRetryPageCommand> {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    Object.keys(value).length !== 1 ||
+    !Object.prototype.hasOwnProperty.call(value, 'limit')
+  ) {
+    throw new TypeError('Cluster Run lost retry command is invalid');
+  }
+  return Object.freeze({
+    limit: boundedInteger(
+      'Cluster Run lost retry page size',
+      value.limit,
+      1,
+      MAX_CLUSTER_RUN_LOST_RETRY_PAGE_SIZE,
+    ),
+  });
+}
+
+export function normalizeClusterRunLostRetryPageResult(
+  value: ClusterRunLostRetryPageResult,
+  limit = MAX_CLUSTER_RUN_LOST_RETRY_PAGE_SIZE,
+): Readonly<ClusterRunLostRetryPageResult> {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    Object.keys(value).sort().join(',') !==
+      'failed,hasMore,raced,requeued,scanned,scheduled'
+  ) {
+    throw new TypeError('Cluster Run lost retry result is invalid');
+  }
+  const maximum = boundedInteger(
+    'Cluster Run lost retry result limit',
+    limit,
+    1,
+    MAX_CLUSTER_RUN_LOST_RETRY_PAGE_SIZE,
+  );
+  const scanned = boundedInteger(
+    'Cluster Run lost retry scanned count',
+    value.scanned,
+    0,
+    maximum,
+  );
+  const scheduled = boundedInteger(
+    'Cluster Run lost retry scheduled count',
+    value.scheduled,
+    0,
+    scanned,
+  );
+  const requeued = boundedInteger(
+    'Cluster Run lost retry requeued count',
+    value.requeued,
+    0,
+    scanned,
+  );
+  const failed = boundedInteger(
+    'Cluster Run lost retry failed count',
+    value.failed,
+    0,
+    scanned,
+  );
+  const raced = boundedInteger(
+    'Cluster Run lost retry raced count',
+    value.raced,
+    0,
+    scanned,
+  );
+  if (
+    scheduled + requeued + failed + raced !== scanned ||
+    typeof value.hasMore !== 'boolean'
+  ) {
+    throw new TypeError('Cluster Run lost retry result counts are invalid');
+  }
+  return Object.freeze({
+    scanned,
+    scheduled,
+    requeued,
+    failed,
+    raced,
+    hasMore: value.hasMore,
+  });
+}
+
+/** One non-overlapping bounded page; deployment owns the shared cadence. */
+export class ClusterRunLostRetryCoordinator {
+  private readonly pageSize: number;
+  private inFlight:
+    | Promise<Readonly<ClusterRunLostRetryPageResult>>
+    | undefined;
+
+  constructor(
+    private readonly repository: ClusterRunLostRetryRepository,
+    options: ClusterRunLostRetryCoordinatorOptions = {},
+  ) {
+    if (
+      typeof repository?.reconcilePage !== 'function' ||
+      !options ||
+      typeof options !== 'object' ||
+      Array.isArray(options)
+    ) {
+      throw new TypeError('Cluster Run lost retry coordinator is invalid');
+    }
+    this.pageSize = boundedInteger(
+      'Cluster Run lost retry page size',
+      options.pageSize ?? 16,
+      1,
+      MAX_CLUSTER_RUN_LOST_RETRY_PAGE_SIZE,
+    );
+  }
+
+  reconcile(): Promise<Readonly<ClusterRunLostRetryPageResult>> {
+    if (this.inFlight) return this.inFlight;
+    const operation = Promise.resolve()
+      .then(() =>
+        this.repository.reconcilePage({ limit: this.pageSize }),
+      )
+      .then((result) =>
+        normalizeClusterRunLostRetryPageResult(result, this.pageSize),
+      )
+      .catch((error: unknown) => {
+        if (error instanceof ClusterRunLostRetryUnavailableError) throw error;
+        throw new ClusterRunLostRetryUnavailableError({ cause: error });
+      })
+      .finally(() => {
+        if (this.inFlight === operation) this.inFlight = undefined;
+      });
+    this.inFlight = operation;
+    return operation;
+  }
+}

packages/ql3-runtime-core/src/run/projectRunList.ts

@@ -0,0 +1,84 @@
+import type { RunRecord } from './run';
+
+export const MAX_PROJECT_RUN_LIST_STORAGE_LIMIT = 65;
+
+export interface ProjectRunListCursor {
+  readonly createdAtMs: number;
+  readonly runId: string;
+}
+
+export interface ProjectRunListQuery {
+  readonly projectId: string;
+  readonly limit: number;
+  readonly after?: Readonly<ProjectRunListCursor>;
+}
+
+export interface ProjectRunListReader {
+  listRunsByProject(
+    query: Readonly<ProjectRunListQuery>,
+  ): Promise<readonly RunRecord[]>;
+}
+
+const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
+
+function boundedText(value: unknown, maximum: number): value is string {
+  return (
+    typeof value === 'string' &&
+    value.length > 0 &&
+    value.length <= maximum &&
+    !CONTROL_PATTERN.test(value)
+  );
+}
+
+export function normalizeProjectRunListQuery(
+  value: Readonly<ProjectRunListQuery>,
+): Readonly<ProjectRunListQuery> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new TypeError('Project Run list query is invalid');
+  }
+  const keys = Reflect.ownKeys(value);
+  const afterKeys =
+    value.after &&
+    typeof value.after === 'object' &&
+    !Array.isArray(value.after)
+      ? Reflect.ownKeys(value.after)
+      : [];
+  if (
+    keys.length < 2 ||
+    keys.length > 3 ||
+    keys.some(
+      (key) => key !== 'projectId' && key !== 'limit' && key !== 'after',
+    ) ||
+    !Object.hasOwn(value, 'projectId') ||
+    !Object.hasOwn(value, 'limit') ||
+    !boundedText(value.projectId, 128) ||
+    !Number.isSafeInteger(value.limit) ||
+    value.limit < 1 ||
+    value.limit > MAX_PROJECT_RUN_LIST_STORAGE_LIMIT ||
+    (value.after !== undefined &&
+      (!value.after ||
+        typeof value.after !== 'object' ||
+        Array.isArray(value.after) ||
+        afterKeys.length !== 2 ||
+        afterKeys.some((key) => key !== 'createdAtMs' && key !== 'runId') ||
+        !Object.hasOwn(value.after, 'createdAtMs') ||
+        !Object.hasOwn(value.after, 'runId') ||
+        !Number.isSafeInteger(value.after.createdAtMs) ||
+        value.after.createdAtMs < 0 ||
+        !boundedText(value.after.runId, 128)))
+  ) {
+    throw new TypeError('Project Run list query is invalid');
+  }
+  return Object.freeze({
+    projectId: value.projectId,
+    limit: value.limit,
+    ...(value.after === undefined
+      ? {}
+      : {
+          after: Object.freeze({
+            createdAtMs: value.after.createdAtMs,
+            runId: value.after.runId,
+          }),
+        }),
+  });
+}

packages/ql3-runtime-core/src/run/projection/boundedRunEventListProjection.ts

@@ -0,0 +1,192 @@
+import {
+  RUN_EVENT_ACTOR_TYPES,
+  type RunEventRecord,
+  type RunRecord,
+} from '../run';
+import type { RunRepositoryReader } from '../runRepository';
+
+export const DEFAULT_BOUNDED_RUN_EVENT_LIST_LIMIT = 32;
+export const MAX_BOUNDED_RUN_EVENT_LIST_LIMIT = 64;
+
+const MAX_INT = 2_147_483_647;
+const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
+
+export interface BoundedRunEventListInput {
+  readonly afterSequence?: number;
+  readonly limit?: number;
+}
+
+export interface BoundedRunEventListItem {
+  readonly sequence: number;
+  readonly type: string;
+  readonly actorType: RunEventRecord['actorType'];
+  readonly createdAtMs: number;
+}
+
+export interface BoundedRunEventListProjection {
+  readonly found: boolean;
+  readonly events: readonly Readonly<BoundedRunEventListItem>[];
+  readonly hasMore: boolean;
+  readonly nextAfterSequence: number;
+}
+
+export class InvalidBoundedRunEventListProjectionError extends TypeError {
+  readonly code = 'BOUNDED_RUN_EVENT_LIST_PROJECTION_INVALID';
+
+  constructor() {
+    super('Bounded Run event list projection input is invalid');
+    this.name = 'InvalidBoundedRunEventListProjectionError';
+  }
+}
+
+export class BoundedRunEventListProjectionUnavailableError extends Error {
+  readonly code = 'BOUNDED_RUN_EVENT_LIST_PROJECTION_UNAVAILABLE';
+
+  constructor() {
+    super('Bounded Run event list projection is unavailable');
+    this.name = 'BoundedRunEventListProjectionUnavailableError';
+  }
+}
+
+function boundedText(value: unknown, maximum: number): value is string {
+  return (
+    typeof value === 'string' &&
+    value.length > 0 &&
+    value.length <= maximum &&
+    !CONTROL_PATTERN.test(value)
+  );
+}
+
+function integer(
+  value: unknown,
+  minimum: number,
+  maximum: number,
+): value is number {
+  return (
+    Number.isSafeInteger(value) &&
+    Number(value) >= minimum &&
+    Number(value) <= maximum
+  );
+}
+
+function normalizeInput(value: Readonly<BoundedRunEventListInput>): Readonly<{
+  afterSequence: number;
+  limit: number;
+}> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new InvalidBoundedRunEventListProjectionError();
+  }
+  const keys = Reflect.ownKeys(value);
+  if (
+    keys.length > 2 ||
+    keys.some((key) => key !== 'afterSequence' && key !== 'limit') ||
+    (value.afterSequence !== undefined &&
+      !integer(value.afterSequence, 0, MAX_INT)) ||
+    (value.limit !== undefined &&
+      !integer(value.limit, 1, MAX_BOUNDED_RUN_EVENT_LIST_LIMIT))
+  ) {
+    throw new InvalidBoundedRunEventListProjectionError();
+  }
+  return Object.freeze({
+    afterSequence: value.afterSequence ?? 0,
+    limit: value.limit ?? DEFAULT_BOUNDED_RUN_EVENT_LIST_LIMIT,
+  });
+}
+
+function ownsRun(value: RunRecord, projectId: string, runId: string): boolean {
+  return (
+    !!value &&
+    typeof value === 'object' &&
+    !Array.isArray(value) &&
+    value.projectId === projectId &&
+    value.id === runId &&
+    boundedText(value.projectId, 128) &&
+    boundedText(value.id, 128)
+  );
+}
+
+function projectEvent(
+  value: RunEventRecord,
+  runId: string,
+  previousSequence: number,
+): Readonly<BoundedRunEventListItem> | null {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    value.runId !== runId ||
+    !integer(value.sequence, previousSequence + 1, MAX_INT) ||
+    !boundedText(value.type, 128) ||
+    !RUN_EVENT_ACTOR_TYPES.includes(value.actorType) ||
+    !integer(value.createdAtMs, 0, Number.MAX_SAFE_INTEGER)
+  ) {
+    return null;
+  }
+  return Object.freeze({
+    sequence: value.sequence,
+    type: value.type,
+    actorType: value.actorType,
+    createdAtMs: value.createdAtMs,
+  });
+}
+
+export async function executeBoundedRunEventListProjection(
+  runs: Pick<RunRepositoryReader, 'findRunById' | 'listEvents'>,
+  projectId: string,
+  runId: string,
+  input: Readonly<BoundedRunEventListInput>,
+): Promise<Readonly<BoundedRunEventListProjection>> {
+  if (
+    !runs ||
+    typeof runs.findRunById !== 'function' ||
+    typeof runs.listEvents !== 'function' ||
+    !boundedText(projectId, 128) ||
+    !boundedText(runId, 128)
+  ) {
+    throw new InvalidBoundedRunEventListProjectionError();
+  }
+  const normalized = normalizeInput(input);
+  let run: RunRecord | null;
+  try {
+    run = await runs.findRunById(runId);
+  } catch {
+    throw new BoundedRunEventListProjectionUnavailableError();
+  }
+  if (!run || !ownsRun(run, projectId, runId)) {
+    return Object.freeze({
+      found: false,
+      events: Object.freeze([]),
+      hasMore: false,
+      nextAfterSequence: normalized.afterSequence,
+    });
+  }
+
+  let rows: RunEventRecord[];
+  try {
+    rows = await runs.listEvents(runId, {
+      afterSequence: normalized.afterSequence,
+      limit: normalized.limit + 1,
+    });
+  } catch {
+    throw new BoundedRunEventListProjectionUnavailableError();
+  }
+  if (!Array.isArray(rows) || rows.length > normalized.limit + 1) {
+    throw new BoundedRunEventListProjectionUnavailableError();
+  }
+
+  const events: Readonly<BoundedRunEventListItem>[] = [];
+  let previousSequence = normalized.afterSequence;
+  for (let index = 0; index < rows.length; index += 1) {
+    const projected = projectEvent(rows[index]!, runId, previousSequence);
+    if (!projected) throw new BoundedRunEventListProjectionUnavailableError();
+    previousSequence = projected.sequence;
+    if (index < normalized.limit) events.push(projected);
+  }
+  const lastReturned = events.at(-1);
+  return Object.freeze({
+    found: true,
+    events: Object.freeze(events),
+    hasMore: rows.length > normalized.limit,
+    nextAfterSequence: lastReturned?.sequence ?? normalized.afterSequence,
+  });
+}

packages/ql3-runtime-core/src/run/projection/boundedRunListProjection.ts

@@ -0,0 +1,237 @@
+import {
+  EXECUTION_ORIGINS,
+  RUN_STATUSES,
+  type ExecutionOrigin,
+  type ExecutionOwner,
+  type RunRecord,
+  type RunStatus,
+} from '../run';
+import {
+  normalizeProjectRunListQuery,
+  type ProjectRunListCursor,
+  type ProjectRunListReader,
+} from '../projectRunList';
+
+export const DEFAULT_BOUNDED_RUN_LIST_LIMIT = 32;
+export const MAX_BOUNDED_RUN_LIST_LIMIT = 64;
+
+const MAX_INT = 2_147_483_647;
+const MIN_INT = -2_147_483_648;
+const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
+
+export interface BoundedRunListInput {
+  readonly limit?: number;
+  readonly after?: Readonly<ProjectRunListCursor>;
+}
+
+export interface BoundedRunListItem {
+  readonly id: string;
+  readonly taskId: string;
+  readonly taskRevision: string;
+  readonly status: RunStatus;
+  readonly version: number;
+  readonly eventSequence: number;
+  readonly priority: number;
+  readonly executionOrigin: ExecutionOrigin;
+  readonly executionOwner: ExecutionOwner;
+  readonly createdAtMs: number;
+  readonly queuedAtMs?: number;
+  readonly startedAtMs?: number;
+  readonly finishedAtMs?: number;
+}
+
+export interface BoundedRunListProjection {
+  readonly runs: readonly Readonly<BoundedRunListItem>[];
+  readonly hasMore: boolean;
+  readonly next?: Readonly<ProjectRunListCursor>;
+}
+
+export class InvalidBoundedRunListProjectionError extends TypeError {
+  readonly code = 'BOUNDED_RUN_LIST_PROJECTION_INVALID';
+
+  constructor() {
+    super('Bounded Run list projection input is invalid');
+    this.name = 'InvalidBoundedRunListProjectionError';
+  }
+}
+
+export class BoundedRunListProjectionUnavailableError extends Error {
+  readonly code = 'BOUNDED_RUN_LIST_PROJECTION_UNAVAILABLE';
+
+  constructor() {
+    super('Bounded Run list projection is unavailable');
+    this.name = 'BoundedRunListProjectionUnavailableError';
+  }
+}
+
+function boundedText(value: unknown, maximum: number): value is string {
+  return (
+    typeof value === 'string' &&
+    value.length > 0 &&
+    value.length <= maximum &&
+    !CONTROL_PATTERN.test(value)
+  );
+}
+
+function integer(value: unknown, minimum: number, maximum: number): value is number {
+  return (
+    Number.isSafeInteger(value) &&
+    Number(value) >= minimum &&
+    Number(value) <= maximum
+  );
+}
+
+function cursor(value: unknown): value is Readonly<ProjectRunListCursor> {
+  return (
+    !!value &&
+    typeof value === 'object' &&
+    !Array.isArray(value) &&
+    Reflect.ownKeys(value).length === 2 &&
+    Object.hasOwn(value, 'createdAtMs') &&
+    Object.hasOwn(value, 'runId') &&
+    integer((value as ProjectRunListCursor).createdAtMs, 0, Number.MAX_SAFE_INTEGER) &&
+    boundedText((value as ProjectRunListCursor).runId, 128)
+  );
+}
+
+function normalizeInput(value: Readonly<BoundedRunListInput>): Readonly<{
+  limit: number;
+  after?: Readonly<ProjectRunListCursor>;
+}> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new InvalidBoundedRunListProjectionError();
+  }
+  const keys = Reflect.ownKeys(value);
+  if (
+    keys.length > 2 ||
+    keys.some((key) => key !== 'limit' && key !== 'after') ||
+    (value.limit !== undefined &&
+      !integer(value.limit, 1, MAX_BOUNDED_RUN_LIST_LIMIT)) ||
+    (value.after !== undefined && !cursor(value.after))
+  ) {
+    throw new InvalidBoundedRunListProjectionError();
+  }
+  return Object.freeze({
+    limit: value.limit ?? DEFAULT_BOUNDED_RUN_LIST_LIMIT,
+    ...(value.after === undefined
+      ? {}
+      : {
+          after: Object.freeze({
+            createdAtMs: value.after.createdAtMs,
+            runId: value.after.runId,
+          }),
+        }),
+  });
+}
+
+function isBefore(
+  value: Readonly<ProjectRunListCursor>,
+  boundary: Readonly<ProjectRunListCursor>,
+): boolean {
+  return (
+    value.createdAtMs < boundary.createdAtMs ||
+    (value.createdAtMs === boundary.createdAtMs && value.runId < boundary.runId)
+  );
+}
+
+function optionalTimestamp(value: unknown): value is number | undefined {
+  return value === undefined || integer(value, 0, Number.MAX_SAFE_INTEGER);
+}
+
+function projectRun(
+  value: RunRecord,
+  projectId: string,
+  boundary?: Readonly<ProjectRunListCursor>,
+): Readonly<BoundedRunListItem> | null {
+  const rowCursor = { createdAtMs: value?.createdAtMs, runId: value?.id };
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    value.projectId !== projectId ||
+    !cursor(rowCursor) ||
+    (boundary !== undefined && !isBefore(rowCursor, boundary)) ||
+    !boundedText(value.taskId, 255) ||
+    !boundedText(value.taskRevision, 255) ||
+    !RUN_STATUSES.includes(value.status) ||
+    !EXECUTION_ORIGINS.includes(value.executionOrigin) ||
+    (value.executionOwner !== 'legacy' && value.executionOwner !== 'runtime') ||
+    !integer(value.version, 0, MAX_INT) ||
+    !integer(value.eventSequence, 0, MAX_INT) ||
+    !integer(value.priority, MIN_INT, MAX_INT) ||
+    !optionalTimestamp(value.queuedAtMs) ||
+    !optionalTimestamp(value.startedAtMs) ||
+    !optionalTimestamp(value.finishedAtMs)
+  ) {
+    return null;
+  }
+  return Object.freeze({
+    id: value.id,
+    taskId: value.taskId,
+    taskRevision: value.taskRevision,
+    status: value.status,
+    version: value.version,
+    eventSequence: value.eventSequence,
+    priority: value.priority,
+    executionOrigin: value.executionOrigin,
+    executionOwner: value.executionOwner,
+    createdAtMs: value.createdAtMs,
+    ...(value.queuedAtMs === undefined ? {} : { queuedAtMs: value.queuedAtMs }),
+    ...(value.startedAtMs === undefined ? {} : { startedAtMs: value.startedAtMs }),
+    ...(value.finishedAtMs === undefined ? {} : { finishedAtMs: value.finishedAtMs }),
+  });
+}
+
+export async function executeBoundedRunListProjection(
+  runs: ProjectRunListReader,
+  projectId: string,
+  input: Readonly<BoundedRunListInput>,
+): Promise<Readonly<BoundedRunListProjection>> {
+  if (
+    !runs ||
+    typeof runs.listRunsByProject !== 'function' ||
+    !boundedText(projectId, 128)
+  ) {
+    throw new InvalidBoundedRunListProjectionError();
+  }
+  const normalized = normalizeInput(input);
+  const query = normalizeProjectRunListQuery({
+    projectId,
+    limit: normalized.limit + 1,
+    ...(normalized.after === undefined ? {} : { after: normalized.after }),
+  });
+  let rows: readonly RunRecord[];
+  try {
+    rows = await runs.listRunsByProject(query);
+  } catch {
+    throw new BoundedRunListProjectionUnavailableError();
+  }
+  if (!Array.isArray(rows) || rows.length > normalized.limit + 1) {
+    throw new BoundedRunListProjectionUnavailableError();
+  }
+
+  const projected: Readonly<BoundedRunListItem>[] = [];
+  let boundary = normalized.after;
+  for (let index = 0; index < rows.length; index += 1) {
+    const row = rows[index]!;
+    const item = projectRun(row, projectId, boundary);
+    if (!item) throw new BoundedRunListProjectionUnavailableError();
+    boundary = Object.freeze({ createdAtMs: row.createdAtMs, runId: row.id });
+    if (index < normalized.limit) projected.push(item);
+  }
+
+  const hasMore = rows.length > normalized.limit;
+  const lastReturned = projected.at(-1);
+  return Object.freeze({
+    runs: Object.freeze(projected),
+    hasMore,
+    ...(hasMore && lastReturned
+      ? {
+          next: Object.freeze({
+            createdAtMs: lastReturned.createdAtMs,
+            runId: lastReturned.id,
+          }),
+        }
+      : {}),
+  });
+}

packages/ql3-runtime-core/src/run/projection/boundedRunReadProjection.ts

@@ -0,0 +1,121 @@
+import { EXECUTION_ORIGINS, RUN_STATUSES, type RunRecord } from '../run';
+import type { RunRepositoryReader } from '../runRepository';
+
+const MAX_INT = 2_147_483_647;
+const MIN_INT = -2_147_483_648;
+const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
+
+export type BoundedRunReadProjection = Readonly<
+  Record<string, boolean | string | number>
+>;
+
+export class BoundedRunReadProjectionUnavailableError extends Error {
+  readonly code = 'BOUNDED_RUN_READ_PROJECTION_UNAVAILABLE';
+
+  constructor() {
+    super('Bounded Run read projection is unavailable');
+    this.name = 'BoundedRunReadProjectionUnavailableError';
+  }
+}
+
+function boundedText(value: unknown, maximum: number): value is string {
+  return (
+    typeof value === 'string' &&
+    value.length > 0 &&
+    value.length <= maximum &&
+    !CONTROL_PATTERN.test(value)
+  );
+}
+
+function integer(
+  value: unknown,
+  minimum: number,
+  maximum: number,
+): value is number {
+  return (
+    Number.isSafeInteger(value) &&
+    Number(value) >= minimum &&
+    Number(value) <= maximum
+  );
+}
+
+function optionalTimestamp(value: unknown): value is number | undefined {
+  return value === undefined || integer(value, 0, Number.MAX_SAFE_INTEGER);
+}
+
+function projectRun(
+  value: RunRecord,
+  projectId: string,
+  runId: string,
+): BoundedRunReadProjection | null {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    value.projectId !== projectId ||
+    value.id !== runId ||
+    !boundedText(value.id, 128) ||
+    !boundedText(value.projectId, 128) ||
+    !boundedText(value.taskId, 255) ||
+    !boundedText(value.taskRevision, 255) ||
+    !RUN_STATUSES.includes(value.status) ||
+    !EXECUTION_ORIGINS.includes(value.executionOrigin) ||
+    (value.executionOwner !== 'legacy' && value.executionOwner !== 'runtime') ||
+    !integer(value.version, 0, MAX_INT) ||
+    !integer(value.eventSequence, 0, MAX_INT) ||
+    !integer(value.priority, MIN_INT, MAX_INT) ||
+    !integer(value.createdAtMs, 0, Number.MAX_SAFE_INTEGER) ||
+    !optionalTimestamp(value.queuedAtMs) ||
+    !optionalTimestamp(value.startedAtMs) ||
+    !optionalTimestamp(value.finishedAtMs)
+  ) {
+    return null;
+  }
+  return Object.freeze({
+    found: true,
+    id: value.id,
+    taskId: value.taskId,
+    taskRevision: value.taskRevision,
+    status: value.status,
+    version: value.version,
+    eventSequence: value.eventSequence,
+    priority: value.priority,
+    executionOrigin: value.executionOrigin,
+    executionOwner: value.executionOwner,
+    createdAtMs: value.createdAtMs,
+    ...(value.queuedAtMs === undefined ? {} : { queuedAtMs: value.queuedAtMs }),
+    ...(value.startedAtMs === undefined
+      ? {}
+      : { startedAtMs: value.startedAtMs }),
+    ...(value.finishedAtMs === undefined
+      ? {}
+      : { finishedAtMs: value.finishedAtMs }),
+  });
+}
+
+export async function executeBoundedRunReadProjection(
+  runs: Pick<RunRepositoryReader, 'findRunById'>,
+  projectId: string,
+  runId: string,
+): Promise<BoundedRunReadProjection> {
+  if (
+    !runs ||
+    typeof runs.findRunById !== 'function' ||
+    !boundedText(projectId, 128) ||
+    !boundedText(runId, 128)
+  ) {
+    throw new TypeError('Bounded Run read projection input is invalid');
+  }
+  let run: RunRecord | null;
+  try {
+    run = await runs.findRunById(runId);
+  } catch {
+    throw new BoundedRunReadProjectionUnavailableError();
+  }
+  if (!run || run.projectId !== projectId) {
+    return Object.freeze({ found: false });
+  }
+  const projection = projectRun(run, projectId, runId);
+  if (!projection) throw new BoundedRunReadProjectionUnavailableError();
+  return projection;
+}

packages/ql3-runtime-core/src/run/projection/boundedRunStepListProjection.ts

@@ -0,0 +1,230 @@
+import type { RunRecord } from '../run';
+import type { RunRepositoryReader } from '../runRepository';
+import {
+  normalizeListStepRunsResult,
+  type StepRunKind,
+  type StepRunRecord,
+  type StepRunRepository,
+  type StepRunStatus,
+} from '../stepRun';
+
+export const DEFAULT_BOUNDED_RUN_STEP_LIST_LIMIT = 32;
+export const MAX_BOUNDED_RUN_STEP_LIST_LIMIT = 64;
+
+const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
+
+export interface BoundedRunStepListCursor {
+  readonly stepKey: string;
+  readonly stepRunId: string;
+}
+
+export interface BoundedRunStepListInput {
+  readonly after?: Readonly<BoundedRunStepListCursor>;
+  readonly limit?: number;
+}
+
+export interface BoundedRunStepListItem {
+  readonly id: string;
+  readonly parentStepRunId: string | null;
+  readonly stepKey: string;
+  readonly kind: StepRunKind;
+  readonly required: boolean;
+  readonly status: StepRunStatus;
+  readonly version: number;
+  readonly attemptCount: number;
+  readonly readyAtMs: number | null;
+  readonly startedAtMs: number | null;
+  readonly finishedAtMs: number | null;
+  readonly resultCode: string | null;
+  readonly createdAtMs: number;
+  readonly updatedAtMs: number;
+}
+
+export interface BoundedRunStepListProjection {
+  readonly found: boolean;
+  readonly steps: readonly Readonly<BoundedRunStepListItem>[];
+  readonly hasMore: boolean;
+  readonly next: Readonly<BoundedRunStepListCursor> | null;
+}
+
+export class InvalidBoundedRunStepListProjectionError extends TypeError {
+  readonly code = 'BOUNDED_RUN_STEP_LIST_PROJECTION_INVALID';
+
+  constructor() {
+    super('Bounded Run Step list projection input is invalid');
+    this.name = 'InvalidBoundedRunStepListProjectionError';
+  }
+}
+
+export class BoundedRunStepListProjectionUnavailableError extends Error {
+  readonly code = 'BOUNDED_RUN_STEP_LIST_PROJECTION_UNAVAILABLE';
+
+  constructor() {
+    super('Bounded Run Step list projection is unavailable');
+    this.name = 'BoundedRunStepListProjectionUnavailableError';
+  }
+}
+
+function boundedText(value: unknown, maximum: number): value is string {
+  return (
+    typeof value === 'string' &&
+    value.length > 0 &&
+    value.length <= maximum &&
+    !CONTROL_PATTERN.test(value)
+  );
+}
+
+function normalizeInput(value: Readonly<BoundedRunStepListInput>): Readonly<{
+  after?: Readonly<BoundedRunStepListCursor>;
+  limit: number;
+}> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new InvalidBoundedRunStepListProjectionError();
+  }
+  const keys = Reflect.ownKeys(value);
+  if (
+    keys.length > 2 ||
+    keys.some((key) => key !== 'after' && key !== 'limit') ||
+    (value.limit !== undefined &&
+      (!Number.isSafeInteger(value.limit) ||
+        value.limit < 1 ||
+        value.limit > MAX_BOUNDED_RUN_STEP_LIST_LIMIT))
+  ) {
+    throw new InvalidBoundedRunStepListProjectionError();
+  }
+  let after: Readonly<BoundedRunStepListCursor> | undefined;
+  if (value.after !== undefined) {
+    if (
+      !value.after ||
+      typeof value.after !== 'object' ||
+      Array.isArray(value.after) ||
+      Reflect.ownKeys(value.after).length !== 2 ||
+      !Object.hasOwn(value.after, 'stepKey') ||
+      !Object.hasOwn(value.after, 'stepRunId') ||
+      !boundedText(value.after.stepKey, 128) ||
+      !boundedText(value.after.stepRunId, 128)
+    ) {
+      throw new InvalidBoundedRunStepListProjectionError();
+    }
+    after = Object.freeze({
+      stepKey: value.after.stepKey,
+      stepRunId: value.after.stepRunId,
+    });
+  }
+  return Object.freeze({
+    ...(after === undefined ? {} : { after }),
+    limit: value.limit ?? DEFAULT_BOUNDED_RUN_STEP_LIST_LIMIT,
+  });
+}
+
+function ownsRun(value: RunRecord, projectId: string, runId: string): boolean {
+  return (
+    !!value &&
+    typeof value === 'object' &&
+    !Array.isArray(value) &&
+    value.projectId === projectId &&
+    value.id === runId &&
+    boundedText(value.projectId, 128) &&
+    boundedText(value.id, 128)
+  );
+}
+
+function projectStep(
+  value: Readonly<StepRunRecord>,
+): Readonly<BoundedRunStepListItem> {
+  return Object.freeze({
+    id: value.id,
+    parentStepRunId: value.parentStepRunId,
+    stepKey: value.stepKey,
+    kind: value.kind,
+    required: value.required,
+    status: value.status,
+    version: value.version,
+    attemptCount: value.attemptCount,
+    readyAtMs: value.readyAtMs,
+    startedAtMs: value.startedAtMs,
+    finishedAtMs: value.finishedAtMs,
+    resultCode: value.resultCode,
+    createdAtMs: value.createdAtMs,
+    updatedAtMs: value.updatedAtMs,
+  });
+}
+
+export async function executeBoundedRunStepListProjection(
+  runs: Pick<RunRepositoryReader, 'findRunById'>,
+  stepRuns: Pick<StepRunRepository, 'listByRun'>,
+  projectId: string,
+  runId: string,
+  input: Readonly<BoundedRunStepListInput>,
+): Promise<Readonly<BoundedRunStepListProjection>> {
+  if (
+    !runs ||
+    typeof runs.findRunById !== 'function' ||
+    !stepRuns ||
+    typeof stepRuns.listByRun !== 'function' ||
+    !boundedText(projectId, 128) ||
+    !boundedText(runId, 128)
+  ) {
+    throw new InvalidBoundedRunStepListProjectionError();
+  }
+  const normalized = normalizeInput(input);
+  let run: RunRecord | null;
+  try {
+    run = await runs.findRunById(runId);
+  } catch {
+    throw new BoundedRunStepListProjectionUnavailableError();
+  }
+  if (!run || !ownsRun(run, projectId, runId)) {
+    return Object.freeze({
+      found: false,
+      steps: Object.freeze([]),
+      hasMore: false,
+      next: null,
+    });
+  }
+
+  const query = Object.freeze({
+    runId,
+    limit: normalized.limit,
+    ...(normalized.after === undefined
+      ? {}
+      : {
+          after: Object.freeze({
+            stepKey: normalized.after.stepKey,
+            id: normalized.after.stepRunId,
+          }),
+        }),
+  });
+  try {
+    const page = normalizeListStepRunsResult(
+      await stepRuns.listByRun(query),
+      query,
+    );
+    const ids = new Set<string>();
+    const stepKeys = new Set<string>();
+    for (const step of page.stepRuns) {
+      if (ids.has(step.id) || stepKeys.has(step.stepKey)) {
+        throw new BoundedRunStepListProjectionUnavailableError();
+      }
+      ids.add(step.id);
+      stepKeys.add(step.stepKey);
+    }
+    return Object.freeze({
+      found: true,
+      steps: Object.freeze(page.stepRuns.map(projectStep)),
+      hasMore: page.truncated,
+      next:
+        page.next === undefined
+          ? null
+          : Object.freeze({
+              stepKey: page.next.stepKey,
+              stepRunId: page.next.id,
+            }),
+    });
+  } catch (error) {
+    if (error instanceof BoundedRunStepListProjectionUnavailableError) {
+      throw error;
+    }
+    throw new BoundedRunStepListProjectionUnavailableError();
+  }
+}

packages/ql3-runtime-core/src/run/repositoryErrors.ts

@@ -0,0 +1,87 @@
+export class RunRepositoryError extends Error {
+  constructor(
+    message: string,
+    public readonly code: string,
+    public readonly retryable = false,
+    public readonly cause?: unknown,
+  ) {
+    super(message);
+    this.name = new.target.name;
+  }
+}
+
+export class DuplicateIdempotencyKeyError extends RunRepositoryError {
+  constructor(
+    public readonly projectId: string,
+    public readonly idempotencyKey: string,
+  ) {
+    super(
+      'A Run with the same project idempotency key already exists',
+      'DUPLICATE_RUN_IDEMPOTENCY_KEY',
+    );
+  }
+}
+
+export class DuplicateRunAttemptError extends RunRepositoryError {
+  constructor(public readonly runId: string, public readonly attempt: number) {
+    super(
+      'A RunAttempt with the same run and attempt number already exists',
+      'DUPLICATE_RUN_ATTEMPT',
+    );
+  }
+}
+
+export class DuplicateRunEventError extends RunRepositoryError {
+  constructor(
+    public readonly runId: string,
+    public readonly dedupeKey?: string,
+  ) {
+    super(
+      'A RunEvent with the same sequence or dedupe key already exists',
+      'DUPLICATE_RUN_EVENT',
+    );
+  }
+}
+
+export class RunRepositoryConstraintError extends RunRepositoryError {
+  constructor(
+    message = 'Run repository constraint violation',
+    cause?: unknown,
+  ) {
+    super(message, 'RUN_REPOSITORY_CONSTRAINT', false, cause);
+  }
+}
+
+export class RunRepositoryBusyError extends RunRepositoryError {
+  constructor(cause?: unknown) {
+    super(
+      'Run repository is temporarily busy',
+      'RUN_REPOSITORY_BUSY',
+      true,
+      cause,
+    );
+  }
+}
+
+export class RunRepositoryOperationError extends RunRepositoryError {
+  constructor(cause?: unknown) {
+    super(
+      'Run repository operation failed',
+      'RUN_REPOSITORY_OPERATION_FAILED',
+      false,
+      cause,
+    );
+  }
+}
+
+export class RunEventPayloadTooLargeError extends RunRepositoryError {
+  constructor(
+    public readonly actualBytes: number,
+    public readonly maxBytes: number,
+  ) {
+    super(
+      'RunEvent payload exceeds the configured size limit',
+      'RUN_EVENT_PAYLOAD_TOO_LARGE',
+    );
+  }
+}

packages/ql3-runtime-core/src/run/run.ts

@@ -0,0 +1,150 @@
+export const RUN_STATUSES = [
+  'created',
+  'queued',
+  'dispatching',
+  'running',
+  'waiting_approval',
+  'retry_wait',
+  'lost',
+  'succeeded',
+  'failed',
+  'cancelled',
+  'timed_out',
+] as const;
+
+export type RunStatus = (typeof RUN_STATUSES)[number];
+
+export const RUN_ATTEMPT_STATUSES = [
+  'claimed',
+  'starting',
+  'running',
+  'succeeded',
+  'failed',
+  'cancelled',
+  'timed_out',
+  'lost',
+] as const;
+
+export type RunAttemptStatus = (typeof RUN_ATTEMPT_STATUSES)[number];
+
+export const EXECUTION_ORIGINS = [
+  'manual',
+  'scheduled_system',
+  'scheduled_node',
+  'once',
+  'boot',
+  'grpc',
+  'subscription',
+  'system',
+  'script',
+  'legacy_import',
+] as const;
+
+export type ExecutionOrigin = (typeof EXECUTION_ORIGINS)[number];
+
+export type ExecutionOwner = 'legacy' | 'runtime';
+
+export const RUN_CANCELLATION_REASONS = [
+  'user',
+  'policy',
+  'shutdown',
+  'reconcile',
+  'timeout',
+] as const;
+
+export type RunCancellationReason = (typeof RUN_CANCELLATION_REASONS)[number];
+
+export const RUN_EVENT_ACTOR_TYPES = [
+  'user',
+  'api_app',
+  'trigger',
+  'agent',
+  'mcp_client',
+  'worker',
+  'executor',
+  'system',
+  'legacy_shell',
+  'scheduler',
+  'reconciler',
+  'compatibility',
+] as const;
+
+export type RunEventActorType = (typeof RUN_EVENT_ACTOR_TYPES)[number];
+
+export interface RunRecord {
+  id: string;
+  projectId: string;
+  taskId: string;
+  taskRevision: string;
+  taskName?: string;
+  taskSnapshotRef?: string;
+  legacyCronId?: number;
+  parentRunId?: string;
+  retryOfRunId?: string;
+  triggerId?: string;
+  triggerType: string;
+  executionOrigin: ExecutionOrigin;
+  executionOwner: ExecutionOwner;
+  triggeredBy?: string;
+  requestId?: string;
+  scheduledForMs?: number;
+  status: RunStatus;
+  version: number;
+  eventSequence: number;
+  priority: number;
+  idempotencyKey?: string;
+  inputRef?: string;
+  outputRef?: string;
+  createdAtMs: number;
+  queuedAtMs?: number;
+  startedAtMs?: number;
+  finishedAtMs?: number;
+  cancelRequestedAtMs?: number;
+  cancelReason?: RunCancellationReason;
+  errorCode?: string;
+  errorSummary?: string;
+}
+
+export interface RunAttemptRecord {
+  id: string;
+  runId: string;
+  stepRunId?: string;
+  attempt: number;
+  status: RunAttemptStatus;
+  executorType: string;
+  workerId?: string;
+  workerSessionId?: string;
+  workerGeneration?: number;
+  executorHandle?: string;
+  pid?: number;
+  logArtifactId?: string;
+  leaseToken?: string;
+  leaseTokenDigest?: string;
+  leaseGeneration?: number;
+  leaseVersion?: number;
+  leaseExpiresAtMs?: number;
+  offerId?: string;
+  deadlineAtMs?: number;
+  callbackTokenHash?: string;
+  callbackSequence: number;
+  createdAtMs: number;
+  startedAtMs?: number;
+  finishedAtMs?: number;
+  exitCode?: number;
+  errorCode?: string;
+  errorSummary?: string;
+}
+
+export interface RunEventRecord {
+  id: string;
+  runId: string;
+  sequence: number;
+  type: string;
+  dedupeKey?: string;
+  actorType: RunEventActorType;
+  actorId?: string;
+  attemptId?: string;
+  stepRunId?: string;
+  payload: Readonly<Record<string, unknown>>;
+  createdAtMs: number;
+}

packages/ql3-runtime-core/src/run/runDispatchLease.ts

@@ -0,0 +1,240 @@
+import { createHash } from 'node:crypto';
+import {
+  assertWorkerId,
+  assertWorkerSessionId,
+} from '../worker/workerSession';
+
+export const RUN_DISPATCH_LEASE_STATUSES = [
+  'leased',
+  'released',
+  'completed',
+] as const;
+export type RunDispatchLeaseStatus =
+  (typeof RUN_DISPATCH_LEASE_STATUSES)[number];
+
+export const RUN_DISPATCH_RELEASE_REASONS = [
+  'declined',
+  'shutdown',
+  'start_failed',
+  'capacity_changed',
+  'lease_expired',
+] as const;
+export type RunDispatchReleaseReason =
+  (typeof RUN_DISPATCH_RELEASE_REASONS)[number];
+
+export const MIN_RUN_DISPATCH_LEASE_DURATION_MS = 5_000;
+export const MAX_RUN_DISPATCH_LEASE_DURATION_MS = 10 * 60_000;
+
+export interface RunDispatchLeaseRecord {
+  readonly attemptId: string;
+  readonly runId: string;
+  readonly status: RunDispatchLeaseStatus;
+  readonly version: number;
+  readonly leaseGeneration: number;
+  readonly workerId: string;
+  readonly workerSessionId: string;
+  readonly workerGeneration: number;
+  /** The database never persists or returns the bearer lease token. */
+  readonly leaseTokenDigest: string;
+  readonly acquiredAtMs: number;
+  readonly renewedAtMs: number;
+  readonly expiresAtMs: number;
+  readonly releasedAtMs?: number;
+  readonly releaseReason?: RunDispatchReleaseReason;
+  readonly completedAtMs?: number;
+  readonly updatedAtMs: number;
+}
+
+export class InvalidRunDispatchLeaseValueError extends TypeError {
+  constructor(message: string) {
+    super(`Run dispatch lease value is invalid: ${message}`);
+    this.name = 'InvalidRunDispatchLeaseValueError';
+  }
+}
+
+export type RunDispatchLeaseFenceReason =
+  | 'missing'
+  | 'run_mismatch'
+  | 'not_leased'
+  | 'worker_mismatch'
+  | 'worker_session_mismatch'
+  | 'worker_generation_mismatch'
+  | 'lease_generation_mismatch'
+  | 'lease_token_mismatch'
+  | 'version_mismatch'
+  | 'lease_expired'
+  | 'worker_unavailable';
+
+export class RunDispatchLeaseFenceRejectedError extends Error {
+  constructor(
+    readonly attemptId: string,
+    readonly reason: RunDispatchLeaseFenceReason,
+  ) {
+    super(`Run dispatch lease for Attempt ${attemptId} was fenced: ${reason}`);
+    this.name = 'RunDispatchLeaseFenceRejectedError';
+  }
+}
+
+function invalid(message: string): never {
+  throw new InvalidRunDispatchLeaseValueError(message);
+}
+
+export function assertRunDispatchId(name: string, value: string): void {
+  if (
+    typeof value !== 'string' ||
+    value.length < 1 ||
+    value.length > 128 ||
+    /[\u0000-\u001f\u007f]/.test(value)
+  ) {
+    invalid(`${name} is invalid`);
+  }
+}
+
+export function assertRunDispatchLeaseToken(value: string): void {
+  if (
+    typeof value !== 'string' ||
+    value.length < 32 ||
+    value.length > 128 ||
+    !/^[A-Za-z0-9_-]+$/.test(value)
+  ) {
+    invalid('leaseToken is invalid');
+  }
+}
+
+export function digestRunDispatchLeaseToken(value: string): string {
+  assertRunDispatchLeaseToken(value);
+  return createHash('sha256').update(value, 'utf8').digest('hex');
+}
+
+export function assertRunDispatchLeaseDuration(value: number): void {
+  if (
+    !Number.isSafeInteger(value) ||
+    value < MIN_RUN_DISPATCH_LEASE_DURATION_MS ||
+    value > MAX_RUN_DISPATCH_LEASE_DURATION_MS
+  ) {
+    invalid(
+      `leaseDurationMs must be between ${MIN_RUN_DISPATCH_LEASE_DURATION_MS} and ${MAX_RUN_DISPATCH_LEASE_DURATION_MS}`,
+    );
+  }
+}
+
+export function assertRunDispatchLeaseFence(value: {
+  workerId: string;
+  workerSessionId: string;
+  workerGeneration: number;
+  leaseGeneration: number;
+  leaseToken: string;
+  expectedVersion: number;
+}): void {
+  assertWorkerId(value.workerId);
+  assertWorkerSessionId(value.workerSessionId);
+  for (const [name, number, minimum] of [
+    ['workerGeneration', value.workerGeneration, 1],
+    ['leaseGeneration', value.leaseGeneration, 1],
+    ['expectedVersion', value.expectedVersion, 0],
+  ] as const) {
+    if (!Number.isSafeInteger(number) || number < minimum) invalid(`${name} is invalid`);
+  }
+  assertRunDispatchLeaseToken(value.leaseToken);
+}
+
+export function assertRunDispatchLeaseRecord(
+  value: RunDispatchLeaseRecord,
+): void {
+  assertRunDispatchId('attemptId', value.attemptId);
+  assertRunDispatchId('runId', value.runId);
+  if (!RUN_DISPATCH_LEASE_STATUSES.includes(value.status)) invalid('status is invalid');
+  assertWorkerId(value.workerId);
+  assertWorkerSessionId(value.workerSessionId);
+  if (!/^[0-9a-f]{64}$/.test(value.leaseTokenDigest)) {
+    invalid('leaseTokenDigest is invalid');
+  }
+  for (const [name, number, minimum] of [
+    ['version', value.version, 0],
+    ['leaseGeneration', value.leaseGeneration, 1],
+    ['workerGeneration', value.workerGeneration, 1],
+    ['acquiredAtMs', value.acquiredAtMs, 0],
+    ['renewedAtMs', value.renewedAtMs, 0],
+    ['expiresAtMs', value.expiresAtMs, 0],
+    ['updatedAtMs', value.updatedAtMs, 0],
+  ] as const) {
+    if (!Number.isSafeInteger(number) || number < minimum) invalid(`${name} is invalid`);
+  }
+  if (
+    value.renewedAtMs < value.acquiredAtMs ||
+    value.expiresAtMs <= value.renewedAtMs ||
+    value.updatedAtMs < value.acquiredAtMs
+  ) {
+    invalid('timestamps are inconsistent');
+  }
+  if (value.status === 'leased') {
+    if (
+      value.releasedAtMs !== undefined ||
+      value.releaseReason !== undefined ||
+      value.completedAtMs !== undefined
+    ) invalid('active lease has terminal metadata');
+    return;
+  }
+  if (value.status === 'released') {
+    if (
+      value.releasedAtMs === undefined ||
+      value.releaseReason === undefined ||
+      !RUN_DISPATCH_RELEASE_REASONS.includes(value.releaseReason) ||
+      value.completedAtMs !== undefined
+    ) invalid('released lease metadata is inconsistent');
+    return;
+  }
+  if (
+    value.completedAtMs === undefined ||
+    value.releasedAtMs !== undefined ||
+    value.releaseReason !== undefined
+  ) invalid('completed lease metadata is inconsistent');
+}
+
+export interface ClaimRunDispatchLeaseCommand {
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly workerId: string;
+  readonly workerSessionId: string;
+  readonly workerGeneration: number;
+  readonly leaseToken: string;
+  readonly leaseDurationMs: number;
+  readonly eventId: string;
+  readonly offerId: string;
+}
+
+export type ClaimRunDispatchLeaseResult =
+  | { readonly status: 'claimed'; readonly lease: RunDispatchLeaseRecord }
+  | { readonly status: 'idempotent' | 'leased'; readonly lease: RunDispatchLeaseRecord }
+  | { readonly status: 'not_eligible' | 'worker_unavailable' | 'capacity_exhausted' };
+
+export interface RenewRunDispatchLeaseCommand {
+  readonly attemptId: string;
+  readonly workerId: string;
+  readonly workerSessionId: string;
+  readonly workerGeneration: number;
+  readonly leaseGeneration: number;
+  readonly leaseToken: string;
+  readonly expectedVersion: number;
+  readonly leaseDurationMs: number;
+}
+
+export interface ReleaseRunDispatchLeaseCommand {
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly workerId: string;
+  readonly workerSessionId: string;
+  readonly workerGeneration: number;
+  readonly leaseGeneration: number;
+  readonly leaseToken: string;
+  readonly expectedVersion: number;
+  readonly reason: Exclude<RunDispatchReleaseReason, 'lease_expired'>;
+  readonly eventId: string;
+}
+
+export interface RunDispatchLeaseRepository {
+  findByAttemptId(attemptId: string): Promise<RunDispatchLeaseRecord | null>;
+  claim(command: ClaimRunDispatchLeaseCommand): Promise<ClaimRunDispatchLeaseResult>;
+  renew(command: RenewRunDispatchLeaseCommand): Promise<RunDispatchLeaseRecord>;
+  release(command: ReleaseRunDispatchLeaseCommand): Promise<RunDispatchLeaseRecord>;
+}

packages/ql3-runtime-core/src/run/runRepository.ts

@@ -0,0 +1,59 @@
+import type {
+  RunAttemptRecord,
+  RunAttemptStatus,
+  RunEventRecord,
+  RunRecord,
+} from './run';
+import type { RunRetryPolicyRecord } from './runRetryPolicy';
+
+export const MAX_RUN_EVENT_PAYLOAD_BYTES = 16 * 1024;
+export const MAX_RUN_EVENT_PAGE_SIZE = 500;
+export const MAX_CANCELLATION_RECOVERY_PAGE_SIZE = 500;
+
+export interface RunRepositoryReader {
+  findRunById(runId: string): Promise<RunRecord | null>;
+  findAttemptById(attemptId: string): Promise<RunAttemptRecord | null>;
+  findLatestAttemptByRunId(runId: string): Promise<RunAttemptRecord | null>;
+  findRetryPolicyByRunId(runId: string): Promise<RunRetryPolicyRecord | null>;
+  listEvents(
+    runId: string,
+    options?: { afterSequence?: number; limit?: number },
+  ): Promise<RunEventRecord[]>;
+  listCancellationRequested(options?: {
+    beforeMs?: number;
+    limit?: number;
+  }): Promise<RunRecord[]>;
+}
+
+export interface RunRepositoryTransaction extends RunRepositoryReader {
+  insertRun(run: RunRecord): Promise<void>;
+  insertAttempt(attempt: RunAttemptRecord): Promise<void>;
+  insertRetryPolicy(policy: RunRetryPolicyRecord): Promise<void>;
+  /**
+   * Replaces a Run only when its persisted version still equals
+   * `expectedVersion`. The supplied Run must carry `expectedVersion + 1`.
+   */
+  compareAndSetRun(run: RunRecord, expectedVersion: number): Promise<boolean>;
+  /**
+   * Replaces an Attempt only when both state and callback sequence still match.
+   * The Run aggregate version remains the primary serialization boundary.
+   */
+  compareAndSetAttempt(
+    attempt: RunAttemptRecord,
+    expected: {
+      status: RunAttemptStatus;
+      callbackSequence: number;
+    },
+  ): Promise<boolean>;
+  compareAndSetRetryPolicy(
+    policy: RunRetryPolicyRecord,
+    expectedVersion: number,
+  ): Promise<boolean>;
+  appendEvent(event: RunEventRecord): Promise<void>;
+}
+
+export interface RunRepository extends RunRepositoryReader {
+  transaction<T>(
+    work: (transaction: RunRepositoryTransaction) => Promise<T>,
+  ): Promise<T>;
+}

packages/ql3-runtime-core/src/run/runRepositoryContract.ts

@@ -0,0 +1,6 @@
+export * from './repositoryErrors';
+export * from './run';
+export * from './runRepository';
+export * from './projectRunList';
+export * from './runRetryPolicy';
+export * from './clusterRunLostRetry';

packages/ql3-runtime-core/src/run/runRetryPolicy.ts

@@ -0,0 +1,138 @@
+export const RUN_RETRY_SAFETIES = [
+  'unknown',
+  'idempotent',
+  'deduplicated',
+] as const;
+
+export type RunRetrySafety = (typeof RUN_RETRY_SAFETIES)[number];
+
+export const MAX_RUN_ATTEMPTS = 16;
+export const MAX_RUN_RETRY_BACKOFF_MS = 24 * 60 * 60 * 1000;
+
+export interface RunRetryPolicyDefinition {
+  maxAttempts: number;
+  retryOnLost: boolean;
+  safety: RunRetrySafety;
+  backoffBaseMs: number;
+  backoffMaxMs: number;
+}
+
+export interface RunRetryPolicyRecord extends RunRetryPolicyDefinition {
+  runId: string;
+  nextAttemptAtMs?: number;
+  version: number;
+  createdAtMs: number;
+  updatedAtMs: number;
+}
+
+export const NO_AUTOMATIC_RUN_RETRY: Readonly<RunRetryPolicyDefinition> = {
+  maxAttempts: 1,
+  retryOnLost: false,
+  safety: 'unknown',
+  backoffBaseMs: 0,
+  backoffMaxMs: 0,
+};
+
+export class InvalidRunRetryPolicyError extends TypeError {
+  readonly code = 'INVALID_RUN_RETRY_POLICY';
+
+  constructor(message: string) {
+    super(message);
+    this.name = 'InvalidRunRetryPolicyError';
+  }
+}
+
+function assertNonNegativeTime(name: string, value: number): void {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new InvalidRunRetryPolicyError(
+      `${name} must be a non-negative safe integer`,
+    );
+  }
+}
+
+export function assertRunRetryPolicyDefinition(
+  policy: RunRetryPolicyDefinition,
+): void {
+  if (
+    !Number.isSafeInteger(policy.maxAttempts) ||
+    policy.maxAttempts < 1 ||
+    policy.maxAttempts > MAX_RUN_ATTEMPTS
+  ) {
+    throw new InvalidRunRetryPolicyError(
+      `maxAttempts must be between 1 and ${MAX_RUN_ATTEMPTS}`,
+    );
+  }
+  if (typeof policy.retryOnLost !== 'boolean') {
+    throw new InvalidRunRetryPolicyError('retryOnLost must be boolean');
+  }
+  if (!RUN_RETRY_SAFETIES.includes(policy.safety)) {
+    throw new InvalidRunRetryPolicyError('safety is not supported');
+  }
+  assertNonNegativeTime('backoffBaseMs', policy.backoffBaseMs);
+  assertNonNegativeTime('backoffMaxMs', policy.backoffMaxMs);
+  if (
+    policy.backoffBaseMs > MAX_RUN_RETRY_BACKOFF_MS ||
+    policy.backoffMaxMs > MAX_RUN_RETRY_BACKOFF_MS
+  ) {
+    throw new InvalidRunRetryPolicyError(
+      `retry backoff cannot exceed ${MAX_RUN_RETRY_BACKOFF_MS}ms`,
+    );
+  }
+  if (policy.backoffMaxMs < policy.backoffBaseMs) {
+    throw new InvalidRunRetryPolicyError(
+      'backoffMaxMs cannot be smaller than backoffBaseMs',
+    );
+  }
+}
+
+export function assertAdmittedRunRetryPolicy(
+  policy: RunRetryPolicyDefinition,
+): void {
+  assertRunRetryPolicyDefinition(policy);
+  if (
+    policy.retryOnLost &&
+    policy.maxAttempts > 1 &&
+    policy.safety === 'unknown'
+  ) {
+    throw new InvalidRunRetryPolicyError(
+      'automatic lost retry requires idempotent or deduplicated safety',
+    );
+  }
+}
+
+export function assertRunRetryPolicyRecord(policy: RunRetryPolicyRecord): void {
+  assertRunRetryPolicyDefinition(policy);
+  if (!policy.runId) {
+    throw new InvalidRunRetryPolicyError('runId is required');
+  }
+  if (!Number.isSafeInteger(policy.version) || policy.version < 0) {
+    throw new InvalidRunRetryPolicyError(
+      'version must be a non-negative safe integer',
+    );
+  }
+  assertNonNegativeTime('createdAtMs', policy.createdAtMs);
+  assertNonNegativeTime('updatedAtMs', policy.updatedAtMs);
+  if (policy.updatedAtMs < policy.createdAtMs) {
+    throw new InvalidRunRetryPolicyError(
+      'updatedAtMs cannot be earlier than createdAtMs',
+    );
+  }
+  if (policy.nextAttemptAtMs !== undefined) {
+    assertNonNegativeTime('nextAttemptAtMs', policy.nextAttemptAtMs);
+  }
+}
+
+export function runRetryDelayMs(
+  policy: RunRetryPolicyDefinition,
+  lostAttempt: number,
+): number {
+  assertRunRetryPolicyDefinition(policy);
+  if (!Number.isSafeInteger(lostAttempt) || lostAttempt < 1) {
+    throw new InvalidRunRetryPolicyError(
+      'lostAttempt must be a positive safe integer',
+    );
+  }
+  if (policy.backoffBaseMs === 0) return 0;
+  const exponent = Math.min(lostAttempt - 1, MAX_RUN_ATTEMPTS - 1);
+  return Math.min(policy.backoffMaxMs, policy.backoffBaseMs * 2 ** exponent);
+}