feat(ql3): establish 3.0 incubation baseline

packages/ql3-local-execution/.gitignore

@@ -0,0 +1 @@
+dist

packages/ql3-local-execution/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@qinglong/local-execution",
+  "version": "3.0.0-alpha.0",
+  "private": true,
+  "description": "QingLong 3.0 local execution, control, recovery and dispatch runtime",
+  "license": "Apache-2.0",
+  "engines": {
+    "node": ">=24.18.0 <25"
+  },
+  "exports": {
+    "./execution": {
+      "types": "./dist/execution/index.d.ts",
+      "require": "./dist/execution/index.js",
+      "default": "./dist/execution/index.js"
+    },
+    "./control": {
+      "types": "./dist/control/index.d.ts",
+      "require": "./dist/control/index.js",
+      "default": "./dist/control/index.js"
+    },
+    "./recovery": {
+      "types": "./dist/recovery/index.d.ts",
+      "require": "./dist/recovery/index.js",
+      "default": "./dist/recovery/index.js"
+    },
+    "./dispatch": {
+      "types": "./dist/dispatch/index.d.ts",
+      "require": "./dist/dispatch/index.js",
+      "default": "./dist/dispatch/index.js"
+    },
+    "./scheduler": {
+      "types": "./dist/scheduler/index.d.ts",
+      "require": "./dist/scheduler/index.js",
+      "default": "./dist/scheduler/index.js"
+    }
+  },
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "check": "node ../../scripts/ql3-build-package-closure.cjs && tsc -p tsconfig.json --noEmit",
+    "test": "node ../../scripts/ql3-build-package-closure.cjs && node --test test/*.test.cjs"
+  },
+  "dependencies": {
+    "@qinglong/local-process": "workspace:*",
+    "@qinglong/runtime-core": "workspace:*",
+    "croner": "7.0.8"
+  },
+  "devDependencies": {
+    "@qinglong/local-sqlite": "workspace:*",
+    "@types/node": "24.13.3",
+    "typescript": "5.9.3"
+  }
+}

packages/ql3-local-execution/src/control/completion.ts

@@ -0,0 +1,479 @@
+import { createHash, randomUUID, timingSafeEqual } from 'crypto';
+import type { LocalCompletionReceiptJournal } from '@qinglong/runtime-core/local-completion-receipt-journal';
+import type {
+  RunAttemptRecord,
+  RunEventRecord,
+  RunRecord,
+  RunRepository,
+  RunRepositoryTransaction,
+} from '@qinglong/runtime-core/run-repository';
+import {
+  InvalidCompletionReceiptError,
+  type CompletionReceipt,
+  type CompletionReceiptStore,
+} from '@qinglong/local-process';
+import type { LocalWorkflowTaskExecutionRepository } from '../execution/workflowTaskExecution';
+
+export const MAX_LOCAL_COMPLETION_QUARANTINE_RETENTION_MS = 24 * 60 * 60_000;
+
+const ACTIVE_ATTEMPT_STATUSES = new Set(['claimed', 'starting', 'running']);
+const TERMINAL_ATTEMPT_STATUSES = new Set([
+  'succeeded',
+  'failed',
+  'cancelled',
+  'timed_out',
+  'lost',
+]);
+const TERMINAL_RUN_STATUSES = new Set([
+  'succeeded',
+  'failed',
+  'cancelled',
+  'timed_out',
+  'lost',
+]);
+
+type MaintainedCompletionReceiptStore = CompletionReceiptStore & {
+  quarantine?(attemptId: string): Promise<string | undefined>;
+  quarantineReference?(attemptId: string): string;
+};
+
+export type LocalCompletionDisposition =
+  | 'completed'
+  | 'already_terminal'
+  | 'missing'
+  | 'invalid'
+  | 'stale';
+
+export interface LocalCompletionReceiptProcessorOptions {
+  readonly clock?: { now(): number };
+  readonly createEventId?: () => string;
+  readonly journal?: Pick<
+    LocalCompletionReceiptJournal,
+    'markQuarantined' | 'resolve'
+  >;
+  readonly quarantineRetentionMs?: number;
+  readonly onDiagnostic?: (
+    record: Readonly<{
+      kind:
+        | 'receipt_cleanup_failed'
+        | 'receipt_quarantined'
+        | 'journal_cleanup_failed';
+      runId: string;
+      attemptId: string;
+    }>,
+  ) => void | Promise<void>;
+  readonly workflowTasks?: LocalWorkflowTaskExecutionRepository;
+}
+
+interface AggregateSnapshot {
+  readonly run: RunRecord;
+  readonly attempt: RunAttemptRecord;
+}
+
+interface TerminalMapping {
+  readonly status: 'succeeded' | 'failed' | 'cancelled' | 'timed_out';
+  readonly errorCode?: string;
+  readonly errorSummary?: string;
+}
+
+class LocalCompletionConcurrentWriteError extends Error {}
+
+function timestamp(value: number, field: string): number {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new RangeError(`${field} must be a non-negative safe integer`);
+  }
+  return value;
+}
+
+function atOrAfter(
+  now: number,
+  run: RunRecord,
+  attempt: RunAttemptRecord,
+): number {
+  return Math.max(
+    timestamp(now, 'Local completion observation'),
+    run.createdAtMs,
+    run.startedAtMs ?? 0,
+    attempt.createdAtMs,
+    attempt.startedAtMs ?? 0,
+  );
+}
+
+function reserveEvent(run: RunRecord): Readonly<{
+  run: RunRecord;
+  sequence: number;
+}> {
+  const version = run.version + 1;
+  const sequence = run.eventSequence + 1;
+  if (!Number.isSafeInteger(version) || !Number.isSafeInteger(sequence)) {
+    throw new RangeError('Local completion aggregate counter overflowed');
+  }
+  return Object.freeze({
+    run: { ...run, version, eventSequence: sequence },
+    sequence,
+  });
+}
+
+function mapping(run: RunRecord, receipt: CompletionReceipt): TerminalMapping {
+  if (run.cancelRequestedAtMs !== undefined) {
+    return run.cancelReason === 'timeout'
+      ? Object.freeze({
+          status: 'timed_out' as const,
+          errorCode: 'EXECUTION_TIMED_OUT',
+          errorSummary: 'Execution exceeded its configured timeout',
+        })
+      : Object.freeze({
+          status: 'cancelled' as const,
+          errorCode: 'EXECUTION_CANCELLED',
+          errorSummary: 'Execution was cancelled',
+        });
+  }
+  return receipt.exitCode === 0
+    ? Object.freeze({ status: 'succeeded' as const })
+    : Object.freeze({
+        status: 'failed' as const,
+        errorCode: 'EXECUTION_FAILED',
+        errorSummary: 'Execution completed without success',
+      });
+}
+
+function authenticate(
+  run: RunRecord,
+  attempt: RunAttemptRecord,
+  receipt: CompletionReceipt,
+  terminal: boolean,
+): void {
+  const allowedSequence = terminal
+    ? receipt.callbackSequence === attempt.callbackSequence ||
+      receipt.callbackSequence === attempt.callbackSequence + 1
+    : receipt.callbackSequence === attempt.callbackSequence + 1;
+  if (
+    run.executionOwner !== 'runtime' ||
+    receipt.runId !== run.id ||
+    receipt.attemptId !== attempt.id ||
+    !allowedSequence ||
+    receipt.startedAtMs < attempt.createdAtMs ||
+    (attempt.startedAtMs !== undefined &&
+      receipt.startedAtMs < attempt.startedAtMs) ||
+    !attempt.callbackTokenHash ||
+    !/^[a-f0-9]{64}$/.test(attempt.callbackTokenHash)
+  ) {
+    throw new InvalidCompletionReceiptError(
+      'Completion receipt does not match durable Run authority',
+    );
+  }
+  const expected = Buffer.from(attempt.callbackTokenHash, 'hex');
+  const actual = createHash('sha256').update(receipt.token, 'utf8').digest();
+  if (!timingSafeEqual(expected, actual)) {
+    throw new InvalidCompletionReceiptError(
+      'Completion receipt token does not match durable authority',
+    );
+  }
+}
+
+function event(
+  id: string,
+  runId: string,
+  attemptId: string,
+  sequence: number,
+  type: string,
+  dedupeKey: string,
+  payload: Readonly<Record<string, unknown>>,
+  atMs: number,
+): RunEventRecord {
+  return {
+    id,
+    runId,
+    attemptId,
+    sequence,
+    type,
+    dedupeKey,
+    actorType: 'executor',
+    actorId: 'completion-receipt',
+    payload,
+    createdAtMs: atMs,
+  };
+}
+
+export class LocalCompletionReceiptProcessor {
+  private readonly clock: { now(): number };
+  private readonly createEventId: () => string;
+  private readonly journal?: LocalCompletionReceiptProcessorOptions['journal'];
+  private readonly quarantineRetentionMs: number;
+  private readonly onDiagnostic?: LocalCompletionReceiptProcessorOptions['onDiagnostic'];
+  private readonly workflowTasks:
+    | LocalWorkflowTaskExecutionRepository
+    | undefined;
+
+  constructor(
+    private readonly repository: RunRepository,
+    private readonly receipts: MaintainedCompletionReceiptStore,
+    options: LocalCompletionReceiptProcessorOptions = {},
+  ) {
+    this.clock = options.clock ?? { now: Date.now };
+    this.createEventId = options.createEventId ?? randomUUID;
+    this.journal = options.journal;
+    this.quarantineRetentionMs = options.quarantineRetentionMs ?? 60 * 60_000;
+    if (
+      !Number.isSafeInteger(this.quarantineRetentionMs) ||
+      this.quarantineRetentionMs < 0 ||
+      this.quarantineRetentionMs > MAX_LOCAL_COMPLETION_QUARANTINE_RETENTION_MS
+    ) {
+      throw new RangeError('Local completion quarantine retention is invalid');
+    }
+    this.onDiagnostic = options.onDiagnostic;
+    this.workflowTasks = options.workflowTasks;
+    if (
+      this.workflowTasks !== undefined &&
+      typeof this.workflowTasks.complete !== 'function'
+    ) {
+      throw new TypeError(
+        'Local Workflow Task completion repository is invalid',
+      );
+    }
+  }
+
+  async process(attemptId: string): Promise<LocalCompletionDisposition> {
+    const snapshot = await this.load(attemptId);
+    if (!snapshot || snapshot.attempt.executorType !== 'local_process') {
+      return 'stale';
+    }
+    let receipt: CompletionReceipt | undefined;
+    try {
+      receipt = await this.receipts.read(attemptId);
+      if (!receipt) return 'missing';
+      authenticate(
+        snapshot.run,
+        snapshot.attempt,
+        receipt,
+        TERMINAL_RUN_STATUSES.has(snapshot.run.status) ||
+          TERMINAL_ATTEMPT_STATUSES.has(snapshot.attempt.status),
+      );
+    } catch (error) {
+      if (!(error instanceof InvalidCompletionReceiptError)) throw error;
+      await this.quarantine(snapshot);
+      return 'invalid';
+    }
+
+    if (
+      TERMINAL_RUN_STATUSES.has(snapshot.run.status) ||
+      TERMINAL_ATTEMPT_STATUSES.has(snapshot.attempt.status)
+    ) {
+      await this.cleanup(receipt);
+      return 'already_terminal';
+    }
+    if (!ACTIVE_ATTEMPT_STATUSES.has(snapshot.attempt.status)) return 'stale';
+
+    if (snapshot.attempt.stepRunId !== undefined) {
+      if (!this.workflowTasks) return 'stale';
+      const terminal = mapping(snapshot.run, receipt);
+      const disposition = await this.workflowTasks.complete({
+        run: snapshot.run,
+        attempt: snapshot.attempt,
+        callbackSequence: receipt.callbackSequence,
+        startedAtMs: receipt.startedAtMs,
+        finishedAtMs: Math.max(
+          atOrAfter(this.clock.now(), snapshot.run, snapshot.attempt),
+          receipt.finishedAtMs,
+        ),
+        exitCode: receipt.exitCode,
+        terminalStatus: terminal.status,
+        ...(terminal.errorCode === undefined
+          ? {}
+          : {
+              errorCode: terminal.errorCode,
+              errorSummary: terminal.errorSummary,
+            }),
+        attemptEventId: this.createEventId(),
+        syntheticStartMutationId: this.createEventId(),
+        terminalStepMutationId: this.createEventId(),
+      });
+      if (
+        disposition === 'completed' ||
+        disposition === 'already_terminal'
+      ) {
+        await this.cleanup(receipt);
+      }
+      return disposition;
+    }
+
+    const completed = await this.repository.transaction(async (transaction) => {
+      const run = await transaction.findRunById(snapshot.run.id);
+      const attempt = await transaction.findAttemptById(snapshot.attempt.id);
+      if (
+        !run ||
+        !attempt ||
+        run.version !== snapshot.run.version ||
+        run.status !== snapshot.run.status ||
+        attempt.status !== snapshot.attempt.status ||
+        attempt.callbackSequence !== snapshot.attempt.callbackSequence ||
+        attempt.runId !== run.id
+      ) {
+        return false;
+      }
+      authenticate(run, attempt, receipt, false);
+      const terminal = mapping(run, receipt);
+      const atMs = Math.max(
+        atOrAfter(this.clock.now(), run, attempt),
+        receipt.finishedAtMs,
+      );
+      const attemptReserved = reserveEvent(run);
+      const nextAttempt: RunAttemptRecord = {
+        ...attempt,
+        status: terminal.status,
+        callbackSequence: receipt.callbackSequence,
+        finishedAtMs: atMs,
+        exitCode: receipt.exitCode,
+        ...(terminal.errorCode === undefined
+          ? {}
+          : {
+              errorCode: terminal.errorCode,
+              errorSummary: terminal.errorSummary,
+            }),
+      };
+      if (
+        !(await transaction.compareAndSetRun(attemptReserved.run, run.version))
+      ) {
+        throw new LocalCompletionConcurrentWriteError();
+      }
+      if (
+        !(await transaction.compareAndSetAttempt(nextAttempt, {
+          status: attempt.status,
+          callbackSequence: attempt.callbackSequence,
+        }))
+      ) {
+        throw new LocalCompletionConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          run.id,
+          attempt.id,
+          attemptReserved.sequence,
+          `attempt.${terminal.status}`,
+          `local-completion:${attempt.id}:${receipt.callbackSequence}:attempt`,
+          Object.freeze({
+            attempt_id: attempt.id,
+            from_status: attempt.status,
+            to_status: terminal.status,
+            callback_sequence: receipt.callbackSequence,
+            exit_code: receipt.exitCode,
+            version: attemptReserved.run.version,
+          }),
+          atMs,
+        ),
+      );
+      const runReserved = reserveEvent(attemptReserved.run);
+      const nextRun: RunRecord = {
+        ...runReserved.run,
+        status: terminal.status,
+        finishedAtMs: atMs,
+        ...(terminal.errorCode === undefined
+          ? {}
+          : {
+              errorCode: terminal.errorCode,
+              errorSummary: terminal.errorSummary,
+            }),
+      };
+      if (
+        !(await transaction.compareAndSetRun(
+          nextRun,
+          attemptReserved.run.version,
+        ))
+      ) {
+        throw new LocalCompletionConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          run.id,
+          attempt.id,
+          runReserved.sequence,
+          `run.${terminal.status}`,
+          `local-completion:${attempt.id}:${receipt.callbackSequence}:run`,
+          Object.freeze({
+            from_status: run.status,
+            to_status: terminal.status,
+            version: nextRun.version,
+          }),
+          atMs,
+        ),
+      );
+      return true;
+    });
+    if (!completed) return 'stale';
+    await this.cleanup(receipt);
+    return 'completed';
+  }
+
+  private load(attemptId: string): Promise<AggregateSnapshot | null> {
+    return this.repository.transaction(async (transaction) => {
+      const attempt = await transaction.findAttemptById(attemptId);
+      if (!attempt) return null;
+      const run = await transaction.findRunById(attempt.runId);
+      return run ? Object.freeze({ run, attempt }) : null;
+    });
+  }
+
+  private async quarantine(snapshot: AggregateSnapshot): Promise<void> {
+    const reference = this.receipts.quarantineReference?.(snapshot.attempt.id);
+    if (reference && this.journal) {
+      const updatedAtMs = atOrAfter(
+        this.clock.now(),
+        snapshot.run,
+        snapshot.attempt,
+      );
+      const purgeAfterMs = timestamp(
+        updatedAtMs + this.quarantineRetentionMs,
+        'Local completion quarantine expiry',
+      );
+      await this.journal.markQuarantined({
+        attemptId: snapshot.attempt.id,
+        quarantineRef: reference,
+        updatedAtMs,
+        purgeAfterMs,
+      });
+    }
+    const quarantined = await this.receipts.quarantine?.(snapshot.attempt.id);
+    if (quarantined) {
+      await this.diagnostic({
+        kind: 'receipt_quarantined',
+        runId: snapshot.run.id,
+        attemptId: snapshot.attempt.id,
+      });
+    }
+  }
+
+  private async cleanup(receipt: CompletionReceipt): Promise<void> {
+    try {
+      await this.receipts.remove(receipt.attemptId);
+    } catch {
+      await this.diagnostic({
+        kind: 'receipt_cleanup_failed',
+        runId: receipt.runId,
+        attemptId: receipt.attemptId,
+      });
+      return;
+    }
+    if (!this.journal) return;
+    try {
+      await this.journal.resolve(receipt.attemptId);
+    } catch {
+      await this.diagnostic({
+        kind: 'journal_cleanup_failed',
+        runId: receipt.runId,
+        attemptId: receipt.attemptId,
+      });
+    }
+  }
+
+  private async diagnostic(
+    record: Parameters<NonNullable<typeof this.onDiagnostic>>[0],
+  ): Promise<void> {
+    try {
+      await this.onDiagnostic?.(Object.freeze(record));
+    } catch {
+      // Diagnostic failure cannot replace the durable completion result.
+    }
+  }
+}

packages/ql3-local-execution/src/control/control.ts

@@ -0,0 +1,696 @@
+import { randomUUID } from 'crypto';
+import {
+  assertLocalExecutionControlLimit,
+  normalizeLocalActiveExecutionCandidate,
+  normalizeLocalExecutionControlCandidate,
+  type LocalActiveExecutionCandidate,
+  type LocalActiveExecutionCursor,
+  type LocalExecutionControlCandidate,
+  type LocalExecutionControlCursor,
+  type LocalExecutionControlSource,
+} from '@qinglong/runtime-core/local-execution-control';
+import type {
+  RunAttemptRecord,
+  RunCancellationReason,
+  RunEventRecord,
+  RunRecord,
+  RunRepository,
+} from '@qinglong/runtime-core/run-repository';
+import type {
+  LocalProcessController,
+  LocalProcessStopResult,
+} from '@qinglong/local-process';
+import type { LocalCompletionReceiptProcessor } from './completion';
+import type { LocalWorkflowTaskExecutionRepository } from '../execution/workflowTaskExecution';
+
+export const MAX_LOCAL_EXECUTION_CONTROL_PAGES = 16;
+
+const ACTIVE_ATTEMPT_STATUSES = new Set(['claimed', 'starting', 'running']);
+const ACTIVE_RUN_STATUSES = new Set(['dispatching', 'running']);
+const TERMINAL_RUN_STATUSES = new Set([
+  'succeeded',
+  'failed',
+  'cancelled',
+  'timed_out',
+  'lost',
+]);
+
+export type LocalExecutionControlDisposition =
+  | 'terminal'
+  | 'cancel_requested'
+  | 'stale'
+  | 'remaining';
+
+export interface LocalExecutionControlCoordinatorOptions {
+  readonly clock?: { now(): number };
+  readonly createEventId?: () => string;
+  readonly workflowTasks?: LocalWorkflowTaskExecutionRepository;
+}
+
+interface ActiveSnapshot {
+  readonly run: RunRecord;
+  readonly attempt: RunAttemptRecord;
+}
+
+class LocalExecutionControlConcurrentWriteError extends Error {}
+
+function timestamp(value: number, field: string): number {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new RangeError(`${field} must be a non-negative safe integer`);
+  }
+  return value;
+}
+
+function reserveEvent(run: RunRecord): Readonly<{
+  run: RunRecord;
+  sequence: number;
+}> {
+  const version = run.version + 1;
+  const sequence = run.eventSequence + 1;
+  if (!Number.isSafeInteger(version) || !Number.isSafeInteger(sequence)) {
+    throw new RangeError(
+      'Local execution control aggregate counter overflowed',
+    );
+  }
+  return Object.freeze({
+    run: { ...run, version, eventSequence: sequence },
+    sequence,
+  });
+}
+
+function atOrAfter(
+  now: number,
+  run: RunRecord,
+  attempt?: RunAttemptRecord,
+): number {
+  return Math.max(
+    timestamp(now, 'Local execution control observation'),
+    run.createdAtMs,
+    run.startedAtMs ?? 0,
+    attempt?.createdAtMs ?? 0,
+    attempt?.startedAtMs ?? 0,
+  );
+}
+
+function event(
+  id: string,
+  run: RunRecord,
+  attemptId: string,
+  sequence: number,
+  type: string,
+  dedupeKey: string,
+  payload: Readonly<Record<string, unknown>>,
+  actorId: string,
+  atMs: number,
+): RunEventRecord {
+  return {
+    id,
+    runId: run.id,
+    attemptId,
+    sequence,
+    type,
+    dedupeKey,
+    actorType: 'reconciler',
+    actorId,
+    payload,
+    createdAtMs: atMs,
+  };
+}
+
+function terminalFor(reason: RunCancellationReason): Readonly<{
+  status: 'cancelled' | 'timed_out';
+  errorCode: string;
+  errorSummary: string;
+}> {
+  return reason === 'timeout'
+    ? Object.freeze({
+        status: 'timed_out' as const,
+        errorCode: 'EXECUTION_TIMED_OUT',
+        errorSummary: 'Execution exceeded its configured timeout',
+      })
+    : Object.freeze({
+        status: 'cancelled' as const,
+        errorCode: 'EXECUTION_CANCELLED',
+        errorSummary: 'Execution was cancelled',
+      });
+}
+
+function conclusiveStop(result: LocalProcessStopResult): boolean {
+  return result.status === 'stopped' || result.status === 'already_exited';
+}
+
+export class LocalExecutionControlCoordinator {
+  private readonly clock: { now(): number };
+  private readonly createEventId: () => string;
+  private readonly workflowTasks:
+    | LocalWorkflowTaskExecutionRepository
+    | undefined;
+
+  constructor(
+    private readonly repository: RunRepository,
+    private readonly completions: Pick<
+      LocalCompletionReceiptProcessor,
+      'process'
+    >,
+    private readonly controller: Pick<LocalProcessController, 'stop'>,
+    options: LocalExecutionControlCoordinatorOptions = {},
+  ) {
+    this.clock = options.clock ?? { now: Date.now };
+    this.createEventId = options.createEventId ?? randomUUID;
+    this.workflowTasks = options.workflowTasks;
+    if (
+      this.workflowTasks !== undefined &&
+      (typeof this.workflowTasks.requestTimeout !== 'function' ||
+        typeof this.workflowTasks.recordControlTerminal !== 'function')
+    ) {
+      throw new TypeError(
+        'Local Workflow Task execution control repository is invalid',
+      );
+    }
+  }
+
+  async process(
+    value: LocalExecutionControlCandidate,
+  ): Promise<LocalExecutionControlDisposition> {
+    let candidate = normalizeLocalExecutionControlCandidate(value);
+    if (candidate.kind === 'deadline') {
+      const current = await this.loadActive(
+        candidate.runId,
+        candidate.attemptId,
+      );
+      if (!current) return 'stale';
+      const requested =
+        current.attempt.stepRunId === undefined
+          ? await this.requestCancellation(
+              candidate.runId,
+              candidate.attemptId,
+              'timeout',
+              candidate.dueAtMs,
+              true,
+            )
+          : this.workflowTasks === undefined
+            ? false
+            : (await this.workflowTasks.requestTimeout({
+                run: current.run,
+                attempt: current.attempt,
+                dueAtMs: candidate.dueAtMs,
+                eventId: this.createEventId(),
+              })) !== 'stale';
+      if (!requested) return 'stale';
+      candidate = Object.freeze({
+        kind: 'cancellation',
+        runId: candidate.runId,
+        attemptId: candidate.attemptId,
+        dueAtMs: candidate.dueAtMs,
+        cancelReason: 'timeout',
+      });
+    }
+
+    const completion = await this.completions.process(candidate.attemptId);
+    if (completion === 'completed' || completion === 'already_terminal') {
+      return 'terminal';
+    }
+    if (completion === 'invalid') return 'remaining';
+
+    const snapshot = await this.loadActive(
+      candidate.runId,
+      candidate.attemptId,
+    );
+    if (!snapshot) return 'stale';
+    const reason =
+      snapshot.attempt.stepRunId === undefined
+        ? snapshot.run.cancelReason
+        : snapshot.run.cancelRequestedAtMs === undefined
+          ? candidate.cancelReason === 'timeout'
+            ? 'timeout'
+            : undefined
+          : snapshot.run.cancelReason;
+    if (reason === undefined) return 'stale';
+    if (snapshot.attempt.status === 'claimed') {
+      return (await this.markTerminal(snapshot, reason))
+        ? 'terminal'
+        : 'remaining';
+    }
+    if (!snapshot.attempt.executorHandle) return 'remaining';
+
+    const stopped = await this.controller.stop(snapshot.attempt.executorHandle);
+    const lateCompletion = await this.completions.process(candidate.attemptId);
+    if (
+      lateCompletion === 'completed' ||
+      lateCompletion === 'already_terminal'
+    ) {
+      return 'terminal';
+    }
+    if (!conclusiveStop(stopped)) return 'remaining';
+    return (await this.markTerminal(snapshot, reason))
+      ? 'terminal'
+      : 'remaining';
+  }
+
+  async requestShutdown(
+    value: LocalActiveExecutionCandidate,
+    requestedAtMs: number,
+  ): Promise<LocalExecutionControlDisposition> {
+    const candidate = normalizeLocalActiveExecutionCandidate(value);
+    const current = await this.loadActive(candidate.runId, candidate.attemptId);
+    if (!current) return 'stale';
+    const reason = current.run.cancelReason ?? 'shutdown';
+    if (current.run.cancelRequestedAtMs === undefined) {
+      const requested = await this.requestCancellation(
+        candidate.runId,
+        candidate.attemptId,
+        reason,
+        requestedAtMs,
+        false,
+      );
+      if (!requested) return 'stale';
+    }
+    return this.process({
+      kind: 'cancellation',
+      runId: candidate.runId,
+      attemptId: candidate.attemptId,
+      dueAtMs: current.run.cancelRequestedAtMs ?? requestedAtMs,
+      cancelReason: reason,
+    });
+  }
+
+  private loadActive(
+    runId: string,
+    attemptId: string,
+  ): Promise<ActiveSnapshot | null> {
+    return this.repository.transaction(async (transaction) => {
+      const run = await transaction.findRunById(runId);
+      const attempt = await transaction.findAttemptById(attemptId);
+      if (
+        !run ||
+        !attempt ||
+        attempt.runId !== run.id ||
+        run.executionOwner !== 'runtime' ||
+        !ACTIVE_RUN_STATUSES.has(run.status) ||
+        !ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
+        attempt.executorType !== 'local_process'
+      ) {
+        return null;
+      }
+      if (attempt.stepRunId === undefined) {
+        const latest = await transaction.findLatestAttemptByRunId(run.id);
+        if (latest?.id !== attempt.id) return null;
+      }
+      return Object.freeze({ run, attempt });
+    });
+  }
+
+  private requestCancellation(
+    runId: string,
+    attemptId: string,
+    reason: RunCancellationReason,
+    requestedAtMs: number,
+    requireDeadline: boolean,
+  ): Promise<boolean> {
+    return this.repository.transaction(async (transaction) => {
+      const run = await transaction.findRunById(runId);
+      const attempt = await transaction.findAttemptById(attemptId);
+      if (
+        !run ||
+        !attempt ||
+        attempt.runId !== run.id ||
+        run.executionOwner !== 'runtime' ||
+        !ACTIVE_RUN_STATUSES.has(run.status) ||
+        !ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
+        attempt.executorType !== 'local_process' ||
+        (attempt.stepRunId === undefined &&
+          (await transaction.findLatestAttemptByRunId(run.id))?.id !==
+            attempt.id)
+      ) {
+        return false;
+      }
+      if (run.cancelRequestedAtMs !== undefined) {
+        return run.cancelReason === reason;
+      }
+      const observedAtMs = timestamp(
+        requestedAtMs,
+        'Cancellation request time',
+      );
+      if (
+        requireDeadline &&
+        (attempt.deadlineAtMs === undefined ||
+          attempt.deadlineAtMs > observedAtMs)
+      ) {
+        return false;
+      }
+      const atMs = atOrAfter(observedAtMs, run, attempt);
+      const reserved = reserveEvent(run);
+      const next: RunRecord = {
+        ...reserved.run,
+        cancelRequestedAtMs: atMs,
+        cancelReason: reason,
+      };
+      if (!(await transaction.compareAndSetRun(next, run.version))) {
+        throw new LocalExecutionControlConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          run,
+          attempt.id,
+          reserved.sequence,
+          'run.cancel_requested',
+          `local-control:${attempt.id}:${reason}:${run.version}`,
+          Object.freeze({
+            reason,
+            from_status: run.status,
+            version: next.version,
+          }),
+          requireDeadline ? 'local-deadline' : 'local-shutdown',
+          atMs,
+        ),
+      );
+      return true;
+    });
+  }
+
+  private markTerminal(
+    expected: ActiveSnapshot,
+    reason: RunCancellationReason,
+  ): Promise<boolean> {
+    if (expected.attempt.stepRunId !== undefined) {
+      if (!this.workflowTasks) return Promise.resolve(false);
+      const terminal = terminalFor(reason);
+      const atMs = atOrAfter(this.clock.now(), expected.run, expected.attempt);
+      return this.workflowTasks
+        .recordControlTerminal({
+          run: expected.run,
+          attempt: expected.attempt,
+          reason,
+          terminalStatus: terminal.status,
+          errorCode: terminal.errorCode,
+          errorSummary: terminal.errorSummary,
+          finishedAtMs: atMs,
+          attemptEventId: this.createEventId(),
+          stepMutationId: this.createEventId(),
+        })
+        .then(
+          (result) =>
+            result === 'terminal' || result === 'already_terminal',
+        );
+    }
+    return this.repository.transaction(async (transaction) => {
+      const run = await transaction.findRunById(expected.run.id);
+      const attempt = await transaction.findAttemptById(expected.attempt.id);
+      if (
+        !run ||
+        !attempt ||
+        run.version < expected.run.version ||
+        attempt.runId !== run.id ||
+        run.executionOwner !== 'runtime' ||
+        !ACTIVE_RUN_STATUSES.has(run.status) ||
+        !ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
+        attempt.callbackSequence !== expected.attempt.callbackSequence ||
+        attempt.executorHandle !== expected.attempt.executorHandle ||
+        attempt.pid !== expected.attempt.pid ||
+        run.cancelRequestedAtMs === undefined ||
+        run.cancelReason !== reason ||
+        (await transaction.findLatestAttemptByRunId(run.id))?.id !== attempt.id
+      ) {
+        return TERMINAL_RUN_STATUSES.has(run?.status ?? 'created');
+      }
+      const terminal = terminalFor(reason);
+      const atMs = atOrAfter(this.clock.now(), run, attempt);
+      const attemptReserved = reserveEvent(run);
+      const nextAttempt: RunAttemptRecord = {
+        ...attempt,
+        status: terminal.status,
+        finishedAtMs: atMs,
+        errorCode: terminal.errorCode,
+        errorSummary: terminal.errorSummary,
+      };
+      if (
+        !(await transaction.compareAndSetRun(attemptReserved.run, run.version))
+      ) {
+        throw new LocalExecutionControlConcurrentWriteError();
+      }
+      if (
+        !(await transaction.compareAndSetAttempt(nextAttempt, {
+          status: attempt.status,
+          callbackSequence: attempt.callbackSequence,
+        }))
+      ) {
+        throw new LocalExecutionControlConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          run,
+          attempt.id,
+          attemptReserved.sequence,
+          `attempt.${terminal.status}`,
+          `local-control:${attempt.id}:${attempt.callbackSequence}:attempt`,
+          Object.freeze({
+            attempt_id: attempt.id,
+            from_status: attempt.status,
+            to_status: terminal.status,
+            reason,
+            version: attemptReserved.run.version,
+          }),
+          'local-execution-control',
+          atMs,
+        ),
+      );
+      const runReserved = reserveEvent(attemptReserved.run);
+      const nextRun: RunRecord = {
+        ...runReserved.run,
+        status: terminal.status,
+        finishedAtMs: atMs,
+        errorCode: terminal.errorCode,
+        errorSummary: terminal.errorSummary,
+      };
+      if (
+        !(await transaction.compareAndSetRun(
+          nextRun,
+          attemptReserved.run.version,
+        ))
+      ) {
+        throw new LocalExecutionControlConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          run,
+          attempt.id,
+          runReserved.sequence,
+          `run.${terminal.status}`,
+          `local-control:${attempt.id}:${attempt.callbackSequence}:run`,
+          Object.freeze({
+            from_status: run.status,
+            to_status: terminal.status,
+            reason,
+            version: nextRun.version,
+          }),
+          'local-execution-control',
+          atMs,
+        ),
+      );
+      return true;
+    });
+  }
+}
+
+export interface LocalExecutionControlScanSummary {
+  readonly scanned: number;
+  readonly terminal: number;
+  readonly cancelRequested: number;
+  readonly stale: number;
+  readonly remaining: number;
+  readonly failed: number;
+  readonly truncated: boolean;
+  readonly nextCursor?: LocalExecutionControlCursor;
+}
+
+export interface LocalExecutionDrainSummary {
+  readonly scanned: number;
+  readonly terminal: number;
+  readonly remaining: number;
+  readonly failed: number;
+  readonly truncated: boolean;
+}
+
+function controlCursor(candidate: LocalExecutionControlCandidate) {
+  return Object.freeze({
+    dueAtMs: candidate.dueAtMs,
+    kind: candidate.kind,
+    attemptId: candidate.attemptId,
+  });
+}
+
+function controlAdvances(
+  previous: LocalExecutionControlCursor,
+  next: LocalExecutionControlCursor,
+): boolean {
+  return (
+    next.dueAtMs > previous.dueAtMs ||
+    (next.dueAtMs === previous.dueAtMs &&
+      (next.kind > previous.kind ||
+        (next.kind === previous.kind && next.attemptId > previous.attemptId)))
+  );
+}
+
+function activeCursor(candidate: LocalActiveExecutionCandidate) {
+  return Object.freeze({
+    attemptCreatedAtMs: candidate.attemptCreatedAtMs,
+    attemptId: candidate.attemptId,
+  });
+}
+
+function activeAdvances(
+  previous: LocalActiveExecutionCursor,
+  next: LocalActiveExecutionCursor,
+): boolean {
+  return (
+    next.attemptCreatedAtMs > previous.attemptCreatedAtMs ||
+    (next.attemptCreatedAtMs === previous.attemptCreatedAtMs &&
+      next.attemptId > previous.attemptId)
+  );
+}
+
+export class LocalExecutionControlScanner {
+  constructor(
+    private readonly source: LocalExecutionControlSource,
+    private readonly coordinator: Pick<
+      LocalExecutionControlCoordinator,
+      'process' | 'requestShutdown'
+    >,
+    private readonly clock: { now(): number } = { now: Date.now },
+  ) {}
+
+  async scan(options: {
+    readonly limit: number;
+    readonly cursor?: LocalExecutionControlCursor;
+  }): Promise<LocalExecutionControlScanSummary> {
+    assertLocalExecutionControlLimit(options.limit);
+    const observedAtMs = timestamp(
+      this.clock.now(),
+      'Local execution control scan time',
+    );
+    const page = await this.source.listLocalExecutionControlCandidates({
+      observedAtMs,
+      limit: options.limit,
+      ...(options.cursor === undefined ? {} : { after: options.cursor }),
+    });
+    if (page.candidates.length > options.limit) {
+      throw new RangeError('Local execution control source exceeded page size');
+    }
+    let previous = options.cursor;
+    let terminal = 0;
+    let cancelRequested = 0;
+    let stale = 0;
+    let remaining = 0;
+    let failed = 0;
+    for (const value of page.candidates) {
+      const candidate = normalizeLocalExecutionControlCandidate(value);
+      const cursor = controlCursor(candidate);
+      if (previous && !controlAdvances(previous, cursor)) {
+        throw new TypeError(
+          'Local execution control page is not strictly ordered',
+        );
+      }
+      previous = cursor;
+      try {
+        const disposition = await this.coordinator.process(candidate);
+        if (disposition === 'terminal') terminal += 1;
+        if (disposition === 'cancel_requested') cancelRequested += 1;
+        if (disposition === 'stale') stale += 1;
+        if (disposition === 'remaining') remaining += 1;
+      } catch {
+        failed += 1;
+      }
+    }
+    if (page.truncated && page.candidates.length === 0) {
+      throw new TypeError(
+        'Local execution control source returned an empty truncated page',
+      );
+    }
+    return Object.freeze({
+      scanned: page.candidates.length,
+      terminal,
+      cancelRequested,
+      stale,
+      remaining,
+      failed,
+      truncated: page.truncated,
+      ...(page.truncated && previous ? { nextCursor: previous } : {}),
+    });
+  }
+
+  async drain(options: {
+    readonly limit: number;
+    readonly maxPages: number;
+  }): Promise<LocalExecutionDrainSummary> {
+    assertLocalExecutionControlLimit(options.limit);
+    if (
+      !Number.isSafeInteger(options.maxPages) ||
+      options.maxPages < 1 ||
+      options.maxPages > MAX_LOCAL_EXECUTION_CONTROL_PAGES
+    ) {
+      throw new RangeError('Local execution drain page budget is invalid');
+    }
+    const requestedAtMs = timestamp(
+      this.clock.now(),
+      'Local execution drain time',
+    );
+    let cursor: LocalActiveExecutionCursor | undefined;
+    let scanned = 0;
+    let terminal = 0;
+    let remaining = 0;
+    let failed = 0;
+    let truncated = false;
+    for (let pageIndex = 0; pageIndex < options.maxPages; pageIndex += 1) {
+      const page = await this.source.listLocalActiveExecutions({
+        limit: options.limit,
+        ...(cursor === undefined ? {} : { after: cursor }),
+      });
+      if (page.candidates.length > options.limit) {
+        throw new RangeError(
+          'Local active execution source exceeded page size',
+        );
+      }
+      let previous = cursor;
+      for (const value of page.candidates) {
+        const candidate = normalizeLocalActiveExecutionCandidate(value);
+        const next = activeCursor(candidate);
+        if (previous && !activeAdvances(previous, next)) {
+          throw new TypeError(
+            'Local active execution page is not strictly ordered',
+          );
+        }
+        previous = next;
+        scanned += 1;
+        try {
+          const disposition = await this.coordinator.requestShutdown(
+            candidate,
+            requestedAtMs,
+          );
+          if (disposition === 'terminal' || disposition === 'stale')
+            terminal += 1;
+          else remaining += 1;
+        } catch {
+          failed += 1;
+        }
+      }
+      truncated = page.truncated;
+      if (!page.truncated) break;
+      if (!previous) {
+        throw new TypeError(
+          'Local active execution source returned an empty truncated page',
+        );
+      }
+      cursor = previous;
+    }
+    return Object.freeze({
+      scanned,
+      terminal,
+      remaining,
+      failed,
+      truncated,
+    });
+  }
+}

packages/ql3-local-execution/src/control/index.ts

@@ -0,0 +1,3 @@
+export * from './completion';
+export * from './control';
+export * from './lifecycle';

packages/ql3-local-execution/src/control/lifecycle.ts

@@ -0,0 +1,291 @@
+import type { LocalCompletionReceiptJournalCursor } from '@qinglong/runtime-core/local-completion-receipt-journal';
+import { assertLocalExecutionControlLimit } from '@qinglong/runtime-core/local-execution-control';
+import type {
+  LocalCompletionReceiptCleanupScanner,
+  LocalCompletionReceiptCleanupSummary,
+} from '@qinglong/local-process';
+import type { LocalCompletionReceiptProcessor } from './completion';
+import type {
+  LocalExecutionControlScanSummary,
+  LocalExecutionControlScanner,
+  LocalExecutionDrainSummary,
+} from './control';
+
+export const MAX_LOCAL_COMPLETION_NOTIFICATIONS = 64;
+
+export interface LocalExecutionControlLifecycleOptions {
+  readonly intervalMs: number;
+  readonly pageSize: number;
+  readonly cleanupIntervalMs: number;
+  readonly cleanupPageSize: number;
+  readonly stopTimeoutMs: number;
+  readonly maxDrainPages: number;
+  readonly maxNotifications?: number;
+  readonly clock?: { now(): number };
+  readonly onDiagnostic?: (
+    error: unknown,
+    summary?: LocalExecutionControlCycleSummary,
+  ) => void | Promise<void>;
+}
+
+export interface LocalExecutionControlCycleSummary {
+  readonly completions: number;
+  readonly completionFailures: number;
+  readonly control: LocalExecutionControlScanSummary;
+  readonly cleanup?: LocalCompletionReceiptCleanupSummary;
+}
+
+export interface LocalExecutionControlStopSummary {
+  readonly status: 'stopped' | 'timed_out';
+  readonly drain?: LocalExecutionDrainSummary;
+  readonly cleanup?: LocalCompletionReceiptCleanupSummary;
+}
+
+export class LocalExecutionControlLifecycle {
+  private readonly clock: { now(): number };
+  private readonly maxNotifications: number;
+  private timer: NodeJS.Timeout | undefined;
+  private inFlight: Promise<LocalExecutionControlCycleSummary> | undefined;
+  private stopPromise: Promise<LocalExecutionControlStopSummary> | undefined;
+  private running = false;
+  private stopping = false;
+  private kickQueued = false;
+  private controlCursor:
+    | NonNullable<LocalExecutionControlScanSummary['nextCursor']>
+    | undefined;
+  private cleanupCursor: LocalCompletionReceiptJournalCursor | undefined;
+  private lastCleanupAtMs: number | undefined;
+  private readonly pending = new Set<string>();
+
+  constructor(
+    private readonly completions: Pick<
+      LocalCompletionReceiptProcessor,
+      'process'
+    >,
+    private readonly control: Pick<
+      LocalExecutionControlScanner,
+      'scan' | 'drain'
+    >,
+    private readonly cleanup: Pick<
+      LocalCompletionReceiptCleanupScanner,
+      'scan'
+    >,
+    private readonly options: LocalExecutionControlLifecycleOptions,
+  ) {
+    if (
+      !Number.isSafeInteger(options.intervalMs) ||
+      options.intervalMs < 250 ||
+      options.intervalMs > 60 * 60_000
+    ) {
+      throw new RangeError('Local execution control interval is invalid');
+    }
+    assertLocalExecutionControlLimit(options.pageSize);
+    if (
+      !Number.isSafeInteger(options.cleanupIntervalMs) ||
+      options.cleanupIntervalMs < 1_000 ||
+      options.cleanupIntervalMs > 24 * 60 * 60_000
+    ) {
+      throw new RangeError('Local execution cleanup interval is invalid');
+    }
+    assertLocalExecutionControlLimit(options.cleanupPageSize);
+    if (
+      !Number.isSafeInteger(options.stopTimeoutMs) ||
+      options.stopTimeoutMs < 100 ||
+      options.stopTimeoutMs > 30_000
+    ) {
+      throw new RangeError('Local execution control stop timeout is invalid');
+    }
+    if (
+      !Number.isSafeInteger(options.maxDrainPages) ||
+      options.maxDrainPages < 1 ||
+      options.maxDrainPages > 16
+    ) {
+      throw new RangeError('Local execution control drain budget is invalid');
+    }
+    this.maxNotifications =
+      options.maxNotifications ?? MAX_LOCAL_COMPLETION_NOTIFICATIONS;
+    if (
+      !Number.isSafeInteger(this.maxNotifications) ||
+      this.maxNotifications < 1 ||
+      this.maxNotifications > MAX_LOCAL_COMPLETION_NOTIFICATIONS
+    ) {
+      throw new RangeError('Local completion notification budget is invalid');
+    }
+    this.clock = options.clock ?? { now: Date.now };
+  }
+
+  start(): void {
+    if (this.running || this.stopping) return;
+    this.running = true;
+    this.schedule();
+  }
+
+  notifyCompletion(attemptId: string): boolean {
+    if (
+      this.stopping ||
+      typeof attemptId !== 'string' ||
+      attemptId.length > 128
+    ) {
+      return false;
+    }
+    if (
+      !this.pending.has(attemptId) &&
+      this.pending.size >= this.maxNotifications
+    ) {
+      return false;
+    }
+    this.pending.add(attemptId);
+    this.kick();
+    return true;
+  }
+
+  runOnce(forceCleanup = false): Promise<LocalExecutionControlCycleSummary> {
+    if (this.inFlight) return this.inFlight;
+    const work = this.cycle(forceCleanup).finally(() => {
+      if (this.inFlight === work) this.inFlight = undefined;
+    });
+    this.inFlight = work;
+    return work;
+  }
+
+  stopAndDrain(): Promise<LocalExecutionControlStopSummary> {
+    if (this.stopPromise) return this.stopPromise;
+    this.stopping = true;
+    this.running = false;
+    if (this.timer) clearTimeout(this.timer);
+    this.timer = undefined;
+
+    const stopWork = (async () => {
+      await this.inFlight;
+      await this.processPending();
+      const drain = await this.control.drain({
+        limit: this.options.pageSize,
+        maxPages: this.options.maxDrainPages,
+      });
+      const cleanup = await this.cleanup.scan({
+        limit: this.options.cleanupPageSize,
+        ...(this.cleanupCursor === undefined
+          ? {}
+          : { cursor: this.cleanupCursor }),
+      });
+      this.cleanupCursor = cleanup.truncated ? cleanup.nextCursor : undefined;
+      return Object.freeze({
+        status:
+          drain.remaining === 0 && drain.failed === 0 && !drain.truncated
+            ? ('stopped' as const)
+            : ('timed_out' as const),
+        drain,
+        cleanup,
+      });
+    })();
+    this.stopPromise = Promise.race([
+      stopWork,
+      new Promise<LocalExecutionControlStopSummary>((resolve) => {
+        const timer = setTimeout(
+          () => resolve(Object.freeze({ status: 'timed_out' as const })),
+          this.options.stopTimeoutMs,
+        );
+        timer.unref?.();
+      }),
+    ]);
+    return this.stopPromise;
+  }
+
+  private schedule(): void {
+    if (!this.running || this.timer) return;
+    this.timer = setTimeout(() => {
+      this.timer = undefined;
+      if (!this.running) return;
+      void this.runOnce()
+        .then((summary) => this.diagnostic(undefined, summary))
+        .catch((error) => this.diagnostic(error))
+        .finally(() => this.schedule());
+    }, this.options.intervalMs);
+    this.timer.unref?.();
+  }
+
+  private kick(): void {
+    if (this.kickQueued || this.inFlight || this.stopping) return;
+    this.kickQueued = true;
+    queueMicrotask(() => {
+      this.kickQueued = false;
+      if (this.stopping || this.inFlight) return;
+      void this.runOnce()
+        .then((summary) => this.diagnostic(undefined, summary))
+        .catch((error) => this.diagnostic(error));
+    });
+  }
+
+  private async cycle(
+    forceCleanup: boolean,
+  ): Promise<LocalExecutionControlCycleSummary> {
+    const completion = await this.processPending();
+    const control = await this.control.scan({
+      limit: this.options.pageSize,
+      ...(this.controlCursor === undefined
+        ? {}
+        : { cursor: this.controlCursor }),
+    });
+    this.controlCursor = control.truncated ? control.nextCursor : undefined;
+    const now = this.clock.now();
+    if (!Number.isSafeInteger(now) || now < 0) {
+      throw new RangeError(
+        'Local execution control lifecycle clock is invalid',
+      );
+    }
+    let cleanup: LocalCompletionReceiptCleanupSummary | undefined;
+    if (
+      forceCleanup ||
+      this.lastCleanupAtMs === undefined ||
+      now - this.lastCleanupAtMs >= this.options.cleanupIntervalMs
+    ) {
+      cleanup = await this.cleanup.scan({
+        limit: this.options.cleanupPageSize,
+        ...(this.cleanupCursor === undefined
+          ? {}
+          : { cursor: this.cleanupCursor }),
+      });
+      this.cleanupCursor = cleanup.truncated ? cleanup.nextCursor : undefined;
+      this.lastCleanupAtMs = now;
+    }
+    if (this.pending.size > 0) this.kick();
+    return Object.freeze({
+      completions: completion.processed,
+      completionFailures: completion.failed,
+      control,
+      ...(cleanup === undefined ? {} : { cleanup }),
+    });
+  }
+
+  private async processPending(): Promise<
+    Readonly<{
+      processed: number;
+      failed: number;
+    }>
+  > {
+    const batch = [...this.pending].slice(0, this.maxNotifications);
+    for (const attemptId of batch) this.pending.delete(attemptId);
+    let processed = 0;
+    let failed = 0;
+    for (const attemptId of batch) {
+      try {
+        await this.completions.process(attemptId);
+        processed += 1;
+      } catch {
+        failed += 1;
+      }
+    }
+    return Object.freeze({ processed, failed });
+  }
+
+  private async diagnostic(
+    error: unknown,
+    summary?: LocalExecutionControlCycleSummary,
+  ): Promise<void> {
+    try {
+      await this.options.onDiagnostic?.(error, summary);
+    } catch {
+      // Diagnostics cannot own execution-control liveness.
+    }
+  }
+}

packages/ql3-local-execution/src/dispatch/artifact.ts

@@ -0,0 +1,185 @@
+import { createHash } from 'node:crypto';
+import { constants } from 'node:fs';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import type { LocalDispatchCandidate } from '@qinglong/runtime-core/local-dispatch';
+import { normalizeLocalDispatchCandidate } from '@qinglong/runtime-core/local-dispatch';
+
+export const MIN_LOCAL_ARTIFACT_MAXIMUM_BYTES = 64 * 1024;
+export const MAX_LOCAL_ARTIFACT_MAXIMUM_BYTES = 1024 * 1024 * 1024;
+export const MAX_LOCAL_ARTIFACT_MINIMUM_FREE_BYTES = 1024 ** 4;
+
+export interface LocalArtifactCapacityPolicy {
+  readonly maximumAttemptBytes: number;
+  readonly minimumFreeBytes: number;
+}
+
+export interface LocalArtifactCapacityProbe {
+  inspect(directory: string): Promise<bigint>;
+}
+
+export interface PreparedLocalArtifact {
+  readonly logArtifactId: string;
+  readonly output: Readonly<{
+    filePath: string;
+    maximumBytes: number;
+    logArtifactId: string;
+  }>;
+}
+
+export interface LocalArtifactAllocator {
+  prepare(candidate: LocalDispatchCandidate): Promise<PreparedLocalArtifact>;
+}
+
+export class LocalArtifactCapacityUnavailableError extends Error {
+  readonly code = 'LOCAL_ARTIFACT_CAPACITY_UNAVAILABLE';
+
+  constructor() {
+    super('Local Artifact capacity is unavailable');
+    this.name = 'LocalArtifactCapacityUnavailableError';
+  }
+}
+
+export class LocalArtifactIdentityConflictError extends Error {
+  readonly code = 'LOCAL_ARTIFACT_IDENTITY_CONFLICT';
+
+  constructor() {
+    super('Local Artifact identity is already occupied');
+    this.name = 'LocalArtifactIdentityConflictError';
+  }
+}
+
+class StatFsLocalArtifactCapacityProbe implements LocalArtifactCapacityProbe {
+  async inspect(directory: string): Promise<bigint> {
+    const stat = await fs.statfs(directory, { bigint: true });
+    return stat.bavail * stat.bsize;
+  }
+}
+
+function assertAbsoluteRoot(value: string): string {
+  if (
+    typeof value !== 'string' ||
+    !path.isAbsolute(value) ||
+    path.parse(value).root === value ||
+    value.includes('\0') ||
+    Buffer.byteLength(value, 'utf8') > 4096
+  ) {
+    throw new TypeError('Local Artifact root is invalid');
+  }
+  return path.resolve(value);
+}
+
+function normalizePolicy(
+  policy: LocalArtifactCapacityPolicy,
+): Readonly<LocalArtifactCapacityPolicy> {
+  if (!policy || typeof policy !== 'object' || Array.isArray(policy)) {
+    throw new TypeError('Local Artifact capacity policy is invalid');
+  }
+  if (
+    !Number.isSafeInteger(policy.maximumAttemptBytes) ||
+    policy.maximumAttemptBytes < MIN_LOCAL_ARTIFACT_MAXIMUM_BYTES ||
+    policy.maximumAttemptBytes > MAX_LOCAL_ARTIFACT_MAXIMUM_BYTES ||
+    !Number.isSafeInteger(policy.minimumFreeBytes) ||
+    policy.minimumFreeBytes < 0 ||
+    policy.minimumFreeBytes > MAX_LOCAL_ARTIFACT_MINIMUM_FREE_BYTES
+  ) {
+    throw new RangeError('Local Artifact capacity policy is out of range');
+  }
+  return Object.freeze({ ...policy });
+}
+
+async function ensurePrivateDirectory(directory: string): Promise<void> {
+  await fs.mkdir(directory, { recursive: true, mode: 0o700 });
+  const stat = await fs.lstat(directory);
+  if (!stat.isDirectory() || stat.isSymbolicLink()) {
+    throw new TypeError('Local Artifact directory is unsafe');
+  }
+  await fs.chmod(directory, 0o700);
+}
+
+export function localArtifactCapacityPolicyForProfile(
+  profile: 'edge' | 'standalone',
+): Readonly<LocalArtifactCapacityPolicy> {
+  if (profile === 'edge') {
+    return Object.freeze({
+      maximumAttemptBytes: 4 * 1024 * 1024,
+      minimumFreeBytes: 32 * 1024 * 1024,
+    });
+  }
+  if (profile === 'standalone') {
+    return Object.freeze({
+      maximumAttemptBytes: 64 * 1024 * 1024,
+      minimumFreeBytes: 256 * 1024 * 1024,
+    });
+  }
+  throw new TypeError('Local Artifact Profile is invalid');
+}
+
+export function localArtifactId(candidate: LocalDispatchCandidate): string {
+  const normalized = normalizeLocalDispatchCandidate(candidate);
+  return `local-${createHash('sha256')
+    .update(normalized.runId, 'utf8')
+    .update('\0', 'utf8')
+    .update(normalized.attemptId, 'utf8')
+    .digest('hex')
+    .slice(0, 30)}`;
+}
+
+export class LocalFileArtifactAllocator implements LocalArtifactAllocator {
+  private readonly root: string;
+  private readonly policy: Readonly<LocalArtifactCapacityPolicy>;
+
+  constructor(
+    root: string,
+    policy: LocalArtifactCapacityPolicy,
+    private readonly capacity: LocalArtifactCapacityProbe = new StatFsLocalArtifactCapacityProbe(),
+  ) {
+    this.root = assertAbsoluteRoot(root);
+    this.policy = normalizePolicy(policy);
+  }
+
+  async prepare(
+    candidate: LocalDispatchCandidate,
+  ): Promise<PreparedLocalArtifact> {
+    const normalized = normalizeLocalDispatchCandidate(candidate);
+    const logArtifactId = localArtifactId(normalized);
+    const shard = logArtifactId.slice(6, 8);
+    const directory = path.join(this.root, shard);
+    await ensurePrivateDirectory(this.root);
+    const availableBytes = await this.capacity.inspect(this.root);
+    const requiredBytes =
+      BigInt(this.policy.minimumFreeBytes) +
+      BigInt(this.policy.maximumAttemptBytes);
+    if (availableBytes < requiredBytes) {
+      throw new LocalArtifactCapacityUnavailableError();
+    }
+    await ensurePrivateDirectory(directory);
+    const filePath = path.join(directory, `${logArtifactId}.log`);
+    let file: fs.FileHandle | undefined;
+    try {
+      file = await fs.open(
+        filePath,
+        constants.O_WRONLY |
+          constants.O_CREAT |
+          constants.O_APPEND |
+          (constants.O_NOFOLLOW ?? 0),
+        0o600,
+      );
+      const stat = await file.stat();
+      if (!stat.isFile() || stat.size !== 0) {
+        throw new LocalArtifactIdentityConflictError();
+      }
+      await file.chmod(0o600);
+    } finally {
+      await file?.close().catch(() => undefined);
+    }
+    return Object.freeze({
+      logArtifactId,
+      output: Object.freeze({
+        filePath,
+        maximumBytes: this.policy.maximumAttemptBytes,
+        logArtifactId,
+      }),
+    });
+  }
+}

packages/ql3-local-execution/src/dispatch/dispatcher.ts

@@ -0,0 +1,227 @@
+import {
+  LOCAL_PROCESS_EXECUTOR_TYPE,
+  assertLocalDispatchPageSize,
+  normalizeLocalDispatchCandidate,
+  type LocalDispatchCandidate,
+  type LocalDispatchCandidateCursor,
+  type LocalDispatchCandidateSource,
+} from '@qinglong/runtime-core/local-dispatch';
+import {
+  LocalExecutionLaunchError,
+  LocalExecutionRejectedError,
+  type LocalExecutionStartCommand,
+  type LocalExecutionStartResult,
+} from '../execution/coordinator';
+import type { LocalDispatchPlanSource } from './materializer';
+
+export interface LocalDispatchActivator {
+  start(
+    command: LocalExecutionStartCommand,
+  ): Promise<LocalExecutionStartResult>;
+}
+
+export interface LocalRunDispatcherOptions {
+  readonly pageSize?: number;
+  readonly maxPages?: number;
+  readonly onCompletion?: (attemptId: string) => void | Promise<void>;
+}
+
+export interface LocalRunDispatcherStats {
+  readonly pages: number;
+  readonly candidatesScanned: number;
+  readonly plansUnavailable: number;
+  readonly activationRaces: number;
+}
+
+export type LocalRunDispatcherResult =
+  | Readonly<{
+      status: 'activated';
+      runId: string;
+      attemptId: string;
+      stats: LocalRunDispatcherStats;
+      truncated: boolean;
+    }>
+  | Readonly<{
+      status: 'activation_failed';
+      runId: string;
+      attemptId: string;
+      stats: LocalRunDispatcherStats;
+      truncated: boolean;
+    }>
+  | Readonly<{
+      status: 'idle';
+      reason:
+        | 'no_candidates'
+        | 'plans_unavailable'
+        | 'activation_raced'
+        | 'scan_budget_exhausted';
+      stats: LocalRunDispatcherStats;
+      truncated: boolean;
+    }>;
+
+function cursorOf(
+  candidate: LocalDispatchCandidate,
+): LocalDispatchCandidateCursor {
+  return Object.freeze({
+    priority: candidate.priority,
+    queuedAtMs: candidate.queuedAtMs,
+    attemptCreatedAtMs: candidate.attemptCreatedAtMs,
+    attemptId: candidate.attemptId,
+  });
+}
+
+function advances(
+  previous: LocalDispatchCandidateCursor,
+  next: LocalDispatchCandidateCursor,
+): boolean {
+  return (
+    next.priority < previous.priority ||
+    (next.priority === previous.priority &&
+      (next.queuedAtMs > previous.queuedAtMs ||
+        (next.queuedAtMs === previous.queuedAtMs &&
+          (next.attemptCreatedAtMs > previous.attemptCreatedAtMs ||
+            (next.attemptCreatedAtMs === previous.attemptCreatedAtMs &&
+              next.attemptId > previous.attemptId)))))
+  );
+}
+
+export class LocalRunDispatcher {
+  private readonly pageSize: number;
+  private readonly maxPages: number;
+  private readonly onCompletion?: LocalRunDispatcherOptions['onCompletion'];
+
+  constructor(
+    private readonly candidates: LocalDispatchCandidateSource,
+    private readonly plans: LocalDispatchPlanSource,
+    private readonly activator: LocalDispatchActivator,
+    options: LocalRunDispatcherOptions = {},
+  ) {
+    this.pageSize = options.pageSize ?? 8;
+    this.maxPages = options.maxPages ?? 1;
+    this.onCompletion = options.onCompletion;
+    assertLocalDispatchPageSize(this.pageSize);
+    if (
+      !Number.isSafeInteger(this.maxPages) ||
+      this.maxPages < 1 ||
+      this.maxPages > 16
+    ) {
+      throw new RangeError('Local dispatch maxPages must be between 1 and 16');
+    }
+  }
+
+  async dispatchOnce(): Promise<LocalRunDispatcherResult> {
+    const stats = {
+      pages: 0,
+      candidatesScanned: 0,
+      plansUnavailable: 0,
+      activationRaces: 0,
+    };
+    const seen = new Set<string>();
+    let after: LocalDispatchCandidateCursor | undefined;
+    let truncated = false;
+    for (let pageIndex = 0; pageIndex < this.maxPages; pageIndex += 1) {
+      const page = await this.candidates.listLocalDispatchCandidates({
+        limit: this.pageSize,
+        ...(after === undefined ? {} : { after }),
+      });
+      if (page.candidates.length > this.pageSize) {
+        throw new RangeError('Local dispatch source exceeded its page size');
+      }
+      stats.pages += 1;
+      truncated = page.truncated;
+      let previous = after;
+      for (const value of page.candidates) {
+        const candidate = normalizeLocalDispatchCandidate(value);
+        if (candidate.executorType !== LOCAL_PROCESS_EXECUTOR_TYPE) {
+          throw new TypeError(
+            'Local dispatch source returned another executor',
+          );
+        }
+        const cursor = cursorOf(candidate);
+        if (
+          seen.has(candidate.attemptId) ||
+          (previous !== undefined && !advances(previous, cursor))
+        ) {
+          throw new TypeError('Local dispatch page is not strictly ordered');
+        }
+        seen.add(candidate.attemptId);
+        previous = cursor;
+        stats.candidatesScanned += 1;
+        const plan = await this.plans.prepare(candidate);
+        if (!plan) {
+          stats.plansUnavailable += 1;
+          continue;
+        }
+        try {
+          const active = await this.activator.start(plan.command);
+          void active.handle.completion
+            .then(
+              () => this.onCompletion?.(active.attempt.id),
+              () => this.onCompletion?.(active.attempt.id),
+            )
+            .catch(() => undefined);
+          return Object.freeze({
+            status: 'activated' as const,
+            runId: active.run.id,
+            attemptId: active.attempt.id,
+            stats: Object.freeze({ ...stats }),
+            truncated,
+          });
+        } catch (error) {
+          if (error instanceof LocalExecutionRejectedError) {
+            if (
+              error.reason === 'aggregate_mismatch' ||
+              error.reason === 'executor_mismatch'
+            ) {
+              throw error;
+            }
+            stats.activationRaces += 1;
+            continue;
+          }
+          if (error instanceof LocalExecutionLaunchError) {
+            return Object.freeze({
+              status: 'activation_failed' as const,
+              runId: candidate.runId,
+              attemptId: candidate.attemptId,
+              stats: Object.freeze({ ...stats }),
+              truncated,
+            });
+          }
+          throw error;
+        }
+      }
+      if (!page.truncated) return this.idle(stats, false);
+      const last = page.candidates.at(-1);
+      if (!last) {
+        throw new TypeError(
+          'Local dispatch source reported an empty truncated page',
+        );
+      }
+      after = cursorOf(last);
+    }
+    return Object.freeze({
+      status: 'idle' as const,
+      reason: 'scan_budget_exhausted' as const,
+      stats: Object.freeze({ ...stats }),
+      truncated: true,
+    });
+  }
+
+  private idle(
+    stats: LocalRunDispatcherStats,
+    truncated: boolean,
+  ): LocalRunDispatcherResult {
+    const reason =
+      stats.candidatesScanned === 0
+        ? 'no_candidates'
+        : stats.plansUnavailable === stats.candidatesScanned
+        ? 'plans_unavailable'
+        : 'activation_raced';
+    return Object.freeze({
+      status: 'idle' as const,
+      reason,
+      stats: Object.freeze({ ...stats }),
+      truncated,
+    });
+  }
+}

packages/ql3-local-execution/src/dispatch/index.ts

@@ -0,0 +1,8 @@
+export * from './artifact';
+export * from './materializer';
+export * from './dispatcher';
+export type {
+  LocalDispatchDefinitionWriter,
+  LocalDispatchStore,
+  LocalSecretEnvironmentProvider,
+} from '@qinglong/runtime-core/local-dispatch';

packages/ql3-local-execution/src/dispatch/materializer.ts

@@ -0,0 +1,133 @@
+import type {
+  LocalDispatchCandidate,
+  LocalDispatchStore,
+  LocalSecretEnvironmentProvider,
+} from '@qinglong/runtime-core/local-dispatch';
+import {
+  MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES,
+  MAX_LOCAL_DISPATCH_SECRET_REFS,
+  normalizeLocalDispatchCandidate,
+  normalizeLocalExecutionContextRecipe,
+  normalizeLocalTaskExecutionRevision,
+} from '@qinglong/runtime-core/local-dispatch';
+import type { LocalExecutionStartCommand } from '../execution/coordinator';
+import type { LocalArtifactAllocator } from './artifact';
+
+export interface LocalDispatchPlan {
+  readonly command: LocalExecutionStartCommand;
+}
+
+export interface LocalDispatchPlanSource {
+  prepare(candidate: LocalDispatchCandidate): Promise<LocalDispatchPlan | null>;
+}
+
+function assertEnvironmentValue(value: unknown): asserts value is string {
+  if (
+    typeof value !== 'string' ||
+    value.includes('\0') ||
+    Buffer.byteLength(value, 'utf8') > 16 * 1024
+  ) {
+    throw new TypeError('Local dispatch environment value is invalid');
+  }
+}
+
+export class LocalDispatchPlanMaterializer implements LocalDispatchPlanSource {
+  constructor(
+    private readonly definitions: Pick<
+      LocalDispatchStore,
+      'resolveLocalTaskExecutionRevision' | 'resolveLocalExecutionContextRecipe'
+    >,
+    private readonly artifacts: LocalArtifactAllocator,
+    private readonly secrets?: LocalSecretEnvironmentProvider,
+  ) {}
+
+  async prepare(
+    candidate: LocalDispatchCandidate,
+  ): Promise<LocalDispatchPlan | null> {
+    const normalizedCandidate = normalizeLocalDispatchCandidate(candidate);
+    const revision = await this.definitions.resolveLocalTaskExecutionRevision({
+      projectId: normalizedCandidate.projectId,
+      taskId: normalizedCandidate.taskId,
+      taskRevision: normalizedCandidate.taskRevision,
+    });
+    if (!revision) return null;
+    const normalizedRevision = normalizeLocalTaskExecutionRevision(revision);
+    if (
+      normalizedRevision.projectId !== normalizedCandidate.projectId ||
+      normalizedRevision.taskId !== normalizedCandidate.taskId ||
+      normalizedRevision.taskRevision !== normalizedCandidate.taskRevision ||
+      normalizedRevision.executorType !== normalizedCandidate.executorType
+    ) {
+      throw new TypeError('Local Task revision does not match its candidate');
+    }
+    const recipe = await this.definitions.resolveLocalExecutionContextRecipe(
+      normalizedRevision.contextRef,
+    );
+    if (!recipe) return null;
+    const normalizedRecipe = normalizeLocalExecutionContextRecipe(recipe);
+    if (normalizedRecipe.contextRef !== normalizedRevision.contextRef) {
+      throw new TypeError('Local context recipe does not match its revision');
+    }
+    const secretRefs = Object.freeze([
+      ...new Set(
+        normalizedRecipe.environment.flatMap((binding) =>
+          binding.kind === 'secret' ? [binding.secretRef] : [],
+        ),
+      ),
+    ]);
+    if (secretRefs.length > MAX_LOCAL_DISPATCH_SECRET_REFS) {
+      throw new RangeError('Local dispatch Secret reference budget exceeded');
+    }
+    let secretValues: readonly string[] = [];
+    if (secretRefs.length > 0) {
+      if (!this.secrets) return null;
+      const resolved = await this.secrets.resolveLocalSecretEnvironment({
+        candidate: normalizedCandidate,
+        secretRefs,
+      });
+      if (!resolved) return null;
+      if (resolved.length !== secretRefs.length) {
+        throw new TypeError('Local Secret provider returned an invalid result');
+      }
+      secretValues = resolved;
+    }
+    const secretByRef = new Map(
+      secretRefs.map((secretRef, index) => [secretRef, secretValues[index]]),
+    );
+    const environment: Record<string, string> = Object.create(null);
+    let environmentBytes = 0;
+    for (const binding of normalizedRecipe.environment) {
+      const value =
+        binding.kind === 'public'
+          ? binding.value
+          : secretByRef.get(binding.secretRef);
+      assertEnvironmentValue(value);
+      environmentBytes +=
+        Buffer.byteLength(binding.name, 'utf8') +
+        Buffer.byteLength(value, 'utf8');
+      if (environmentBytes > MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES) {
+        throw new RangeError('Local dispatch environment byte budget exceeded');
+      }
+      environment[binding.name] = value;
+    }
+    const artifact = await this.artifacts.prepare(normalizedCandidate);
+    return Object.freeze({
+      command: Object.freeze({
+        runId: normalizedCandidate.runId,
+        attemptId: normalizedCandidate.attemptId,
+        ...(normalizedCandidate.stepRunId === undefined
+          ? {}
+          : { stepRunId: normalizedCandidate.stepRunId }),
+        command: normalizedRevision.command,
+        environment: Object.freeze(environment),
+        ...(normalizedRevision.workingDirectory === undefined
+          ? {}
+          : { workingDirectory: normalizedRevision.workingDirectory }),
+        ...(normalizedRevision.timeoutMs === undefined
+          ? {}
+          : { timeoutMs: normalizedRevision.timeoutMs }),
+        output: artifact.output,
+      }),
+    });
+  }
+}

packages/ql3-local-execution/src/execution/coordinator.ts

@@ -0,0 +1,951 @@
+import { createHash, randomBytes, randomUUID } from 'node:crypto';
+import path from 'node:path';
+import { parseLocalProcessDurableHandle } from '@qinglong/local-process';
+import type {
+  LocalProcessCommand,
+  LocalProcessLaunchHandle,
+  LocalProcessLaunchRequest,
+  LocalProcessOutputPlan,
+  LocalProcessStopResult,
+} from '@qinglong/local-process';
+import type {
+  RunAttemptRecord,
+  RunEventRecord,
+  RunRecord,
+  RunRepository,
+  RunRepositoryTransaction,
+} from '@qinglong/runtime-core/run-repository';
+import type { LocalWorkflowTaskExecutionRepository } from './workflowTaskExecution';
+
+export const LOCAL_PROCESS_EXECUTOR_TYPE = 'local_process';
+export const MAX_LOCAL_EXECUTION_TIMEOUT_MS = 365 * 24 * 60 * 60_000;
+
+export interface LocalExecutionStartCommand {
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly stepRunId?: string;
+  readonly command: LocalProcessCommand;
+  readonly environment?: Readonly<Record<string, string>>;
+  readonly workingDirectory?: string;
+  readonly output?: LocalProcessOutputPlan;
+  readonly timeoutMs?: number;
+}
+
+export interface LocalExecutionStartResult {
+  readonly run: RunRecord;
+  readonly attempt: RunAttemptRecord;
+  readonly handle: LocalProcessLaunchHandle;
+}
+
+export interface LocalExecutionLauncher {
+  start(request: LocalProcessLaunchRequest): Promise<LocalProcessLaunchHandle>;
+}
+
+export interface LocalExecutionController {
+  stop(durableHandle: string): Promise<LocalProcessStopResult>;
+}
+
+export interface LocalExecutionCoordinatorOptions {
+  readonly clock?: { now(): number };
+  readonly createEventId?: () => string;
+  readonly createCallbackToken?: () => string;
+  readonly workflowTasks?: LocalWorkflowTaskExecutionRepository;
+}
+
+interface AggregateSnapshot {
+  readonly run: RunRecord;
+  readonly attempt: RunAttemptRecord;
+}
+
+interface PreparedAggregate extends AggregateSnapshot {
+  readonly callbackSequence: number;
+  readonly callbackTokenHash: string;
+}
+
+interface EventDraft {
+  readonly sequence: number;
+  readonly type: string;
+  readonly payload: Readonly<Record<string, unknown>>;
+}
+
+export class LocalExecutionRejectedError extends Error {
+  readonly code = 'LOCAL_EXECUTION_REJECTED';
+
+  constructor(readonly reason: string) {
+    super(`Local execution rejected: ${reason}`);
+    this.name = 'LocalExecutionRejectedError';
+  }
+}
+
+export class LocalExecutionConcurrentWriteError extends Error {
+  readonly code = 'LOCAL_EXECUTION_CONCURRENT_WRITE';
+
+  constructor() {
+    super('Local execution aggregate changed concurrently');
+    this.name = 'LocalExecutionConcurrentWriteError';
+  }
+}
+
+export class LocalExecutionLaunchError extends Error {
+  readonly code = 'LOCAL_EXECUTION_LAUNCH_FAILED';
+
+  constructor(
+    message: string,
+    readonly snapshot: AggregateSnapshot,
+    readonly cause?: unknown,
+  ) {
+    super(message);
+    this.name = 'LocalExecutionLaunchError';
+  }
+}
+
+export class LocalExecutionOwnershipPersistenceError extends Error {
+  readonly code = 'LOCAL_EXECUTION_OWNERSHIP_PERSISTENCE_FAILED';
+
+  constructor(
+    readonly compensation: LocalProcessStopResult,
+    readonly snapshot: AggregateSnapshot,
+    readonly cause?: unknown,
+  ) {
+    super('Local execution could not persist durable process ownership');
+    this.name = 'LocalExecutionOwnershipPersistenceError';
+  }
+}
+
+function assertSafeTimestamp(value: number, field: string): void {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new TypeError(`${field} must be a non-negative safe integer`);
+  }
+}
+
+function assertStartCommand(command: LocalExecutionStartCommand): void {
+  if (!command || typeof command !== 'object' || Array.isArray(command)) {
+    throw new TypeError('Local execution start command is invalid');
+  }
+  for (const [field, value] of [
+    ['runId', command.runId],
+    ['attemptId', command.attemptId],
+  ] as const) {
+    if (
+      typeof value !== 'string' ||
+      value.length < 1 ||
+      value.length > 128 ||
+      /[\0\r\n]/.test(value)
+    ) {
+      throw new TypeError(`Local execution ${field} is invalid`);
+    }
+  }
+  if (
+    command.stepRunId !== undefined &&
+    (typeof command.stepRunId !== 'string' ||
+      command.stepRunId.length < 1 ||
+      command.stepRunId.length > 128 ||
+      /[\0\r\n]/.test(command.stepRunId))
+  ) {
+    throw new TypeError('Local execution stepRunId is invalid');
+  }
+  if (
+    command.timeoutMs !== undefined &&
+    (!Number.isSafeInteger(command.timeoutMs) ||
+      command.timeoutMs < 1 ||
+      command.timeoutMs > MAX_LOCAL_EXECUTION_TIMEOUT_MS)
+  ) {
+    throw new RangeError('Local execution timeout is invalid');
+  }
+  if (command.output !== undefined) {
+    const output = command.output;
+    if (
+      !output ||
+      typeof output !== 'object' ||
+      Array.isArray(output) ||
+      !path.isAbsolute(output.filePath) ||
+      path.parse(output.filePath).root === output.filePath ||
+      output.filePath.includes('\0') ||
+      Buffer.byteLength(output.filePath, 'utf8') > 4096 ||
+      !/^local-[0-9a-f]{30}$/.test(output.logArtifactId) ||
+      path.basename(output.filePath) !== `${output.logArtifactId}.log` ||
+      !Number.isSafeInteger(output.maximumBytes) ||
+      output.maximumBytes < 64 * 1024 ||
+      output.maximumBytes > 1024 * 1024 * 1024
+    ) {
+      throw new TypeError('Local execution output plan is invalid');
+    }
+  }
+}
+
+function reserveEvent(run: RunRecord): Readonly<{
+  run: RunRecord;
+  sequence: number;
+}> {
+  const version = run.version + 1;
+  const sequence = run.eventSequence + 1;
+  if (
+    !Number.isSafeInteger(version) ||
+    version < 1 ||
+    !Number.isSafeInteger(sequence) ||
+    sequence < 1
+  ) {
+    throw new TypeError('Local execution version or sequence overflowed');
+  }
+  return Object.freeze({
+    run: { ...run, version, eventSequence: sequence },
+    sequence,
+  });
+}
+
+function event(
+  id: string,
+  runId: string,
+  attemptId: string,
+  draft: EventDraft,
+  dedupeKey: string,
+  actorType: RunEventRecord['actorType'],
+  actorId: string,
+  atMs: number,
+): RunEventRecord {
+  if (
+    typeof id !== 'string' ||
+    id.length < 1 ||
+    id.length > 128 ||
+    /[\0\r\n]/.test(id)
+  ) {
+    throw new TypeError('Local execution event id is invalid');
+  }
+  return {
+    id,
+    runId,
+    attemptId,
+    sequence: draft.sequence,
+    type: draft.type,
+    dedupeKey,
+    actorType,
+    actorId,
+    payload: draft.payload,
+    createdAtMs: atMs,
+  };
+}
+
+function atOrAfter(
+  now: number,
+  run: RunRecord,
+  attempt: RunAttemptRecord,
+  additional?: number,
+): number {
+  assertSafeTimestamp(now, 'Local execution clock');
+  if (additional !== undefined) {
+    assertSafeTimestamp(additional, 'Local execution additional timestamp');
+  }
+  return Math.max(
+    now,
+    run.createdAtMs,
+    run.startedAtMs ?? 0,
+    attempt.createdAtMs,
+    attempt.startedAtMs ?? 0,
+    additional ?? 0,
+  );
+}
+
+function assertClaimedAggregate(
+  run: RunRecord | null,
+  attempt: RunAttemptRecord | null,
+  latest: RunAttemptRecord | null,
+): asserts run is RunRecord & object {
+  if (!run) throw new LocalExecutionRejectedError('run_not_found');
+  if (!attempt) throw new LocalExecutionRejectedError('attempt_not_found');
+  if (
+    attempt.runId !== run.id ||
+    latest?.runId !== run.id ||
+    latest.id !== attempt.id ||
+    run.executionOwner !== 'runtime'
+  ) {
+    throw new LocalExecutionRejectedError('aggregate_mismatch');
+  }
+  if (run.status !== 'queued') {
+    throw new LocalExecutionRejectedError('run_not_queued');
+  }
+  if (attempt.status !== 'claimed') {
+    throw new LocalExecutionRejectedError('attempt_not_claimed');
+  }
+  if (attempt.executorType !== LOCAL_PROCESS_EXECUTOR_TYPE) {
+    throw new LocalExecutionRejectedError('executor_mismatch');
+  }
+  if (run.cancelRequestedAtMs !== undefined) {
+    throw new LocalExecutionRejectedError('cancellation_requested');
+  }
+  if (
+    attempt.callbackSequence !== 0 ||
+    attempt.callbackTokenHash !== undefined ||
+    attempt.workerId !== undefined ||
+    attempt.workerSessionId !== undefined ||
+    attempt.workerGeneration !== undefined ||
+    attempt.executorHandle !== undefined ||
+    attempt.pid !== undefined ||
+    attempt.logArtifactId !== undefined ||
+    attempt.leaseToken !== undefined ||
+    attempt.leaseTokenDigest !== undefined ||
+    attempt.leaseGeneration !== undefined ||
+    attempt.leaseVersion !== undefined ||
+    attempt.leaseExpiresAtMs !== undefined ||
+    attempt.offerId !== undefined ||
+    attempt.deadlineAtMs !== undefined ||
+    attempt.startedAtMs !== undefined ||
+    attempt.finishedAtMs !== undefined ||
+    attempt.exitCode !== undefined ||
+    attempt.errorCode !== undefined ||
+    attempt.errorSummary !== undefined
+  ) {
+    throw new LocalExecutionRejectedError('stale_execution_authority');
+  }
+}
+
+function assertPreparedAggregate(
+  current: AggregateSnapshot,
+  expected: PreparedAggregate,
+): void {
+  if (
+    current.run.id !== expected.run.id ||
+    current.run.version !== expected.run.version ||
+    current.run.status !== 'dispatching' ||
+    current.run.executionOwner !== 'runtime' ||
+    current.run.cancelRequestedAtMs !== undefined ||
+    current.attempt.id !== expected.attempt.id ||
+    current.attempt.runId !== current.run.id ||
+    current.attempt.status !== 'starting' ||
+    current.attempt.executorType !== LOCAL_PROCESS_EXECUTOR_TYPE ||
+    current.attempt.callbackSequence !== expected.attempt.callbackSequence ||
+    current.attempt.callbackTokenHash !== expected.callbackTokenHash ||
+    current.attempt.deadlineAtMs !== expected.attempt.deadlineAtMs ||
+    current.attempt.logArtifactId !== expected.attempt.logArtifactId ||
+    current.attempt.executorHandle !== undefined ||
+    current.attempt.pid !== undefined ||
+    current.attempt.startedAtMs !== undefined ||
+    current.attempt.finishedAtMs !== undefined
+  ) {
+    throw new LocalExecutionConcurrentWriteError();
+  }
+}
+
+async function loadExact(
+  transaction: RunRepositoryTransaction,
+  runId: string,
+  attemptId: string,
+): Promise<AggregateSnapshot> {
+  const [run, attempt, latest] = await Promise.all([
+    transaction.findRunById(runId),
+    transaction.findAttemptById(attemptId),
+    transaction.findLatestAttemptByRunId(runId),
+  ]);
+  if (!run || !attempt || latest?.id !== attempt.id) {
+    throw new LocalExecutionConcurrentWriteError();
+  }
+  return { run, attempt };
+}
+
+export class LocalExecutionCoordinator {
+  private readonly clock: { now(): number };
+  private readonly createEventId: () => string;
+  private readonly createCallbackToken: () => string;
+  private readonly workflowTasks:
+    | LocalWorkflowTaskExecutionRepository
+    | undefined;
+
+  constructor(
+    private readonly repository: RunRepository,
+    private readonly launcher: LocalExecutionLauncher,
+    private readonly controller: LocalExecutionController,
+    options: LocalExecutionCoordinatorOptions = {},
+  ) {
+    this.clock = options.clock ?? { now: Date.now };
+    this.createEventId = options.createEventId ?? randomUUID;
+    this.createCallbackToken =
+      options.createCallbackToken ??
+      (() => randomBytes(32).toString('base64url'));
+    this.workflowTasks = options.workflowTasks;
+    if (
+      this.workflowTasks !== undefined &&
+      (typeof this.workflowTasks.prepare !== 'function' ||
+        typeof this.workflowTasks.recordRunning !== 'function' ||
+        typeof this.workflowTasks.recordStartFailure !== 'function')
+    ) {
+      throw new TypeError(
+        'Local Workflow Task execution repository is invalid',
+      );
+    }
+  }
+
+  async start(
+    command: LocalExecutionStartCommand,
+  ): Promise<LocalExecutionStartResult> {
+    assertStartCommand(command);
+    const callbackToken = this.createCallbackToken();
+    if (!/^[A-Za-z0-9_-]{32,128}$/.test(callbackToken)) {
+      throw new TypeError('Local execution callback token factory is invalid');
+    }
+    const callbackTokenHash = createHash('sha256')
+      .update(callbackToken)
+      .digest('hex');
+    const prepared = await this.prepare(command, callbackTokenHash);
+
+    let handle: LocalProcessLaunchHandle;
+    try {
+      handle = await this.launcher.start({
+        runId: prepared.run.id,
+        attemptId: prepared.attempt.id,
+        callbackSequence: prepared.callbackSequence,
+        callbackToken,
+        command: command.command,
+        ...(command.environment === undefined
+          ? {}
+          : { environment: command.environment }),
+        ...(command.workingDirectory === undefined
+          ? {}
+          : { workingDirectory: command.workingDirectory }),
+        ...(command.output === undefined ? {} : { output: command.output }),
+      });
+    } catch (error) {
+      try {
+        const failed = await this.recordTerminal(
+          prepared,
+          'failed',
+          'EXECUTOR_START_FAILED',
+          'Local process failed before durable ownership was established',
+        );
+        throw new LocalExecutionLaunchError(
+          'Local process could not start',
+          failed,
+          error,
+        );
+      } catch (persistenceError) {
+        if (persistenceError instanceof LocalExecutionLaunchError) {
+          throw persistenceError;
+        }
+        throw new LocalExecutionLaunchError(
+          'Local process could not start and failure state could not be persisted',
+          prepared,
+          new AggregateError([error, persistenceError]),
+        );
+      }
+    }
+
+    try {
+      this.assertHandle(handle, prepared);
+      const running = await this.recordRunning(prepared, handle);
+      return Object.freeze({ ...running, handle });
+    } catch (error) {
+      void handle.completion.catch(() => undefined);
+      let compensation: LocalProcessStopResult;
+      try {
+        compensation = await this.controller.stop(handle.durableHandle);
+      } catch {
+        compensation = Object.freeze({
+          status: 'unknown' as const,
+          reason: 'signal_failed' as const,
+        });
+      }
+      let snapshot: AggregateSnapshot = prepared;
+      if (
+        compensation.status === 'stopped' ||
+        compensation.status === 'already_exited'
+      ) {
+        try {
+          snapshot = await this.recordTerminal(
+            prepared,
+            'lost',
+            'EXECUTION_ACTIVATION_PERSISTENCE_FAILED',
+            'Durable process ownership could not be persisted',
+          );
+        } catch {
+          snapshot = await this.loadLatest(prepared);
+        }
+      } else {
+        snapshot = await this.loadLatest(prepared);
+      }
+      throw new LocalExecutionOwnershipPersistenceError(
+        compensation,
+        snapshot,
+        error,
+      );
+    }
+  }
+
+  private async prepare(
+    command: LocalExecutionStartCommand,
+    callbackTokenHash: string,
+  ): Promise<PreparedAggregate> {
+    if (command.stepRunId !== undefined) {
+      if (!this.workflowTasks) {
+        throw new LocalExecutionRejectedError(
+          'workflow_task_authority_unavailable',
+        );
+      }
+      const observedAtMs = this.clock.now();
+      assertSafeTimestamp(observedAtMs, 'Local execution clock');
+      const deadlineAtMs =
+        command.timeoutMs === undefined
+          ? undefined
+          : observedAtMs + command.timeoutMs;
+      if (
+        deadlineAtMs !== undefined &&
+        (!Number.isSafeInteger(deadlineAtMs) ||
+          deadlineAtMs <= observedAtMs)
+      ) {
+        throw new RangeError('Local execution deadline overflowed');
+      }
+      const result = await this.workflowTasks.prepare({
+        runId: command.runId,
+        attemptId: command.attemptId,
+        stepRunId: command.stepRunId,
+        callbackTokenHash,
+        ...(deadlineAtMs === undefined ? {} : { deadlineAtMs }),
+        ...(command.output === undefined
+          ? {}
+          : { logArtifactId: command.output.logArtifactId }),
+        atMs: observedAtMs,
+        eventId: this.createEventId(),
+      });
+      if (result.status === 'rejected') {
+        throw new LocalExecutionRejectedError(result.reason);
+      }
+      const current = await this.loadWorkflowTaskSnapshot(
+        command.runId,
+        command.attemptId,
+        result.snapshot,
+      );
+      return Object.freeze({
+        ...current,
+        callbackSequence: current.attempt.callbackSequence + 1,
+        callbackTokenHash,
+      });
+    }
+    return this.repository.transaction(async (transaction) => {
+      const [run, attempt, latest] = await Promise.all([
+        transaction.findRunById(command.runId),
+        transaction.findAttemptById(command.attemptId),
+        transaction.findLatestAttemptByRunId(command.runId),
+      ]);
+      assertClaimedAggregate(run, attempt, latest);
+      if (!attempt) throw new LocalExecutionRejectedError('attempt_not_found');
+      const atMs = atOrAfter(this.clock.now(), run, attempt);
+      const deadlineAtMs =
+        command.timeoutMs === undefined ? undefined : atMs + command.timeoutMs;
+      if (
+        deadlineAtMs !== undefined &&
+        (!Number.isSafeInteger(deadlineAtMs) || deadlineAtMs <= atMs)
+      ) {
+        throw new RangeError('Local execution deadline overflowed');
+      }
+
+      const dispatching = reserveEvent(run);
+      const nextDispatching: RunRecord = {
+        ...dispatching.run,
+        status: 'dispatching',
+      };
+      if (!(await transaction.compareAndSetRun(nextDispatching, run.version))) {
+        throw new LocalExecutionConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          run.id,
+          attempt.id,
+          {
+            sequence: dispatching.sequence,
+            type: 'run.dispatching',
+            payload: Object.freeze({
+              from_status: run.status,
+              to_status: 'dispatching',
+              version: nextDispatching.version,
+            }),
+          },
+          `local-execution:run:${run.id}:${run.version}:dispatching`,
+          'scheduler',
+          'local-execution',
+          atMs,
+        ),
+      );
+
+      const starting = reserveEvent(nextDispatching);
+      const nextStartingRun = starting.run;
+      const nextAttempt: RunAttemptRecord = {
+        ...attempt,
+        status: 'starting',
+        callbackTokenHash,
+        ...(deadlineAtMs === undefined ? {} : { deadlineAtMs }),
+        ...(command.output === undefined
+          ? {}
+          : { logArtifactId: command.output.logArtifactId }),
+      };
+      if (
+        !(await transaction.compareAndSetRun(
+          nextStartingRun,
+          nextDispatching.version,
+        )) ||
+        !(await transaction.compareAndSetAttempt(nextAttempt, {
+          status: attempt.status,
+          callbackSequence: attempt.callbackSequence,
+        }))
+      ) {
+        throw new LocalExecutionConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          run.id,
+          attempt.id,
+          {
+            sequence: starting.sequence,
+            type: 'attempt.starting',
+            payload: Object.freeze({
+              attempt_id: attempt.id,
+              from_status: attempt.status,
+              to_status: 'starting',
+              version: nextStartingRun.version,
+              ...(deadlineAtMs === undefined
+                ? {}
+                : { deadline_at_ms: deadlineAtMs }),
+            }),
+          },
+          `local-execution:attempt:${attempt.id}:${attempt.callbackSequence}:starting`,
+          'worker',
+          LOCAL_PROCESS_EXECUTOR_TYPE,
+          atMs,
+        ),
+      );
+      const callbackSequence = attempt.callbackSequence + 1;
+      if (!Number.isSafeInteger(callbackSequence)) {
+        throw new TypeError('Local execution callback sequence overflowed');
+      }
+      return Object.freeze({
+        run: nextStartingRun,
+        attempt: nextAttempt,
+        callbackSequence,
+        callbackTokenHash,
+      });
+    });
+  }
+
+  private async recordRunning(
+    expected: PreparedAggregate,
+    handle: LocalProcessLaunchHandle,
+  ): Promise<AggregateSnapshot> {
+    if (expected.attempt.stepRunId !== undefined) {
+      if (!this.workflowTasks) {
+        throw new LocalExecutionConcurrentWriteError();
+      }
+      const atMs = atOrAfter(
+        handle.startedAtMs,
+        expected.run,
+        expected.attempt,
+      );
+      const result = await this.workflowTasks.recordRunning({
+        run: expected.run,
+        attempt: expected.attempt,
+        callbackTokenHash: expected.callbackTokenHash,
+        executorHandle: handle.durableHandle,
+        pid: handle.pid,
+        startedAtMs: atMs,
+        attemptEventId: this.createEventId(),
+        stepMutationId: this.createEventId(),
+      });
+      if (result.status === 'rejected') {
+        throw new LocalExecutionConcurrentWriteError();
+      }
+      return this.loadWorkflowTaskSnapshot(
+        expected.run.id,
+        expected.attempt.id,
+        result.snapshot,
+      );
+    }
+    return this.repository.transaction(async (transaction) => {
+      const current = await loadExact(
+        transaction,
+        expected.run.id,
+        expected.attempt.id,
+      );
+      assertPreparedAggregate(current, expected);
+      const atMs = atOrAfter(
+        handle.startedAtMs,
+        current.run,
+        current.attempt,
+      );
+      const attemptRunning = reserveEvent(current.run);
+      const nextAttempt: RunAttemptRecord = {
+        ...current.attempt,
+        status: 'running',
+        executorHandle: handle.durableHandle,
+        pid: handle.pid,
+        startedAtMs: atMs,
+      };
+      if (
+        !(await transaction.compareAndSetRun(
+          attemptRunning.run,
+          current.run.version,
+        )) ||
+        !(await transaction.compareAndSetAttempt(nextAttempt, {
+          status: current.attempt.status,
+          callbackSequence: current.attempt.callbackSequence,
+        }))
+      ) {
+        throw new LocalExecutionConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          current.run.id,
+          current.attempt.id,
+          {
+            sequence: attemptRunning.sequence,
+            type: 'attempt.running',
+            payload: Object.freeze({
+              attempt_id: current.attempt.id,
+              from_status: current.attempt.status,
+              to_status: 'running',
+              pid: handle.pid,
+              version: attemptRunning.run.version,
+            }),
+          },
+          `local-execution:attempt:${current.attempt.id}:${current.attempt.callbackSequence}:running`,
+          'executor',
+          LOCAL_PROCESS_EXECUTOR_TYPE,
+          atMs,
+        ),
+      );
+
+      const runRunning = reserveEvent(attemptRunning.run);
+      const nextRun: RunRecord = {
+        ...runRunning.run,
+        status: 'running',
+        startedAtMs: atMs,
+      };
+      if (
+        !(await transaction.compareAndSetRun(
+          nextRun,
+          attemptRunning.run.version,
+        ))
+      ) {
+        throw new LocalExecutionConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          current.run.id,
+          current.attempt.id,
+          {
+            sequence: runRunning.sequence,
+            type: 'run.running',
+            payload: Object.freeze({
+              from_status: current.run.status,
+              to_status: 'running',
+              version: nextRun.version,
+            }),
+          },
+          `local-execution:run:${current.run.id}:${current.run.version}:running`,
+          'executor',
+          LOCAL_PROCESS_EXECUTOR_TYPE,
+          atMs,
+        ),
+      );
+      return Object.freeze({ run: nextRun, attempt: nextAttempt });
+    });
+  }
+
+  private async recordTerminal(
+    expected: PreparedAggregate,
+    status: 'failed' | 'lost',
+    errorCode: string,
+    errorSummary: string,
+  ): Promise<AggregateSnapshot> {
+    if (expected.attempt.stepRunId !== undefined) {
+      if (!this.workflowTasks) {
+        throw new LocalExecutionConcurrentWriteError();
+      }
+      const atMs = atOrAfter(
+        this.clock.now(),
+        expected.run,
+        expected.attempt,
+      );
+      const result = await this.workflowTasks.recordStartFailure({
+        run: expected.run,
+        attempt: expected.attempt,
+        callbackTokenHash: expected.callbackTokenHash,
+        status: 'failed',
+        errorCode,
+        errorSummary,
+        finishedAtMs: atMs,
+        attemptEventId: this.createEventId(),
+        stepMutationId: this.createEventId(),
+      });
+      if (result.status === 'rejected') {
+        throw new LocalExecutionConcurrentWriteError();
+      }
+      return this.loadWorkflowTaskSnapshot(
+        expected.run.id,
+        expected.attempt.id,
+        result.snapshot,
+      );
+    }
+    return this.repository.transaction(async (transaction) => {
+      const current = await loadExact(
+        transaction,
+        expected.run.id,
+        expected.attempt.id,
+      );
+      assertPreparedAggregate(current, expected);
+      const atMs = atOrAfter(this.clock.now(), current.run, current.attempt);
+      const attemptTerminal = reserveEvent(current.run);
+      const nextAttempt: RunAttemptRecord = {
+        ...current.attempt,
+        status,
+        finishedAtMs: atMs,
+        errorCode,
+        errorSummary,
+      };
+      if (
+        !(await transaction.compareAndSetRun(
+          attemptTerminal.run,
+          current.run.version,
+        )) ||
+        !(await transaction.compareAndSetAttempt(nextAttempt, {
+          status: current.attempt.status,
+          callbackSequence: current.attempt.callbackSequence,
+        }))
+      ) {
+        throw new LocalExecutionConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          current.run.id,
+          current.attempt.id,
+          {
+            sequence: attemptTerminal.sequence,
+            type: `attempt.${status}`,
+            payload: Object.freeze({
+              attempt_id: current.attempt.id,
+              from_status: current.attempt.status,
+              to_status: status,
+              error_code: errorCode,
+              version: attemptTerminal.run.version,
+            }),
+          },
+          `local-execution:attempt:${current.attempt.id}:${current.attempt.callbackSequence}:${status}`,
+          status === 'lost' ? 'reconciler' : 'executor',
+          status === 'lost' ? 'local-execution' : LOCAL_PROCESS_EXECUTOR_TYPE,
+          atMs,
+        ),
+      );
+
+      const runTerminal = reserveEvent(attemptTerminal.run);
+      const nextRun: RunRecord = {
+        ...runTerminal.run,
+        status,
+        ...(status === 'failed' ? { finishedAtMs: atMs } : {}),
+        errorCode,
+        errorSummary,
+      };
+      if (
+        !(await transaction.compareAndSetRun(
+          nextRun,
+          attemptTerminal.run.version,
+        ))
+      ) {
+        throw new LocalExecutionConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          current.run.id,
+          current.attempt.id,
+          {
+            sequence: runTerminal.sequence,
+            type: `run.${status}`,
+            payload: Object.freeze({
+              from_status: current.run.status,
+              to_status: status,
+              error_code: errorCode,
+              version: nextRun.version,
+            }),
+          },
+          `local-execution:run:${current.run.id}:${current.run.version}:${status}`,
+          status === 'lost' ? 'reconciler' : 'executor',
+          status === 'lost' ? 'local-execution' : LOCAL_PROCESS_EXECUTOR_TYPE,
+          atMs,
+        ),
+      );
+      return Object.freeze({ run: nextRun, attempt: nextAttempt });
+    });
+  }
+
+  private assertHandle(
+    handle: LocalProcessLaunchHandle,
+    expected: PreparedAggregate,
+  ): void {
+    const parsed =
+      handle &&
+      typeof handle === 'object' &&
+      typeof handle.durableHandle === 'string'
+        ? parseLocalProcessDurableHandle(handle.durableHandle)
+        : null;
+    if (
+      !handle ||
+      typeof handle !== 'object' ||
+      !Number.isSafeInteger(handle.pid) ||
+      handle.pid < 1 ||
+      !Number.isSafeInteger(handle.startedAtMs) ||
+      handle.startedAtMs < 0 ||
+      typeof handle.handleId !== 'string' ||
+      handle.handleId.length < 1 ||
+      typeof handle.durableHandle !== 'string' ||
+      handle.durableHandle.length < 1 ||
+      typeof handle.completion?.then !== 'function' ||
+      !parsed ||
+      parsed.handleId !== handle.handleId ||
+      parsed.identity.pid !== handle.pid ||
+      expected.run.id !== expected.attempt.runId
+    ) {
+      throw new TypeError('Local process launcher returned an invalid handle');
+    }
+  }
+
+  private async loadLatest(
+    fallback: AggregateSnapshot,
+  ): Promise<AggregateSnapshot> {
+    const [run, attempt] = await Promise.all([
+      this.repository.findRunById(fallback.run.id),
+      this.repository.findAttemptById(fallback.attempt.id),
+    ]);
+    return Object.freeze({
+      run: run ?? fallback.run,
+      attempt: attempt ?? fallback.attempt,
+    });
+  }
+
+  private async loadWorkflowTaskSnapshot(
+    runId: string,
+    attemptId: string,
+    expected: Readonly<{
+      runVersion: number;
+      runEventSequence: number;
+      attemptStatus: RunAttemptRecord['status'];
+      callbackSequence: number;
+    }>,
+  ): Promise<AggregateSnapshot> {
+    const [run, attempt] = await Promise.all([
+      this.repository.findRunById(runId),
+      this.repository.findAttemptById(attemptId),
+    ]);
+    if (
+      !run ||
+      !attempt ||
+      attempt.runId !== run.id ||
+      attempt.stepRunId === undefined ||
+      run.version !== expected.runVersion ||
+      run.eventSequence !== expected.runEventSequence ||
+      attempt.status !== expected.attemptStatus ||
+      attempt.callbackSequence !== expected.callbackSequence
+    ) {
+      throw new LocalExecutionConcurrentWriteError();
+    }
+    return Object.freeze({ run, attempt });
+  }
+}

packages/ql3-local-execution/src/execution/index.ts

@@ -0,0 +1,2 @@
+export * from './coordinator';
+export * from './workflowTaskExecution';

packages/ql3-local-execution/src/execution/workflowTaskExecution.ts

@@ -0,0 +1,98 @@
+import type {
+  RunAttemptRecord,
+  RunRecord,
+} from '@qinglong/runtime-core/run-repository';
+
+export type LocalWorkflowTaskTerminalStatus =
+  | 'succeeded'
+  | 'failed'
+  | 'cancelled'
+  | 'timed_out';
+
+export interface LocalWorkflowTaskExecutionSnapshot {
+  readonly runVersion: number;
+  readonly runEventSequence: number;
+  readonly attemptStatus: RunAttemptRecord['status'];
+  readonly callbackSequence: number;
+}
+
+export type LocalWorkflowTaskExecutionMutationResult =
+  | Readonly<{
+      status: 'applied';
+      snapshot: Readonly<LocalWorkflowTaskExecutionSnapshot>;
+    }>
+  | Readonly<{
+      status: 'rejected';
+      reason:
+        | 'aggregate_mismatch'
+        | 'run_not_running'
+        | 'attempt_not_claimed'
+        | 'attempt_not_starting'
+        | 'cancellation_requested'
+        | 'stale_execution_authority';
+    }>;
+
+export interface LocalWorkflowTaskExecutionRepository {
+  prepare(command: Readonly<{
+    runId: string;
+    attemptId: string;
+    stepRunId: string;
+    callbackTokenHash: string;
+    deadlineAtMs?: number;
+    logArtifactId?: string;
+    atMs: number;
+    eventId: string;
+  }>): Promise<Readonly<LocalWorkflowTaskExecutionMutationResult>>;
+  recordRunning(command: Readonly<{
+    run: Readonly<RunRecord>;
+    attempt: Readonly<RunAttemptRecord>;
+    callbackTokenHash: string;
+    executorHandle: string;
+    pid: number;
+    startedAtMs: number;
+    attemptEventId: string;
+    stepMutationId: string;
+  }>): Promise<Readonly<LocalWorkflowTaskExecutionMutationResult>>;
+  recordStartFailure(command: Readonly<{
+    run: Readonly<RunRecord>;
+    attempt: Readonly<RunAttemptRecord>;
+    callbackTokenHash: string;
+    status: 'failed';
+    errorCode: string;
+    errorSummary: string;
+    finishedAtMs: number;
+    attemptEventId: string;
+    stepMutationId: string;
+  }>): Promise<Readonly<LocalWorkflowTaskExecutionMutationResult>>;
+  complete(command: Readonly<{
+    run: Readonly<RunRecord>;
+    attempt: Readonly<RunAttemptRecord>;
+    callbackSequence: number;
+    startedAtMs: number;
+    finishedAtMs: number;
+    exitCode: number;
+    terminalStatus: LocalWorkflowTaskTerminalStatus;
+    errorCode?: string;
+    errorSummary?: string;
+    attemptEventId: string;
+    syntheticStartMutationId: string;
+    terminalStepMutationId: string;
+  }>): Promise<'completed' | 'already_terminal' | 'stale'>;
+  requestTimeout(command: Readonly<{
+    run: Readonly<RunRecord>;
+    attempt: Readonly<RunAttemptRecord>;
+    dueAtMs: number;
+    eventId: string;
+  }>): Promise<'requested' | 'existing' | 'stale'>;
+  recordControlTerminal(command: Readonly<{
+    run: Readonly<RunRecord>;
+    attempt: Readonly<RunAttemptRecord>;
+    reason: 'user' | 'policy' | 'shutdown' | 'reconcile' | 'timeout';
+    terminalStatus: 'cancelled' | 'timed_out';
+    errorCode: string;
+    errorSummary: string;
+    finishedAtMs: number;
+    attemptEventId: string;
+    stepMutationId: string;
+  }>): Promise<'terminal' | 'already_terminal' | 'stale'>;
+}

packages/ql3-local-execution/src/recovery/coordinator.ts

@@ -0,0 +1,848 @@
+import { randomUUID } from 'crypto';
+import type {
+  LocalRunStartupRecoveryCandidate,
+  LocalRunStartupRecoveryPage,
+  LocalRunStartupRecoverySource,
+} from '@qinglong/runtime-core/local-startup-recovery';
+import type {
+  RunAttemptRecord,
+  RunEventRecord,
+  RunRecord,
+  RunRepository,
+  RunRepositoryTransaction,
+} from '@qinglong/runtime-core/run-repository';
+import type { LocalCompletionReceiptJournal } from '@qinglong/runtime-core/local-completion-receipt-journal';
+import { LocalCompletionReceiptProcessor } from '../control/completion';
+import {
+  type CompletionReceiptStore,
+  type LocalPersistedExecutionInspection,
+  type LocalPersistedExecutionInspector,
+} from '@qinglong/local-process';
+
+export const MAX_LOCAL_RUN_RECOVERY_ITEMS = 256;
+export const MAX_LOCAL_RUN_RECEIPT_GRACE_MS = 5_000;
+export const MAX_LOCAL_RUN_QUARANTINE_RETENTION_MS = 24 * 60 * 60_000;
+
+const ACTIVE_ATTEMPT_STATUSES = new Set(['claimed', 'starting', 'running']);
+const PAGE_KEYS = Object.freeze(['candidates', 'truncated']);
+const CANDIDATE_KEYS = Object.freeze([
+  'activeAttemptCount',
+  'runId',
+  'runStatus',
+]);
+
+export interface LocalRunStartupRecoverySummary {
+  readonly safe: boolean;
+  readonly scanned: number;
+  readonly recovered: number;
+  readonly remaining: number;
+  readonly failed: number;
+  readonly truncated: boolean;
+}
+
+export interface LocalRunStartupRecoveryCoordinatorOptions {
+  readonly receiptPublishGraceMs?: number;
+  readonly clock?: { now(): number };
+  readonly wait?: (delayMs: number) => Promise<void>;
+  readonly createEventId?: () => string;
+  readonly journal?: Pick<
+    LocalCompletionReceiptJournal,
+    'markQuarantined' | 'resolve'
+  >;
+  readonly quarantineRetentionMs?: number;
+  readonly completionProcessor?: Pick<
+    LocalCompletionReceiptProcessor,
+    'process'
+  >;
+  readonly onDiagnostic?: (
+    record: Readonly<{
+      kind:
+        | 'receipt_cleanup_failed'
+        | 'receipt_quarantined'
+        | 'journal_cleanup_failed';
+      runId: string;
+      attemptId: string;
+    }>,
+  ) => void | Promise<void>;
+}
+
+type CandidateDisposition =
+  | Readonly<{ status: 'recovered' }>
+  | Readonly<{ status: 'verified'; fingerprint: VerifiedFingerprint }>
+  | Readonly<{ status: 'remaining' }>
+  | Readonly<{ status: 'failed' }>;
+
+interface AggregateSnapshot {
+  readonly run: RunRecord;
+  readonly attempt: RunAttemptRecord | null;
+}
+
+interface VerifiedFingerprint {
+  readonly runId: string;
+  readonly runStatus: 'running';
+  readonly runVersion: number;
+  readonly attemptId: string;
+  readonly attemptStatus: 'running';
+  readonly callbackSequence: number;
+  readonly executorHandle: string;
+  readonly pid: number;
+}
+
+interface EventDraft {
+  readonly sequence: number;
+  readonly type: string;
+  readonly payload: Readonly<Record<string, unknown>>;
+}
+
+class LocalRunRecoveryConcurrentWriteError extends Error {}
+
+function exactKeys(
+  value: Readonly<Record<string, unknown>>,
+  expected: readonly string[],
+): boolean {
+  return (
+    JSON.stringify(Object.keys(value).sort()) ===
+    JSON.stringify([...expected].sort())
+  );
+}
+
+function assertSafeTimestamp(value: number, name: string): void {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new TypeError(`${name} must be a non-negative safe integer`);
+  }
+}
+
+function validatePage(value: unknown): LocalRunStartupRecoveryPage {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new TypeError('Local Run recovery page is invalid');
+  }
+  const page = value as unknown as Record<string, unknown>;
+  if (
+    !exactKeys(page, PAGE_KEYS) ||
+    !Array.isArray(page.candidates) ||
+    page.candidates.length > MAX_LOCAL_RUN_RECOVERY_ITEMS ||
+    typeof page.truncated !== 'boolean'
+  ) {
+    throw new TypeError('Local Run recovery page shape or bounds are invalid');
+  }
+  const candidates: LocalRunStartupRecoveryCandidate[] = [];
+  const runIds = new Set<string>();
+  for (const value of page.candidates) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+      throw new TypeError('Local Run recovery candidate is invalid');
+    }
+    const candidate = value as Record<string, unknown>;
+    if (
+      !exactKeys(candidate, CANDIDATE_KEYS) ||
+      typeof candidate.runId !== 'string' ||
+      candidate.runId.length < 1 ||
+      candidate.runId.length > 128 ||
+      /[\0\r\n]/.test(candidate.runId) ||
+      (candidate.runStatus !== 'dispatching' &&
+        candidate.runStatus !== 'running') ||
+      !Number.isSafeInteger(candidate.activeAttemptCount) ||
+      (candidate.activeAttemptCount as number) < 0 ||
+      (candidate.activeAttemptCount as number) > MAX_LOCAL_RUN_RECOVERY_ITEMS ||
+      runIds.has(candidate.runId)
+    ) {
+      throw new TypeError('Local Run recovery candidate fields are invalid');
+    }
+    runIds.add(candidate.runId);
+    candidates.push(
+      Object.freeze({
+        runId: candidate.runId,
+        runStatus: candidate.runStatus,
+        activeAttemptCount: candidate.activeAttemptCount as number,
+      }),
+    );
+  }
+  return Object.freeze({
+    candidates: Object.freeze(candidates),
+    truncated: page.truncated,
+  });
+}
+
+function reserveEvent(run: RunRecord): Readonly<{
+  run: RunRecord;
+  sequence: number;
+}> {
+  const version = run.version + 1;
+  const sequence = run.eventSequence + 1;
+  if (
+    !Number.isSafeInteger(version) ||
+    version < 1 ||
+    !Number.isSafeInteger(sequence) ||
+    sequence < 1
+  ) {
+    throw new TypeError('Local Run recovery version or sequence overflowed');
+  }
+  return Object.freeze({
+    run: { ...run, version, eventSequence: sequence },
+    sequence,
+  });
+}
+
+function event(
+  id: string,
+  runId: string,
+  attemptId: string | undefined,
+  draft: EventDraft,
+  dedupeKey: string,
+  actorType: RunEventRecord['actorType'],
+  actorId: string,
+  atMs: number,
+): RunEventRecord {
+  return {
+    id,
+    runId,
+    sequence: draft.sequence,
+    type: draft.type,
+    dedupeKey,
+    actorType,
+    actorId,
+    ...(attemptId === undefined ? {} : { attemptId }),
+    payload: draft.payload,
+    createdAtMs: atMs,
+  };
+}
+
+function atOrAfter(
+  now: number,
+  run: RunRecord,
+  attempt?: RunAttemptRecord,
+): number {
+  assertSafeTimestamp(now, 'Local Run recovery observation');
+  return Math.max(
+    now,
+    run.createdAtMs,
+    run.startedAtMs ?? 0,
+    attempt?.createdAtMs ?? 0,
+    attempt?.startedAtMs ?? 0,
+  );
+}
+
+function fingerprint(snapshot: AggregateSnapshot): VerifiedFingerprint {
+  const { run, attempt } = snapshot;
+  if (
+    run.status !== 'running' ||
+    !attempt ||
+    attempt.status !== 'running' ||
+    attempt.executorType !== 'local_process' ||
+    !attempt.executorHandle ||
+    !Number.isSafeInteger(attempt.pid) ||
+    (attempt.pid as number) < 1
+  ) {
+    throw new TypeError('Verified local Run snapshot is not running');
+  }
+  return Object.freeze({
+    runId: run.id,
+    runStatus: 'running',
+    runVersion: run.version,
+    attemptId: attempt.id,
+    attemptStatus: 'running',
+    callbackSequence: attempt.callbackSequence,
+    executorHandle: attempt.executorHandle,
+    pid: attempt.pid as number,
+  });
+}
+
+function sameFingerprint(
+  expected: VerifiedFingerprint,
+  snapshot: AggregateSnapshot,
+): boolean {
+  const attempt = snapshot.attempt;
+  return (
+    snapshot.run.id === expected.runId &&
+    snapshot.run.status === expected.runStatus &&
+    snapshot.run.version === expected.runVersion &&
+    attempt?.id === expected.attemptId &&
+    attempt.status === expected.attemptStatus &&
+    attempt.callbackSequence === expected.callbackSequence &&
+    attempt.executorHandle === expected.executorHandle &&
+    attempt.pid === expected.pid
+  );
+}
+
+/**
+ * One bounded startup coordinator. It never scans receipt directories, starts
+ * an Executor, or treats missing/invalid evidence as proof of process exit.
+ */
+export class LocalRunStartupRecoveryCoordinator {
+  private readonly clock: { now(): number };
+  private readonly wait: (delayMs: number) => Promise<void>;
+  private readonly receiptPublishGraceMs: number;
+  private readonly createEventId: () => string;
+  private readonly journal?: LocalRunStartupRecoveryCoordinatorOptions['journal'];
+  private readonly quarantineRetentionMs: number;
+  private readonly onDiagnostic?: LocalRunStartupRecoveryCoordinatorOptions['onDiagnostic'];
+  private readonly completionProcessor: Pick<
+    LocalCompletionReceiptProcessor,
+    'process'
+  >;
+
+  constructor(
+    private readonly repository: RunRepository,
+    private readonly source: LocalRunStartupRecoverySource,
+    receipts: CompletionReceiptStore,
+    private readonly inspector: LocalPersistedExecutionInspector,
+    options: LocalRunStartupRecoveryCoordinatorOptions = {},
+  ) {
+    this.clock = options.clock ?? { now: Date.now };
+    this.wait =
+      options.wait ??
+      ((delayMs) =>
+        new Promise((resolve) => {
+          setTimeout(resolve, delayMs);
+        }));
+    this.receiptPublishGraceMs = options.receiptPublishGraceMs ?? 0;
+    if (
+      !Number.isSafeInteger(this.receiptPublishGraceMs) ||
+      this.receiptPublishGraceMs < 0 ||
+      this.receiptPublishGraceMs > MAX_LOCAL_RUN_RECEIPT_GRACE_MS
+    ) {
+      throw new RangeError(
+        `receiptPublishGraceMs must be between 0 and ${MAX_LOCAL_RUN_RECEIPT_GRACE_MS}`,
+      );
+    }
+    if (inspector.executorType !== 'local_process') {
+      throw new TypeError('Local Run recovery inspector type is invalid');
+    }
+    this.createEventId = options.createEventId ?? randomUUID;
+    this.journal = options.journal;
+    this.quarantineRetentionMs = options.quarantineRetentionMs ?? 60 * 60_000;
+    if (
+      !Number.isSafeInteger(this.quarantineRetentionMs) ||
+      this.quarantineRetentionMs < 0 ||
+      this.quarantineRetentionMs > MAX_LOCAL_RUN_QUARANTINE_RETENTION_MS
+    ) {
+      throw new RangeError(
+        `quarantineRetentionMs must be between 0 and ${MAX_LOCAL_RUN_QUARANTINE_RETENTION_MS}`,
+      );
+    }
+    this.onDiagnostic = options.onDiagnostic;
+    this.completionProcessor =
+      options.completionProcessor ??
+      new LocalCompletionReceiptProcessor(repository, receipts, {
+        clock: this.clock,
+        createEventId: this.createEventId,
+        ...(this.journal === undefined ? {} : { journal: this.journal }),
+        quarantineRetentionMs: this.quarantineRetentionMs,
+        ...(this.onDiagnostic === undefined
+          ? {}
+          : { onDiagnostic: this.onDiagnostic }),
+      });
+  }
+
+  async recover(): Promise<LocalRunStartupRecoverySummary> {
+    const initial = validatePage(
+      await this.source.inspectCandidates({
+        limit: MAX_LOCAL_RUN_RECOVERY_ITEMS,
+      }),
+    );
+    if (initial.truncated) {
+      return Object.freeze({
+        safe: false,
+        scanned: initial.candidates.length,
+        recovered: 0,
+        remaining: initial.candidates.length,
+        failed: 0,
+        truncated: true,
+      });
+    }
+    if (initial.candidates.length === 0) {
+      return Object.freeze({
+        safe: true,
+        scanned: 0,
+        recovered: 0,
+        remaining: 0,
+        failed: 0,
+        truncated: false,
+      });
+    }
+
+    let recovered = 0;
+    let remaining = 0;
+    let failed = 0;
+    const verified = new Map<string, VerifiedFingerprint>();
+    for (const candidate of initial.candidates) {
+      const disposition = await this.reconcile(candidate);
+      if (disposition.status === 'recovered') recovered += 1;
+      if (disposition.status === 'verified') {
+        recovered += 1;
+        verified.set(candidate.runId, disposition.fingerprint);
+      }
+      if (disposition.status === 'remaining') remaining += 1;
+      if (disposition.status === 'failed') failed += 1;
+    }
+
+    if (remaining === 0 && failed === 0) {
+      const final = await this.verify(verified);
+      if (final.remaining > 0 || final.failed > 0) {
+        const unresolved = Math.min(
+          initial.candidates.length,
+          Math.max(1, final.remaining + final.failed),
+        );
+        recovered = Math.max(0, recovered - unresolved);
+        failed = Math.min(unresolved, final.failed);
+        remaining = unresolved - failed;
+      }
+    }
+    return Object.freeze({
+      safe: remaining === 0 && failed === 0,
+      scanned: initial.candidates.length,
+      recovered,
+      remaining,
+      failed,
+      truncated: false,
+    });
+  }
+
+  private async reconcile(
+    candidate: LocalRunStartupRecoveryCandidate,
+  ): Promise<CandidateDisposition> {
+    try {
+      if (candidate.activeAttemptCount > 1) {
+        return Object.freeze({ status: 'remaining' });
+      }
+      let snapshot = await this.load(candidate.runId);
+      if (
+        !snapshot ||
+        snapshot.run.executionOwner !== 'runtime' ||
+        snapshot.run.status !== candidate.runStatus
+      ) {
+        return Object.freeze({ status: 'remaining' });
+      }
+      const attempt = snapshot.attempt;
+      if (candidate.activeAttemptCount === 0) {
+        if (snapshot.run.status !== 'dispatching' || attempt !== null) {
+          return Object.freeze({ status: 'remaining' });
+        }
+        return (await this.markRunLostWithoutAttempt(snapshot.run))
+          ? Object.freeze({ status: 'recovered' })
+          : Object.freeze({ status: 'remaining' });
+      }
+      if (
+        !attempt ||
+        !ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
+        attempt.runId !== snapshot.run.id
+      ) {
+        return Object.freeze({ status: 'remaining' });
+      }
+
+      const completion = await this.completionProcessor.process(attempt.id);
+      if (completion === 'completed' || completion === 'already_terminal') {
+        return Object.freeze({ status: 'recovered' });
+      }
+      if (completion === 'invalid')
+        return Object.freeze({ status: 'remaining' });
+      if (attempt.status === 'claimed') {
+        return (await this.markAggregateLost(snapshot, 'unstarted_claim'))
+          ? Object.freeze({ status: 'recovered' })
+          : Object.freeze({ status: 'remaining' });
+      }
+      if (
+        attempt.executorType !== 'local_process' ||
+        !attempt.executorHandle ||
+        !Number.isSafeInteger(attempt.pid) ||
+        (attempt.pid as number) < 1
+      ) {
+        return Object.freeze({ status: 'remaining' });
+      }
+
+      const inspection = await this.inspector.inspect(attempt.executorHandle);
+      if (inspection.status === 'unknown') {
+        return inspection.reason === 'provider_unavailable'
+          ? Object.freeze({ status: 'failed' })
+          : Object.freeze({ status: 'remaining' });
+      }
+      if (inspection.identityPid !== attempt.pid) {
+        return Object.freeze({ status: 'remaining' });
+      }
+      if (inspection.status === 'running') {
+        snapshot = await this.markObservedRunning(snapshot);
+        return snapshot
+          ? Object.freeze({
+              status: 'verified',
+              fingerprint: fingerprint(snapshot),
+            })
+          : Object.freeze({ status: 'remaining' });
+      }
+
+      if (this.receiptPublishGraceMs > 0) {
+        await this.wait(this.receiptPublishGraceMs);
+        const delayed = await this.load(candidate.runId);
+        if (!delayed || !delayed.attempt || delayed.attempt.id !== attempt.id) {
+          return Object.freeze({ status: 'remaining' });
+        }
+        const lateCompletion = await this.completionProcessor.process(
+          attempt.id,
+        );
+        if (lateCompletion === 'invalid') {
+          return Object.freeze({ status: 'remaining' });
+        }
+        if (
+          lateCompletion === 'completed' ||
+          lateCompletion === 'already_terminal'
+        ) {
+          return Object.freeze({ status: 'recovered' });
+        }
+        snapshot = delayed;
+      }
+      return (await this.markAggregateLost(snapshot, 'process_not_running'))
+        ? Object.freeze({ status: 'recovered' })
+        : Object.freeze({ status: 'remaining' });
+    } catch {
+      return Object.freeze({ status: 'failed' });
+    }
+  }
+
+  private async verify(
+    verified: ReadonlyMap<string, VerifiedFingerprint>,
+  ): Promise<Readonly<{ remaining: number; failed: number }>> {
+    let remaining = 0;
+    let failed = 0;
+    let page: LocalRunStartupRecoveryPage;
+    try {
+      page = validatePage(
+        await this.source.inspectCandidates({
+          limit: MAX_LOCAL_RUN_RECOVERY_ITEMS,
+        }),
+      );
+    } catch {
+      return Object.freeze({ remaining: 0, failed: 1 });
+    }
+    if (page.truncated) {
+      return Object.freeze({
+        remaining: page.candidates.length,
+        failed: 0,
+      });
+    }
+    for (const candidate of page.candidates) {
+      const expected = verified.get(candidate.runId);
+      if (!expected || candidate.activeAttemptCount !== 1) {
+        remaining += 1;
+        continue;
+      }
+      try {
+        const completion = await this.completionProcessor.process(
+          expected.attemptId,
+        );
+        if (completion === 'completed' || completion === 'already_terminal') {
+          continue;
+        }
+        if (completion === 'invalid') {
+          remaining += 1;
+          continue;
+        }
+        const snapshot = await this.load(candidate.runId);
+        if (!snapshot || !sameFingerprint(expected, snapshot)) {
+          remaining += 1;
+          continue;
+        }
+        const inspection = await this.inspector.inspect(
+          expected.executorHandle,
+        );
+        if (
+          inspection.status !== 'running' ||
+          inspection.identityPid !== expected.pid
+        ) {
+          remaining += 1;
+        }
+      } catch {
+        failed += 1;
+      }
+    }
+    return Object.freeze({ remaining, failed });
+  }
+
+  private load(runId: string): Promise<AggregateSnapshot | null> {
+    return this.repository.transaction(async (transaction) => {
+      const run = await transaction.findRunById(runId);
+      if (!run) return null;
+      const attempt = await transaction.findLatestAttemptByRunId(runId);
+      return Object.freeze({ run, attempt });
+    });
+  }
+
+  private async markRunLostWithoutAttempt(run: RunRecord): Promise<boolean> {
+    return this.repository.transaction(async (transaction) => {
+      const current = await transaction.findRunById(run.id);
+      if (
+        !current ||
+        current.version !== run.version ||
+        current.status !== 'dispatching' ||
+        current.executionOwner !== 'runtime' ||
+        (await transaction.findLatestAttemptByRunId(run.id)) !== null
+      ) {
+        return false;
+      }
+      const atMs = atOrAfter(this.clock.now(), current);
+      const reserved = reserveEvent(current);
+      const next: RunRecord = {
+        ...reserved.run,
+        status: 'lost',
+        errorCode: 'RECOVERY_ATTEMPT_MISSING_BEFORE_START',
+        errorSummary: 'Dispatching Run has no durable Attempt',
+      };
+      if (!(await transaction.compareAndSetRun(next, current.version))) {
+        throw new LocalRunRecoveryConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          current.id,
+          undefined,
+          {
+            sequence: reserved.sequence,
+            type: 'run.lost',
+            payload: Object.freeze({
+              from_status: current.status,
+              to_status: 'lost',
+              error_code: next.errorCode,
+              version: next.version,
+            }),
+          },
+          `local-recovery:run:${current.id}:${current.version}:missing-attempt`,
+          'reconciler',
+          'local-startup',
+          atMs,
+        ),
+      );
+      return true;
+    });
+  }
+
+  private async markAggregateLost(
+    expected: AggregateSnapshot,
+    reason: 'unstarted_claim' | 'process_not_running',
+  ): Promise<boolean> {
+    return this.repository.transaction(async (transaction) => {
+      const current = await this.reloadExact(transaction, expected);
+      if (!current || !current.attempt) return false;
+      const { run, attempt } = current;
+      if (
+        run.cancelRequestedAtMs !== undefined ||
+        !ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
+        (reason === 'unstarted_claim' && attempt.status !== 'claimed') ||
+        (reason === 'process_not_running' &&
+          attempt.status !== 'starting' &&
+          attempt.status !== 'running')
+      ) {
+        return false;
+      }
+      const errorCode =
+        reason === 'unstarted_claim'
+          ? 'RECOVERY_UNSTARTED_CLAIM'
+          : 'RECOVERY_PROCESS_NOT_RUNNING';
+      const errorSummary =
+        reason === 'unstarted_claim'
+          ? 'Claimed Attempt never crossed the start barrier'
+          : 'Durable process identity proves the execution is not running';
+      const atMs = atOrAfter(this.clock.now(), run, attempt);
+      const attemptReserved = reserveEvent(run);
+      const nextAttempt: RunAttemptRecord = {
+        ...attempt,
+        status: 'lost',
+        finishedAtMs: atMs,
+        errorCode,
+        errorSummary,
+      };
+      await this.persistRunAndAttempt(
+        transaction,
+        run,
+        attempt,
+        attemptReserved.run,
+        nextAttempt,
+      );
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          run.id,
+          attempt.id,
+          {
+            sequence: attemptReserved.sequence,
+            type: 'attempt.lost',
+            payload: Object.freeze({
+              attempt_id: attempt.id,
+              from_status: attempt.status,
+              to_status: 'lost',
+              error_code: errorCode,
+              version: attemptReserved.run.version,
+            }),
+          },
+          `local-recovery:attempt:${attempt.id}:${attempt.callbackSequence}:${reason}`,
+          'reconciler',
+          'local-startup',
+          atMs,
+        ),
+      );
+      const runReserved = reserveEvent(attemptReserved.run);
+      const nextRun: RunRecord = {
+        ...runReserved.run,
+        status: 'lost',
+        errorCode,
+        errorSummary,
+      };
+      if (
+        !(await transaction.compareAndSetRun(
+          nextRun,
+          attemptReserved.run.version,
+        ))
+      ) {
+        throw new LocalRunRecoveryConcurrentWriteError();
+      }
+      await transaction.appendEvent(
+        event(
+          this.createEventId(),
+          run.id,
+          attempt.id,
+          {
+            sequence: runReserved.sequence,
+            type: 'run.lost',
+            payload: Object.freeze({
+              from_status: run.status,
+              to_status: 'lost',
+              error_code: errorCode,
+              version: nextRun.version,
+            }),
+          },
+          `local-recovery:run:${run.id}:${attempt.id}:${run.version}:${reason}`,
+          'reconciler',
+          'local-startup',
+          atMs,
+        ),
+      );
+      return true;
+    });
+  }
+
+  private async markObservedRunning(
+    expected: AggregateSnapshot,
+  ): Promise<AggregateSnapshot | null> {
+    return this.repository.transaction(async (transaction) => {
+      const current = await this.reloadExact(transaction, expected);
+      if (!current || !current.attempt) return null;
+      let { run, attempt } = current;
+      const originalRunStatus = run.status;
+      const atMs = atOrAfter(this.clock.now(), run, attempt);
+      if (attempt.status === 'starting') {
+        const reserved = reserveEvent(run);
+        const nextAttempt: RunAttemptRecord = {
+          ...attempt,
+          status: 'running',
+          startedAtMs: attempt.startedAtMs ?? atMs,
+        };
+        await this.persistRunAndAttempt(
+          transaction,
+          run,
+          attempt,
+          reserved.run,
+          nextAttempt,
+        );
+        await transaction.appendEvent(
+          event(
+            this.createEventId(),
+            run.id,
+            attempt.id,
+            {
+              sequence: reserved.sequence,
+              type: 'attempt.running',
+              payload: Object.freeze({
+                attempt_id: attempt.id,
+                from_status: attempt.status,
+                to_status: 'running',
+                evidence: 'durable_process_identity',
+                version: reserved.run.version,
+              }),
+            },
+            `local-recovery:attempt:${attempt.id}:${attempt.callbackSequence}:running`,
+            'reconciler',
+            'local-startup',
+            atMs,
+          ),
+        );
+        run = reserved.run;
+        attempt = nextAttempt;
+      }
+      if (run.status === 'dispatching') {
+        const reserved = reserveEvent(run);
+        const nextRun: RunRecord = {
+          ...reserved.run,
+          status: 'running',
+          startedAtMs: run.startedAtMs ?? atMs,
+        };
+        if (!(await transaction.compareAndSetRun(nextRun, run.version))) {
+          throw new LocalRunRecoveryConcurrentWriteError();
+        }
+        await transaction.appendEvent(
+          event(
+            this.createEventId(),
+            run.id,
+            attempt.id,
+            {
+              sequence: reserved.sequence,
+              type: 'run.running',
+              payload: Object.freeze({
+                from_status: originalRunStatus,
+                to_status: 'running',
+                evidence: 'durable_process_identity',
+                version: nextRun.version,
+              }),
+            },
+            `local-recovery:run:${run.id}:${attempt.id}:${run.version}:running`,
+            'reconciler',
+            'local-startup',
+            atMs,
+          ),
+        );
+        run = nextRun;
+      }
+      if (run.status !== 'running' || attempt.status !== 'running') return null;
+      return Object.freeze({ run, attempt });
+    });
+  }
+
+  private async reloadExact(
+    transaction: RunRepositoryTransaction,
+    expected: AggregateSnapshot,
+  ): Promise<AggregateSnapshot | null> {
+    const run = await transaction.findRunById(expected.run.id);
+    const expectedAttempt = expected.attempt;
+    if (!run || run.version !== expected.run.version) return null;
+    if (!expectedAttempt) {
+      return Object.freeze({ run, attempt: null });
+    }
+    const attempt = await transaction.findAttemptById(expectedAttempt.id);
+    if (
+      !attempt ||
+      attempt.runId !== run.id ||
+      attempt.status !== expectedAttempt.status ||
+      attempt.callbackSequence !== expectedAttempt.callbackSequence ||
+      attempt.executorHandle !== expectedAttempt.executorHandle ||
+      attempt.pid !== expectedAttempt.pid
+    ) {
+      return null;
+    }
+    return Object.freeze({ run, attempt });
+  }
+
+  private async persistRunAndAttempt(
+    transaction: RunRepositoryTransaction,
+    run: RunRecord,
+    attempt: RunAttemptRecord,
+    nextRun: RunRecord,
+    nextAttempt: RunAttemptRecord,
+  ): Promise<void> {
+    if (!(await transaction.compareAndSetRun(nextRun, run.version))) {
+      throw new LocalRunRecoveryConcurrentWriteError();
+    }
+    if (
+      !(await transaction.compareAndSetAttempt(nextAttempt, {
+        status: attempt.status,
+        callbackSequence: attempt.callbackSequence,
+      }))
+    ) {
+      throw new LocalRunRecoveryConcurrentWriteError();
+    }
+  }
+}

packages/ql3-local-execution/src/recovery/index.ts

@@ -0,0 +1,2 @@
+export * from './coordinator';
+export * from './workflowTask';

packages/ql3-local-execution/src/recovery/workflowTask.ts

@@ -0,0 +1,461 @@
+import { randomUUID } from 'node:crypto';
+
+import type {
+  RunAttemptRecord,
+  RunRecord,
+  RunRepository,
+} from '@qinglong/runtime-core/run-repository';
+import type {
+  LocalPersistedExecutionInspector,
+} from '@qinglong/local-process';
+
+import type { LocalCompletionReceiptProcessor } from '../control/completion';
+import type { LocalWorkflowTaskExecutionRepository } from '../execution/workflowTaskExecution';
+
+export const MAX_LOCAL_WORKFLOW_TASK_RECOVERY_ITEMS = 64;
+
+export interface LocalWorkflowTaskRecoveryCandidate {
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly attemptCreatedAtMs: number;
+}
+
+export interface LocalWorkflowTaskRecoveryPage {
+  readonly candidates: readonly Readonly<LocalWorkflowTaskRecoveryCandidate>[];
+  readonly truncated: boolean;
+}
+
+export interface LocalWorkflowTaskRecoveryRepository {
+  listRecoveryCandidates(command: Readonly<{
+    limit: number;
+  }>): Promise<Readonly<LocalWorkflowTaskRecoveryPage>>;
+  recover(command: Readonly<{
+    run: Readonly<RunRecord>;
+    attempt: Readonly<RunAttemptRecord>;
+    reason: 'unstarted_claim_expired' | 'execution_not_running';
+    observedAtMs: number;
+  }>): Promise<'requeued' | 'failed' | 'already_recovered' | 'stale'>;
+}
+
+export interface LocalWorkflowTaskStartupRecoveryOptions {
+  readonly receiptPublishGraceMs?: number;
+  readonly clock?: { now(): number };
+  readonly wait?: (delayMs: number) => Promise<void>;
+  readonly createEventId?: () => string;
+}
+
+export interface LocalWorkflowTaskStartupRecoverySummary {
+  readonly safe: boolean;
+  readonly scanned: number;
+  readonly recovered: number;
+  readonly verified: number;
+  readonly remaining: number;
+  readonly failed: number;
+  readonly truncated: boolean;
+}
+
+interface ActiveSnapshot {
+  readonly run: Readonly<RunRecord>;
+  readonly attempt: Readonly<RunAttemptRecord>;
+}
+
+interface VerifiedFingerprint {
+  readonly runId: string;
+  readonly runVersion: number;
+  readonly attemptId: string;
+  readonly callbackSequence: number;
+  readonly executorHandle: string;
+  readonly pid: number;
+}
+
+function timestamp(value: unknown, label: string): number {
+  if (!Number.isSafeInteger(value) || (value as number) < 0) {
+    throw new TypeError(`${label} is invalid`);
+  }
+  return value as number;
+}
+
+function validatePage(
+  value: Readonly<LocalWorkflowTaskRecoveryPage>,
+): Readonly<LocalWorkflowTaskRecoveryPage> {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    !Array.isArray(value.candidates) ||
+    value.candidates.length > MAX_LOCAL_WORKFLOW_TASK_RECOVERY_ITEMS ||
+    typeof value.truncated !== 'boolean'
+  ) {
+    throw new TypeError('Local Workflow Task recovery page is invalid');
+  }
+  const seen = new Set<string>();
+  let previous:
+    | Readonly<LocalWorkflowTaskRecoveryCandidate>
+    | undefined;
+  const candidates = value.candidates.map((candidate) => {
+    if (
+      !candidate ||
+      typeof candidate !== 'object' ||
+      Array.isArray(candidate) ||
+      typeof candidate.runId !== 'string' ||
+      candidate.runId.length < 1 ||
+      typeof candidate.attemptId !== 'string' ||
+      candidate.attemptId.length < 1 ||
+      seen.has(candidate.attemptId)
+    ) {
+      throw new TypeError(
+        'Local Workflow Task recovery candidate is invalid',
+      );
+    }
+    const attemptCreatedAtMs = timestamp(
+      candidate.attemptCreatedAtMs,
+      'Local Workflow Task recovery candidate timestamp',
+    );
+    const normalized = Object.freeze({
+      runId: candidate.runId,
+      attemptId: candidate.attemptId,
+      attemptCreatedAtMs,
+    });
+    if (
+      previous &&
+      (normalized.attemptCreatedAtMs < previous.attemptCreatedAtMs ||
+        (normalized.attemptCreatedAtMs === previous.attemptCreatedAtMs &&
+          normalized.attemptId <= previous.attemptId))
+    ) {
+      throw new TypeError(
+        'Local Workflow Task recovery candidates are not ordered',
+      );
+    }
+    seen.add(normalized.attemptId);
+    previous = normalized;
+    return normalized;
+  });
+  return Object.freeze({
+    candidates: Object.freeze(candidates),
+    truncated: value.truncated,
+  });
+}
+
+function fingerprint(snapshot: ActiveSnapshot): VerifiedFingerprint {
+  if (
+    snapshot.run.status !== 'running' ||
+    snapshot.attempt.status !== 'running' ||
+    !snapshot.attempt.stepRunId ||
+    !snapshot.attempt.executorHandle ||
+    !Number.isSafeInteger(snapshot.attempt.pid) ||
+    snapshot.attempt.pid! < 1
+  ) {
+    throw new TypeError(
+      'Local Workflow Task recovery fingerprint is invalid',
+    );
+  }
+  return Object.freeze({
+    runId: snapshot.run.id,
+    runVersion: snapshot.run.version,
+    attemptId: snapshot.attempt.id,
+    callbackSequence: snapshot.attempt.callbackSequence,
+    executorHandle: snapshot.attempt.executorHandle,
+    pid: snapshot.attempt.pid!,
+  });
+}
+
+export class LocalWorkflowTaskStartupRecoveryCoordinator {
+  private readonly clock: { now(): number };
+  private readonly wait: (delayMs: number) => Promise<void>;
+  private readonly receiptPublishGraceMs: number;
+  private readonly createEventId: () => string;
+
+  constructor(
+    private readonly runs: RunRepository,
+    private readonly recovery: LocalWorkflowTaskRecoveryRepository,
+    private readonly workflowTasks: LocalWorkflowTaskExecutionRepository,
+    private readonly completions: Pick<
+      LocalCompletionReceiptProcessor,
+      'process'
+    >,
+    private readonly inspector: LocalPersistedExecutionInspector,
+    options: LocalWorkflowTaskStartupRecoveryOptions = {},
+  ) {
+    if (
+      typeof recovery?.listRecoveryCandidates !== 'function' ||
+      typeof recovery?.recover !== 'function' ||
+      typeof workflowTasks?.recordRunning !== 'function' ||
+      typeof completions?.process !== 'function' ||
+      inspector?.executorType !== 'local_process'
+    ) {
+      throw new TypeError(
+        'Local Workflow Task startup recovery dependencies are invalid',
+      );
+    }
+    this.clock = options.clock ?? { now: Date.now };
+    this.wait =
+      options.wait ??
+      ((delayMs) =>
+        new Promise((resolve) => {
+          setTimeout(resolve, delayMs);
+        }));
+    this.receiptPublishGraceMs = options.receiptPublishGraceMs ?? 0;
+    if (
+      !Number.isSafeInteger(this.receiptPublishGraceMs) ||
+      this.receiptPublishGraceMs < 0 ||
+      this.receiptPublishGraceMs > 5_000
+    ) {
+      throw new RangeError(
+        'Local Workflow Task receipt grace is invalid',
+      );
+    }
+    this.createEventId = options.createEventId ?? randomUUID;
+  }
+
+  async recover(): Promise<Readonly<LocalWorkflowTaskStartupRecoverySummary>> {
+    const initial = validatePage(
+      await this.recovery.listRecoveryCandidates({
+        limit: MAX_LOCAL_WORKFLOW_TASK_RECOVERY_ITEMS,
+      }),
+    );
+    if (initial.truncated) {
+      return Object.freeze({
+        safe: false,
+        scanned: initial.candidates.length,
+        recovered: 0,
+        verified: 0,
+        remaining: initial.candidates.length,
+        failed: 0,
+        truncated: true,
+      });
+    }
+    let recovered = 0;
+    let remaining = 0;
+    let failed = 0;
+    const verified = new Map<string, VerifiedFingerprint>();
+    for (const candidate of initial.candidates) {
+      try {
+        const result = await this.reconcile(candidate);
+        if (result.status === 'recovered') recovered += 1;
+        else if (result.status === 'verified') {
+          verified.set(candidate.attemptId, result.fingerprint);
+        } else if (result.status === 'remaining') remaining += 1;
+        else failed += 1;
+      } catch {
+        failed += 1;
+      }
+    }
+    if (remaining === 0 && failed === 0) {
+      const final = await this.verify(verified);
+      remaining += final.remaining;
+      failed += final.failed;
+    }
+    return Object.freeze({
+      safe: remaining === 0 && failed === 0,
+      scanned: initial.candidates.length,
+      recovered,
+      verified: verified.size,
+      remaining,
+      failed,
+      truncated: false,
+    });
+  }
+
+  private async reconcile(
+    candidate: Readonly<LocalWorkflowTaskRecoveryCandidate>,
+  ): Promise<
+    | Readonly<{ status: 'recovered' }>
+    | Readonly<{
+        status: 'verified';
+        fingerprint: VerifiedFingerprint;
+      }>
+    | Readonly<{ status: 'remaining' }>
+    | Readonly<{ status: 'failed' }>
+  > {
+    let snapshot = await this.load(candidate);
+    if (!snapshot) return Object.freeze({ status: 'remaining' as const });
+    const completion = await this.completions.process(candidate.attemptId);
+    if (completion === 'completed' || completion === 'already_terminal') {
+      return Object.freeze({ status: 'recovered' as const });
+    }
+    if (completion === 'invalid') {
+      return Object.freeze({ status: 'remaining' as const });
+    }
+    if (snapshot.attempt.status === 'claimed') {
+      const result = await this.recovery.recover({
+        ...snapshot,
+        reason: 'unstarted_claim_expired',
+        observedAtMs: Math.max(
+          timestamp(
+            this.clock.now(),
+            'Local Workflow Task recovery clock',
+          ),
+          snapshot.attempt.createdAtMs,
+        ),
+      });
+      return result === 'requeued' || result === 'already_recovered'
+        ? Object.freeze({ status: 'recovered' as const })
+        : Object.freeze({ status: 'remaining' as const });
+    }
+    if (
+      (snapshot.attempt.status !== 'starting' &&
+        snapshot.attempt.status !== 'running') ||
+      !snapshot.attempt.executorHandle ||
+      !Number.isSafeInteger(snapshot.attempt.pid) ||
+      snapshot.attempt.pid! < 1
+    ) {
+      return Object.freeze({ status: 'remaining' as const });
+    }
+    const inspection = await this.inspector.inspect(
+      snapshot.attempt.executorHandle,
+    );
+    if (inspection.status === 'unknown') {
+      return inspection.reason === 'provider_unavailable'
+        ? Object.freeze({ status: 'failed' as const })
+        : Object.freeze({ status: 'remaining' as const });
+    }
+    if (inspection.identityPid !== snapshot.attempt.pid) {
+      return Object.freeze({ status: 'remaining' as const });
+    }
+    if (inspection.status === 'running') {
+      if (snapshot.attempt.status === 'starting') {
+        if (!snapshot.attempt.callbackTokenHash) {
+          return Object.freeze({ status: 'remaining' as const });
+        }
+        const marked = await this.workflowTasks.recordRunning({
+          run: snapshot.run,
+          attempt: snapshot.attempt,
+          callbackTokenHash: snapshot.attempt.callbackTokenHash,
+          executorHandle: snapshot.attempt.executorHandle,
+          pid: snapshot.attempt.pid,
+          startedAtMs: Math.max(
+            timestamp(
+              this.clock.now(),
+              'Local Workflow Task recovery clock',
+            ),
+            snapshot.attempt.createdAtMs,
+          ),
+          attemptEventId: this.createEventId(),
+          stepMutationId: this.createEventId(),
+        });
+        if (marked.status !== 'applied') {
+          return Object.freeze({ status: 'remaining' as const });
+        }
+        snapshot = await this.load(candidate);
+        if (!snapshot || snapshot.attempt.status !== 'running') {
+          return Object.freeze({ status: 'remaining' as const });
+        }
+      }
+      return Object.freeze({
+        status: 'verified' as const,
+        fingerprint: fingerprint(snapshot),
+      });
+    }
+    if (this.receiptPublishGraceMs > 0) {
+      await this.wait(this.receiptPublishGraceMs);
+      const late = await this.completions.process(candidate.attemptId);
+      if (late === 'completed' || late === 'already_terminal') {
+        return Object.freeze({ status: 'recovered' as const });
+      }
+      if (late === 'invalid') {
+        return Object.freeze({ status: 'remaining' as const });
+      }
+      snapshot = await this.load(candidate);
+      if (!snapshot) return Object.freeze({ status: 'recovered' as const });
+    }
+    const result = await this.recovery.recover({
+      ...snapshot,
+      reason: 'execution_not_running',
+      observedAtMs: Math.max(
+        timestamp(
+          this.clock.now(),
+          'Local Workflow Task recovery clock',
+        ),
+        snapshot.attempt.createdAtMs,
+        snapshot.attempt.startedAtMs ?? 0,
+      ),
+    });
+    return result === 'failed' || result === 'already_recovered'
+      ? Object.freeze({ status: 'recovered' as const })
+      : Object.freeze({ status: 'remaining' as const });
+  }
+
+  private async verify(
+    expected: ReadonlyMap<string, VerifiedFingerprint>,
+  ): Promise<Readonly<{ remaining: number; failed: number }>> {
+    let page: Readonly<LocalWorkflowTaskRecoveryPage>;
+    try {
+      page = validatePage(
+        await this.recovery.listRecoveryCandidates({
+          limit: MAX_LOCAL_WORKFLOW_TASK_RECOVERY_ITEMS,
+        }),
+      );
+    } catch {
+      return Object.freeze({ remaining: 0, failed: 1 });
+    }
+    if (page.truncated) {
+      return Object.freeze({
+        remaining: page.candidates.length,
+        failed: 0,
+      });
+    }
+    let remaining = 0;
+    let failed = 0;
+    for (const candidate of page.candidates) {
+      const fingerprintValue = expected.get(candidate.attemptId);
+      if (!fingerprintValue) {
+        remaining += 1;
+        continue;
+      }
+      try {
+        const snapshot = await this.load(candidate);
+        if (
+          !snapshot ||
+          snapshot.run.version !== fingerprintValue.runVersion ||
+          snapshot.attempt.status !== 'running' ||
+          snapshot.attempt.callbackSequence !==
+            fingerprintValue.callbackSequence ||
+          snapshot.attempt.executorHandle !==
+            fingerprintValue.executorHandle ||
+          snapshot.attempt.pid !== fingerprintValue.pid
+        ) {
+          remaining += 1;
+          continue;
+        }
+        const inspection = await this.inspector.inspect(
+          fingerprintValue.executorHandle,
+        );
+        if (
+          inspection.status !== 'running' ||
+          inspection.identityPid !== fingerprintValue.pid
+        ) {
+          remaining += 1;
+        }
+      } catch {
+        failed += 1;
+      }
+    }
+    return Object.freeze({ remaining, failed });
+  }
+
+  private load(
+    candidate: Readonly<LocalWorkflowTaskRecoveryCandidate>,
+  ): Promise<ActiveSnapshot | null> {
+    return this.runs.transaction(async (transaction) => {
+      const [run, attempt] = await Promise.all([
+        transaction.findRunById(candidate.runId),
+        transaction.findAttemptById(candidate.attemptId),
+      ]);
+      if (
+        !run ||
+        !attempt ||
+        run.status !== 'running' ||
+        run.executionOwner !== 'runtime' ||
+        run.cancelRequestedAtMs !== undefined ||
+        attempt.runId !== run.id ||
+        !attempt.stepRunId ||
+        attempt.executorType !== 'local_process' ||
+        !['claimed', 'starting', 'running'].includes(attempt.status) ||
+        attempt.createdAtMs !== candidate.attemptCreatedAtMs
+      ) {
+        return null;
+      }
+      return Object.freeze({ run, attempt });
+    });
+  }
+}

packages/ql3-local-execution/src/scheduler/coordinator.ts

@@ -0,0 +1,118 @@
+import { randomUUID } from 'node:crypto';
+import {
+  assertLocalSchedulePageSize,
+  resolveLocalScheduleDecision,
+  type LocalCronNextOccurrence,
+  type LocalScheduleStore,
+} from '@qinglong/runtime-core/local-scheduler';
+import { cronerLocalNextOccurrence } from './croner';
+
+export interface LocalSchedulerCoordinatorOptions {
+  readonly pageSize?: number;
+  readonly misfireGraceMs?: number;
+  readonly clock?: () => number;
+  readonly createId?: () => string;
+  readonly nextOccurrence?: LocalCronNextOccurrence;
+  readonly onAdmitted?: (
+    runId: string,
+    attemptId: string,
+  ) => void | Promise<void>;
+}
+
+export interface LocalSchedulerCycleSummary {
+  readonly observedAtMs: number;
+  readonly scanned: number;
+  readonly initialized: number;
+  readonly skipped: number;
+  readonly admitted: number;
+  readonly raced: number;
+  readonly truncated: boolean;
+}
+
+export class LocalSchedulerCoordinator {
+  private readonly pageSize: number;
+  private readonly misfireGraceMs: number;
+  private readonly clock: () => number;
+  private readonly createId: () => string;
+  private readonly nextOccurrence: LocalCronNextOccurrence;
+  private readonly onAdmitted?: LocalSchedulerCoordinatorOptions['onAdmitted'];
+
+  constructor(
+    private readonly schedules: LocalScheduleStore,
+    options: LocalSchedulerCoordinatorOptions = {},
+  ) {
+    this.pageSize = options.pageSize ?? 8;
+    this.misfireGraceMs = options.misfireGraceMs ?? 30_000;
+    this.clock = options.clock ?? Date.now;
+    this.createId = options.createId ?? randomUUID;
+    this.nextOccurrence = options.nextOccurrence ?? cronerLocalNextOccurrence;
+    this.onAdmitted = options.onAdmitted;
+    assertLocalSchedulePageSize(this.pageSize);
+    if (
+      !Number.isSafeInteger(this.misfireGraceMs) ||
+      this.misfireGraceMs < 0 ||
+      this.misfireGraceMs > 5 * 60_000 ||
+      typeof this.clock !== 'function' ||
+      typeof this.createId !== 'function' ||
+      typeof this.nextOccurrence !== 'function' ||
+      (this.onAdmitted !== undefined && typeof this.onAdmitted !== 'function')
+    ) {
+      throw new TypeError('Local scheduler options are invalid');
+    }
+  }
+
+  async scheduleOnce(): Promise<LocalSchedulerCycleSummary> {
+    const observedAtMs = this.clock();
+    if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
+      throw new TypeError('Local scheduler clock is invalid');
+    }
+    const page = await this.schedules.listLocalScheduleCandidates({
+      observedAtMs,
+      limit: this.pageSize,
+    });
+    if (page.candidates.length > this.pageSize) {
+      throw new RangeError('Local scheduler source exceeded its page size');
+    }
+    const stats = {
+      observedAtMs,
+      scanned: 0,
+      initialized: 0,
+      skipped: 0,
+      admitted: 0,
+      raced: 0,
+      truncated: page.truncated,
+    };
+    for (const candidate of page.candidates) {
+      stats.scanned += 1;
+      const decision = resolveLocalScheduleDecision(
+        candidate,
+        observedAtMs,
+        this.misfireGraceMs,
+        this.nextOccurrence,
+      );
+      const admitted = decision.disposition === 'admit';
+      const result = await this.schedules.commitLocalScheduleDecision({
+        decision,
+        ...(admitted
+          ? {
+              runId: this.createId(),
+              attemptId: this.createId(),
+              createdEventId: this.createId(),
+              queuedEventId: this.createId(),
+            }
+          : {}),
+      });
+      if (result.status === 'raced') {
+        stats.raced += 1;
+        continue;
+      }
+      if (result.disposition === 'initialize') stats.initialized += 1;
+      if (result.disposition === 'skip') stats.skipped += 1;
+      if (result.status === 'admitted') {
+        stats.admitted += 1;
+        await this.onAdmitted?.(result.runId, result.attemptId);
+      }
+    }
+    return Object.freeze(stats);
+  }
+}

packages/ql3-local-execution/src/scheduler/croner.ts

@@ -0,0 +1,44 @@
+import type {
+  LocalCronNextOccurrence,
+  LocalCronSchedule,
+} from '@qinglong/runtime-core/local-scheduler';
+
+interface CronerJob {
+  nextRun(after: Date): Date | null;
+  stop(): void;
+}
+
+interface CronerConstructor {
+  new (
+    expression: string,
+    options: Readonly<{
+      timezone: string;
+      paused: true;
+      unref: true;
+    }>,
+  ): CronerJob;
+}
+
+export const cronerLocalNextOccurrence: LocalCronNextOccurrence = (
+  schedule: LocalCronSchedule,
+  afterMs: number,
+): number => {
+  let job: CronerJob | undefined;
+  try {
+    const { Cron } = require('croner') as Readonly<{
+      Cron: CronerConstructor;
+    }>;
+    job = new Cron(schedule.expression, {
+      timezone: schedule.timezone,
+      paused: true,
+      unref: true,
+    });
+    const next = job.nextRun(new Date(afterMs));
+    if (!(next instanceof Date)) {
+      throw new Error('cron has no next occurrence');
+    }
+    return next.getTime();
+  } finally {
+    job?.stop();
+  }
+};

packages/ql3-local-execution/src/scheduler/index.ts

@@ -0,0 +1,16 @@
+export {
+  LocalSchedulerCoordinator,
+  type LocalSchedulerCoordinatorOptions,
+  type LocalSchedulerCycleSummary,
+} from './coordinator';
+export {
+  LocalSchedulerLifecycle,
+  type LocalSchedulerLifecycleOptions,
+  type LocalSchedulerLifecycleStopSummary,
+} from './lifecycle';
+export { cronerLocalNextOccurrence } from './croner';
+export {
+  LocalWorkflowSchedulerCoordinator,
+  type LocalWorkflowSchedulerCoordinatorOptions,
+  type LocalWorkflowSchedulerCycleSummary,
+} from './workflowCoordinator';

packages/ql3-local-execution/src/scheduler/lifecycle.ts

@@ -0,0 +1,128 @@
+import type {
+  LocalSchedulerCoordinator,
+  LocalSchedulerCycleSummary,
+} from './coordinator';
+
+export interface LocalSchedulerLifecycleOptions {
+  readonly intervalMs: number;
+  readonly stopTimeoutMs: number;
+  readonly onDiagnostic?: (
+    error: unknown,
+    summary?: LocalSchedulerCycleSummary,
+  ) => void | Promise<void>;
+}
+
+export interface LocalSchedulerLifecycleStopSummary {
+  readonly status: 'stopped' | 'timed_out';
+}
+
+export class LocalSchedulerLifecycle {
+  private timer: NodeJS.Timeout | undefined;
+  private inFlight: Promise<LocalSchedulerCycleSummary> | undefined;
+  private stopPromise: Promise<LocalSchedulerLifecycleStopSummary> | undefined;
+  private running = false;
+  private stopping = false;
+
+  constructor(
+    private readonly scheduler: Pick<LocalSchedulerCoordinator, 'scheduleOnce'>,
+    private readonly options: LocalSchedulerLifecycleOptions,
+  ) {
+    if (
+      !scheduler ||
+      typeof scheduler.scheduleOnce !== 'function' ||
+      !options ||
+      typeof options !== 'object' ||
+      Array.isArray(options) ||
+      !Number.isSafeInteger(options.intervalMs) ||
+      options.intervalMs < 250 ||
+      options.intervalMs > 60 * 60_000 ||
+      !Number.isSafeInteger(options.stopTimeoutMs) ||
+      options.stopTimeoutMs < 100 ||
+      options.stopTimeoutMs > 30_000 ||
+      (options.onDiagnostic !== undefined &&
+        typeof options.onDiagnostic !== 'function')
+    ) {
+      throw new TypeError('Local scheduler lifecycle options are invalid');
+    }
+  }
+
+  start(): 'started' {
+    if (!this.running && !this.stopping) {
+      this.running = true;
+      this.schedule();
+    }
+    return 'started';
+  }
+
+  runOnce(): Promise<LocalSchedulerCycleSummary> {
+    if (this.stopping) {
+      return Promise.reject(
+        new Error('Local scheduler lifecycle is stopping'),
+      );
+    }
+    if (this.inFlight) return this.inFlight;
+    const work = this.scheduler.scheduleOnce().finally(() => {
+      if (this.inFlight === work) this.inFlight = undefined;
+    });
+    this.inFlight = work;
+    return work;
+  }
+
+  stopAndDrain(): Promise<LocalSchedulerLifecycleStopSummary> {
+    if (this.stopPromise) return this.stopPromise;
+    this.stopping = true;
+    this.running = false;
+    if (this.timer) clearTimeout(this.timer);
+    this.timer = undefined;
+
+    this.stopPromise = (async () => {
+      const work = this.inFlight;
+      if (!work) return Object.freeze({ status: 'stopped' as const });
+      let timeout: NodeJS.Timeout | undefined;
+      try {
+        return await Promise.race([
+          work.then(
+            () => Object.freeze({ status: 'stopped' as const }),
+            () => Object.freeze({ status: 'stopped' as const }),
+          ),
+          new Promise<LocalSchedulerLifecycleStopSummary>((resolve) => {
+            timeout = setTimeout(
+              () =>
+                resolve(Object.freeze({ status: 'timed_out' as const })),
+              this.options.stopTimeoutMs,
+            );
+            timeout.unref?.();
+          }),
+        ]);
+      } finally {
+        if (timeout) clearTimeout(timeout);
+      }
+    })();
+    return this.stopPromise;
+  }
+
+  private schedule(): void {
+    if (!this.running || this.timer) return;
+    this.timer = setTimeout(() => {
+      this.timer = undefined;
+      if (!this.running) return;
+      void this.runOnce()
+        .then((summary) => this.diagnostic(undefined, summary))
+        .catch((error) => this.diagnostic(error))
+        .finally(() => this.schedule());
+    }, this.options.intervalMs);
+    this.timer.unref?.();
+  }
+
+  private async diagnostic(
+    error: unknown,
+    summary?: LocalSchedulerCycleSummary,
+  ): Promise<void> {
+    if (this.stopping) return;
+    try {
+      await this.options.onDiagnostic?.(error, summary);
+    } catch {
+      // Diagnostics cannot own or stop scheduling.
+    }
+  }
+}

packages/ql3-local-execution/src/scheduler/workflowCoordinator.ts

@@ -0,0 +1,334 @@
+import {
+  ClusterRunCancellationConvergenceCoordinator,
+  type ClusterRunCancellationConvergenceCycleResult,
+  type ClusterRunCancellationConvergenceRepository,
+} from '@qinglong/runtime-core/cluster-run-cancellation-convergence';
+import type {
+  PluginPackageWorkflowFrontierCursor,
+  PluginPackageWorkflowFrontierRepository,
+} from '@qinglong/runtime-core/plugin-package-workflow-frontier';
+import type {
+  PluginPackageWorkflowTaskAttemptAdmissionCursor,
+  PluginPackageWorkflowTaskAttemptAdmissionRepository,
+} from '@qinglong/runtime-core/plugin-package-workflow-task-attempt-admission';
+
+import type {
+  LocalRunDispatcher,
+  LocalRunDispatcherResult,
+} from '../dispatch/dispatcher';
+import type {
+  LocalSchedulerCoordinator,
+  LocalSchedulerCycleSummary,
+} from './coordinator';
+
+export interface LocalWorkflowSchedulerCoordinatorOptions {
+  readonly cancellationPageSize: number;
+  readonly cancellationMaxPages: number;
+  readonly frontierPageSize: number;
+  readonly frontierMaxPages: number;
+  readonly taskAttemptPageSize: number;
+  readonly taskAttemptMaxPages: number;
+  readonly maxDispatches: number;
+}
+
+export interface LocalWorkflowSchedulerCycleSummary {
+  readonly cancellation: Readonly<ClusterRunCancellationConvergenceCycleResult>;
+  readonly frontierPages: number;
+  readonly frontierScanned: number;
+  readonly frontierAdvanced: number;
+  readonly frontierTruncated: boolean;
+  readonly taskAttemptPages: number;
+  readonly taskAttemptsScanned: number;
+  readonly taskAttemptsCreated: number;
+  readonly taskAttemptsExisting: number;
+  readonly taskAttemptsTruncated: boolean;
+  readonly dispatches: number;
+  readonly activated: number;
+  readonly activationFailed: number;
+  readonly dispatchIdle: boolean;
+}
+
+function bounded(
+  value: number,
+  minimum: number,
+  maximum: number,
+  label: string,
+): number {
+  if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
+    throw new RangeError(`${label} must be between ${minimum} and ${maximum}`);
+  }
+  return value;
+}
+
+function nextFrontierCursor(
+  current: PluginPackageWorkflowFrontierCursor | undefined,
+  next: PluginPackageWorkflowFrontierCursor | undefined,
+): PluginPackageWorkflowFrontierCursor {
+  if (
+    !next ||
+    (current !== undefined &&
+      (next.admittedAtMs < current.admittedAtMs ||
+        (next.admittedAtMs === current.admittedAtMs &&
+          next.planDigest <= current.planDigest)))
+  ) {
+    throw new TypeError(
+      'Local Workflow frontier continuation did not advance',
+    );
+  }
+  return next;
+}
+
+function nextTaskAttemptCursor(
+  current: PluginPackageWorkflowTaskAttemptAdmissionCursor | undefined,
+  next: PluginPackageWorkflowTaskAttemptAdmissionCursor | undefined,
+): PluginPackageWorkflowTaskAttemptAdmissionCursor {
+  if (
+    !next ||
+    (current !== undefined &&
+      (next.readyAtMs < current.readyAtMs ||
+        (next.readyAtMs === current.readyAtMs &&
+          next.stepRunId <= current.stepRunId)))
+  ) {
+    throw new TypeError(
+      'Local Workflow Task Attempt continuation did not advance',
+    );
+  }
+  return next;
+}
+
+/**
+ * Reuses the existing Local scheduler cadence. It owns no timer, connection,
+ * watcher, or per-Workflow state.
+ */
+export class LocalWorkflowSchedulerCoordinator {
+  private readonly cancellation: ClusterRunCancellationConvergenceCoordinator;
+  private readonly frontierPageSize: number;
+  private readonly frontierMaxPages: number;
+  private readonly taskAttemptPageSize: number;
+  private readonly taskAttemptMaxPages: number;
+  private readonly maxDispatches: number;
+  private inFlight:
+    | Promise<Readonly<LocalSchedulerCycleSummary>>
+    | undefined;
+  private latestWorkflow:
+    | Readonly<LocalWorkflowSchedulerCycleSummary>
+    | undefined;
+
+  constructor(
+    private readonly scheduler: Pick<LocalSchedulerCoordinator, 'scheduleOnce'>,
+    cancellation: ClusterRunCancellationConvergenceRepository,
+    private readonly frontier: PluginPackageWorkflowFrontierRepository,
+    private readonly taskAttempts: PluginPackageWorkflowTaskAttemptAdmissionRepository,
+    private readonly dispatcher: Pick<LocalRunDispatcher, 'dispatchOnce'>,
+    options: LocalWorkflowSchedulerCoordinatorOptions,
+  ) {
+    if (
+      typeof scheduler?.scheduleOnce !== 'function' ||
+      typeof cancellation?.convergePage !== 'function' ||
+      typeof frontier?.listCandidates !== 'function' ||
+      typeof frontier?.advance !== 'function' ||
+      typeof taskAttempts?.listCandidates !== 'function' ||
+      typeof taskAttempts?.admit !== 'function' ||
+      typeof dispatcher?.dispatchOnce !== 'function' ||
+      !options ||
+      typeof options !== 'object' ||
+      Array.isArray(options)
+    ) {
+      throw new TypeError(
+        'Local Workflow scheduler coordinator is invalid',
+      );
+    }
+    this.cancellation = new ClusterRunCancellationConvergenceCoordinator(
+      cancellation,
+      {
+        pageSize: options.cancellationPageSize,
+        maxPages: options.cancellationMaxPages,
+      },
+    );
+    this.frontierPageSize = bounded(
+      options.frontierPageSize,
+      1,
+      64,
+      'Local Workflow frontier page size',
+    );
+    this.frontierMaxPages = bounded(
+      options.frontierMaxPages,
+      1,
+      16,
+      'Local Workflow frontier page limit',
+    );
+    this.taskAttemptPageSize = bounded(
+      options.taskAttemptPageSize,
+      1,
+      64,
+      'Local Workflow Task Attempt page size',
+    );
+    this.taskAttemptMaxPages = bounded(
+      options.taskAttemptMaxPages,
+      1,
+      16,
+      'Local Workflow Task Attempt page limit',
+    );
+    this.maxDispatches = bounded(
+      options.maxDispatches,
+      1,
+      16,
+      'Local Workflow dispatch limit',
+    );
+  }
+
+  scheduleOnce(): Promise<Readonly<LocalSchedulerCycleSummary>> {
+    if (this.inFlight) return this.inFlight;
+    const work = this.runCycle().finally(() => {
+      if (this.inFlight === work) this.inFlight = undefined;
+    });
+    this.inFlight = work;
+    return work;
+  }
+
+  latestWorkflowSummary():
+    | Readonly<LocalWorkflowSchedulerCycleSummary>
+    | undefined {
+    return this.latestWorkflow;
+  }
+
+  private async runCycle(): Promise<Readonly<LocalSchedulerCycleSummary>> {
+    const cancellation = await this.cancellation.reconcile();
+    const scheduler = await this.scheduler.scheduleOnce();
+    const frontier = await this.advanceFrontier();
+    const taskAttempts = await this.admitTaskAttempts();
+    const dispatch = await this.dispatch();
+    this.latestWorkflow = Object.freeze({
+      cancellation,
+      ...frontier,
+      ...taskAttempts,
+      ...dispatch,
+    });
+    return scheduler;
+  }
+
+  private async advanceFrontier(): Promise<Readonly<{
+    frontierPages: number;
+    frontierScanned: number;
+    frontierAdvanced: number;
+    frontierTruncated: boolean;
+  }>> {
+    let frontierPages = 0;
+    let frontierScanned = 0;
+    let frontierAdvanced = 0;
+    let frontierTruncated = false;
+    let after: PluginPackageWorkflowFrontierCursor | undefined;
+    for (
+      let index = 0;
+      index < this.frontierMaxPages;
+      index += 1
+    ) {
+      const page = await this.frontier.listCandidates({
+        limit: this.frontierPageSize,
+        ...(after === undefined ? {} : { after }),
+      });
+      if (page.candidates.length > this.frontierPageSize) {
+        throw new RangeError(
+          'Local Workflow frontier exceeded its page size',
+        );
+      }
+      frontierPages += 1;
+      frontierScanned += page.candidates.length;
+      for (const candidate of page.candidates) {
+        await this.frontier.advance(candidate.runId);
+        frontierAdvanced += 1;
+      }
+      frontierTruncated = page.truncated;
+      if (!page.truncated) break;
+      after = nextFrontierCursor(after, page.next);
+    }
+    return Object.freeze({
+      frontierPages,
+      frontierScanned,
+      frontierAdvanced,
+      frontierTruncated,
+    });
+  }
+
+  private async admitTaskAttempts(): Promise<Readonly<{
+    taskAttemptPages: number;
+    taskAttemptsScanned: number;
+    taskAttemptsCreated: number;
+    taskAttemptsExisting: number;
+    taskAttemptsTruncated: boolean;
+  }>> {
+    let taskAttemptPages = 0;
+    let taskAttemptsScanned = 0;
+    let taskAttemptsCreated = 0;
+    let taskAttemptsExisting = 0;
+    let taskAttemptsTruncated = false;
+    let after:
+      | PluginPackageWorkflowTaskAttemptAdmissionCursor
+      | undefined;
+    for (
+      let index = 0;
+      index < this.taskAttemptMaxPages;
+      index += 1
+    ) {
+      const page = await this.taskAttempts.listCandidates({
+        limit: this.taskAttemptPageSize,
+        ...(after === undefined ? {} : { after }),
+      });
+      if (page.candidates.length > this.taskAttemptPageSize) {
+        throw new RangeError(
+          'Local Workflow Task Attempt source exceeded its page size',
+        );
+      }
+      taskAttemptPages += 1;
+      taskAttemptsScanned += page.candidates.length;
+      for (const candidate of page.candidates) {
+        const admitted = await this.taskAttempts.admit(
+          candidate.runId,
+          candidate.stepRunId,
+        );
+        if (admitted.status === 'created') taskAttemptsCreated += 1;
+        else taskAttemptsExisting += 1;
+      }
+      taskAttemptsTruncated = page.truncated;
+      if (!page.truncated) break;
+      after = nextTaskAttemptCursor(after, page.next);
+    }
+    return Object.freeze({
+      taskAttemptPages,
+      taskAttemptsScanned,
+      taskAttemptsCreated,
+      taskAttemptsExisting,
+      taskAttemptsTruncated,
+    });
+  }
+
+  private async dispatch(): Promise<Readonly<{
+    dispatches: number;
+    activated: number;
+    activationFailed: number;
+    dispatchIdle: boolean;
+  }>> {
+    let dispatches = 0;
+    let activated = 0;
+    let activationFailed = 0;
+    let dispatchIdle = false;
+    for (let index = 0; index < this.maxDispatches; index += 1) {
+      const result: LocalRunDispatcherResult =
+        await this.dispatcher.dispatchOnce();
+      dispatches += 1;
+      if (result.status === 'activated') activated += 1;
+      else if (result.status === 'activation_failed') {
+        activationFailed += 1;
+      } else {
+        dispatchIdle = true;
+        break;
+      }
+    }
+    return Object.freeze({
+      dispatches,
+      activated,
+      activationFailed,
+      dispatchIdle,
+    });
+  }
+}

packages/ql3-local-execution/test/control.test.cjs

@@ -0,0 +1,437 @@
+const assert = require('node:assert/strict');
+const { createHash } = require('node:crypto');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { test } = require('node:test');
+const {
+  migrateLocalSqlitePath,
+  openLocalSqliteRuntimeDatabase,
+} = require('@qinglong/local-sqlite');
+const {
+  LocalCompletionReceiptProcessor,
+  LocalExecutionControlCoordinator,
+  LocalExecutionControlLifecycle,
+  LocalExecutionControlScanner,
+} = require('../dist/control');
+
+const IDS = Object.freeze({
+  completionRun: '019f7130-0000-7000-8000-000000000001',
+  completionAttempt: '019f7130-0000-7000-8000-000000000002',
+  deadlineRun: '019f7130-0000-7000-8000-000000000003',
+  deadlineAttempt: '019f7130-0000-7000-8000-000000000004',
+  cancelRun: '019f7130-0000-7000-8000-000000000005',
+  cancelAttempt: '019f7130-0000-7000-8000-000000000006',
+  shutdownRun: '019f7130-0000-7000-8000-000000000007',
+  shutdownAttempt: '019f7130-0000-7000-8000-000000000008',
+});
+const TOKEN = 'A'.repeat(32);
+
+function eventIds() {
+  let value = 0;
+  return () => `control-event-${++value}`;
+}
+
+function receiptStore() {
+  const receipts = new Map();
+  return {
+    receipts,
+    removed: [],
+    async publish(receipt) {
+      receipts.set(receipt.attemptId, receipt);
+    },
+    async read(attemptId) {
+      return receipts.get(attemptId);
+    },
+    async remove(attemptId) {
+      this.removed.push(attemptId);
+      return receipts.delete(attemptId);
+    },
+  };
+}
+
+async function fixture(t) {
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-control-'));
+  const databasePath = path.join(directory, 'qinglong3.sqlite');
+  await migrateLocalSqlitePath({ databasePath, profile: 'edge' });
+  const runtime = await openLocalSqliteRuntimeDatabase({
+    databasePath,
+    profile: 'edge',
+  });
+  t.after(async () => {
+    await runtime.close();
+    fs.rmSync(directory, { recursive: true, force: true });
+  });
+  return runtime;
+}
+
+async function insertActive(runtime, options) {
+  await runtime.runRepository.transaction(async (transaction) => {
+    await transaction.insertRun({
+      id: options.runId,
+      projectId: 'default',
+      taskId: 'task-1',
+      taskRevision: 'revision-1',
+      triggerType: 'manual',
+      executionOrigin: 'manual',
+      executionOwner: 'runtime',
+      status: options.runStatus || 'running',
+      version: 0,
+      eventSequence: 0,
+      priority: 0,
+      createdAtMs: 1,
+      startedAtMs: 2,
+      ...(options.cancelRequestedAtMs === undefined
+        ? {}
+        : {
+            cancelRequestedAtMs: options.cancelRequestedAtMs,
+            cancelReason: options.cancelReason,
+          }),
+    });
+    await transaction.insertAttempt({
+      id: options.attemptId,
+      runId: options.runId,
+      attempt: 1,
+      status: options.attemptStatus || 'running',
+      executorType: 'local_process',
+      executorHandle: `handle:${options.attemptId}`,
+      pid: 1234,
+      callbackTokenHash: createHash('sha256').update(TOKEN).digest('hex'),
+      callbackSequence: options.callbackSequence || 0,
+      createdAtMs: 1,
+      startedAtMs: 2,
+      ...(options.deadlineAtMs === undefined
+        ? {}
+        : { deadlineAtMs: options.deadlineAtMs }),
+    });
+  });
+}
+
+test('authenticates one receipt and commits Attempt then Run terminal facts', async (t) => {
+  const runtime = await fixture(t);
+  const store = receiptStore();
+  await insertActive(runtime, {
+    runId: IDS.completionRun,
+    attemptId: IDS.completionAttempt,
+  });
+  store.receipts.set(IDS.completionAttempt, {
+    schemaVersion: 1,
+    runId: IDS.completionRun,
+    attemptId: IDS.completionAttempt,
+    callbackSequence: 1,
+    token: TOKEN,
+    startedAtMs: 2,
+    finishedAtMs: 10,
+    exitCode: 0,
+  });
+  const processor = new LocalCompletionReceiptProcessor(
+    runtime.runRepository,
+    store,
+    { clock: { now: () => 10 }, createEventId: eventIds() },
+  );
+  assert.equal(await processor.process(IDS.completionAttempt), 'completed');
+  assert.equal(
+    (await runtime.runRepository.findRunById(IDS.completionRun)).status,
+    'succeeded',
+  );
+  assert.equal(
+    (await runtime.runRepository.findAttemptById(IDS.completionAttempt)).status,
+    'succeeded',
+  );
+  assert.deepEqual(
+    (await runtime.runRepository.listEvents(IDS.completionRun)).map(
+      (event) => event.type,
+    ),
+    ['attempt.succeeded', 'run.succeeded'],
+  );
+  assert.deepEqual(store.removed, [IDS.completionAttempt]);
+});
+
+test('discovers due deadlines and cancellation intents through stable SQLite pages', async (t) => {
+  const runtime = await fixture(t);
+  await insertActive(runtime, {
+    runId: IDS.deadlineRun,
+    attemptId: IDS.deadlineAttempt,
+    deadlineAtMs: 50,
+  });
+  await insertActive(runtime, {
+    runId: IDS.cancelRun,
+    attemptId: IDS.cancelAttempt,
+    cancelRequestedAtMs: 40,
+    cancelReason: 'user',
+    deadlineAtMs: 30,
+  });
+  const first =
+    await runtime.executionControl.listLocalExecutionControlCandidates({
+      observedAtMs: 100,
+      limit: 1,
+    });
+  assert.equal(first.truncated, true);
+  assert.deepEqual(first.candidates[0], {
+    kind: 'cancellation',
+    runId: IDS.cancelRun,
+    attemptId: IDS.cancelAttempt,
+    dueAtMs: 40,
+    cancelReason: 'user',
+  });
+  const second =
+    await runtime.executionControl.listLocalExecutionControlCandidates({
+      observedAtMs: 100,
+      limit: 1,
+      after: first.nextCursor,
+    });
+  assert.equal(second.truncated, false);
+  assert.deepEqual(second.candidates[0], {
+    kind: 'deadline',
+    runId: IDS.deadlineRun,
+    attemptId: IDS.deadlineAttempt,
+    dueAtMs: 50,
+  });
+});
+
+test('turns deadline and user cancellation into exact terminal aggregates', async (t) => {
+  const runtime = await fixture(t);
+  await insertActive(runtime, {
+    runId: IDS.deadlineRun,
+    attemptId: IDS.deadlineAttempt,
+    deadlineAtMs: 50,
+  });
+  await insertActive(runtime, {
+    runId: IDS.cancelRun,
+    attemptId: IDS.cancelAttempt,
+    cancelRequestedAtMs: 40,
+    cancelReason: 'user',
+  });
+  const stopped = [];
+  const store = receiptStore();
+  const completion = new LocalCompletionReceiptProcessor(
+    runtime.runRepository,
+    store,
+    { clock: { now: () => 100 }, createEventId: eventIds() },
+  );
+  const coordinator = new LocalExecutionControlCoordinator(
+    runtime.runRepository,
+    completion,
+    {
+      async stop(handle) {
+        stopped.push(handle);
+        return { status: 'stopped', signal: 'SIGTERM' };
+      },
+    },
+    { clock: { now: () => 100 }, createEventId: eventIds() },
+  );
+  const scanner = new LocalExecutionControlScanner(
+    runtime.executionControl,
+    coordinator,
+    { now: () => 100 },
+  );
+  const summary = await scanner.scan({ limit: 8 });
+  assert.equal(summary.scanned, 2);
+  assert.equal(summary.terminal, 2);
+  assert.equal(
+    (await runtime.runRepository.findRunById(IDS.deadlineRun)).status,
+    'timed_out',
+  );
+  assert.equal(
+    (await runtime.runRepository.findRunById(IDS.cancelRun)).status,
+    'cancelled',
+  );
+  assert.equal(stopped.length, 2);
+});
+
+test('routes a Workflow Task deadline to Step scope without cancelling its parent', async () => {
+  const run = {
+    id: 'workflow-run',
+    projectId: 'default',
+    taskId: 'workflow',
+    taskRevision: 'revision-1',
+    triggerType: 'system',
+    executionOrigin: 'system',
+    executionOwner: 'runtime',
+    status: 'running',
+    version: 7,
+    eventSequence: 7,
+    priority: 0,
+    createdAtMs: 1,
+    startedAtMs: 2,
+  };
+  const attempt = {
+    id: 'workflow-attempt',
+    runId: run.id,
+    stepRunId: 'workflow-step',
+    attempt: 1,
+    status: 'running',
+    executorType: 'local_process',
+    executorHandle: 'workflow-handle',
+    pid: 321,
+    callbackTokenHash: 'a'.repeat(64),
+    callbackSequence: 0,
+    deadlineAtMs: 50,
+    createdAtMs: 1,
+    startedAtMs: 2,
+  };
+  let latestReads = 0;
+  const repository = {
+    transaction(work) {
+      return work({
+        findRunById: async (runId) => (runId === run.id ? run : null),
+        findAttemptById: async (attemptId) =>
+          attemptId === attempt.id ? attempt : null,
+        findLatestAttemptByRunId: async () => {
+          latestReads += 1;
+          return attempt;
+        },
+      });
+    },
+  };
+  const workflowCalls = [];
+  const coordinator = new LocalExecutionControlCoordinator(
+    repository,
+    { process: async () => 'missing' },
+    { stop: async () => ({ status: 'stopped', signal: 'SIGTERM' }) },
+    {
+      clock: { now: () => 100 },
+      createEventId: eventIds(),
+      workflowTasks: {
+        async requestTimeout(command) {
+          workflowCalls.push(['timeout', command]);
+          return 'requested';
+        },
+        async recordControlTerminal(command) {
+          workflowCalls.push(['terminal', command]);
+          return 'terminal';
+        },
+      },
+    },
+  );
+
+  assert.equal(
+    await coordinator.process({
+      kind: 'deadline',
+      runId: run.id,
+      attemptId: attempt.id,
+      dueAtMs: 50,
+    }),
+    'terminal',
+  );
+  assert.equal(latestReads, 0);
+  assert.equal(workflowCalls[0][0], 'timeout');
+  assert.equal(workflowCalls[1][0], 'terminal');
+  assert.equal(workflowCalls[1][1].terminalStatus, 'timed_out');
+  assert.equal(run.cancelRequestedAtMs, undefined);
+});
+
+test('shutdown drain requests shutdown cancellation before stopping work', async (t) => {
+  const runtime = await fixture(t);
+  await insertActive(runtime, {
+    runId: IDS.shutdownRun,
+    attemptId: IDS.shutdownAttempt,
+  });
+  const completion = new LocalCompletionReceiptProcessor(
+    runtime.runRepository,
+    receiptStore(),
+    { clock: { now: () => 100 }, createEventId: eventIds() },
+  );
+  const coordinator = new LocalExecutionControlCoordinator(
+    runtime.runRepository,
+    completion,
+    { stop: async () => ({ status: 'already_exited' }) },
+    { clock: { now: () => 100 }, createEventId: eventIds() },
+  );
+  const scanner = new LocalExecutionControlScanner(
+    runtime.executionControl,
+    coordinator,
+    { now: () => 100 },
+  );
+  const summary = await scanner.drain({ limit: 4, maxPages: 2 });
+  assert.deepEqual(summary, {
+    scanned: 1,
+    terminal: 1,
+    remaining: 0,
+    failed: 0,
+    truncated: false,
+  });
+  const run = await runtime.runRepository.findRunById(IDS.shutdownRun);
+  assert.equal(run.status, 'cancelled');
+  assert.equal(run.cancelReason, 'shutdown');
+  assert.deepEqual(
+    (await runtime.runRepository.listEvents(IDS.shutdownRun)).map(
+      (event) => event.type,
+    ),
+    ['run.cancel_requested', 'attempt.cancelled', 'run.cancelled'],
+  );
+});
+
+test('coalesces completion notifications and owns one idempotent shutdown drain', async () => {
+  const completions = [];
+  let scans = 0;
+  let drains = 0;
+  let cleanups = 0;
+  const lifecycle = new LocalExecutionControlLifecycle(
+    {
+      async process(attemptId) {
+        completions.push(attemptId);
+        return 'missing';
+      },
+    },
+    {
+      async scan() {
+        scans += 1;
+        return {
+          scanned: 0,
+          terminal: 0,
+          cancelRequested: 0,
+          stale: 0,
+          remaining: 0,
+          failed: 0,
+          truncated: false,
+        };
+      },
+      async drain() {
+        drains += 1;
+        return {
+          scanned: 0,
+          terminal: 0,
+          remaining: 0,
+          failed: 0,
+          truncated: false,
+        };
+      },
+    },
+    {
+      async scan() {
+        cleanups += 1;
+        return {
+          scanned: 0,
+          removed: 0,
+          expiredMissing: 0,
+          purgedQuarantines: 0,
+          remaining: 0,
+          failed: 0,
+          truncated: false,
+        };
+      },
+    },
+    {
+      intervalMs: 60_000,
+      pageSize: 4,
+      cleanupIntervalMs: 60_000,
+      cleanupPageSize: 4,
+      stopTimeoutMs: 1_000,
+      maxDrainPages: 1,
+      clock: { now: () => 100 },
+    },
+  );
+  assert.equal(lifecycle.notifyCompletion(IDS.completionAttempt), true);
+  assert.equal(lifecycle.notifyCompletion(IDS.completionAttempt), true);
+  await new Promise((resolve) => setImmediate(resolve));
+  assert.deepEqual(completions, [IDS.completionAttempt]);
+  assert.equal(scans, 1);
+  assert.equal(cleanups, 1);
+  const first = lifecycle.stopAndDrain();
+  const second = lifecycle.stopAndDrain();
+  assert.equal(first, second);
+  assert.equal((await first).status, 'stopped');
+  assert.equal(drains, 1);
+  assert.equal(cleanups, 2);
+});

packages/ql3-local-execution/test/dispatch.test.cjs

@@ -0,0 +1,232 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { test } = require('node:test');
+const {
+  createLocalExecutionContextRecipe,
+  createLocalTaskExecutionRevision,
+} = require('@qinglong/runtime-core/local-dispatch');
+const {
+  migrateLocalSqlitePath,
+  openLocalSqliteRuntimeDatabase,
+} = require('@qinglong/local-sqlite');
+const { LocalExecutionCoordinator } = require('../dist/execution');
+const { createLocalProcessDurableHandle } = require('@qinglong/local-process');
+const {
+  LocalArtifactCapacityUnavailableError,
+  LocalDispatchPlanMaterializer,
+  LocalFileArtifactAllocator,
+  LocalRunDispatcher,
+  localArtifactCapacityPolicyForProfile,
+} = require('../dist/dispatch');
+
+const RUN_ID = '019f7120-0000-7000-8000-000000000001';
+const ATTEMPT_ID = '019f7120-0000-7000-8000-000000000002';
+
+function eventIdFactory() {
+  let value = 0;
+  return () => `dispatch-event-${++value}`;
+}
+
+function processHandle() {
+  const identity = {
+    platform: 'linux',
+    bootId: '11111111-2222-3333-4444-555555555555',
+    pid: 4321,
+    processGroupId: 4321,
+    startTimeTicks: '123456',
+  };
+  return Object.freeze({
+    handleId: 'dispatch-handle',
+    pid: 4321,
+    durableHandle: createLocalProcessDurableHandle('dispatch-handle', identity),
+    startedAtMs: 10,
+    completion: Promise.resolve({ exitCode: 0, signal: null }),
+  });
+}
+
+async function fixture(t) {
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-dispatch-'));
+  const databasePath = path.join(directory, 'qinglong3.sqlite');
+  const artifactRoot = path.join(directory, 'artifacts');
+  await migrateLocalSqlitePath({ databasePath, profile: 'edge' });
+  const runtime = await openLocalSqliteRuntimeDatabase({
+    databasePath,
+    profile: 'edge',
+  });
+  t.after(async () => {
+    await runtime.close();
+    fs.rmSync(directory, { recursive: true, force: true });
+  });
+  const recipe = createLocalExecutionContextRecipe({
+    environment: [
+      { name: 'PUBLIC_VALUE', kind: 'public', value: 'public-value' },
+      { name: 'SECRET_VALUE', kind: 'secret', secretRef: 'secret-ref-1' },
+    ],
+    createdAtMs: 1,
+  });
+  assert.equal(
+    await runtime.localDispatch.appendLocalExecutionContextRecipe(recipe),
+    'inserted',
+  );
+  assert.equal(
+    await runtime.localDispatch.appendLocalExecutionContextRecipe(recipe),
+    'existing',
+  );
+  const revision = createLocalTaskExecutionRevision({
+    projectId: 'default',
+    taskId: 'task-1',
+    taskRevision: 'revision-1',
+    executorType: 'local_process',
+    command: { kind: 'argv', file: '/bin/echo', args: ['hello'] },
+    timeoutMs: 1_000,
+    contextRef: recipe.contextRef,
+    createdAtMs: 1,
+  });
+  assert.equal(
+    await runtime.localDispatch.appendLocalTaskExecutionRevision(revision),
+    'inserted',
+  );
+  assert.equal(
+    await runtime.localDispatch.appendLocalTaskExecutionRevision(revision),
+    'existing',
+  );
+  await runtime.runRepository.transaction(async (transaction) => {
+    await transaction.insertRun({
+      id: RUN_ID,
+      projectId: 'default',
+      taskId: 'task-1',
+      taskRevision: 'revision-1',
+      triggerType: 'manual',
+      executionOrigin: 'manual',
+      executionOwner: 'runtime',
+      status: 'queued',
+      version: 0,
+      eventSequence: 0,
+      priority: 10,
+      createdAtMs: 1,
+      queuedAtMs: 1,
+    });
+    await transaction.insertAttempt({
+      id: ATTEMPT_ID,
+      runId: RUN_ID,
+      attempt: 1,
+      status: 'claimed',
+      executorType: 'local_process',
+      callbackSequence: 0,
+      createdAtMs: 1,
+    });
+  });
+  return { directory, databasePath, artifactRoot, runtime };
+}
+
+test('materializes pinned context and atomically admits one real SQLite candidate', async (t) => {
+  const value = await fixture(t);
+  let launchRequest;
+  const completionNotifications = [];
+  const execution = new LocalExecutionCoordinator(
+    value.runtime.runRepository,
+    {
+      async start(request) {
+        launchRequest = request;
+        assert.equal(request.environment.PUBLIC_VALUE, 'public-value');
+        assert.equal(request.environment.SECRET_VALUE, 'top-secret');
+        assert.equal(request.output.maximumBytes, 4 * 1024 * 1024);
+        assert.match(request.output.logArtifactId, /^local-[0-9a-f]{30}$/);
+        return processHandle();
+      },
+    },
+    { stop: async () => assert.fail('valid launch must not be stopped') },
+    {
+      clock: { now: () => 10 },
+      createEventId: eventIdFactory(),
+      createCallbackToken: () => 'A'.repeat(32),
+    },
+  );
+  const dispatcher = new LocalRunDispatcher(
+    value.runtime.localDispatch,
+    new LocalDispatchPlanMaterializer(
+      value.runtime.localDispatch,
+      new LocalFileArtifactAllocator(
+        value.artifactRoot,
+        localArtifactCapacityPolicyForProfile('edge'),
+      ),
+      {
+        async resolveLocalSecretEnvironment(request) {
+          assert.deepEqual(request.secretRefs, ['secret-ref-1']);
+          assert.equal(request.candidate.projectId, 'default');
+          return ['top-secret'];
+        },
+      },
+    ),
+    execution,
+    {
+      pageSize: 4,
+      maxPages: 1,
+      onCompletion: (attemptId) => completionNotifications.push(attemptId),
+    },
+  );
+
+  const result = await dispatcher.dispatchOnce();
+  assert.equal(result.status, 'activated');
+  assert.equal(result.runId, RUN_ID);
+  assert.equal(result.attemptId, ATTEMPT_ID);
+  await new Promise((resolve) => setImmediate(resolve));
+  assert.deepEqual(completionNotifications, [ATTEMPT_ID]);
+  assert.equal(result.stats.candidatesScanned, 1);
+  assert.equal(
+    (await value.runtime.runRepository.findRunById(RUN_ID)).status,
+    'running',
+  );
+  const attempt = await value.runtime.runRepository.findAttemptById(ATTEMPT_ID);
+  assert.equal(attempt.status, 'running');
+  assert.equal(attempt.logArtifactId, launchRequest.output.logArtifactId);
+  assert.equal(fs.statSync(launchRequest.output.filePath).mode & 0o777, 0o600);
+  assert.equal(fs.statSync(launchRequest.output.filePath).size, 0);
+  assert.equal(
+    fs.readFileSync(value.databasePath).includes(Buffer.from('top-secret')),
+    false,
+  );
+});
+
+test('missing Secret capability returns unavailable before Artifact allocation', async (t) => {
+  const value = await fixture(t);
+  const dispatcher = new LocalRunDispatcher(
+    value.runtime.localDispatch,
+    new LocalDispatchPlanMaterializer(
+      value.runtime.localDispatch,
+      new LocalFileArtifactAllocator(
+        value.artifactRoot,
+        localArtifactCapacityPolicyForProfile('edge'),
+      ),
+    ),
+    { start: async () => assert.fail('unavailable plan must not activate') },
+  );
+  const result = await dispatcher.dispatchOnce();
+  assert.equal(result.status, 'idle');
+  assert.equal(result.reason, 'plans_unavailable');
+  assert.equal(result.stats.plansUnavailable, 1);
+  assert.equal(fs.existsSync(value.artifactRoot), false);
+  assert.equal(
+    (await value.runtime.runRepository.findRunById(RUN_ID)).status,
+    'queued',
+  );
+});
+
+test('capacity admission fails before creating an Artifact file', async (t) => {
+  const value = await fixture(t);
+  const [candidate] = (
+    await value.runtime.localDispatch.listLocalDispatchCandidates({ limit: 1 })
+  ).candidates;
+  const allocator = new LocalFileArtifactAllocator(
+    value.artifactRoot,
+    localArtifactCapacityPolicyForProfile('edge'),
+    { inspect: async () => 0n },
+  );
+  await assert.rejects(
+    allocator.prepare(candidate),
+    LocalArtifactCapacityUnavailableError,
+  );
+  assert.deepEqual(fs.readdirSync(value.artifactRoot), []);
+});

packages/ql3-local-execution/test/execution.test.cjs

@@ -0,0 +1,264 @@
+const assert = require('node:assert/strict');
+const { createHash } = require('node:crypto');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { test } = require('node:test');
+const {
+  migrateLocalSqlitePath,
+  openLocalSqliteRuntimeDatabase,
+} = require('@qinglong/local-sqlite');
+const {
+  LocalExecutionCoordinator,
+  LocalExecutionLaunchError,
+  LocalExecutionOwnershipPersistenceError,
+  LocalExecutionRejectedError,
+} = require('../dist/execution');
+const { createLocalProcessDurableHandle } = require('@qinglong/local-process');
+
+const RUN_ID = '019f70d0-0000-7000-8000-000000000001';
+const ATTEMPT_ID = '019f70d0-0000-7000-8000-000000000002';
+const CALLBACK_TOKEN = 'A'.repeat(32);
+
+async function fixture(t) {
+  const directory = fs.mkdtempSync(
+    path.join(os.tmpdir(), 'ql3-local-execution-'),
+  );
+  const options = {
+    databasePath: path.join(directory, 'qinglong3.sqlite'),
+    profile: 'edge',
+  };
+  await migrateLocalSqlitePath(options);
+  const runtime = await openLocalSqliteRuntimeDatabase(options);
+  t.after(async () => {
+    await runtime.close();
+    fs.rmSync(directory, { recursive: true, force: true });
+  });
+  await runtime.runRepository.transaction(async (transaction) => {
+    await transaction.insertRun({
+      id: RUN_ID,
+      projectId: 'default',
+      taskId: 'task-1',
+      taskRevision: 'revision-1',
+      triggerType: 'manual',
+      executionOrigin: 'manual',
+      executionOwner: 'runtime',
+      status: 'queued',
+      version: 0,
+      eventSequence: 0,
+      priority: 0,
+      createdAtMs: 1,
+      queuedAtMs: 1,
+    });
+    await transaction.insertAttempt({
+      id: ATTEMPT_ID,
+      runId: RUN_ID,
+      attempt: 1,
+      status: 'claimed',
+      executorType: 'local_process',
+      callbackSequence: 0,
+      createdAtMs: 1,
+    });
+  });
+  return runtime.runRepository;
+}
+
+function eventIdFactory() {
+  let value = 0;
+  return () => `event-${++value}`;
+}
+
+function handle() {
+  const identity = {
+    platform: 'linux',
+    bootId: '11111111-2222-3333-4444-555555555555',
+    pid: 1234,
+    processGroupId: 1234,
+    startTimeTicks: '987654',
+  };
+  return Object.freeze({
+    handleId: 'handle-1',
+    pid: 1234,
+    durableHandle: createLocalProcessDurableHandle('handle-1', identity),
+    startedAtMs: 10,
+    completion: Promise.resolve({ exitCode: 0, signal: null }),
+  });
+}
+
+function command() {
+  return {
+    runId: RUN_ID,
+    attemptId: ATTEMPT_ID,
+    command: { kind: 'argv', file: '/bin/echo', args: ['hello'] },
+    timeoutMs: 1_000,
+  };
+}
+
+function repositoryFailingTransaction(repository, transactionNumber) {
+  let transactions = 0;
+  return {
+    findRunById: (...args) => repository.findRunById(...args),
+    findAttemptById: (...args) => repository.findAttemptById(...args),
+    findLatestAttemptByRunId: (...args) =>
+      repository.findLatestAttemptByRunId(...args),
+    findRetryPolicyByRunId: (...args) =>
+      repository.findRetryPolicyByRunId(...args),
+    listEvents: (...args) => repository.listEvents(...args),
+    listCancellationRequested: (...args) =>
+      repository.listCancellationRequested(...args),
+    transaction(work) {
+      transactions += 1;
+      if (transactions === transactionNumber) {
+        return Promise.reject(new Error('injected transaction failure'));
+      }
+      return repository.transaction(work);
+    },
+  };
+}
+
+test('atomically persists claimed -> starting -> running around launch', async (t) => {
+  const repository = await fixture(t);
+  let launchRequest;
+  let clockReads = 0;
+  const coordinator = new LocalExecutionCoordinator(
+    repository,
+    {
+      async start(request) {
+        launchRequest = request;
+        const [run, attempt] = await Promise.all([
+          repository.findRunById(RUN_ID),
+          repository.findAttemptById(ATTEMPT_ID),
+        ]);
+        assert.equal(run.status, 'dispatching');
+        assert.equal(run.version, 2);
+        assert.equal(attempt.status, 'starting');
+        assert.equal(
+          attempt.callbackTokenHash,
+          createHash('sha256').update(CALLBACK_TOKEN).digest('hex'),
+        );
+        return handle();
+      },
+    },
+    {
+      stop: async () => assert.fail('controller must not stop a valid launch'),
+    },
+    {
+      clock: { now: () => (clockReads++ === 0 ? 5 : 20) },
+      createEventId: eventIdFactory(),
+      createCallbackToken: () => CALLBACK_TOKEN,
+    },
+  );
+
+  const result = await coordinator.start(command());
+  assert.equal(launchRequest.callbackToken, CALLBACK_TOKEN);
+  assert.equal(launchRequest.callbackSequence, 1);
+  assert.equal(result.run.status, 'running');
+  assert.equal(result.run.version, 4);
+  assert.equal(result.attempt.status, 'running');
+  assert.equal(result.attempt.executorHandle, handle().durableHandle);
+  assert.equal(result.attempt.pid, 1234);
+  assert.equal(result.attempt.startedAtMs, handle().startedAtMs);
+  assert.equal(result.attempt.callbackSequence, 0);
+  assert.equal('callbackToken' in result, false);
+  assert.deepEqual(
+    (await repository.listEvents(RUN_ID)).map((event) => event.type),
+    ['run.dispatching', 'attempt.starting', 'attempt.running', 'run.running'],
+  );
+});
+
+test('records a pre-ownership launch failure as failed', async (t) => {
+  const repository = await fixture(t);
+  const coordinator = new LocalExecutionCoordinator(
+    repository,
+    { start: async () => Promise.reject(new Error('spawn failed')) },
+    { stop: async () => assert.fail('no durable handle exists') },
+    {
+      clock: { now: () => 10 },
+      createEventId: eventIdFactory(),
+      createCallbackToken: () => CALLBACK_TOKEN,
+    },
+  );
+  await assert.rejects(coordinator.start(command()), LocalExecutionLaunchError);
+  assert.equal((await repository.findRunById(RUN_ID)).status, 'failed');
+  assert.equal(
+    (await repository.findAttemptById(ATTEMPT_ID)).errorCode,
+    'EXECUTOR_START_FAILED',
+  );
+});
+
+test('stops exact process then records lost when running persistence fails', async (t) => {
+  const repository = await fixture(t);
+  const failing = repositoryFailingTransaction(repository, 2);
+  const stopped = [];
+  const coordinator = new LocalExecutionCoordinator(
+    failing,
+    { start: async () => handle() },
+    {
+      async stop(durableHandle) {
+        stopped.push(durableHandle);
+        return { status: 'stopped', signal: 'SIGTERM' };
+      },
+    },
+    {
+      clock: { now: () => 10 },
+      createEventId: eventIdFactory(),
+      createCallbackToken: () => CALLBACK_TOKEN,
+    },
+  );
+  await assert.rejects(coordinator.start(command()), (error) => {
+    assert.ok(error instanceof LocalExecutionOwnershipPersistenceError);
+    assert.equal(error.compensation.status, 'stopped');
+    return true;
+  });
+  assert.deepEqual(stopped, [handle().durableHandle]);
+  assert.equal((await repository.findRunById(RUN_ID)).status, 'lost');
+  assert.equal((await repository.findAttemptById(ATTEMPT_ID)).status, 'lost');
+});
+
+test('keeps starting authority for recovery when exact stop is inconclusive', async (t) => {
+  const repository = await fixture(t);
+  const failing = repositoryFailingTransaction(repository, 2);
+  const coordinator = new LocalExecutionCoordinator(
+    failing,
+    { start: async () => handle() },
+    {
+      stop: async () => ({
+        status: 'unknown',
+        reason: 'provider_unavailable',
+      }),
+    },
+    {
+      clock: { now: () => 10 },
+      createEventId: eventIdFactory(),
+      createCallbackToken: () => CALLBACK_TOKEN,
+    },
+  );
+  await assert.rejects(
+    coordinator.start(command()),
+    LocalExecutionOwnershipPersistenceError,
+  );
+  assert.equal((await repository.findRunById(RUN_ID)).status, 'dispatching');
+  assert.equal(
+    (await repository.findAttemptById(ATTEMPT_ID)).status,
+    'starting',
+  );
+});
+
+test('rejects replay after execution authority has moved', async (t) => {
+  const repository = await fixture(t);
+  const coordinator = new LocalExecutionCoordinator(
+    repository,
+    { start: async () => handle() },
+    { stop: async () => ({ status: 'already_exited' }) },
+    {
+      clock: { now: () => 10 },
+      createEventId: eventIdFactory(),
+      createCallbackToken: () => CALLBACK_TOKEN,
+    },
+  );
+  await coordinator.start(command());
+  await assert.rejects(
+    coordinator.start(command()),
+    LocalExecutionRejectedError,
+  );
+});

packages/ql3-local-execution/test/recovery.test.cjs

@@ -0,0 +1,554 @@
+const assert = require('node:assert/strict');
+const { createHash } = require('node:crypto');
+const { test } = require('node:test');
+const { InvalidCompletionReceiptError } = require('@qinglong/local-process');
+const {
+  LocalRunStartupRecoveryCoordinator,
+  LocalWorkflowTaskStartupRecoveryCoordinator,
+} = require('../dist/recovery');
+
+const RUN_ID = '019f70c0-0000-7000-8000-000000000001';
+const ATTEMPT_ID = '019f70c0-0000-7000-8000-000000000002';
+const TOKEN = 'A'.repeat(32);
+
+function run(overrides = {}) {
+  return {
+    id: RUN_ID,
+    projectId: 'default',
+    taskId: 'task-1',
+    taskRevision: 'revision-1',
+    triggerType: 'manual',
+    executionOrigin: 'manual',
+    executionOwner: 'runtime',
+    status: 'dispatching',
+    version: 0,
+    eventSequence: 0,
+    priority: 0,
+    createdAtMs: 1,
+    ...overrides,
+  };
+}
+
+function attempt(overrides = {}) {
+  return {
+    id: ATTEMPT_ID,
+    runId: RUN_ID,
+    attempt: 1,
+    status: 'claimed',
+    executorType: 'local_process',
+    callbackSequence: 0,
+    createdAtMs: 2,
+    ...overrides,
+  };
+}
+
+class MemoryRepository {
+  constructor(initialRun, initialAttempt) {
+    this.runs = new Map(initialRun ? [[initialRun.id, initialRun]] : []);
+    this.attempts = new Map(
+      initialAttempt ? [[initialAttempt.id, initialAttempt]] : [],
+    );
+    this.events = [];
+  }
+
+  async transaction(work) {
+    const previousRuns = structuredClone(this.runs);
+    const previousAttempts = structuredClone(this.attempts);
+    const previousEvents = structuredClone(this.events);
+    const transaction = {
+      findRunById: async (id) => structuredClone(this.runs.get(id) ?? null),
+      findAttemptById: async (id) =>
+        structuredClone(this.attempts.get(id) ?? null),
+      findLatestAttemptByRunId: async (runId) => {
+        const values = [...this.attempts.values()]
+          .filter((item) => item.runId === runId)
+          .sort((left, right) => right.attempt - left.attempt);
+        return structuredClone(values[0] ?? null);
+      },
+      compareAndSetRun: async (value, expectedVersion) => {
+        const current = this.runs.get(value.id);
+        if (!current || current.version !== expectedVersion) return false;
+        this.runs.set(value.id, structuredClone(value));
+        return true;
+      },
+      compareAndSetAttempt: async (value, expected) => {
+        const current = this.attempts.get(value.id);
+        if (
+          !current ||
+          current.status !== expected.status ||
+          current.callbackSequence !== expected.callbackSequence
+        ) {
+          return false;
+        }
+        this.attempts.set(value.id, structuredClone(value));
+        return true;
+      },
+      appendEvent: async (value) => this.events.push(structuredClone(value)),
+    };
+    try {
+      return await work(transaction);
+    } catch (error) {
+      this.runs = previousRuns;
+      this.attempts = previousAttempts;
+      this.events = previousEvents;
+      throw error;
+    }
+  }
+}
+
+function source(repository, overrides = {}) {
+  let calls = 0;
+  return {
+    get calls() {
+      return calls;
+    },
+    async inspectCandidates() {
+      calls += 1;
+      if (overrides.page) return overrides.page;
+      const candidates = [];
+      for (const value of [...repository.runs.values()].sort((a, b) =>
+        a.id.localeCompare(b.id),
+      )) {
+        if (
+          value.executionOwner !== 'runtime' ||
+          !['dispatching', 'running'].includes(value.status)
+        ) {
+          continue;
+        }
+        candidates.push({
+          runId: value.id,
+          runStatus: value.status,
+          activeAttemptCount: [...repository.attempts.values()].filter(
+            (item) =>
+              item.runId === value.id &&
+              ['claimed', 'starting', 'running'].includes(item.status),
+          ).length,
+        });
+      }
+      return { candidates, truncated: false };
+    },
+  };
+}
+
+function receipts(value) {
+  let readCount = 0;
+  let removed = 0;
+  return {
+    get readCount() {
+      return readCount;
+    },
+    get removed() {
+      return removed;
+    },
+    async read() {
+      readCount += 1;
+      return value;
+    },
+    async remove() {
+      removed += 1;
+      return true;
+    },
+  };
+}
+
+function coordinator(
+  repository,
+  candidateSource,
+  receiptStore,
+  inspect,
+  options = {},
+) {
+  return new LocalRunStartupRecoveryCoordinator(
+    repository,
+    candidateSource,
+    receiptStore,
+    { executorType: 'local_process', inspect },
+    {
+      clock: { now: () => 10 },
+      createEventId: (() => {
+        let value = 0;
+        return () => `event-${++value}`;
+      })(),
+      ...options,
+    },
+  );
+}
+
+test('zero candidates pay one durable read and no receipt or process work', async () => {
+  const repository = new MemoryRepository();
+  const candidateSource = source(repository);
+  const receiptStore = receipts(undefined);
+  let inspections = 0;
+  const summary = await coordinator(
+    repository,
+    candidateSource,
+    receiptStore,
+    async () => {
+      inspections += 1;
+      return { status: 'running', identityPid: 1 };
+    },
+  ).recover();
+
+  assert.deepEqual(summary, {
+    safe: true,
+    scanned: 0,
+    recovered: 0,
+    remaining: 0,
+    failed: 0,
+    truncated: false,
+  });
+  assert.equal(candidateSource.calls, 1);
+  assert.equal(receiptStore.readCount, 0);
+  assert.equal(inspections, 0);
+});
+
+test('truncation fails before any partial mutation or evidence read', async () => {
+  const repository = new MemoryRepository(run(), attempt());
+  const candidateSource = source(repository, {
+    page: {
+      candidates: [
+        {
+          runId: RUN_ID,
+          runStatus: 'dispatching',
+          activeAttemptCount: 1,
+        },
+      ],
+      truncated: true,
+    },
+  });
+  const receiptStore = receipts(undefined);
+  const summary = await coordinator(
+    repository,
+    candidateSource,
+    receiptStore,
+    async () => {
+      throw new Error('must not inspect');
+    },
+  ).recover();
+
+  assert.equal(summary.safe, false);
+  assert.equal(summary.truncated, true);
+  assert.equal(repository.runs.get(RUN_ID).status, 'dispatching');
+  assert.equal(repository.attempts.get(ATTEMPT_ID).status, 'claimed');
+  assert.equal(repository.events.length, 0);
+  assert.equal(receiptStore.readCount, 0);
+});
+
+test('an unstarted claimed Attempt is atomically marked lost without probing', async () => {
+  const repository = new MemoryRepository(run(), attempt());
+  const candidateSource = source(repository);
+  const receiptStore = receipts(undefined);
+  let inspections = 0;
+  const summary = await coordinator(
+    repository,
+    candidateSource,
+    receiptStore,
+    async () => {
+      inspections += 1;
+      return { status: 'running', identityPid: 1 };
+    },
+  ).recover();
+
+  assert.equal(summary.safe, true);
+  assert.equal(summary.recovered, 1);
+  assert.equal(repository.runs.get(RUN_ID).status, 'lost');
+  assert.equal(repository.attempts.get(ATTEMPT_ID).status, 'lost');
+  assert.deepEqual(
+    repository.events.map((item) => item.type),
+    ['attempt.lost', 'run.lost'],
+  );
+  assert.equal(inspections, 0);
+});
+
+test('a trusted completion receipt wins before process inspection', async () => {
+  const callbackTokenHash = createHash('sha256').update(TOKEN).digest('hex');
+  const repository = new MemoryRepository(
+    run({ status: 'running', startedAtMs: 3 }),
+    attempt({
+      status: 'running',
+      startedAtMs: 3,
+      callbackTokenHash,
+      executorHandle: 'unused',
+      pid: 10,
+    }),
+  );
+  const candidateSource = source(repository);
+  const receiptStore = receipts({
+    schemaVersion: 1,
+    runId: RUN_ID,
+    attemptId: ATTEMPT_ID,
+    callbackSequence: 1,
+    token: TOKEN,
+    startedAtMs: 3,
+    finishedAtMs: 8,
+    exitCode: 0,
+  });
+  let inspections = 0;
+  const resolved = [];
+  const summary = await coordinator(
+    repository,
+    candidateSource,
+    receiptStore,
+    async () => {
+      inspections += 1;
+      return { status: 'running', identityPid: 10 };
+    },
+    {
+      journal: {
+        async markQuarantined() {
+          throw new Error('must not quarantine a trusted receipt');
+        },
+        async resolve(attemptId) {
+          resolved.push(attemptId);
+        },
+      },
+    },
+  ).recover();
+
+  assert.equal(summary.safe, true);
+  assert.equal(repository.runs.get(RUN_ID).status, 'succeeded');
+  assert.equal(repository.attempts.get(ATTEMPT_ID).status, 'succeeded');
+  assert.equal(repository.attempts.get(ATTEMPT_ID).callbackSequence, 1);
+  assert.deepEqual(
+    repository.events.map((item) => item.type),
+    ['attempt.succeeded', 'run.succeeded'],
+  );
+  assert.equal(receiptStore.removed, 1);
+  assert.deepEqual(resolved, [ATTEMPT_ID]);
+  assert.equal(inspections, 0);
+});
+
+test('an invalid receipt records durable quarantine intent before moving the file', async () => {
+  const repository = new MemoryRepository(
+    run({ status: 'running', startedAtMs: 3 }),
+    attempt({
+      status: 'running',
+      startedAtMs: 3,
+      executorHandle: 'handle-1',
+      pid: 10,
+    }),
+  );
+  const operations = [];
+  const receiptStore = {
+    async read() {
+      throw new InvalidCompletionReceiptError('invalid test receipt');
+    },
+    async remove() {
+      throw new Error('must not remove an invalid receipt');
+    },
+    async publish() {
+      throw new Error('must not publish during recovery');
+    },
+    quarantineReference(attemptId) {
+      return `.quarantine/${attemptId}.json`;
+    },
+    async quarantine(attemptId) {
+      operations.push(`file:${attemptId}`);
+      return `.quarantine/${attemptId}.json`;
+    },
+  };
+
+  const summary = await coordinator(
+    repository,
+    source(repository),
+    receiptStore,
+    async () => {
+      throw new Error('invalid receipt must block process inspection');
+    },
+    {
+      quarantineRetentionMs: 100,
+      journal: {
+        async markQuarantined(record) {
+          operations.push(`journal:${record.attemptId}`);
+          assert.deepEqual(record, {
+            attemptId: ATTEMPT_ID,
+            quarantineRef: `.quarantine/${ATTEMPT_ID}.json`,
+            updatedAtMs: 10,
+            purgeAfterMs: 110,
+          });
+        },
+        async resolve() {
+          throw new Error('must not resolve invalid receipt intent');
+        },
+      },
+    },
+  ).recover();
+
+  assert.equal(summary.safe, false);
+  assert.equal(summary.remaining, 1);
+  assert.deepEqual(operations, [`journal:${ATTEMPT_ID}`, `file:${ATTEMPT_ID}`]);
+});
+
+test('a live exact process is verified twice without terminalizing the Run', async () => {
+  const repository = new MemoryRepository(
+    run({ status: 'running', startedAtMs: 3 }),
+    attempt({
+      status: 'running',
+      startedAtMs: 3,
+      executorHandle: 'handle-1',
+      pid: 10,
+    }),
+  );
+  const candidateSource = source(repository);
+  const receiptStore = receipts(undefined);
+  let inspections = 0;
+  const summary = await coordinator(
+    repository,
+    candidateSource,
+    receiptStore,
+    async () => {
+      inspections += 1;
+      return { status: 'running', identityPid: 10 };
+    },
+  ).recover();
+
+  assert.equal(summary.safe, true);
+  assert.equal(summary.recovered, 1);
+  assert.equal(repository.runs.get(RUN_ID).status, 'running');
+  assert.equal(repository.attempts.get(ATTEMPT_ID).status, 'running');
+  assert.equal(repository.events.length, 0);
+  assert.equal(inspections, 2);
+  assert.equal(receiptStore.readCount, 2);
+});
+
+test('a process change during final verification revokes startup safety', async () => {
+  const repository = new MemoryRepository(
+    run({ status: 'running', startedAtMs: 3 }),
+    attempt({
+      status: 'running',
+      startedAtMs: 3,
+      executorHandle: 'handle-1',
+      pid: 10,
+    }),
+  );
+  let inspections = 0;
+  const summary = await coordinator(
+    repository,
+    source(repository),
+    receipts(undefined),
+    async () => {
+      inspections += 1;
+      return inspections === 1
+        ? { status: 'running', identityPid: 10 }
+        : { status: 'not_running', identityPid: 10 };
+    },
+  ).recover();
+
+  assert.deepEqual(summary, {
+    safe: false,
+    scanned: 1,
+    recovered: 0,
+    remaining: 1,
+    failed: 0,
+    truncated: false,
+  });
+  assert.equal(repository.runs.get(RUN_ID).status, 'running');
+});
+
+test('trusted not-running evidence marks the aggregate lost but unknown evidence blocks', async () => {
+  const activeRun = run({ status: 'running', startedAtMs: 3 });
+  const activeAttempt = attempt({
+    status: 'running',
+    startedAtMs: 3,
+    executorHandle: 'handle-1',
+    pid: 10,
+  });
+  const repository = new MemoryRepository(activeRun, activeAttempt);
+  const summary = await coordinator(
+    repository,
+    source(repository),
+    receipts(undefined),
+    async () => ({ status: 'not_running', identityPid: 10 }),
+  ).recover();
+  assert.equal(summary.safe, true);
+  assert.equal(repository.runs.get(RUN_ID).status, 'lost');
+
+  const unknownRepository = new MemoryRepository(activeRun, activeAttempt);
+  const unknown = await coordinator(
+    unknownRepository,
+    source(unknownRepository),
+    receipts(undefined),
+    async () => ({ status: 'unknown', reason: 'invalid_handle' }),
+  ).recover();
+  assert.equal(unknown.safe, false);
+  assert.equal(unknown.remaining, 1);
+  assert.equal(unknownRepository.runs.get(RUN_ID).status, 'running');
+  assert.equal(unknownRepository.events.length, 0);
+});
+
+test('recovers an orphaned claimed Workflow Task without terminalizing its parent', async () => {
+  const workflowRun = run({
+    taskId: 'workflow',
+    triggerType: 'plugin_package_workflow',
+    executionOrigin: 'system',
+    status: 'running',
+    version: 4,
+    eventSequence: 4,
+    startedAtMs: 2,
+  });
+  const workflowAttempt = attempt({
+    stepRunId: 'workflow-step',
+    status: 'claimed',
+    createdAtMs: 5,
+  });
+  const repository = new MemoryRepository(workflowRun, workflowAttempt);
+  let recovered = false;
+  let inspections = 0;
+  const recovery = {
+    async listRecoveryCandidates() {
+      return {
+        candidates: recovered
+          ? []
+          : [
+              {
+                runId: RUN_ID,
+                attemptId: ATTEMPT_ID,
+                attemptCreatedAtMs: 5,
+              },
+            ],
+        truncated: false,
+      };
+    },
+    async recover(command) {
+      assert.equal(command.reason, 'unstarted_claim_expired');
+      repository.attempts.set(ATTEMPT_ID, {
+        ...workflowAttempt,
+        status: 'lost',
+        finishedAtMs: command.observedAtMs,
+      });
+      recovered = true;
+      return 'requeued';
+    },
+  };
+  const coordinator = new LocalWorkflowTaskStartupRecoveryCoordinator(
+    repository,
+    recovery,
+    {
+      async recordRunning() {
+        throw new Error('claimed recovery must not mark running');
+      },
+    },
+    { process: async () => 'missing' },
+    {
+      executorType: 'local_process',
+      async inspect() {
+        inspections += 1;
+        throw new Error('claimed recovery must not inspect a process');
+      },
+    },
+    { clock: { now: () => 10 } },
+  );
+
+  assert.deepEqual(await coordinator.recover(), {
+    safe: true,
+    scanned: 1,
+    recovered: 1,
+    verified: 0,
+    remaining: 0,
+    failed: 0,
+    truncated: false,
+  });
+  assert.equal(repository.runs.get(RUN_ID).status, 'running');
+  assert.equal(repository.attempts.get(ATTEMPT_ID).status, 'lost');
+  assert.equal(inspections, 0);
+});

packages/ql3-local-execution/test/scheduler.test.cjs

@@ -0,0 +1,377 @@
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+const {
+  LocalSchedulerCoordinator,
+  LocalSchedulerLifecycle,
+  LocalWorkflowSchedulerCoordinator,
+} = require('../dist/scheduler');
+
+function nextMinute(schedule, afterMs) {
+  if (schedule.expression !== '* * * * *' || schedule.timezone !== 'UTC') {
+    throw new Error('unsupported test schedule');
+  }
+  return Math.floor(afterMs / 60_000 + 1) * 60_000;
+}
+
+function candidate(overrides = {}) {
+  return {
+    projectId: 'default',
+    triggerId: 'trigger-1',
+    triggerRevision: 1,
+    triggerContentDigest: 'a'.repeat(64),
+    triggerUpdatedAtMs: 1,
+    taskId: 'task-1',
+    taskRevision: 1,
+    taskContentDigest: 'b'.repeat(64),
+    expression: '* * * * *',
+    timezone: 'UTC',
+    misfirePolicy: 'skip',
+    stateVersion: 0,
+    nextFireAtMs: 60_000,
+    ...overrides,
+  };
+}
+
+test('coordinates one bounded page and notifies only committed admissions', async () => {
+  const committed = [];
+  const notified = [];
+  let sequence = 0;
+  const coordinator = new LocalSchedulerCoordinator(
+    {
+      async listLocalScheduleCandidates(options) {
+        assert.deepEqual(options, { observedAtMs: 61_000, limit: 4 });
+        return {
+          candidates: [candidate(), candidate({ triggerId: 'trigger-2' })],
+          truncated: true,
+        };
+      },
+      async commitLocalScheduleDecision(command) {
+        committed.push(command);
+        if (committed.length === 2) return { status: 'raced' };
+        return {
+          status: 'admitted',
+          disposition: 'admit',
+          runId: command.runId,
+          attemptId: command.attemptId,
+        };
+      },
+    },
+    {
+      pageSize: 4,
+      misfireGraceMs: 5_000,
+      clock: () => 61_000,
+      nextOccurrence: nextMinute,
+      createId: () =>
+        `019f7500-0000-4000-8000-${String(++sequence).padStart(12, '0')}`,
+      onAdmitted: (runId, attemptId) => notified.push([runId, attemptId]),
+    },
+  );
+
+  assert.deepEqual(await coordinator.scheduleOnce(), {
+    observedAtMs: 61_000,
+    scanned: 2,
+    initialized: 0,
+    skipped: 0,
+    admitted: 1,
+    raced: 1,
+    truncated: true,
+  });
+  assert.equal(committed.length, 2);
+  assert.deepEqual(notified, [[committed[0].runId, committed[0].attemptId]]);
+});
+
+test('does not allocate Run identities for initialization or skip decisions', async () => {
+  let allocations = 0;
+  const commands = [];
+  const coordinator = new LocalSchedulerCoordinator(
+    {
+      async listLocalScheduleCandidates() {
+        return {
+          candidates: [
+            candidate({ nextFireAtMs: null, triggerUpdatedAtMs: 90_000 }),
+            candidate({ triggerId: 'trigger-2', nextFireAtMs: 60_000 }),
+          ],
+          truncated: false,
+        };
+      },
+      async commitLocalScheduleDecision(command) {
+        commands.push(command);
+        return {
+          status: 'advanced',
+          disposition: command.decision.disposition,
+        };
+      },
+    },
+    {
+      clock: () => 100_000,
+      misfireGraceMs: 5_000,
+      nextOccurrence: nextMinute,
+      createId() {
+        allocations += 1;
+        return 'unused';
+      },
+    },
+  );
+  const summary = await coordinator.scheduleOnce();
+  assert.equal(summary.initialized, 1);
+  assert.equal(summary.skipped, 1);
+  assert.equal(allocations, 0);
+  assert.equal(
+    commands.every((command) => command.runId === undefined),
+    true,
+  );
+});
+
+test('reuses one scheduler cycle for cancellation, frontier, Task admission and dispatch', async () => {
+  const calls = [];
+  let dispatches = 0;
+  const schedulerSummary = {
+    observedAtMs: 10,
+    scanned: 0,
+    initialized: 0,
+    skipped: 0,
+    admitted: 0,
+    raced: 0,
+    truncated: false,
+  };
+  const coordinator = new LocalWorkflowSchedulerCoordinator(
+    {
+      async scheduleOnce() {
+        calls.push('schedule');
+        return schedulerSummary;
+      },
+    },
+    {
+      async convergePage(command) {
+        calls.push(`cancel:${command.limit}`);
+        return {
+          scanned: 0,
+          settledRuns: 0,
+          settledAttempts: 0,
+          blocked: 0,
+          hasMore: false,
+        };
+      },
+    },
+    {
+      async listCandidates(command) {
+        calls.push(`frontier-list:${command.limit}`);
+        return {
+          candidates: [
+            {
+              runId: 'workflow-run',
+              planDigest: 'a'.repeat(64),
+              admittedAtMs: 1,
+            },
+          ],
+          truncated: false,
+        };
+      },
+      async advance(runId) {
+        calls.push(`frontier-advance:${runId}`);
+        return {};
+      },
+    },
+    {
+      async listCandidates(command) {
+        calls.push(`task-list:${command.limit}`);
+        return {
+          candidates: [
+            {
+              runId: 'workflow-run',
+              stepRunId: 'workflow-step',
+              readyAtMs: 2,
+              planDigest: 'a'.repeat(64),
+            },
+          ],
+          truncated: false,
+        };
+      },
+      async admit(runId, stepRunId) {
+        calls.push(`task-admit:${runId}:${stepRunId}`);
+        return { status: 'created', receipt: {} };
+      },
+    },
+    {
+      async dispatchOnce() {
+        dispatches += 1;
+        calls.push(`dispatch:${dispatches}`);
+        const stats = {
+          pages: 1,
+          candidatesScanned: dispatches === 1 ? 1 : 0,
+          plansUnavailable: 0,
+          activationRaces: 0,
+        };
+        return dispatches === 1
+          ? {
+              status: 'activated',
+              runId: 'workflow-run',
+              attemptId: 'workflow-attempt',
+              stats,
+              truncated: false,
+            }
+          : {
+              status: 'idle',
+              reason: 'no_candidates',
+              stats,
+              truncated: false,
+            };
+      },
+    },
+    {
+      cancellationPageSize: 1,
+      cancellationMaxPages: 1,
+      frontierPageSize: 1,
+      frontierMaxPages: 1,
+      taskAttemptPageSize: 1,
+      taskAttemptMaxPages: 1,
+      maxDispatches: 2,
+    },
+  );
+
+  assert.deepEqual(await coordinator.scheduleOnce(), schedulerSummary);
+  assert.deepEqual(calls, [
+    'cancel:1',
+    'schedule',
+    'frontier-list:1',
+    'frontier-advance:workflow-run',
+    'task-list:1',
+    'task-admit:workflow-run:workflow-step',
+    'dispatch:1',
+    'dispatch:2',
+  ]);
+  assert.deepEqual(coordinator.latestWorkflowSummary(), {
+    cancellation: {
+      pages: 1,
+      scanned: 0,
+      settledRuns: 0,
+      settledAttempts: 0,
+      blocked: 0,
+      hasMore: false,
+      remaining: false,
+      stopReason: 'complete',
+    },
+    frontierPages: 1,
+    frontierScanned: 1,
+    frontierAdvanced: 1,
+    frontierTruncated: false,
+    taskAttemptPages: 1,
+    taskAttemptsScanned: 1,
+    taskAttemptsCreated: 1,
+    taskAttemptsExisting: 0,
+    taskAttemptsTruncated: false,
+    dispatches: 2,
+    activated: 1,
+    activationFailed: 0,
+    dispatchIdle: true,
+  });
+});
+
+test('lifecycle coalesces cycles and stops without leaving a scheduler timer', async () => {
+  let calls = 0;
+  let release;
+  const pending = new Promise((resolve) => {
+    release = resolve;
+  });
+  const summary = {
+    observedAtMs: 1,
+    scanned: 0,
+    initialized: 0,
+    skipped: 0,
+    admitted: 0,
+    raced: 0,
+    truncated: false,
+  };
+  const lifecycle = new LocalSchedulerLifecycle(
+    {
+      async scheduleOnce() {
+        calls += 1;
+        await pending;
+        return summary;
+      },
+    },
+    { intervalMs: 250, stopTimeoutMs: 1_000 },
+  );
+
+  assert.equal(lifecycle.start(), 'started');
+  const first = lifecycle.runOnce();
+  const second = lifecycle.runOnce();
+  assert.equal(first, second);
+  const stopping = lifecycle.stopAndDrain();
+  release();
+  assert.deepEqual(await first, summary);
+  assert.deepEqual(await stopping, { status: 'stopped' });
+  assert.equal(calls, 1);
+  await assert.rejects(lifecycle.runOnce(), /stopping/);
+});
+
+test('lifecycle bounds shutdown when a schedule transaction does not settle', async () => {
+  const lifecycle = new LocalSchedulerLifecycle(
+    { scheduleOnce: () => new Promise(() => {}) },
+    { intervalMs: 250, stopTimeoutMs: 100 },
+  );
+  void lifecycle.runOnce();
+  assert.deepEqual(await lifecycle.stopAndDrain(), { status: 'timed_out' });
+});
+
+test('lifecycle shutdown absorbs an already isolated cycle failure', async () => {
+  let rejectCycle;
+  const lifecycle = new LocalSchedulerLifecycle(
+    {
+      scheduleOnce: () =>
+        new Promise((resolve, reject) => {
+          rejectCycle = reject;
+        }),
+    },
+    { intervalMs: 250, stopTimeoutMs: 1_000 },
+  );
+  const cycle = lifecycle.runOnce();
+  const stopping = lifecycle.stopAndDrain();
+  rejectCycle(new Error('schedule storage unavailable'));
+  await assert.rejects(cycle, /schedule storage unavailable/);
+  assert.deepEqual(await stopping, { status: 'stopped' });
+});
+
+test('lifecycle runs an unrefed non-overlapping cadence and isolates diagnostics', async () => {
+  let active = 0;
+  let maximumActive = 0;
+  let calls = 0;
+  const diagnostics = [];
+  const lifecycle = new LocalSchedulerLifecycle(
+    {
+      async scheduleOnce() {
+        active += 1;
+        maximumActive = Math.max(maximumActive, active);
+        await new Promise((resolve) => setTimeout(resolve, 20));
+        active -= 1;
+        calls += 1;
+        if (calls === 1) throw new Error('temporary schedule failure');
+        return {
+          observedAtMs: calls,
+          scanned: 0,
+          initialized: 0,
+          skipped: 0,
+          admitted: 0,
+          raced: 0,
+          truncated: false,
+        };
+      },
+    },
+    {
+      intervalMs: 250,
+      stopTimeoutMs: 1_000,
+      onDiagnostic(error, summary) {
+        diagnostics.push({ error, summary });
+        if (error === undefined) throw new Error('diagnostic sink unavailable');
+      },
+    },
+  );
+
+  lifecycle.start();
+  await new Promise((resolve) => setTimeout(resolve, 650));
+  assert.deepEqual(await lifecycle.stopAndDrain(), { status: 'stopped' });
+  assert.ok(calls >= 2);
+  assert.equal(maximumActive, 1);
+  assert.ok(diagnostics.some(({ error }) => error instanceof Error));
+  assert.ok(diagnostics.some(({ summary }) => summary?.observedAtMs >= 2));
+});

packages/ql3-local-execution/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "Node16",
+    "moduleResolution": "Node16",
+    "lib": ["ES2022"],
+    "types": ["node"],
+    "rootDir": "src",
+    "outDir": "dist",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "strict": true,
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "skipLibCheck": false
+  },
+  "include": ["src/**/*.ts"]
+}