feat(ql3): establish 3.0 incubation baseline

packages/ql3-runtime-core/src/cluster-control/clusterControlActivation.ts

@@ -0,0 +1,199 @@
+export type DeploymentProfile =
+  | 'edge'
+  | 'standalone'
+  | 'cluster-control'
+  | 'worker';
+
+export type ClusterControlActivationState =
+  | 'disabled'
+  | 'schema_ready'
+  | 'reconciled'
+  | 'active'
+  | 'failed'
+  | 'stopped';
+
+export interface ClusterControlReadinessEvidence {
+  readonly contractName: string;
+  readonly contractVersion: number;
+  readonly serverMajor: number;
+  readonly migrationIds: readonly string[];
+}
+
+export interface ClusterControlReadinessProbe {
+  assertReady(): Promise<ClusterControlReadinessEvidence>;
+}
+
+export interface ClusterControlStartupRecoverySummary {
+  readonly safe: boolean;
+  readonly remaining: number;
+  readonly failed: number;
+}
+
+export type ClusterControlStopResult = 'stopped' | 'timed_out';
+export type ClusterControlAdmissionDisposer = () => void | Promise<void>;
+
+export interface ClusterControlActivationStack {
+  reconcile(): Promise<ClusterControlStartupRecoverySummary>;
+  startLifecycles(): Promise<boolean>;
+  installAdmission(): ClusterControlAdmissionDisposer;
+  stop(): Promise<ClusterControlStopResult>;
+}
+
+export interface ClusterControlActivationAudit {
+  readonly state: ClusterControlActivationState;
+  readonly contractName?: string;
+  readonly contractVersion?: number;
+  readonly serverMajor?: number;
+  readonly migrationCount?: number;
+  readonly recovery?: ClusterControlStartupRecoverySummary;
+}
+
+export interface ClusterControlRuntimeActivationOptions {
+  readonly enabled?: boolean;
+  readonly profile: DeploymentProfile;
+  readonly readiness: ClusterControlReadinessProbe;
+  readonly create: (
+    evidence: ClusterControlReadinessEvidence,
+  ) => ClusterControlActivationStack;
+  readonly audit: (
+    record: ClusterControlActivationAudit,
+  ) => void | Promise<void>;
+}
+
+export type ClusterControlRuntimeActivationResult =
+  | { readonly status: 'disabled'; stop(): Promise<'stopped'> }
+  | {
+      readonly status: 'active';
+      readonly evidence: ClusterControlReadinessEvidence;
+      readonly recovery: ClusterControlStartupRecoverySummary;
+      stop(): Promise<ClusterControlStopResult>;
+    };
+
+const DISABLED_STOP = async (): Promise<'stopped'> => 'stopped';
+
+function auditEvidence(
+  evidence: ClusterControlReadinessEvidence,
+): Pick<
+  ClusterControlActivationAudit,
+  'contractName' | 'contractVersion' | 'serverMajor' | 'migrationCount'
+> {
+  return {
+    contractName: evidence.contractName,
+    contractVersion: evidence.contractVersion,
+    serverMajor: evidence.serverMajor,
+    migrationCount: evidence.migrationIds.length,
+  };
+}
+
+function assertSafeRecovery(
+  recovery: ClusterControlStartupRecoverySummary,
+): void {
+  if (!recovery.safe || recovery.remaining !== 0 || recovery.failed !== 0) {
+    throw new Error('Cluster-control startup recovery did not converge safely');
+  }
+}
+
+/** Enforces readiness -> assembly -> recovery -> lifecycle -> admission order. */
+export async function activateClusterControlRuntime(
+  options: ClusterControlRuntimeActivationOptions,
+): Promise<ClusterControlRuntimeActivationResult> {
+  const enabled = options.enabled ?? false;
+  if (!enabled) {
+    await options.audit({ state: 'disabled' });
+    return { status: 'disabled', stop: DISABLED_STOP };
+  }
+  if (options.profile !== 'cluster-control') {
+    throw new TypeError(
+      `Deployment profile ${options.profile} cannot activate cluster-control`,
+    );
+  }
+
+  let evidence: ClusterControlReadinessEvidence | undefined;
+  let stack: ClusterControlActivationStack | undefined;
+  let disposeAdmission: ClusterControlAdmissionDisposer | undefined;
+  try {
+    evidence = await options.readiness.assertReady();
+    await options.audit({ state: 'schema_ready', ...auditEvidence(evidence) });
+    stack = options.create(evidence);
+    const recovery = await stack.reconcile();
+    assertSafeRecovery(recovery);
+    await options.audit({
+      state: 'reconciled',
+      ...auditEvidence(evidence),
+      recovery,
+    });
+    if (!(await stack.startLifecycles())) {
+      throw new Error('Cluster-control lifecycles did not start');
+    }
+    disposeAdmission = stack.installAdmission();
+    await options.audit({
+      state: 'active',
+      ...auditEvidence(evidence),
+      recovery,
+    });
+
+    let stopPromise: Promise<ClusterControlStopResult> | undefined;
+    return {
+      status: 'active',
+      evidence,
+      recovery,
+      stop() {
+        if (stopPromise) return stopPromise;
+        stopPromise = (async () => {
+          let admissionError: unknown;
+          try {
+            await disposeAdmission?.();
+          } catch (error) {
+            admissionError = error;
+          }
+          disposeAdmission = undefined;
+          const result = await stack!.stop();
+          if (admissionError) {
+            try {
+              await options.audit({
+                state: 'failed',
+                ...auditEvidence(evidence!),
+              });
+            } catch {
+              // Preserve the admission cleanup failure after stopping the stack.
+            }
+            throw admissionError;
+          }
+          try {
+            await options.audit({
+              state: 'stopped',
+              ...auditEvidence(evidence!),
+              recovery,
+            });
+          } catch {
+            // Diagnostic failure cannot reverse stopped ownership.
+          }
+          return result;
+        })();
+        return stopPromise;
+      },
+    };
+  } catch (error) {
+    try {
+      await disposeAdmission?.();
+    } catch {
+      // Preserve the activation failure and continue stopping the stack.
+    }
+    if (stack) {
+      try {
+        await stack.stop();
+      } catch {
+        // Preserve the activation failure after best-effort cleanup.
+      }
+    }
+    try {
+      await options.audit({
+        state: 'failed',
+        ...(evidence ? auditEvidence(evidence) : {}),
+      });
+    } catch {
+      // Diagnostic failure cannot replace the activation failure.
+    }
+    throw error;
+  }
+}

packages/ql3-runtime-core/src/cluster-control/clusterControlRecovery.ts

@@ -0,0 +1,66 @@
+import type { ClusterControlStartupRecoverySummary } from './clusterControlActivation';
+
+export const MAX_CLUSTER_CONTROL_RECOVERY_PAGE_SIZE = 128;
+
+export type ClusterControlRecoveryCandidate = Readonly<
+  | {
+      kind: 'run';
+      id: string;
+      runId: string;
+      status: 'created' | 'dispatching' | 'running';
+      createdAtMs: number;
+    }
+  | {
+      kind: 'attempt';
+      id: string;
+      runId: string;
+      status: 'claimed' | 'starting' | 'running';
+      createdAtMs: number;
+    }
+>;
+
+export interface ClusterControlRecoveryPage {
+  /** Observation instant selected by the durable source authority. */
+  readonly observedAtMs: number;
+  readonly candidates: readonly ClusterControlRecoveryCandidate[];
+  /** True means at least one additional candidate exists beyond this page. */
+  readonly hasMore: boolean;
+}
+
+/**
+ * Cluster-profile port for bounded, durable startup-recovery discovery.
+ * Implementations must never use an unbounded table scan or return terminal
+ * work as a candidate.
+ */
+export interface ClusterControlRecoverySource {
+  listOutstanding(limit: number): Promise<ClusterControlRecoveryPage>;
+}
+
+/**
+ * Independently proves that a recovery implementation really converged before
+ * cluster admission is installed. A non-zero remaining value is a lower bound,
+ * not an expensive exact count.
+ */
+export class ClusterControlRecoveryConvergenceVerifier {
+  constructor(private readonly source: ClusterControlRecoverySource) {}
+
+  async verify(): Promise<ClusterControlStartupRecoverySummary> {
+    const page = await this.source.listOutstanding(1);
+    if (!Number.isSafeInteger(page.observedAtMs) || page.observedAtMs < 0) {
+      throw new Error('Cluster-control recovery observation is invalid');
+    }
+    if (page.candidates.length === 0) {
+      if (page.hasMore) {
+        throw new Error(
+          'Cluster-control recovery source returned hasMore without a candidate',
+        );
+      }
+      return Object.freeze({ safe: true, remaining: 0, failed: 0 });
+    }
+    return Object.freeze({
+      safe: false,
+      remaining: page.candidates.length + (page.hasMore ? 1 : 0),
+      failed: 0,
+    });
+  }
+}

packages/ql3-runtime-core/src/cluster-control/clusterControlRecoveryEvidenceRegistry.ts

@@ -0,0 +1,356 @@
+import type {
+  ClusterControlRecoveryEvidence,
+  ClusterControlRecoveryEvidenceProvider,
+  ClusterControlRecoveryProbeTarget,
+} from './clusterControlRecoveryProcessor';
+
+export const MAX_CLUSTER_CONTROL_RECOVERY_EVIDENCE_PROVIDERS = 32;
+export const MAX_CLUSTER_CONTROL_RECOVERY_EVIDENCE_TIMEOUT_MS = 30_000;
+
+export const CLUSTER_CONTROL_RECOVERY_IDENTITY_FIELDS = [
+  'workerId',
+  'workerSessionId',
+  'workerGeneration',
+  'executorHandle',
+  'pid',
+  'leaseToken',
+  'leaseTokenDigest',
+  'leaseGeneration',
+  'leaseVersion',
+  'offerId',
+] as const;
+
+export type ClusterControlRecoveryIdentityField =
+  (typeof CLUSTER_CONTROL_RECOVERY_IDENTITY_FIELDS)[number];
+
+export interface ClusterControlRecoveryEvidenceInspectionContext {
+  /** Resource bound only; PostgreSQL time remains the claim-fence authority. */
+  readonly timeoutMs: number;
+  readonly signal: AbortSignal;
+}
+
+/**
+ * Executor-specific, evidence-only capability. It deliberately receives no
+ * recovery owner/token and exposes no start, stop, retry or completion port.
+ */
+export interface ClusterControlRecoveryExecutorEvidenceProvider {
+  readonly executorType: string;
+  readonly requiredIdentity: readonly ClusterControlRecoveryIdentityField[];
+  inspect(
+    target: ClusterControlRecoveryProbeTarget,
+    context: ClusterControlRecoveryEvidenceInspectionContext,
+  ): Promise<ClusterControlRecoveryEvidence>;
+}
+
+export interface ClusterControlRecoveryEvidenceRegistryOptions {
+  readonly timeoutMs?: number;
+}
+
+interface RegisteredProvider {
+  readonly inspect: ClusterControlRecoveryExecutorEvidenceProvider['inspect'];
+  readonly requiredIdentity: ReadonlySet<ClusterControlRecoveryIdentityField>;
+}
+
+interface ActiveInspection {
+  readonly controller: AbortController;
+  readonly resolve: (evidence: ClusterControlRecoveryEvidence) => void;
+  timer: ReturnType<typeof setTimeout> | undefined;
+  responded: boolean;
+}
+
+const PROVIDER_UNAVAILABLE = Object.freeze({
+  status: 'unknown',
+  reason: 'provider_unavailable',
+} as const);
+
+const IDENTITY_UNVERIFIABLE = Object.freeze({
+  status: 'unknown',
+  reason: 'identity_unverifiable',
+} as const);
+
+const CONFLICTING_EVIDENCE = Object.freeze({
+  status: 'unknown',
+  reason: 'conflicting_evidence',
+} as const);
+
+function integerInRange(
+  name: string,
+  value: number,
+  minimum: number,
+  maximum: number,
+): number {
+  if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
+    throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
+  }
+  return value;
+}
+
+function executorType(value: string): string {
+  if (!/^[a-z][a-z0-9_.-]{0,63}$/.test(value)) {
+    throw new TypeError(
+      'Cluster-control recovery evidence executorType is invalid',
+    );
+  }
+  return value;
+}
+
+function requiredIdentity(
+  fields: readonly ClusterControlRecoveryIdentityField[],
+): ReadonlySet<ClusterControlRecoveryIdentityField> {
+  if (!Array.isArray(fields) || fields.length < 1) {
+    throw new TypeError(
+      'Cluster-control recovery evidence provider requires an execution identity',
+    );
+  }
+  const normalized = new Set<ClusterControlRecoveryIdentityField>();
+  for (const field of fields) {
+    if (!CLUSTER_CONTROL_RECOVERY_IDENTITY_FIELDS.includes(field)) {
+      throw new TypeError(
+        'Cluster-control recovery evidence identity field is invalid',
+      );
+    }
+    if (normalized.has(field)) {
+      throw new TypeError(
+        'Cluster-control recovery evidence identity field is duplicated',
+      );
+    }
+    normalized.add(field);
+  }
+  return normalized;
+}
+
+function optionalString(value: string | undefined, maximum: number): boolean {
+  return (
+    value === undefined ||
+    (value.length > 0 && value.length <= maximum && !value.includes('\0'))
+  );
+}
+
+function optionalSafeInteger(
+  value: number | undefined,
+  minimum: number,
+): boolean {
+  return (
+    value === undefined || (Number.isSafeInteger(value) && value >= minimum)
+  );
+}
+
+function validTarget(target: ClusterControlRecoveryProbeTarget): boolean {
+  return (
+    !!target &&
+    typeof target === 'object' &&
+    typeof target.runId === 'string' &&
+    target.runId.length > 0 &&
+    target.runId.length <= 64 &&
+    typeof target.attemptId === 'string' &&
+    target.attemptId.length > 0 &&
+    target.attemptId.length <= 64 &&
+    ['starting', 'running'].includes(target.attemptStatus) &&
+    /^[a-z][a-z0-9_.-]{0,63}$/.test(target.executorType) &&
+    Number.isSafeInteger(target.callbackSequence) &&
+    target.callbackSequence >= 0 &&
+    optionalString(target.workerId, 128) &&
+    (target.workerSessionId === undefined ||
+      /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(
+        target.workerSessionId,
+      )) &&
+    optionalSafeInteger(target.workerGeneration, 1) &&
+    optionalString(target.executorHandle, 512) &&
+    optionalSafeInteger(target.pid, 1) &&
+    optionalString(target.leaseToken, 128) &&
+    (target.leaseTokenDigest === undefined ||
+      /^[0-9a-f]{64}$/.test(target.leaseTokenDigest)) &&
+    optionalSafeInteger(target.leaseGeneration, 1) &&
+    optionalSafeInteger(target.leaseVersion, 0) &&
+    optionalSafeInteger(target.leaseExpiresAtMs, 0) &&
+    optionalString(target.offerId, 128) &&
+    optionalSafeInteger(target.startedAtMs, 0)
+  );
+}
+
+function hasRequiredIdentity(
+  target: ClusterControlRecoveryProbeTarget,
+  fields: ReadonlySet<ClusterControlRecoveryIdentityField>,
+): boolean {
+  for (const field of fields) {
+    const value = target[field];
+    if (
+      field === 'pid' ||
+      field === 'workerGeneration' ||
+      field === 'leaseGeneration' ||
+      field === 'leaseVersion'
+    ) {
+      if (typeof value !== 'number') return false;
+    } else if (typeof value !== 'string' || value.length === 0) {
+      return false;
+    }
+  }
+  return true;
+}
+
+function frozenTarget(
+  target: ClusterControlRecoveryProbeTarget,
+): ClusterControlRecoveryProbeTarget {
+  return Object.freeze({ ...target });
+}
+
+function normalizeEvidence(value: unknown): ClusterControlRecoveryEvidence {
+  if (
+    value &&
+    typeof value === 'object' &&
+    'status' in value &&
+    value.status === 'running'
+  ) {
+    return Object.freeze({ status: 'running' });
+  }
+  if (
+    value &&
+    typeof value === 'object' &&
+    'status' in value &&
+    value.status === 'not_running'
+  ) {
+    return Object.freeze({ status: 'not_running' });
+  }
+  if (
+    value &&
+    typeof value === 'object' &&
+    'status' in value &&
+    value.status === 'unknown' &&
+    'reason' in value
+  ) {
+    if (value.reason === 'provider_unavailable') return PROVIDER_UNAVAILABLE;
+    if (value.reason === 'identity_unverifiable') {
+      return IDENTITY_UNVERIFIABLE;
+    }
+    if (value.reason === 'conflicting_evidence') {
+      return CONFLICTING_EVIDENCE;
+    }
+  }
+  return CONFLICTING_EVIDENCE;
+}
+
+/**
+ * Exact executor-type router with one in-flight inspection per provider.
+ * Timeout releases the startup path but keeps the provider busy until its
+ * abandoned operation settles, preventing unbounded probe accumulation.
+ */
+export class ClusterControlRecoveryEvidenceRegistry
+  implements ClusterControlRecoveryEvidenceProvider
+{
+  private readonly providers = new Map<string, RegisteredProvider>();
+  private readonly active = new Map<string, ActiveInspection>();
+  private readonly timeoutMs: number;
+  private disposed = false;
+
+  constructor(
+    providers: readonly ClusterControlRecoveryExecutorEvidenceProvider[],
+    options: ClusterControlRecoveryEvidenceRegistryOptions = {},
+  ) {
+    if (
+      !Array.isArray(providers) ||
+      providers.length > MAX_CLUSTER_CONTROL_RECOVERY_EVIDENCE_PROVIDERS
+    ) {
+      throw new RangeError(
+        `Cluster-control recovery evidence providers cannot exceed ${MAX_CLUSTER_CONTROL_RECOVERY_EVIDENCE_PROVIDERS}`,
+      );
+    }
+    this.timeoutMs = integerInRange(
+      'Cluster-control recovery evidence timeout',
+      options.timeoutMs ?? 5_000,
+      1,
+      MAX_CLUSTER_CONTROL_RECOVERY_EVIDENCE_TIMEOUT_MS,
+    );
+    for (const provider of providers) {
+      if (!provider || typeof provider.inspect !== 'function') {
+        throw new TypeError(
+          'Cluster-control recovery evidence provider is invalid',
+        );
+      }
+      const type = executorType(provider.executorType);
+      if (this.providers.has(type)) {
+        throw new TypeError(
+          `Duplicate cluster-control recovery evidence provider: ${type}`,
+        );
+      }
+      this.providers.set(type, {
+        inspect: provider.inspect.bind(provider),
+        requiredIdentity: requiredIdentity(provider.requiredIdentity),
+      });
+    }
+  }
+
+  async inspect(
+    _claim: Parameters<ClusterControlRecoveryEvidenceProvider['inspect']>[0],
+    target: ClusterControlRecoveryProbeTarget,
+  ): Promise<ClusterControlRecoveryEvidence> {
+    if (this.disposed) return PROVIDER_UNAVAILABLE;
+    if (!validTarget(target)) return IDENTITY_UNVERIFIABLE;
+    const registration = this.providers.get(target.executorType);
+    if (!registration) return IDENTITY_UNVERIFIABLE;
+    if (!hasRequiredIdentity(target, registration.requiredIdentity)) {
+      return IDENTITY_UNVERIFIABLE;
+    }
+    if (this.active.has(target.executorType)) return PROVIDER_UNAVAILABLE;
+
+    return new Promise<ClusterControlRecoveryEvidence>((resolve) => {
+      const controller = new AbortController();
+      const inspection: ActiveInspection = {
+        controller,
+        resolve,
+        timer: undefined,
+        responded: false,
+      };
+      this.active.set(target.executorType, inspection);
+
+      const respond = (
+        evidence: ClusterControlRecoveryEvidence,
+        release: boolean,
+      ): void => {
+        if (release && this.active.get(target.executorType) === inspection) {
+          this.active.delete(target.executorType);
+        }
+        if (inspection.timer !== undefined) {
+          clearTimeout(inspection.timer);
+          inspection.timer = undefined;
+        }
+        if (inspection.responded) return;
+        inspection.responded = true;
+        inspection.resolve(evidence);
+      };
+
+      inspection.timer = setTimeout(() => {
+        inspection.timer = undefined;
+        controller.abort();
+        // Keep the slot occupied until the provider promise actually settles.
+        respond(PROVIDER_UNAVAILABLE, false);
+      }, this.timeoutMs);
+      inspection.timer.unref?.();
+
+      const context = Object.freeze({
+        timeoutMs: this.timeoutMs,
+        signal: controller.signal,
+      });
+      Promise.resolve()
+        .then(() => registration.inspect(frozenTarget(target), context))
+        .then(
+          (evidence) => respond(normalizeEvidence(evidence), true),
+          () => respond(PROVIDER_UNAVAILABLE, true),
+        );
+    });
+  }
+
+  dispose(): void {
+    if (this.disposed) return;
+    this.disposed = true;
+    for (const inspection of this.active.values()) {
+      if (inspection.timer !== undefined) clearTimeout(inspection.timer);
+      inspection.timer = undefined;
+      inspection.controller.abort();
+      if (!inspection.responded) {
+        inspection.responded = true;
+        inspection.resolve(PROVIDER_UNAVAILABLE);
+      }
+    }
+    this.active.clear();
+  }
+}

packages/ql3-runtime-core/src/cluster-control/clusterControlRecoveryProcessor.ts

@@ -0,0 +1,608 @@
+import type { RunAttemptRecord, RunEventRecord, RunRecord } from '../run/run';
+import type {
+  PluginPackageWorkflowTaskAttemptAdmissionReceipt,
+} from '../plugin-package/workflow/pluginPackageWorkflowTaskAttemptAdmission';
+import type { StepRunRecord } from '../run/stepRun';
+import type {
+  ClusterControlRecoveryClaim,
+  ClusterControlRecoveryDisposition,
+  ClusterControlRecoveryProcessor,
+} from './clusterControlRecoverySupervisor';
+import { MAX_CLUSTER_CONTROL_RECOVERY_RETRY_DELAY_MS } from './clusterControlRecoverySupervisor';
+
+const ACTIVE_RUN_STATUSES = new Set(['dispatching', 'running']);
+const ACTIVE_ATTEMPT_STATUSES = new Set(['claimed', 'starting', 'running']);
+const TERMINAL_ATTEMPT_STATUSES = new Set([
+  'succeeded',
+  'failed',
+  'cancelled',
+  'timed_out',
+  'lost',
+]);
+
+export const CLUSTER_CONTROL_RECOVERY_UNKNOWN_REASONS = [
+  'provider_unavailable',
+  'identity_unverifiable',
+  'conflicting_evidence',
+] as const;
+
+export type ClusterControlRecoveryUnknownReason =
+  (typeof CLUSTER_CONTROL_RECOVERY_UNKNOWN_REASONS)[number];
+
+export type ClusterControlRecoveryEvidence = Readonly<
+  | { status: 'running' }
+  | { status: 'not_running' }
+  | {
+      status: 'unknown';
+      reason: ClusterControlRecoveryUnknownReason;
+    }
+>;
+
+export interface ClusterControlRecoveryProbeTarget {
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly attemptStatus: 'starting' | 'running';
+  readonly executorType: string;
+  readonly callbackSequence: number;
+  readonly workerId?: string;
+  readonly workerSessionId?: string;
+  readonly workerGeneration?: number;
+  readonly executorHandle?: string;
+  readonly pid?: number;
+  readonly leaseToken?: string;
+  readonly leaseTokenDigest?: string;
+  readonly leaseGeneration?: number;
+  readonly leaseVersion?: number;
+  readonly leaseExpiresAtMs?: number;
+  readonly offerId?: string;
+  readonly startedAtMs?: number;
+}
+
+export interface ClusterControlRecoveryEvidenceProvider {
+  inspect(
+    claim: ClusterControlRecoveryClaim,
+    target: ClusterControlRecoveryProbeTarget,
+  ): Promise<ClusterControlRecoveryEvidence>;
+}
+
+export interface ClusterControlRecoverySnapshot {
+  /** Database-authoritative observation used to revalidate execution leases. */
+  readonly observedAtMs: number;
+  readonly run: Readonly<RunRecord> | null;
+  /** Exact candidate Attempt, or latest Attempt when the candidate is a Run. */
+  readonly attempt: Readonly<RunAttemptRecord> | null;
+  /** Immutable Task admission authority when this is a Workflow Task Attempt. */
+  readonly workflowTask?: Readonly<{
+    admission:
+      Readonly<PluginPackageWorkflowTaskAttemptAdmissionReceipt>;
+    stepRun: Readonly<StepRunRecord>;
+  }> | null;
+}
+
+export type ClusterControlRecoveryLostReason =
+  | 'attempt_already_lost'
+  | 'unstarted_claim_expired'
+  | 'execution_not_running';
+
+export type ClusterControlRecoveryLostAction = Readonly<
+  | {
+      kind: 'mark_run_lost';
+      reason: 'attempt_already_lost';
+    }
+  | {
+      kind: 'mark_attempt_lost';
+      reason: 'unstarted_claim_expired' | 'execution_not_running';
+    }
+  | {
+      kind: 'mark_attempt_and_run_lost';
+      reason: 'unstarted_claim_expired' | 'execution_not_running';
+    }
+  | {
+      kind: 'recover_workflow_task';
+      reason: 'unstarted_claim_expired' | 'execution_not_running';
+    }
+>;
+
+export interface ClusterControlRecoveryResolutionRepository {
+  load(
+    claim: ClusterControlRecoveryClaim,
+  ): Promise<ClusterControlRecoverySnapshot | 'fenced'>;
+  applyLost(
+    claim: ClusterControlRecoveryClaim,
+    snapshot: ClusterControlRecoverySnapshot,
+    action: ClusterControlRecoveryLostAction,
+  ): Promise<'applied' | 'stale' | 'fenced'>;
+}
+
+export class ClusterControlRecoveryFenceLostError extends Error {
+  readonly retryable = true;
+
+  constructor() {
+    super('Cluster-control recovery claim fence was lost');
+    this.name = 'ClusterControlRecoveryFenceLostError';
+  }
+}
+
+export class InvalidClusterControlRecoveryTransitionError extends Error {
+  readonly code = 'INVALID_CLUSTER_CONTROL_RECOVERY_TRANSITION';
+
+  constructor(message: string) {
+    super(message);
+    this.name = 'InvalidClusterControlRecoveryTransitionError';
+  }
+}
+
+export interface ClusterControlRecoveryLostTransition {
+  readonly attempt?: Readonly<{
+    run: RunRecord;
+    attempt: RunAttemptRecord;
+    event: Readonly<
+      Omit<RunEventRecord, 'id' | 'actorType' | 'actorId' | 'createdAtMs'>
+    >;
+  }>;
+  readonly run?: Readonly<{
+    run: RunRecord;
+    event: Readonly<
+      Omit<RunEventRecord, 'id' | 'actorType' | 'actorId' | 'createdAtMs'>
+    >;
+  }>;
+}
+
+function errorMetadata(reason: ClusterControlRecoveryLostReason): Readonly<{
+  code: string;
+  summary: string;
+}> {
+  if (reason === 'attempt_already_lost') {
+    return Object.freeze({
+      code: 'CLUSTER_RECOVERY_ATTEMPT_ALREADY_LOST',
+      summary:
+        'The latest Attempt was already lost before the Run was reconciled',
+    });
+  }
+  if (reason === 'unstarted_claim_expired') {
+    return Object.freeze({
+      code: 'CLUSTER_RECOVERY_UNSTARTED_CLAIM_EXPIRED',
+      summary:
+        'The unstarted Attempt claim expired before execution was admitted',
+    });
+  }
+  return Object.freeze({
+    code: 'CLUSTER_RECOVERY_EXECUTION_NOT_RUNNING',
+    summary:
+      'Trusted execution evidence proved that the active Attempt is not running',
+  });
+}
+
+function transitionTime(
+  atMs: number,
+  run: Readonly<RunRecord>,
+  attempt?: Readonly<RunAttemptRecord>,
+): number {
+  if (!Number.isSafeInteger(atMs) || atMs < 0) {
+    throw new InvalidClusterControlRecoveryTransitionError(
+      'Recovery transition time is invalid',
+    );
+  }
+  return Math.max(
+    atMs,
+    run.createdAtMs,
+    run.startedAtMs ?? 0,
+    attempt?.createdAtMs ?? 0,
+    attempt?.startedAtMs ?? 0,
+    attempt?.finishedAtMs ?? 0,
+  );
+}
+
+function reserveEvent(run: Readonly<RunRecord>): Readonly<{
+  run: RunRecord;
+  sequence: number;
+}> {
+  const version = run.version + 1;
+  const sequence = run.eventSequence + 1;
+  if (
+    !Number.isSafeInteger(version) ||
+    version < 1 ||
+    !Number.isSafeInteger(sequence) ||
+    sequence < 1
+  ) {
+    throw new InvalidClusterControlRecoveryTransitionError(
+      'Recovery transition version or event sequence overflowed',
+    );
+  }
+  return Object.freeze({
+    run: { ...run, version, eventSequence: sequence },
+    sequence,
+  });
+}
+
+function attemptLostTransition(
+  run: Readonly<RunRecord>,
+  attempt: Readonly<RunAttemptRecord>,
+  reason: 'unstarted_claim_expired' | 'execution_not_running',
+  atMs: number,
+): NonNullable<ClusterControlRecoveryLostTransition['attempt']> {
+  if (
+    attempt.runId !== run.id ||
+    !ACTIVE_ATTEMPT_STATUSES.has(attempt.status)
+  ) {
+    throw new InvalidClusterControlRecoveryTransitionError(
+      'Recovery Attempt is not an active member of the Run aggregate',
+    );
+  }
+  const metadata = errorMetadata(reason);
+  const reserved = reserveEvent(run);
+  const nextAttempt: RunAttemptRecord = {
+    ...attempt,
+    status: 'lost',
+    finishedAtMs: atMs,
+    errorCode: metadata.code,
+    errorSummary: metadata.summary,
+  };
+  return Object.freeze({
+    run: reserved.run,
+    attempt: nextAttempt,
+    event: Object.freeze({
+      runId: run.id,
+      sequence: reserved.sequence,
+      type: 'attempt.lost',
+      dedupeKey: `cluster-recovery:attempt:${attempt.id}:${attempt.callbackSequence}`,
+      attemptId: attempt.id,
+      ...(attempt.stepRunId === undefined
+        ? {}
+        : { stepRunId: attempt.stepRunId }),
+      payload: Object.freeze({
+        attempt_id: attempt.id,
+        attempt: attempt.attempt,
+        from_status: attempt.status,
+        to_status: 'lost',
+        version: reserved.run.version,
+        error_code: metadata.code,
+      }),
+    }),
+  });
+}
+
+function runLostTransition(
+  run: Readonly<RunRecord>,
+  attemptId: string,
+  reason: ClusterControlRecoveryLostReason,
+): NonNullable<ClusterControlRecoveryLostTransition['run']> {
+  if (!ACTIVE_RUN_STATUSES.has(run.status)) {
+    throw new InvalidClusterControlRecoveryTransitionError(
+      'Recovery Run is not active',
+    );
+  }
+  const metadata = errorMetadata(reason);
+  const reserved = reserveEvent(run);
+  const nextRun: RunRecord = {
+    ...reserved.run,
+    status: 'lost',
+    errorCode: metadata.code,
+    errorSummary: metadata.summary,
+  };
+  return Object.freeze({
+    run: nextRun,
+    event: Object.freeze({
+      runId: run.id,
+      sequence: reserved.sequence,
+      type: 'run.lost',
+      dedupeKey: `cluster-recovery:run:${run.id}:${attemptId}:${run.version}`,
+      attemptId,
+      payload: Object.freeze({
+        from_status: run.status,
+        to_status: 'lost',
+        version: nextRun.version,
+        error_code: metadata.code,
+      }),
+    }),
+  });
+}
+
+/**
+ * Builds only the recovery-specific lost transition. It deliberately does not
+ * create another Attempt, enqueue work, call an Executor or infer completion.
+ */
+export function buildClusterControlRecoveryLostTransition(
+  currentRun: Readonly<RunRecord>,
+  currentAttempt: Readonly<RunAttemptRecord> | null,
+  action: ClusterControlRecoveryLostAction,
+  observedAtMs: number,
+): ClusterControlRecoveryLostTransition {
+  if (
+    currentRun.executionOwner !== 'runtime' ||
+    currentRun.cancelRequestedAtMs !== undefined
+  ) {
+    throw new InvalidClusterControlRecoveryTransitionError(
+      'Recovery lost transition has no runtime ownership authority',
+    );
+  }
+  const atMs = transitionTime(
+    observedAtMs,
+    currentRun,
+    currentAttempt ?? undefined,
+  );
+  if (action.kind === 'recover_workflow_task') {
+    throw new InvalidClusterControlRecoveryTransitionError(
+      'Workflow Task recovery requires its admission-bound StepRun authority',
+    );
+  }
+  if (action.kind === 'mark_run_lost') {
+    if (!currentAttempt || currentAttempt.status !== 'lost') {
+      throw new InvalidClusterControlRecoveryTransitionError(
+        'Run-only recovery requires an already-lost Attempt',
+      );
+    }
+    return Object.freeze({
+      run: runLostTransition(currentRun, currentAttempt.id, action.reason),
+    });
+  }
+  if (!currentAttempt) {
+    throw new InvalidClusterControlRecoveryTransitionError(
+      'Attempt recovery requires an Attempt',
+    );
+  }
+  if (action.kind === 'mark_attempt_lost') {
+    if (currentRun.status !== 'lost') {
+      throw new InvalidClusterControlRecoveryTransitionError(
+        'Attempt-only recovery requires an already-lost Run',
+      );
+    }
+    return Object.freeze({
+      attempt: attemptLostTransition(
+        currentRun,
+        currentAttempt,
+        action.reason,
+        atMs,
+      ),
+    });
+  }
+  if (!ACTIVE_RUN_STATUSES.has(currentRun.status)) {
+    throw new InvalidClusterControlRecoveryTransitionError(
+      'Aggregate recovery requires an active Run',
+    );
+  }
+  const attempt = attemptLostTransition(
+    currentRun,
+    currentAttempt,
+    action.reason,
+    atMs,
+  );
+  return Object.freeze({
+    attempt,
+    run: runLostTransition(attempt.run, currentAttempt.id, action.reason),
+  });
+}
+
+function retryDelay(value: number | undefined): number {
+  const delay = value ?? 5_000;
+  if (
+    !Number.isSafeInteger(delay) ||
+    delay < 0 ||
+    delay > MAX_CLUSTER_CONTROL_RECOVERY_RETRY_DELAY_MS
+  ) {
+    throw new RangeError(
+      `Cluster-control evidence retry delay must be between 0 and ${MAX_CLUSTER_CONTROL_RECOVERY_RETRY_DELAY_MS}`,
+    );
+  }
+  return delay;
+}
+
+function validSnapshot(
+  claim: ClusterControlRecoveryClaim,
+  snapshot: ClusterControlRecoverySnapshot,
+): void {
+  if (
+    !Number.isSafeInteger(snapshot.observedAtMs) ||
+    snapshot.observedAtMs < 0 ||
+    (snapshot.run !== null && snapshot.run.id !== claim.candidate.runId) ||
+    (claim.candidate.kind === 'attempt' &&
+      snapshot.attempt !== null &&
+      snapshot.attempt.id !== claim.candidate.id) ||
+    (snapshot.workflowTask !== undefined &&
+      snapshot.workflowTask !== null &&
+      (claim.candidate.kind !== 'attempt' ||
+        snapshot.workflowTask.admission.attemptId !==
+          claim.candidate.id ||
+        snapshot.workflowTask.admission.runId !==
+          claim.candidate.runId ||
+        snapshot.workflowTask.stepRun.id !==
+          snapshot.workflowTask.admission.stepRunId ||
+        snapshot.workflowTask.stepRun.runId !==
+          claim.candidate.runId))
+  ) {
+    throw new TypeError(
+      'Cluster-control recovery resolution repository returned an invalid snapshot',
+    );
+  }
+}
+
+function hasValidLease(
+  attempt: Readonly<RunAttemptRecord>,
+  observedAtMs: number,
+): boolean {
+  return (
+    attempt.leaseExpiresAtMs !== undefined &&
+    attempt.leaseExpiresAtMs > observedAtMs
+  );
+}
+
+function probeTarget(
+  attempt: Readonly<RunAttemptRecord>,
+): ClusterControlRecoveryProbeTarget {
+  if (attempt.status !== 'starting' && attempt.status !== 'running') {
+    throw new TypeError('Cluster-control recovery probe target is not running');
+  }
+  return Object.freeze({
+    runId: attempt.runId,
+    attemptId: attempt.id,
+    attemptStatus: attempt.status,
+    executorType: attempt.executorType,
+    callbackSequence: attempt.callbackSequence,
+    ...(attempt.workerId === undefined ? {} : { workerId: attempt.workerId }),
+    ...(attempt.workerSessionId === undefined
+      ? {}
+      : { workerSessionId: attempt.workerSessionId }),
+    ...(attempt.workerGeneration === undefined
+      ? {}
+      : { workerGeneration: attempt.workerGeneration }),
+    ...(attempt.executorHandle === undefined
+      ? {}
+      : { executorHandle: attempt.executorHandle }),
+    ...(attempt.pid === undefined ? {} : { pid: attempt.pid }),
+    ...(attempt.leaseToken === undefined
+      ? {}
+      : { leaseToken: attempt.leaseToken }),
+    ...(attempt.leaseTokenDigest === undefined
+      ? {}
+      : { leaseTokenDigest: attempt.leaseTokenDigest }),
+    ...(attempt.leaseGeneration === undefined
+      ? {}
+      : { leaseGeneration: attempt.leaseGeneration }),
+    ...(attempt.leaseVersion === undefined
+      ? {}
+      : { leaseVersion: attempt.leaseVersion }),
+    ...(attempt.leaseExpiresAtMs === undefined
+      ? {}
+      : { leaseExpiresAtMs: attempt.leaseExpiresAtMs }),
+    ...(attempt.offerId === undefined ? {} : { offerId: attempt.offerId }),
+    ...(attempt.startedAtMs === undefined
+      ? {}
+      : { startedAtMs: attempt.startedAtMs }),
+  });
+}
+
+function normalizeEvidence(
+  evidence: ClusterControlRecoveryEvidence,
+): ClusterControlRecoveryEvidence {
+  if (evidence?.status === 'running') {
+    return Object.freeze({ status: 'running' });
+  }
+  if (evidence?.status === 'not_running') {
+    return Object.freeze({ status: 'not_running' });
+  }
+  if (
+    evidence?.status === 'unknown' &&
+    CLUSTER_CONTROL_RECOVERY_UNKNOWN_REASONS.includes(evidence.reason)
+  ) {
+    return Object.freeze({ status: 'unknown', reason: evidence.reason });
+  }
+  throw new TypeError(
+    'Cluster-control recovery evidence provider returned invalid evidence',
+  );
+}
+
+/**
+ * Revalidates durable state, requests external evidence only for attempts that
+ * crossed the start barrier, and delegates fenced atomic mutation to storage.
+ */
+export class EvidenceBasedClusterControlRecoveryProcessor
+  implements ClusterControlRecoveryProcessor
+{
+  private readonly retryDelayMs: number;
+
+  constructor(
+    private readonly repository: ClusterControlRecoveryResolutionRepository,
+    private readonly evidence: ClusterControlRecoveryEvidenceProvider,
+    options: Readonly<{ retryDelayMs?: number }> = {},
+  ) {
+    this.retryDelayMs = retryDelay(options.retryDelayMs);
+  }
+
+  async process(
+    claim: ClusterControlRecoveryClaim,
+  ): Promise<ClusterControlRecoveryDisposition> {
+    const snapshot = await this.repository.load(claim);
+    if (snapshot === 'fenced') throw new ClusterControlRecoveryFenceLostError();
+    validSnapshot(claim, snapshot);
+    const { run, attempt } = snapshot;
+    if (!run || run.executionOwner !== 'runtime') {
+      return Object.freeze({ status: 'resolved' });
+    }
+
+    if (claim.candidate.kind === 'run') {
+      if (!['created', 'dispatching', 'running'].includes(run.status)) {
+        return Object.freeze({ status: 'resolved' });
+      }
+      if (run.status === 'created' || !attempt) {
+        return Object.freeze({ status: 'manual' });
+      }
+      if (attempt.status === 'lost') {
+        if (run.cancelRequestedAtMs !== undefined) return this.retry();
+        return this.apply(claim, snapshot, {
+          kind: 'mark_run_lost',
+          reason: 'attempt_already_lost',
+        });
+      }
+      if (TERMINAL_ATTEMPT_STATUSES.has(attempt.status)) {
+        return Object.freeze({ status: 'manual' });
+      }
+    } else {
+      if (!attempt || attempt.runId !== run.id) {
+        return Object.freeze({ status: 'resolved' });
+      }
+      if (!ACTIVE_ATTEMPT_STATUSES.has(attempt.status)) {
+        return Object.freeze({ status: 'resolved' });
+      }
+    }
+
+    if (!attempt || !ACTIVE_ATTEMPT_STATUSES.has(attempt.status)) {
+      return Object.freeze({ status: 'manual' });
+    }
+    if (hasValidLease(attempt, snapshot.observedAtMs)) {
+      return Object.freeze({ status: 'resolved' });
+    }
+    if (
+      run.status !== 'dispatching' &&
+      run.status !== 'running' &&
+      run.status !== 'lost'
+    ) {
+      return Object.freeze({ status: 'manual' });
+    }
+    if (run.cancelRequestedAtMs !== undefined) return this.retry();
+
+    const actionKind = snapshot.workflowTask
+      ? 'recover_workflow_task'
+      : run.status === 'lost'
+        ? 'mark_attempt_lost'
+        : 'mark_attempt_and_run_lost';
+    if (attempt.status === 'claimed') {
+      return this.apply(claim, snapshot, {
+        kind: actionKind,
+        reason: 'unstarted_claim_expired',
+      });
+    }
+
+    const evidence = normalizeEvidence(
+      await this.evidence.inspect(claim, probeTarget(attempt)),
+    );
+    if (evidence.status === 'running') return this.retry();
+    if (evidence.status === 'unknown') {
+      return evidence.reason === 'provider_unavailable'
+        ? this.retry()
+        : Object.freeze({ status: 'manual' });
+    }
+    return this.apply(claim, snapshot, {
+      kind: actionKind,
+      reason: 'execution_not_running',
+    });
+  }
+
+  private retry(): ClusterControlRecoveryDisposition {
+    return Object.freeze({ status: 'retry', delayMs: this.retryDelayMs });
+  }
+
+  private async apply(
+    claim: ClusterControlRecoveryClaim,
+    snapshot: ClusterControlRecoverySnapshot,
+    action: ClusterControlRecoveryLostAction,
+  ): Promise<ClusterControlRecoveryDisposition> {
+    const result = await this.repository.applyLost(claim, snapshot, action);
+    if (result === 'fenced') throw new ClusterControlRecoveryFenceLostError();
+    if (result !== 'applied' && result !== 'stale') {
+      throw new TypeError(
+        'Cluster-control recovery resolution repository returned an invalid result',
+      );
+    }
+    return Object.freeze({ status: 'resolved' });
+  }
+}

packages/ql3-runtime-core/src/cluster-control/clusterControlRecoverySupervisor.ts

@@ -0,0 +1,242 @@
+import type { ClusterControlRecoveryCandidate } from './clusterControlRecovery';
+import type { ClusterControlStartupRecoverySummary } from './clusterControlActivation';
+
+export const MAX_CLUSTER_CONTROL_RECOVERY_CLAIMS_PER_PASS = 128;
+export const MAX_CLUSTER_CONTROL_RECOVERY_CLAIM_LEASE_MS = 5 * 60 * 1000;
+export const MAX_CLUSTER_CONTROL_RECOVERY_RETRY_DELAY_MS = 5 * 60 * 1000;
+
+export class ClusterControlRecoveryStoreError extends Error {
+  readonly retryable = true;
+
+  constructor(readonly cause?: unknown) {
+    super('Cluster-control recovery store operation failed');
+    this.name = 'ClusterControlRecoveryStoreError';
+  }
+}
+
+export interface ClusterControlRecoveryClaim {
+  readonly candidate: ClusterControlRecoveryCandidate;
+  readonly observedAtMs: number;
+  readonly ownerId: string;
+  readonly token: string;
+  readonly version: number;
+  readonly expiresAtMs: number;
+}
+
+export interface ClusterControlRecoveryClaimPage {
+  readonly claims: readonly ClusterControlRecoveryClaim[];
+  /** Number of candidates observed in this bounded discovery page. */
+  readonly discovered: number;
+  readonly hasMore: boolean;
+}
+
+export type ClusterControlRecoveryDisposition =
+  | Readonly<{ status: 'resolved' }>
+  | Readonly<{ status: 'retry'; delayMs: number }>
+  | Readonly<{ status: 'manual' }>;
+
+export interface ClusterControlRecoveryClaimRepository {
+  claim(
+    options: Readonly<{
+      ownerId: string;
+      limit: number;
+      leaseMs: number;
+    }>,
+  ): Promise<ClusterControlRecoveryClaimPage>;
+  settle(
+    claim: ClusterControlRecoveryClaim,
+    disposition: ClusterControlRecoveryDisposition,
+  ): Promise<'settled' | 'fenced'>;
+}
+
+export interface ClusterControlRecoveryProcessor {
+  process(
+    claim: ClusterControlRecoveryClaim,
+  ): Promise<ClusterControlRecoveryDisposition>;
+}
+
+export interface ClusterControlRecoverySupervisorOptions {
+  readonly ownerId: string;
+  readonly limit?: number;
+  readonly leaseMs?: number;
+  readonly retryDelayMs?: number;
+}
+
+function integerInRange(
+  name: string,
+  value: number,
+  minimum: number,
+  maximum: number,
+): number {
+  if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
+    throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
+  }
+  return value;
+}
+
+function ownerId(value: string): string {
+  if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value)) {
+    throw new TypeError(
+      'Cluster-control recovery ownerId must contain 1-128 safe identifier characters',
+    );
+  }
+  return value;
+}
+
+function targetKey(candidate: ClusterControlRecoveryCandidate): string {
+  return `${candidate.kind}:${candidate.id}`;
+}
+
+function assertClaim(
+  claim: ClusterControlRecoveryClaim,
+  expectedOwnerId: string,
+): void {
+  if (
+    claim.ownerId !== expectedOwnerId ||
+    typeof claim.token !== 'string' ||
+    claim.token.length < 16 ||
+    claim.token.length > 128 ||
+    !Number.isSafeInteger(claim.version) ||
+    claim.version < 1 ||
+    !Number.isSafeInteger(claim.observedAtMs) ||
+    claim.observedAtMs < 0 ||
+    !Number.isSafeInteger(claim.expiresAtMs) ||
+    claim.expiresAtMs <= claim.observedAtMs ||
+    !claim.candidate ||
+    typeof claim.candidate.id !== 'string' ||
+    claim.candidate.id.length === 0
+  ) {
+    throw new TypeError(
+      'Cluster-control recovery repository returned an invalid claim',
+    );
+  }
+}
+
+function normalizeDisposition(
+  value: ClusterControlRecoveryDisposition,
+): ClusterControlRecoveryDisposition {
+  if (!value || typeof value !== 'object') {
+    throw new TypeError(
+      'Cluster-control recovery processor returned no disposition',
+    );
+  }
+  if (value.status === 'resolved') return Object.freeze({ status: 'resolved' });
+  if (value.status === 'manual') return Object.freeze({ status: 'manual' });
+  if (value.status === 'retry') {
+    return Object.freeze({
+      status: 'retry',
+      delayMs: integerInRange(
+        'Cluster-control recovery retry delay',
+        value.delayMs,
+        0,
+        MAX_CLUSTER_CONTROL_RECOVERY_RETRY_DELAY_MS,
+      ),
+    });
+  }
+  throw new TypeError(
+    'Cluster-control recovery processor returned an invalid disposition',
+  );
+}
+
+/**
+ * Runs one bounded, sequential recovery pass. It owns no timer and never
+ * executes a task; the injected processor may only resolve evidence under the
+ * repository claim fence.
+ */
+export class ClusterControlRecoverySupervisor {
+  private readonly ownerId: string;
+  private readonly limit: number;
+  private readonly leaseMs: number;
+  private readonly retryDelayMs: number;
+
+  constructor(
+    private readonly repository: ClusterControlRecoveryClaimRepository,
+    private readonly processor: ClusterControlRecoveryProcessor,
+    options: ClusterControlRecoverySupervisorOptions,
+  ) {
+    this.ownerId = ownerId(options.ownerId);
+    this.limit = integerInRange(
+      'Cluster-control recovery claim limit',
+      options.limit ?? 16,
+      1,
+      MAX_CLUSTER_CONTROL_RECOVERY_CLAIMS_PER_PASS,
+    );
+    this.leaseMs = integerInRange(
+      'Cluster-control recovery claim lease',
+      options.leaseMs ?? 30_000,
+      1_000,
+      MAX_CLUSTER_CONTROL_RECOVERY_CLAIM_LEASE_MS,
+    );
+    this.retryDelayMs = integerInRange(
+      'Cluster-control recovery retry delay',
+      options.retryDelayMs ?? 5_000,
+      0,
+      MAX_CLUSTER_CONTROL_RECOVERY_RETRY_DELAY_MS,
+    );
+  }
+
+  async reconcile(): Promise<ClusterControlStartupRecoverySummary> {
+    const page = await this.repository.claim({
+      ownerId: this.ownerId,
+      limit: this.limit,
+      leaseMs: this.leaseMs,
+    });
+    if (
+      !Number.isSafeInteger(page.discovered) ||
+      page.discovered < page.claims.length ||
+      page.discovered > this.limit ||
+      typeof page.hasMore !== 'boolean' ||
+      page.claims.length > this.limit
+    ) {
+      throw new TypeError(
+        'Cluster-control recovery repository returned an invalid claim page',
+      );
+    }
+
+    const keys = new Set<string>();
+    for (const claim of page.claims) {
+      assertClaim(claim, this.ownerId);
+      const key = targetKey(claim.candidate);
+      if (keys.has(key)) {
+        throw new TypeError(
+          'Cluster-control recovery repository returned duplicate claims',
+        );
+      }
+      keys.add(key);
+    }
+
+    let remaining =
+      page.discovered - page.claims.length + (page.hasMore ? 1 : 0);
+    let failed = 0;
+    for (const claim of page.claims) {
+      let disposition: ClusterControlRecoveryDisposition;
+      try {
+        disposition = normalizeDisposition(await this.processor.process(claim));
+      } catch {
+        disposition = Object.freeze({
+          status: 'retry',
+          delayMs: this.retryDelayMs,
+        });
+      }
+      const settled = await this.repository.settle(claim, disposition);
+      if (settled === 'fenced') {
+        remaining += 1;
+        continue;
+      }
+      if (settled !== 'settled') {
+        throw new TypeError(
+          'Cluster-control recovery repository returned an invalid settlement',
+        );
+      }
+      if (disposition.status !== 'resolved') {
+        remaining += 1;
+        failed += 1;
+      }
+    }
+    return Object.freeze({
+      safe: remaining === 0 && failed === 0,
+      remaining,
+      failed,
+    });
+  }
+}

packages/ql3-runtime-core/src/cluster-control/clusterControlStartupRecoveryCoordinator.ts

@@ -0,0 +1,78 @@
+import type { ClusterControlStartupRecoverySummary } from './clusterControlActivation';
+
+export const MAX_CLUSTER_CONTROL_STARTUP_RECOVERY_PASSES = 64;
+
+export interface ClusterControlRecoveryPass {
+  reconcile(): Promise<ClusterControlStartupRecoverySummary>;
+}
+
+export interface ClusterControlStartupRecoveryCoordinatorOptions {
+  readonly maxPasses?: number;
+}
+
+function maxPasses(value: number | undefined): number {
+  const normalized = value ?? 8;
+  if (
+    !Number.isSafeInteger(normalized) ||
+    normalized < 1 ||
+    normalized > MAX_CLUSTER_CONTROL_STARTUP_RECOVERY_PASSES
+  ) {
+    throw new RangeError(
+      `Cluster-control startup recovery maxPasses must be between 1 and ${MAX_CLUSTER_CONTROL_STARTUP_RECOVERY_PASSES}`,
+    );
+  }
+  return normalized;
+}
+
+function normalizeSummary(
+  value: ClusterControlStartupRecoverySummary,
+): ClusterControlStartupRecoverySummary {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    typeof value.safe !== 'boolean' ||
+    !Number.isSafeInteger(value.remaining) ||
+    value.remaining < 0 ||
+    !Number.isSafeInteger(value.failed) ||
+    value.failed < 0 ||
+    (value.safe && (value.remaining !== 0 || value.failed !== 0)) ||
+    (!value.safe && value.remaining === 0 && value.failed === 0)
+  ) {
+    throw new TypeError(
+      'Cluster-control recovery pass returned an invalid summary',
+    );
+  }
+  return Object.freeze({
+    safe: value.safe,
+    remaining: value.remaining,
+    failed: value.failed,
+  });
+}
+
+/**
+ * Runs a hard-bounded number of startup passes without a timer or recursion.
+ * Deferred/manual work stops immediately; pure backlog may consume another
+ * page, and the independent durable-source verifier remains the final gate.
+ */
+export class ClusterControlStartupRecoveryCoordinator {
+  private readonly maxPasses: number;
+
+  constructor(
+    private readonly pass: ClusterControlRecoveryPass,
+    options: ClusterControlStartupRecoveryCoordinatorOptions = {},
+  ) {
+    if (!pass || typeof pass.reconcile !== 'function') {
+      throw new TypeError('Cluster-control recovery pass is invalid');
+    }
+    this.maxPasses = maxPasses(options.maxPasses);
+  }
+
+  async reconcile(): Promise<ClusterControlStartupRecoverySummary> {
+    let latest: ClusterControlStartupRecoverySummary | undefined;
+    for (let index = 0; index < this.maxPasses; index += 1) {
+      latest = normalizeSummary(await this.pass.reconcile());
+      if (latest.safe || latest.failed > 0) return latest;
+    }
+    return latest!;
+  }
+}