feat(ql3): recover terminal secret action jobs

packages/ql3-cluster-admin/src/plugin-package/executor/pluginPackageExecutorProcess.ts

@@ -17,7 +17,9 @@ import {
   loadPostgresConnectionEnvironment,
   PostgresApprovedActionExecutionRepository,
   PostgresPluginPackageSecretBindingApprovalPlanReader,
+  PostgresPluginPackageSecretBindingRepository,
   PostgresPluginPackageSecretBindingTransitionApprovalPlanReader,
+  PostgresPluginPackageSecretBindingTransitionRepository,
   type PostgresConnectionOptions,
   type PostgresPoolOptions,
   type PostgresSchemaReadinessReport,
@@ -631,6 +633,9 @@ function emptySecretActionJobSummary(): Readonly<PluginPackageKubernetesSecretAc
     created: 0,
     existing: 0,
     active: 0,
+    recoveredSucceeded: 0,
+    recoveredFailed: 0,
+    recoveredBlocked: 0,
     recoveryRequired: 0,
     unavailable: 0,
     truncated: false,
@@ -796,6 +801,13 @@ export async function runClusterPluginPackageExecutorProcess(
               new PostgresPluginPackageSecretBindingTransitionApprovalPlanReader(
                 database.pool,
               ),
+            bindings: new PostgresPluginPackageSecretBindingRepository(
+              database.pool,
+            ),
+            transitionReceipts:
+              new PostgresPluginPackageSecretBindingTransitionRepository(
+                database.pool,
+              ),
             job: config.kubernetesSecretActions.job,
             ...(options.now ? { now: options.now } : {}),
           });

packages/ql3-cluster-admin/src/plugin-package/executor/pluginPackageKubernetesSecretActionController.ts

@@ -1,12 +1,27 @@
-import type { ApprovedActionExecutionSnapshot } from '@qinglong/runtime-core/approved-action-execution';
+import type {
+  ApprovedActionExecutionSnapshot,
+  ClaimApprovedActionExecutionCommand,
+  ClaimApprovedActionExecutionResult,
+  CompleteApprovedActionExecutionCommand,
+  ReleaseApprovedActionExecutionBeforeStartCommand,
+} from '@qinglong/runtime-core/approved-action-execution';
 import {
   PLUGIN_PACKAGE_SECRET_BINDING_ACTION_TYPE,
+  createPluginPackageSecretBindingFromApprovalPlan,
+  normalizePluginPackageSecretBindingApprovalPlan,
   type PluginPackageSecretBindingApprovalPlan,
 } from '@qinglong/runtime-core/plugin-package-secret-binding-approval-plan';
+import type { PluginPackageSecretBinding } from '@qinglong/runtime-core/plugin-package-secret-binding';
 import {
   PLUGIN_PACKAGE_SECRET_BINDING_TRANSITION_ACTION_TYPE,
+  normalizePluginPackageSecretBindingTransitionApprovalPlan,
   type PluginPackageSecretBindingTransitionApprovalPlan,
 } from '@qinglong/runtime-core/plugin-package-secret-binding-transition-approval-plan';
+import {
+  createPluginPackageSecretBindingFromTransitionPlan,
+  createPluginPackageSecretBindingTransitionReceipt,
+  type PluginPackageSecretBindingTransitionReceipt,
+} from '@qinglong/runtime-core/plugin-package-secret-binding-transition-receipt';
 
 import {
   createPluginPackageKubernetesSecretActionJob,
@@ -14,6 +29,8 @@ import {
 } from './pluginPackageKubernetesSecretActionJob';
 
 const FIELD_MANAGER = 'qinglong-plugin-package-secret-action-controller';
+const RECOVERY_OWNER = 'qinglong-secret-action-controller';
+const RECOVERY_LEASE_DURATION_MS = 60_000;
 const MAX_PAGE_SIZE = 32;
 
 type SecretActionApprovalPlan =
@@ -26,7 +43,7 @@ export interface PluginPackageSecretActionApprovalPlanReader<T> {
   findByActionRef(actionRef: string): Promise<Readonly<T> | null>;
 }
 
-export interface PluginPackageKubernetesSecretActionExecutionReader {
+export interface PluginPackageKubernetesSecretActionExecutionPort {
   listReconciliableExecutions(query: Readonly<{
     nowMs: number;
     limit: number;
@@ -35,6 +52,19 @@ export interface PluginPackageKubernetesSecretActionExecutionReader {
     executions: readonly Readonly<ApprovedActionExecutionSnapshot>[];
     truncated: boolean;
   }>>;
+  completeExecution(
+    command: CompleteApprovedActionExecutionCommand,
+  ): Promise<Readonly<ApprovedActionExecutionSnapshot>>;
+  claimExecution(
+    command: ClaimApprovedActionExecutionCommand,
+  ): Promise<ClaimApprovedActionExecutionResult>;
+  releaseExecutionBeforeStart(
+    command: ReleaseApprovedActionExecutionBeforeStartCommand,
+  ): Promise<Readonly<ApprovedActionExecutionSnapshot>>;
+}
+
+export interface PluginPackageSecretActionDurableResultReader<T> {
+  find(generationDigest: string): Promise<Readonly<T> | null>;
 }
 
 export interface PluginPackageKubernetesSecretActionJobResource {
@@ -71,9 +101,11 @@ export interface PluginPackageKubernetesSecretActionJobApi {
 }
 
 export interface PluginPackageKubernetesSecretActionControllerOptions {
-  readonly executions: PluginPackageKubernetesSecretActionExecutionReader;
+  readonly executions: PluginPackageKubernetesSecretActionExecutionPort;
   readonly bindingPlans: PluginPackageSecretActionApprovalPlanReader<PluginPackageSecretBindingApprovalPlan>;
   readonly transitionPlans: PluginPackageSecretActionApprovalPlanReader<PluginPackageSecretBindingTransitionApprovalPlan>;
+  readonly bindings: PluginPackageSecretActionDurableResultReader<PluginPackageSecretBinding>;
+  readonly transitionReceipts: PluginPackageSecretActionDurableResultReader<PluginPackageSecretBindingTransitionReceipt>;
   readonly jobs: PluginPackageKubernetesSecretActionJobApi;
   readonly job: Readonly<PluginPackageKubernetesSecretActionJobOptions>;
   readonly now?: () => number;
@@ -84,6 +116,9 @@ export interface PluginPackageKubernetesSecretActionControllerSummary {
   readonly created: number;
   readonly existing: number;
   readonly active: number;
+  readonly recoveredSucceeded: number;
+  readonly recoveredFailed: number;
+  readonly recoveredBlocked: number;
   readonly recoveryRequired: number;
   readonly unavailable: number;
   readonly truncated: boolean;
@@ -214,10 +249,17 @@ export class PluginPackageKubernetesSecretActionController {
       typeof options !== 'object' ||
       !options.executions ||
       typeof options.executions.listReconciliableExecutions !== 'function' ||
+      typeof options.executions.completeExecution !== 'function' ||
+      typeof options.executions.claimExecution !== 'function' ||
+      typeof options.executions.releaseExecutionBeforeStart !== 'function' ||
       !options.bindingPlans ||
       typeof options.bindingPlans.findByActionRef !== 'function' ||
       !options.transitionPlans ||
       typeof options.transitionPlans.findByActionRef !== 'function' ||
+      !options.bindings ||
+      typeof options.bindings.find !== 'function' ||
+      !options.transitionReceipts ||
+      typeof options.transitionReceipts.find !== 'function' ||
       !options.jobs ||
       typeof options.jobs.createNamespacedJob !== 'function' ||
       typeof options.jobs.readNamespacedJob !== 'function' ||
@@ -269,6 +311,9 @@ export class PluginPackageKubernetesSecretActionController {
       created: 0,
       existing: 0,
       active: 0,
+      recoveredSucceeded: 0,
+      recoveredFailed: 0,
+      recoveredBlocked: 0,
       recoveryRequired: 0,
       unavailable: 0,
       truncated: page.truncated,
@@ -293,7 +338,15 @@ export class PluginPackageKubernetesSecretActionController {
   async #reconcileOne(
     snapshot: Readonly<ApprovedActionExecutionSnapshot>,
     nowMs: number,
-  ): Promise<'created' | 'existing' | 'active' | 'recoveryRequired'> {
+  ): Promise<
+    | 'created'
+    | 'existing'
+    | 'active'
+    | 'recoveredSucceeded'
+    | 'recoveredFailed'
+    | 'recoveredBlocked'
+    | 'recoveryRequired'
+  > {
     const plan = await this.#plan(snapshot);
     if (!plan) {
       throw new PluginPackageKubernetesSecretActionControllerUnavailableError();
@@ -313,10 +366,22 @@ export class PluginPackageKubernetesSecretActionController {
     } catch (error) {
       if (apiStatus(error) !== 404) throw error;
       if (
-        snapshot.execution.status === 'executing' ||
-        nowMs > plan.expiresAtMs
+        snapshot.execution.status === 'executing'
       ) {
-        return 'recoveryRequired';
+        return this.#recoverMissingExecuting(
+          snapshot,
+          plan,
+          name,
+          nowMs,
+        );
+      }
+      if (nowMs > plan.expiresAtMs) {
+        return this.#blockBeforeStart(
+          snapshot,
+          name,
+          nowMs,
+          'package_secret_action_approval_expired',
+        );
       }
       try {
         observed = await this.options.jobs.createNamespacedJob({
@@ -343,10 +408,213 @@ export class PluginPackageKubernetesSecretActionController {
     }
     assertObservedJob(desired, observed);
     const terminal = terminalStatus(observed);
-    if (terminal !== 'active') return 'recoveryRequired';
+    if (terminal !== 'active') {
+      return this.#recoverTerminal(snapshot, plan, name, terminal, nowMs);
+    }
     return disposition;
   }
 
+  async #recoverTerminal(
+    snapshot: Readonly<ApprovedActionExecutionSnapshot>,
+    plan: Readonly<SecretActionApprovalPlan>,
+    jobName: string,
+    terminal: 'complete' | 'failed',
+    nowMs: number,
+  ): Promise<
+    'recoveredSucceeded' | 'recoveredFailed' | 'recoveredBlocked' | 'recoveryRequired'
+  > {
+    const execution = snapshot.execution;
+    if (execution.status !== 'executing') {
+      return this.#blockBeforeStart(
+        snapshot,
+        jobName,
+        nowMs,
+        terminal === 'failed'
+          ? 'package_secret_action_job_failed_before_start'
+          : 'package_secret_action_job_completed_before_start',
+      );
+    }
+    if (
+      execution.startedAtMs === null ||
+      execution.leaseOwner === null ||
+      execution.leaseToken === null
+    ) {
+      throw new PluginPackageKubernetesSecretActionControllerConflictError();
+    }
+    const durableResultDigest = await this.#durableResultDigest(
+      plan,
+      execution.startedAtMs,
+    );
+    const outcome = durableResultDigest
+      ? 'succeeded'
+      : terminal === 'failed'
+        ? 'failed'
+        : 'indeterminate';
+    return this.#completeExecuting(
+      snapshot,
+      jobName,
+      nowMs,
+      outcome,
+      durableResultDigest
+        ? snapshot.dispatch.action.actionType ===
+          PLUGIN_PACKAGE_SECRET_BINDING_ACTION_TYPE
+          ? 'package_secret_binding_job_recovered'
+          : 'package_secret_transition_job_recovered'
+        : terminal === 'failed'
+          ? 'package_secret_action_job_failed'
+          : 'package_secret_action_receipt_missing',
+      durableResultDigest,
+    );
+  }
+
+  async #recoverMissingExecuting(
+    snapshot: Readonly<ApprovedActionExecutionSnapshot>,
+    plan: Readonly<SecretActionApprovalPlan>,
+    jobName: string,
+    nowMs: number,
+  ): Promise<'recoveredSucceeded' | 'recoveryRequired'> {
+    const execution = snapshot.execution;
+    if (
+      execution.status !== 'executing' ||
+      execution.startedAtMs === null ||
+      execution.leaseOwner === null ||
+      execution.leaseToken === null
+    ) {
+      throw new PluginPackageKubernetesSecretActionControllerConflictError();
+    }
+    const durableResultDigest = await this.#durableResultDigest(
+      plan,
+      execution.startedAtMs,
+    );
+    if (!durableResultDigest) return 'recoveryRequired';
+    const recovered = await this.#completeExecuting(
+      snapshot,
+      jobName,
+      nowMs,
+      'succeeded',
+      snapshot.dispatch.action.actionType ===
+        PLUGIN_PACKAGE_SECRET_BINDING_ACTION_TYPE
+        ? 'package_secret_binding_job_recovered'
+        : 'package_secret_transition_job_recovered',
+      durableResultDigest,
+    );
+    if (recovered !== 'recoveredSucceeded') {
+      throw new PluginPackageKubernetesSecretActionControllerConflictError();
+    }
+    return recovered;
+  }
+
+  async #completeExecuting(
+    snapshot: Readonly<ApprovedActionExecutionSnapshot>,
+    jobName: string,
+    nowMs: number,
+    outcome: 'succeeded' | 'failed' | 'indeterminate',
+    resultCode: string,
+    resultDigest: string | null,
+  ): Promise<'recoveredSucceeded' | 'recoveredFailed' | 'recoveredBlocked'> {
+    const execution = snapshot.execution;
+    if (
+      execution.status !== 'executing' ||
+      execution.leaseOwner === null ||
+      execution.leaseToken === null ||
+      (outcome === 'succeeded') !== (resultDigest !== null)
+    ) {
+      throw new PluginPackageKubernetesSecretActionControllerConflictError();
+    }
+    const result = await this.options.executions.completeExecution({
+      dispatchId: execution.dispatchId,
+      owner: execution.leaseOwner,
+      leaseToken: execution.leaseToken,
+      expectedVersion: execution.version,
+      resultMutationId: `k8s-recovery-${jobName}`,
+      outcome,
+      resultCode,
+      ...(resultDigest ? { resultDigest } : {}),
+      completedAtMs: Math.max(nowMs, execution.updatedAtMs),
+    });
+    if (result.execution.status === 'succeeded') return 'recoveredSucceeded';
+    if (result.execution.status === 'failed') return 'recoveredFailed';
+    if (result.execution.status === 'blocked') return 'recoveredBlocked';
+    throw new PluginPackageKubernetesSecretActionControllerConflictError();
+  }
+
+  async #blockBeforeStart(
+    snapshot: Readonly<ApprovedActionExecutionSnapshot>,
+    jobName: string,
+    nowMs: number,
+    resultCode: string,
+  ): Promise<'recoveredBlocked' | 'recoveryRequired'> {
+    const claimed = await this.options.executions.claimExecution({
+      dispatchId: snapshot.execution.dispatchId,
+      owner: RECOVERY_OWNER,
+      leaseToken: `recovery-${jobName}`,
+      nowMs,
+      leaseDurationMs: RECOVERY_LEASE_DURATION_MS,
+    });
+    if (claimed.status !== 'claimed') {
+      return claimed.status === 'blocked' ? 'recoveredBlocked' : 'recoveryRequired';
+    }
+    const released = await this.options.executions.releaseExecutionBeforeStart({
+      dispatchId: claimed.snapshot.execution.dispatchId,
+      owner: RECOVERY_OWNER,
+      leaseToken: claimed.snapshot.execution.leaseToken!,
+      expectedVersion: claimed.snapshot.execution.version,
+      resultMutationId: `k8s-recovery-${jobName}`,
+      resultCode,
+      atMs: Math.max(nowMs, claimed.snapshot.execution.updatedAtMs),
+    });
+    if (released.execution.status !== 'blocked') {
+      throw new PluginPackageKubernetesSecretActionControllerConflictError();
+    }
+    return 'recoveredBlocked';
+  }
+
+  async #durableResultDigest(
+    plan: Readonly<SecretActionApprovalPlan>,
+    startedAtMs: number,
+  ): Promise<string | null> {
+    if ('bindingPlan' in plan) {
+      const normalized = normalizePluginPackageSecretBindingApprovalPlan(plan);
+      const expected = createPluginPackageSecretBindingFromApprovalPlan(
+        normalized,
+        startedAtMs,
+      );
+      const stored = await this.options.bindings.find(
+        normalized.bindingPlan.target.generationDigest,
+      );
+      if (!stored) return null;
+      if (JSON.stringify(stored) !== JSON.stringify(expected)) {
+        throw new PluginPackageKubernetesSecretActionControllerConflictError();
+      }
+      return stored.bindingDigest;
+    }
+    const normalized =
+      normalizePluginPackageSecretBindingTransitionApprovalPlan(plan);
+    const binding = createPluginPackageSecretBindingFromTransitionPlan(
+      normalized.transitionPlan,
+      'approved-action-execution',
+      normalized.approvalPlanDigest,
+      startedAtMs,
+    );
+    const expected = createPluginPackageSecretBindingTransitionReceipt({
+      transitionPlan: normalized.transitionPlan,
+      authority: Object.freeze({
+        kind: 'approved-action-execution',
+        evidenceDigest: normalized.approvalPlanDigest,
+      }),
+      binding,
+      committedAtMs: startedAtMs,
+    });
+    const stored = await this.options.transitionReceipts.find(
+      normalized.transitionPlan.nextTarget.generationDigest,
+    );
+    if (!stored) return null;
+    if (JSON.stringify(stored) !== JSON.stringify(expected)) {
+      throw new PluginPackageKubernetesSecretActionControllerConflictError();
+    }
+    return stored.receiptDigest;
+  }
+
   #plan(
     snapshot: Readonly<ApprovedActionExecutionSnapshot>,
   ): Promise<Readonly<SecretActionApprovalPlan> | null> {

packages/ql3-cluster-admin/test/pluginPackageExecutorProcess.test.cjs

@@ -253,6 +253,9 @@ test('batch mode consumes approvals before reconciling exact Secret action Jobs'
     },
     async createSecretActionController(options) {
       assert.equal(options.job.image.endsWith('c'.repeat(64)), true);
+      assert.equal(typeof options.executions.completeExecution, 'function');
+      assert.equal(typeof options.bindings.find, 'function');
+      assert.equal(typeof options.transitionReceipts.find, 'function');
       calls.push('controller-open');
       return {
         controller: {
@@ -263,6 +266,9 @@ test('batch mode consumes approvals before reconciling exact Secret action Jobs'
               created: 1,
               existing: 0,
               active: 0,
+              recoveredSucceeded: 0,
+              recoveredFailed: 0,
+              recoveredBlocked: 0,
               recoveryRequired: 0,
               unavailable: 0,
               truncated: false,

packages/ql3-cluster-admin/test/pluginPackageKubernetesSecretActionController.test.cjs

@@ -9,24 +9,44 @@ const {
   decideApprovalRequest,
 } = require('@qinglong/runtime-core/approved-action');
 const {
+  claimApprovedActionExecution,
+  completeApprovedActionExecution,
   createApprovedActionExecution,
+  releaseApprovedActionExecutionBeforeStart,
+  startApprovedActionExecution,
 } = require('@qinglong/runtime-core/approved-action-execution');
 const {
   createPluginPackageResourceGeneration,
 } = require('@qinglong/runtime-core/plugin-package-resource-generation');
 const {
+  createPluginPackageSecretBindingFromApprovalPlan,
   createPluginPackageSecretBindingApprovalPlan,
 } = require('@qinglong/runtime-core/plugin-package-secret-binding-approval-plan');
 const {
   createPluginPackageSecretBindingPlan,
 } = require('@qinglong/runtime-core/plugin-package-secret-binding-plan');
 const {
-  createSecretRef,
-} = require('@qinglong/runtime-core/secret-reference');
+  createPluginPackageSecretBinding,
+} = require('@qinglong/runtime-core/plugin-package-secret-binding');
+const {
+  createPluginPackageSecretBindingTransitionApprovalPlan,
+  pluginPackageSecretBindingTransitionApprovedAction,
+} = require('@qinglong/runtime-core/plugin-package-secret-binding-transition-approval-plan');
+const {
+  createPluginPackageSecretBindingTransitionPlan,
+} = require('@qinglong/runtime-core/plugin-package-secret-binding-transition-plan');
+const {
+  createPluginPackageSecretBindingFromTransitionPlan,
+  createPluginPackageSecretBindingTransitionReceipt,
+} = require('@qinglong/runtime-core/plugin-package-secret-binding-transition-receipt');
+const { createSecretRef } = require('@qinglong/runtime-core/secret-reference');
 const {
   PluginPackageKubernetesSecretActionController,
   PluginPackageKubernetesSecretActionControllerConflictError,
 } = require('@qinglong/cluster-admin/plugin-package-kubernetes-secret-action-controller');
+const {
+  createPluginPackageKubernetesSecretActionJob,
+} = require('@qinglong/cluster-admin/plugin-package-kubernetes-secret-action-job');
 
 const REQUESTER = Object.freeze({ type: 'user', id: 'cluster-owner' });
 const REVIEWER = Object.freeze({ type: 'user', id: 'security-reviewer' });
@@ -97,8 +117,10 @@ function fixture() {
     requestedBy: REQUESTER,
     expiresAtMs: 1_000,
   });
-  const action = require('@qinglong/runtime-core/plugin-package-secret-binding-approval-plan')
-    .pluginPackageSecretBindingApprovedAction(plan);
+  const action =
+    require('@qinglong/runtime-core/plugin-package-secret-binding-approval-plan').pluginPackageSecretBindingApprovedAction(
+      plan,
+    );
   const approved = decideApprovalRequest(
     createApprovalRequest({
       id: 'approval-controller-1',
@@ -167,12 +189,197 @@ function jobOptions() {
   };
 }
 
+function transitionFixture() {
+  const manifest = (version) => ({
+    apiVersion: 'qinglong.io/v1alpha1',
+    kind: 'Package',
+    metadata: {
+      name: 'controller-transition',
+      displayName: 'Controller Transition',
+      version,
+      description: 'Secret action transition controller fixture',
+      license: 'Apache-2.0',
+    },
+    spec: {
+      compatibility: {
+        qinglong: '>=3.0.0-0 <4.0.0',
+        architectures: ['arm64'],
+        deploymentProfiles: ['cluster-control'],
+      },
+      runtimes: [],
+      resources: {
+        memory: { recommended: '32Mi' },
+        disk: { install: '4Mi', working: '8Mi' },
+      },
+      permissions: {
+        network: { allowedHosts: [] },
+        secrets: [{ name: 'TOKEN', required: true }],
+        tools: ['secret.use'],
+      },
+      contents: { tasks: [], workflows: [], prompts: [], tools: [] },
+    },
+  });
+  const previousManifest = manifest('1.0.0');
+  const previousGeneration = createPluginPackageResourceGeneration({
+    installationId: 'install-controller-transition-v1',
+    projectId: 'project-1',
+    packageName: 'controller-transition',
+    lockDigest: '1'.repeat(64),
+    generation: 1,
+    previousActiveLockDigest: null,
+    contentDigest: '2'.repeat(64),
+    contents: previousManifest.spec.contents,
+  });
+  const previousBinding = createPluginPackageSecretBinding({
+    generation: previousGeneration,
+    manifest: previousManifest,
+    assignments: [
+      {
+        name: 'TOKEN',
+        secretRef: createSecretRef({
+          projectId: 'project-1',
+          name: 'runtime-token',
+          version: 1,
+        }),
+      },
+    ],
+    authority: {
+      kind: 'approved-action-execution',
+      evidenceDigest: '3'.repeat(64),
+    },
+    boundAtMs: 80,
+  });
+  const nextManifest = manifest('2.0.0');
+  const transitionPlan = createPluginPackageSecretBindingTransitionPlan({
+    previousTarget: previousBinding.target,
+    previousBinding,
+    previousAttemptGeneration: 1,
+    nextGeneration: createPluginPackageResourceGeneration({
+      installationId: 'install-controller-transition-v2',
+      projectId: 'project-1',
+      packageName: 'controller-transition',
+      lockDigest: '4'.repeat(64),
+      generation: 2,
+      previousActiveLockDigest: '1'.repeat(64),
+      contentDigest: '5'.repeat(64),
+      contents: nextManifest.spec.contents,
+    }),
+    nextManifest,
+    assignments: [
+      {
+        name: 'TOKEN',
+        secretRef: createSecretRef({
+          projectId: 'project-1',
+          name: 'runtime-token',
+          version: 2,
+        }),
+      },
+    ],
+    plannedAtMs: 100,
+  });
+  const plan = createPluginPackageSecretBindingTransitionApprovalPlan({
+    actionRef: 'secret-transition:controller-v2',
+    transitionPlan,
+    requestedBy: REQUESTER,
+    plannedAtMs: 100,
+    expiresAtMs: 1_000,
+  });
+  const action = pluginPackageSecretBindingTransitionApprovedAction(plan);
+  const approved = decideApprovalRequest(
+    createApprovalRequest({
+      id: 'approval-controller-transition',
+      projectId: 'project-1',
+      action,
+      risk: 'high',
+      decisionMode: 'separation_of_duty',
+      requestedBy: REQUESTER,
+      requestedAtMs: 110,
+      expiresAtMs: 900,
+      requestFence: FENCE,
+    }),
+    {
+      expectedVersion: 1,
+      decisionId: 'decision-controller-transition',
+      decision: 'approved',
+      reasonCode: 'reviewed',
+      principal: {
+        subject: REVIEWER,
+        authenticationId: 'auth-transition-reviewer',
+        authenticatedAtMs: 100,
+        expiresAtMs: 800,
+        assurance: 'multi_factor',
+      },
+      decidedAtMs: 120,
+      authorizationFence: FENCE,
+    },
+  );
+  const dispatch = consumeApprovalRequest(approved, {
+    expectedVersion: 2,
+    consumptionId: 'consume-controller-transition',
+    dispatchId: 'dispatch-controller-transition',
+    action,
+    requestedBy: REQUESTER,
+    consumedBy: CONSUMER,
+    consumedAtMs: 130,
+    authorizationFence: FENCE,
+  }).dispatch;
+  return {
+    plan,
+    snapshot: Object.freeze({
+      dispatch,
+      execution: createApprovedActionExecution(dispatch),
+    }),
+  };
+}
+
 function apiError(code) {
   return Object.assign(new Error(`Kubernetes ${code}`), { code });
 }
 
-function controller({ read, create, now = 200, snapshot: snapshotOverride }) {
+function executingSnapshot(snapshot, startedAtMs = 150) {
+  const leased = claimApprovedActionExecution(snapshot.execution, {
+    owner: 'secret-action-job',
+    leaseToken: 'secret-action-lease',
+    nowMs: 140,
+    leaseDurationMs: 100,
+  });
+  return Object.freeze({
+    dispatch: snapshot.dispatch,
+    execution: startApprovedActionExecution(
+      { dispatch: snapshot.dispatch, execution: leased },
+      {
+        dispatchId: snapshot.dispatch.id,
+        approvalRequestId: snapshot.dispatch.approvalRequestId,
+        actionDigest: snapshot.dispatch.action.actionDigest,
+        owner: leased.leaseOwner,
+        leaseToken: leased.leaseToken,
+        expectedVersion: leased.version,
+        startedAtMs,
+      },
+    ),
+  });
+}
+
+function desiredJob(plan, snapshot) {
+  return createPluginPackageKubernetesSecretActionJob({
+    dispatch: snapshot.dispatch,
+    approvalPlan: plan,
+    options: jobOptions(),
+  });
+}
+
+function controller({
+  read,
+  create,
+  complete,
+  findBinding,
+  findTransitionReceipt,
+  now = 200,
+  plan: planOverride,
+  snapshot: snapshotOverride,
+}) {
   const { plan, snapshot } = fixture();
+  const selectedPlan = planOverride ?? plan;
   return new PluginPackageKubernetesSecretActionController({
     executions: {
       async listReconciliableExecutions(query) {
@@ -185,16 +392,72 @@ function controller({ read, create, now = 200, snapshot: snapshotOverride }) {
           truncated: false,
         };
       },
+      async completeExecution(command) {
+        if (!complete) throw new Error('completion must not run');
+        return complete(command, snapshotOverride ?? snapshot);
+      },
+      async claimExecution(command) {
+        const current = snapshotOverride ?? snapshot;
+        const execution = claimApprovedActionExecution(current.execution, {
+          owner: command.owner,
+          leaseToken: command.leaseToken,
+          nowMs: command.nowMs,
+          leaseDurationMs: command.leaseDurationMs,
+        });
+        return {
+          status: 'claimed',
+          snapshot: { dispatch: current.dispatch, execution },
+        };
+      },
+      async releaseExecutionBeforeStart(command) {
+        const current = snapshotOverride ?? snapshot;
+        const claimed = claimApprovedActionExecution(current.execution, {
+          owner: command.owner,
+          leaseToken: command.leaseToken,
+          nowMs: command.atMs,
+          leaseDurationMs: 60_000,
+        });
+        return {
+          dispatch: current.dispatch,
+          execution: releaseApprovedActionExecutionBeforeStart(claimed, {
+            owner: command.owner,
+            leaseToken: command.leaseToken,
+            expectedVersion: command.expectedVersion,
+            resultMutationId: command.resultMutationId,
+            resultCode: command.resultCode,
+            atMs: command.atMs,
+          }),
+        };
+      },
     },
     bindingPlans: {
       async findByActionRef(actionRef) {
-        assert.equal(actionRef, plan.actionRef);
-        return plan;
+        if (!('bindingPlan' in selectedPlan)) {
+          throw new Error('binding reader must not run');
+        }
+        assert.equal(actionRef, selectedPlan.actionRef);
+        return selectedPlan;
       },
     },
     transitionPlans: {
-      async findByActionRef() {
-        throw new Error('transition reader must not run');
+      async findByActionRef(actionRef) {
+        if ('bindingPlan' in selectedPlan) {
+          throw new Error('transition reader must not run');
+        }
+        assert.equal(actionRef, selectedPlan.actionRef);
+        return selectedPlan;
+      },
+    },
+    bindings: {
+      async find(generationDigest) {
+        return findBinding ? findBinding(generationDigest) : null;
+      },
+    },
+    transitionReceipts: {
+      async find(generationDigest) {
+        return findTransitionReceipt
+          ? findTransitionReceipt(generationDigest)
+          : null;
       },
     },
     jobs: {
@@ -228,7 +491,10 @@ test('creates one Strict deterministic Job without claiming the execution', asyn
     calls[1][1].fieldManager,
     'qinglong-plugin-package-secret-action-controller',
   );
-  assert.match(calls[1][1].body.metadata.name, /^ql3-package-secret-[0-9a-f]{32}$/);
+  assert.match(
+    calls[1][1].body.metadata.name,
+    /^ql3-package-secret-[0-9a-f]{32}$/,
+  );
 });
 
 test('converges a concurrent create through one exact get', async () => {
@@ -284,10 +550,7 @@ test('does not recreate a missing Job for an already executing action', async ()
   const { snapshot } = fixture();
   let creates = 0;
   const subject = controller({
-    snapshot: {
-      ...snapshot,
-      execution: { ...snapshot.execution, status: 'executing' },
-    },
+    snapshot: executingSnapshot(snapshot),
     async read() {
       throw apiError(404);
     },
@@ -301,7 +564,47 @@ test('does not recreate a missing Job for an already executing action', async ()
   assert.equal(creates, 0);
 });
 
-test('does not create a missing Job after the approval plan expires', async () => {
+test('recovers a missing executing Job when its exact durable binding exists', async () => {
+  const { plan, snapshot } = fixture();
+  const executing = executingSnapshot(snapshot);
+  const durable = createPluginPackageSecretBindingFromApprovalPlan(
+    plan,
+    executing.execution.startedAtMs,
+  );
+  const subject = controller({
+    snapshot: executing,
+    async read() {
+      throw apiError(404);
+    },
+    async create() {
+      throw new Error('must not create');
+    },
+    async findBinding() {
+      return durable;
+    },
+    async complete(command) {
+      assert.equal(command.outcome, 'succeeded');
+      return {
+        dispatch: executing.dispatch,
+        execution: completeApprovedActionExecution(executing.execution, {
+          owner: command.owner,
+          leaseToken: command.leaseToken,
+          expectedVersion: command.expectedVersion,
+          resultMutationId: command.resultMutationId,
+          outcome: command.outcome,
+          resultCode: command.resultCode,
+          resultDigest: command.resultDigest,
+          completedAtMs: command.completedAtMs,
+        }),
+      };
+    },
+  });
+  const result = await subject.reconcile();
+  assert.equal(result.recoveredSucceeded, 1);
+  assert.equal(result.recoveryRequired, 0);
+});
+
+test('blocks a missing Job after the approval plan expires before start', async () => {
   let creates = 0;
   const subject = controller({
     now: 1_001,
@@ -314,11 +617,12 @@ test('does not create a missing Job after the approval plan expires', async () =
     },
   });
   const result = await subject.reconcile();
-  assert.equal(result.recoveryRequired, 1);
+  assert.equal(result.recoveredBlocked, 1);
+  assert.equal(result.recoveryRequired, 0);
   assert.equal(creates, 0);
 });
 
-test('marks a terminal Job with a nonterminal execution as recovery required', async () => {
+test('blocks a terminal Job before the execution start barrier', async () => {
   let desired;
   const subject = controller({
     async read() {
@@ -336,10 +640,222 @@ test('marks a terminal Job with a nonterminal execution as recovery required', a
     },
   });
   const result = await subject.reconcile();
-  assert.equal(result.recoveryRequired, 1);
+  assert.equal(result.recoveredBlocked, 1);
+  assert.equal(result.recoveryRequired, 0);
   assert.equal(result.created, 0);
 });
 
+test('recovers an executing terminal Job as succeeded from its exact durable binding', async () => {
+  const { plan, snapshot } = fixture();
+  const executing = executingSnapshot(snapshot);
+  const desired = desiredJob(plan, executing);
+  const durable = createPluginPackageSecretBindingFromApprovalPlan(
+    plan,
+    executing.execution.startedAtMs,
+  );
+  let completion;
+  const subject = controller({
+    snapshot: executing,
+    async read() {
+      return {
+        ...desired,
+        status: { conditions: [{ type: 'Complete', status: 'True' }] },
+      };
+    },
+    async create() {
+      throw new Error('must not create');
+    },
+    async findBinding(generationDigest) {
+      assert.equal(generationDigest, plan.bindingPlan.target.generationDigest);
+      return durable;
+    },
+    async complete(command) {
+      completion = command;
+      return {
+        dispatch: executing.dispatch,
+        execution: completeApprovedActionExecution(executing.execution, {
+          owner: command.owner,
+          leaseToken: command.leaseToken,
+          expectedVersion: command.expectedVersion,
+          resultMutationId: command.resultMutationId,
+          outcome: command.outcome,
+          resultCode: command.resultCode,
+          resultDigest: command.resultDigest,
+          completedAtMs: command.completedAtMs,
+        }),
+      };
+    },
+  });
+  const result = await subject.reconcile();
+  assert.equal(result.recoveredSucceeded, 1);
+  assert.equal(result.recoveryRequired, 0);
+  assert.equal(completion.outcome, 'succeeded');
+  assert.equal(completion.resultDigest, durable.bindingDigest);
+  assert.equal(completion.resultCode, 'package_secret_binding_job_recovered');
+});
+
+test('recovers a failed executing Job without a durable mutation as failed', async () => {
+  const { plan, snapshot } = fixture();
+  const executing = executingSnapshot(snapshot);
+  const desired = desiredJob(plan, executing);
+  const subject = controller({
+    snapshot: executing,
+    async read() {
+      return {
+        ...desired,
+        status: { conditions: [{ type: 'Failed', status: 'True' }] },
+      };
+    },
+    async create() {
+      throw new Error('must not create');
+    },
+    async complete(command) {
+      assert.equal(command.outcome, 'failed');
+      return {
+        dispatch: executing.dispatch,
+        execution: completeApprovedActionExecution(executing.execution, {
+          owner: command.owner,
+          leaseToken: command.leaseToken,
+          expectedVersion: command.expectedVersion,
+          resultMutationId: command.resultMutationId,
+          outcome: command.outcome,
+          resultCode: command.resultCode,
+          completedAtMs: command.completedAtMs,
+        }),
+      };
+    },
+  });
+  const result = await subject.reconcile();
+  assert.equal(result.recoveredFailed, 1);
+  assert.equal(result.recoveryRequired, 0);
+});
+
+test('recovers a terminal transition Job from its exact durable receipt', async () => {
+  const { plan, snapshot } = transitionFixture();
+  const executing = executingSnapshot(snapshot);
+  const desired = desiredJob(plan, executing);
+  const binding = createPluginPackageSecretBindingFromTransitionPlan(
+    plan.transitionPlan,
+    'approved-action-execution',
+    plan.approvalPlanDigest,
+    executing.execution.startedAtMs,
+  );
+  const receipt = createPluginPackageSecretBindingTransitionReceipt({
+    transitionPlan: plan.transitionPlan,
+    authority: {
+      kind: 'approved-action-execution',
+      evidenceDigest: plan.approvalPlanDigest,
+    },
+    binding,
+    committedAtMs: executing.execution.startedAtMs,
+  });
+  const subject = controller({
+    plan,
+    snapshot: executing,
+    async read() {
+      return {
+        ...desired,
+        status: { conditions: [{ type: 'Failed', status: 'True' }] },
+      };
+    },
+    async create() {
+      throw new Error('must not create');
+    },
+    async findTransitionReceipt(generationDigest) {
+      assert.equal(
+        generationDigest,
+        plan.transitionPlan.nextTarget.generationDigest,
+      );
+      return receipt;
+    },
+    async complete(command) {
+      assert.equal(command.outcome, 'succeeded');
+      assert.equal(command.resultDigest, receipt.receiptDigest);
+      return {
+        dispatch: executing.dispatch,
+        execution: completeApprovedActionExecution(executing.execution, {
+          owner: command.owner,
+          leaseToken: command.leaseToken,
+          expectedVersion: command.expectedVersion,
+          resultMutationId: command.resultMutationId,
+          outcome: command.outcome,
+          resultCode: command.resultCode,
+          resultDigest: command.resultDigest,
+          completedAtMs: command.completedAtMs,
+        }),
+      };
+    },
+  });
+  const result = await subject.reconcile();
+  assert.equal(result.recoveredSucceeded, 1);
+  assert.equal(result.recoveryRequired, 0);
+});
+
+test('blocks a completed executing Job when its durable receipt is missing', async () => {
+  const { plan, snapshot } = fixture();
+  const executing = executingSnapshot(snapshot);
+  const desired = desiredJob(plan, executing);
+  const subject = controller({
+    snapshot: executing,
+    async read() {
+      return {
+        ...desired,
+        status: { conditions: [{ type: 'Complete', status: 'True' }] },
+      };
+    },
+    async create() {
+      throw new Error('must not create');
+    },
+    async complete(command) {
+      assert.equal(command.outcome, 'indeterminate');
+      return {
+        dispatch: executing.dispatch,
+        execution: completeApprovedActionExecution(executing.execution, {
+          owner: command.owner,
+          leaseToken: command.leaseToken,
+          expectedVersion: command.expectedVersion,
+          resultMutationId: command.resultMutationId,
+          outcome: command.outcome,
+          resultCode: command.resultCode,
+          completedAtMs: command.completedAtMs,
+        }),
+      };
+    },
+  });
+  const result = await subject.reconcile();
+  assert.equal(result.recoveredBlocked, 1);
+  assert.equal(result.recoveryRequired, 0);
+});
+
+test('fails closed when the durable binding differs from the approved result', async () => {
+  const { plan, snapshot } = fixture();
+  const executing = executingSnapshot(snapshot);
+  const desired = desiredJob(plan, executing);
+  const durable = createPluginPackageSecretBindingFromApprovalPlan(
+    plan,
+    executing.execution.startedAtMs,
+  );
+  const subject = controller({
+    snapshot: executing,
+    async read() {
+      return {
+        ...desired,
+        status: { conditions: [{ type: 'Complete', status: 'True' }] },
+      };
+    },
+    async create() {
+      throw new Error('must not create');
+    },
+    async findBinding() {
+      return { ...durable, bindingDigest: 'f'.repeat(64) };
+    },
+  });
+  await assert.rejects(
+    () => subject.reconcile(),
+    PluginPackageKubernetesSecretActionControllerConflictError,
+  );
+});
+
 test('fails closed when a deterministic Job name contains another contract', async () => {
   const subject = controller({
     async read() {