feat(ql3): resolve secret action recovery manually

packages/ql3-cluster-admin/test/approvalManagementClient.test.cjs

@@ -39,6 +39,13 @@ const BASE_REQUEST = Object.freeze({
   auditEventId: '60000000-0000-4000-8000-000000000001',
   failureAuditEventId: '60000000-0000-4000-8000-000000000002',
 });
+const RECOVERY_BASE_REQUEST = Object.freeze({
+  projectId: 'default',
+  dispatchId: 'dispatch-1',
+  requestId: 'recovery-command-1',
+  auditEventId: '60000000-0000-4000-8000-000000000003',
+  failureAuditEventId: '60000000-0000-4000-8000-000000000004',
+});
 
 function privateWrite(filePath, value) {
   writeFileSync(filePath, value, { mode: 0o600 });
@@ -204,6 +211,117 @@ test('validates the durable decision tuple and rejects server-side drift', () =>
   }
 });
 
+test('validates recovery inspection and binds terminal resolution to the command fence', () => {
+  const recoveryAction = {
+    ...ACTION,
+    permission: 'secret.manage',
+    actionType: 'plugin_package.secret_binding.bind',
+    actionRef: 'secret-binding:1',
+  };
+  const inspectCommand = {
+    schemaVersion: 1,
+    operation: 'approval.recover.inspect',
+    request: RECOVERY_BASE_REQUEST,
+  };
+  const recovery = {
+    projectId: 'default',
+    dispatchId: 'dispatch-1',
+    approvalRequestId: 'approval-1',
+    expectedAction: recoveryAction,
+    execution: {
+      status: 'recovery_required',
+      version: 3,
+      executionDigest: 'c'.repeat(64),
+      attemptCount: 1,
+      maxAttempts: 3,
+      startedAtMs: 1_400,
+      leaseExpiresAtMs: 1_800,
+      resultMutationId: null,
+      resultCode: null,
+      resultDigest: null,
+      completedAtMs: null,
+      createdAtMs: 1_200,
+      updatedAtMs: 1_400,
+    },
+    resolution: null,
+  };
+  assert.deepEqual(
+    validateClusterApprovalManagementClientResult(
+      {
+        schemaVersion: 1,
+        operation: inspectCommand.operation,
+        status: 'found',
+        recovery,
+      },
+      inspectCommand,
+    ).recovery,
+    recovery,
+  );
+
+  const resolveCommand = {
+    schemaVersion: 1,
+    operation: 'approval.recover.resolve',
+    request: {
+      ...RECOVERY_BASE_REQUEST,
+      expectedExecutionVersion: 3,
+      expectedExecutionDigest: 'c'.repeat(64),
+      mutationId: 'manual-recovery-1',
+      decision: 'abandon_unknown',
+      evidenceDigest: 'e'.repeat(64),
+      reasonCode: 'orphan_absence_verified',
+    },
+  };
+  const resolved = {
+    ...recovery,
+    execution: {
+      ...recovery.execution,
+      status: 'blocked',
+      version: 4,
+      executionDigest: 'd'.repeat(64),
+      leaseExpiresAtMs: null,
+      resultMutationId: 'manual-recovery-1',
+      resultCode: 'manual_recovery_abandoned_unknown',
+      completedAtMs: 2_000,
+      updatedAtMs: 2_000,
+    },
+    resolution: {
+      mutationId: 'manual-recovery-1',
+      decision: 'abandon_unknown',
+      evidenceDigest: 'e'.repeat(64),
+      reasonCode: 'orphan_absence_verified',
+      resolvedBy: { type: 'user', id: 'owner-1' },
+      resolvedAtMs: 2_000,
+      resolutionDigest: 'f'.repeat(64),
+    },
+  };
+  const result = {
+    schemaVersion: 1,
+    operation: resolveCommand.operation,
+    status: 'resolved',
+    recovery: resolved,
+  };
+  assert.deepEqual(
+    validateClusterApprovalManagementClientResult(result, resolveCommand),
+    result,
+  );
+  for (const changed of [
+    { ...resolved, execution: { ...resolved.execution, version: 3 } },
+    {
+      ...resolved,
+      resolution: { ...resolved.resolution, evidenceDigest: '0'.repeat(64) },
+    },
+  ]) {
+    assert.throws(
+      () =>
+        validateClusterApprovalManagementClientResult(
+          { ...result, recovery: changed },
+          resolveCommand,
+        ),
+      ClusterPluginPackageManagementClientRequestError,
+    );
+  }
+});
+
 test('accepts only the exact Approval route before opening one mTLS connection', async () => {
   let connects = 0;
   await assert.rejects(

packages/ql3-cluster-admin/test/approvalManagementTransport.test.cjs

@@ -2,9 +2,16 @@ const assert = require('node:assert/strict');
 const { test } = require('node:test');
 
 const {
+  consumeApprovalRequest,
   createApprovalRequest,
   decideApprovalRequest,
 } = require('@qinglong/runtime-core/approved-action');
+const {
+  claimApprovedActionExecution,
+  completeApprovedActionExecution,
+  createApprovedActionExecution,
+  startApprovedActionExecution,
+} = require('@qinglong/runtime-core/approved-action-execution');
 const {
   ClusterApprovalManagementTransportAuthenticationError,
   ClusterApprovalManagementTransportRequestError,
@@ -32,6 +39,13 @@ const BASE_REQUEST = Object.freeze({
   auditEventId: '40000000-0000-4000-8000-000000000001',
   failureAuditEventId: '40000000-0000-4000-8000-000000000002',
 });
+const RECOVERY_BASE_REQUEST = Object.freeze({
+  projectId: 'default',
+  dispatchId: 'dispatch-1',
+  requestId: 'recovery-command-1',
+  auditEventId: '40000000-0000-4000-8000-000000000003',
+  failureAuditEventId: '40000000-0000-4000-8000-000000000004',
+});
 
 function pending() {
   return createApprovalRequest({
@@ -70,6 +84,64 @@ function decideCommand() {
   };
 }
 
+function executingRecoverySnapshot() {
+  const action = {
+    ...ACTION,
+    permission: 'secret.manage',
+    actionType: 'plugin_package.secret_binding.bind',
+    actionRef: 'secret-binding:1',
+  };
+  const recoveryPending = createApprovalRequest({
+    id: 'approval-1',
+    projectId: 'default',
+    action,
+    risk: 'high',
+    decisionMode: 'human_confirmation',
+    requestedBy: { type: 'agent', id: 'agent-1' },
+    requestedAtMs: 900,
+    expiresAtMs: 10_000,
+    requestFence: { projectVersion: 1, bindingVersion: 2 },
+  });
+  const approved = decideApprovalRequest(recoveryPending, {
+    expectedVersion: 1,
+    decisionId: 'decision-recovery-1',
+    decision: 'approved',
+    reasonCode: 'reviewed',
+    principal: PRINCIPAL,
+    decidedAtMs: 1_100,
+    authorizationFence: { projectVersion: 1, bindingVersion: 2 },
+  });
+  const dispatch = consumeApprovalRequest(approved, {
+    expectedVersion: 2,
+    consumptionId: 'consumption-1',
+    dispatchId: 'dispatch-1',
+    action,
+    requestedBy: approved.requestedBy,
+    consumedBy: { type: 'system', id: 'package-executor' },
+    consumedAtMs: 1_200,
+    authorizationFence: { projectVersion: 1, bindingVersion: 2 },
+  }).dispatch;
+  const leased = claimApprovedActionExecution(createApprovedActionExecution(dispatch), {
+    owner: 'executor-1',
+    leaseToken: 'lease-1',
+    nowMs: 1_300,
+    leaseDurationMs: 500,
+  });
+  const execution = startApprovedActionExecution(
+    { dispatch, execution: leased },
+    {
+      dispatchId: dispatch.id,
+      approvalRequestId: dispatch.approvalRequestId,
+      actionDigest: dispatch.action.actionDigest,
+      owner: leased.leaseOwner,
+      leaseToken: leased.leaseToken,
+      expectedVersion: leased.version,
+      startedAtMs: 1_400,
+    },
+  );
+  return { execution: { dispatch, execution }, resolution: null };
+}
+
 test('inspects and decides through fresh strong authentication without leaking principal facts', async () => {
   const calls = [];
   const failures = [];
@@ -104,6 +176,12 @@ test('inspects and decides through fresh strong authentication without leaking p
           }),
         };
       },
+      async inspectRecovery() {
+        throw new Error('not used');
+      },
+      async resolveRecovery() {
+        throw new Error('not used');
+      },
       async recordFailure(record) {
         failures.push(record);
       },
@@ -147,6 +225,12 @@ test('records unauthenticated and reauthentication failures with schema-valid id
     async decide() {
       throw new Error('not used');
     },
+    async inspectRecovery() {
+      throw new Error('not used');
+    },
+    async resolveRecovery() {
+      throw new Error('not used');
+    },
     async recordFailure(record) {
       failures.push(record);
     },
@@ -191,6 +275,8 @@ test('rejects widened or ambiguously audited commands before authentication', as
     service: {
       async inspect() {},
       async decide() {},
+      async inspectRecovery() {},
+      async resolveRecovery() {},
       async recordFailure() {},
     },
   });
@@ -229,3 +315,90 @@ test('rejects widened or ambiguously audited commands before authentication', as
   );
   assert.equal(authenticationCalls, 0);
 });
+
+test('inspects and resolves recovery without exposing execution lease or authentication facts', async () => {
+  const source = executingRecoverySnapshot();
+  const nextExecution = completeApprovedActionExecution(source.execution.execution, {
+    owner: source.execution.execution.leaseOwner,
+    leaseToken: source.execution.execution.leaseToken,
+    expectedVersion: source.execution.execution.version,
+    resultMutationId: 'manual-recovery-1',
+    outcome: 'indeterminate',
+    resultCode: 'manual_recovery_abandoned_unknown',
+    completedAtMs: 2_000,
+  });
+  const resolution = {
+    mutationId: 'manual-recovery-1',
+    decision: 'abandon_unknown',
+    evidenceDigest: 'e'.repeat(64),
+    reasonCode: 'orphan_absence_verified',
+    resolvedBy: { type: 'user', id: 'owner-1' },
+    resolvedAtMs: 2_000,
+    resolutionDigest: 'f'.repeat(64),
+  };
+  const transport = createClusterApprovalManagementTransport({
+    service: {
+      async inspect() {},
+      async decide() {},
+      async inspectRecovery(_request, confirmAuthorization) {
+        await confirmAuthorization();
+        return source;
+      },
+      async resolveRecovery(_request, confirmAuthorization) {
+        await confirmAuthorization();
+        return {
+          status: 'resolved',
+          snapshot: {
+            execution: { dispatch: source.execution.dispatch, execution: nextExecution },
+            resolution: {
+              schema: 'qinglong/approved-action-manual-recovery@v1',
+              dispatchId: 'dispatch-1',
+              dispatchDigest: source.execution.execution.dispatchDigest,
+              projectId: 'default',
+              actionType: source.execution.dispatch.action.actionType,
+              actionDigest: source.execution.dispatch.action.actionDigest,
+              executionVersion: source.execution.execution.version,
+              executionDigest: source.execution.execution.executionDigest,
+              authenticationId: PRINCIPAL.authenticationId,
+              assurance: PRINCIPAL.assurance,
+              authenticatedAtMs: PRINCIPAL.authenticatedAtMs,
+              authorizationFence: { projectVersion: 1, bindingVersion: 2 },
+              auditEventId: RECOVERY_BASE_REQUEST.auditEventId,
+              ...resolution,
+            },
+          },
+        };
+      },
+      async recordFailure() {},
+    },
+    now: () => 2_000,
+  });
+  const authentication = { async authenticate() { return PRINCIPAL; } };
+  const inspected = await transport.execute(
+    { schemaVersion: 1, operation: 'approval.recover.inspect', request: RECOVERY_BASE_REQUEST },
+    authentication,
+  );
+  const resolved = await transport.execute(
+    {
+      schemaVersion: 1,
+      operation: 'approval.recover.resolve',
+      request: {
+        ...RECOVERY_BASE_REQUEST,
+        expectedExecutionVersion: source.execution.execution.version,
+        expectedExecutionDigest: source.execution.execution.executionDigest,
+        mutationId: resolution.mutationId,
+        decision: resolution.decision,
+        evidenceDigest: resolution.evidenceDigest,
+        reasonCode: resolution.reasonCode,
+      },
+    },
+    authentication,
+  );
+  assert.equal(inspected.recovery.execution.status, 'recovery_required');
+  assert.equal(resolved.recovery.execution.status, 'blocked');
+  assert.equal(resolved.recovery.resolution.decision, 'abandon_unknown');
+  assert.doesNotMatch(
+    JSON.stringify([inspected, resolved]),
+    /leaseOwner|leaseToken|authenticationId|authenticatedAtMs|assurance/,
+  );
+});

packages/ql3-cluster-admin/test/bootstrap.test.cjs

@@ -188,6 +188,7 @@ function database(serverVersionNum = '160014') {
                 'create_plugin_package_secret_binding_approval_plan',
                 'create_plugin_package_secret_transition_plan',
                 'plugin_package_secret_binding_transition_snapshot',
+                'resolve_approved_action_manual_recovery',
               ].includes(functionName),
               isOwner: false,
             })),

packages/ql3-cluster-admin/test/pluginPackageRecovery.test.cjs

@@ -220,6 +220,7 @@ function database(serverVersionNum = '160014') {
                 'plugin_package_secret_binding_planning_snapshot',
                 'create_plugin_package_secret_transition_plan',
                 'plugin_package_secret_binding_transition_snapshot',
+                'resolve_approved_action_manual_recovery',
               ].includes(functionName),
               isOwner: false,
             })),