feat(ql3): resolve secret action recovery manually

This commit is contained in:
whyour
2026-08-14 12:11:50 +08:00
parent d8489d8a0b
commit 3ca9901055
29 changed files with 3143 additions and 50 deletions
@@ -0,0 +1,273 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
approvalRequestDigest,
approvedActionDispatchDigest,
consumeApprovalRequest,
createApprovalRequest,
decideApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
createApprovedActionExecution,
} = require('@qinglong/runtime-core/approved-action-execution');
const {
createApprovedActionManualRecoveryService,
} = require('@qinglong/runtime-core/approved-action-manual-recovery');
const { ProjectPolicyEngine } = require('@qinglong/runtime-core/project-policy');
const {
assertPostgresApprovalManagerSchemaReady,
createPostgresDatabaseOpener,
PostgresApprovedActionManualRecoveryRepository,
PostgresProjectPolicyRepository,
PostgresSecurityAuditRepository,
} = require('@qinglong/cluster-postgres/approval-manager');
const {
PostgresApprovedActionExecutionRepository,
} = require('../dist/approved-action/approvedActionExecutionRepository');
const { runPostgresMigrations } = require('../dist/migration/migration');
const migrationConnectionString = process.env.QL3_TEST_POSTGRES_MIGRATION_URL;
const approvalManagerConnectionString =
process.env.QL3_TEST_POSTGRES_APPROVAL_MANAGER_URL;
async function open(role, connectionString) {
return createPostgresDatabaseOpener({
role,
connection: { connectionString, tls: { mode: 'disable' } },
pool: {
maxConnections: 1,
applicationName: `ql3-manual-recovery-${role}`,
},
onPoolError(error) {
throw error;
},
})();
}
async function insertFixture(pool, namespace) {
const projectId = `${namespace}-project`;
const principal = Object.freeze({
subject: Object.freeze({ type: 'user', id: `${namespace}-owner` }),
authenticationId: `${namespace}-session`,
authenticatedAtMs: 100,
expiresAtMs: 20_000,
assurance: 'hardware',
});
const action = Object.freeze({
permission: 'secret.manage',
actionType: 'plugin_package.secret_binding.bind',
actionRef: `${namespace}:secret-binding`,
actionDigest: 'a'.repeat(64),
previewDigest: 'b'.repeat(64),
});
const pending = createApprovalRequest({
id: `${namespace}-approval`,
projectId,
action,
risk: 'high',
decisionMode: 'human_confirmation',
requestedBy: { type: 'agent', id: `${namespace}-agent` },
requestedAtMs: 800,
expiresAtMs: 10_000,
requestFence: { projectVersion: 1, bindingVersion: 1 },
});
const approved = decideApprovalRequest(pending, {
expectedVersion: 1,
decisionId: `${namespace}-decision`,
decision: 'approved',
reasonCode: 'reviewed',
principal,
decidedAtMs: 900,
authorizationFence: { projectVersion: 1, bindingVersion: 1 },
});
const consumed = consumeApprovalRequest(approved, {
expectedVersion: 2,
consumptionId: `${namespace}-consumption`,
dispatchId: `${namespace}-dispatch`,
action,
requestedBy: pending.requestedBy,
consumedBy: { type: 'system', id: 'package-executor' },
consumedAtMs: 950,
authorizationFence: { projectVersion: 1, bindingVersion: 1 },
});
await pool.query(
`INSERT INTO "ql3"."projects" (
id, name, slug, status, version, created_at_ms, updated_at_ms
) VALUES ($1, $1, $2, 'active', 1, 1, 1)`,
[projectId, projectId.replace(/[^a-z0-9-]/g, '-')],
);
await pool.query(
`INSERT INTO "ql3"."project_role_bindings" (
project_id, subject_type, subject_id, version, state, role,
mutation_id, changed_by_type, changed_by_id, created_at_ms
) VALUES ($1, 'user', $2, 1, 'active', 'owner', $3, 'system',
'integration-fixture', 2)`,
[projectId, principal.subject.id, `${namespace}-binding`],
);
const request = consumed.request;
await pool.query(
`INSERT INTO "ql3"."approval_requests" (
request_id, project_id, version, state, action_type, action_ref,
action_digest, preview_digest, requested_by_type, requested_by_id,
decision_id, consumption_id, dispatch_id, expires_at_ms, request_json,
request_digest, updated_at_ms
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14,
$15::jsonb, $16, $17
)`,
[
request.id,
request.projectId,
request.version,
request.state,
request.action.actionType,
request.action.actionRef,
request.action.actionDigest,
request.action.previewDigest,
request.requestedBy.type,
request.requestedBy.id,
request.decisionId,
request.consumptionId,
request.dispatchId,
request.expiresAtMs,
JSON.stringify(request),
approvalRequestDigest(request),
request.consumedAtMs,
],
);
const dispatch = consumed.dispatch;
await pool.query(
`INSERT INTO "ql3"."approved_action_dispatches" (
dispatch_id, approval_request_id, project_id, action_type, action_ref,
action_digest, preview_digest, dispatch_json, dispatch_digest,
created_at_ms
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10)`,
[
dispatch.id,
dispatch.approvalRequestId,
dispatch.projectId,
dispatch.action.actionType,
dispatch.action.actionRef,
dispatch.action.actionDigest,
dispatch.action.previewDigest,
JSON.stringify(dispatch),
approvedActionDispatchDigest(dispatch),
dispatch.createdAtMs,
],
);
const executions = new PostgresApprovedActionExecutionRepository(pool);
await pool.query(
`INSERT INTO "ql3"."approved_action_executions" (
dispatch_id, dispatch_digest, project_id, status, version,
attempt_count, max_attempts, eligible_at_ms, next_attempt_at_ms,
lease_owner, lease_token, lease_expires_at_ms, started_at_ms,
result_mutation_id, result_code, result_digest, completed_at_ms,
created_at_ms, updated_at_ms, execution_json, execution_digest
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12,
$13, $14, $15, $16, $17, $18, $19, $20::jsonb, $21)`,
(() => {
const value = createApprovedActionExecution(dispatch);
return [
value.dispatchId, value.dispatchDigest, value.projectId, value.status,
value.version, value.attemptCount, value.maxAttempts, value.eligibleAtMs,
value.nextAttemptAtMs, value.leaseOwner, value.leaseToken,
value.leaseExpiresAtMs, value.startedAtMs, value.resultMutationId,
value.resultCode, value.resultDigest, value.completedAtMs,
value.createdAtMs, value.updatedAtMs, JSON.stringify(value),
value.executionDigest,
];
})(),
);
const claimed = await executions.claimExecution({
dispatchId: dispatch.id,
owner: `${namespace}-executor`,
leaseToken: `${namespace}-lease`,
nowMs: 1_000,
leaseDurationMs: 500,
});
const started = await executions.startExecution({
dispatchId: dispatch.id,
approvalRequestId: dispatch.approvalRequestId,
actionDigest: dispatch.action.actionDigest,
owner: `${namespace}-executor`,
leaseToken: `${namespace}-lease`,
expectedVersion: claimed.snapshot.execution.version,
startedAtMs: 1_100,
});
return { projectId, principal, dispatch, execution: started.execution };
}
if (!migrationConnectionString || !approvalManagerConnectionString) {
test('PostgreSQL manual recovery gate requires migration and Approval manager URLs', {
skip: true,
});
} else {
test('PostgreSQL atomically resolves and exactly replays an expired Secret Action', async () => {
const migration = await open('migration', migrationConnectionString);
let manager;
try {
await runPostgresMigrations({ pool: migration.pool });
manager = await open('approval-manager', approvalManagerConnectionString);
const readiness = await assertPostgresApprovalManagerSchemaReady(manager.pool);
assert.equal(readiness.contractVersion, 64);
const namespace = `recovery-${process.pid}-${Date.now()}`;
const fixture = await insertFixture(migration.pool, namespace);
const service = createApprovedActionManualRecoveryService({
repository: new PostgresApprovedActionManualRecoveryRepository(manager.pool),
policy: new ProjectPolicyEngine(
new PostgresProjectPolicyRepository(manager.pool),
),
audit: new PostgresSecurityAuditRepository(manager.pool),
now: () => 2_000,
});
const inspected = await service.inspect({
projectId: fixture.projectId,
dispatchId: fixture.dispatch.id,
requestId: `${namespace}-inspect`,
auditEventId: '80000000-0000-4000-8000-000000000001',
principal: fixture.principal,
});
assert.equal(inspected.execution.execution.status, 'executing');
const request = {
projectId: fixture.projectId,
dispatchId: fixture.dispatch.id,
expectedExecutionVersion: fixture.execution.version,
expectedExecutionDigest: fixture.execution.executionDigest,
mutationId: `${namespace}-mutation`,
decision: 'abandon_unknown',
evidenceDigest: 'e'.repeat(64),
reasonCode: 'orphan_absence_verified',
requestId: `${namespace}-resolve`,
auditEventId: '80000000-0000-4000-8000-000000000002',
principal: fixture.principal,
};
const first = await service.resolve(request);
const replay = await service.resolve(request);
assert.equal(first.status, 'resolved');
assert.equal(replay.status, 'existing');
assert.equal(first.snapshot.execution.execution.status, 'blocked');
assert.equal(first.snapshot.resolution.decision, 'abandon_unknown');
const persisted = await migration.pool.query(
`SELECT
(SELECT count(*)::integer FROM "ql3"."approved_action_manual_recovery_resolutions"
WHERE dispatch_id = $1) AS "resolutionCount",
(SELECT count(*)::integer FROM "ql3"."security_audit_events"
WHERE event_id = $2) AS "auditCount"`,
[fixture.dispatch.id, request.auditEventId],
);
assert.deepEqual(persisted.rows[0], { resolutionCount: 1, auditCount: 1 });
await assert.rejects(
manager.pool.query(
`UPDATE "ql3"."approved_action_executions" SET status = 'failed'
WHERE dispatch_id = $1`,
[fixture.dispatch.id],
),
(error) => error && error.code === '42501',
);
} finally {
if (manager) await manager.close();
await migration.close();
}
});
}
@@ -0,0 +1,158 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
consumeApprovalRequest,
createApprovalRequest,
decideApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
claimApprovedActionExecution,
createApprovedActionExecution,
startApprovedActionExecution,
} = require('@qinglong/runtime-core/approved-action-execution');
const {
createApprovedActionManualRecoveryService,
} = require('@qinglong/runtime-core/approved-action-manual-recovery');
const {
PostgresApprovedActionManualRecoveryRepository,
} = require('@qinglong/cluster-postgres/approval-manager');
const PRINCIPAL = Object.freeze({
subject: Object.freeze({ type: 'user', id: 'owner-1' }),
authenticationId: 'oidc:session-1',
authenticatedAtMs: 100,
expiresAtMs: 20_000,
assurance: 'hardware',
});
function executing() {
const action = {
permission: 'secret.manage',
actionType: 'plugin_package.secret_binding.bind',
actionRef: 'secret-binding:1',
actionDigest: 'a'.repeat(64),
previewDigest: 'b'.repeat(64),
};
const pending = createApprovalRequest({
id: 'approval-1',
projectId: 'default',
action,
risk: 'high',
decisionMode: 'human_confirmation',
requestedBy: { type: 'agent', id: 'agent-1' },
requestedAtMs: 800,
expiresAtMs: 10_000,
requestFence: { projectVersion: 1, bindingVersion: 2 },
});
const approved = decideApprovalRequest(pending, {
expectedVersion: 1,
decisionId: 'decision-1',
decision: 'approved',
reasonCode: 'reviewed',
principal: PRINCIPAL,
decidedAtMs: 900,
authorizationFence: { projectVersion: 1, bindingVersion: 2 },
});
const dispatch = consumeApprovalRequest(approved, {
expectedVersion: 2,
consumptionId: 'consumption-1',
dispatchId: 'dispatch-1',
action,
requestedBy: pending.requestedBy,
consumedBy: { type: 'system', id: 'package-executor' },
consumedAtMs: 950,
authorizationFence: { projectVersion: 1, bindingVersion: 2 },
}).dispatch;
const leased = claimApprovedActionExecution(createApprovedActionExecution(dispatch), {
owner: 'executor-1',
leaseToken: 'lease-1',
nowMs: 1_000,
leaseDurationMs: 500,
});
const execution = startApprovedActionExecution(
{ dispatch, execution: leased },
{
dispatchId: dispatch.id,
approvalRequestId: dispatch.approvalRequestId,
actionDigest: dispatch.action.actionDigest,
owner: leased.leaseOwner,
leaseToken: leased.leaseToken,
expectedVersion: leased.version,
startedAtMs: 1_100,
},
);
return { dispatch, execution };
}
test('resolves through the bounded PostgreSQL function and verifies the stored tuple', async () => {
const initial = executing();
let stored = { ...initial, resolution: null };
const calls = [];
const pool = {
async query(text, values) {
calls.push([text, values]);
if (text.includes('resolve_approved_action_manual_recovery')) {
stored = {
dispatch: initial.dispatch,
execution: JSON.parse(values[1]),
resolution: JSON.parse(values[0]),
};
return { rows: [{ status: 'resolved' }] };
}
if (text.includes('approved_action_manual_recovery_resolutions')) {
return {
rows: [
{
dispatchJson: stored.dispatch,
executionJson: stored.execution,
executionDigest: stored.execution.executionDigest,
resolutionJson: stored.resolution,
},
],
};
}
throw new Error(`unexpected query: ${text}`);
},
async connect() {
throw new Error('repository must not open a broad transaction');
},
};
const repository = new PostgresApprovedActionManualRecoveryRepository(pool);
const service = createApprovedActionManualRecoveryService({
repository,
policy: {
async authorize(_principal, _projectId, permission) {
assert.equal(permission, 'approval.recover');
return {
effect: 'allow',
reasons: ['role_grant'],
fence: { projectVersion: 1, bindingVersion: 2 },
};
},
},
audit: { async record() {} },
now: () => 2_000,
});
const result = await service.resolve({
projectId: 'default',
dispatchId: 'dispatch-1',
expectedExecutionVersion: initial.execution.version,
expectedExecutionDigest: initial.execution.executionDigest,
mutationId: 'manual-recovery-1',
decision: 'abandon_unknown',
evidenceDigest: 'e'.repeat(64),
reasonCode: 'orphan_absence_verified',
auditEventId: '70000000-0000-4000-8000-000000000001',
requestId: 'manual-recovery-request-1',
principal: PRINCIPAL,
});
assert.equal(result.status, 'resolved');
assert.equal(result.snapshot.execution.execution.status, 'blocked');
assert.equal(result.snapshot.resolution.decision, 'abandon_unknown');
assert.equal(
calls.filter(([sql]) => sql.includes('resolve_approved_action_manual_recovery'))
.length,
1,
);
});
@@ -115,6 +115,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
'pg-0062-plugin-package-secret-binding-target-guard',
'pg-0063-plugin-package-secret-binding-transition-receipts',
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
],
);
for (const migration of postgresqlMainMigrationStream.migrations) {
@@ -573,6 +574,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
checksum:
'1951b77a0265f8826169e4724424b2fbbd30061b27e27d3ba95de03430c1bac9',
},
{
id: 'pg-0065-approved-action-manual-recovery',
checksum:
'95387c5b40659490dbcb7626ecd15bacf6412360752bef88873bde57c43e0185',
},
];
assert.deepEqual(
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
@@ -2227,3 +2233,52 @@ test('advances capability v63 with manager-only immutable Secret transition plan
/migration_id = 'pg-0063-plugin-package-secret-binding-transition-receipts'/,
);
});
test('advances capability v64 with atomic least-privilege manual recovery', async () => {
const migration = migrationById('pg-0065-approved-action-manual-recovery');
const statements = [];
await migration.up({
async query(statement) {
statements.push(statement);
return { rows: [] };
},
});
const sql = statements.join('\n');
assert.match(
sql,
/CREATE TABLE "ql3"\."approved_action_manual_recovery_resolutions"/,
);
assert.match(
sql,
/CREATE FUNCTION "ql3"\."resolve_approved_action_manual_recovery"\([\s\S]+SECURITY DEFINER[\s\S]+SET search_path = pg_catalog, ql3/,
);
assert.match(sql, /current_status <> 'executing'/);
assert.match(sql, /current_lease_expires_at_ms > .*'resolvedAtMs'/);
assert.match(
sql,
/current_action_type NOT IN \([\s\S]+plugin_package\.secret_binding\.bind[\s\S]+plugin_package\.secret_binding\.transition/,
);
assert.match(
sql,
/INSERT INTO "ql3"\."security_audit_events"[\s\S]+UPDATE "ql3"\."approved_action_executions"[\s\S]+INSERT INTO "ql3"\."approved_action_manual_recovery_resolutions"/,
);
assert.match(
sql,
/GRANT SELECT ON "ql3"\."approved_action_dispatches", "ql3"\."approved_action_executions", "ql3"\."approved_action_manual_recovery_resolutions" TO ql3_approval_manager/,
);
assert.match(
sql,
/GRANT EXECUTE ON FUNCTION "ql3"\."resolve_approved_action_manual_recovery"\(jsonb, jsonb, jsonb\) TO ql3_approval_manager/,
);
assert.doesNotMatch(
sql,
/GRANT (?:INSERT|UPDATE|DELETE)[^;]+(?:approved_action_executions|approved_action_manual_recovery_resolutions)[^;]+ql3_approval_manager/,
);
assert.match(sql, /contract_version = 64/);
assert.match(sql, /"approved_action_manual_recovery":1/);
assert.match(sql, /contract_version = 63/);
assert.match(
sql,
/migration_id = 'pg-0064-plugin-package-secret-binding-transition-approval-plans'/,
);
});
@@ -134,6 +134,7 @@ function validPrivileges() {
approval_requests: [false, false, false, false],
approved_action_dispatches: [false, false, false, false],
approved_action_executions: [false, false, false, false],
approved_action_manual_recovery_resolutions: [false, false, false, false],
plugin_package_install_proposals: [false, false, false, false],
plugin_package_management_quota_buckets: [false, false, false, false],
plugin_package_identity_keyset_ledger: [false, false, false, false],
@@ -271,6 +272,7 @@ function validAdminPrivileges() {
approval_requests: [false, false, false, false],
approved_action_dispatches: [false, false, false, false],
approved_action_executions: [false, false, false, false],
approved_action_manual_recovery_resolutions: [false, false, false, false],
plugin_package_install_proposals: [false, false, false, false],
plugin_package_management_quota_buckets: [false, false, false, false],
plugin_package_identity_keyset_ledger: [false, false, false, false],
@@ -491,6 +493,9 @@ function approvalManagerPrivileges() {
'project_role_bindings',
'security_audit_events',
'approval_requests',
'approved_action_dispatches',
'approved_action_executions',
'approved_action_manual_recovery_resolutions',
'tool_invocation_preview_artifacts',
'plugin_package_identity_keyset_ledger',
]);
@@ -705,6 +710,11 @@ function queryable(overrides = {}) {
'plugin_package_secret_binding_planning_snapshot',
'plugin_package_secret_binding_transition_snapshot',
].includes(functionName)
: overrides.functionMode === 'approval-manager'
? [
'lock_approval_policy_fence',
'resolve_approved_action_manual_recovery',
].includes(functionName)
: overrides.functionMode === 'manager'
? functionName === 'lock_approval_policy_fence'
: overrides.functionMode === 'run-manager'
@@ -805,7 +815,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
serverMajor: 16,
currentUser: 'ql3_runtime',
contractName: 'control-core',
contractVersion: 63,
contractVersion: 64,
migrationIds: [
'pg-0001-schema-capability',
'pg-0002-run-core',
@@ -871,6 +881,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
'pg-0062-plugin-package-secret-binding-target-guard',
'pg-0063-plugin-package-secret-binding-transition-receipts',
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
],
});
});
@@ -901,10 +912,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
}),
);
assert.equal(report.currentUser, 'ql3_admin');
assert.equal(report.contractVersion, 63);
assert.equal(report.contractVersion, 64);
assert.equal(
report.migrationIds.at(-1),
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
);
});
@@ -917,10 +928,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
}),
);
assert.equal(report.currentUser, 'ql3_automation_manager');
assert.equal(report.contractVersion, 63);
assert.equal(report.contractVersion, 64);
assert.equal(
report.migrationIds.at(-1),
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
);
const widened = automationManagerPrivileges();
@@ -945,14 +956,14 @@ test('accepts the isolated least-privilege human Approval manager role', async (
queryable({
currentUser: 'ql3_approval_manager',
privileges: approvalManagerPrivileges(),
functionMode: 'manager',
functionMode: 'approval-manager',
}),
);
assert.equal(report.currentUser, 'ql3_approval_manager');
assert.equal(report.contractVersion, 63);
assert.equal(report.contractVersion, 64);
assert.equal(
report.migrationIds.at(-1),
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
);
const widened = approvalManagerPrivileges();
@@ -964,7 +975,7 @@ test('accepts the isolated least-privilege human Approval manager role', async (
queryable({
currentUser: 'ql3_approval_manager',
privileges: widened,
functionMode: 'manager',
functionMode: 'approval-manager',
}),
),
(error) =>
@@ -983,10 +994,10 @@ test('accepts the isolated least-privilege Run manager role', async () => {
}),
);
assert.equal(report.currentUser, 'ql3_run_manager');
assert.equal(report.contractVersion, 63);
assert.equal(report.contractVersion, 64);
assert.equal(
report.migrationIds.at(-1),
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
);
const widened = runManagerPrivileges();
@@ -1118,10 +1129,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
}),
);
assert.equal(report.currentUser, 'ql3_worker_ingress');
assert.equal(report.contractVersion, 63);
assert.equal(report.contractVersion, 64);
assert.equal(
report.migrationIds.at(-1),
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
);
});