feat(ql3): resolve secret action recovery manually

This commit is contained in:
whyour
2026-08-14 12:11:50 +08:00
parent d8489d8a0b
commit 3ca9901055
29 changed files with 3143 additions and 50 deletions
@@ -25,6 +25,7 @@ export {
export { postgresqlMainMigrationManifest } from '../migration/migrationManifest';
export { PostgresApprovalRequestRepository } from '../approved-action/approvalRequestRepository';
export { PostgresApprovalRequestSource } from '../approved-action/approvalRequestSource';
export { PostgresApprovedActionManualRecoveryRepository } from '../approved-action/approvedActionManualRecoveryRepository';
export { PostgresProjectPolicyRepository } from '../security/projectPolicyRepository';
export { PostgresSecurityAuditRepository } from '../security/securityAuditRepository';
export {
@@ -0,0 +1,269 @@
import type { PostgresPool } from '@qinglong/runtime-core';
import {
normalizeApprovedActionDispatchRecord,
type ApprovedActionDispatchRecord,
} from '@qinglong/runtime-core/approved-action';
import {
normalizeApprovedActionExecutionRecord,
normalizeApprovedActionExecutionSnapshot,
type ApprovedActionExecutionRecord,
type ApprovedActionExecutionSnapshot,
} from '@qinglong/runtime-core/approved-action-execution';
import {
ApprovedActionManualRecoveryFenceConflictError,
ApprovedActionManualRecoveryTargetUnavailableError,
ApprovedActionManualRecoveryUnavailableError,
normalizeApprovedActionManualRecoveryResolution,
normalizeApprovedActionManualRecoverySnapshot,
type ApprovedActionManualRecoveryRepository,
type ApprovedActionManualRecoveryResolutionRecord,
type ApprovedActionManualRecoverySnapshot,
type ResolveApprovedActionManualRecoveryCommand,
type ResolveApprovedActionManualRecoveryResult,
} from '@qinglong/runtime-core/approved-action-manual-recovery';
import {
normalizeSecurityAuditRecord,
type SecurityAuditRecord,
} from '@qinglong/runtime-core/security-audit';
import {
postgresRequiredJsonObject,
postgresRequiredString,
postgresSqlState,
} from '../repository/definitionRepositorySupport';
type Row = Record<string, unknown>;
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
function unavailable(options?: ErrorOptions): ApprovedActionManualRecoveryUnavailableError {
return new ApprovedActionManualRecoveryUnavailableError(options);
}
function same(left: unknown, right: unknown): boolean {
return JSON.stringify(left) === JSON.stringify(right);
}
function parseDispatch(row: Row): Readonly<ApprovedActionDispatchRecord> {
try {
return normalizeApprovedActionDispatchRecord(
postgresRequiredJsonObject(
row.dispatchJson,
unavailable,
) as unknown as ApprovedActionDispatchRecord,
);
} catch (error) {
if (error instanceof ApprovedActionManualRecoveryUnavailableError) throw error;
throw unavailable();
}
}
function parseExecution(row: Row): Readonly<ApprovedActionExecutionRecord> {
try {
const execution = normalizeApprovedActionExecutionRecord(
postgresRequiredJsonObject(
row.executionJson,
unavailable,
) as unknown as ApprovedActionExecutionRecord,
);
if (
execution.executionDigest !==
postgresRequiredString(row.executionDigest, unavailable)
) {
throw unavailable();
}
return execution;
} catch (error) {
if (error instanceof ApprovedActionManualRecoveryUnavailableError) throw error;
throw unavailable();
}
}
function parseResolution(
value: unknown,
): Readonly<ApprovedActionManualRecoveryResolutionRecord> | null {
if (value === null || value === undefined) return null;
try {
return normalizeApprovedActionManualRecoveryResolution(
postgresRequiredJsonObject(
value,
unavailable,
) as unknown as ApprovedActionManualRecoveryResolutionRecord,
);
} catch (error) {
if (error instanceof ApprovedActionManualRecoveryUnavailableError) throw error;
throw unavailable();
}
}
function parseSnapshot(row: Row): Readonly<ApprovedActionManualRecoverySnapshot> {
try {
return normalizeApprovedActionManualRecoverySnapshot({
execution: normalizeApprovedActionExecutionSnapshot({
dispatch: parseDispatch(row),
execution: parseExecution(row),
}),
resolution: parseResolution(row.resolutionJson),
});
} catch (error) {
if (error instanceof ApprovedActionManualRecoveryUnavailableError) throw error;
throw unavailable();
}
}
function mapped(error: unknown): Error {
if (
error instanceof ApprovedActionManualRecoveryFenceConflictError ||
error instanceof ApprovedActionManualRecoveryTargetUnavailableError ||
error instanceof ApprovedActionManualRecoveryUnavailableError ||
(error instanceof Error && error.name.startsWith('InvalidApprovedAction'))
) {
return error;
}
const state = postgresSqlState(error);
if (
state === '23503' ||
state === '23505' ||
state === '23514' ||
state === '40001' ||
state === '40P01'
) {
return new ApprovedActionManualRecoveryFenceConflictError();
}
return unavailable({ cause: error instanceof Error ? error : undefined });
}
function auditMatchesResolution(
audit: Readonly<SecurityAuditRecord>,
resolution: Readonly<ApprovedActionManualRecoveryResolutionRecord>,
): boolean {
return (
audit.eventId === resolution.auditEventId &&
audit.operationId === 'approval.recover.resolve' &&
audit.projectId === resolution.projectId &&
audit.subject?.type === resolution.resolvedBy.type &&
audit.subject.id === resolution.resolvedBy.id &&
audit.authenticationId === resolution.authenticationId &&
audit.outcome === 'allowed' &&
same(audit.reasons, [
'role_grant',
'strong_authentication',
'manual_recovery',
]) &&
same(audit.fence, resolution.authorizationFence) &&
audit.occurredAtMs === resolution.resolvedAtMs
);
}
function normalizeCommand(
value: Readonly<ResolveApprovedActionManualRecoveryCommand>,
): Readonly<ResolveApprovedActionManualRecoveryCommand> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
Object.keys(value).sort().join('\0') !==
['previous', 'nextExecution', 'resolution', 'audit'].sort().join('\0')
) {
throw new TypeError('Approved Action manual recovery command is invalid');
}
const previous = normalizeApprovedActionExecutionSnapshot(value.previous);
const nextExecution = normalizeApprovedActionExecutionRecord(value.nextExecution);
const resolution = normalizeApprovedActionManualRecoveryResolution(
value.resolution,
);
const audit = normalizeSecurityAuditRecord(value.audit);
if (
previous.dispatch.id !== resolution.dispatchId ||
previous.dispatch.projectId !== resolution.projectId ||
previous.dispatch.action.actionType !== resolution.actionType ||
previous.dispatch.action.actionDigest !== resolution.actionDigest ||
previous.execution.version !== resolution.executionVersion ||
previous.execution.executionDigest !== resolution.executionDigest ||
nextExecution.dispatchId !== previous.dispatch.id ||
nextExecution.version !== previous.execution.version + 1 ||
nextExecution.resultMutationId !== resolution.mutationId ||
nextExecution.completedAtMs !== resolution.resolvedAtMs ||
!auditMatchesResolution(audit, resolution)
) {
throw new ApprovedActionManualRecoveryFenceConflictError();
}
return Object.freeze({ previous, nextExecution, resolution, audit });
}
export class PostgresApprovedActionManualRecoveryRepository
implements ApprovedActionManualRecoveryRepository
{
constructor(private readonly pool: PostgresPool) {
if (
!pool ||
typeof pool.query !== 'function' ||
typeof pool.connect !== 'function'
) {
throw new TypeError(
'PostgreSQL Approved Action manual recovery pool is invalid',
);
}
}
async findByDispatchId(
dispatchId: string,
): Promise<Readonly<ApprovedActionManualRecoverySnapshot> | null> {
if (typeof dispatchId !== 'string' || !IDENTIFIER_PATTERN.test(dispatchId)) {
throw new TypeError('Approved Action recovery dispatch id is invalid');
}
try {
const result = await this.pool.query<Row>(
`SELECT dispatch.dispatch_json AS "dispatchJson",
execution.execution_json AS "executionJson",
execution.execution_digest AS "executionDigest",
resolution.resolution_json AS "resolutionJson"
FROM "ql3"."approved_action_executions" AS execution
JOIN "ql3"."approved_action_dispatches" AS dispatch
ON dispatch.dispatch_id = execution.dispatch_id
LEFT JOIN "ql3"."approved_action_manual_recovery_resolutions" AS resolution
ON resolution.dispatch_id = execution.dispatch_id
WHERE execution.dispatch_id = $1
LIMIT 2`,
[dispatchId],
);
if (result.rows.length === 0) return null;
if (result.rows.length !== 1) throw unavailable();
return parseSnapshot(result.rows[0]!);
} catch (error) {
throw mapped(error);
}
}
async resolve(
commandValue: Readonly<ResolveApprovedActionManualRecoveryCommand>,
): Promise<Readonly<ResolveApprovedActionManualRecoveryResult>> {
const command = normalizeCommand(commandValue);
try {
const result = await this.pool.query<Row>(
`SELECT "ql3"."resolve_approved_action_manual_recovery"(
$1::jsonb, $2::jsonb, $3::jsonb
) AS status`,
[
JSON.stringify(command.resolution),
JSON.stringify(command.nextExecution),
JSON.stringify(command.audit),
],
);
if (result.rows.length !== 1) throw unavailable();
const status = postgresRequiredString(result.rows[0]!.status, unavailable);
if (status !== 'resolved' && status !== 'existing') throw unavailable();
const stored = await this.findByDispatchId(command.resolution.dispatchId);
if (
!stored ||
!stored.resolution ||
!same(stored.resolution, command.resolution) ||
!same(stored.execution.execution, command.nextExecution)
) {
throw new ApprovedActionManualRecoveryFenceConflictError();
}
return Object.freeze({ status, snapshot: stored });
} catch (error) {
throw mapped(error);
}
}
}
@@ -0,0 +1,392 @@
import { CAPABILITIES_V63 } from '../migrations/pg-0064-plugin-package-secret-binding-transition-approval-plans';
import { definePostgresSqlMigration } from '../migrations/sqlMigration';
export const CAPABILITIES_V64 = CAPABILITIES_V63.replace(
'"approved_action_execution":1,',
'"approved_action_execution":1,"approved_action_manual_recovery":1,',
);
export const pg0065ApprovedActionManualRecoveryMigration =
definePostgresSqlMigration({
id: 'pg-0065-approved-action-manual-recovery',
statements: [
`
CREATE TABLE "ql3"."approved_action_manual_recovery_resolutions" (
dispatch_id varchar(128) PRIMARY KEY,
dispatch_digest char(64) NOT NULL,
project_id varchar(128) NOT NULL,
action_type varchar(128) NOT NULL,
action_digest char(64) NOT NULL,
execution_version integer NOT NULL,
execution_digest char(64) NOT NULL,
mutation_id varchar(128) NOT NULL,
decision varchar(32) NOT NULL,
evidence_digest char(64) NOT NULL,
reason_code varchar(64) NOT NULL,
resolved_by_type varchar(16) NOT NULL,
resolved_by_id varchar(255) NOT NULL,
authentication_id varchar(128) NOT NULL,
assurance varchar(32) NOT NULL,
authenticated_at_ms bigint NOT NULL,
project_version integer NOT NULL,
binding_version integer NOT NULL,
audit_event_id uuid NOT NULL,
resolved_at_ms bigint NOT NULL,
resolution_json jsonb NOT NULL,
resolution_digest char(64) NOT NULL,
CONSTRAINT ql3_approved_action_manual_recovery_dispatch_fk
FOREIGN KEY (dispatch_id)
REFERENCES "ql3"."approved_action_dispatches" (dispatch_id)
ON DELETE RESTRICT ON UPDATE RESTRICT,
CONSTRAINT ql3_approved_action_manual_recovery_project_fk
FOREIGN KEY (project_id) REFERENCES "ql3"."projects" (id)
ON DELETE RESTRICT ON UPDATE RESTRICT,
CONSTRAINT ql3_approved_action_manual_recovery_audit_fk
FOREIGN KEY (audit_event_id)
REFERENCES "ql3"."security_audit_events" (event_id)
ON DELETE RESTRICT ON UPDATE RESTRICT,
CONSTRAINT ql3_approved_action_manual_recovery_identity_check CHECK (
dispatch_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
action_type IN (
'plugin_package.secret_binding.bind',
'plugin_package.secret_binding.transition'
) AND
execution_version BETWEEN 1 AND 2147483647 AND
mutation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
decision IN ('confirm_failed', 'abandon_unknown') AND
reason_code ~ '^[a-z][a-z0-9_]{0,63}$' AND
resolved_by_type = 'user' AND
octet_length(resolved_by_id) BETWEEN 1 AND 255 AND
resolved_by_id !~ '[[:cntrl:]]' AND
authentication_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
assurance IN ('multi_factor', 'hardware') AND
project_version >= 1 AND binding_version >= 1
),
CONSTRAINT ql3_approved_action_manual_recovery_digest_check CHECK (
dispatch_digest ~ '^[0-9a-f]{64}$' AND
action_digest ~ '^[0-9a-f]{64}$' AND
execution_digest ~ '^[0-9a-f]{64}$' AND
evidence_digest ~ '^[0-9a-f]{64}$' AND
resolution_digest ~ '^[0-9a-f]{64}$'
),
CONSTRAINT ql3_approved_action_manual_recovery_time_check CHECK (
authenticated_at_ms >= 0 AND
resolved_at_ms >= authenticated_at_ms AND
resolved_at_ms - authenticated_at_ms <= 300000
),
CONSTRAINT ql3_approved_action_manual_recovery_json_check CHECK (
jsonb_typeof(resolution_json) = 'object' AND
octet_length(resolution_json::text) BETWEEN 2 AND 65536 AND
resolution_json @> jsonb_build_object(
'schema', 'qinglong/approved-action-manual-recovery@v1',
'dispatchId', dispatch_id,
'dispatchDigest', dispatch_digest,
'projectId', project_id,
'actionType', action_type,
'actionDigest', action_digest,
'executionVersion', execution_version,
'executionDigest', execution_digest,
'mutationId', mutation_id,
'decision', decision,
'evidenceDigest', evidence_digest,
'reasonCode', reason_code,
'resolvedBy', jsonb_build_object(
'type', resolved_by_type,
'id', resolved_by_id
),
'authenticationId', authentication_id,
'assurance', assurance,
'authenticatedAtMs', authenticated_at_ms,
'authorizationFence', jsonb_build_object(
'projectVersion', project_version,
'bindingVersion', binding_version
),
'auditEventId', audit_event_id,
'resolvedAtMs', resolved_at_ms,
'resolutionDigest', resolution_digest
)
)
)
`.trim(),
`CREATE UNIQUE INDEX ql3_approved_action_manual_recovery_mutation_uidx ON "ql3"."approved_action_manual_recovery_resolutions" (mutation_id)`,
`CREATE UNIQUE INDEX ql3_approved_action_manual_recovery_digest_uidx ON "ql3"."approved_action_manual_recovery_resolutions" (resolution_digest)`,
`CREATE INDEX ql3_approved_action_manual_recovery_project_idx ON "ql3"."approved_action_manual_recovery_resolutions" (project_id, resolved_at_ms, dispatch_id)`,
`
CREATE FUNCTION "ql3"."resolve_approved_action_manual_recovery"(
p_resolution_json jsonb,
p_next_execution_json jsonb,
p_audit_json jsonb
)
RETURNS varchar
LANGUAGE plpgsql
VOLATILE
SECURITY DEFINER
SET search_path = pg_catalog, ql3
AS $ql3$
DECLARE
current_execution_json jsonb;
current_execution_digest char(64);
current_version integer;
current_status varchar;
current_project_id varchar;
current_dispatch_digest char(64);
current_action_type varchar;
current_action_digest char(64);
current_lease_expires_at_ms bigint;
existing_resolution_json jsonb;
expected_status varchar;
expected_result_code varchar;
BEGIN
IF NOT pg_has_role(session_user, 'ql3_approval_manager', 'member') THEN
RAISE EXCEPTION 'Approval manager authority is required'
USING ERRCODE = 'insufficient_privilege';
END IF;
IF jsonb_typeof(p_resolution_json) <> 'object'
OR (SELECT count(*) FROM jsonb_object_keys(p_resolution_json)) <> 20
OR jsonb_typeof(p_next_execution_json) <> 'object'
OR (SELECT count(*) FROM jsonb_object_keys(p_next_execution_json)) <> 21
OR jsonb_typeof(p_audit_json) <> 'object'
OR (SELECT count(*) FROM jsonb_object_keys(p_audit_json)) <> 10
THEN
RAISE EXCEPTION 'Approved Action manual recovery input is malformed'
USING ERRCODE = 'check_violation';
END IF;
SELECT resolution_json
INTO existing_resolution_json
FROM "ql3"."approved_action_manual_recovery_resolutions"
WHERE dispatch_id = p_resolution_json ->> 'dispatchId';
IF FOUND THEN
SELECT execution_json
INTO current_execution_json
FROM "ql3"."approved_action_executions"
WHERE dispatch_id = p_resolution_json ->> 'dispatchId';
IF existing_resolution_json = p_resolution_json
AND current_execution_json = p_next_execution_json
AND EXISTS (
SELECT 1
FROM "ql3"."security_audit_events" AS audit
WHERE audit.event_id = (p_audit_json ->> 'eventId')::uuid
AND audit.request_id = p_audit_json ->> 'requestId'
AND audit.operation_id = 'approval.recover.resolve'
AND audit.project_id = p_resolution_json ->> 'projectId'
AND audit.subject_type = 'user'
AND audit.subject_id = p_resolution_json #>> '{resolvedBy,id}'
AND audit.authentication_id = p_resolution_json ->> 'authenticationId'
AND audit.outcome = 'allowed'
AND audit.reasons = p_audit_json -> 'reasons'
AND audit.project_version = (p_resolution_json #>> '{authorizationFence,projectVersion}')::integer
AND audit.binding_version = (p_resolution_json #>> '{authorizationFence,bindingVersion}')::integer
AND audit.occurred_at_ms = (p_resolution_json ->> 'resolvedAtMs')::bigint
)
THEN
RETURN 'existing';
END IF;
RAISE EXCEPTION 'Approved Action manual recovery replay conflicts'
USING ERRCODE = 'check_violation';
END IF;
IF NOT "ql3"."lock_approval_policy_fence"(
p_resolution_json ->> 'projectId',
p_resolution_json #>> '{resolvedBy,type}',
p_resolution_json #>> '{resolvedBy,id}',
(p_resolution_json #>> '{authorizationFence,projectVersion}')::integer,
(p_resolution_json #>> '{authorizationFence,bindingVersion}')::integer
) THEN
RAISE EXCEPTION 'Approved Action manual recovery policy fence changed'
USING ERRCODE = 'check_violation';
END IF;
SELECT execution.execution_json,
execution.execution_digest,
execution.version,
execution.status,
execution.project_id,
execution.dispatch_digest,
dispatch.action_type,
dispatch.action_digest,
execution.lease_expires_at_ms
INTO current_execution_json,
current_execution_digest,
current_version,
current_status,
current_project_id,
current_dispatch_digest,
current_action_type,
current_action_digest,
current_lease_expires_at_ms
FROM "ql3"."approved_action_executions" AS execution
JOIN "ql3"."approved_action_dispatches" AS dispatch
ON dispatch.dispatch_id = execution.dispatch_id
WHERE execution.dispatch_id = p_resolution_json ->> 'dispatchId'
FOR UPDATE OF execution;
IF NOT FOUND
OR current_status <> 'executing'
OR current_lease_expires_at_ms IS NULL
OR current_lease_expires_at_ms > (p_resolution_json ->> 'resolvedAtMs')::bigint
OR current_project_id <> p_resolution_json ->> 'projectId'
OR current_dispatch_digest <> p_resolution_json ->> 'dispatchDigest'
OR current_action_type <> p_resolution_json ->> 'actionType'
OR current_action_type NOT IN (
'plugin_package.secret_binding.bind',
'plugin_package.secret_binding.transition'
)
OR current_action_digest <> p_resolution_json ->> 'actionDigest'
OR current_version <> (p_resolution_json ->> 'executionVersion')::integer
OR current_execution_digest <> p_resolution_json ->> 'executionDigest'
THEN
RAISE EXCEPTION 'Approved Action manual recovery execution fence changed'
USING ERRCODE = 'check_violation';
END IF;
IF p_resolution_json ->> 'decision' = 'confirm_failed' THEN
expected_status := 'failed';
expected_result_code := 'manual_recovery_confirmed_failed';
ELSIF p_resolution_json ->> 'decision' = 'abandon_unknown' THEN
expected_status := 'blocked';
expected_result_code := 'manual_recovery_abandoned_unknown';
ELSE
RAISE EXCEPTION 'Approved Action manual recovery decision is invalid'
USING ERRCODE = 'check_violation';
END IF;
IF NOT p_next_execution_json ?& ARRAY[
'schema', 'dispatchId', 'dispatchDigest', 'projectId', 'status',
'version', 'attemptCount', 'maxAttempts', 'eligibleAtMs',
'nextAttemptAtMs', 'leaseOwner', 'leaseToken', 'leaseExpiresAtMs',
'startedAtMs', 'resultMutationId', 'resultCode', 'resultDigest',
'completedAtMs', 'createdAtMs', 'updatedAtMs', 'executionDigest'
]::text[]
OR p_next_execution_json - ARRAY[
'status', 'version', 'eligibleAtMs', 'nextAttemptAtMs',
'leaseOwner', 'leaseToken', 'leaseExpiresAtMs', 'resultMutationId',
'resultCode', 'resultDigest', 'completedAtMs', 'updatedAtMs',
'executionDigest'
]::text[]
<> current_execution_json - ARRAY[
'status', 'version', 'eligibleAtMs', 'nextAttemptAtMs',
'leaseOwner', 'leaseToken', 'leaseExpiresAtMs', 'resultMutationId',
'resultCode', 'resultDigest', 'completedAtMs', 'updatedAtMs',
'executionDigest'
]::text[]
OR p_next_execution_json ->> 'status' <> expected_status
OR (p_next_execution_json ->> 'version')::integer <> current_version + 1
OR p_next_execution_json -> 'eligibleAtMs' <> 'null'::jsonb
OR p_next_execution_json -> 'nextAttemptAtMs' <> 'null'::jsonb
OR p_next_execution_json -> 'leaseOwner' <> 'null'::jsonb
OR p_next_execution_json -> 'leaseToken' <> 'null'::jsonb
OR p_next_execution_json -> 'leaseExpiresAtMs' <> 'null'::jsonb
OR p_next_execution_json ->> 'resultMutationId' <> p_resolution_json ->> 'mutationId'
OR p_next_execution_json ->> 'resultCode' <> expected_result_code
OR p_next_execution_json -> 'resultDigest' <> 'null'::jsonb
OR (p_next_execution_json ->> 'completedAtMs')::bigint <> (p_resolution_json ->> 'resolvedAtMs')::bigint
OR (p_next_execution_json ->> 'updatedAtMs')::bigint <> (p_resolution_json ->> 'resolvedAtMs')::bigint
OR p_next_execution_json ->> 'executionDigest' !~ '^[0-9a-f]{64}$'
THEN
RAISE EXCEPTION 'Approved Action manual recovery terminal execution is invalid'
USING ERRCODE = 'check_violation';
END IF;
IF p_audit_json ->> 'eventId' <> p_resolution_json ->> 'auditEventId'
OR p_audit_json ->> 'operationId' <> 'approval.recover.resolve'
OR p_audit_json ->> 'projectId' <> p_resolution_json ->> 'projectId'
OR p_audit_json #>> '{subject,type}' <> 'user'
OR p_audit_json #>> '{subject,id}' <> p_resolution_json #>> '{resolvedBy,id}'
OR p_audit_json ->> 'authenticationId' <> p_resolution_json ->> 'authenticationId'
OR p_audit_json ->> 'outcome' <> 'allowed'
OR p_audit_json -> 'reasons' <> '["role_grant","strong_authentication","manual_recovery"]'::jsonb
OR p_audit_json -> 'fence' <> p_resolution_json -> 'authorizationFence'
OR (p_audit_json ->> 'occurredAtMs')::bigint <> (p_resolution_json ->> 'resolvedAtMs')::bigint
THEN
RAISE EXCEPTION 'Approved Action manual recovery audit is invalid'
USING ERRCODE = 'check_violation';
END IF;
INSERT INTO "ql3"."security_audit_events" (
event_id, request_id, operation_id, project_id, subject_type, subject_id,
authentication_id, outcome, reasons, project_version, binding_version,
occurred_at_ms
) VALUES (
(p_audit_json ->> 'eventId')::uuid,
p_audit_json ->> 'requestId',
p_audit_json ->> 'operationId',
p_audit_json ->> 'projectId',
p_audit_json #>> '{subject,type}',
p_audit_json #>> '{subject,id}',
p_audit_json ->> 'authenticationId',
p_audit_json ->> 'outcome',
p_audit_json -> 'reasons',
(p_audit_json #>> '{fence,projectVersion}')::integer,
(p_audit_json #>> '{fence,bindingVersion}')::integer,
(p_audit_json ->> 'occurredAtMs')::bigint
);
UPDATE "ql3"."approved_action_executions"
SET status = expected_status,
version = (p_next_execution_json ->> 'version')::integer,
eligible_at_ms = NULL,
next_attempt_at_ms = NULL,
lease_owner = NULL,
lease_token = NULL,
lease_expires_at_ms = NULL,
result_mutation_id = p_next_execution_json ->> 'resultMutationId',
result_code = expected_result_code,
result_digest = NULL,
completed_at_ms = (p_next_execution_json ->> 'completedAtMs')::bigint,
updated_at_ms = (p_next_execution_json ->> 'updatedAtMs')::bigint,
execution_json = p_next_execution_json,
execution_digest = p_next_execution_json ->> 'executionDigest'
WHERE dispatch_id = p_resolution_json ->> 'dispatchId'
AND version = current_version
AND execution_digest = current_execution_digest;
IF NOT FOUND THEN
RAISE EXCEPTION 'Approved Action manual recovery update fence changed'
USING ERRCODE = 'check_violation';
END IF;
INSERT INTO "ql3"."approved_action_manual_recovery_resolutions" (
dispatch_id, dispatch_digest, project_id, action_type, action_digest,
execution_version, execution_digest, mutation_id, decision,
evidence_digest, reason_code, resolved_by_type, resolved_by_id,
authentication_id, assurance, authenticated_at_ms, project_version,
binding_version, audit_event_id, resolved_at_ms, resolution_json,
resolution_digest
) VALUES (
p_resolution_json ->> 'dispatchId',
p_resolution_json ->> 'dispatchDigest',
p_resolution_json ->> 'projectId',
p_resolution_json ->> 'actionType',
p_resolution_json ->> 'actionDigest',
(p_resolution_json ->> 'executionVersion')::integer,
p_resolution_json ->> 'executionDigest',
p_resolution_json ->> 'mutationId',
p_resolution_json ->> 'decision',
p_resolution_json ->> 'evidenceDigest',
p_resolution_json ->> 'reasonCode',
p_resolution_json #>> '{resolvedBy,type}',
p_resolution_json #>> '{resolvedBy,id}',
p_resolution_json ->> 'authenticationId',
p_resolution_json ->> 'assurance',
(p_resolution_json ->> 'authenticatedAtMs')::bigint,
(p_resolution_json #>> '{authorizationFence,projectVersion}')::integer,
(p_resolution_json #>> '{authorizationFence,bindingVersion}')::integer,
(p_resolution_json ->> 'auditEventId')::uuid,
(p_resolution_json ->> 'resolvedAtMs')::bigint,
p_resolution_json,
p_resolution_json ->> 'resolutionDigest'
);
RETURN 'resolved';
END
$ql3$
`.trim(),
`REVOKE ALL ON "ql3"."approved_action_manual_recovery_resolutions" FROM PUBLIC, ql3_runtime, ql3_admin, ql3_package_manager, ql3_package_executor, ql3_worker_ingress, ql3_worker_credential_manager, ql3_worker_credential_executor, ql3_automation_manager, ql3_approval_manager, ql3_run_manager`,
`GRANT SELECT ON "ql3"."approved_action_dispatches", "ql3"."approved_action_executions", "ql3"."approved_action_manual_recovery_resolutions" TO ql3_approval_manager`,
`REVOKE ALL ON FUNCTION "ql3"."resolve_approved_action_manual_recovery"(jsonb, jsonb, jsonb) FROM PUBLIC, ql3_runtime, ql3_admin, ql3_package_manager, ql3_package_executor, ql3_worker_ingress, ql3_worker_credential_manager, ql3_worker_credential_executor, ql3_automation_manager, ql3_approval_manager, ql3_run_manager`,
`GRANT EXECUTE ON FUNCTION "ql3"."resolve_approved_action_manual_recovery"(jsonb, jsonb, jsonb) TO ql3_approval_manager`,
`DO $ql3$ BEGIN UPDATE "ql3"."schema_capabilities" SET contract_version = 64, migration_id = 'pg-0065-approved-action-manual-recovery', capabilities = '${CAPABILITIES_V64}'::jsonb, updated_at_ms = floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint WHERE contract_name = 'control-core' AND contract_version = 63 AND migration_id = 'pg-0064-plugin-package-secret-binding-transition-approval-plans' AND capabilities = '${CAPABILITIES_V63}'::jsonb; IF NOT FOUND THEN RAISE EXCEPTION 'control-core capability is not at version 63' USING ERRCODE = 'check_violation'; END IF; END $ql3$`,
],
});
@@ -328,5 +328,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest =
checksum:
'1951b77a0265f8826169e4724424b2fbbd30061b27e27d3ba95de03430c1bac9',
}),
Object.freeze({
id: 'pg-0065-approved-action-manual-recovery',
checksum:
'95387c5b40659490dbcb7626ecd15bacf6412360752bef88873bde57c43e0185',
}),
]),
});
@@ -67,6 +67,7 @@ import { pg0061PluginPackageSecretBindingApprovalPlansMigration } from './pg-006
import { pg0062PluginPackageSecretBindingTargetGuardMigration } from './pg-0062-plugin-package-secret-binding-target-guard';
import { pg0063PluginPackageSecretBindingTransitionReceiptsMigration } from './pg-0063-plugin-package-secret-binding-transition-receipts';
import { pg0064PluginPackageSecretBindingTransitionApprovalPlansMigration } from './pg-0064-plugin-package-secret-binding-transition-approval-plans';
import { pg0065ApprovedActionManualRecoveryMigration } from '../approved-action/pg-0065-approved-action-manual-recovery';
export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
Object.freeze({
@@ -139,5 +140,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMi
pg0062PluginPackageSecretBindingTargetGuardMigration,
pg0063PluginPackageSecretBindingTransitionReceiptsMigration,
pg0064PluginPackageSecretBindingTransitionApprovalPlansMigration,
pg0065ApprovedActionManualRecoveryMigration,
]),
});
@@ -1027,6 +1027,80 @@ export const approvedActionExecutions = ql3Schema.table(
],
);
export const approvedActionManualRecoveryResolutions = ql3Schema.table(
'approved_action_manual_recovery_resolutions',
{
dispatchId: varchar('dispatch_id', { length: 128 }).primaryKey(),
dispatchDigest: char('dispatch_digest', { length: 64 }).notNull(),
projectId: varchar('project_id', { length: 128 }).notNull(),
actionType: varchar('action_type', { length: 128 }).notNull(),
actionDigest: char('action_digest', { length: 64 }).notNull(),
executionVersion: integer('execution_version').notNull(),
executionDigest: char('execution_digest', { length: 64 }).notNull(),
mutationId: varchar('mutation_id', { length: 128 }).notNull(),
decision: varchar('decision', { length: 32 }).notNull(),
evidenceDigest: char('evidence_digest', { length: 64 }).notNull(),
reasonCode: varchar('reason_code', { length: 64 }).notNull(),
resolvedByType: varchar('resolved_by_type', { length: 16 }).notNull(),
resolvedById: varchar('resolved_by_id', { length: 255 }).notNull(),
authenticationId: varchar('authentication_id', { length: 128 }).notNull(),
assurance: varchar('assurance', { length: 32 }).notNull(),
authenticatedAtMs: bigint('authenticated_at_ms', { mode: 'number' }).notNull(),
projectVersion: integer('project_version').notNull(),
bindingVersion: integer('binding_version').notNull(),
auditEventId: uuid('audit_event_id').notNull(),
resolvedAtMs: bigint('resolved_at_ms', { mode: 'number' }).notNull(),
resolutionJson: jsonb('resolution_json')
.$type<Record<string, unknown>>()
.notNull(),
resolutionDigest: char('resolution_digest', { length: 64 }).notNull(),
},
(table) => [
foreignKey({
name: 'ql3_approved_action_manual_recovery_dispatch_fk',
columns: [table.dispatchId],
foreignColumns: [approvedActionDispatches.dispatchId],
}).onDelete('restrict'),
foreignKey({
name: 'ql3_approved_action_manual_recovery_project_fk',
columns: [table.projectId],
foreignColumns: [projects.id],
}).onDelete('restrict'),
foreignKey({
name: 'ql3_approved_action_manual_recovery_audit_fk',
columns: [table.auditEventId],
foreignColumns: [securityAuditEvents.eventId],
}).onDelete('restrict'),
check(
'ql3_approved_action_manual_recovery_identity_check',
sql`${table.dispatchId} ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' and ${table.projectId} ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' and ${table.actionType} in ('plugin_package.secret_binding.bind','plugin_package.secret_binding.transition') and ${table.executionVersion} between 1 and 2147483647 and ${table.mutationId} ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' and ${table.decision} in ('confirm_failed','abandon_unknown') and ${table.reasonCode} ~ '^[a-z][a-z0-9_]{0,63}$' and ${table.resolvedByType} = 'user' and octet_length(${table.resolvedById}) between 1 and 255 and ${table.resolvedById} !~ '[[:cntrl:]]' and ${table.authenticationId} ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' and ${table.assurance} in ('multi_factor','hardware') and ${table.projectVersion} >= 1 and ${table.bindingVersion} >= 1`,
),
check(
'ql3_approved_action_manual_recovery_digest_check',
sql`${table.dispatchDigest} ~ '^[0-9a-f]{64}$' and ${table.actionDigest} ~ '^[0-9a-f]{64}$' and ${table.executionDigest} ~ '^[0-9a-f]{64}$' and ${table.evidenceDigest} ~ '^[0-9a-f]{64}$' and ${table.resolutionDigest} ~ '^[0-9a-f]{64}$'`,
),
check(
'ql3_approved_action_manual_recovery_time_check',
sql`${table.authenticatedAtMs} >= 0 and ${table.resolvedAtMs} >= ${table.authenticatedAtMs} and ${table.resolvedAtMs} - ${table.authenticatedAtMs} <= 300000`,
),
check(
'ql3_approved_action_manual_recovery_json_check',
sql`jsonb_typeof(${table.resolutionJson}) = 'object' and octet_length(${table.resolutionJson}::text) between 2 and 65536 and ${table.resolutionJson} @> jsonb_build_object('schema', 'qinglong/approved-action-manual-recovery@v1', 'dispatchId', ${table.dispatchId}, 'dispatchDigest', ${table.dispatchDigest}, 'projectId', ${table.projectId}, 'actionType', ${table.actionType}, 'actionDigest', ${table.actionDigest}, 'executionVersion', ${table.executionVersion}, 'executionDigest', ${table.executionDigest}, 'mutationId', ${table.mutationId}, 'decision', ${table.decision}, 'evidenceDigest', ${table.evidenceDigest}, 'reasonCode', ${table.reasonCode}, 'resolvedBy', jsonb_build_object('type', ${table.resolvedByType}, 'id', ${table.resolvedById}), 'authenticationId', ${table.authenticationId}, 'assurance', ${table.assurance}, 'authenticatedAtMs', ${table.authenticatedAtMs}, 'authorizationFence', jsonb_build_object('projectVersion', ${table.projectVersion}, 'bindingVersion', ${table.bindingVersion}), 'auditEventId', ${table.auditEventId}, 'resolvedAtMs', ${table.resolvedAtMs}, 'resolutionDigest', ${table.resolutionDigest})`,
),
uniqueIndex('ql3_approved_action_manual_recovery_mutation_uidx').on(
table.mutationId,
),
uniqueIndex('ql3_approved_action_manual_recovery_digest_uidx').on(
table.resolutionDigest,
),
index('ql3_approved_action_manual_recovery_project_idx').on(
table.projectId,
table.resolvedAtMs,
table.dispatchId,
),
],
);
export const pluginPackageInstallProposals = ql3Schema.table(
'plugin_package_install_proposals',
{
@@ -6103,6 +6177,7 @@ export const ql3PostgresTables = [
approvalRequests,
approvedActionDispatches,
approvedActionExecutions,
approvedActionManualRecoveryResolutions,
pluginPackageInstallProposals,
pluginPackageManagementQuotaBuckets,
workerCredentialManagementQuotaBuckets,
@@ -21,8 +21,8 @@ export interface PostgresSchemaContractTrigger {
export interface PostgresSchemaContract {
readonly schema: 'ql3';
readonly contractName: 'control-core';
readonly contractVersion: 63;
readonly migrationId: 'pg-0064-plugin-package-secret-binding-transition-approval-plans';
readonly contractVersion: 64;
readonly migrationId: 'pg-0065-approved-action-manual-recovery';
readonly minimumServerMajor: 16;
readonly maximumServerMajor: 18;
readonly capabilities: Readonly<{
@@ -38,6 +38,7 @@ export interface PostgresSchemaContract {
api_credential_pepper_binding: 1;
approved_action: 1;
approved_action_execution: 1;
approved_action_manual_recovery: 1;
approval_management_boundary: 1;
automation_management_boundary: 1;
automation_management_identity_keyset_ledger: 1;
@@ -117,8 +118,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
Object.freeze({
schema: 'ql3',
contractName: 'control-core',
contractVersion: 63,
migrationId: 'pg-0064-plugin-package-secret-binding-transition-approval-plans',
contractVersion: 64,
migrationId: 'pg-0065-approved-action-manual-recovery',
minimumServerMajor: 16,
maximumServerMajor: 18,
capabilities: Object.freeze({
@@ -127,6 +128,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
api_credential_pepper_binding: 1,
approved_action: 1,
approved_action_execution: 1,
approved_action_manual_recovery: 1,
approval_management_boundary: 1,
automation_management_boundary: 1,
automation_management_identity_keyset_ledger: 1,
@@ -770,6 +772,30 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
'execution_json',
'execution_digest',
]),
table('approved_action_manual_recovery_resolutions', [
'dispatch_id',
'dispatch_digest',
'project_id',
'action_type',
'action_digest',
'execution_version',
'execution_digest',
'mutation_id',
'decision',
'evidence_digest',
'reason_code',
'resolved_by_type',
'resolved_by_id',
'authentication_id',
'assurance',
'authenticated_at_ms',
'project_version',
'binding_version',
'audit_event_id',
'resolved_at_ms',
'resolution_json',
'resolution_digest',
]),
table('plugin_package_install_proposals', [
'action_ref',
'project_id',
@@ -1624,6 +1650,10 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
'ql3_approved_action_execution_due_idx',
'ql3_approved_action_execution_recovery_idx',
'ql3_approved_action_execution_project_idx',
'approved_action_manual_recovery_resolutions_pkey',
'ql3_approved_action_manual_recovery_mutation_uidx',
'ql3_approved_action_manual_recovery_digest_uidx',
'ql3_approved_action_manual_recovery_project_idx',
'plugin_package_install_proposals_pkey',
'ql3_plugin_package_proposal_project_idx',
'plugin_package_management_quota_buckets_pkey',
@@ -1942,6 +1972,10 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
'ql3_approved_action_execution_digest_check',
'ql3_approved_action_execution_json_check',
'ql3_approved_action_execution_time_check',
'ql3_approved_action_manual_recovery_identity_check',
'ql3_approved_action_manual_recovery_digest_check',
'ql3_approved_action_manual_recovery_time_check',
'ql3_approved_action_manual_recovery_json_check',
'ql3_plugin_package_proposal_identity_check',
'ql3_plugin_package_proposal_digest_check',
'ql3_plugin_package_proposal_json_check',
@@ -2347,6 +2381,9 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
'ql3_approved_action_dispatch_request_fk',
'ql3_approved_action_dispatch_project_fk',
'ql3_approved_action_execution_dispatch_fk',
'ql3_approved_action_manual_recovery_dispatch_fk',
'ql3_approved_action_manual_recovery_project_fk',
'ql3_approved_action_manual_recovery_audit_fk',
'ql3_approved_action_execution_project_fk',
'ql3_plugin_package_proposal_project_fk',
'ql3_plugin_package_management_quota_project_fk',
@@ -2434,6 +2471,15 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
'ql3_run_retry_policies_run_fk',
]),
functions: Object.freeze([
Object.freeze({
name: 'resolve_approved_action_manual_recovery',
identityArguments:
'p_resolution_json jsonb, p_next_execution_json jsonb, p_audit_json jsonb',
owner: 'ql3_migration',
securityDefiner: true,
volatility: 'volatile',
configuration: Object.freeze(['search_path=pg_catalog, ql3']),
}),
Object.freeze({
name: 'plugin_package_secret_binding_transition_snapshot',
identityArguments:
@@ -384,6 +384,12 @@ const REQUIRED_RUNTIME_PRIVILEGES = Object.freeze({
update: false,
delete: false,
}),
approved_action_manual_recovery_resolutions: Object.freeze({
select: false,
insert: false,
update: false,
delete: false,
}),
plugin_package_install_proposals: Object.freeze({
select: false,
insert: false,
@@ -921,6 +927,12 @@ const REQUIRED_ADMIN_PRIVILEGES = Object.freeze({
update: false,
delete: false,
}),
approved_action_manual_recovery_resolutions: Object.freeze({
select: false,
insert: false,
update: false,
delete: false,
}),
plugin_package_install_proposals: Object.freeze({
select: false,
insert: false,
@@ -1421,7 +1433,10 @@ const REQUIRED_APPROVAL_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze(
name === 'schema_capabilities' ||
name === 'projects' ||
name === 'project_role_bindings' ||
name === 'tool_invocation_preview_artifacts'
name === 'tool_invocation_preview_artifacts' ||
name === 'approved_action_dispatches' ||
name === 'approved_action_executions' ||
name === 'approved_action_manual_recovery_resolutions'
? { ...NO_TABLE_PRIVILEGES, select: true }
: name === 'security_audit_events'
? { ...NO_TABLE_PRIVILEGES, select: true, insert: true }
@@ -1573,6 +1588,7 @@ const REQUIRED_RUNTIME_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
plugin_package_secret_binding_transition_snapshot: false,
plugin_package_tool_start_allowed: true,
register_plugin_package_automation_disposition_event: false,
resolve_approved_action_manual_recovery: false,
});
const REQUIRED_PACKAGE_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
@@ -1598,6 +1614,7 @@ const REQUIRED_PACKAGE_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
plugin_package_secret_binding_transition_snapshot: true,
plugin_package_tool_start_allowed: false,
register_plugin_package_automation_disposition_event: false,
resolve_approved_action_manual_recovery: false,
});
const REQUIRED_PACKAGE_EXECUTOR_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
@@ -1623,6 +1640,7 @@ const REQUIRED_PACKAGE_EXECUTOR_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges
plugin_package_secret_binding_transition_snapshot: false,
plugin_package_tool_start_allowed: false,
register_plugin_package_automation_disposition_event: false,
resolve_approved_action_manual_recovery: false,
});
const REQUIRED_WORKER_CREDENTIAL_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
@@ -1635,6 +1653,7 @@ const REQUIRED_APPROVAL_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges
Object.freeze({
...NO_FUNCTION_PRIVILEGES,
lock_approval_policy_fence: true,
resolve_approved_action_manual_recovery: true,
});
const REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges =
@@ -0,0 +1,273 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
approvalRequestDigest,
approvedActionDispatchDigest,
consumeApprovalRequest,
createApprovalRequest,
decideApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
createApprovedActionExecution,
} = require('@qinglong/runtime-core/approved-action-execution');
const {
createApprovedActionManualRecoveryService,
} = require('@qinglong/runtime-core/approved-action-manual-recovery');
const { ProjectPolicyEngine } = require('@qinglong/runtime-core/project-policy');
const {
assertPostgresApprovalManagerSchemaReady,
createPostgresDatabaseOpener,
PostgresApprovedActionManualRecoveryRepository,
PostgresProjectPolicyRepository,
PostgresSecurityAuditRepository,
} = require('@qinglong/cluster-postgres/approval-manager');
const {
PostgresApprovedActionExecutionRepository,
} = require('../dist/approved-action/approvedActionExecutionRepository');
const { runPostgresMigrations } = require('../dist/migration/migration');
const migrationConnectionString = process.env.QL3_TEST_POSTGRES_MIGRATION_URL;
const approvalManagerConnectionString =
process.env.QL3_TEST_POSTGRES_APPROVAL_MANAGER_URL;
async function open(role, connectionString) {
return createPostgresDatabaseOpener({
role,
connection: { connectionString, tls: { mode: 'disable' } },
pool: {
maxConnections: 1,
applicationName: `ql3-manual-recovery-${role}`,
},
onPoolError(error) {
throw error;
},
})();
}
async function insertFixture(pool, namespace) {
const projectId = `${namespace}-project`;
const principal = Object.freeze({
subject: Object.freeze({ type: 'user', id: `${namespace}-owner` }),
authenticationId: `${namespace}-session`,
authenticatedAtMs: 100,
expiresAtMs: 20_000,
assurance: 'hardware',
});
const action = Object.freeze({
permission: 'secret.manage',
actionType: 'plugin_package.secret_binding.bind',
actionRef: `${namespace}:secret-binding`,
actionDigest: 'a'.repeat(64),
previewDigest: 'b'.repeat(64),
});
const pending = createApprovalRequest({
id: `${namespace}-approval`,
projectId,
action,
risk: 'high',
decisionMode: 'human_confirmation',
requestedBy: { type: 'agent', id: `${namespace}-agent` },
requestedAtMs: 800,
expiresAtMs: 10_000,
requestFence: { projectVersion: 1, bindingVersion: 1 },
});
const approved = decideApprovalRequest(pending, {
expectedVersion: 1,
decisionId: `${namespace}-decision`,
decision: 'approved',
reasonCode: 'reviewed',
principal,
decidedAtMs: 900,
authorizationFence: { projectVersion: 1, bindingVersion: 1 },
});
const consumed = consumeApprovalRequest(approved, {
expectedVersion: 2,
consumptionId: `${namespace}-consumption`,
dispatchId: `${namespace}-dispatch`,
action,
requestedBy: pending.requestedBy,
consumedBy: { type: 'system', id: 'package-executor' },
consumedAtMs: 950,
authorizationFence: { projectVersion: 1, bindingVersion: 1 },
});
await pool.query(
`INSERT INTO "ql3"."projects" (
id, name, slug, status, version, created_at_ms, updated_at_ms
) VALUES ($1, $1, $2, 'active', 1, 1, 1)`,
[projectId, projectId.replace(/[^a-z0-9-]/g, '-')],
);
await pool.query(
`INSERT INTO "ql3"."project_role_bindings" (
project_id, subject_type, subject_id, version, state, role,
mutation_id, changed_by_type, changed_by_id, created_at_ms
) VALUES ($1, 'user', $2, 1, 'active', 'owner', $3, 'system',
'integration-fixture', 2)`,
[projectId, principal.subject.id, `${namespace}-binding`],
);
const request = consumed.request;
await pool.query(
`INSERT INTO "ql3"."approval_requests" (
request_id, project_id, version, state, action_type, action_ref,
action_digest, preview_digest, requested_by_type, requested_by_id,
decision_id, consumption_id, dispatch_id, expires_at_ms, request_json,
request_digest, updated_at_ms
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14,
$15::jsonb, $16, $17
)`,
[
request.id,
request.projectId,
request.version,
request.state,
request.action.actionType,
request.action.actionRef,
request.action.actionDigest,
request.action.previewDigest,
request.requestedBy.type,
request.requestedBy.id,
request.decisionId,
request.consumptionId,
request.dispatchId,
request.expiresAtMs,
JSON.stringify(request),
approvalRequestDigest(request),
request.consumedAtMs,
],
);
const dispatch = consumed.dispatch;
await pool.query(
`INSERT INTO "ql3"."approved_action_dispatches" (
dispatch_id, approval_request_id, project_id, action_type, action_ref,
action_digest, preview_digest, dispatch_json, dispatch_digest,
created_at_ms
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10)`,
[
dispatch.id,
dispatch.approvalRequestId,
dispatch.projectId,
dispatch.action.actionType,
dispatch.action.actionRef,
dispatch.action.actionDigest,
dispatch.action.previewDigest,
JSON.stringify(dispatch),
approvedActionDispatchDigest(dispatch),
dispatch.createdAtMs,
],
);
const executions = new PostgresApprovedActionExecutionRepository(pool);
await pool.query(
`INSERT INTO "ql3"."approved_action_executions" (
dispatch_id, dispatch_digest, project_id, status, version,
attempt_count, max_attempts, eligible_at_ms, next_attempt_at_ms,
lease_owner, lease_token, lease_expires_at_ms, started_at_ms,
result_mutation_id, result_code, result_digest, completed_at_ms,
created_at_ms, updated_at_ms, execution_json, execution_digest
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12,
$13, $14, $15, $16, $17, $18, $19, $20::jsonb, $21)`,
(() => {
const value = createApprovedActionExecution(dispatch);
return [
value.dispatchId, value.dispatchDigest, value.projectId, value.status,
value.version, value.attemptCount, value.maxAttempts, value.eligibleAtMs,
value.nextAttemptAtMs, value.leaseOwner, value.leaseToken,
value.leaseExpiresAtMs, value.startedAtMs, value.resultMutationId,
value.resultCode, value.resultDigest, value.completedAtMs,
value.createdAtMs, value.updatedAtMs, JSON.stringify(value),
value.executionDigest,
];
})(),
);
const claimed = await executions.claimExecution({
dispatchId: dispatch.id,
owner: `${namespace}-executor`,
leaseToken: `${namespace}-lease`,
nowMs: 1_000,
leaseDurationMs: 500,
});
const started = await executions.startExecution({
dispatchId: dispatch.id,
approvalRequestId: dispatch.approvalRequestId,
actionDigest: dispatch.action.actionDigest,
owner: `${namespace}-executor`,
leaseToken: `${namespace}-lease`,
expectedVersion: claimed.snapshot.execution.version,
startedAtMs: 1_100,
});
return { projectId, principal, dispatch, execution: started.execution };
}
if (!migrationConnectionString || !approvalManagerConnectionString) {
test('PostgreSQL manual recovery gate requires migration and Approval manager URLs', {
skip: true,
});
} else {
test('PostgreSQL atomically resolves and exactly replays an expired Secret Action', async () => {
const migration = await open('migration', migrationConnectionString);
let manager;
try {
await runPostgresMigrations({ pool: migration.pool });
manager = await open('approval-manager', approvalManagerConnectionString);
const readiness = await assertPostgresApprovalManagerSchemaReady(manager.pool);
assert.equal(readiness.contractVersion, 64);
const namespace = `recovery-${process.pid}-${Date.now()}`;
const fixture = await insertFixture(migration.pool, namespace);
const service = createApprovedActionManualRecoveryService({
repository: new PostgresApprovedActionManualRecoveryRepository(manager.pool),
policy: new ProjectPolicyEngine(
new PostgresProjectPolicyRepository(manager.pool),
),
audit: new PostgresSecurityAuditRepository(manager.pool),
now: () => 2_000,
});
const inspected = await service.inspect({
projectId: fixture.projectId,
dispatchId: fixture.dispatch.id,
requestId: `${namespace}-inspect`,
auditEventId: '80000000-0000-4000-8000-000000000001',
principal: fixture.principal,
});
assert.equal(inspected.execution.execution.status, 'executing');
const request = {
projectId: fixture.projectId,
dispatchId: fixture.dispatch.id,
expectedExecutionVersion: fixture.execution.version,
expectedExecutionDigest: fixture.execution.executionDigest,
mutationId: `${namespace}-mutation`,
decision: 'abandon_unknown',
evidenceDigest: 'e'.repeat(64),
reasonCode: 'orphan_absence_verified',
requestId: `${namespace}-resolve`,
auditEventId: '80000000-0000-4000-8000-000000000002',
principal: fixture.principal,
};
const first = await service.resolve(request);
const replay = await service.resolve(request);
assert.equal(first.status, 'resolved');
assert.equal(replay.status, 'existing');
assert.equal(first.snapshot.execution.execution.status, 'blocked');
assert.equal(first.snapshot.resolution.decision, 'abandon_unknown');
const persisted = await migration.pool.query(
`SELECT
(SELECT count(*)::integer FROM "ql3"."approved_action_manual_recovery_resolutions"
WHERE dispatch_id = $1) AS "resolutionCount",
(SELECT count(*)::integer FROM "ql3"."security_audit_events"
WHERE event_id = $2) AS "auditCount"`,
[fixture.dispatch.id, request.auditEventId],
);
assert.deepEqual(persisted.rows[0], { resolutionCount: 1, auditCount: 1 });
await assert.rejects(
manager.pool.query(
`UPDATE "ql3"."approved_action_executions" SET status = 'failed'
WHERE dispatch_id = $1`,
[fixture.dispatch.id],
),
(error) => error && error.code === '42501',
);
} finally {
if (manager) await manager.close();
await migration.close();
}
});
}
@@ -0,0 +1,158 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
consumeApprovalRequest,
createApprovalRequest,
decideApprovalRequest,
} = require('@qinglong/runtime-core/approved-action');
const {
claimApprovedActionExecution,
createApprovedActionExecution,
startApprovedActionExecution,
} = require('@qinglong/runtime-core/approved-action-execution');
const {
createApprovedActionManualRecoveryService,
} = require('@qinglong/runtime-core/approved-action-manual-recovery');
const {
PostgresApprovedActionManualRecoveryRepository,
} = require('@qinglong/cluster-postgres/approval-manager');
const PRINCIPAL = Object.freeze({
subject: Object.freeze({ type: 'user', id: 'owner-1' }),
authenticationId: 'oidc:session-1',
authenticatedAtMs: 100,
expiresAtMs: 20_000,
assurance: 'hardware',
});
function executing() {
const action = {
permission: 'secret.manage',
actionType: 'plugin_package.secret_binding.bind',
actionRef: 'secret-binding:1',
actionDigest: 'a'.repeat(64),
previewDigest: 'b'.repeat(64),
};
const pending = createApprovalRequest({
id: 'approval-1',
projectId: 'default',
action,
risk: 'high',
decisionMode: 'human_confirmation',
requestedBy: { type: 'agent', id: 'agent-1' },
requestedAtMs: 800,
expiresAtMs: 10_000,
requestFence: { projectVersion: 1, bindingVersion: 2 },
});
const approved = decideApprovalRequest(pending, {
expectedVersion: 1,
decisionId: 'decision-1',
decision: 'approved',
reasonCode: 'reviewed',
principal: PRINCIPAL,
decidedAtMs: 900,
authorizationFence: { projectVersion: 1, bindingVersion: 2 },
});
const dispatch = consumeApprovalRequest(approved, {
expectedVersion: 2,
consumptionId: 'consumption-1',
dispatchId: 'dispatch-1',
action,
requestedBy: pending.requestedBy,
consumedBy: { type: 'system', id: 'package-executor' },
consumedAtMs: 950,
authorizationFence: { projectVersion: 1, bindingVersion: 2 },
}).dispatch;
const leased = claimApprovedActionExecution(createApprovedActionExecution(dispatch), {
owner: 'executor-1',
leaseToken: 'lease-1',
nowMs: 1_000,
leaseDurationMs: 500,
});
const execution = startApprovedActionExecution(
{ dispatch, execution: leased },
{
dispatchId: dispatch.id,
approvalRequestId: dispatch.approvalRequestId,
actionDigest: dispatch.action.actionDigest,
owner: leased.leaseOwner,
leaseToken: leased.leaseToken,
expectedVersion: leased.version,
startedAtMs: 1_100,
},
);
return { dispatch, execution };
}
test('resolves through the bounded PostgreSQL function and verifies the stored tuple', async () => {
const initial = executing();
let stored = { ...initial, resolution: null };
const calls = [];
const pool = {
async query(text, values) {
calls.push([text, values]);
if (text.includes('resolve_approved_action_manual_recovery')) {
stored = {
dispatch: initial.dispatch,
execution: JSON.parse(values[1]),
resolution: JSON.parse(values[0]),
};
return { rows: [{ status: 'resolved' }] };
}
if (text.includes('approved_action_manual_recovery_resolutions')) {
return {
rows: [
{
dispatchJson: stored.dispatch,
executionJson: stored.execution,
executionDigest: stored.execution.executionDigest,
resolutionJson: stored.resolution,
},
],
};
}
throw new Error(`unexpected query: ${text}`);
},
async connect() {
throw new Error('repository must not open a broad transaction');
},
};
const repository = new PostgresApprovedActionManualRecoveryRepository(pool);
const service = createApprovedActionManualRecoveryService({
repository,
policy: {
async authorize(_principal, _projectId, permission) {
assert.equal(permission, 'approval.recover');
return {
effect: 'allow',
reasons: ['role_grant'],
fence: { projectVersion: 1, bindingVersion: 2 },
};
},
},
audit: { async record() {} },
now: () => 2_000,
});
const result = await service.resolve({
projectId: 'default',
dispatchId: 'dispatch-1',
expectedExecutionVersion: initial.execution.version,
expectedExecutionDigest: initial.execution.executionDigest,
mutationId: 'manual-recovery-1',
decision: 'abandon_unknown',
evidenceDigest: 'e'.repeat(64),
reasonCode: 'orphan_absence_verified',
auditEventId: '70000000-0000-4000-8000-000000000001',
requestId: 'manual-recovery-request-1',
principal: PRINCIPAL,
});
assert.equal(result.status, 'resolved');
assert.equal(result.snapshot.execution.execution.status, 'blocked');
assert.equal(result.snapshot.resolution.decision, 'abandon_unknown');
assert.equal(
calls.filter(([sql]) => sql.includes('resolve_approved_action_manual_recovery'))
.length,
1,
);
});
@@ -115,6 +115,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
'pg-0062-plugin-package-secret-binding-target-guard',
'pg-0063-plugin-package-secret-binding-transition-receipts',
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
],
);
for (const migration of postgresqlMainMigrationStream.migrations) {
@@ -573,6 +574,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
checksum:
'1951b77a0265f8826169e4724424b2fbbd30061b27e27d3ba95de03430c1bac9',
},
{
id: 'pg-0065-approved-action-manual-recovery',
checksum:
'95387c5b40659490dbcb7626ecd15bacf6412360752bef88873bde57c43e0185',
},
];
assert.deepEqual(
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
@@ -2227,3 +2233,52 @@ test('advances capability v63 with manager-only immutable Secret transition plan
/migration_id = 'pg-0063-plugin-package-secret-binding-transition-receipts'/,
);
});
test('advances capability v64 with atomic least-privilege manual recovery', async () => {
const migration = migrationById('pg-0065-approved-action-manual-recovery');
const statements = [];
await migration.up({
async query(statement) {
statements.push(statement);
return { rows: [] };
},
});
const sql = statements.join('\n');
assert.match(
sql,
/CREATE TABLE "ql3"\."approved_action_manual_recovery_resolutions"/,
);
assert.match(
sql,
/CREATE FUNCTION "ql3"\."resolve_approved_action_manual_recovery"\([\s\S]+SECURITY DEFINER[\s\S]+SET search_path = pg_catalog, ql3/,
);
assert.match(sql, /current_status <> 'executing'/);
assert.match(sql, /current_lease_expires_at_ms > .*'resolvedAtMs'/);
assert.match(
sql,
/current_action_type NOT IN \([\s\S]+plugin_package\.secret_binding\.bind[\s\S]+plugin_package\.secret_binding\.transition/,
);
assert.match(
sql,
/INSERT INTO "ql3"\."security_audit_events"[\s\S]+UPDATE "ql3"\."approved_action_executions"[\s\S]+INSERT INTO "ql3"\."approved_action_manual_recovery_resolutions"/,
);
assert.match(
sql,
/GRANT SELECT ON "ql3"\."approved_action_dispatches", "ql3"\."approved_action_executions", "ql3"\."approved_action_manual_recovery_resolutions" TO ql3_approval_manager/,
);
assert.match(
sql,
/GRANT EXECUTE ON FUNCTION "ql3"\."resolve_approved_action_manual_recovery"\(jsonb, jsonb, jsonb\) TO ql3_approval_manager/,
);
assert.doesNotMatch(
sql,
/GRANT (?:INSERT|UPDATE|DELETE)[^;]+(?:approved_action_executions|approved_action_manual_recovery_resolutions)[^;]+ql3_approval_manager/,
);
assert.match(sql, /contract_version = 64/);
assert.match(sql, /"approved_action_manual_recovery":1/);
assert.match(sql, /contract_version = 63/);
assert.match(
sql,
/migration_id = 'pg-0064-plugin-package-secret-binding-transition-approval-plans'/,
);
});
@@ -134,6 +134,7 @@ function validPrivileges() {
approval_requests: [false, false, false, false],
approved_action_dispatches: [false, false, false, false],
approved_action_executions: [false, false, false, false],
approved_action_manual_recovery_resolutions: [false, false, false, false],
plugin_package_install_proposals: [false, false, false, false],
plugin_package_management_quota_buckets: [false, false, false, false],
plugin_package_identity_keyset_ledger: [false, false, false, false],
@@ -271,6 +272,7 @@ function validAdminPrivileges() {
approval_requests: [false, false, false, false],
approved_action_dispatches: [false, false, false, false],
approved_action_executions: [false, false, false, false],
approved_action_manual_recovery_resolutions: [false, false, false, false],
plugin_package_install_proposals: [false, false, false, false],
plugin_package_management_quota_buckets: [false, false, false, false],
plugin_package_identity_keyset_ledger: [false, false, false, false],
@@ -491,6 +493,9 @@ function approvalManagerPrivileges() {
'project_role_bindings',
'security_audit_events',
'approval_requests',
'approved_action_dispatches',
'approved_action_executions',
'approved_action_manual_recovery_resolutions',
'tool_invocation_preview_artifacts',
'plugin_package_identity_keyset_ledger',
]);
@@ -705,6 +710,11 @@ function queryable(overrides = {}) {
'plugin_package_secret_binding_planning_snapshot',
'plugin_package_secret_binding_transition_snapshot',
].includes(functionName)
: overrides.functionMode === 'approval-manager'
? [
'lock_approval_policy_fence',
'resolve_approved_action_manual_recovery',
].includes(functionName)
: overrides.functionMode === 'manager'
? functionName === 'lock_approval_policy_fence'
: overrides.functionMode === 'run-manager'
@@ -805,7 +815,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
serverMajor: 16,
currentUser: 'ql3_runtime',
contractName: 'control-core',
contractVersion: 63,
contractVersion: 64,
migrationIds: [
'pg-0001-schema-capability',
'pg-0002-run-core',
@@ -871,6 +881,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
'pg-0062-plugin-package-secret-binding-target-guard',
'pg-0063-plugin-package-secret-binding-transition-receipts',
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
],
});
});
@@ -901,10 +912,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
}),
);
assert.equal(report.currentUser, 'ql3_admin');
assert.equal(report.contractVersion, 63);
assert.equal(report.contractVersion, 64);
assert.equal(
report.migrationIds.at(-1),
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
);
});
@@ -917,10 +928,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
}),
);
assert.equal(report.currentUser, 'ql3_automation_manager');
assert.equal(report.contractVersion, 63);
assert.equal(report.contractVersion, 64);
assert.equal(
report.migrationIds.at(-1),
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
);
const widened = automationManagerPrivileges();
@@ -945,14 +956,14 @@ test('accepts the isolated least-privilege human Approval manager role', async (
queryable({
currentUser: 'ql3_approval_manager',
privileges: approvalManagerPrivileges(),
functionMode: 'manager',
functionMode: 'approval-manager',
}),
);
assert.equal(report.currentUser, 'ql3_approval_manager');
assert.equal(report.contractVersion, 63);
assert.equal(report.contractVersion, 64);
assert.equal(
report.migrationIds.at(-1),
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
);
const widened = approvalManagerPrivileges();
@@ -964,7 +975,7 @@ test('accepts the isolated least-privilege human Approval manager role', async (
queryable({
currentUser: 'ql3_approval_manager',
privileges: widened,
functionMode: 'manager',
functionMode: 'approval-manager',
}),
),
(error) =>
@@ -983,10 +994,10 @@ test('accepts the isolated least-privilege Run manager role', async () => {
}),
);
assert.equal(report.currentUser, 'ql3_run_manager');
assert.equal(report.contractVersion, 63);
assert.equal(report.contractVersion, 64);
assert.equal(
report.migrationIds.at(-1),
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
);
const widened = runManagerPrivileges();
@@ -1118,10 +1129,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
}),
);
assert.equal(report.currentUser, 'ql3_worker_ingress');
assert.equal(report.contractVersion, 63);
assert.equal(report.contractVersion, 64);
assert.equal(
report.migrationIds.at(-1),
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
);
});