open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-23 03:18:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): resolve secret action recovery manually

Browse Source
This commit is contained in:
whyour
2026-08-14 12:11:50 +08:00
parent d8489d8a0b
commit 3ca9901055
29 changed files with 3143 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-cluster-admin/src/approval-management/approvalManagement.ts
+33
View File
@@ -1,11 +1,18 @@
import {
PostgresApprovalManagementIdentityKeysetLedgerRepository,
PostgresApprovedActionManualRecoveryRepository,
PostgresApprovalRequestRepository,
PostgresApprovalRequestSource,
PostgresProjectPolicyRepository,
PostgresSecurityAuditRepository,
} from '@qinglong/cluster-postgres/approval-manager';
import type { PostgresPool } from '@qinglong/runtime-core';
import {
createApprovedActionManualRecoveryService,
type ApprovedActionManualRecoveryInspectRequest,
type ApprovedActionManualRecoveryResolveRequest,
type ApprovedActionManualRecoveryService,
} from '@qinglong/runtime-core/approved-action-manual-recovery';
import {
createApprovalDecisionService,
type ApprovalDecisionRequest,
@@ -28,6 +35,14 @@ export interface ClusterApprovalManagementService {
request: ApprovalDecisionRequest,
confirmAuthorization: () => void | Promise<void>,
): ReturnType<ApprovalDecisionService['decide']>;
inspectRecovery(
request: ApprovedActionManualRecoveryInspectRequest,
confirmAuthorization: () => void | Promise<void>,
): ReturnType<ApprovedActionManualRecoveryService['inspect']>;
resolveRecovery(
request: ApprovedActionManualRecoveryResolveRequest,
confirmAuthorization: () => void | Promise<void>,
): ReturnType<ApprovedActionManualRecoveryService['resolve']>;
recordFailure(record: SecurityAuditRecord): Promise<void>;
}
@@ -69,6 +84,12 @@ export function createClusterApprovalManagementService(
const approvals = new PostgresApprovalRequestRepository(options.pool);
const source = new PostgresApprovalRequestSource(options.pool);
const audit = new PostgresSecurityAuditRepository(options.pool);
const recovery = createApprovedActionManualRecoveryService({
repository: new PostgresApprovedActionManualRecoveryRepository(options.pool),
policy,
audit,
...(options.now === undefined ? {} : { now: options.now }),
});
return Object.freeze({
inspect(
request: ApprovalInspectionRequest,
@@ -93,6 +114,18 @@ export function createClusterApprovalManagementService(
...(options.now === undefined ? {} : { now: options.now }),
}).decide(request);
},
inspectRecovery(
request: ApprovedActionManualRecoveryInspectRequest,
confirmAuthorization: () => void | Promise<void>,
) {
return recovery.inspect(request, confirmAuthorization);
},
resolveRecovery(
request: ApprovedActionManualRecoveryResolveRequest,
confirmAuthorization: () => void | Promise<void>,
) {
return recovery.resolve(request, confirmAuthorization);
},
recordFailure(record: SecurityAuditRecord) {
return audit.record(record);
},
packages/ql3-cluster-admin/src/approval-management/approvalManagementClient.ts
+141 -2
View File
@@ -142,15 +142,139 @@ function decisionApproval(
safeTime(approval.decidedAtMs);
}
function recoveryResult(
value: unknown,
command: Extract<
ClusterApprovalManagementCommand,
{ operation: 'approval.recover.inspect' | 'approval.recover.resolve' }
>,
): void {
const recovery = exact(value, [
'projectId',
'dispatchId',
'approvalRequestId',
'expectedAction',
'execution',
'resolution',
]);
if (
identifier(recovery.projectId) !== command.request.projectId ||
identifier(recovery.dispatchId) !== command.request.dispatchId
) {
invalid();
}
identifier(recovery.approvalRequestId);
action(recovery.expectedAction);
const execution = exact(recovery.execution, [
'status',
'version',
'executionDigest',
'attemptCount',
'maxAttempts',
'startedAtMs',
'leaseExpiresAtMs',
'resultMutationId',
'resultCode',
'resultDigest',
'completedAtMs',
'createdAtMs',
'updatedAtMs',
]);
if (
![
'pending',
'leased',
'executing',
'recovery_required',
'retry_wait',
'succeeded',
'failed',
'blocked',
].includes(execution.status as string) ||
!Number.isSafeInteger(execution.version) ||
Number(execution.version) < 1 ||
typeof execution.executionDigest !== 'string' ||
!/^[0-9a-f]{64}$/.test(execution.executionDigest) ||
!Number.isSafeInteger(execution.attemptCount) ||
Number(execution.attemptCount) < 0 ||
!Number.isSafeInteger(execution.maxAttempts) ||
Number(execution.maxAttempts) < 1
) {
invalid();
}
safeTime(execution.startedAtMs, true);
safeTime(execution.leaseExpiresAtMs, true);
safeTime(execution.completedAtMs, true);
safeTime(execution.createdAtMs);
safeTime(execution.updatedAtMs);
for (const field of ['resultMutationId', 'resultCode']) {
const item = execution[field];
if (item !== null) identifier(item);
}
if (
execution.resultDigest !== null &&
(typeof execution.resultDigest !== 'string' ||
!/^[0-9a-f]{64}$/.test(execution.resultDigest))
) {
invalid();
}
if (recovery.resolution !== null) {
const resolution = exact(recovery.resolution, [
'mutationId',
'decision',
'evidenceDigest',
'reasonCode',
'resolvedBy',
'resolvedAtMs',
'resolutionDigest',
]);
identifier(resolution.mutationId);
if (
(resolution.decision !== 'confirm_failed' &&
resolution.decision !== 'abandon_unknown') ||
typeof resolution.evidenceDigest !== 'string' ||
!/^[0-9a-f]{64}$/.test(resolution.evidenceDigest) ||
typeof resolution.reasonCode !== 'string' ||
!/^[a-z][a-z0-9_]{0,63}$/.test(resolution.reasonCode) ||
typeof resolution.resolutionDigest !== 'string' ||
!/^[0-9a-f]{64}$/.test(resolution.resolutionDigest)
) {
invalid();
}
subject(resolution.resolvedBy);
safeTime(resolution.resolvedAtMs);
if (
command.operation === 'approval.recover.resolve' &&
(resolution.mutationId !== command.request.mutationId ||
resolution.decision !== command.request.decision ||
resolution.evidenceDigest !== command.request.evidenceDigest ||
resolution.reasonCode !== command.request.reasonCode)
) {
invalid();
}
}
if (
command.operation === 'approval.recover.resolve' &&
(recovery.resolution === null ||
Number(execution.version) !== command.request.expectedExecutionVersion + 1 ||
(execution.status !== 'failed' && execution.status !== 'blocked'))
) {
invalid();
}
}
export function validateClusterApprovalManagementClientResult(
value: unknown,
command: Readonly<ClusterApprovalManagementCommand>,
): Readonly<ClusterApprovalManagementTransportResult> {
const recoveryOperation =
command.operation === 'approval.recover.inspect' ||
command.operation === 'approval.recover.resolve';
const envelope = exact(value, [
'schemaVersion',
'operation',
'status',
'approval',
recoveryOperation ? 'recovery' : 'approval',
]);
if (
envelope.schemaVersion !== 1 ||
@@ -158,7 +282,22 @@ export function validateClusterApprovalManagementClientResult(
) {
invalid();
}
if (command.operation === 'approval.inspect') {
if (command.operation === 'approval.recover.inspect') {
if (
(envelope.status !== 'found' && envelope.status !== 'absent') ||
(envelope.status === 'absent') !== (envelope.recovery === null)
) {
invalid();
}
if (envelope.recovery !== null) {
recoveryResult(envelope.recovery, command);
}
} else if (command.operation === 'approval.recover.resolve') {
if (envelope.status !== 'resolved' && envelope.status !== 'existing') {
invalid();
}
recoveryResult(envelope.recovery, command);
} else if (command.operation === 'approval.inspect') {
if (
(envelope.status !== 'found' && envelope.status !== 'absent') ||
(envelope.status === 'absent') !== (envelope.approval === null)
packages/ql3-cluster-admin/src/approval-management/approvalManagementTransport.ts
+225 -25
View File
@@ -7,6 +7,15 @@ import {
normalizeApprovedActionBinding,
type ApprovedActionBinding,
} from '@qinglong/runtime-core/approved-action';
import {
ApprovedActionManualRecoveryAuthorizationError,
ApprovedActionManualRecoveryFenceConflictError,
ApprovedActionManualRecoveryTargetUnavailableError,
ApprovedActionManualRecoveryUnavailableError,
ApprovedActionManualRecoveryUnsupportedError,
type ApprovedActionManualRecoverySnapshot,
} from '@qinglong/runtime-core/approved-action-manual-recovery';
import { approvedActionExecutionEffectiveStatus } from '@qinglong/runtime-core/approved-action-execution';
import {
ApprovalDecisionAuthorizationError,
ApprovalDecisionBindingConflictError,
@@ -30,30 +39,54 @@ const UUID_V4_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
interface BaseRequest {
interface AuditBaseRequest {
readonly projectId: string;
readonly approvalRequestId: string;
readonly requestId: string;
readonly auditEventId: string;
readonly failureAuditEventId: string;
}
interface ApprovalBaseRequest extends AuditBaseRequest {
readonly approvalRequestId: string;
}
interface RecoveryBaseRequest extends AuditBaseRequest {
readonly dispatchId: string;
}
export type ClusterApprovalManagementCommand =
| Readonly<{
schemaVersion: 1;
operation: 'approval.inspect';
request: BaseRequest;
request: ApprovalBaseRequest;
}>
| Readonly<{
schemaVersion: 1;
operation: 'approval.decide';
request: BaseRequest & {
request: ApprovalBaseRequest & {
readonly expectedVersion: 1;
readonly expectedAction: Readonly<ApprovedActionBinding>;
readonly decisionId: string;
readonly decision: 'approved' | 'rejected';
readonly reasonCode: string;
};
}>
| Readonly<{
schemaVersion: 1;
operation: 'approval.recover.inspect';
request: RecoveryBaseRequest;
}>
| Readonly<{
schemaVersion: 1;
operation: 'approval.recover.resolve';
request: RecoveryBaseRequest & {
readonly expectedExecutionVersion: number;
readonly expectedExecutionDigest: string;
readonly mutationId: string;
readonly decision: 'confirm_failed' | 'abandon_unknown';
readonly evidenceDigest: string;
readonly reasonCode: string;
};
}>;
export type ClusterApprovalManagementTransportResult = Readonly<
@@ -163,45 +196,109 @@ export function normalizeClusterApprovalManagementCommand(
const envelope = exact(value, ['schemaVersion', 'operation', 'request']);
if (
envelope.schemaVersion !== 1 ||
(envelope.operation !== 'approval.inspect' &&
envelope.operation !== 'approval.decide')
![
'approval.inspect',
'approval.decide',
'approval.recover.inspect',
'approval.recover.resolve',
].includes(envelope.operation as string)
) {
invalid();
}
const operation = envelope.operation;
const base = [
const operation = envelope.operation as ClusterApprovalManagementCommand['operation'];
const auditBase = [
'projectId',
'approvalRequestId',
'requestId',
'auditEventId',
'failureAuditEventId',
];
const recoveryOperation =
operation === 'approval.recover.inspect' ||
operation === 'approval.recover.resolve';
const base = [
...auditBase,
recoveryOperation ? 'dispatchId' : 'approvalRequestId',
];
const request = exact(
envelope.request,
operation === 'approval.inspect'
operation === 'approval.inspect' || operation === 'approval.recover.inspect'
? base
: [
: operation === 'approval.decide'
? [
...base,
'expectedVersion',
'expectedAction',
'decisionId',
'decision',
'reasonCode',
]
: [
...base,
'expectedExecutionVersion',
'expectedExecutionDigest',
'mutationId',
'decision',
'evidenceDigest',
'reasonCode',
],
);
const normalizedBase = {
const normalizedAuditBase = {
projectId: identifier(request.projectId),
approvalRequestId: identifier(request.approvalRequestId),
requestId: identifier(request.requestId),
auditEventId: uuid(request.auditEventId),
failureAuditEventId: uuid(request.failureAuditEventId),
};
if (normalizedBase.auditEventId === normalizedBase.failureAuditEventId) invalid();
if (normalizedAuditBase.auditEventId === normalizedAuditBase.failureAuditEventId) {
invalid();
}
if (operation === 'approval.inspect') {
return Object.freeze({
schemaVersion: 1,
operation,
request: Object.freeze(normalizedBase),
operation: 'approval.inspect',
request: Object.freeze({
...normalizedAuditBase,
approvalRequestId: identifier(request.approvalRequestId),
}),
});
}
if (operation === 'approval.recover.inspect') {
return Object.freeze({
schemaVersion: 1,
operation: 'approval.recover.inspect',
request: Object.freeze({
...normalizedAuditBase,
dispatchId: identifier(request.dispatchId),
}),
});
}
if (operation === 'approval.recover.resolve') {
if (
!Number.isSafeInteger(request.expectedExecutionVersion) ||
Number(request.expectedExecutionVersion) < 1 ||
typeof request.expectedExecutionDigest !== 'string' ||
!/^[0-9a-f]{64}$/.test(request.expectedExecutionDigest) ||
(request.decision !== 'confirm_failed' &&
request.decision !== 'abandon_unknown') ||
typeof request.evidenceDigest !== 'string' ||
!/^[0-9a-f]{64}$/.test(request.evidenceDigest) ||
typeof request.reasonCode !== 'string' ||
!REASON_PATTERN.test(request.reasonCode)
) {
invalid();
}
return Object.freeze({
schemaVersion: 1,
operation: 'approval.recover.resolve',
request: Object.freeze({
...normalizedAuditBase,
dispatchId: identifier(request.dispatchId),
expectedExecutionVersion: Number(request.expectedExecutionVersion),
expectedExecutionDigest: request.expectedExecutionDigest,
mutationId: identifier(request.mutationId),
decision: request.decision,
evidenceDigest: request.evidenceDigest,
reasonCode: request.reasonCode,
}),
});
}
if (
@@ -222,9 +319,10 @@ export function normalizeClusterApprovalManagementCommand(
}
return Object.freeze({
schemaVersion: 1,
operation,
operation: 'approval.decide',
request: Object.freeze({
...normalizedBase,
...normalizedAuditBase,
approvalRequestId: identifier(request.approvalRequestId),
expectedVersion: 1,
expectedAction,
decisionId: identifier(request.decisionId),
@@ -284,11 +382,15 @@ function failureReason(error: unknown, authenticated: boolean): Readonly<{
}
if (
error instanceof ApprovalInspectionAuthorizationError ||
error instanceof ApprovalDecisionAuthorizationError
error instanceof ApprovalDecisionAuthorizationError ||
error instanceof ApprovedActionManualRecoveryAuthorizationError
) {
return Object.freeze({ outcome: 'denied', reason: 'policy_rejected' });
}
if (error instanceof ApprovalDecisionTargetUnavailableError) {
if (
error instanceof ApprovalDecisionTargetUnavailableError ||
error instanceof ApprovedActionManualRecoveryTargetUnavailableError
) {
return Object.freeze({ outcome: 'denied', reason: 'approval_target_unavailable' });
}
if (error instanceof ApprovalDecisionBindingConflictError) {
@@ -299,7 +401,9 @@ function failureReason(error: unknown, authenticated: boolean): Readonly<{
error instanceof ApprovalRequestStateConflictError ||
error instanceof ApprovalRequestExpiredError ||
error instanceof ApprovalMutationConflictError ||
error instanceof ApprovalPolicyFenceConflictError
error instanceof ApprovalPolicyFenceConflictError ||
error instanceof ApprovedActionManualRecoveryFenceConflictError ||
error instanceof ApprovedActionManualRecoveryUnsupportedError
) {
return Object.freeze({ outcome: 'denied', reason: 'approval_state_or_fence_conflict' });
}
@@ -320,11 +424,15 @@ function observedTime(now: () => number): number {
function mapped(error: unknown): Error {
if (
error instanceof ApprovalInspectionAuthorizationError ||
error instanceof ApprovalDecisionAuthorizationError
error instanceof ApprovalDecisionAuthorizationError ||
error instanceof ApprovedActionManualRecoveryAuthorizationError
) {
return new ClusterApprovalManagementTransportAuthorizationError();
}
if (error instanceof ApprovalDecisionTargetUnavailableError) {
if (
error instanceof ApprovalDecisionTargetUnavailableError ||
error instanceof ApprovedActionManualRecoveryTargetUnavailableError
) {
return new ClusterApprovalManagementTransportTargetUnavailableError();
}
if (
@@ -333,13 +441,16 @@ function mapped(error: unknown): Error {
error instanceof ApprovalRequestStateConflictError ||
error instanceof ApprovalRequestExpiredError ||
error instanceof ApprovalMutationConflictError ||
error instanceof ApprovalPolicyFenceConflictError
error instanceof ApprovalPolicyFenceConflictError ||
error instanceof ApprovedActionManualRecoveryFenceConflictError ||
error instanceof ApprovedActionManualRecoveryUnsupportedError
) {
return new ClusterApprovalManagementTransportConflictError();
}
if (
error instanceof ApprovalInspectionUnavailableError ||
error instanceof ApprovalDecisionUnavailableError
error instanceof ApprovalDecisionUnavailableError ||
error instanceof ApprovedActionManualRecoveryUnavailableError
) {
return new ClusterApprovalManagementTransportUnavailableError();
}
@@ -348,6 +459,47 @@ function mapped(error: unknown): Error {
: new ClusterApprovalManagementTransportUnavailableError();
}
function recoveryProjection(
snapshot: Readonly<ApprovedActionManualRecoverySnapshot>,
nowMs: number,
): Readonly<Record<string, unknown>> {
const execution = snapshot.execution.execution;
const resolution = snapshot.resolution;
return Object.freeze({
projectId: snapshot.execution.dispatch.projectId,
dispatchId: snapshot.execution.dispatch.id,
approvalRequestId: snapshot.execution.dispatch.approvalRequestId,
expectedAction: snapshot.execution.dispatch.action,
execution: Object.freeze({
status: approvedActionExecutionEffectiveStatus(execution, nowMs),
version: execution.version,
executionDigest: execution.executionDigest,
attemptCount: execution.attemptCount,
maxAttempts: execution.maxAttempts,
startedAtMs: execution.startedAtMs,
leaseExpiresAtMs: execution.leaseExpiresAtMs,
resultMutationId: execution.resultMutationId,
resultCode: execution.resultCode,
resultDigest: execution.resultDigest,
completedAtMs: execution.completedAtMs,
createdAtMs: execution.createdAtMs,
updatedAtMs: execution.updatedAtMs,
}),
resolution:
resolution === null
? null
: Object.freeze({
mutationId: resolution.mutationId,
decision: resolution.decision,
evidenceDigest: resolution.evidenceDigest,
reasonCode: resolution.reasonCode,
resolvedBy: resolution.resolvedBy,
resolvedAtMs: resolution.resolvedAtMs,
resolutionDigest: resolution.resolutionDigest,
}),
});
}
export function createClusterApprovalManagementTransport(options: Readonly<{
service: ClusterApprovalManagementService;
now?: () => number;
@@ -359,6 +511,8 @@ export function createClusterApprovalManagementTransport(options: Readonly<{
Object.keys(options).some((key) => key !== 'service' && key !== 'now') ||
typeof options.service?.inspect !== 'function' ||
typeof options.service?.decide !== 'function' ||
typeof options.service?.inspectRecovery !== 'function' ||
typeof options.service?.resolveRecovery !== 'function' ||
typeof options.service?.recordFailure !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function')
) {
@@ -407,6 +561,52 @@ export function createClusterApprovalManagementTransport(options: Readonly<{
throw new ClusterApprovalManagementTransportAuthenticationError();
}
};
if (command.operation === 'approval.recover.inspect') {
const snapshot = await options.service.inspectRecovery(
{
projectId: command.request.projectId,
dispatchId: command.request.dispatchId,
auditEventId: command.request.auditEventId,
requestId: command.request.requestId,
principal,
},
confirmAuthorization,
);
return Object.freeze({
schemaVersion: 1 as const,
operation: command.operation,
status: snapshot === null ? ('absent' as const) : ('found' as const),
recovery:
snapshot === null
? null
: recoveryProjection(snapshot, observedTime(now)),
});
}
if (command.operation === 'approval.recover.resolve') {
const result = await options.service.resolveRecovery(
{
projectId: command.request.projectId,
dispatchId: command.request.dispatchId,
expectedExecutionVersion:
command.request.expectedExecutionVersion,
expectedExecutionDigest: command.request.expectedExecutionDigest,
mutationId: command.request.mutationId,
decision: command.request.decision,
evidenceDigest: command.request.evidenceDigest,
reasonCode: command.request.reasonCode,
auditEventId: command.request.auditEventId,
requestId: command.request.requestId,
principal,
},
confirmAuthorization,
);
return Object.freeze({
schemaVersion: 1 as const,
operation: command.operation,
status: result.status,
recovery: recoveryProjection(result.snapshot, observedTime(now)),
});
}
if (command.operation === 'approval.inspect') {
const detail = await options.service.inspect(
{