From 3ca990105590c77c0ab97816994afece66dc8242 Mon Sep 17 00:00:00 2001 From: whyour Date: Fri, 14 Aug 2026 12:11:50 +0800 Subject: [PATCH] feat(ql3): resolve secret action recovery manually --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 1 + ...t-cluster-secret-action-manual-recovery.md | 45 + docs/adr/README.md | 2 + .../ql3-cluster-approval-management.md | 44 +- .../approval-management/approvalManagement.ts | 33 + .../approvalManagementClient.ts | 143 +++- .../approvalManagementTransport.ts | 250 +++++- .../test/approvalManagementClient.test.cjs | 118 +++ .../test/approvalManagementTransport.test.cjs | 173 ++++ .../ql3-cluster-admin/test/bootstrap.test.cjs | 1 + .../test/pluginPackageRecovery.test.cjs | 1 + .../test/application.test.cjs | 1 + .../test/bootstrap.test.cjs | 1 + .../src/approval-management/index.ts | 1 + .../approvedActionManualRecoveryRepository.ts | 269 ++++++ ...pg-0065-approved-action-manual-recovery.ts | 392 +++++++++ .../src/migration/migrationManifest.ts | 5 + .../src/migrations/index.ts | 2 + .../ql3-cluster-postgres/src/schema/schema.ts | 75 ++ .../src/schema/schemaContract.ts | 54 +- .../src/schema/schemaReadiness.ts | 21 +- ...dActionManualRecovery.integration.test.cjs | 273 ++++++ ...vedActionManualRecoveryRepository.test.cjs | 158 ++++ .../postgresqlMigrationDefinitions.test.cjs | 55 ++ .../test/postgresqlSchemaReadiness.test.cjs | 37 +- packages/ql3-runtime-core/package.json | 5 + .../approvedActionManualRecovery.ts | 775 ++++++++++++++++++ .../approvedActionManualRecovery.test.cjs | 250 ++++++ test/back/ql3PackageBoundaryAudit.test.cjs | 8 +- 29 files changed, 3143 insertions(+), 50 deletions(-) create mode 100644 docs/adr/ADR-0397-explicit-cluster-secret-action-manual-recovery.md create mode 100644 packages/ql3-cluster-postgres/src/approved-action/approvedActionManualRecoveryRepository.ts create mode 100644 packages/ql3-cluster-postgres/src/approved-action/pg-0065-approved-action-manual-recovery.ts create mode 100644 packages/ql3-cluster-postgres/test/approvedActionManualRecovery.integration.test.cjs create mode 100644 packages/ql3-cluster-postgres/test/approvedActionManualRecoveryRepository.test.cjs create mode 100644 packages/ql3-runtime-core/src/approved-action/approvedActionManualRecovery.ts create mode 100644 packages/ql3-runtime-core/test/approvedActionManualRecovery.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index fd1cacae..a597bf42 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -33,6 +33,7 @@ - 2026-08-14 收口更新（取代上一句关于 controller 尚未启用的描述）：生产 controller 已在既有 `cluster-admin/plugin-package/executor` 内接入 create/get-only Kubernetes Job adapter。它先按确定性名称 GET；仅当 durable execution 仍可创建且 approval 未过期时使用 Strict CREATE；CREATE 的 409 或响应丢失均回到 exact GET 收敛，`executing` 但 Job 缺失、过期 plan、终态 Job 与 renderer contract 漂移全部返回 `recoveryRequired`，不会盲目重建。Controller ServiceAccount 的 RBAC 仅允许 Job `create|get`，action ServiceAccount 无 API token；`admissionregistration.k8s.io/v1` ValidatingAdmissionPolicy 以固定参数 ConfigMap 和请求者身份锁死 digest 镜像、command、两个 ServiceAccount、source Secret、exact SHA-256 item/path、PostgreSQL SecretRef、安全上下文、资源额度和 volume/mount 形状，参数缺失与策略错误均失败关闭。基础 NetworkPolicy 继续只允许 DNS，集群 overlay 必须显式提供 API Server 的精确 CIDR/TCP 443 出口补丁。 - 真实 K3s `v1.34.3+k3s1` 已完成 admission 编译与现场门：合规 Job 的 server dry-run 通过，篡改镜像被策略拒绝，删除参数 ConfigMap 后创建被拒绝；controller SA 的 `list|watch|delete jobs`、Pod 创建和 Secret 读取均被拒绝，action SA 的 Job/Pod 创建与 Secret 读取也均被拒绝。实现仍保持 18 个 package，未新增 workspace package、Edge daemon/timer/watcher 或低配设备常驻负担；短生命周期 controller 与按需 action Job 仅属于 Cluster profile。完整 18-package clean build/test 退出 0；backend 1196 项为 1194 pass、2 条条件 skip、0 fail；cluster-admin 339 pass/3 skip、cluster-postgres 328 pass/2 skip，package boundary、cluster dependency、edge import 和 cluster deployment 审计均无 finding。PostgreSQL `18.4` arm64 physical HA 125 项、timeline `1→2` 通过，报告 SHA-256 为 `a3d34e61ea2064e1cde574e533137186e09fdce9048455da64f582906037fa0d`，临时 Docker 资源已清理。ADR 仍为 Proposed：升级失败自动回滚、终态 Job 的 durable 恢复决议和固定物理低配设备证据尚未完成。 - 2026-08-14 终态恢复更新：Secret Action controller 不再把所有终态 Job 仅计为瞬时 `recoveryRequired`。Job 到达 Complete/Failed 后已停止执行，controller 会用 started execution 的原 lease fence 复验不可变业务结果：首次 binding 必须与 approval plan、`startedAtMs` 推导出的 binding 完全一致；transition 必须与 plan、authority evidence、commit time 推导出的 receipt 完全一致。精确 durable result 存在时补写 `succeeded`，即使 Job 已被 TTL 清理也能收敛；Failed 且无 durable mutation 时写 `failed`；Complete 但无 receipt 时以 `indeterminate` 写 `blocked`。Job 在 start barrier 前终态或审批过期且尚未创建时，controller 复用既有 claim→release fence 写 `blocked`，不让坏 Job 永久占据 reconciler 页首。任何 stored result 漂移继续抛出 conflict，`executing + Job 缺失 + receipt 缺失` 继续要求人工处理，绝不自动重建可能已产生副作用的动作。该切片不修改共享 execution schema、PostgreSQL migration 或角色权限，不新增 package、连接与常驻进程；Cluster controller 复用现有 package-executor Pool，Edge 零变化。controller/process 定向 21/21，cluster-admin 全包 348 项为 345 pass/3 条件 skip/0 fail；完整 18-package 串行 build/test 退出 0；backend 1196 项为 1194 pass/2 条件 skip/0 fail；package boundary、cluster dependency、edge import、cluster deployment 均无 finding，部署/包边界聚焦测试 61/61。PostgreSQL `18.4` arm64 physical HA 125 项、timeline `1→2` 通过，报告 SHA-256 为 `bec512767fbbd7774baa9366698f60c25c8b017ed66f459b154d143fe86293bc`，临时 Docker 资源已清理。 + - 2026-08-14 人工恢复更新（ADR-0397，已接受）：上述唯一保留的 `executing + Job/receipt 均缺失` 不确定窗口现在具有显式 Cluster 产品处置路径。既有 Approval management mTLS/OIDC endpoint 新增 `approval.recover.inspect|resolve`，只接受五分钟内 `multi_factor|hardware` User、独立 `approval.recover` 权限、二次认证、exact execution version/digest 和外部 evidence SHA-256。只允许 Secret binding/transition action；`confirm_failed` 写 failed，`abandon_unknown` 写 blocked，永远禁止人工 succeeded、Job 重建或 execution 重置。PostgreSQL `pg-0065`/capability v64 新增不可变 resolution ledger 与单个 SECURITY DEFINER resolver，在同一事务内锁 Policy/execution fence、写 allowed audit、推进终态并写 receipt；Approval manager 只有 dispatch/execution/resolution SELECT 与函数 EXECUTE，没有 execution UPDATE。通用 execution repository 与 Worker Credential 调用链保持不变。真实 PostgreSQL 18.4 已从空库完成 65 migration，证明原子提交、exact replay 不重复审计和 direct UPDATE `42501`；实现不新增 package、依赖、Pod、Pool、daemon、timer、watcher 或 Edge/Standalone 负担。18-package clean build/test 退出 0，backend 1,194 pass/2 skip/0 fail，package/dependency/edge/deployment 审计零 finding；新 migration 与 repository 内聚到 `approved-action` 领域，migration ledger 直属源码保持审定上限 65。PostgreSQL 18.4 arm64 physical HA 125 项 gate、timeline `1→2` 通过，报告 SHA-256 为 `6d4921cba74475d15722a13c6a8034793c0ee25681bc7dcaf91024927c5752fe`，临时 Docker 资源已清理。 - D-302/ADR-0390（已接受） Cluster operator context 增加无网络、无 mutation 的内建 `ql3-cluster-admin context validate` 预检。它先复用 owner-private context reader，再让每个 entry 经过与真实请求相同的 production HTTPS/Kubernetes configuration preparation，验证精确 route、hostname、CA、 diff --git a/docs/adr/ADR-0397-explicit-cluster-secret-action-manual-recovery.md b/docs/adr/ADR-0397-explicit-cluster-secret-action-manual-recovery.md new file mode 100644 index 00000000..a1925111 --- /dev/null +++ b/docs/adr/ADR-0397-explicit-cluster-secret-action-manual-recovery.md @@ -0,0 +1,45 @@ +# ADR-0397：Cluster Secret Action 显式人工恢复 + +- 状态：Accepted（实现、真实 PostgreSQL 单节点门、完整 workspace/后端门、边界审计与 physical HA 门均完成） +- 日期：2026-08-14 +- 关联 RFC：QL-RFC-0001 D-306B2 +- 关联 ADR：ADR-0035、ADR-0359、ADR-0395、ADR-0396 + +## 问题 + +Secret Action controller 已能从终态 Kubernetes Job 与不可变 binding/transition receipt 自动恢复，也能在 Job Failed 且数据库无业务 mutation 时安全写入 failed。但 `approved_action_executions` 已进入 `executing`、Job 已不存在且 durable binding/receipt 也不存在时，系统无法证明外部效果从未发生。自动重建可能重复写 Secret；自动标记 succeeded 会伪造不存在的业务 receipt；长期保持 executing 又缺少可审计的产品处置入口。 + +通用 execution repository 同时服务 Worker Credential 等高影响调用链。为人工恢复放宽它的 UPDATE 权限或复用 package-executor，会扩大爆炸半径，也破坏 Approval manager 与执行器的职责分离。 + +## 决策 + +1. 只为 `plugin_package.secret_binding.bind` 与 `plugin_package.secret_binding.transition` 开放人工恢复。目标必须仍是 `executing`，原 lease 已过期，effective status 为 `recovery_required`；pending、live lease、已有可自动验证 receipt、其他 action type 和任何终态均拒绝。 +2. 人工决议只有 `confirm_failed` 与 `abandon_unknown`。前者表示外部证据已证明没有业务效果，execution 进入 `failed`；后者表示结果仍不可判定但操作者决定停止自动恢复，execution 进入 `blocked`。入口永远不能写 `succeeded`，因为人类陈述不能替代 immutable binding/transition receipt。 +3. 调用者必须提交 exact execution version、execution digest、唯一 mutation ID、稳定低敏 reason code 和外部证据 SHA-256。服务端重新读取 dispatch/execution 并绑定 action、dispatch、Project 与原 execution digest；相同事实 exact replay 返回 `existing`，任一字段漂移冲突。 +4. 复用 Cluster Approval management 的 mTLS/OIDC endpoint、短生命周期 client 与双重认证确认。只接受五分钟内的 `multi_factor|hardware` User，并独立请求 `approval.recover`；不复用 `approval.decide`、package.manage、ServiceAccount、Agent 或 System authority。 +5. PostgreSQL `pg-0065`/control capability v64 新增不可变 resolution ledger 和单个 `SECURITY DEFINER resolve_approved_action_manual_recovery(jsonb,jsonb,jsonb)`。函数在同一事务内锁定当前 Policy fence 与 exact executing row，插入 allowed security audit、推进 execution 终态并插入 resolution。任一步失败全部回滚。 +6. `ql3_approval_manager` 只新增 dispatch/execution/resolution 的 SELECT 与该函数的 EXECUTE；不取得 execution UPDATE，也不取得 resolution INSERT/UPDATE/DELETE。通用 `PostgresApprovedActionExecutionRepository` 保持不变，人工路径使用独立 repository，避免影响 Worker Credential execution flow。 +7. 返回只包含低敏 action binding、execution 状态/版本/摘要/时间和 resolution receipt。lease owner/token、authentication ID、assurance、认证时间与 Policy 内部原因不进入响应。 +8. 能力只属于 Cluster profile，复用现有 Approval manager Pool、Pod、listener 和 client。Edge/Standalone 不加载 PostgreSQL migration、repository 或新协议；不新增 workspace package、第三方依赖、daemon、timer、watcher、连接池或 Kubernetes workload。 + +## 接受条件 + +- Runtime Core 覆盖授权 inspect、两种终态、live lease/unsupported action/weak User/权限拒绝、围栏漂移和 exact replay。 +- transport/client 覆盖 exact command/result、二次认证、终态 version fence，并证明 lease 与认证事实不泄露。 +- PostgreSQL migration/readiness 明确证明 Approval manager 没有 execution UPDATE，只能执行受限函数。 +- 真实 PostgreSQL 从空库执行全部 migration，证明 resolution、execution 与 audit 原子提交，响应重放不重复审计，直接 UPDATE 返回 `42501`。 +- 完整 workspace build/test、package/dependency/edge/deployment 审计与 PostgreSQL physical HA 门通过后，ADR 才能转为 Accepted。 + +## 影响与替代方案 + +- 每次人工处置增加一行有界 resolution 和一行既有 security audit；正常执行路径没有额外查询、常驻内存或 cadence。 +- evidence digest 只证明操作者审查的外部材料，不把材料本身写入 QingLong；证据保存、访问控制与 retention 由部署者负责。 +- 不提供“重置为 pending”“重新创建 Job”或“人工确认 succeeded”。这些方案都可能复制或伪造外部副作用，拒绝。 +- 不给 Approval manager 通用 UPDATE。即使应用层能够校验，数据库 credential 仍是独立 authority boundary，必须由函数约束精确转换。 + +## 当前验证 + +- Runtime Core、Cluster PostgreSQL 与 Cluster Admin 包级测试已通过；新增领域、repository、transport/client 与 migration/readiness 测试均进入默认 test 集合。18 个 QL3 package clean build/test 退出 0；后端全门 1,194 pass、2 条件 skip、0 fail。 +- PostgreSQL 18.4 单节点真实门从空库完成 65 个 migration，验证 `resolved → existing`、resolution/audit 各一条，以及 Approval manager 直接 UPDATE 被 `42501` 拒绝。该门同时发现并修复 `audit_event_id` 外键与 JSON-to-UUID 写入的真实类型错误。 +- package boundary、cluster dependency、edge import 与 cluster deployment 四项审计全部 compatible 且零 finding；workspace 保持 18 package、无 single-source/shallow-source package。`pg-0065` 与 recovery repository 按 `approved-action` 领域内聚，既有有序 migration ledger 直属源码仍保持审定上限 65，没有以放宽阈值掩盖目录增长。 +- PostgreSQL 18.4 arm64 physical HA 125 项 gate 全绿，timeline `1→2`，报告 SHA-256 为 `6d4921cba74475d15722a13c6a8034793c0ee25681bc7dcaf91024927c5752fe`；临时 Docker 资源已清理。因此本 ADR 转为 Accepted。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 93c86bf1..beb72d72 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -399,6 +399,8 @@ | [ADR-0393](./ADR-0393-generation-bound-plugin-package-secret-binding-ledger.md) | 按 Generation 固定的 Plugin Package Secret 绑定账本 | Accepted | | [ADR-0394](./ADR-0394-generation-bound-plugin-package-secret-materialization.md) | 按 Generation 固定的 Plugin Package Secret Materialization | Accepted | | [ADR-0395](./ADR-0395-owner-confirmed-plugin-package-secret-binding.md) | Owner 确认的 Plugin Package Secret 首次绑定 | Proposed | +| [ADR-0396](./ADR-0396-generation-transition-plugin-package-secret-binding.md) | 按 Package Generation 切换 Plugin Package Secret Binding | Proposed | +| [ADR-0397](./ADR-0397-explicit-cluster-secret-action-manual-recovery.md) | Cluster Secret Action 显式人工恢复 | Accepted（实现、单节点 PostgreSQL、完整 workspace/后端/边界与 physical HA 门完成） | ## 规则 diff --git a/docs/operations/ql3-cluster-approval-management.md b/docs/operations/ql3-cluster-approval-management.md index cb937730..eb16e04a 100644 --- a/docs/operations/ql3-cluster-approval-management.md +++ b/docs/operations/ql3-cluster-approval-management.md @@ -6,7 +6,7 @@ PostgreSQL Pool。 ## 部署前置 -1. PostgreSQL 已完成 54 条 control-core migration、capability v53 和正常 readiness；CloudNativePG 已创建 +1. PostgreSQL 已完成 65 条 control-core migration、capability v64 和正常 readiness；CloudNativePG 已创建 `ql3_approval_manager` 与 `ql3-postgres-approval-manager-auth`。 2. 从 `deploy/kubernetes/ql3-cluster/operations/approval-management/config.example.yaml` @@ -95,6 +95,48 @@ assertion。服务会在进入领域服务前和提交前重新认证；身份 将 `decision` 改为 `rejected` 可拒绝。`reasonCode` 只允许稳定、低敏的 snake_case 分类，不写自由文本、Secret 或个人信息。 成功返回 `decided` 与 version 2；同语义精确重放返回 `existing`。 +## Secret Action 人工恢复 + +只有 controller 已报告 `executing + Job missing + durable binding/transition receipt missing` 时才使用本入口。先检查，不要从日志手工拼接 execution digest： + +```json +{ + "schemaVersion": 1, + "operation": "approval.recover.inspect", + "request": { + "projectId": "default", + "dispatchId": "dispatch-secret-action-1", + "requestId": "cluster-recovery-inspect-1", + "auditEventId": "30000000-0000-4000-8000-000000000001", + "failureAuditEventId": "30000000-0000-4000-8000-000000000002" + } +} +``` + +结果必须是受支持的 Secret binding/transition action，execution effective status 必须是 `recovery_required`，resolution 必须为空。完成外部取证后，用 inspect 返回的 exact version/digest 和证据文件的 SHA-256 创建新命令： + +```json +{ + "schemaVersion": 1, + "operation": "approval.recover.resolve", + "request": { + "projectId": "default", + "dispatchId": "dispatch-secret-action-1", + "requestId": "cluster-recovery-resolve-1", + "auditEventId": "40000000-0000-4000-8000-000000000001", + "failureAuditEventId": "40000000-0000-4000-8000-000000000002", + "expectedExecutionVersion": 3, + "expectedExecutionDigest": "<64-lowercase-hex-from-inspect>", + "mutationId": "manual-recovery-1", + "decision": "abandon_unknown", + "evidenceDigest": "", + "reasonCode": "orphan_absence_unverifiable" + } +} +``` + +`confirm_failed` 只用于外部证据明确证明业务 mutation 未发生，终态为 failed；仍无法判定时使用 `abandon_unknown`，终态为 blocked。不存在人工 succeeded，也不得借此重建 Job 或重置 execution。响应丢失时只能用相同 User、mutation、decision、evidence、reason 和 execution fence 精确重放，返回 `existing` 表示原事务已提交。 + ## 使用一次性 Kubernetes Client Job 复制 diff --git a/packages/ql3-cluster-admin/src/approval-management/approvalManagement.ts b/packages/ql3-cluster-admin/src/approval-management/approvalManagement.ts index c5938a9d..41e24277 100644 --- a/packages/ql3-cluster-admin/src/approval-management/approvalManagement.ts +++ b/packages/ql3-cluster-admin/src/approval-management/approvalManagement.ts @@ -1,11 +1,18 @@ import { PostgresApprovalManagementIdentityKeysetLedgerRepository, + PostgresApprovedActionManualRecoveryRepository, PostgresApprovalRequestRepository, PostgresApprovalRequestSource, PostgresProjectPolicyRepository, PostgresSecurityAuditRepository, } from '@qinglong/cluster-postgres/approval-manager'; import type { PostgresPool } from '@qinglong/runtime-core'; +import { + createApprovedActionManualRecoveryService, + type ApprovedActionManualRecoveryInspectRequest, + type ApprovedActionManualRecoveryResolveRequest, + type ApprovedActionManualRecoveryService, +} from '@qinglong/runtime-core/approved-action-manual-recovery'; import { createApprovalDecisionService, type ApprovalDecisionRequest, @@ -28,6 +35,14 @@ export interface ClusterApprovalManagementService { request: ApprovalDecisionRequest, confirmAuthorization: () => void | Promise, ): ReturnType; + inspectRecovery( + request: ApprovedActionManualRecoveryInspectRequest, + confirmAuthorization: () => void | Promise, + ): ReturnType; + resolveRecovery( + request: ApprovedActionManualRecoveryResolveRequest, + confirmAuthorization: () => void | Promise, + ): ReturnType; recordFailure(record: SecurityAuditRecord): Promise; } @@ -69,6 +84,12 @@ export function createClusterApprovalManagementService( const approvals = new PostgresApprovalRequestRepository(options.pool); const source = new PostgresApprovalRequestSource(options.pool); const audit = new PostgresSecurityAuditRepository(options.pool); + const recovery = createApprovedActionManualRecoveryService({ + repository: new PostgresApprovedActionManualRecoveryRepository(options.pool), + policy, + audit, + ...(options.now === undefined ? {} : { now: options.now }), + }); return Object.freeze({ inspect( request: ApprovalInspectionRequest, @@ -93,6 +114,18 @@ export function createClusterApprovalManagementService( ...(options.now === undefined ? {} : { now: options.now }), }).decide(request); }, + inspectRecovery( + request: ApprovedActionManualRecoveryInspectRequest, + confirmAuthorization: () => void | Promise, + ) { + return recovery.inspect(request, confirmAuthorization); + }, + resolveRecovery( + request: ApprovedActionManualRecoveryResolveRequest, + confirmAuthorization: () => void | Promise, + ) { + return recovery.resolve(request, confirmAuthorization); + }, recordFailure(record: SecurityAuditRecord) { return audit.record(record); }, diff --git a/packages/ql3-cluster-admin/src/approval-management/approvalManagementClient.ts b/packages/ql3-cluster-admin/src/approval-management/approvalManagementClient.ts index 42d072ab..0aeca399 100644 --- a/packages/ql3-cluster-admin/src/approval-management/approvalManagementClient.ts +++ b/packages/ql3-cluster-admin/src/approval-management/approvalManagementClient.ts @@ -142,15 +142,139 @@ function decisionApproval( safeTime(approval.decidedAtMs); } +function recoveryResult( + value: unknown, + command: Extract< + ClusterApprovalManagementCommand, + { operation: 'approval.recover.inspect' | 'approval.recover.resolve' } + >, +): void { + const recovery = exact(value, [ + 'projectId', + 'dispatchId', + 'approvalRequestId', + 'expectedAction', + 'execution', + 'resolution', + ]); + if ( + identifier(recovery.projectId) !== command.request.projectId || + identifier(recovery.dispatchId) !== command.request.dispatchId + ) { + invalid(); + } + identifier(recovery.approvalRequestId); + action(recovery.expectedAction); + const execution = exact(recovery.execution, [ + 'status', + 'version', + 'executionDigest', + 'attemptCount', + 'maxAttempts', + 'startedAtMs', + 'leaseExpiresAtMs', + 'resultMutationId', + 'resultCode', + 'resultDigest', + 'completedAtMs', + 'createdAtMs', + 'updatedAtMs', + ]); + if ( + ![ + 'pending', + 'leased', + 'executing', + 'recovery_required', + 'retry_wait', + 'succeeded', + 'failed', + 'blocked', + ].includes(execution.status as string) || + !Number.isSafeInteger(execution.version) || + Number(execution.version) < 1 || + typeof execution.executionDigest !== 'string' || + !/^[0-9a-f]{64}$/.test(execution.executionDigest) || + !Number.isSafeInteger(execution.attemptCount) || + Number(execution.attemptCount) < 0 || + !Number.isSafeInteger(execution.maxAttempts) || + Number(execution.maxAttempts) < 1 + ) { + invalid(); + } + safeTime(execution.startedAtMs, true); + safeTime(execution.leaseExpiresAtMs, true); + safeTime(execution.completedAtMs, true); + safeTime(execution.createdAtMs); + safeTime(execution.updatedAtMs); + for (const field of ['resultMutationId', 'resultCode']) { + const item = execution[field]; + if (item !== null) identifier(item); + } + if ( + execution.resultDigest !== null && + (typeof execution.resultDigest !== 'string' || + !/^[0-9a-f]{64}$/.test(execution.resultDigest)) + ) { + invalid(); + } + if (recovery.resolution !== null) { + const resolution = exact(recovery.resolution, [ + 'mutationId', + 'decision', + 'evidenceDigest', + 'reasonCode', + 'resolvedBy', + 'resolvedAtMs', + 'resolutionDigest', + ]); + identifier(resolution.mutationId); + if ( + (resolution.decision !== 'confirm_failed' && + resolution.decision !== 'abandon_unknown') || + typeof resolution.evidenceDigest !== 'string' || + !/^[0-9a-f]{64}$/.test(resolution.evidenceDigest) || + typeof resolution.reasonCode !== 'string' || + !/^[a-z][a-z0-9_]{0,63}$/.test(resolution.reasonCode) || + typeof resolution.resolutionDigest !== 'string' || + !/^[0-9a-f]{64}$/.test(resolution.resolutionDigest) + ) { + invalid(); + } + subject(resolution.resolvedBy); + safeTime(resolution.resolvedAtMs); + if ( + command.operation === 'approval.recover.resolve' && + (resolution.mutationId !== command.request.mutationId || + resolution.decision !== command.request.decision || + resolution.evidenceDigest !== command.request.evidenceDigest || + resolution.reasonCode !== command.request.reasonCode) + ) { + invalid(); + } + } + if ( + command.operation === 'approval.recover.resolve' && + (recovery.resolution === null || + Number(execution.version) !== command.request.expectedExecutionVersion + 1 || + (execution.status !== 'failed' && execution.status !== 'blocked')) + ) { + invalid(); + } +} + export function validateClusterApprovalManagementClientResult( value: unknown, command: Readonly, ): Readonly { + const recoveryOperation = + command.operation === 'approval.recover.inspect' || + command.operation === 'approval.recover.resolve'; const envelope = exact(value, [ 'schemaVersion', 'operation', 'status', - 'approval', + recoveryOperation ? 'recovery' : 'approval', ]); if ( envelope.schemaVersion !== 1 || @@ -158,7 +282,22 @@ export function validateClusterApprovalManagementClientResult( ) { invalid(); } - if (command.operation === 'approval.inspect') { + if (command.operation === 'approval.recover.inspect') { + if ( + (envelope.status !== 'found' && envelope.status !== 'absent') || + (envelope.status === 'absent') !== (envelope.recovery === null) + ) { + invalid(); + } + if (envelope.recovery !== null) { + recoveryResult(envelope.recovery, command); + } + } else if (command.operation === 'approval.recover.resolve') { + if (envelope.status !== 'resolved' && envelope.status !== 'existing') { + invalid(); + } + recoveryResult(envelope.recovery, command); + } else if (command.operation === 'approval.inspect') { if ( (envelope.status !== 'found' && envelope.status !== 'absent') || (envelope.status === 'absent') !== (envelope.approval === null) diff --git a/packages/ql3-cluster-admin/src/approval-management/approvalManagementTransport.ts b/packages/ql3-cluster-admin/src/approval-management/approvalManagementTransport.ts index 0d008080..0cbe156e 100644 --- a/packages/ql3-cluster-admin/src/approval-management/approvalManagementTransport.ts +++ b/packages/ql3-cluster-admin/src/approval-management/approvalManagementTransport.ts @@ -7,6 +7,15 @@ import { normalizeApprovedActionBinding, type ApprovedActionBinding, } from '@qinglong/runtime-core/approved-action'; +import { + ApprovedActionManualRecoveryAuthorizationError, + ApprovedActionManualRecoveryFenceConflictError, + ApprovedActionManualRecoveryTargetUnavailableError, + ApprovedActionManualRecoveryUnavailableError, + ApprovedActionManualRecoveryUnsupportedError, + type ApprovedActionManualRecoverySnapshot, +} from '@qinglong/runtime-core/approved-action-manual-recovery'; +import { approvedActionExecutionEffectiveStatus } from '@qinglong/runtime-core/approved-action-execution'; import { ApprovalDecisionAuthorizationError, ApprovalDecisionBindingConflictError, @@ -30,30 +39,54 @@ const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']); -interface BaseRequest { +interface AuditBaseRequest { readonly projectId: string; - readonly approvalRequestId: string; readonly requestId: string; readonly auditEventId: string; readonly failureAuditEventId: string; } +interface ApprovalBaseRequest extends AuditBaseRequest { + readonly approvalRequestId: string; +} + +interface RecoveryBaseRequest extends AuditBaseRequest { + readonly dispatchId: string; +} + export type ClusterApprovalManagementCommand = | Readonly<{ schemaVersion: 1; operation: 'approval.inspect'; - request: BaseRequest; + request: ApprovalBaseRequest; }> | Readonly<{ schemaVersion: 1; operation: 'approval.decide'; - request: BaseRequest & { + request: ApprovalBaseRequest & { readonly expectedVersion: 1; readonly expectedAction: Readonly; readonly decisionId: string; readonly decision: 'approved' | 'rejected'; readonly reasonCode: string; }; + }> + | Readonly<{ + schemaVersion: 1; + operation: 'approval.recover.inspect'; + request: RecoveryBaseRequest; + }> + | Readonly<{ + schemaVersion: 1; + operation: 'approval.recover.resolve'; + request: RecoveryBaseRequest & { + readonly expectedExecutionVersion: number; + readonly expectedExecutionDigest: string; + readonly mutationId: string; + readonly decision: 'confirm_failed' | 'abandon_unknown'; + readonly evidenceDigest: string; + readonly reasonCode: string; + }; }>; export type ClusterApprovalManagementTransportResult = Readonly< @@ -163,45 +196,109 @@ export function normalizeClusterApprovalManagementCommand( const envelope = exact(value, ['schemaVersion', 'operation', 'request']); if ( envelope.schemaVersion !== 1 || - (envelope.operation !== 'approval.inspect' && - envelope.operation !== 'approval.decide') + ![ + 'approval.inspect', + 'approval.decide', + 'approval.recover.inspect', + 'approval.recover.resolve', + ].includes(envelope.operation as string) ) { invalid(); } - const operation = envelope.operation; - const base = [ + const operation = envelope.operation as ClusterApprovalManagementCommand['operation']; + const auditBase = [ 'projectId', - 'approvalRequestId', 'requestId', 'auditEventId', 'failureAuditEventId', ]; + const recoveryOperation = + operation === 'approval.recover.inspect' || + operation === 'approval.recover.resolve'; + const base = [ + ...auditBase, + recoveryOperation ? 'dispatchId' : 'approvalRequestId', + ]; const request = exact( envelope.request, - operation === 'approval.inspect' + operation === 'approval.inspect' || operation === 'approval.recover.inspect' ? base - : [ + : operation === 'approval.decide' + ? [ ...base, 'expectedVersion', 'expectedAction', 'decisionId', 'decision', 'reasonCode', + ] + : [ + ...base, + 'expectedExecutionVersion', + 'expectedExecutionDigest', + 'mutationId', + 'decision', + 'evidenceDigest', + 'reasonCode', ], ); - const normalizedBase = { + const normalizedAuditBase = { projectId: identifier(request.projectId), - approvalRequestId: identifier(request.approvalRequestId), requestId: identifier(request.requestId), auditEventId: uuid(request.auditEventId), failureAuditEventId: uuid(request.failureAuditEventId), }; - if (normalizedBase.auditEventId === normalizedBase.failureAuditEventId) invalid(); + if (normalizedAuditBase.auditEventId === normalizedAuditBase.failureAuditEventId) { + invalid(); + } if (operation === 'approval.inspect') { return Object.freeze({ schemaVersion: 1, - operation, - request: Object.freeze(normalizedBase), + operation: 'approval.inspect', + request: Object.freeze({ + ...normalizedAuditBase, + approvalRequestId: identifier(request.approvalRequestId), + }), + }); + } + if (operation === 'approval.recover.inspect') { + return Object.freeze({ + schemaVersion: 1, + operation: 'approval.recover.inspect', + request: Object.freeze({ + ...normalizedAuditBase, + dispatchId: identifier(request.dispatchId), + }), + }); + } + if (operation === 'approval.recover.resolve') { + if ( + !Number.isSafeInteger(request.expectedExecutionVersion) || + Number(request.expectedExecutionVersion) < 1 || + typeof request.expectedExecutionDigest !== 'string' || + !/^[0-9a-f]{64}$/.test(request.expectedExecutionDigest) || + (request.decision !== 'confirm_failed' && + request.decision !== 'abandon_unknown') || + typeof request.evidenceDigest !== 'string' || + !/^[0-9a-f]{64}$/.test(request.evidenceDigest) || + typeof request.reasonCode !== 'string' || + !REASON_PATTERN.test(request.reasonCode) + ) { + invalid(); + } + return Object.freeze({ + schemaVersion: 1, + operation: 'approval.recover.resolve', + request: Object.freeze({ + ...normalizedAuditBase, + dispatchId: identifier(request.dispatchId), + expectedExecutionVersion: Number(request.expectedExecutionVersion), + expectedExecutionDigest: request.expectedExecutionDigest, + mutationId: identifier(request.mutationId), + decision: request.decision, + evidenceDigest: request.evidenceDigest, + reasonCode: request.reasonCode, + }), }); } if ( @@ -222,9 +319,10 @@ export function normalizeClusterApprovalManagementCommand( } return Object.freeze({ schemaVersion: 1, - operation, + operation: 'approval.decide', request: Object.freeze({ - ...normalizedBase, + ...normalizedAuditBase, + approvalRequestId: identifier(request.approvalRequestId), expectedVersion: 1, expectedAction, decisionId: identifier(request.decisionId), @@ -284,11 +382,15 @@ function failureReason(error: unknown, authenticated: boolean): Readonly<{ } if ( error instanceof ApprovalInspectionAuthorizationError || - error instanceof ApprovalDecisionAuthorizationError + error instanceof ApprovalDecisionAuthorizationError || + error instanceof ApprovedActionManualRecoveryAuthorizationError ) { return Object.freeze({ outcome: 'denied', reason: 'policy_rejected' }); } - if (error instanceof ApprovalDecisionTargetUnavailableError) { + if ( + error instanceof ApprovalDecisionTargetUnavailableError || + error instanceof ApprovedActionManualRecoveryTargetUnavailableError + ) { return Object.freeze({ outcome: 'denied', reason: 'approval_target_unavailable' }); } if (error instanceof ApprovalDecisionBindingConflictError) { @@ -299,7 +401,9 @@ function failureReason(error: unknown, authenticated: boolean): Readonly<{ error instanceof ApprovalRequestStateConflictError || error instanceof ApprovalRequestExpiredError || error instanceof ApprovalMutationConflictError || - error instanceof ApprovalPolicyFenceConflictError + error instanceof ApprovalPolicyFenceConflictError || + error instanceof ApprovedActionManualRecoveryFenceConflictError || + error instanceof ApprovedActionManualRecoveryUnsupportedError ) { return Object.freeze({ outcome: 'denied', reason: 'approval_state_or_fence_conflict' }); } @@ -320,11 +424,15 @@ function observedTime(now: () => number): number { function mapped(error: unknown): Error { if ( error instanceof ApprovalInspectionAuthorizationError || - error instanceof ApprovalDecisionAuthorizationError + error instanceof ApprovalDecisionAuthorizationError || + error instanceof ApprovedActionManualRecoveryAuthorizationError ) { return new ClusterApprovalManagementTransportAuthorizationError(); } - if (error instanceof ApprovalDecisionTargetUnavailableError) { + if ( + error instanceof ApprovalDecisionTargetUnavailableError || + error instanceof ApprovedActionManualRecoveryTargetUnavailableError + ) { return new ClusterApprovalManagementTransportTargetUnavailableError(); } if ( @@ -333,13 +441,16 @@ function mapped(error: unknown): Error { error instanceof ApprovalRequestStateConflictError || error instanceof ApprovalRequestExpiredError || error instanceof ApprovalMutationConflictError || - error instanceof ApprovalPolicyFenceConflictError + error instanceof ApprovalPolicyFenceConflictError || + error instanceof ApprovedActionManualRecoveryFenceConflictError || + error instanceof ApprovedActionManualRecoveryUnsupportedError ) { return new ClusterApprovalManagementTransportConflictError(); } if ( error instanceof ApprovalInspectionUnavailableError || - error instanceof ApprovalDecisionUnavailableError + error instanceof ApprovalDecisionUnavailableError || + error instanceof ApprovedActionManualRecoveryUnavailableError ) { return new ClusterApprovalManagementTransportUnavailableError(); } @@ -348,6 +459,47 @@ function mapped(error: unknown): Error { : new ClusterApprovalManagementTransportUnavailableError(); } +function recoveryProjection( + snapshot: Readonly, + nowMs: number, +): Readonly> { + const execution = snapshot.execution.execution; + const resolution = snapshot.resolution; + return Object.freeze({ + projectId: snapshot.execution.dispatch.projectId, + dispatchId: snapshot.execution.dispatch.id, + approvalRequestId: snapshot.execution.dispatch.approvalRequestId, + expectedAction: snapshot.execution.dispatch.action, + execution: Object.freeze({ + status: approvedActionExecutionEffectiveStatus(execution, nowMs), + version: execution.version, + executionDigest: execution.executionDigest, + attemptCount: execution.attemptCount, + maxAttempts: execution.maxAttempts, + startedAtMs: execution.startedAtMs, + leaseExpiresAtMs: execution.leaseExpiresAtMs, + resultMutationId: execution.resultMutationId, + resultCode: execution.resultCode, + resultDigest: execution.resultDigest, + completedAtMs: execution.completedAtMs, + createdAtMs: execution.createdAtMs, + updatedAtMs: execution.updatedAtMs, + }), + resolution: + resolution === null + ? null + : Object.freeze({ + mutationId: resolution.mutationId, + decision: resolution.decision, + evidenceDigest: resolution.evidenceDigest, + reasonCode: resolution.reasonCode, + resolvedBy: resolution.resolvedBy, + resolvedAtMs: resolution.resolvedAtMs, + resolutionDigest: resolution.resolutionDigest, + }), + }); +} + export function createClusterApprovalManagementTransport(options: Readonly<{ service: ClusterApprovalManagementService; now?: () => number; @@ -359,6 +511,8 @@ export function createClusterApprovalManagementTransport(options: Readonly<{ Object.keys(options).some((key) => key !== 'service' && key !== 'now') || typeof options.service?.inspect !== 'function' || typeof options.service?.decide !== 'function' || + typeof options.service?.inspectRecovery !== 'function' || + typeof options.service?.resolveRecovery !== 'function' || typeof options.service?.recordFailure !== 'function' || (options.now !== undefined && typeof options.now !== 'function') ) { @@ -407,6 +561,52 @@ export function createClusterApprovalManagementTransport(options: Readonly<{ throw new ClusterApprovalManagementTransportAuthenticationError(); } }; + if (command.operation === 'approval.recover.inspect') { + const snapshot = await options.service.inspectRecovery( + { + projectId: command.request.projectId, + dispatchId: command.request.dispatchId, + auditEventId: command.request.auditEventId, + requestId: command.request.requestId, + principal, + }, + confirmAuthorization, + ); + return Object.freeze({ + schemaVersion: 1 as const, + operation: command.operation, + status: snapshot === null ? ('absent' as const) : ('found' as const), + recovery: + snapshot === null + ? null + : recoveryProjection(snapshot, observedTime(now)), + }); + } + if (command.operation === 'approval.recover.resolve') { + const result = await options.service.resolveRecovery( + { + projectId: command.request.projectId, + dispatchId: command.request.dispatchId, + expectedExecutionVersion: + command.request.expectedExecutionVersion, + expectedExecutionDigest: command.request.expectedExecutionDigest, + mutationId: command.request.mutationId, + decision: command.request.decision, + evidenceDigest: command.request.evidenceDigest, + reasonCode: command.request.reasonCode, + auditEventId: command.request.auditEventId, + requestId: command.request.requestId, + principal, + }, + confirmAuthorization, + ); + return Object.freeze({ + schemaVersion: 1 as const, + operation: command.operation, + status: result.status, + recovery: recoveryProjection(result.snapshot, observedTime(now)), + }); + } if (command.operation === 'approval.inspect') { const detail = await options.service.inspect( { diff --git a/packages/ql3-cluster-admin/test/approvalManagementClient.test.cjs b/packages/ql3-cluster-admin/test/approvalManagementClient.test.cjs index 98c3bac4..cadac776 100644 --- a/packages/ql3-cluster-admin/test/approvalManagementClient.test.cjs +++ b/packages/ql3-cluster-admin/test/approvalManagementClient.test.cjs @@ -39,6 +39,13 @@ const BASE_REQUEST = Object.freeze({ auditEventId: '60000000-0000-4000-8000-000000000001', failureAuditEventId: '60000000-0000-4000-8000-000000000002', }); +const RECOVERY_BASE_REQUEST = Object.freeze({ + projectId: 'default', + dispatchId: 'dispatch-1', + requestId: 'recovery-command-1', + auditEventId: '60000000-0000-4000-8000-000000000003', + failureAuditEventId: '60000000-0000-4000-8000-000000000004', +}); function privateWrite(filePath, value) { writeFileSync(filePath, value, { mode: 0o600 }); @@ -204,6 +211,117 @@ test('validates the durable decision tuple and rejects server-side drift', () => } }); +test('validates recovery inspection and binds terminal resolution to the command fence', () => { + const recoveryAction = { + ...ACTION, + permission: 'secret.manage', + actionType: 'plugin_package.secret_binding.bind', + actionRef: 'secret-binding:1', + }; + const inspectCommand = { + schemaVersion: 1, + operation: 'approval.recover.inspect', + request: RECOVERY_BASE_REQUEST, + }; + const recovery = { + projectId: 'default', + dispatchId: 'dispatch-1', + approvalRequestId: 'approval-1', + expectedAction: recoveryAction, + execution: { + status: 'recovery_required', + version: 3, + executionDigest: 'c'.repeat(64), + attemptCount: 1, + maxAttempts: 3, + startedAtMs: 1_400, + leaseExpiresAtMs: 1_800, + resultMutationId: null, + resultCode: null, + resultDigest: null, + completedAtMs: null, + createdAtMs: 1_200, + updatedAtMs: 1_400, + }, + resolution: null, + }; + assert.deepEqual( + validateClusterApprovalManagementClientResult( + { + schemaVersion: 1, + operation: inspectCommand.operation, + status: 'found', + recovery, + }, + inspectCommand, + ).recovery, + recovery, + ); + + const resolveCommand = { + schemaVersion: 1, + operation: 'approval.recover.resolve', + request: { + ...RECOVERY_BASE_REQUEST, + expectedExecutionVersion: 3, + expectedExecutionDigest: 'c'.repeat(64), + mutationId: 'manual-recovery-1', + decision: 'abandon_unknown', + evidenceDigest: 'e'.repeat(64), + reasonCode: 'orphan_absence_verified', + }, + }; + const resolved = { + ...recovery, + execution: { + ...recovery.execution, + status: 'blocked', + version: 4, + executionDigest: 'd'.repeat(64), + leaseExpiresAtMs: null, + resultMutationId: 'manual-recovery-1', + resultCode: 'manual_recovery_abandoned_unknown', + completedAtMs: 2_000, + updatedAtMs: 2_000, + }, + resolution: { + mutationId: 'manual-recovery-1', + decision: 'abandon_unknown', + evidenceDigest: 'e'.repeat(64), + reasonCode: 'orphan_absence_verified', + resolvedBy: { type: 'user', id: 'owner-1' }, + resolvedAtMs: 2_000, + resolutionDigest: 'f'.repeat(64), + }, + }; + const result = { + schemaVersion: 1, + operation: resolveCommand.operation, + status: 'resolved', + recovery: resolved, + }; + assert.deepEqual( + validateClusterApprovalManagementClientResult(result, resolveCommand), + result, + ); + for (const changed of [ + { ...resolved, execution: { ...resolved.execution, version: 3 } }, + { + ...resolved, + resolution: { ...resolved.resolution, evidenceDigest: '0'.repeat(64) }, + }, + ]) { + assert.throws( + () => + validateClusterApprovalManagementClientResult( + { ...result, recovery: changed }, + resolveCommand, + ), + ClusterPluginPackageManagementClientRequestError, + ); + } +}); + test('accepts only the exact Approval route before opening one mTLS connection', async () => { let connects = 0; await assert.rejects( diff --git a/packages/ql3-cluster-admin/test/approvalManagementTransport.test.cjs b/packages/ql3-cluster-admin/test/approvalManagementTransport.test.cjs index e39b6a2b..3f0a4c6a 100644 --- a/packages/ql3-cluster-admin/test/approvalManagementTransport.test.cjs +++ b/packages/ql3-cluster-admin/test/approvalManagementTransport.test.cjs @@ -2,9 +2,16 @@ const assert = require('node:assert/strict'); const { test } = require('node:test'); const { + consumeApprovalRequest, createApprovalRequest, decideApprovalRequest, } = require('@qinglong/runtime-core/approved-action'); +const { + claimApprovedActionExecution, + completeApprovedActionExecution, + createApprovedActionExecution, + startApprovedActionExecution, +} = require('@qinglong/runtime-core/approved-action-execution'); const { ClusterApprovalManagementTransportAuthenticationError, ClusterApprovalManagementTransportRequestError, @@ -32,6 +39,13 @@ const BASE_REQUEST = Object.freeze({ auditEventId: '40000000-0000-4000-8000-000000000001', failureAuditEventId: '40000000-0000-4000-8000-000000000002', }); +const RECOVERY_BASE_REQUEST = Object.freeze({ + projectId: 'default', + dispatchId: 'dispatch-1', + requestId: 'recovery-command-1', + auditEventId: '40000000-0000-4000-8000-000000000003', + failureAuditEventId: '40000000-0000-4000-8000-000000000004', +}); function pending() { return createApprovalRequest({ @@ -70,6 +84,64 @@ function decideCommand() { }; } +function executingRecoverySnapshot() { + const action = { + ...ACTION, + permission: 'secret.manage', + actionType: 'plugin_package.secret_binding.bind', + actionRef: 'secret-binding:1', + }; + const recoveryPending = createApprovalRequest({ + id: 'approval-1', + projectId: 'default', + action, + risk: 'high', + decisionMode: 'human_confirmation', + requestedBy: { type: 'agent', id: 'agent-1' }, + requestedAtMs: 900, + expiresAtMs: 10_000, + requestFence: { projectVersion: 1, bindingVersion: 2 }, + }); + const approved = decideApprovalRequest(recoveryPending, { + expectedVersion: 1, + decisionId: 'decision-recovery-1', + decision: 'approved', + reasonCode: 'reviewed', + principal: PRINCIPAL, + decidedAtMs: 1_100, + authorizationFence: { projectVersion: 1, bindingVersion: 2 }, + }); + const dispatch = consumeApprovalRequest(approved, { + expectedVersion: 2, + consumptionId: 'consumption-1', + dispatchId: 'dispatch-1', + action, + requestedBy: approved.requestedBy, + consumedBy: { type: 'system', id: 'package-executor' }, + consumedAtMs: 1_200, + authorizationFence: { projectVersion: 1, bindingVersion: 2 }, + }).dispatch; + const leased = claimApprovedActionExecution(createApprovedActionExecution(dispatch), { + owner: 'executor-1', + leaseToken: 'lease-1', + nowMs: 1_300, + leaseDurationMs: 500, + }); + const execution = startApprovedActionExecution( + { dispatch, execution: leased }, + { + dispatchId: dispatch.id, + approvalRequestId: dispatch.approvalRequestId, + actionDigest: dispatch.action.actionDigest, + owner: leased.leaseOwner, + leaseToken: leased.leaseToken, + expectedVersion: leased.version, + startedAtMs: 1_400, + }, + ); + return { execution: { dispatch, execution }, resolution: null }; +} + test('inspects and decides through fresh strong authentication without leaking principal facts', async () => { const calls = []; const failures = []; @@ -104,6 +176,12 @@ test('inspects and decides through fresh strong authentication without leaking p }), }; }, + async inspectRecovery() { + throw new Error('not used'); + }, + async resolveRecovery() { + throw new Error('not used'); + }, async recordFailure(record) { failures.push(record); }, @@ -147,6 +225,12 @@ test('records unauthenticated and reauthentication failures with schema-valid id async decide() { throw new Error('not used'); }, + async inspectRecovery() { + throw new Error('not used'); + }, + async resolveRecovery() { + throw new Error('not used'); + }, async recordFailure(record) { failures.push(record); }, @@ -191,6 +275,8 @@ test('rejects widened or ambiguously audited commands before authentication', as service: { async inspect() {}, async decide() {}, + async inspectRecovery() {}, + async resolveRecovery() {}, async recordFailure() {}, }, }); @@ -229,3 +315,90 @@ test('rejects widened or ambiguously audited commands before authentication', as ); assert.equal(authenticationCalls, 0); }); + +test('inspects and resolves recovery without exposing execution lease or authentication facts', async () => { + const source = executingRecoverySnapshot(); + const nextExecution = completeApprovedActionExecution(source.execution.execution, { + owner: source.execution.execution.leaseOwner, + leaseToken: source.execution.execution.leaseToken, + expectedVersion: source.execution.execution.version, + resultMutationId: 'manual-recovery-1', + outcome: 'indeterminate', + resultCode: 'manual_recovery_abandoned_unknown', + completedAtMs: 2_000, + }); + const resolution = { + mutationId: 'manual-recovery-1', + decision: 'abandon_unknown', + evidenceDigest: 'e'.repeat(64), + reasonCode: 'orphan_absence_verified', + resolvedBy: { type: 'user', id: 'owner-1' }, + resolvedAtMs: 2_000, + resolutionDigest: 'f'.repeat(64), + }; + const transport = createClusterApprovalManagementTransport({ + service: { + async inspect() {}, + async decide() {}, + async inspectRecovery(_request, confirmAuthorization) { + await confirmAuthorization(); + return source; + }, + async resolveRecovery(_request, confirmAuthorization) { + await confirmAuthorization(); + return { + status: 'resolved', + snapshot: { + execution: { dispatch: source.execution.dispatch, execution: nextExecution }, + resolution: { + schema: 'qinglong/approved-action-manual-recovery@v1', + dispatchId: 'dispatch-1', + dispatchDigest: source.execution.execution.dispatchDigest, + projectId: 'default', + actionType: source.execution.dispatch.action.actionType, + actionDigest: source.execution.dispatch.action.actionDigest, + executionVersion: source.execution.execution.version, + executionDigest: source.execution.execution.executionDigest, + authenticationId: PRINCIPAL.authenticationId, + assurance: PRINCIPAL.assurance, + authenticatedAtMs: PRINCIPAL.authenticatedAtMs, + authorizationFence: { projectVersion: 1, bindingVersion: 2 }, + auditEventId: RECOVERY_BASE_REQUEST.auditEventId, + ...resolution, + }, + }, + }; + }, + async recordFailure() {}, + }, + now: () => 2_000, + }); + const authentication = { async authenticate() { return PRINCIPAL; } }; + const inspected = await transport.execute( + { schemaVersion: 1, operation: 'approval.recover.inspect', request: RECOVERY_BASE_REQUEST }, + authentication, + ); + const resolved = await transport.execute( + { + schemaVersion: 1, + operation: 'approval.recover.resolve', + request: { + ...RECOVERY_BASE_REQUEST, + expectedExecutionVersion: source.execution.execution.version, + expectedExecutionDigest: source.execution.execution.executionDigest, + mutationId: resolution.mutationId, + decision: resolution.decision, + evidenceDigest: resolution.evidenceDigest, + reasonCode: resolution.reasonCode, + }, + }, + authentication, + ); + assert.equal(inspected.recovery.execution.status, 'recovery_required'); + assert.equal(resolved.recovery.execution.status, 'blocked'); + assert.equal(resolved.recovery.resolution.decision, 'abandon_unknown'); + assert.doesNotMatch( + JSON.stringify([inspected, resolved]), + /leaseOwner|leaseToken|authenticationId|authenticatedAtMs|assurance/, + ); +}); diff --git a/packages/ql3-cluster-admin/test/bootstrap.test.cjs b/packages/ql3-cluster-admin/test/bootstrap.test.cjs index cdd075cb..0d4a7d79 100644 --- a/packages/ql3-cluster-admin/test/bootstrap.test.cjs +++ b/packages/ql3-cluster-admin/test/bootstrap.test.cjs @@ -188,6 +188,7 @@ function database(serverVersionNum = '160014') { 'create_plugin_package_secret_binding_approval_plan', 'create_plugin_package_secret_transition_plan', 'plugin_package_secret_binding_transition_snapshot', + 'resolve_approved_action_manual_recovery', ].includes(functionName), isOwner: false, })), diff --git a/packages/ql3-cluster-admin/test/pluginPackageRecovery.test.cjs b/packages/ql3-cluster-admin/test/pluginPackageRecovery.test.cjs index 8897e795..0f40e1d2 100644 --- a/packages/ql3-cluster-admin/test/pluginPackageRecovery.test.cjs +++ b/packages/ql3-cluster-admin/test/pluginPackageRecovery.test.cjs @@ -220,6 +220,7 @@ function database(serverVersionNum = '160014') { 'plugin_package_secret_binding_planning_snapshot', 'create_plugin_package_secret_transition_plan', 'plugin_package_secret_binding_transition_snapshot', + 'resolve_approved_action_manual_recovery', ].includes(functionName), isOwner: false, })), diff --git a/packages/ql3-cluster-control/test/application.test.cjs b/packages/ql3-cluster-control/test/application.test.cjs index 1aab27ca..6d39eac6 100644 --- a/packages/ql3-cluster-control/test/application.test.cjs +++ b/packages/ql3-cluster-control/test/application.test.cjs @@ -154,6 +154,7 @@ function runtimePrivileges() { approval_requests: [false, false, false, false], approved_action_dispatches: [false, false, false, false], approved_action_executions: [false, false, false, false], + approved_action_manual_recovery_resolutions: [false, false, false, false], plugin_package_install_proposals: [false, false, false, false], plugin_package_admission_receipts: [false, false, false, false], plugin_package_management_quota_buckets: [false, false, false, false], diff --git a/packages/ql3-cluster-control/test/bootstrap.test.cjs b/packages/ql3-cluster-control/test/bootstrap.test.cjs index 4c3a77a5..71ba6a32 100644 --- a/packages/ql3-cluster-control/test/bootstrap.test.cjs +++ b/packages/ql3-cluster-control/test/bootstrap.test.cjs @@ -68,6 +68,7 @@ function runtimePrivileges() { approval_requests: [false, false, false, false], approved_action_dispatches: [false, false, false, false], approved_action_executions: [false, false, false, false], + approved_action_manual_recovery_resolutions: [false, false, false, false], plugin_package_install_proposals: [false, false, false, false], plugin_package_admission_receipts: [false, false, false, false], plugin_package_management_quota_buckets: [false, false, false, false], diff --git a/packages/ql3-cluster-postgres/src/approval-management/index.ts b/packages/ql3-cluster-postgres/src/approval-management/index.ts index 47aec05b..4d3ee0dd 100644 --- a/packages/ql3-cluster-postgres/src/approval-management/index.ts +++ b/packages/ql3-cluster-postgres/src/approval-management/index.ts @@ -25,6 +25,7 @@ export { export { postgresqlMainMigrationManifest } from '../migration/migrationManifest'; export { PostgresApprovalRequestRepository } from '../approved-action/approvalRequestRepository'; export { PostgresApprovalRequestSource } from '../approved-action/approvalRequestSource'; +export { PostgresApprovedActionManualRecoveryRepository } from '../approved-action/approvedActionManualRecoveryRepository'; export { PostgresProjectPolicyRepository } from '../security/projectPolicyRepository'; export { PostgresSecurityAuditRepository } from '../security/securityAuditRepository'; export { diff --git a/packages/ql3-cluster-postgres/src/approved-action/approvedActionManualRecoveryRepository.ts b/packages/ql3-cluster-postgres/src/approved-action/approvedActionManualRecoveryRepository.ts new file mode 100644 index 00000000..a88d8419 --- /dev/null +++ b/packages/ql3-cluster-postgres/src/approved-action/approvedActionManualRecoveryRepository.ts @@ -0,0 +1,269 @@ +import type { PostgresPool } from '@qinglong/runtime-core'; +import { + normalizeApprovedActionDispatchRecord, + type ApprovedActionDispatchRecord, +} from '@qinglong/runtime-core/approved-action'; +import { + normalizeApprovedActionExecutionRecord, + normalizeApprovedActionExecutionSnapshot, + type ApprovedActionExecutionRecord, + type ApprovedActionExecutionSnapshot, +} from '@qinglong/runtime-core/approved-action-execution'; +import { + ApprovedActionManualRecoveryFenceConflictError, + ApprovedActionManualRecoveryTargetUnavailableError, + ApprovedActionManualRecoveryUnavailableError, + normalizeApprovedActionManualRecoveryResolution, + normalizeApprovedActionManualRecoverySnapshot, + type ApprovedActionManualRecoveryRepository, + type ApprovedActionManualRecoveryResolutionRecord, + type ApprovedActionManualRecoverySnapshot, + type ResolveApprovedActionManualRecoveryCommand, + type ResolveApprovedActionManualRecoveryResult, +} from '@qinglong/runtime-core/approved-action-manual-recovery'; +import { + normalizeSecurityAuditRecord, + type SecurityAuditRecord, +} from '@qinglong/runtime-core/security-audit'; + +import { + postgresRequiredJsonObject, + postgresRequiredString, + postgresSqlState, +} from '../repository/definitionRepositorySupport'; + +type Row = Record; + +const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; + +function unavailable(options?: ErrorOptions): ApprovedActionManualRecoveryUnavailableError { + return new ApprovedActionManualRecoveryUnavailableError(options); +} + +function same(left: unknown, right: unknown): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + +function parseDispatch(row: Row): Readonly { + try { + return normalizeApprovedActionDispatchRecord( + postgresRequiredJsonObject( + row.dispatchJson, + unavailable, + ) as unknown as ApprovedActionDispatchRecord, + ); + } catch (error) { + if (error instanceof ApprovedActionManualRecoveryUnavailableError) throw error; + throw unavailable(); + } +} +function parseExecution(row: Row): Readonly { + try { + const execution = normalizeApprovedActionExecutionRecord( + postgresRequiredJsonObject( + row.executionJson, + unavailable, + ) as unknown as ApprovedActionExecutionRecord, + ); + if ( + execution.executionDigest !== + postgresRequiredString(row.executionDigest, unavailable) + ) { + throw unavailable(); + } + return execution; + } catch (error) { + if (error instanceof ApprovedActionManualRecoveryUnavailableError) throw error; + throw unavailable(); + } +} + +function parseResolution( + value: unknown, +): Readonly | null { + if (value === null || value === undefined) return null; + try { + return normalizeApprovedActionManualRecoveryResolution( + postgresRequiredJsonObject( + value, + unavailable, + ) as unknown as ApprovedActionManualRecoveryResolutionRecord, + ); + } catch (error) { + if (error instanceof ApprovedActionManualRecoveryUnavailableError) throw error; + throw unavailable(); + } +} + +function parseSnapshot(row: Row): Readonly { + try { + return normalizeApprovedActionManualRecoverySnapshot({ + execution: normalizeApprovedActionExecutionSnapshot({ + dispatch: parseDispatch(row), + execution: parseExecution(row), + }), + resolution: parseResolution(row.resolutionJson), + }); + } catch (error) { + if (error instanceof ApprovedActionManualRecoveryUnavailableError) throw error; + throw unavailable(); + } +} + +function mapped(error: unknown): Error { + if ( + error instanceof ApprovedActionManualRecoveryFenceConflictError || + error instanceof ApprovedActionManualRecoveryTargetUnavailableError || + error instanceof ApprovedActionManualRecoveryUnavailableError || + (error instanceof Error && error.name.startsWith('InvalidApprovedAction')) + ) { + return error; + } + const state = postgresSqlState(error); + if ( + state === '23503' || + state === '23505' || + state === '23514' || + state === '40001' || + state === '40P01' + ) { + return new ApprovedActionManualRecoveryFenceConflictError(); + } + return unavailable({ cause: error instanceof Error ? error : undefined }); +} + +function auditMatchesResolution( + audit: Readonly, + resolution: Readonly, +): boolean { + return ( + audit.eventId === resolution.auditEventId && + audit.operationId === 'approval.recover.resolve' && + audit.projectId === resolution.projectId && + audit.subject?.type === resolution.resolvedBy.type && + audit.subject.id === resolution.resolvedBy.id && + audit.authenticationId === resolution.authenticationId && + audit.outcome === 'allowed' && + same(audit.reasons, [ + 'role_grant', + 'strong_authentication', + 'manual_recovery', + ]) && + same(audit.fence, resolution.authorizationFence) && + audit.occurredAtMs === resolution.resolvedAtMs + ); +} + +function normalizeCommand( + value: Readonly, +): Readonly { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + Object.keys(value).sort().join('\0') !== + ['previous', 'nextExecution', 'resolution', 'audit'].sort().join('\0') + ) { + throw new TypeError('Approved Action manual recovery command is invalid'); + } + const previous = normalizeApprovedActionExecutionSnapshot(value.previous); + const nextExecution = normalizeApprovedActionExecutionRecord(value.nextExecution); + const resolution = normalizeApprovedActionManualRecoveryResolution( + value.resolution, + ); + const audit = normalizeSecurityAuditRecord(value.audit); + if ( + previous.dispatch.id !== resolution.dispatchId || + previous.dispatch.projectId !== resolution.projectId || + previous.dispatch.action.actionType !== resolution.actionType || + previous.dispatch.action.actionDigest !== resolution.actionDigest || + previous.execution.version !== resolution.executionVersion || + previous.execution.executionDigest !== resolution.executionDigest || + nextExecution.dispatchId !== previous.dispatch.id || + nextExecution.version !== previous.execution.version + 1 || + nextExecution.resultMutationId !== resolution.mutationId || + nextExecution.completedAtMs !== resolution.resolvedAtMs || + !auditMatchesResolution(audit, resolution) + ) { + throw new ApprovedActionManualRecoveryFenceConflictError(); + } + return Object.freeze({ previous, nextExecution, resolution, audit }); +} + +export class PostgresApprovedActionManualRecoveryRepository + implements ApprovedActionManualRecoveryRepository +{ + constructor(private readonly pool: PostgresPool) { + if ( + !pool || + typeof pool.query !== 'function' || + typeof pool.connect !== 'function' + ) { + throw new TypeError( + 'PostgreSQL Approved Action manual recovery pool is invalid', + ); + } + } + + async findByDispatchId( + dispatchId: string, + ): Promise | null> { + if (typeof dispatchId !== 'string' || !IDENTIFIER_PATTERN.test(dispatchId)) { + throw new TypeError('Approved Action recovery dispatch id is invalid'); + } + try { + const result = await this.pool.query( + `SELECT dispatch.dispatch_json AS "dispatchJson", + execution.execution_json AS "executionJson", + execution.execution_digest AS "executionDigest", + resolution.resolution_json AS "resolutionJson" + FROM "ql3"."approved_action_executions" AS execution + JOIN "ql3"."approved_action_dispatches" AS dispatch + ON dispatch.dispatch_id = execution.dispatch_id + LEFT JOIN "ql3"."approved_action_manual_recovery_resolutions" AS resolution + ON resolution.dispatch_id = execution.dispatch_id + WHERE execution.dispatch_id = $1 + LIMIT 2`, + [dispatchId], + ); + if (result.rows.length === 0) return null; + if (result.rows.length !== 1) throw unavailable(); + return parseSnapshot(result.rows[0]!); + } catch (error) { + throw mapped(error); + } + } + + async resolve( + commandValue: Readonly, + ): Promise> { + const command = normalizeCommand(commandValue); + try { + const result = await this.pool.query( + `SELECT "ql3"."resolve_approved_action_manual_recovery"( + $1::jsonb, $2::jsonb, $3::jsonb + ) AS status`, + [ + JSON.stringify(command.resolution), + JSON.stringify(command.nextExecution), + JSON.stringify(command.audit), + ], + ); + if (result.rows.length !== 1) throw unavailable(); + const status = postgresRequiredString(result.rows[0]!.status, unavailable); + if (status !== 'resolved' && status !== 'existing') throw unavailable(); + const stored = await this.findByDispatchId(command.resolution.dispatchId); + if ( + !stored || + !stored.resolution || + !same(stored.resolution, command.resolution) || + !same(stored.execution.execution, command.nextExecution) + ) { + throw new ApprovedActionManualRecoveryFenceConflictError(); + } + return Object.freeze({ status, snapshot: stored }); + } catch (error) { + throw mapped(error); + } + } +} diff --git a/packages/ql3-cluster-postgres/src/approved-action/pg-0065-approved-action-manual-recovery.ts b/packages/ql3-cluster-postgres/src/approved-action/pg-0065-approved-action-manual-recovery.ts new file mode 100644 index 00000000..4e1f4fce --- /dev/null +++ b/packages/ql3-cluster-postgres/src/approved-action/pg-0065-approved-action-manual-recovery.ts @@ -0,0 +1,392 @@ +import { CAPABILITIES_V63 } from '../migrations/pg-0064-plugin-package-secret-binding-transition-approval-plans'; +import { definePostgresSqlMigration } from '../migrations/sqlMigration'; + +export const CAPABILITIES_V64 = CAPABILITIES_V63.replace( + '"approved_action_execution":1,', + '"approved_action_execution":1,"approved_action_manual_recovery":1,', +); + +export const pg0065ApprovedActionManualRecoveryMigration = + definePostgresSqlMigration({ + id: 'pg-0065-approved-action-manual-recovery', + statements: [ + ` +CREATE TABLE "ql3"."approved_action_manual_recovery_resolutions" ( + dispatch_id varchar(128) PRIMARY KEY, + dispatch_digest char(64) NOT NULL, + project_id varchar(128) NOT NULL, + action_type varchar(128) NOT NULL, + action_digest char(64) NOT NULL, + execution_version integer NOT NULL, + execution_digest char(64) NOT NULL, + mutation_id varchar(128) NOT NULL, + decision varchar(32) NOT NULL, + evidence_digest char(64) NOT NULL, + reason_code varchar(64) NOT NULL, + resolved_by_type varchar(16) NOT NULL, + resolved_by_id varchar(255) NOT NULL, + authentication_id varchar(128) NOT NULL, + assurance varchar(32) NOT NULL, + authenticated_at_ms bigint NOT NULL, + project_version integer NOT NULL, + binding_version integer NOT NULL, + audit_event_id uuid NOT NULL, + resolved_at_ms bigint NOT NULL, + resolution_json jsonb NOT NULL, + resolution_digest char(64) NOT NULL, + CONSTRAINT ql3_approved_action_manual_recovery_dispatch_fk + FOREIGN KEY (dispatch_id) + REFERENCES "ql3"."approved_action_dispatches" (dispatch_id) + ON DELETE RESTRICT ON UPDATE RESTRICT, + CONSTRAINT ql3_approved_action_manual_recovery_project_fk + FOREIGN KEY (project_id) REFERENCES "ql3"."projects" (id) + ON DELETE RESTRICT ON UPDATE RESTRICT, + CONSTRAINT ql3_approved_action_manual_recovery_audit_fk + FOREIGN KEY (audit_event_id) + REFERENCES "ql3"."security_audit_events" (event_id) + ON DELETE RESTRICT ON UPDATE RESTRICT, + CONSTRAINT ql3_approved_action_manual_recovery_identity_check CHECK ( + dispatch_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND + project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND + action_type IN ( + 'plugin_package.secret_binding.bind', + 'plugin_package.secret_binding.transition' + ) AND + execution_version BETWEEN 1 AND 2147483647 AND + mutation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND + decision IN ('confirm_failed', 'abandon_unknown') AND + reason_code ~ '^[a-z][a-z0-9_]{0,63}$' AND + resolved_by_type = 'user' AND + octet_length(resolved_by_id) BETWEEN 1 AND 255 AND + resolved_by_id !~ '[[:cntrl:]]' AND + authentication_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND + assurance IN ('multi_factor', 'hardware') AND + project_version >= 1 AND binding_version >= 1 + ), + CONSTRAINT ql3_approved_action_manual_recovery_digest_check CHECK ( + dispatch_digest ~ '^[0-9a-f]{64}$' AND + action_digest ~ '^[0-9a-f]{64}$' AND + execution_digest ~ '^[0-9a-f]{64}$' AND + evidence_digest ~ '^[0-9a-f]{64}$' AND + resolution_digest ~ '^[0-9a-f]{64}$' + ), + CONSTRAINT ql3_approved_action_manual_recovery_time_check CHECK ( + authenticated_at_ms >= 0 AND + resolved_at_ms >= authenticated_at_ms AND + resolved_at_ms - authenticated_at_ms <= 300000 + ), + CONSTRAINT ql3_approved_action_manual_recovery_json_check CHECK ( + jsonb_typeof(resolution_json) = 'object' AND + octet_length(resolution_json::text) BETWEEN 2 AND 65536 AND + resolution_json @> jsonb_build_object( + 'schema', 'qinglong/approved-action-manual-recovery@v1', + 'dispatchId', dispatch_id, + 'dispatchDigest', dispatch_digest, + 'projectId', project_id, + 'actionType', action_type, + 'actionDigest', action_digest, + 'executionVersion', execution_version, + 'executionDigest', execution_digest, + 'mutationId', mutation_id, + 'decision', decision, + 'evidenceDigest', evidence_digest, + 'reasonCode', reason_code, + 'resolvedBy', jsonb_build_object( + 'type', resolved_by_type, + 'id', resolved_by_id + ), + 'authenticationId', authentication_id, + 'assurance', assurance, + 'authenticatedAtMs', authenticated_at_ms, + 'authorizationFence', jsonb_build_object( + 'projectVersion', project_version, + 'bindingVersion', binding_version + ), + 'auditEventId', audit_event_id, + 'resolvedAtMs', resolved_at_ms, + 'resolutionDigest', resolution_digest + ) + ) +) + `.trim(), + `CREATE UNIQUE INDEX ql3_approved_action_manual_recovery_mutation_uidx ON "ql3"."approved_action_manual_recovery_resolutions" (mutation_id)`, + `CREATE UNIQUE INDEX ql3_approved_action_manual_recovery_digest_uidx ON "ql3"."approved_action_manual_recovery_resolutions" (resolution_digest)`, + `CREATE INDEX ql3_approved_action_manual_recovery_project_idx ON "ql3"."approved_action_manual_recovery_resolutions" (project_id, resolved_at_ms, dispatch_id)`, + ` +CREATE FUNCTION "ql3"."resolve_approved_action_manual_recovery"( + p_resolution_json jsonb, + p_next_execution_json jsonb, + p_audit_json jsonb +) +RETURNS varchar +LANGUAGE plpgsql +VOLATILE +SECURITY DEFINER +SET search_path = pg_catalog, ql3 +AS $ql3$ +DECLARE + current_execution_json jsonb; + current_execution_digest char(64); + current_version integer; + current_status varchar; + current_project_id varchar; + current_dispatch_digest char(64); + current_action_type varchar; + current_action_digest char(64); + current_lease_expires_at_ms bigint; + existing_resolution_json jsonb; + expected_status varchar; + expected_result_code varchar; +BEGIN + IF NOT pg_has_role(session_user, 'ql3_approval_manager', 'member') THEN + RAISE EXCEPTION 'Approval manager authority is required' + USING ERRCODE = 'insufficient_privilege'; + END IF; + + IF jsonb_typeof(p_resolution_json) <> 'object' + OR (SELECT count(*) FROM jsonb_object_keys(p_resolution_json)) <> 20 + OR jsonb_typeof(p_next_execution_json) <> 'object' + OR (SELECT count(*) FROM jsonb_object_keys(p_next_execution_json)) <> 21 + OR jsonb_typeof(p_audit_json) <> 'object' + OR (SELECT count(*) FROM jsonb_object_keys(p_audit_json)) <> 10 + THEN + RAISE EXCEPTION 'Approved Action manual recovery input is malformed' + USING ERRCODE = 'check_violation'; + END IF; + + SELECT resolution_json + INTO existing_resolution_json + FROM "ql3"."approved_action_manual_recovery_resolutions" + WHERE dispatch_id = p_resolution_json ->> 'dispatchId'; + IF FOUND THEN + SELECT execution_json + INTO current_execution_json + FROM "ql3"."approved_action_executions" + WHERE dispatch_id = p_resolution_json ->> 'dispatchId'; + IF existing_resolution_json = p_resolution_json + AND current_execution_json = p_next_execution_json + AND EXISTS ( + SELECT 1 + FROM "ql3"."security_audit_events" AS audit + WHERE audit.event_id = (p_audit_json ->> 'eventId')::uuid + AND audit.request_id = p_audit_json ->> 'requestId' + AND audit.operation_id = 'approval.recover.resolve' + AND audit.project_id = p_resolution_json ->> 'projectId' + AND audit.subject_type = 'user' + AND audit.subject_id = p_resolution_json #>> '{resolvedBy,id}' + AND audit.authentication_id = p_resolution_json ->> 'authenticationId' + AND audit.outcome = 'allowed' + AND audit.reasons = p_audit_json -> 'reasons' + AND audit.project_version = (p_resolution_json #>> '{authorizationFence,projectVersion}')::integer + AND audit.binding_version = (p_resolution_json #>> '{authorizationFence,bindingVersion}')::integer + AND audit.occurred_at_ms = (p_resolution_json ->> 'resolvedAtMs')::bigint + ) + THEN + RETURN 'existing'; + END IF; + RAISE EXCEPTION 'Approved Action manual recovery replay conflicts' + USING ERRCODE = 'check_violation'; + END IF; + + IF NOT "ql3"."lock_approval_policy_fence"( + p_resolution_json ->> 'projectId', + p_resolution_json #>> '{resolvedBy,type}', + p_resolution_json #>> '{resolvedBy,id}', + (p_resolution_json #>> '{authorizationFence,projectVersion}')::integer, + (p_resolution_json #>> '{authorizationFence,bindingVersion}')::integer + ) THEN + RAISE EXCEPTION 'Approved Action manual recovery policy fence changed' + USING ERRCODE = 'check_violation'; + END IF; + + SELECT execution.execution_json, + execution.execution_digest, + execution.version, + execution.status, + execution.project_id, + execution.dispatch_digest, + dispatch.action_type, + dispatch.action_digest, + execution.lease_expires_at_ms + INTO current_execution_json, + current_execution_digest, + current_version, + current_status, + current_project_id, + current_dispatch_digest, + current_action_type, + current_action_digest, + current_lease_expires_at_ms + FROM "ql3"."approved_action_executions" AS execution + JOIN "ql3"."approved_action_dispatches" AS dispatch + ON dispatch.dispatch_id = execution.dispatch_id + WHERE execution.dispatch_id = p_resolution_json ->> 'dispatchId' + FOR UPDATE OF execution; + + IF NOT FOUND + OR current_status <> 'executing' + OR current_lease_expires_at_ms IS NULL + OR current_lease_expires_at_ms > (p_resolution_json ->> 'resolvedAtMs')::bigint + OR current_project_id <> p_resolution_json ->> 'projectId' + OR current_dispatch_digest <> p_resolution_json ->> 'dispatchDigest' + OR current_action_type <> p_resolution_json ->> 'actionType' + OR current_action_type NOT IN ( + 'plugin_package.secret_binding.bind', + 'plugin_package.secret_binding.transition' + ) + OR current_action_digest <> p_resolution_json ->> 'actionDigest' + OR current_version <> (p_resolution_json ->> 'executionVersion')::integer + OR current_execution_digest <> p_resolution_json ->> 'executionDigest' + THEN + RAISE EXCEPTION 'Approved Action manual recovery execution fence changed' + USING ERRCODE = 'check_violation'; + END IF; + + IF p_resolution_json ->> 'decision' = 'confirm_failed' THEN + expected_status := 'failed'; + expected_result_code := 'manual_recovery_confirmed_failed'; + ELSIF p_resolution_json ->> 'decision' = 'abandon_unknown' THEN + expected_status := 'blocked'; + expected_result_code := 'manual_recovery_abandoned_unknown'; + ELSE + RAISE EXCEPTION 'Approved Action manual recovery decision is invalid' + USING ERRCODE = 'check_violation'; + END IF; + + IF NOT p_next_execution_json ?& ARRAY[ + 'schema', 'dispatchId', 'dispatchDigest', 'projectId', 'status', + 'version', 'attemptCount', 'maxAttempts', 'eligibleAtMs', + 'nextAttemptAtMs', 'leaseOwner', 'leaseToken', 'leaseExpiresAtMs', + 'startedAtMs', 'resultMutationId', 'resultCode', 'resultDigest', + 'completedAtMs', 'createdAtMs', 'updatedAtMs', 'executionDigest' + ]::text[] + OR p_next_execution_json - ARRAY[ + 'status', 'version', 'eligibleAtMs', 'nextAttemptAtMs', + 'leaseOwner', 'leaseToken', 'leaseExpiresAtMs', 'resultMutationId', + 'resultCode', 'resultDigest', 'completedAtMs', 'updatedAtMs', + 'executionDigest' + ]::text[] + <> current_execution_json - ARRAY[ + 'status', 'version', 'eligibleAtMs', 'nextAttemptAtMs', + 'leaseOwner', 'leaseToken', 'leaseExpiresAtMs', 'resultMutationId', + 'resultCode', 'resultDigest', 'completedAtMs', 'updatedAtMs', + 'executionDigest' + ]::text[] + OR p_next_execution_json ->> 'status' <> expected_status + OR (p_next_execution_json ->> 'version')::integer <> current_version + 1 + OR p_next_execution_json -> 'eligibleAtMs' <> 'null'::jsonb + OR p_next_execution_json -> 'nextAttemptAtMs' <> 'null'::jsonb + OR p_next_execution_json -> 'leaseOwner' <> 'null'::jsonb + OR p_next_execution_json -> 'leaseToken' <> 'null'::jsonb + OR p_next_execution_json -> 'leaseExpiresAtMs' <> 'null'::jsonb + OR p_next_execution_json ->> 'resultMutationId' <> p_resolution_json ->> 'mutationId' + OR p_next_execution_json ->> 'resultCode' <> expected_result_code + OR p_next_execution_json -> 'resultDigest' <> 'null'::jsonb + OR (p_next_execution_json ->> 'completedAtMs')::bigint <> (p_resolution_json ->> 'resolvedAtMs')::bigint + OR (p_next_execution_json ->> 'updatedAtMs')::bigint <> (p_resolution_json ->> 'resolvedAtMs')::bigint + OR p_next_execution_json ->> 'executionDigest' !~ '^[0-9a-f]{64}$' + THEN + RAISE EXCEPTION 'Approved Action manual recovery terminal execution is invalid' + USING ERRCODE = 'check_violation'; + END IF; + + IF p_audit_json ->> 'eventId' <> p_resolution_json ->> 'auditEventId' + OR p_audit_json ->> 'operationId' <> 'approval.recover.resolve' + OR p_audit_json ->> 'projectId' <> p_resolution_json ->> 'projectId' + OR p_audit_json #>> '{subject,type}' <> 'user' + OR p_audit_json #>> '{subject,id}' <> p_resolution_json #>> '{resolvedBy,id}' + OR p_audit_json ->> 'authenticationId' <> p_resolution_json ->> 'authenticationId' + OR p_audit_json ->> 'outcome' <> 'allowed' + OR p_audit_json -> 'reasons' <> '["role_grant","strong_authentication","manual_recovery"]'::jsonb + OR p_audit_json -> 'fence' <> p_resolution_json -> 'authorizationFence' + OR (p_audit_json ->> 'occurredAtMs')::bigint <> (p_resolution_json ->> 'resolvedAtMs')::bigint + THEN + RAISE EXCEPTION 'Approved Action manual recovery audit is invalid' + USING ERRCODE = 'check_violation'; + END IF; + + INSERT INTO "ql3"."security_audit_events" ( + event_id, request_id, operation_id, project_id, subject_type, subject_id, + authentication_id, outcome, reasons, project_version, binding_version, + occurred_at_ms + ) VALUES ( + (p_audit_json ->> 'eventId')::uuid, + p_audit_json ->> 'requestId', + p_audit_json ->> 'operationId', + p_audit_json ->> 'projectId', + p_audit_json #>> '{subject,type}', + p_audit_json #>> '{subject,id}', + p_audit_json ->> 'authenticationId', + p_audit_json ->> 'outcome', + p_audit_json -> 'reasons', + (p_audit_json #>> '{fence,projectVersion}')::integer, + (p_audit_json #>> '{fence,bindingVersion}')::integer, + (p_audit_json ->> 'occurredAtMs')::bigint + ); + + UPDATE "ql3"."approved_action_executions" + SET status = expected_status, + version = (p_next_execution_json ->> 'version')::integer, + eligible_at_ms = NULL, + next_attempt_at_ms = NULL, + lease_owner = NULL, + lease_token = NULL, + lease_expires_at_ms = NULL, + result_mutation_id = p_next_execution_json ->> 'resultMutationId', + result_code = expected_result_code, + result_digest = NULL, + completed_at_ms = (p_next_execution_json ->> 'completedAtMs')::bigint, + updated_at_ms = (p_next_execution_json ->> 'updatedAtMs')::bigint, + execution_json = p_next_execution_json, + execution_digest = p_next_execution_json ->> 'executionDigest' + WHERE dispatch_id = p_resolution_json ->> 'dispatchId' + AND version = current_version + AND execution_digest = current_execution_digest; + IF NOT FOUND THEN + RAISE EXCEPTION 'Approved Action manual recovery update fence changed' + USING ERRCODE = 'check_violation'; + END IF; + + INSERT INTO "ql3"."approved_action_manual_recovery_resolutions" ( + dispatch_id, dispatch_digest, project_id, action_type, action_digest, + execution_version, execution_digest, mutation_id, decision, + evidence_digest, reason_code, resolved_by_type, resolved_by_id, + authentication_id, assurance, authenticated_at_ms, project_version, + binding_version, audit_event_id, resolved_at_ms, resolution_json, + resolution_digest + ) VALUES ( + p_resolution_json ->> 'dispatchId', + p_resolution_json ->> 'dispatchDigest', + p_resolution_json ->> 'projectId', + p_resolution_json ->> 'actionType', + p_resolution_json ->> 'actionDigest', + (p_resolution_json ->> 'executionVersion')::integer, + p_resolution_json ->> 'executionDigest', + p_resolution_json ->> 'mutationId', + p_resolution_json ->> 'decision', + p_resolution_json ->> 'evidenceDigest', + p_resolution_json ->> 'reasonCode', + p_resolution_json #>> '{resolvedBy,type}', + p_resolution_json #>> '{resolvedBy,id}', + p_resolution_json ->> 'authenticationId', + p_resolution_json ->> 'assurance', + (p_resolution_json ->> 'authenticatedAtMs')::bigint, + (p_resolution_json #>> '{authorizationFence,projectVersion}')::integer, + (p_resolution_json #>> '{authorizationFence,bindingVersion}')::integer, + (p_resolution_json ->> 'auditEventId')::uuid, + (p_resolution_json ->> 'resolvedAtMs')::bigint, + p_resolution_json, + p_resolution_json ->> 'resolutionDigest' + ); + + RETURN 'resolved'; +END +$ql3$ + `.trim(), + `REVOKE ALL ON "ql3"."approved_action_manual_recovery_resolutions" FROM PUBLIC, ql3_runtime, ql3_admin, ql3_package_manager, ql3_package_executor, ql3_worker_ingress, ql3_worker_credential_manager, ql3_worker_credential_executor, ql3_automation_manager, ql3_approval_manager, ql3_run_manager`, + `GRANT SELECT ON "ql3"."approved_action_dispatches", "ql3"."approved_action_executions", "ql3"."approved_action_manual_recovery_resolutions" TO ql3_approval_manager`, + `REVOKE ALL ON FUNCTION "ql3"."resolve_approved_action_manual_recovery"(jsonb, jsonb, jsonb) FROM PUBLIC, ql3_runtime, ql3_admin, ql3_package_manager, ql3_package_executor, ql3_worker_ingress, ql3_worker_credential_manager, ql3_worker_credential_executor, ql3_automation_manager, ql3_approval_manager, ql3_run_manager`, + `GRANT EXECUTE ON FUNCTION "ql3"."resolve_approved_action_manual_recovery"(jsonb, jsonb, jsonb) TO ql3_approval_manager`, + `DO $ql3$ BEGIN UPDATE "ql3"."schema_capabilities" SET contract_version = 64, migration_id = 'pg-0065-approved-action-manual-recovery', capabilities = '${CAPABILITIES_V64}'::jsonb, updated_at_ms = floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint WHERE contract_name = 'control-core' AND contract_version = 63 AND migration_id = 'pg-0064-plugin-package-secret-binding-transition-approval-plans' AND capabilities = '${CAPABILITIES_V63}'::jsonb; IF NOT FOUND THEN RAISE EXCEPTION 'control-core capability is not at version 63' USING ERRCODE = 'check_violation'; END IF; END $ql3$`, + ], + }); diff --git a/packages/ql3-cluster-postgres/src/migration/migrationManifest.ts b/packages/ql3-cluster-postgres/src/migration/migrationManifest.ts index 54e09a5d..30295823 100644 --- a/packages/ql3-cluster-postgres/src/migration/migrationManifest.ts +++ b/packages/ql3-cluster-postgres/src/migration/migrationManifest.ts @@ -328,5 +328,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest = checksum: '1951b77a0265f8826169e4724424b2fbbd30061b27e27d3ba95de03430c1bac9', }), + Object.freeze({ + id: 'pg-0065-approved-action-manual-recovery', + checksum: + '95387c5b40659490dbcb7626ecd15bacf6412360752bef88873bde57c43e0185', + }), ]), }); diff --git a/packages/ql3-cluster-postgres/src/migrations/index.ts b/packages/ql3-cluster-postgres/src/migrations/index.ts index 43168932..1de87840 100644 --- a/packages/ql3-cluster-postgres/src/migrations/index.ts +++ b/packages/ql3-cluster-postgres/src/migrations/index.ts @@ -67,6 +67,7 @@ import { pg0061PluginPackageSecretBindingApprovalPlansMigration } from './pg-006 import { pg0062PluginPackageSecretBindingTargetGuardMigration } from './pg-0062-plugin-package-secret-binding-target-guard'; import { pg0063PluginPackageSecretBindingTransitionReceiptsMigration } from './pg-0063-plugin-package-secret-binding-transition-receipts'; import { pg0064PluginPackageSecretBindingTransitionApprovalPlansMigration } from './pg-0064-plugin-package-secret-binding-transition-approval-plans'; +import { pg0065ApprovedActionManualRecoveryMigration } from '../approved-action/pg-0065-approved-action-manual-recovery'; export const postgresqlMainMigrationStream: MigrationStreamDefinition = Object.freeze({ @@ -139,5 +140,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition>() + .notNull(), + resolutionDigest: char('resolution_digest', { length: 64 }).notNull(), + }, + (table) => [ + foreignKey({ + name: 'ql3_approved_action_manual_recovery_dispatch_fk', + columns: [table.dispatchId], + foreignColumns: [approvedActionDispatches.dispatchId], + }).onDelete('restrict'), + foreignKey({ + name: 'ql3_approved_action_manual_recovery_project_fk', + columns: [table.projectId], + foreignColumns: [projects.id], + }).onDelete('restrict'), + foreignKey({ + name: 'ql3_approved_action_manual_recovery_audit_fk', + columns: [table.auditEventId], + foreignColumns: [securityAuditEvents.eventId], + }).onDelete('restrict'), + check( + 'ql3_approved_action_manual_recovery_identity_check', + sql`${table.dispatchId} ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' and ${table.projectId} ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' and ${table.actionType} in ('plugin_package.secret_binding.bind','plugin_package.secret_binding.transition') and ${table.executionVersion} between 1 and 2147483647 and ${table.mutationId} ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' and ${table.decision} in ('confirm_failed','abandon_unknown') and ${table.reasonCode} ~ '^[a-z][a-z0-9_]{0,63}$' and ${table.resolvedByType} = 'user' and octet_length(${table.resolvedById}) between 1 and 255 and ${table.resolvedById} !~ '[[:cntrl:]]' and ${table.authenticationId} ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' and ${table.assurance} in ('multi_factor','hardware') and ${table.projectVersion} >= 1 and ${table.bindingVersion} >= 1`, + ), + check( + 'ql3_approved_action_manual_recovery_digest_check', + sql`${table.dispatchDigest} ~ '^[0-9a-f]{64}$' and ${table.actionDigest} ~ '^[0-9a-f]{64}$' and ${table.executionDigest} ~ '^[0-9a-f]{64}$' and ${table.evidenceDigest} ~ '^[0-9a-f]{64}$' and ${table.resolutionDigest} ~ '^[0-9a-f]{64}$'`, + ), + check( + 'ql3_approved_action_manual_recovery_time_check', + sql`${table.authenticatedAtMs} >= 0 and ${table.resolvedAtMs} >= ${table.authenticatedAtMs} and ${table.resolvedAtMs} - ${table.authenticatedAtMs} <= 300000`, + ), + check( + 'ql3_approved_action_manual_recovery_json_check', + sql`jsonb_typeof(${table.resolutionJson}) = 'object' and octet_length(${table.resolutionJson}::text) between 2 and 65536 and ${table.resolutionJson} @> jsonb_build_object('schema', 'qinglong/approved-action-manual-recovery@v1', 'dispatchId', ${table.dispatchId}, 'dispatchDigest', ${table.dispatchDigest}, 'projectId', ${table.projectId}, 'actionType', ${table.actionType}, 'actionDigest', ${table.actionDigest}, 'executionVersion', ${table.executionVersion}, 'executionDigest', ${table.executionDigest}, 'mutationId', ${table.mutationId}, 'decision', ${table.decision}, 'evidenceDigest', ${table.evidenceDigest}, 'reasonCode', ${table.reasonCode}, 'resolvedBy', jsonb_build_object('type', ${table.resolvedByType}, 'id', ${table.resolvedById}), 'authenticationId', ${table.authenticationId}, 'assurance', ${table.assurance}, 'authenticatedAtMs', ${table.authenticatedAtMs}, 'authorizationFence', jsonb_build_object('projectVersion', ${table.projectVersion}, 'bindingVersion', ${table.bindingVersion}), 'auditEventId', ${table.auditEventId}, 'resolvedAtMs', ${table.resolvedAtMs}, 'resolutionDigest', ${table.resolutionDigest})`, + ), + uniqueIndex('ql3_approved_action_manual_recovery_mutation_uidx').on( + table.mutationId, + ), + uniqueIndex('ql3_approved_action_manual_recovery_digest_uidx').on( + table.resolutionDigest, + ), + index('ql3_approved_action_manual_recovery_project_idx').on( + table.projectId, + table.resolvedAtMs, + table.dispatchId, + ), + ], +); + export const pluginPackageInstallProposals = ql3Schema.table( 'plugin_package_install_proposals', { @@ -6103,6 +6177,7 @@ export const ql3PostgresTables = [ approvalRequests, approvedActionDispatches, approvedActionExecutions, + approvedActionManualRecoveryResolutions, pluginPackageInstallProposals, pluginPackageManagementQuotaBuckets, workerCredentialManagementQuotaBuckets, diff --git a/packages/ql3-cluster-postgres/src/schema/schemaContract.ts b/packages/ql3-cluster-postgres/src/schema/schemaContract.ts index 2547033b..248441c8 100644 --- a/packages/ql3-cluster-postgres/src/schema/schemaContract.ts +++ b/packages/ql3-cluster-postgres/src/schema/schemaContract.ts @@ -21,8 +21,8 @@ export interface PostgresSchemaContractTrigger { export interface PostgresSchemaContract { readonly schema: 'ql3'; readonly contractName: 'control-core'; - readonly contractVersion: 63; - readonly migrationId: 'pg-0064-plugin-package-secret-binding-transition-approval-plans'; + readonly contractVersion: 64; + readonly migrationId: 'pg-0065-approved-action-manual-recovery'; readonly minimumServerMajor: 16; readonly maximumServerMajor: 18; readonly capabilities: Readonly<{ @@ -38,6 +38,7 @@ export interface PostgresSchemaContract { api_credential_pepper_binding: 1; approved_action: 1; approved_action_execution: 1; + approved_action_manual_recovery: 1; approval_management_boundary: 1; automation_management_boundary: 1; automation_management_identity_keyset_ledger: 1; @@ -117,8 +118,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract = Object.freeze({ schema: 'ql3', contractName: 'control-core', - contractVersion: 63, - migrationId: 'pg-0064-plugin-package-secret-binding-transition-approval-plans', + contractVersion: 64, + migrationId: 'pg-0065-approved-action-manual-recovery', minimumServerMajor: 16, maximumServerMajor: 18, capabilities: Object.freeze({ @@ -127,6 +128,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract = api_credential_pepper_binding: 1, approved_action: 1, approved_action_execution: 1, + approved_action_manual_recovery: 1, approval_management_boundary: 1, automation_management_boundary: 1, automation_management_identity_keyset_ledger: 1, @@ -770,6 +772,30 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract = 'execution_json', 'execution_digest', ]), + table('approved_action_manual_recovery_resolutions', [ + 'dispatch_id', + 'dispatch_digest', + 'project_id', + 'action_type', + 'action_digest', + 'execution_version', + 'execution_digest', + 'mutation_id', + 'decision', + 'evidence_digest', + 'reason_code', + 'resolved_by_type', + 'resolved_by_id', + 'authentication_id', + 'assurance', + 'authenticated_at_ms', + 'project_version', + 'binding_version', + 'audit_event_id', + 'resolved_at_ms', + 'resolution_json', + 'resolution_digest', + ]), table('plugin_package_install_proposals', [ 'action_ref', 'project_id', @@ -1624,6 +1650,10 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract = 'ql3_approved_action_execution_due_idx', 'ql3_approved_action_execution_recovery_idx', 'ql3_approved_action_execution_project_idx', + 'approved_action_manual_recovery_resolutions_pkey', + 'ql3_approved_action_manual_recovery_mutation_uidx', + 'ql3_approved_action_manual_recovery_digest_uidx', + 'ql3_approved_action_manual_recovery_project_idx', 'plugin_package_install_proposals_pkey', 'ql3_plugin_package_proposal_project_idx', 'plugin_package_management_quota_buckets_pkey', @@ -1942,6 +1972,10 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract = 'ql3_approved_action_execution_digest_check', 'ql3_approved_action_execution_json_check', 'ql3_approved_action_execution_time_check', + 'ql3_approved_action_manual_recovery_identity_check', + 'ql3_approved_action_manual_recovery_digest_check', + 'ql3_approved_action_manual_recovery_time_check', + 'ql3_approved_action_manual_recovery_json_check', 'ql3_plugin_package_proposal_identity_check', 'ql3_plugin_package_proposal_digest_check', 'ql3_plugin_package_proposal_json_check', @@ -2347,6 +2381,9 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract = 'ql3_approved_action_dispatch_request_fk', 'ql3_approved_action_dispatch_project_fk', 'ql3_approved_action_execution_dispatch_fk', + 'ql3_approved_action_manual_recovery_dispatch_fk', + 'ql3_approved_action_manual_recovery_project_fk', + 'ql3_approved_action_manual_recovery_audit_fk', 'ql3_approved_action_execution_project_fk', 'ql3_plugin_package_proposal_project_fk', 'ql3_plugin_package_management_quota_project_fk', @@ -2434,6 +2471,15 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract = 'ql3_run_retry_policies_run_fk', ]), functions: Object.freeze([ + Object.freeze({ + name: 'resolve_approved_action_manual_recovery', + identityArguments: + 'p_resolution_json jsonb, p_next_execution_json jsonb, p_audit_json jsonb', + owner: 'ql3_migration', + securityDefiner: true, + volatility: 'volatile', + configuration: Object.freeze(['search_path=pg_catalog, ql3']), + }), Object.freeze({ name: 'plugin_package_secret_binding_transition_snapshot', identityArguments: diff --git a/packages/ql3-cluster-postgres/src/schema/schemaReadiness.ts b/packages/ql3-cluster-postgres/src/schema/schemaReadiness.ts index 5fc390c7..c75bf708 100644 --- a/packages/ql3-cluster-postgres/src/schema/schemaReadiness.ts +++ b/packages/ql3-cluster-postgres/src/schema/schemaReadiness.ts @@ -384,6 +384,12 @@ const REQUIRED_RUNTIME_PRIVILEGES = Object.freeze({ update: false, delete: false, }), + approved_action_manual_recovery_resolutions: Object.freeze({ + select: false, + insert: false, + update: false, + delete: false, + }), plugin_package_install_proposals: Object.freeze({ select: false, insert: false, @@ -921,6 +927,12 @@ const REQUIRED_ADMIN_PRIVILEGES = Object.freeze({ update: false, delete: false, }), + approved_action_manual_recovery_resolutions: Object.freeze({ + select: false, + insert: false, + update: false, + delete: false, + }), plugin_package_install_proposals: Object.freeze({ select: false, insert: false, @@ -1421,7 +1433,10 @@ const REQUIRED_APPROVAL_MANAGER_PRIVILEGES: RequiredPrivileges = Object.freeze( name === 'schema_capabilities' || name === 'projects' || name === 'project_role_bindings' || - name === 'tool_invocation_preview_artifacts' + name === 'tool_invocation_preview_artifacts' || + name === 'approved_action_dispatches' || + name === 'approved_action_executions' || + name === 'approved_action_manual_recovery_resolutions' ? { ...NO_TABLE_PRIVILEGES, select: true } : name === 'security_audit_events' ? { ...NO_TABLE_PRIVILEGES, select: true, insert: true } @@ -1573,6 +1588,7 @@ const REQUIRED_RUNTIME_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges = plugin_package_secret_binding_transition_snapshot: false, plugin_package_tool_start_allowed: true, register_plugin_package_automation_disposition_event: false, + resolve_approved_action_manual_recovery: false, }); const REQUIRED_PACKAGE_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges = @@ -1598,6 +1614,7 @@ const REQUIRED_PACKAGE_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges = plugin_package_secret_binding_transition_snapshot: true, plugin_package_tool_start_allowed: false, register_plugin_package_automation_disposition_event: false, + resolve_approved_action_manual_recovery: false, }); const REQUIRED_PACKAGE_EXECUTOR_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges = @@ -1623,6 +1640,7 @@ const REQUIRED_PACKAGE_EXECUTOR_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges plugin_package_secret_binding_transition_snapshot: false, plugin_package_tool_start_allowed: false, register_plugin_package_automation_disposition_event: false, + resolve_approved_action_manual_recovery: false, }); const REQUIRED_WORKER_CREDENTIAL_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges = @@ -1635,6 +1653,7 @@ const REQUIRED_APPROVAL_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges Object.freeze({ ...NO_FUNCTION_PRIVILEGES, lock_approval_policy_fence: true, + resolve_approved_action_manual_recovery: true, }); const REQUIRED_RUN_MANAGER_FUNCTION_PRIVILEGES: RequiredFunctionPrivileges = diff --git a/packages/ql3-cluster-postgres/test/approvedActionManualRecovery.integration.test.cjs b/packages/ql3-cluster-postgres/test/approvedActionManualRecovery.integration.test.cjs new file mode 100644 index 00000000..c32afd50 --- /dev/null +++ b/packages/ql3-cluster-postgres/test/approvedActionManualRecovery.integration.test.cjs @@ -0,0 +1,273 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + approvalRequestDigest, + approvedActionDispatchDigest, + consumeApprovalRequest, + createApprovalRequest, + decideApprovalRequest, +} = require('@qinglong/runtime-core/approved-action'); +const { + createApprovedActionExecution, +} = require('@qinglong/runtime-core/approved-action-execution'); +const { + createApprovedActionManualRecoveryService, +} = require('@qinglong/runtime-core/approved-action-manual-recovery'); +const { ProjectPolicyEngine } = require('@qinglong/runtime-core/project-policy'); +const { + assertPostgresApprovalManagerSchemaReady, + createPostgresDatabaseOpener, + PostgresApprovedActionManualRecoveryRepository, + PostgresProjectPolicyRepository, + PostgresSecurityAuditRepository, +} = require('@qinglong/cluster-postgres/approval-manager'); +const { + PostgresApprovedActionExecutionRepository, +} = require('../dist/approved-action/approvedActionExecutionRepository'); +const { runPostgresMigrations } = require('../dist/migration/migration'); + +const migrationConnectionString = process.env.QL3_TEST_POSTGRES_MIGRATION_URL; +const approvalManagerConnectionString = + process.env.QL3_TEST_POSTGRES_APPROVAL_MANAGER_URL; + +async function open(role, connectionString) { + return createPostgresDatabaseOpener({ + role, + connection: { connectionString, tls: { mode: 'disable' } }, + pool: { + maxConnections: 1, + applicationName: `ql3-manual-recovery-${role}`, + }, + onPoolError(error) { + throw error; + }, + })(); +} + +async function insertFixture(pool, namespace) { + const projectId = `${namespace}-project`; + const principal = Object.freeze({ + subject: Object.freeze({ type: 'user', id: `${namespace}-owner` }), + authenticationId: `${namespace}-session`, + authenticatedAtMs: 100, + expiresAtMs: 20_000, + assurance: 'hardware', + }); + const action = Object.freeze({ + permission: 'secret.manage', + actionType: 'plugin_package.secret_binding.bind', + actionRef: `${namespace}:secret-binding`, + actionDigest: 'a'.repeat(64), + previewDigest: 'b'.repeat(64), + }); + const pending = createApprovalRequest({ + id: `${namespace}-approval`, + projectId, + action, + risk: 'high', + decisionMode: 'human_confirmation', + requestedBy: { type: 'agent', id: `${namespace}-agent` }, + requestedAtMs: 800, + expiresAtMs: 10_000, + requestFence: { projectVersion: 1, bindingVersion: 1 }, + }); + const approved = decideApprovalRequest(pending, { + expectedVersion: 1, + decisionId: `${namespace}-decision`, + decision: 'approved', + reasonCode: 'reviewed', + principal, + decidedAtMs: 900, + authorizationFence: { projectVersion: 1, bindingVersion: 1 }, + }); + const consumed = consumeApprovalRequest(approved, { + expectedVersion: 2, + consumptionId: `${namespace}-consumption`, + dispatchId: `${namespace}-dispatch`, + action, + requestedBy: pending.requestedBy, + consumedBy: { type: 'system', id: 'package-executor' }, + consumedAtMs: 950, + authorizationFence: { projectVersion: 1, bindingVersion: 1 }, + }); + await pool.query( + `INSERT INTO "ql3"."projects" ( + id, name, slug, status, version, created_at_ms, updated_at_ms + ) VALUES ($1, $1, $2, 'active', 1, 1, 1)`, + [projectId, projectId.replace(/[^a-z0-9-]/g, '-')], + ); + await pool.query( + `INSERT INTO "ql3"."project_role_bindings" ( + project_id, subject_type, subject_id, version, state, role, + mutation_id, changed_by_type, changed_by_id, created_at_ms + ) VALUES ($1, 'user', $2, 1, 'active', 'owner', $3, 'system', + 'integration-fixture', 2)`, + [projectId, principal.subject.id, `${namespace}-binding`], + ); + const request = consumed.request; + await pool.query( + `INSERT INTO "ql3"."approval_requests" ( + request_id, project_id, version, state, action_type, action_ref, + action_digest, preview_digest, requested_by_type, requested_by_id, + decision_id, consumption_id, dispatch_id, expires_at_ms, request_json, + request_digest, updated_at_ms + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, + $15::jsonb, $16, $17 + )`, + [ + request.id, + request.projectId, + request.version, + request.state, + request.action.actionType, + request.action.actionRef, + request.action.actionDigest, + request.action.previewDigest, + request.requestedBy.type, + request.requestedBy.id, + request.decisionId, + request.consumptionId, + request.dispatchId, + request.expiresAtMs, + JSON.stringify(request), + approvalRequestDigest(request), + request.consumedAtMs, + ], + ); + const dispatch = consumed.dispatch; + await pool.query( + `INSERT INTO "ql3"."approved_action_dispatches" ( + dispatch_id, approval_request_id, project_id, action_type, action_ref, + action_digest, preview_digest, dispatch_json, dispatch_digest, + created_at_ms + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10)`, + [ + dispatch.id, + dispatch.approvalRequestId, + dispatch.projectId, + dispatch.action.actionType, + dispatch.action.actionRef, + dispatch.action.actionDigest, + dispatch.action.previewDigest, + JSON.stringify(dispatch), + approvedActionDispatchDigest(dispatch), + dispatch.createdAtMs, + ], + ); + const executions = new PostgresApprovedActionExecutionRepository(pool); + await pool.query( + `INSERT INTO "ql3"."approved_action_executions" ( + dispatch_id, dispatch_digest, project_id, status, version, + attempt_count, max_attempts, eligible_at_ms, next_attempt_at_ms, + lease_owner, lease_token, lease_expires_at_ms, started_at_ms, + result_mutation_id, result_code, result_digest, completed_at_ms, + created_at_ms, updated_at_ms, execution_json, execution_digest + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, + $13, $14, $15, $16, $17, $18, $19, $20::jsonb, $21)`, + (() => { + const value = createApprovedActionExecution(dispatch); + return [ + value.dispatchId, value.dispatchDigest, value.projectId, value.status, + value.version, value.attemptCount, value.maxAttempts, value.eligibleAtMs, + value.nextAttemptAtMs, value.leaseOwner, value.leaseToken, + value.leaseExpiresAtMs, value.startedAtMs, value.resultMutationId, + value.resultCode, value.resultDigest, value.completedAtMs, + value.createdAtMs, value.updatedAtMs, JSON.stringify(value), + value.executionDigest, + ]; + })(), + ); + const claimed = await executions.claimExecution({ + dispatchId: dispatch.id, + owner: `${namespace}-executor`, + leaseToken: `${namespace}-lease`, + nowMs: 1_000, + leaseDurationMs: 500, + }); + const started = await executions.startExecution({ + dispatchId: dispatch.id, + approvalRequestId: dispatch.approvalRequestId, + actionDigest: dispatch.action.actionDigest, + owner: `${namespace}-executor`, + leaseToken: `${namespace}-lease`, + expectedVersion: claimed.snapshot.execution.version, + startedAtMs: 1_100, + }); + return { projectId, principal, dispatch, execution: started.execution }; +} + +if (!migrationConnectionString || !approvalManagerConnectionString) { + test('PostgreSQL manual recovery gate requires migration and Approval manager URLs', { + skip: true, + }); +} else { + test('PostgreSQL atomically resolves and exactly replays an expired Secret Action', async () => { + const migration = await open('migration', migrationConnectionString); + let manager; + try { + await runPostgresMigrations({ pool: migration.pool }); + manager = await open('approval-manager', approvalManagerConnectionString); + const readiness = await assertPostgresApprovalManagerSchemaReady(manager.pool); + assert.equal(readiness.contractVersion, 64); + const namespace = `recovery-${process.pid}-${Date.now()}`; + const fixture = await insertFixture(migration.pool, namespace); + const service = createApprovedActionManualRecoveryService({ + repository: new PostgresApprovedActionManualRecoveryRepository(manager.pool), + policy: new ProjectPolicyEngine( + new PostgresProjectPolicyRepository(manager.pool), + ), + audit: new PostgresSecurityAuditRepository(manager.pool), + now: () => 2_000, + }); + const inspected = await service.inspect({ + projectId: fixture.projectId, + dispatchId: fixture.dispatch.id, + requestId: `${namespace}-inspect`, + auditEventId: '80000000-0000-4000-8000-000000000001', + principal: fixture.principal, + }); + assert.equal(inspected.execution.execution.status, 'executing'); + const request = { + projectId: fixture.projectId, + dispatchId: fixture.dispatch.id, + expectedExecutionVersion: fixture.execution.version, + expectedExecutionDigest: fixture.execution.executionDigest, + mutationId: `${namespace}-mutation`, + decision: 'abandon_unknown', + evidenceDigest: 'e'.repeat(64), + reasonCode: 'orphan_absence_verified', + requestId: `${namespace}-resolve`, + auditEventId: '80000000-0000-4000-8000-000000000002', + principal: fixture.principal, + }; + const first = await service.resolve(request); + const replay = await service.resolve(request); + assert.equal(first.status, 'resolved'); + assert.equal(replay.status, 'existing'); + assert.equal(first.snapshot.execution.execution.status, 'blocked'); + assert.equal(first.snapshot.resolution.decision, 'abandon_unknown'); + const persisted = await migration.pool.query( + `SELECT + (SELECT count(*)::integer FROM "ql3"."approved_action_manual_recovery_resolutions" + WHERE dispatch_id = $1) AS "resolutionCount", + (SELECT count(*)::integer FROM "ql3"."security_audit_events" + WHERE event_id = $2) AS "auditCount"`, + [fixture.dispatch.id, request.auditEventId], + ); + assert.deepEqual(persisted.rows[0], { resolutionCount: 1, auditCount: 1 }); + await assert.rejects( + manager.pool.query( + `UPDATE "ql3"."approved_action_executions" SET status = 'failed' + WHERE dispatch_id = $1`, + [fixture.dispatch.id], + ), + (error) => error && error.code === '42501', + ); + } finally { + if (manager) await manager.close(); + await migration.close(); + } + }); +} diff --git a/packages/ql3-cluster-postgres/test/approvedActionManualRecoveryRepository.test.cjs b/packages/ql3-cluster-postgres/test/approvedActionManualRecoveryRepository.test.cjs new file mode 100644 index 00000000..b5f72e35 --- /dev/null +++ b/packages/ql3-cluster-postgres/test/approvedActionManualRecoveryRepository.test.cjs @@ -0,0 +1,158 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + consumeApprovalRequest, + createApprovalRequest, + decideApprovalRequest, +} = require('@qinglong/runtime-core/approved-action'); +const { + claimApprovedActionExecution, + createApprovedActionExecution, + startApprovedActionExecution, +} = require('@qinglong/runtime-core/approved-action-execution'); +const { + createApprovedActionManualRecoveryService, +} = require('@qinglong/runtime-core/approved-action-manual-recovery'); +const { + PostgresApprovedActionManualRecoveryRepository, +} = require('@qinglong/cluster-postgres/approval-manager'); + +const PRINCIPAL = Object.freeze({ + subject: Object.freeze({ type: 'user', id: 'owner-1' }), + authenticationId: 'oidc:session-1', + authenticatedAtMs: 100, + expiresAtMs: 20_000, + assurance: 'hardware', +}); + +function executing() { + const action = { + permission: 'secret.manage', + actionType: 'plugin_package.secret_binding.bind', + actionRef: 'secret-binding:1', + actionDigest: 'a'.repeat(64), + previewDigest: 'b'.repeat(64), + }; + const pending = createApprovalRequest({ + id: 'approval-1', + projectId: 'default', + action, + risk: 'high', + decisionMode: 'human_confirmation', + requestedBy: { type: 'agent', id: 'agent-1' }, + requestedAtMs: 800, + expiresAtMs: 10_000, + requestFence: { projectVersion: 1, bindingVersion: 2 }, + }); + const approved = decideApprovalRequest(pending, { + expectedVersion: 1, + decisionId: 'decision-1', + decision: 'approved', + reasonCode: 'reviewed', + principal: PRINCIPAL, + decidedAtMs: 900, + authorizationFence: { projectVersion: 1, bindingVersion: 2 }, + }); + const dispatch = consumeApprovalRequest(approved, { + expectedVersion: 2, + consumptionId: 'consumption-1', + dispatchId: 'dispatch-1', + action, + requestedBy: pending.requestedBy, + consumedBy: { type: 'system', id: 'package-executor' }, + consumedAtMs: 950, + authorizationFence: { projectVersion: 1, bindingVersion: 2 }, + }).dispatch; + const leased = claimApprovedActionExecution(createApprovedActionExecution(dispatch), { + owner: 'executor-1', + leaseToken: 'lease-1', + nowMs: 1_000, + leaseDurationMs: 500, + }); + const execution = startApprovedActionExecution( + { dispatch, execution: leased }, + { + dispatchId: dispatch.id, + approvalRequestId: dispatch.approvalRequestId, + actionDigest: dispatch.action.actionDigest, + owner: leased.leaseOwner, + leaseToken: leased.leaseToken, + expectedVersion: leased.version, + startedAtMs: 1_100, + }, + ); + return { dispatch, execution }; +} + +test('resolves through the bounded PostgreSQL function and verifies the stored tuple', async () => { + const initial = executing(); + let stored = { ...initial, resolution: null }; + const calls = []; + const pool = { + async query(text, values) { + calls.push([text, values]); + if (text.includes('resolve_approved_action_manual_recovery')) { + stored = { + dispatch: initial.dispatch, + execution: JSON.parse(values[1]), + resolution: JSON.parse(values[0]), + }; + return { rows: [{ status: 'resolved' }] }; + } + if (text.includes('approved_action_manual_recovery_resolutions')) { + return { + rows: [ + { + dispatchJson: stored.dispatch, + executionJson: stored.execution, + executionDigest: stored.execution.executionDigest, + resolutionJson: stored.resolution, + }, + ], + }; + } + throw new Error(`unexpected query: ${text}`); + }, + async connect() { + throw new Error('repository must not open a broad transaction'); + }, + }; + const repository = new PostgresApprovedActionManualRecoveryRepository(pool); + const service = createApprovedActionManualRecoveryService({ + repository, + policy: { + async authorize(_principal, _projectId, permission) { + assert.equal(permission, 'approval.recover'); + return { + effect: 'allow', + reasons: ['role_grant'], + fence: { projectVersion: 1, bindingVersion: 2 }, + }; + }, + }, + audit: { async record() {} }, + now: () => 2_000, + }); + const result = await service.resolve({ + projectId: 'default', + dispatchId: 'dispatch-1', + expectedExecutionVersion: initial.execution.version, + expectedExecutionDigest: initial.execution.executionDigest, + mutationId: 'manual-recovery-1', + decision: 'abandon_unknown', + evidenceDigest: 'e'.repeat(64), + reasonCode: 'orphan_absence_verified', + auditEventId: '70000000-0000-4000-8000-000000000001', + requestId: 'manual-recovery-request-1', + principal: PRINCIPAL, + }); + assert.equal(result.status, 'resolved'); + assert.equal(result.snapshot.execution.execution.status, 'blocked'); + assert.equal(result.snapshot.resolution.decision, 'abandon_unknown'); + assert.equal( + calls.filter(([sql]) => sql.includes('resolve_approved_action_manual_recovery')) + .length, + 1, + ); +}); diff --git a/packages/ql3-cluster-postgres/test/postgresqlMigrationDefinitions.test.cjs b/packages/ql3-cluster-postgres/test/postgresqlMigrationDefinitions.test.cjs index 53030092..7130bb77 100644 --- a/packages/ql3-cluster-postgres/test/postgresqlMigrationDefinitions.test.cjs +++ b/packages/ql3-cluster-postgres/test/postgresqlMigrationDefinitions.test.cjs @@ -115,6 +115,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async () 'pg-0062-plugin-package-secret-binding-target-guard', 'pg-0063-plugin-package-secret-binding-transition-receipts', 'pg-0064-plugin-package-secret-binding-transition-approval-plans', + 'pg-0065-approved-action-manual-recovery', ], ); for (const migration of postgresqlMainMigrationStream.migrations) { @@ -573,6 +574,11 @@ test('freezes every published PostgreSQL migration checksum', () => { checksum: '1951b77a0265f8826169e4724424b2fbbd30061b27e27d3ba95de03430c1bac9', }, + { + id: 'pg-0065-approved-action-manual-recovery', + checksum: + '95387c5b40659490dbcb7626ecd15bacf6412360752bef88873bde57c43e0185', + }, ]; assert.deepEqual( postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({ @@ -2227,3 +2233,52 @@ test('advances capability v63 with manager-only immutable Secret transition plan /migration_id = 'pg-0063-plugin-package-secret-binding-transition-receipts'/, ); }); + +test('advances capability v64 with atomic least-privilege manual recovery', async () => { + const migration = migrationById('pg-0065-approved-action-manual-recovery'); + const statements = []; + await migration.up({ + async query(statement) { + statements.push(statement); + return { rows: [] }; + }, + }); + const sql = statements.join('\n'); + assert.match( + sql, + /CREATE TABLE "ql3"\."approved_action_manual_recovery_resolutions"/, + ); + assert.match( + sql, + /CREATE FUNCTION "ql3"\."resolve_approved_action_manual_recovery"\([\s\S]+SECURITY DEFINER[\s\S]+SET search_path = pg_catalog, ql3/, + ); + assert.match(sql, /current_status <> 'executing'/); + assert.match(sql, /current_lease_expires_at_ms > .*'resolvedAtMs'/); + assert.match( + sql, + /current_action_type NOT IN \([\s\S]+plugin_package\.secret_binding\.bind[\s\S]+plugin_package\.secret_binding\.transition/, + ); + assert.match( + sql, + /INSERT INTO "ql3"\."security_audit_events"[\s\S]+UPDATE "ql3"\."approved_action_executions"[\s\S]+INSERT INTO "ql3"\."approved_action_manual_recovery_resolutions"/, + ); + assert.match( + sql, + /GRANT SELECT ON "ql3"\."approved_action_dispatches", "ql3"\."approved_action_executions", "ql3"\."approved_action_manual_recovery_resolutions" TO ql3_approval_manager/, + ); + assert.match( + sql, + /GRANT EXECUTE ON FUNCTION "ql3"\."resolve_approved_action_manual_recovery"\(jsonb, jsonb, jsonb\) TO ql3_approval_manager/, + ); + assert.doesNotMatch( + sql, + /GRANT (?:INSERT|UPDATE|DELETE)[^;]+(?:approved_action_executions|approved_action_manual_recovery_resolutions)[^;]+ql3_approval_manager/, + ); + assert.match(sql, /contract_version = 64/); + assert.match(sql, /"approved_action_manual_recovery":1/); + assert.match(sql, /contract_version = 63/); + assert.match( + sql, + /migration_id = 'pg-0064-plugin-package-secret-binding-transition-approval-plans'/, + ); +}); diff --git a/packages/ql3-cluster-postgres/test/postgresqlSchemaReadiness.test.cjs b/packages/ql3-cluster-postgres/test/postgresqlSchemaReadiness.test.cjs index cf047a54..6da6f323 100644 --- a/packages/ql3-cluster-postgres/test/postgresqlSchemaReadiness.test.cjs +++ b/packages/ql3-cluster-postgres/test/postgresqlSchemaReadiness.test.cjs @@ -134,6 +134,7 @@ function validPrivileges() { approval_requests: [false, false, false, false], approved_action_dispatches: [false, false, false, false], approved_action_executions: [false, false, false, false], + approved_action_manual_recovery_resolutions: [false, false, false, false], plugin_package_install_proposals: [false, false, false, false], plugin_package_management_quota_buckets: [false, false, false, false], plugin_package_identity_keyset_ledger: [false, false, false, false], @@ -271,6 +272,7 @@ function validAdminPrivileges() { approval_requests: [false, false, false, false], approved_action_dispatches: [false, false, false, false], approved_action_executions: [false, false, false, false], + approved_action_manual_recovery_resolutions: [false, false, false, false], plugin_package_install_proposals: [false, false, false, false], plugin_package_management_quota_buckets: [false, false, false, false], plugin_package_identity_keyset_ledger: [false, false, false, false], @@ -491,6 +493,9 @@ function approvalManagerPrivileges() { 'project_role_bindings', 'security_audit_events', 'approval_requests', + 'approved_action_dispatches', + 'approved_action_executions', + 'approved_action_manual_recovery_resolutions', 'tool_invocation_preview_artifacts', 'plugin_package_identity_keyset_ledger', ]); @@ -705,6 +710,11 @@ function queryable(overrides = {}) { 'plugin_package_secret_binding_planning_snapshot', 'plugin_package_secret_binding_transition_snapshot', ].includes(functionName) + : overrides.functionMode === 'approval-manager' + ? [ + 'lock_approval_policy_fence', + 'resolve_approved_action_manual_recovery', + ].includes(functionName) : overrides.functionMode === 'manager' ? functionName === 'lock_approval_policy_fence' : overrides.functionMode === 'run-manager' @@ -805,7 +815,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro serverMajor: 16, currentUser: 'ql3_runtime', contractName: 'control-core', - contractVersion: 63, + contractVersion: 64, migrationIds: [ 'pg-0001-schema-capability', 'pg-0002-run-core', @@ -871,6 +881,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro 'pg-0062-plugin-package-secret-binding-target-guard', 'pg-0063-plugin-package-secret-binding-transition-receipts', 'pg-0064-plugin-package-secret-binding-transition-approval-plans', + 'pg-0065-approved-action-manual-recovery', ], }); }); @@ -901,10 +912,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async ( }), ); assert.equal(report.currentUser, 'ql3_admin'); - assert.equal(report.contractVersion, 63); + assert.equal(report.contractVersion, 64); assert.equal( report.migrationIds.at(-1), - 'pg-0064-plugin-package-secret-binding-transition-approval-plans', + 'pg-0065-approved-action-manual-recovery', ); }); @@ -917,10 +928,10 @@ test('accepts the isolated least-privilege automation manager role', async () => }), ); assert.equal(report.currentUser, 'ql3_automation_manager'); - assert.equal(report.contractVersion, 63); + assert.equal(report.contractVersion, 64); assert.equal( report.migrationIds.at(-1), - 'pg-0064-plugin-package-secret-binding-transition-approval-plans', + 'pg-0065-approved-action-manual-recovery', ); const widened = automationManagerPrivileges(); @@ -945,14 +956,14 @@ test('accepts the isolated least-privilege human Approval manager role', async ( queryable({ currentUser: 'ql3_approval_manager', privileges: approvalManagerPrivileges(), - functionMode: 'manager', + functionMode: 'approval-manager', }), ); assert.equal(report.currentUser, 'ql3_approval_manager'); - assert.equal(report.contractVersion, 63); + assert.equal(report.contractVersion, 64); assert.equal( report.migrationIds.at(-1), - 'pg-0064-plugin-package-secret-binding-transition-approval-plans', + 'pg-0065-approved-action-manual-recovery', ); const widened = approvalManagerPrivileges(); @@ -964,7 +975,7 @@ test('accepts the isolated least-privilege human Approval manager role', async ( queryable({ currentUser: 'ql3_approval_manager', privileges: widened, - functionMode: 'manager', + functionMode: 'approval-manager', }), ), (error) => @@ -983,10 +994,10 @@ test('accepts the isolated least-privilege Run manager role', async () => { }), ); assert.equal(report.currentUser, 'ql3_run_manager'); - assert.equal(report.contractVersion, 63); + assert.equal(report.contractVersion, 64); assert.equal( report.migrationIds.at(-1), - 'pg-0064-plugin-package-secret-binding-transition-approval-plans', + 'pg-0065-approved-action-manual-recovery', ); const widened = runManagerPrivileges(); @@ -1118,10 +1129,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => { }), ); assert.equal(report.currentUser, 'ql3_worker_ingress'); - assert.equal(report.contractVersion, 63); + assert.equal(report.contractVersion, 64); assert.equal( report.migrationIds.at(-1), - 'pg-0064-plugin-package-secret-binding-transition-approval-plans', + 'pg-0065-approved-action-manual-recovery', ); }); diff --git a/packages/ql3-runtime-core/package.json b/packages/ql3-runtime-core/package.json index b1340f81..5fa54a42 100644 --- a/packages/ql3-runtime-core/package.json +++ b/packages/ql3-runtime-core/package.json @@ -552,6 +552,11 @@ "require": "./dist/approved-action/approvedActionExecution.js", "default": "./dist/approved-action/approvedActionExecution.js" }, + "./approved-action-manual-recovery": { + "types": "./dist/approved-action/approvedActionManualRecovery.d.ts", + "require": "./dist/approved-action/approvedActionManualRecovery.js", + "default": "./dist/approved-action/approvedActionManualRecovery.js" + }, "./approved-action-dispatcher": { "types": "./dist/approved-action/approvedActionDispatcher.d.ts", "require": "./dist/approved-action/approvedActionDispatcher.js", diff --git a/packages/ql3-runtime-core/src/approved-action/approvedActionManualRecovery.ts b/packages/ql3-runtime-core/src/approved-action/approvedActionManualRecovery.ts new file mode 100644 index 00000000..d2459cfc --- /dev/null +++ b/packages/ql3-runtime-core/src/approved-action/approvedActionManualRecovery.ts @@ -0,0 +1,775 @@ +import { createHash } from 'node:crypto'; + +import { + approvedActionExecutionEffectiveStatus, + completeApprovedActionExecution, + normalizeApprovedActionExecutionRecord, + normalizeApprovedActionExecutionSnapshot, + type ApprovedActionExecutionRecord, + type ApprovedActionExecutionSnapshot, +} from './approvedActionExecution'; +import type { ProjectPolicyEngine } from '../security/project-policy/projectPolicy'; +import { + normalizeSecurityPrincipal, + type SecurityPolicyFence, + type SecurityPrincipal, + type SecuritySubject, +} from '../security/security'; +import { + normalizeSecurityAuditRecord, + type SecurityAuditRecord, + type SecurityAuditSink, +} from '../security/audit/securityAudit'; + +export const APPROVED_ACTION_MANUAL_RECOVERY_SCHEMA = + 'qinglong/approved-action-manual-recovery@v1' as const; +export const APPROVED_ACTION_MANUAL_RECOVERY_DECISIONS = [ + 'confirm_failed', + 'abandon_unknown', +] as const; +export const APPROVED_ACTION_MANUAL_RECOVERY_ACTION_TYPES = [ + 'plugin_package.secret_binding.bind', + 'plugin_package.secret_binding.transition', +] as const; +export const MAX_APPROVED_ACTION_MANUAL_RECOVERY_AUTH_AGE_MS = 5 * 60_000; + +export type ApprovedActionManualRecoveryDecision = + (typeof APPROVED_ACTION_MANUAL_RECOVERY_DECISIONS)[number]; + +export interface ApprovedActionManualRecoveryResolutionRecord { + readonly schema: typeof APPROVED_ACTION_MANUAL_RECOVERY_SCHEMA; + readonly dispatchId: string; + readonly dispatchDigest: string; + readonly projectId: string; + readonly actionType: string; + readonly actionDigest: string; + readonly executionVersion: number; + readonly executionDigest: string; + readonly mutationId: string; + readonly decision: ApprovedActionManualRecoveryDecision; + readonly evidenceDigest: string; + readonly reasonCode: string; + readonly resolvedBy: Readonly; + readonly authenticationId: string; + readonly assurance: 'multi_factor' | 'hardware'; + readonly authenticatedAtMs: number; + readonly authorizationFence: Readonly; + readonly auditEventId: string; + readonly resolvedAtMs: number; + readonly resolutionDigest: string; +} + +export interface ApprovedActionManualRecoverySnapshot { + readonly execution: Readonly; + readonly resolution: Readonly | null; +} + +export interface ApprovedActionManualRecoveryInspectRequest { + readonly projectId: string; + readonly dispatchId: string; + readonly auditEventId: string; + readonly requestId: string; + readonly principal: Readonly; +} + +export interface ApprovedActionManualRecoveryResolveRequest + extends ApprovedActionManualRecoveryInspectRequest { + readonly expectedExecutionVersion: number; + readonly expectedExecutionDigest: string; + readonly mutationId: string; + readonly decision: ApprovedActionManualRecoveryDecision; + readonly evidenceDigest: string; + readonly reasonCode: string; +} + +export interface ResolveApprovedActionManualRecoveryCommand { + readonly previous: Readonly; + readonly nextExecution: Readonly; + readonly resolution: Readonly; + readonly audit: Readonly; +} + +export interface ResolveApprovedActionManualRecoveryResult { + readonly status: 'resolved' | 'existing'; + readonly snapshot: Readonly; +} + +export interface ApprovedActionManualRecoveryRepository { + findByDispatchId( + dispatchId: string, + ): Promise | null>; + resolve( + command: Readonly, + ): Promise>; +} + +export interface ApprovedActionManualRecoveryService { + inspect( + request: ApprovedActionManualRecoveryInspectRequest, + confirmAuthorization?: () => void | Promise, + ): Promise | null>; + resolve( + request: ApprovedActionManualRecoveryResolveRequest, + confirmAuthorization?: () => void | Promise, + ): Promise>; +} + +export class InvalidApprovedActionManualRecoveryError extends TypeError { + readonly code = 'APPROVED_ACTION_MANUAL_RECOVERY_INVALID'; + + constructor(message: string) { + super(`Approved Action manual recovery is invalid: ${message}`); + this.name = 'InvalidApprovedActionManualRecoveryError'; + } +} + +export class ApprovedActionManualRecoveryAuthorizationError extends Error { + readonly code = 'APPROVED_ACTION_MANUAL_RECOVERY_AUTHORIZATION_REJECTED'; + + constructor() { + super('Approved Action manual recovery authorization was rejected'); + this.name = 'ApprovedActionManualRecoveryAuthorizationError'; + } +} + +export class ApprovedActionManualRecoveryTargetUnavailableError extends Error { + readonly code = 'APPROVED_ACTION_MANUAL_RECOVERY_TARGET_UNAVAILABLE'; + + constructor() { + super('Approved Action manual recovery target is unavailable'); + this.name = 'ApprovedActionManualRecoveryTargetUnavailableError'; + } +} + +export class ApprovedActionManualRecoveryUnsupportedError extends Error { + readonly code = 'APPROVED_ACTION_MANUAL_RECOVERY_UNSUPPORTED'; + + constructor() { + super('Approved Action does not support manual recovery'); + this.name = 'ApprovedActionManualRecoveryUnsupportedError'; + } +} + +export class ApprovedActionManualRecoveryFenceConflictError extends Error { + readonly code = 'APPROVED_ACTION_MANUAL_RECOVERY_FENCE_CONFLICT'; + + constructor() { + super('Approved Action manual recovery fence changed'); + this.name = 'ApprovedActionManualRecoveryFenceConflictError'; + } +} + +export class ApprovedActionManualRecoveryUnavailableError extends Error { + readonly code = 'APPROVED_ACTION_MANUAL_RECOVERY_UNAVAILABLE'; + + constructor(options?: ErrorOptions) { + super('Approved Action manual recovery is unavailable', options); + this.name = 'ApprovedActionManualRecoveryUnavailableError'; + } +} + +const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const REASON_PATTERN = /^[a-z][a-z0-9_]{0,63}$/; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const UUID_V4_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +function invalid(message: string): never { + throw new InvalidApprovedActionManualRecoveryError(message); +} + +function exact(value: unknown, keys: readonly string[]): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + invalid('value must be an object'); + } + const actual = Object.keys(value as object).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + invalid('value shape is invalid'); + } + return value as Record; +} + +function identifier(value: unknown, name: string): string { + if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) { + invalid(`${name} is invalid`); + } + return value; +} + +function reason(value: unknown): string { + if (typeof value !== 'string' || !REASON_PATTERN.test(value)) { + invalid('reason code is invalid'); + } + return value; +} + +function digest(value: unknown, name: string): string { + if (typeof value !== 'string' || !DIGEST_PATTERN.test(value)) { + invalid(`${name} is invalid`); + } + return value; +} + +function uuid(value: unknown, name: string): string { + if (typeof value !== 'string' || !UUID_V4_PATTERN.test(value)) { + invalid(`${name} is invalid`); + } + return value; +} + +function integer(value: unknown, name: string, minimum = 0): number { + if ( + typeof value !== 'number' || + !Number.isSafeInteger(value) || + value < minimum || + value > 2_147_483_647 + ) { + invalid(`${name} is invalid`); + } + return value; +} + +function timestamp(value: unknown, name: string): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + invalid(`${name} is invalid`); + } + return value; +} + +function decision(value: unknown): ApprovedActionManualRecoveryDecision { + if ( + !APPROVED_ACTION_MANUAL_RECOVERY_DECISIONS.includes( + value as ApprovedActionManualRecoveryDecision, + ) + ) { + invalid('decision is invalid'); + } + return value as ApprovedActionManualRecoveryDecision; +} + +function supportedActionType(value: string): boolean { + return APPROVED_ACTION_MANUAL_RECOVERY_ACTION_TYPES.includes( + value as (typeof APPROVED_ACTION_MANUAL_RECOVERY_ACTION_TYPES)[number], + ); +} + +function subject(value: unknown): Readonly { + const record = exact(value, ['type', 'id']); + if (record.type !== 'user') invalid('resolved subject must be a User'); + return Object.freeze({ + type: 'user' as const, + id: identifier(record.id, 'resolved subject id'), + }); +} + +function fence(value: unknown): Readonly { + const record = exact(value, ['projectVersion', 'bindingVersion']); + const bindingVersion = integer(record.bindingVersion, 'binding version', 1); + return Object.freeze({ + projectVersion: integer(record.projectVersion, 'project version', 1), + bindingVersion, + }); +} + +function resolutionWithoutDigest( + value: Omit, +): Omit { + return { + schema: APPROVED_ACTION_MANUAL_RECOVERY_SCHEMA, + dispatchId: value.dispatchId, + dispatchDigest: value.dispatchDigest, + projectId: value.projectId, + actionType: value.actionType, + actionDigest: value.actionDigest, + executionVersion: value.executionVersion, + executionDigest: value.executionDigest, + mutationId: value.mutationId, + decision: value.decision, + evidenceDigest: value.evidenceDigest, + reasonCode: value.reasonCode, + resolvedBy: value.resolvedBy, + authenticationId: value.authenticationId, + assurance: value.assurance, + authenticatedAtMs: value.authenticatedAtMs, + authorizationFence: value.authorizationFence, + auditEventId: value.auditEventId, + resolvedAtMs: value.resolvedAtMs, + }; +} + +function resolutionDigest( + value: Omit, +): string { + return createHash('sha256') + .update(APPROVED_ACTION_MANUAL_RECOVERY_SCHEMA) + .update('\0') + .update(JSON.stringify(resolutionWithoutDigest(value))) + .digest('hex'); +} + +export function normalizeApprovedActionManualRecoveryResolution( + value: ApprovedActionManualRecoveryResolutionRecord, +): Readonly { + const record = exact(value, [ + 'schema', + 'dispatchId', + 'dispatchDigest', + 'projectId', + 'actionType', + 'actionDigest', + 'executionVersion', + 'executionDigest', + 'mutationId', + 'decision', + 'evidenceDigest', + 'reasonCode', + 'resolvedBy', + 'authenticationId', + 'assurance', + 'authenticatedAtMs', + 'authorizationFence', + 'auditEventId', + 'resolvedAtMs', + 'resolutionDigest', + ]); + if (record.schema !== APPROVED_ACTION_MANUAL_RECOVERY_SCHEMA) { + invalid('schema is invalid'); + } + const normalized = resolutionWithoutDigest({ + schema: APPROVED_ACTION_MANUAL_RECOVERY_SCHEMA, + dispatchId: identifier(record.dispatchId, 'dispatch id'), + dispatchDigest: digest(record.dispatchDigest, 'dispatch digest'), + projectId: identifier(record.projectId, 'project id'), + actionType: identifier(record.actionType, 'action type'), + actionDigest: digest(record.actionDigest, 'action digest'), + executionVersion: integer(record.executionVersion, 'execution version', 1), + executionDigest: digest(record.executionDigest, 'execution digest'), + mutationId: identifier(record.mutationId, 'mutation id'), + decision: decision(record.decision), + evidenceDigest: digest(record.evidenceDigest, 'evidence digest'), + reasonCode: reason(record.reasonCode), + resolvedBy: subject(record.resolvedBy), + authenticationId: identifier(record.authenticationId, 'authentication id'), + assurance: + record.assurance === 'multi_factor' || record.assurance === 'hardware' + ? record.assurance + : invalid('assurance is invalid'), + authenticatedAtMs: timestamp(record.authenticatedAtMs, 'authentication time'), + authorizationFence: fence(record.authorizationFence), + auditEventId: uuid(record.auditEventId, 'audit event id'), + resolvedAtMs: timestamp(record.resolvedAtMs, 'resolution time'), + }); + if ( + !supportedActionType(normalized.actionType) || + normalized.authenticatedAtMs > normalized.resolvedAtMs || + normalized.resolvedAtMs - normalized.authenticatedAtMs > + MAX_APPROVED_ACTION_MANUAL_RECOVERY_AUTH_AGE_MS + ) { + invalid('resolution authority is invalid'); + } + const normalizedDigest = digest(record.resolutionDigest, 'resolution digest'); + if (normalizedDigest !== resolutionDigest(normalized)) { + invalid('resolution digest does not match'); + } + return Object.freeze({ ...normalized, resolutionDigest: normalizedDigest }); +} + +export function normalizeApprovedActionManualRecoverySnapshot( + value: ApprovedActionManualRecoverySnapshot, +): Readonly { + const record = exact(value, ['execution', 'resolution']); + const execution = normalizeApprovedActionExecutionSnapshot( + record.execution as unknown as ApprovedActionExecutionSnapshot, + ); + const resolution = + record.resolution === null + ? null + : normalizeApprovedActionManualRecoveryResolution( + record.resolution as ApprovedActionManualRecoveryResolutionRecord, + ); + if ( + resolution && + (resolution.dispatchId !== execution.dispatch.id || + resolution.dispatchDigest !== execution.execution.dispatchDigest || + resolution.projectId !== execution.dispatch.projectId || + resolution.actionType !== execution.dispatch.action.actionType || + resolution.actionDigest !== execution.dispatch.action.actionDigest || + execution.execution.version !== resolution.executionVersion + 1 || + execution.execution.resultMutationId !== resolution.mutationId || + execution.execution.resultCode !== resultCodeFor(resolution.decision) || + execution.execution.resultDigest !== null || + execution.execution.completedAtMs !== resolution.resolvedAtMs || + execution.execution.status !== statusFor(resolution.decision)) + ) { + invalid('resolution is not bound to the execution'); + } + return Object.freeze({ execution, resolution }); +} + +function statusFor( + value: ApprovedActionManualRecoveryDecision, +): 'failed' | 'blocked' { + return value === 'confirm_failed' ? 'failed' : 'blocked'; +} + +function outcomeFor( + value: ApprovedActionManualRecoveryDecision, +): 'failed' | 'indeterminate' { + return value === 'confirm_failed' ? 'failed' : 'indeterminate'; +} + +function resultCodeFor(value: ApprovedActionManualRecoveryDecision): string { + return value === 'confirm_failed' + ? 'manual_recovery_confirmed_failed' + : 'manual_recovery_abandoned_unknown'; +} + +function observedTime(now: () => number): number { + const value = now(); + if (!Number.isSafeInteger(value) || value < 0) { + throw new ApprovedActionManualRecoveryUnavailableError(); + } + return value; +} + +function strongPrincipal( + value: Readonly, + nowMs: number, +): Readonly { + let principal: Readonly; + try { + principal = normalizeSecurityPrincipal(value, nowMs); + } catch { + throw new ApprovedActionManualRecoveryAuthorizationError(); + } + if ( + principal.subject.type !== 'user' || + (principal.assurance !== 'multi_factor' && principal.assurance !== 'hardware') || + principal.authenticatedAtMs > nowMs || + nowMs - principal.authenticatedAtMs > + MAX_APPROVED_ACTION_MANUAL_RECOVERY_AUTH_AGE_MS + ) { + throw new ApprovedActionManualRecoveryAuthorizationError(); + } + return principal; +} + +function normalizeInspectRequest( + value: ApprovedActionManualRecoveryInspectRequest, +): Readonly { + const record = exact(value, [ + 'projectId', + 'dispatchId', + 'auditEventId', + 'requestId', + 'principal', + ]); + return Object.freeze({ + projectId: identifier(record.projectId, 'project id'), + dispatchId: identifier(record.dispatchId, 'dispatch id'), + auditEventId: uuid(record.auditEventId, 'audit event id'), + requestId: identifier(record.requestId, 'request id'), + principal: record.principal as Readonly, + }); +} + +function normalizeResolveRequest( + value: ApprovedActionManualRecoveryResolveRequest, +): Readonly { + const record = exact(value, [ + 'projectId', + 'dispatchId', + 'auditEventId', + 'requestId', + 'principal', + 'expectedExecutionVersion', + 'expectedExecutionDigest', + 'mutationId', + 'decision', + 'evidenceDigest', + 'reasonCode', + ]); + return Object.freeze({ + ...normalizeInspectRequest({ + projectId: record.projectId as string, + dispatchId: record.dispatchId as string, + auditEventId: record.auditEventId as string, + requestId: record.requestId as string, + principal: record.principal as Readonly, + }), + expectedExecutionVersion: integer( + record.expectedExecutionVersion, + 'expected execution version', + 1, + ), + expectedExecutionDigest: digest( + record.expectedExecutionDigest, + 'expected execution digest', + ), + mutationId: identifier(record.mutationId, 'mutation id'), + decision: decision(record.decision), + evidenceDigest: digest(record.evidenceDigest, 'evidence digest'), + reasonCode: reason(record.reasonCode), + }); +} + +function allowedAudit( + request: Readonly, + principal: Readonly, + policyFence: Readonly, + operationId: 'approval.recover.inspect' | 'approval.recover.resolve', + occurredAtMs: number, +): Readonly { + return normalizeSecurityAuditRecord({ + eventId: request.auditEventId, + requestId: request.requestId, + operationId, + projectId: request.projectId, + subject: principal.subject, + authenticationId: principal.authenticationId, + outcome: 'allowed', + reasons: ['role_grant', 'strong_authentication', 'manual_recovery'], + fence: policyFence, + occurredAtMs, + }); +} + +function createResolution( + snapshot: Readonly, + request: Readonly, + principal: Readonly, + policyFence: Readonly, + resolvedAtMs: number, +): Readonly { + const withoutDigest = resolutionWithoutDigest({ + schema: APPROVED_ACTION_MANUAL_RECOVERY_SCHEMA, + dispatchId: snapshot.dispatch.id, + dispatchDigest: snapshot.execution.dispatchDigest, + projectId: snapshot.dispatch.projectId, + actionType: snapshot.dispatch.action.actionType, + actionDigest: snapshot.dispatch.action.actionDigest, + executionVersion: snapshot.execution.version, + executionDigest: snapshot.execution.executionDigest, + mutationId: request.mutationId, + decision: request.decision, + evidenceDigest: request.evidenceDigest, + reasonCode: request.reasonCode, + resolvedBy: principal.subject, + authenticationId: principal.authenticationId, + assurance: principal.assurance as 'multi_factor' | 'hardware', + authenticatedAtMs: principal.authenticatedAtMs, + authorizationFence: policyFence, + auditEventId: request.auditEventId, + resolvedAtMs, + }); + return normalizeApprovedActionManualRecoveryResolution({ + ...withoutDigest, + resolutionDigest: resolutionDigest(withoutDigest), + }); +} + +function exactReplay( + resolution: Readonly, + request: Readonly, + principal: Readonly, +): boolean { + return ( + resolution.projectId === request.projectId && + resolution.executionVersion === request.expectedExecutionVersion && + resolution.executionDigest === request.expectedExecutionDigest && + resolution.mutationId === request.mutationId && + resolution.decision === request.decision && + resolution.evidenceDigest === request.evidenceDigest && + resolution.reasonCode === request.reasonCode && + resolution.auditEventId === request.auditEventId && + resolution.resolvedBy.type === principal.subject.type && + resolution.resolvedBy.id === principal.subject.id && + resolution.authenticationId === principal.authenticationId + ); +} + +async function authorized( + policy: Pick, + principal: Readonly, + projectId: string, +): Promise> { + let decision; + try { + decision = await policy.authorize(principal, projectId, 'approval.recover'); + } catch (error) { + throw new ApprovedActionManualRecoveryUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + if ( + decision.effect !== 'allow' || + decision.fence === null || + decision.fence.bindingVersion === null + ) { + throw new ApprovedActionManualRecoveryAuthorizationError(); + } + return decision.fence; +} + +export function createApprovedActionManualRecoveryService(options: Readonly<{ + repository: ApprovedActionManualRecoveryRepository; + policy: Pick; + audit: SecurityAuditSink; + now?: () => number; +}>): Readonly { + exact(options, [ + 'repository', + 'policy', + 'audit', + ...(options?.now === undefined ? [] : ['now']), + ]); + if ( + typeof options.repository?.findByDispatchId !== 'function' || + typeof options.repository?.resolve !== 'function' || + typeof options.policy?.authorize !== 'function' || + typeof options.audit?.record !== 'function' || + (options.now !== undefined && typeof options.now !== 'function') + ) { + invalid('options are invalid'); + } + const now = options.now ?? Date.now; + return Object.freeze({ + async inspect( + requestValue: ApprovedActionManualRecoveryInspectRequest, + confirmAuthorization?: () => void | Promise, + ) { + const request = normalizeInspectRequest(requestValue); + const inspectedAtMs = observedTime(now); + const principal = strongPrincipal(request.principal, inspectedAtMs); + const policyFence = await authorized(options.policy, principal, request.projectId); + let snapshot: Readonly | null; + try { + const found = await options.repository.findByDispatchId(request.dispatchId); + snapshot = found ? normalizeApprovedActionManualRecoverySnapshot(found) : null; + } catch (error) { + throw new ApprovedActionManualRecoveryUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + if (snapshot && snapshot.execution.dispatch.projectId !== request.projectId) { + snapshot = null; + } + if ( + snapshot && + !supportedActionType(snapshot.execution.dispatch.action.actionType) + ) { + throw new ApprovedActionManualRecoveryUnsupportedError(); + } + await confirmAuthorization?.(); + try { + await options.audit.record( + allowedAudit( + request, + principal, + policyFence, + 'approval.recover.inspect', + inspectedAtMs, + ), + ); + } catch (error) { + throw new ApprovedActionManualRecoveryUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + return snapshot; + }, + + async resolve( + requestValue: ApprovedActionManualRecoveryResolveRequest, + confirmAuthorization?: () => void | Promise, + ) { + const request = normalizeResolveRequest(requestValue); + const resolvedAtMs = observedTime(now); + const principal = strongPrincipal(request.principal, resolvedAtMs); + const policyFence = await authorized(options.policy, principal, request.projectId); + let current: Readonly | null; + try { + const found = await options.repository.findByDispatchId(request.dispatchId); + current = found ? normalizeApprovedActionManualRecoverySnapshot(found) : null; + } catch (error) { + throw new ApprovedActionManualRecoveryUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + if (!current || current.execution.dispatch.projectId !== request.projectId) { + throw new ApprovedActionManualRecoveryTargetUnavailableError(); + } + if (!supportedActionType(current.execution.dispatch.action.actionType)) { + throw new ApprovedActionManualRecoveryUnsupportedError(); + } + if (current.resolution) { + if (!exactReplay(current.resolution, request, principal)) { + throw new ApprovedActionManualRecoveryFenceConflictError(); + } + await confirmAuthorization?.(); + return Object.freeze({ status: 'existing' as const, snapshot: current }); + } + if ( + current.execution.execution.version !== request.expectedExecutionVersion || + current.execution.execution.executionDigest !== + request.expectedExecutionDigest || + approvedActionExecutionEffectiveStatus( + current.execution.execution, + resolvedAtMs, + ) !== 'recovery_required' || + current.execution.execution.leaseOwner === null || + current.execution.execution.leaseToken === null + ) { + throw new ApprovedActionManualRecoveryFenceConflictError(); + } + const resolution = createResolution( + current.execution, + request, + principal, + policyFence, + resolvedAtMs, + ); + const nextExecution = completeApprovedActionExecution( + current.execution.execution, + { + owner: current.execution.execution.leaseOwner, + leaseToken: current.execution.execution.leaseToken, + expectedVersion: current.execution.execution.version, + resultMutationId: request.mutationId, + outcome: outcomeFor(request.decision), + resultCode: resultCodeFor(request.decision), + completedAtMs: resolvedAtMs, + }, + ); + await confirmAuthorization?.(); + try { + return await options.repository.resolve({ + previous: current.execution, + nextExecution, + resolution, + audit: allowedAudit( + request, + principal, + policyFence, + 'approval.recover.resolve', + resolvedAtMs, + ), + }); + } catch (error) { + if ( + error instanceof ApprovedActionManualRecoveryFenceConflictError || + error instanceof ApprovedActionManualRecoveryTargetUnavailableError + ) { + throw error; + } + throw new ApprovedActionManualRecoveryUnavailableError({ + cause: error instanceof Error ? error : undefined, + }); + } + }, + }); +} diff --git a/packages/ql3-runtime-core/test/approvedActionManualRecovery.test.cjs b/packages/ql3-runtime-core/test/approvedActionManualRecovery.test.cjs new file mode 100644 index 00000000..8d64097a --- /dev/null +++ b/packages/ql3-runtime-core/test/approvedActionManualRecovery.test.cjs @@ -0,0 +1,250 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + approvedActionDispatchDigest, + consumeApprovalRequest, + createApprovalRequest, + decideApprovalRequest, +} = require('@qinglong/runtime-core/approved-action'); +const { + claimApprovedActionExecution, + createApprovedActionExecution, + startApprovedActionExecution, +} = require('@qinglong/runtime-core/approved-action-execution'); +const { + ApprovedActionManualRecoveryAuthorizationError, + ApprovedActionManualRecoveryFenceConflictError, + ApprovedActionManualRecoveryUnsupportedError, + createApprovedActionManualRecoveryService, + normalizeApprovedActionManualRecoverySnapshot, +} = require('@qinglong/runtime-core/approved-action-manual-recovery'); + +const PRINCIPAL = Object.freeze({ + subject: Object.freeze({ type: 'user', id: 'owner-1' }), + authenticationId: 'oidc:session-1', + authenticatedAtMs: 100, + expiresAtMs: 20_000, + assurance: 'hardware', +}); + +function executing(actionType = 'plugin_package.secret_binding.bind') { + const action = { + permission: 'secret.manage', + actionType, + actionRef: 'secret-binding:1', + actionDigest: 'a'.repeat(64), + previewDigest: 'b'.repeat(64), + }; + const pending = createApprovalRequest({ + id: 'approval-1', + projectId: 'default', + action, + risk: 'high', + decisionMode: 'human_confirmation', + requestedBy: { type: 'agent', id: 'agent-1' }, + requestedAtMs: 800, + expiresAtMs: 10_000, + requestFence: { projectVersion: 1, bindingVersion: 2 }, + }); + const approved = decideApprovalRequest(pending, { + expectedVersion: 1, + decisionId: 'decision-1', + decision: 'approved', + reasonCode: 'reviewed', + principal: PRINCIPAL, + decidedAtMs: 900, + authorizationFence: { projectVersion: 1, bindingVersion: 2 }, + }); + const dispatch = consumeApprovalRequest(approved, { + expectedVersion: 2, + consumptionId: 'consumption-1', + dispatchId: 'dispatch-1', + action, + requestedBy: pending.requestedBy, + consumedBy: { type: 'system', id: 'package-executor' }, + consumedAtMs: 950, + authorizationFence: { projectVersion: 1, bindingVersion: 2 }, + }).dispatch; + const baseline = createApprovedActionExecution(dispatch); + const leased = claimApprovedActionExecution(baseline, { + owner: 'executor-1', + leaseToken: 'lease-1', + nowMs: 1_000, + leaseDurationMs: 500, + }); + const started = startApprovedActionExecution( + { dispatch, execution: leased }, + { + dispatchId: dispatch.id, + approvalRequestId: dispatch.approvalRequestId, + actionDigest: dispatch.action.actionDigest, + owner: leased.leaseOwner, + leaseToken: leased.leaseToken, + expectedVersion: leased.version, + startedAtMs: 1_100, + }, + ); + assert.equal(started.dispatchDigest, approvedActionDispatchDigest(dispatch)); + return Object.freeze({ dispatch, execution: started }); +} + +function harness(options = {}) { + let current = { + execution: options.execution ?? executing(), + resolution: null, + }; + const audits = []; + const calls = []; + const service = createApprovedActionManualRecoveryService({ + repository: { + async findByDispatchId() { + return current; + }, + async resolve(command) { + calls.push(command); + current = normalizeApprovedActionManualRecoverySnapshot({ + execution: { + dispatch: command.previous.dispatch, + execution: command.nextExecution, + }, + resolution: command.resolution, + }); + return { status: 'resolved', snapshot: current }; + }, + }, + policy: { + async authorize(_principal, _projectId, permission) { + assert.equal(permission, 'approval.recover'); + return options.denied + ? { effect: 'deny', reasons: ['permission_missing'], fence: null } + : { + effect: 'allow', + reasons: ['role_grant'], + fence: { projectVersion: 1, bindingVersion: 2 }, + }; + }, + }, + audit: { + async record(record) { + audits.push(record); + }, + }, + now: () => options.now ?? 2_000, + }); + return { service, audits, calls, current: () => current }; +} + +function inspectRequest() { + return { + projectId: 'default', + dispatchId: 'dispatch-1', + auditEventId: '10000000-0000-4000-8000-000000000001', + requestId: 'recover-inspect-1', + principal: PRINCIPAL, + }; +} + +function resolveRequest(decision = 'abandon_unknown') { + const snapshot = executing(); + return { + ...inspectRequest(), + auditEventId: '10000000-0000-4000-8000-000000000002', + requestId: 'recover-resolve-1', + expectedExecutionVersion: snapshot.execution.version, + expectedExecutionDigest: snapshot.execution.executionDigest, + mutationId: 'manual-recovery-1', + decision, + evidenceDigest: 'e'.repeat(64), + reasonCode: 'orphan_absence_verified', + }; +} + +test('inspects only after strong authorization and writes a bounded audit', async () => { + const { service, audits } = harness(); + let confirmed = 0; + const result = await service.inspect(inspectRequest(), () => { + confirmed += 1; + }); + assert.equal(result.execution.execution.status, 'executing'); + assert.equal(result.resolution, null); + assert.equal(confirmed, 1); + assert.equal(audits.length, 1); + assert.equal(audits[0].operationId, 'approval.recover.inspect'); + assert.deepEqual(audits[0].reasons, [ + 'role_grant', + 'strong_authentication', + 'manual_recovery', + ]); +}); + +test('abandons an expired executing action as blocked with immutable evidence', async () => { + const { service, calls, current } = harness(); + const result = await service.resolve(resolveRequest()); + assert.equal(result.status, 'resolved'); + assert.equal(result.snapshot.execution.execution.status, 'blocked'); + assert.equal( + result.snapshot.execution.execution.resultCode, + 'manual_recovery_abandoned_unknown', + ); + assert.equal(result.snapshot.resolution.evidenceDigest, 'e'.repeat(64)); + assert.equal(result.snapshot.resolution.decision, 'abandon_unknown'); + assert.equal(calls[0].audit.operationId, 'approval.recover.resolve'); + assert.equal(current().resolution.resolvedBy.id, 'owner-1'); +}); + +test('confirms a verified no-effect execution as failed and replays exactly', async () => { + const { service } = harness(); + const request = resolveRequest('confirm_failed'); + const first = await service.resolve(request); + const replay = await service.resolve(request); + assert.equal(first.snapshot.execution.execution.status, 'failed'); + assert.equal(replay.status, 'existing'); + assert.equal( + replay.snapshot.execution.execution.resultCode, + 'manual_recovery_confirmed_failed', + ); + await assert.rejects( + service.resolve({ ...request, evidenceDigest: 'f'.repeat(64) }), + ApprovedActionManualRecoveryFenceConflictError, + ); +}); + +test('rejects live leases, stale fences, unsupported actions and weak Users', async () => { + const live = executing(); + const { service: liveService } = harness({ execution: live, now: 1_200 }); + await assert.rejects( + liveService.resolve({ + ...resolveRequest(), + expectedExecutionVersion: live.execution.version, + expectedExecutionDigest: live.execution.executionDigest, + }), + ApprovedActionManualRecoveryFenceConflictError, + ); + + const { service: unsupported } = harness({ + execution: executing('tool.invoke'), + }); + await assert.rejects( + unsupported.inspect(inspectRequest()), + ApprovedActionManualRecoveryUnsupportedError, + ); + + const { service: weak } = harness(); + await assert.rejects( + weak.inspect({ + ...inspectRequest(), + principal: { ...PRINCIPAL, assurance: 'single_factor' }, + }), + ApprovedActionManualRecoveryAuthorizationError, + ); +}); + +test('rejects missing approval.recover permission before storage mutation', async () => { + const { service, calls } = harness({ denied: true }); + await assert.rejects( + service.resolve(resolveRequest()), + ApprovedActionManualRecoveryAuthorizationError, + ); + assert.equal(calls.length, 0); +}); diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index 690c460c..7f69690c 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -299,10 +299,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: runtimeCore.rootSourceFileRoles, }, { - sourceFiles: 159, + sourceFiles: 160, rootSourceFiles: 1, rootSourceLines: 160, - nestedSourceFiles: 158, + nestedSourceFiles: 159, rootSourceFileRoles: { 'index.ts': 'public_export' }, }, ); @@ -421,10 +421,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: clusterPostgres.rootSourceFileRoles, }, { - sourceFiles: 165, + sourceFiles: 167, rootSourceFiles: 1, rootSourceLines: 126, - nestedSourceFiles: 164, + nestedSourceFiles: 166, rootSourceFileRoles: { 'index.ts': 'public_export' }, }, );