From f77b19e61594833d8f68695a4f68ed7332d693ce Mon Sep 17 00:00:00 2001 From: whyour Date: Sun, 16 Aug 2026 00:41:34 +0800 Subject: [PATCH] feat(ql3): expose copilot diagnosis read model --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 5 +- ...14-copilot-failure-diagnosis-read-model.md | 116 +++ packages/ql3-ai/package.json | 5 + .../failure-diagnosis/read-model/service.ts | 666 ++++++++++++++++++ .../failure-diagnosis-read-model.test.cjs | 404 +++++++++++ packages/ql3-cluster-control/package.json | 5 + .../aiProductionApplication.ts | 58 +- .../failureDiagnosisReadComposition.ts | 57 ++ .../productionApplication.ts | 27 +- .../failureDiagnosisReadRoutes.ts | 436 ++++++++++++ .../test/aiProductionApplication.test.cjs | 99 ++- ...copilotFailureDiagnosisReadRoutes.test.cjs | 288 ++++++++ .../test/productionApplication.test.cjs | 57 +- scripts/ql3-cluster-dependency-audit.cjs | 7 + scripts/ql3-postgres-ha-contract.cjs | 336 ++++++++- test/back/ql3PackageBoundaryAudit.test.cjs | 8 +- 16 files changed, 2526 insertions(+), 48 deletions(-) create mode 100644 docs/adr/ADR-0414-copilot-failure-diagnosis-read-model.md create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/read-model/service.ts create mode 100644 packages/ql3-ai/test/failure-diagnosis-read-model.test.cjs create mode 100644 packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisReadComposition.ts create mode 100644 packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisReadRoutes.ts create mode 100644 packages/ql3-cluster-control/test/copilotFailureDiagnosisReadRoutes.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 526f7360..53890df5 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -6,11 +6,12 @@ - 目标版本：QingLong 3.x - 作者：QingLong Maintainers - 创建日期：2026-07-17 -- 最后更新：2026-08-15 +- 最后更新：2026-08-16 - 讨论范围：架构与演进路线，不包含最终 UI 视觉方案 -最新增量证据（2026-08-15）： +最新增量证据（2026-08-16）： +- D-322/ADR-0414（已接受）：Cluster Copilot failure diagnosis 已补齐 request-keyed 产品读模型。`GET /api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}` 只需 `run.read`，投影 running/terminal、取消/失败 stage/reason、authoritative admission/finalization time 与 durable usage/settled cost；同路径 `/output` 独立要求 `artifact.read`，只在 admission plan/receipt、Project/source Run、finalization、invocation 和 encrypted Artifact 全部 exact binding 且 current Project Policy 再授权后解析 historical key。调用者不能提交 Artifact、diagnosis Run、Model invocation、provider/model、价格、usage、outcome 或 key identity；deny/approval/absent/cross-target 统一 404，存储/Policy/key/decrypt/cipher 冲突统一 503。写 `capability` 与 `readCapability` 接口隔离，production 复用既有 AI PostgreSQL Pool、repository、Project Policy 与 projected output keyring，不新增 package、进程、端口、Pool、timer、watcher、queue、cache、Pod、Service 或 Kubernetes 权限；普通 Cluster Control、Edge/Standalone 均无这些 route。workspace 保持 18 package、`singleSourcePackages=[]`、`shallowSourcePackages=[]`，AI 193 个源码中 192 个、Cluster Control 62 个源码中 60 个位于嵌套领域目录。AI 249 pass/3 条件 skip、Cluster Control 256 pass/2 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计和 14 档 Local artifact 全部通过；默认 Edge/Standalone 仍为 2,589,890/2,589,968 bytes，Edge/Standalone AI 为 3,069,143/3,069,233 bytes。PostgreSQL 18.6 arm64 physical HA 139/139、timeline `1→2`，成功密文输出在 standby 可读，提升后 request-keyed exact replay 且 provider 调用为零；报告 SHA-256 为 `22decb54cfb8735bf787fe0665c877c201fc7b44d3c3de16fdbfdab31b7ac2cd`，离线审计零 finding。下一独立 Gate 可评审取消 mutation、CLI/UI/MCP 客户端或真实 Provider，不得把它们混入读边界。 - D-321/ADR-0413（已接受）：Cluster Copilot failure diagnosis 已开放唯一、默认关闭的产品写入口 `POST /api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses`。该 route 只在显式 AI 进程且 `QL3_CLUSTER_AI_COPILOT_ENABLED=true` 时注入既有 Cluster Control route registry，完整复用同一认证器、`model.invoke` Project Policy、fail-closed 同步安全审计、HTTP/TLS body/response/concurrency 上限和生命周期；内部只读 Tool 仍独立复验其 exact `tool.call:*` Policy。耐久幂等身份直接绑定 `x-request-id`，body 只允许 schema 与 `traceId`，Project/source Run/principal 来自 canonical path 和认证，Attempt、日志范围、Tool、provider/model、预算、deadline、Policy fence、reason/outcome 均由服务端从数据库与只读配置派生。响应只投影 created/existing、source/diagnosis Run、终态 stage/reason/outcome 和加密输出 Artifact id/digest，不返回日志、Tool/Model plaintext、prompt、模型信息、密钥或内部异常。普通 Cluster Control、Edge、Standalone 和未启用 Copilot 的 AI 进程没有 route，也没有新增 package、进程、监听器、Pool/连接、timer、watcher、队列、cache、Pod、Service 或 Kubernetes 权限；workspace 仍为 18 package、无单文件/浅平 package，新增源码位于既有嵌套 Copilot 领域目录。Cluster Control 250 pass/2 条件 skip，18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计和 14 档 Local artifact 全部通过；默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes。PostgreSQL 18.6 arm64 HA 137/137、timeline `1→2`，报告 SHA-256 为 `0a12b5c1102555823d43b5a93dd7868b98b194b491840a5242bab6fa2da26123`，离线审计零 finding。本 Gate 不改 migration/schema/role/SQL/HA 拓扑。下一 Gate 优先补同一 Policy 下的加密诊断输出读取与费用/取消可观测性；CLI/UI/MCP 只能复用该 API，不得直连 application capability。 - D-320/ADR-0412（已接受）：Cluster Copilot failure diagnosis 已补齐产品入口前的非成功收敛边界。新增的 pre-Model terminalizer 只从 durable Tool failure、受审日志 projection 或数据库观察到的 deadline/cancellation 派生封闭 reason，在 Model start 不存在时以一个 SERIALIZABLE 事务原子提交 StepRun mutations、RunEvents、父 Run 终态和 `pg-9021` append-only content-free receipt；Tool `failed|timed_out`、日志 `not_found|pending|missing|retired`、五秒 Tool budget 不足、deadline 和 cancellation 均可 exact replay。Model 已开始后的 `outcome_unknown` 继续要求强 User 显式 `fail|cancel|retry` resolution；Copilot finalizer 现在对 `fail|cancel` 精确校验 resolution mutation 的 resolved Step digest，既不伪造失败也不自动重试 Provider。实现复用既有 AI package、Cluster AI 进程、Pool 与 ledger，不增加 package、进程、连接、timer、watcher、队列、cache 或产品 route；workspace 仍为 18 package、无单文件/浅平 package，AI 192 个源码中 191 个位于嵌套领域目录。AI 完整测试 244 pass/3 条件 skip，18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计和 14 档 Local artifact 全部通过。PostgreSQL 18.6 arm64 HA 137/137、timeline `1→2`，覆盖日志不可用 pre-Model terminalization、unknown completion→人工 fail resolution→Run finalization 及晋升后零外部副作用 exact replay；报告 SHA-256 为 `6eaeb20615a62d153c5a69687344f41f31351c6ecf111cfb9cbafad115538c83`，离线审计零 finding。下一 Gate 才增加认证、Policy、audit 与 source fence 保护的 Cluster API，并由 CLI/UI/MCP 复用，仍不得建立旁路执行器。 - D-319/ADR-0411（已接受）：Cluster Copilot failure diagnosis 已在既有 `ql3-cluster-control-ai` 进程内完成默认关闭的 production composition。Prompt 与 Copilot 共享同一个 PostgreSQL AI Pool、Model Gateway、Provider client、恢复扫描、quota/pricing ledger 与 `maxConcurrent` 预算；有界 successful-completion router 只向声明 exact invocation 的 durable sink 分发，不复制 Gateway、连接或隐藏队列。application service 只接受 source Run/request identity，从数据库当前 Run、latest Attempt、Project Tool snapshot、Policy 和 canonical read-only egress config 派生计划，先 admission，再以 durable plan、historical key 与确定性 nonce 修复 admission→Artifact crash window，随后复用 Trusted Tool、Worker Artifact range reader、Model Gateway 及独立的 invocation/result/model-output 三域 keyring。能力保持 caller-driven，不增加 timer、watcher、队列、HTTP/CLI/UI/MCP route 或 Kubernetes API 权限；Kubernetes 独立可选 component 仅投影 config 与三个 0440 keyring。最终 AI 238 pass/3 条件 skip、Cluster Control 240 pass/2 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计、14 档 Local artifact 全部通过；workspace 仍为 18 package、无单文件/浅平 package，AI 187 个源码中 186 个、Cluster Control 59 个中 57 个位于嵌套领域目录。默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes，Edge/Standalone AI 为 3,064,454/3,064,544 bytes，证明 Cluster-only composition 未进入小设备闭包。PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`，报告 SHA-256 为 `981299b454dce5541e9596450b85816dc40559cba8dc42adf3d5fea571c3d3a6`；本 Gate 不改 migration/schema/role/SQL/HA 拓扑。下一 Gate 是 Tool failure、日志 missing/retired/pending、deadline/cancel 与 Model outcome-unknown 的 durable terminalization/recovery，完成前不开放产品入口。 diff --git a/docs/adr/ADR-0414-copilot-failure-diagnosis-read-model.md b/docs/adr/ADR-0414-copilot-failure-diagnosis-read-model.md new file mode 100644 index 00000000..b9a87f12 --- /dev/null +++ b/docs/adr/ADR-0414-copilot-failure-diagnosis-read-model.md @@ -0,0 +1,116 @@ +# ADR-0414：Copilot 故障诊断请求键读模型 + +- 状态：Accepted +- 日期：2026-08-16 +- 关联 RFC：QL-RFC-0001 D-322、Phase 2 +- 关联 ADR：ADR-0087、ADR-0409、ADR-0410、ADR-0411、ADR-0412、ADR-0413 + +## 问题 + +ADR-0413 已提供默认关闭的故障诊断写入口，但响应只返回低敏终态和加密输出 Artifact 引用。 +部署者仍缺少两个产品能力：在连接中断或重放后按原请求查看诊断状态、取消事实和实际费用，以及在 +具备 Artifact 读取权限时取得诊断正文。若客户端直接提交 Artifact、diagnosis Run、Model invocation、 +provider/model 或价格信息，或者 CLI/UI 直连数据库与 application service，就会扩大枚举、越权、 +密钥、计费和部署边界。 + +低配路由设备与集群节点仍必须使用同一产品契约但保持不同闭包：默认 Edge/Standalone 不能加载 +Cluster Copilot；显式 AI Cluster 也不能因为读模型增加进程、端口、连接池、后台扫描或新 package。 + +## 决策 + +1. 增加两个 caller-driven、request-keyed 的只读端点： + - `GET /api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}`， + operation `copilot.failure_diagnosis.read`，permission `run.read`； + - 同一路径追加 `/output`，operation `copilot.failure_diagnosis.output.read`，permission + `artifact.read`。 + `runId` 始终表示 source Run；`requestId` 必须复用创建诊断时的 HTTP `x-request-id`。 +2. 调用者不得提交 body、Artifact id/digest、diagnosis Run、Step、invocation、provider/model、 + price revision、usage、terminal outcome、Policy fence、key id 或密钥。服务从 admission plan、 + pre-Model terminalization、Model finalization、usage ledger、price settlement 与 encrypted output + Artifact 派生全部事实。 +3. 状态读取只投影 request/source/diagnosis Run identity、`running|terminal`、终态 + outcome/stage/reason、output availability、timestamps，以及该次 invocation 已耐久结算的 + input/output/total tokens、USD `costMicros` 或明确的 unknown。不得返回日志、Tool output、 + prompt、diagnosis 正文、provider/model、单价、price revision、Policy reasons、内部错误或密钥信息。 +4. 输出读取只有在 exact admission、Project/source Run、finalization、Artifact 与 invocation 绑定全部 + 一致时才解析 key。服务在 transport `artifact.read` 之后再次使用当前 Project Policy 对认证 + principal 授权；deny、approval、absent、cross-Project、cross-Run 和不匹配统一返回 `not_found`。 + 存储/Policy/key/decrypt 异常统一为 unavailable，不向调用者泄露存在性或内部 message。 +5. 输出正文只返回 reference 的低敏 identity/digest/size/sealed time，以及 `text`、finish reason 和 + usage；provider/model、key id、egress evidence、nonce、ciphertext、auth tag 与 plaintext buffer + 都不进入 HTTP 响应。resolved key 和所有临时明密文 buffer 必须在成功或失败路径擦除。 +6. 状态读取不估算费用，也不从当前 catalog 反推历史价格。只有 invocation usage ledger 的耐久 + token facts 可见；只有 exact price settlement 与 usage 一致时返回 `currency=USD` 和 + `costMicros`，否则返回 `currency=null,costMicros=null`。pre-Model 终态固定为零 token、零费用， + 因为未发生 Model invocation；Model 路径在 settlement 尚未形成时明确为 unknown。 +7. `running` 只表示已有 admission 且尚无任一耐久终态，不推断进程存活。取消仅投影既有 + pre-Model/finalization durable fact；本 Gate 不新增取消 mutation API，也不修改 source Run。 +8. 两个 route 仅随 ADR-0413 的显式 AI Copilot 配置一起注入既有 Cluster Control route registry， + 但写能力与读能力保持接口隔离：`capability` 只执行诊断，独立 `readCapability` 只提供 + `inspect/readOutput`。两者复用同一认证、Project Policy、同步安全审计、TLS、限流、body/response + 上限和 lifecycle。普通 Cluster Control、Edge、Standalone 与未启用 Copilot 的 AI 进程仍没有 + 这些 route。 +9. 实现放在既有 `ql3-ai/src/copilot/failure-diagnosis/read-model` 与 + `ql3-cluster-control/src/copilot/failure-diagnosis` 内部目录，允许增加显式 package export subpath， + 但不得增加 workspace package。production composition 复用 ADR-0411 已建立的 admission/model/ + terminalization repository、Project Policy、output keyring 和同一个 AI PostgreSQL Pool；admission + receipt 提供 authoritative `admittedAtMs`，plan 只提供 immutable execution binding。不得新增 + Pool/连接预算、进程、listener、timer、watcher、queue、cache、Pod、Service 或 Kubernetes 权限。 +10. CLI、UI 与 MCP 后续只能调用这两个 HTTP API；不得读取 `ql3_ai` 表、Projected keyring 或直接 + import production/application capability。 + +## 状态投影 + +| 耐久事实 | 状态 | stage/reason | usage/cost | +| --- | --- | --- | --- | +| admission，尚无终态 | `running` | `null/null` | `null` | +| pre-Model terminalization | `terminal` | receipt stage/reason | 零 token、USD 零费用 | +| Model finalization | `terminal` | `model/null` | ledger 可缺失；存在时必须 exact，价格未结算时 cost unknown | + +同时出现两个终态、终态与 plan/invocation 不一致、usage/settlement 不一致或成功终态缺少 exact output +均视为存储 authority 冲突并失败关闭，不向 HTTP 暴露哪项事实异常。 + +## 被否决方案 + +1. **按 Artifact id 直接读取**：要求调用者保存内部引用，并扩大跨 Project 枚举面。 +2. **一个 `run.read` 端点同时返回正文**：把低敏运行可观测性升级为 Artifact 内容读取。 +3. **返回 provider/model、单价和完整 settlement**：泄露平台配置，且不是诊断消费所必需。 +4. **用当前价格目录估算历史费用**：无法证明历史 revision，重放结果会漂移。 +5. **新增 Copilot query service/package/数据库 Pool**：没有独立部署边界，并增加小设备和集群资源成本。 +6. **后台物化查询表或轮询终态**：现有 request-keyed index 与 durable receipts 已能有界点查。 +7. **本 Gate 同时增加取消写入口**：会引入新的 mutation、幂等和恢复协议，应单独评审。 + +## 验证标准 + +1. AI 单元测试覆盖 request/project/source binding、running、pre-Model cancel/failure、Model success/ + failure、usage/price exactness、双终态冲突、deny/absence/cross-target masking、未知 key、tamper、 + 解密成功和 key 擦除。 +2. route 测试覆盖无 body、path validation、`run.read` 与 `artifact.read` 分权、响应脱敏、404 masking、 + 503 fail-closed，以及拒绝路径零 capability call。 +3. production composition 测试证明三条 Copilot route 只随显式 AI capability 一起注入，复用同一 + Pool/repository/key authority；默认 route allowlist 和普通 Cluster Control 不变。 +4. package/dependency/import/deployment 审计证明 workspace package、默认 Edge/Standalone 闭包、 + 进程、端口、Pool、Pod、Service 与 Kubernetes 权限无增长。 +5. 18-package clean build/test、完整 backend、Local artifact、PostgreSQL 18 physical HA 与 + GitNexus staged/change detection 全通过后才允许 D-322 阶段性提交。 + +## 接受证据 + +- AI：252 tests，249 pass、3 条件 skip、0 fail；Cluster Control：258 tests，256 pass、2 条件 + skip、0 fail。新增覆盖 partial admission authority、双终态冲突、跨 Project/source masking、 + current Policy deny、usage/settlement 一致性、密文 tamper、解密成功和 key 擦除。 +- 18-package clean build/test 退出 0；backend 1,209 tests，1,207 pass、2 条件 skip、0 fail。 + package boundary 保持 18 个 package、`singleSourcePackages=[]`、`shallowSourcePackages=[]`；AI + 193 个源码中 192 个、Cluster Control 62 个源码中 60 个位于嵌套领域目录。 +- edge import、cluster dependency、package boundary、cluster deployment 四项审计均无 finding。 + 14 档 Local artifact 全通过；默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes， + Edge/Standalone AI 为 3,069,143/3,069,233 bytes。 +- PostgreSQL 18.6 arm64 physical HA 139/139、timeline `1→2`；成功 Model output 在 standby 解密读取， + 提升后按 request id exact replay 且 provider 调用为零。owner-private 报告 SHA-256 为 + `22decb54cfb8735bf787fe0665c877c201fc7b44d3c3de16fdbfdab31b7ac2cd`，独立离线审计 + `compatible=true/findings=[]`。 + +## 后续门禁 + +本 Gate 不提供诊断列表、取消 mutation、unknown outcome 人工裁决、外部 Provider 实测、CLI、UI +或 MCP。真实 Provider、双副本并发/故障注入和产品客户端必须继续保持独立 Gate。 diff --git a/packages/ql3-ai/package.json b/packages/ql3-ai/package.json index d1858fe5..732b0248 100644 --- a/packages/ql3-ai/package.json +++ b/packages/ql3-ai/package.json @@ -60,6 +60,11 @@ "require": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js", "default": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js" }, + "./failure-diagnosis-read-model": { + "types": "./dist/copilot/failure-diagnosis/read-model/service.d.ts", + "require": "./dist/copilot/failure-diagnosis/read-model/service.js", + "default": "./dist/copilot/failure-diagnosis/read-model/service.js" + }, "./failure-diagnosis-pre-model-terminalization": { "types": "./dist/copilot/failure-diagnosis/preModelTerminalization.d.ts", "require": "./dist/copilot/failure-diagnosis/preModelTerminalization.js", diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/read-model/service.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/read-model/service.ts new file mode 100644 index 00000000..007501af --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/read-model/service.ts @@ -0,0 +1,666 @@ +import { + normalizeSecurityPrincipal, + type SecurityPolicyDecision, + type SecurityPrincipal, +} from '@qinglong/runtime-core/security'; +import type { ProjectPermission } from '@qinglong/runtime-core/project-policy'; + +import type { ModelInvocationPriceSettlement } from '../../../pricing/pricing'; +import type { ModelInvocationUsageLedgerRecord } from '../../../usage/usageLedger'; +import type { GenerateResult } from '../../../model-gateway/model'; +import type { + CopilotFailureDiagnosisAdmissionRepository, + CopilotFailureDiagnosisAdmissionReceipt, + CopilotFailureDiagnosisExecutionPlan, +} from '../admission/contracts'; +import type { + CopilotFailureDiagnosisFinalizationReceipt, + CopilotFailureDiagnosisFinalizationRepository, +} from '../model-execution/finalization'; +import { + openCopilotFailureDiagnosisOutputArtifact, + type CopilotFailureDiagnosisOutputArtifact, + type CopilotFailureDiagnosisOutputKeyProvider, +} from '../model-execution/outputArtifact'; +import type { CopilotFailureDiagnosisOutputCompletionRepository } from '../model-execution/completion'; +import type { + CopilotFailureDiagnosisPreModelTerminalizationReceipt, + CopilotFailureDiagnosisPreModelTerminalizationRepository, +} from '../terminalization/contracts'; + +export const COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA = + 'qinglong/copilot-failure-diagnosis-inspection-result@v1' as const; +export const COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA = + 'qinglong/copilot-failure-diagnosis-output-read-result@v1' as const; + +export interface CopilotFailureDiagnosisReadTarget { + readonly principal: Readonly; + readonly projectId: string; + readonly sourceRunId: string; + readonly requestId: string; +} + +export interface CopilotFailureDiagnosisReadAuthorizer { + authorize( + principal: Readonly, + projectId: string, + permission: ProjectPermission, + ): Promise>; +} + +export interface CopilotFailureDiagnosisReadModelRepository + extends Pick< + CopilotFailureDiagnosisOutputCompletionRepository, + 'findCopilotFailureDiagnosisOutput' + > { + findUsage( + invocationId: string, + ): Promise | null>; + findPriceSettlement( + invocationId: string, + ): Promise | null>; +} + +export interface CopilotFailureDiagnosisUsageView { + readonly inputTokens: number; + readonly outputTokens: number; + readonly totalTokens: number; + readonly currency: 'USD' | null; + readonly costMicros: number | null; +} + +export type CopilotFailureDiagnosisInspectionResult = Readonly< + | { + schema: typeof COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA; + status: 'not_found'; + projectId: string; + sourceRunId: string; + requestId: string; + } + | { + schema: typeof COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA; + status: 'running' | 'terminal'; + projectId: string; + sourceRunId: string; + requestId: string; + diagnosisRunId: string; + outcome: 'succeeded' | 'failed' | 'timed_out' | 'cancelled' | null; + stage: 'model' | 'tool' | 'log' | 'deadline' | 'cancellation' | null; + reason: string | null; + outputAvailable: boolean; + admittedAtMs: number; + finalizedAtMs: number | null; + usage: Readonly | null; + } +>; + +export type CopilotFailureDiagnosisOutputReadResult = Readonly< + | { + schema: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA; + status: 'not_found'; + projectId: string; + sourceRunId: string; + requestId: string; + } + | { + schema: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA; + status: 'available'; + projectId: string; + sourceRunId: string; + requestId: string; + diagnosisRunId: string; + reference: Readonly<{ + artifactId: string; + artifactDigest: string; + contentDigest: string; + outputBytes: number; + sealedAtMs: number; + }>; + result: Readonly>; + } +>; + +export class InvalidCopilotFailureDiagnosisReadRequestError extends TypeError { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_READ_INVALID'; + + constructor() { + super('Copilot failure diagnosis read request is invalid'); + this.name = 'InvalidCopilotFailureDiagnosisReadRequestError'; + } +} + +export class CopilotFailureDiagnosisReadUnavailableError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_READ_UNAVAILABLE'; + + constructor(options?: ErrorOptions) { + super('Copilot failure diagnosis read is unavailable', options); + this.name = 'CopilotFailureDiagnosisReadUnavailableError'; + } +} + +const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/; +const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/; +const DIGEST = /^[0-9a-f]{64}$/; +const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']); +const STAGES = new Set(['tool', 'log', 'deadline', 'cancellation']); +const REASONS = new Set([ + 'tool_failed', + 'tool_timed_out', + 'log_not_found', + 'log_pending', + 'log_missing', + 'log_retired', + 'tool_budget_exhausted', + 'deadline_exceeded', + 'cancellation_requested', +]); + +function unavailable( + cause?: unknown, +): CopilotFailureDiagnosisReadUnavailableError { + return new CopilotFailureDiagnosisReadUnavailableError({ + cause: cause instanceof Error ? cause : undefined, + }); +} + +function exactKeys(value: object, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + const canonical = [...expected].sort(); + return ( + actual.length === canonical.length && + actual.every((key, index) => key === canonical[index]) + ); +} + +function safeInteger(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} + +function normalizeTarget( + value: CopilotFailureDiagnosisReadTarget, + nowMs: number, +): Readonly { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + !exactKeys(value, ['principal', 'projectId', 'requestId', 'sourceRunId']) || + typeof value.projectId !== 'string' || + !IDENTITY.test(value.projectId) || + typeof value.sourceRunId !== 'string' || + !RUN_ID.test(value.sourceRunId) || + typeof value.requestId !== 'string' || + !IDENTITY.test(value.requestId) + ) { + throw new InvalidCopilotFailureDiagnosisReadRequestError(); + } + try { + return Object.freeze({ + principal: normalizeSecurityPrincipal(value.principal, nowMs), + projectId: value.projectId, + sourceRunId: value.sourceRunId, + requestId: value.requestId, + }); + } catch { + throw new InvalidCopilotFailureDiagnosisReadRequestError(); + } +} + +function inspectionNotFound( + target: Omit, +): CopilotFailureDiagnosisInspectionResult { + return Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA, + status: 'not_found' as const, + ...target, + }); +} + +function outputNotFound( + target: Omit, +): CopilotFailureDiagnosisOutputReadResult { + return Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA, + status: 'not_found' as const, + ...target, + }); +} + +function targetView( + command: Readonly, +): Readonly> { + return Object.freeze({ + projectId: command.projectId, + sourceRunId: command.sourceRunId, + requestId: command.requestId, + }); +} + +function planMatchesTarget( + plan: Readonly, + command: Readonly, +): boolean { + return ( + plan.requestId === command.requestId && + plan.projectId === command.projectId && + plan.source?.runId === command.sourceRunId + ); +} + +function validPlan( + plan: Readonly, +): boolean { + return ( + !!plan && + typeof plan === 'object' && + IDENTITY.test(plan.requestId) && + IDENTITY.test(plan.projectId) && + RUN_ID.test(plan.runId) && + RUN_ID.test(plan.source?.runId) && + IDENTITY.test(plan.modelStepRunId) && + IDENTITY.test(plan.modelInvocationId) && + DIGEST.test(plan.planDigest) && + safeInteger(plan.plannedAtMs) + ); +} + +function validTerminalization( + value: Readonly, + plan: Readonly, +): boolean { + return ( + !!value && + value.requestId === plan.requestId && + value.planDigest === plan.planDigest && + value.runId === plan.runId && + STAGES.has(value.stage) && + REASONS.has(value.reason) && + OUTCOMES.has(value.outcome) && + safeInteger(value.finalizedAtMs) + ); +} + +function validFinalization( + value: Readonly, + plan: Readonly, +): boolean { + return ( + !!value && + value.requestId === plan.requestId && + value.planDigest === plan.planDigest && + value.runId === plan.runId && + value.modelStepRunId === plan.modelStepRunId && + value.invocationId === plan.modelInvocationId && + DIGEST.test(value.completionDigest) && + OUTCOMES.has(value.outcome) && + (value.outputArtifactId === null || + IDENTITY.test(value.outputArtifactId)) && + (value.outcome === 'succeeded') === (value.outputArtifactId !== null) && + safeInteger(value.finalizedAtMs) + ); +} + +function validUsage( + usage: Readonly, + plan: Readonly, + finalization: Readonly, +): boolean { + return ( + usage.invocationId === plan.modelInvocationId && + usage.projectId === plan.projectId && + usage.runId === plan.runId && + usage.stepRunId === plan.modelStepRunId && + usage.traceId === plan.traceId && + usage.completionDigest === finalization.completionDigest && + usage.outcome === finalization.outcome && + safeInteger(usage.inputTokens) && + safeInteger(usage.outputTokens) && + safeInteger(usage.totalTokens) && + usage.totalTokens === usage.inputTokens + usage.outputTokens && + (usage.costMicros === null || safeInteger(usage.costMicros)) + ); +} + +function settlementMatches( + settlement: Readonly, + usage: Readonly, + finalization: Readonly, +): boolean { + return ( + settlement.invocationId === usage.invocationId && + settlement.projectId === usage.projectId && + settlement.completionDigest === finalization.completionDigest && + settlement.currency === 'USD' && + settlement.inputTokens === usage.inputTokens && + settlement.outputTokens === usage.outputTokens && + safeInteger(settlement.costMicros) && + usage.costMicros === settlement.costMicros + ); +} + +function validArtifact( + artifact: Readonly, + plan: Readonly, + finalization: Readonly, +): boolean { + return ( + artifact.artifactId === finalization.outputArtifactId && + artifact.requestId === plan.requestId && + artifact.planDigest === plan.planDigest && + artifact.projectId === plan.projectId && + artifact.runId === plan.runId && + artifact.stepRunId === plan.modelStepRunId && + artifact.invocationId === plan.modelInvocationId && + DIGEST.test(artifact.artifactDigest) && + DIGEST.test(artifact.contentDigest) && + safeInteger(artifact.outputBytes) && + safeInteger(artifact.sealedAtMs) + ); +} + +interface LocatedDiagnosis { + readonly plan: Readonly; + readonly admission: Readonly; + readonly terminalization: Readonly | null; + readonly finalization: Readonly | null; +} + +/** + * Request-keyed product read boundary. It resolves durable authority before + * current Policy, never accepts storage/model identities from the caller, and + * only resolves output key material after every binding is proven. + */ +export class CopilotFailureDiagnosisReadService { + readonly #admissions: Pick< + CopilotFailureDiagnosisAdmissionRepository, + 'findByRequestId' | 'findPlanByRequestId' + >; + readonly #terminalizations: Pick< + CopilotFailureDiagnosisPreModelTerminalizationRepository, + 'findByRequestId' + >; + readonly #finalizations: Pick< + CopilotFailureDiagnosisFinalizationRepository, + 'findFinalization' + >; + readonly #models: CopilotFailureDiagnosisReadModelRepository; + readonly #authorizer: CopilotFailureDiagnosisReadAuthorizer; + readonly #keys: CopilotFailureDiagnosisOutputKeyProvider; + readonly #now: () => number; + + constructor( + options: Readonly<{ + admissions: Pick< + CopilotFailureDiagnosisAdmissionRepository, + 'findByRequestId' | 'findPlanByRequestId' + >; + terminalizations: Pick< + CopilotFailureDiagnosisPreModelTerminalizationRepository, + 'findByRequestId' + >; + finalizations: Pick< + CopilotFailureDiagnosisFinalizationRepository, + 'findFinalization' + >; + models: CopilotFailureDiagnosisReadModelRepository; + authorizer: CopilotFailureDiagnosisReadAuthorizer; + keys: CopilotFailureDiagnosisOutputKeyProvider; + now?: () => number; + }>, + ) { + if ( + !options || + typeof options !== 'object' || + Array.isArray(options) || + typeof options.admissions?.findPlanByRequestId !== 'function' || + typeof options.admissions?.findByRequestId !== 'function' || + typeof options.terminalizations?.findByRequestId !== 'function' || + typeof options.finalizations?.findFinalization !== 'function' || + typeof options.models?.findCopilotFailureDiagnosisOutput !== 'function' || + typeof options.models?.findUsage !== 'function' || + typeof options.models?.findPriceSettlement !== 'function' || + typeof options.authorizer?.authorize !== 'function' || + typeof options.keys?.resolve !== 'function' || + (options.now !== undefined && typeof options.now !== 'function') + ) { + throw unavailable(); + } + this.#admissions = options.admissions; + this.#terminalizations = options.terminalizations; + this.#finalizations = options.finalizations; + this.#models = options.models; + this.#authorizer = options.authorizer; + this.#keys = options.keys; + this.#now = options.now ?? Date.now; + } + + async #locate( + command: Readonly, + permission: ProjectPermission, + ): Promise | null> { + let plan; + let admission; + try { + [plan, admission] = await Promise.all([ + this.#admissions.findPlanByRequestId(command.requestId), + this.#admissions.findByRequestId(command.requestId), + ]); + } catch (cause) { + throw unavailable(cause); + } + if (plan === null && admission === null) return null; + if (plan === null || admission === null) throw unavailable(); + if (!validPlan(plan)) throw unavailable(); + if (!planMatchesTarget(plan, command)) return null; + if ( + admission.requestId !== plan.requestId || + admission.planDigest !== plan.planDigest || + admission.runId !== plan.runId || + admission.sourceRunId !== plan.source.runId || + !safeInteger(admission.admittedAtMs) + ) { + throw unavailable(); + } + let decision; + try { + decision = await this.#authorizer.authorize( + command.principal, + command.projectId, + permission, + ); + } catch (cause) { + throw unavailable(cause); + } + if ( + !decision || + typeof decision !== 'object' || + !['allow', 'deny', 'require_approval'].includes(decision.effect) + ) { + throw unavailable(); + } + if (decision.effect !== 'allow') return null; + let terminalization; + let finalization; + try { + [terminalization, finalization] = await Promise.all([ + this.#terminalizations.findByRequestId(command.requestId), + this.#finalizations.findFinalization(command.requestId), + ]); + } catch (cause) { + throw unavailable(cause); + } + if (terminalization && finalization) throw unavailable(); + if (terminalization && !validTerminalization(terminalization, plan)) { + throw unavailable(); + } + if (finalization && !validFinalization(finalization, plan)) { + throw unavailable(); + } + return Object.freeze({ plan, admission, terminalization, finalization }); + } + + async inspect( + value: CopilotFailureDiagnosisReadTarget, + ): Promise { + const nowMs = this.#now(); + if (!safeInteger(nowMs)) throw unavailable(); + const command = normalizeTarget(value, nowMs); + const target = targetView(command); + const located = await this.#locate(command, 'run.read'); + if (!located) return inspectionNotFound(target); + const { plan, admission, terminalization, finalization } = located; + if (!terminalization && !finalization) { + return Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA, + status: 'running' as const, + ...target, + diagnosisRunId: plan.runId, + outcome: null, + stage: null, + reason: null, + outputAvailable: false, + admittedAtMs: admission.admittedAtMs, + finalizedAtMs: null, + usage: null, + }); + } + if (terminalization) { + return Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA, + status: 'terminal' as const, + ...target, + diagnosisRunId: plan.runId, + outcome: terminalization.outcome, + stage: terminalization.stage, + reason: terminalization.reason, + outputAvailable: false, + admittedAtMs: admission.admittedAtMs, + finalizedAtMs: terminalization.finalizedAtMs, + usage: Object.freeze({ + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + currency: 'USD' as const, + costMicros: 0, + }), + }); + } + const finalized = finalization!; + let usage; + let settlement; + try { + [usage, settlement] = await Promise.all([ + this.#models.findUsage(plan.modelInvocationId), + this.#models.findPriceSettlement(plan.modelInvocationId), + ]); + } catch (cause) { + throw unavailable(cause); + } + if (usage && !validUsage(usage, plan, finalized)) throw unavailable(); + if ( + settlement && + (!usage || !settlementMatches(settlement, usage, finalized)) + ) { + throw unavailable(); + } + return Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA, + status: 'terminal' as const, + ...target, + diagnosisRunId: plan.runId, + outcome: finalized.outcome, + stage: 'model' as const, + reason: null, + outputAvailable: finalized.outputArtifactId !== null, + admittedAtMs: admission.admittedAtMs, + finalizedAtMs: finalized.finalizedAtMs, + usage: + usage === null + ? null + : Object.freeze({ + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + totalTokens: usage.totalTokens, + currency: settlement ? ('USD' as const) : null, + costMicros: settlement?.costMicros ?? null, + }), + }); + } + + async readOutput( + value: CopilotFailureDiagnosisReadTarget, + ): Promise { + const nowMs = this.#now(); + if (!safeInteger(nowMs)) throw unavailable(); + const command = normalizeTarget(value, nowMs); + const target = targetView(command); + const located = await this.#locate(command, 'artifact.read'); + if (!located || !located.finalization) return outputNotFound(target); + const { plan, finalization } = located; + const outputArtifactId = finalization.outputArtifactId; + if (outputArtifactId === null) return outputNotFound(target); + let artifact; + try { + artifact = await this.#models.findCopilotFailureDiagnosisOutput( + outputArtifactId, + ); + } catch (cause) { + throw unavailable(cause); + } + if (!artifact) return outputNotFound(target); + if (!validArtifact(artifact, plan, finalization)) throw unavailable(); + let material; + try { + material = await this.#keys.resolve(artifact.keyId); + } catch (cause) { + throw unavailable(cause); + } + if ( + !material || + typeof material !== 'object' || + Array.isArray(material) || + material.keyId !== artifact.keyId || + !(material.key instanceof Uint8Array) || + material.key.byteLength !== 32 + ) { + try { + material?.key?.fill(0); + } catch { + // Invalid key material remains unavailable and must not escape. + } + throw unavailable(); + } + try { + const opened = openCopilotFailureDiagnosisOutputArtifact( + artifact, + material.key, + ); + return Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA, + status: 'available' as const, + ...target, + diagnosisRunId: plan.runId, + reference: Object.freeze({ + artifactId: artifact.artifactId, + artifactDigest: artifact.artifactDigest, + contentDigest: artifact.contentDigest, + outputBytes: artifact.outputBytes, + sealedAtMs: artifact.sealedAtMs, + }), + result: Object.freeze({ + text: opened.text, + finishReason: opened.finishReason, + usage: Object.freeze({ ...opened.usage }), + }), + }); + } catch (cause) { + throw unavailable(cause); + } finally { + try { + material.key.fill(0); + } catch (cause) { + throw unavailable(cause); + } + } + } +} diff --git a/packages/ql3-ai/test/failure-diagnosis-read-model.test.cjs b/packages/ql3-ai/test/failure-diagnosis-read-model.test.cjs new file mode 100644 index 00000000..e8b2eefb --- /dev/null +++ b/packages/ql3-ai/test/failure-diagnosis-read-model.test.cjs @@ -0,0 +1,404 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA, + COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA, + CopilotFailureDiagnosisReadService, + CopilotFailureDiagnosisReadUnavailableError, +} = require('@qinglong/ai/failure-diagnosis-read-model'); +const { + createCopilotFailureDiagnosisOutputArtifact, +} = require('@qinglong/ai/failure-diagnosis-model-execution'); + +const DIGEST = 'a'.repeat(64); +const COMPLETION_DIGEST = 'b'.repeat(64); + +function principal() { + return { + subject: { type: 'api_app', id: 'app-1' }, + authenticationId: 'credential-1', + authenticatedAtMs: 10, + expiresAtMs: 10_000, + assurance: 'service', + }; +} + +function target(overrides = {}) { + return { + principal: principal(), + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'diagnosis-request-1', + ...overrides, + }; +} + +function plan(overrides = {}) { + return { + requestId: 'diagnosis-request-1', + projectId: 'project-1', + source: { runId: 'source-run-1' }, + runId: 'diagnosis-run-1', + modelStepRunId: 'model-step-1', + modelInvocationId: 'model-invocation-1', + traceId: 'trace-1', + planDigest: DIGEST, + plannedAtMs: 100, + ...overrides, + }; +} + +function admission(overrides = {}) { + return { + requestId: 'diagnosis-request-1', + planDigest: DIGEST, + runId: 'diagnosis-run-1', + sourceRunId: 'source-run-1', + admittedAtMs: 110, + ...overrides, + }; +} + +function finalization(overrides = {}) { + return { + requestId: 'diagnosis-request-1', + planDigest: DIGEST, + runId: 'diagnosis-run-1', + modelStepRunId: 'model-step-1', + invocationId: 'model-invocation-1', + completionDigest: COMPLETION_DIGEST, + outcome: 'succeeded', + outputArtifactId: 'cdo:artifact-1', + finalizedAtMs: 300, + ...overrides, + }; +} + +function terminalization(overrides = {}) { + return { + requestId: 'diagnosis-request-1', + planDigest: DIGEST, + runId: 'diagnosis-run-1', + stage: 'cancellation', + reason: 'cancellation_requested', + outcome: 'cancelled', + finalizedAtMs: 250, + ...overrides, + }; +} + +function usage(overrides = {}) { + return { + invocationId: 'model-invocation-1', + projectId: 'project-1', + runId: 'diagnosis-run-1', + stepRunId: 'model-step-1', + traceId: 'trace-1', + completionDigest: COMPLETION_DIGEST, + outcome: 'succeeded', + inputTokens: 12, + outputTokens: 8, + totalTokens: 20, + costMicros: 42, + ...overrides, + }; +} + +function settlement(overrides = {}) { + return { + invocationId: 'model-invocation-1', + projectId: 'project-1', + completionDigest: COMPLETION_DIGEST, + currency: 'USD', + inputTokens: 12, + outputTokens: 8, + costMicros: 42, + ...overrides, + }; +} + +function fixture(overrides = {}) { + const state = { + plan: plan(), + admission: admission(), + terminalization: null, + finalization: null, + usage: null, + settlement: null, + artifact: null, + decision: { + effect: 'allow', + reasons: ['role_grant'], + fence: { + projectVersion: 1, + bindingVersion: 1, + }, + }, + keyMaterial: null, + keyResolves: 0, + permissions: [], + ...overrides, + }; + const service = new CopilotFailureDiagnosisReadService({ + admissions: { + async findPlanByRequestId() { + return state.plan; + }, + async findByRequestId() { + return state.admission; + }, + }, + terminalizations: { + async findByRequestId() { + return state.terminalization; + }, + }, + finalizations: { + async findFinalization() { + return state.finalization; + }, + }, + models: { + async findCopilotFailureDiagnosisOutput() { + return state.artifact; + }, + async findUsage() { + return state.usage; + }, + async findPriceSettlement() { + return state.settlement; + }, + }, + authorizer: { + async authorize(_principal, _projectId, permission) { + state.permissions.push(permission); + return state.decision; + }, + }, + keys: { + async active() { + throw new Error('read must not request the active key'); + }, + async resolve() { + state.keyResolves += 1; + return state.keyMaterial; + }, + }, + now: () => 500, + }); + return { service, state }; +} + +test('inspects running and pre-Model cancellation without content or model metadata', async () => { + const running = fixture(); + assert.deepEqual(await running.service.inspect(target()), { + schema: COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA, + status: 'running', + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'diagnosis-request-1', + diagnosisRunId: 'diagnosis-run-1', + outcome: null, + stage: null, + reason: null, + outputAvailable: false, + admittedAtMs: 110, + finalizedAtMs: null, + usage: null, + }); + assert.deepEqual(running.state.permissions, ['run.read']); + + const cancelled = fixture({ terminalization: terminalization() }); + const result = await cancelled.service.inspect(target()); + assert.equal(result.status, 'terminal'); + assert.equal(result.stage, 'cancellation'); + assert.equal(result.reason, 'cancellation_requested'); + assert.deepEqual(result.usage, { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + currency: 'USD', + costMicros: 0, + }); + assert.equal(JSON.stringify(result).includes('provider'), false); +}); + +test('projects only exact durable usage and price settlement', async () => { + const settled = fixture({ + finalization: finalization(), + usage: usage(), + settlement: settlement(), + }); + const result = await settled.service.inspect(target()); + assert.equal(result.status, 'terminal'); + assert.equal(result.outcome, 'succeeded'); + assert.equal(result.outputAvailable, true); + assert.deepEqual(result.usage, { + inputTokens: 12, + outputTokens: 8, + totalTokens: 20, + currency: 'USD', + costMicros: 42, + }); + + const unpriced = fixture({ + finalization: finalization(), + usage: usage({ costMicros: null }), + }); + assert.deepEqual((await unpriced.service.inspect(target())).usage, { + inputTokens: 12, + outputTokens: 8, + totalTokens: 20, + currency: null, + costMicros: null, + }); + + const inconsistent = fixture({ + finalization: finalization(), + usage: usage(), + settlement: settlement({ costMicros: 43 }), + }); + await assert.rejects( + inconsistent.service.inspect(target()), + CopilotFailureDiagnosisReadUnavailableError, + ); +}); + +test('masks absence, cross-target and current Policy denial and rejects dual terminal facts', async () => { + const absent = fixture({ plan: null, admission: null }); + assert.equal((await absent.service.inspect(target())).status, 'not_found'); + + const incompleteAdmission = fixture({ plan: null }); + await assert.rejects( + incompleteAdmission.service.inspect(target()), + CopilotFailureDiagnosisReadUnavailableError, + ); + + const crossProject = fixture({ plan: plan({ projectId: 'project-2' }) }); + assert.equal( + (await crossProject.service.inspect(target())).status, + 'not_found', + ); + assert.deepEqual(crossProject.state.permissions, []); + + const denied = fixture({ + decision: { effect: 'deny', reasons: ['permission_missing'], fence: null }, + }); + assert.equal((await denied.service.inspect(target())).status, 'not_found'); + + const conflict = fixture({ + terminalization: terminalization(), + finalization: finalization(), + }); + await assert.rejects( + conflict.service.inspect(target()), + CopilotFailureDiagnosisReadUnavailableError, + ); +}); + +test('decrypts an exact success Artifact, omits provider/model and wipes resolved key', async () => { + const encryptionKey = Buffer.alloc(32, 0x44); + const artifact = createCopilotFailureDiagnosisOutputArtifact( + { + requestId: 'diagnosis-request-1', + planDigest: DIGEST, + toolCompletionDigest: 'c'.repeat(64), + projectId: 'project-1', + runId: 'diagnosis-run-1', + stepRunId: 'model-step-1', + invocationId: 'model-invocation-1', + result: { + provider: 'private-provider', + model: 'private-model', + text: 'bounded diagnosis', + finishReason: 'stop', + usage: { inputTokens: 12, outputTokens: 8, totalTokens: 20 }, + }, + egressEvidence: { policyRevision: 'private-policy' }, + keyId: 'output-key-1', + key: encryptionKey, + sealedAtMs: 200, + }, + () => Buffer.alloc(12, 0x22), + ); + const resolvedKey = Buffer.from(encryptionKey); + const exactFinalization = finalization({ + outputArtifactId: artifact.artifactId, + }); + const output = fixture({ + finalization: exactFinalization, + artifact, + keyMaterial: { keyId: 'output-key-1', key: resolvedKey }, + }); + const result = await output.service.readOutput(target()); + assert.equal( + result.schema, + COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA, + ); + assert.equal(result.status, 'available'); + assert.equal(result.result.text, 'bounded diagnosis'); + assert.equal(result.reference.artifactId, artifact.artifactId); + assert.equal(JSON.stringify(result).includes('private-provider'), false); + assert.equal(JSON.stringify(result).includes('private-model'), false); + assert.equal( + resolvedKey.every((byte) => byte === 0), + true, + ); + assert.deepEqual(output.state.permissions, ['artifact.read']); +}); + +test('never resolves keys for denial/non-success and wipes keys after tamper failure', async () => { + const denied = fixture({ + finalization: finalization(), + decision: { effect: 'deny', reasons: ['permission_missing'], fence: null }, + }); + assert.equal((await denied.service.readOutput(target())).status, 'not_found'); + assert.equal(denied.state.keyResolves, 0); + + const failed = fixture({ + finalization: finalization({ + outcome: 'failed', + outputArtifactId: null, + }), + }); + assert.equal((await failed.service.readOutput(target())).status, 'not_found'); + assert.equal(failed.state.keyResolves, 0); + + const encryptionKey = Buffer.alloc(32, 0x55); + const artifact = createCopilotFailureDiagnosisOutputArtifact({ + requestId: 'diagnosis-request-1', + planDigest: DIGEST, + toolCompletionDigest: 'c'.repeat(64), + projectId: 'project-1', + runId: 'diagnosis-run-1', + stepRunId: 'model-step-1', + invocationId: 'model-invocation-1', + result: { + provider: 'provider', + model: 'model', + text: 'diagnosis', + finishReason: 'stop', + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }, + egressEvidence: { policyRevision: 'private-policy' }, + keyId: 'output-key-1', + key: encryptionKey, + sealedAtMs: 200, + }); + const resolvedKey = Buffer.from(encryptionKey); + const tampered = { ...artifact, ciphertext: `${artifact.ciphertext}A` }; + const corrupted = fixture({ + finalization: finalization({ outputArtifactId: artifact.artifactId }), + artifact: tampered, + keyMaterial: { keyId: 'output-key-1', key: resolvedKey }, + }); + await assert.rejects( + corrupted.service.readOutput(target()), + CopilotFailureDiagnosisReadUnavailableError, + ); + assert.equal( + resolvedKey.every((byte) => byte === 0), + true, + ); +}); diff --git a/packages/ql3-cluster-control/package.json b/packages/ql3-cluster-control/package.json index 0a0f0b45..f5d6301e 100644 --- a/packages/ql3-cluster-control/package.json +++ b/packages/ql3-cluster-control/package.json @@ -50,6 +50,11 @@ "require": "./dist/copilot/failure-diagnosis/failureDiagnosisRoute.js", "default": "./dist/copilot/failure-diagnosis/failureDiagnosisRoute.js" }, + "./copilot-read-routes": { + "types": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.d.ts", + "require": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js", + "default": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js" + }, "./http": { "types": "./dist/transport/httpSurface.d.ts", "require": "./dist/transport/httpSurface.js", diff --git a/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts b/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts index b8fb1ae5..c37d9481 100644 --- a/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts +++ b/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts @@ -31,6 +31,10 @@ import { type ClusterCopilotFailureDiagnosisProjection, type CreateProductionClusterCopilotFailureDiagnosisOptions, } from './copilot/failureDiagnosisComposition'; +import { + createProductionClusterCopilotFailureDiagnosisReadService, + type CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions, +} from './copilot/failureDiagnosisReadComposition'; export interface EnabledProductionClusterAiConfig { readonly enabled: true; @@ -54,6 +58,11 @@ export interface ProductionClusterAiControlApplicationOptions { readonly createCopilot?: ( options: CreateProductionClusterCopilotFailureDiagnosisOptions, ) => Promise>; + readonly createCopilotRead?: ( + options: CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions, + ) => ReturnType< + typeof createProductionClusterCopilotFailureDiagnosisReadService + >; readonly openAiDatabase?: ReturnType; } @@ -241,7 +250,9 @@ export async function startProductionClusterAiControlApplication( Array.isArray(options) || typeof options.audit !== 'function' ) { - throw new TypeError('Production Cluster AI application options are invalid'); + throw new TypeError( + 'Production Cluster AI application options are invalid', + ); } const startControl = options.startControl ?? startProductionClusterControlApplication; @@ -249,14 +260,20 @@ export async function startProductionClusterAiControlApplication( options.bootstrapPrompt ?? bootstrapPostgresPluginPackagePromptApplication; const createCopilot = options.createCopilot ?? createProductionClusterCopilotFailureDiagnosis; + const createCopilotRead = + options.createCopilotRead ?? + createProductionClusterCopilotFailureDiagnosisReadService; if ( typeof startControl !== 'function' || typeof bootstrapPrompt !== 'function' || typeof createCopilot !== 'function' || + typeof createCopilotRead !== 'function' || (options.openAiDatabase !== undefined && typeof options.openAiDatabase !== 'function') ) { - throw new TypeError('Production Cluster AI application factories are invalid'); + throw new TypeError( + 'Production Cluster AI application factories are invalid', + ); } const copilotArtifactStore = options.control.workerIngress?.artifactStore; if ( @@ -303,18 +320,27 @@ export async function startProductionClusterAiControlApplication( let copilotApplication: | Readonly | undefined; + let copilotReadApplication: + | ReturnType< + typeof createProductionClusterCopilotFailureDiagnosisReadService + > + | undefined; let copilotSuccessfulCompletion: | CopilotFailureDiagnosisModelCompletionCoordinator | undefined; let stopPromise: Promise | undefined; let promptOutputPolicy: ProjectPolicyEngine | undefined; const promptOutputReadAuthorizer = Object.freeze({ - async authorize(request: Readonly<{ - principal: Parameters[0]; - projectId: string; - }>) { + async authorize( + request: Readonly<{ + principal: Parameters[0]; + projectId: string; + }>, + ) { if (!aiDatabase) { - throw new Error('Cluster AI database is unavailable during output read'); + throw new Error( + 'Cluster AI database is unavailable during output read', + ); } promptOutputPolicy ??= new ProjectPolicyEngine( new PostgresProjectPolicyRepository(aiDatabase.pool), @@ -361,7 +387,9 @@ export async function startProductionClusterAiControlApplication( }, async loadProviders() { if (!aiDatabase) { - throw new Error('Cluster AI database is unavailable during provider load'); + throw new Error( + 'Cluster AI database is unavailable during provider load', + ); } const credentialStorage = new PostgresModelProviderCredentialReader( aiDatabase.pool, @@ -409,7 +437,11 @@ export async function startProductionClusterAiControlApplication( throw new Error('Cluster AI Prompt application did not activate'); } if (preparedCopilot !== undefined) { - if (!copilotSuccessfulCompletion || !aiDatabase || !copilotArtifactStore) { + if ( + !copilotSuccessfulCompletion || + !aiDatabase || + !copilotArtifactStore + ) { throw new Error('Cluster Copilot shared authorities did not activate'); } copilotApplication = await createCopilot({ @@ -419,6 +451,10 @@ export async function startProductionClusterAiControlApplication( successfulCompletion: copilotSuccessfulCompletion, artifactStore: copilotArtifactStore, }); + copilotReadApplication = createCopilotRead({ + pool: aiDatabase.pool, + prepared: preparedCopilot, + }); } controlApplication = await startControl({ ...options.control, @@ -445,11 +481,13 @@ export async function startProductionClusterAiControlApplication( capability: promptApplication.promptExecutionOutputs, }, }), - ...(copilotApplication === undefined + ...(copilotApplication === undefined || + copilotReadApplication === undefined ? {} : { copilotFailureDiagnosis: { capability: copilotApplication, + readCapability: copilotReadApplication, }, }), }); diff --git a/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisReadComposition.ts b/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisReadComposition.ts new file mode 100644 index 00000000..84d86d3c --- /dev/null +++ b/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisReadComposition.ts @@ -0,0 +1,57 @@ +import { CopilotFailureDiagnosisReadService } from '@qinglong/ai/failure-diagnosis-read-model'; +import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage'; +import { PostgresCopilotFailureDiagnosisModelRepository } from '@qinglong/ai/postgres-failure-diagnosis-model-execution-storage'; +import { PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository } from '@qinglong/ai/failure-diagnosis-pre-model-terminalization'; +import { + PostgresProjectPolicyRepository, + type QingLongPostgresPool, +} from '@qinglong/cluster-postgres/runtime'; +import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy'; + +import type { PreparedClusterCopilotFailureDiagnosisProjection } from './failureDiagnosisComposition'; + +export interface CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions { + readonly pool: QingLongPostgresPool; + readonly prepared: PreparedClusterCopilotFailureDiagnosisProjection; +} + +/** + * Reuses the execution Pool, durable repositories and projected output keys; + * creating this service owns no connection, listener or background lifecycle. + */ +export function createProductionClusterCopilotFailureDiagnosisReadService( + options: CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions, +): Readonly { + if ( + !options || + typeof options !== 'object' || + Array.isArray(options) || + typeof options.pool?.query !== 'function' || + typeof options.pool?.connect !== 'function' || + typeof options.prepared?.outputKeys?.resolve !== 'function' + ) { + throw new TypeError( + 'Production Cluster Copilot failure diagnosis read dependencies are invalid', + ); + } + const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository( + options.pool, + ); + const models = new PostgresCopilotFailureDiagnosisModelRepository( + options.pool, + ); + const terminalizations = + new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository( + options.pool, + ); + return new CopilotFailureDiagnosisReadService({ + admissions, + terminalizations, + finalizations: models, + models, + authorizer: new ProjectPolicyEngine( + new PostgresProjectPolicyRepository(options.pool), + ), + keys: options.prepared.outputKeys, + }); +} diff --git a/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts b/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts index bb6437ec..b12a4863 100644 --- a/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts +++ b/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts @@ -65,6 +65,12 @@ import { createClusterControlCopilotFailureDiagnosisRoute, type ClusterCopilotFailureDiagnosisCapability, } from '../copilot/failure-diagnosis/failureDiagnosisRoute'; +import { + createClusterControlCopilotFailureDiagnosisInspectionRoute, + createClusterControlCopilotFailureDiagnosisOutputReadRoute, + type ClusterCopilotFailureDiagnosisInspectionCapability, + type ClusterCopilotFailureDiagnosisOutputReadCapability, +} from '../copilot/failure-diagnosis/failureDiagnosisReadRoutes'; export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([ 'task.get', @@ -92,8 +98,14 @@ export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS = 'prompt.execution.output.read', 'prompt.output.read', 'copilot.failure_diagnosis.execute', + 'copilot.failure_diagnosis.read', + 'copilot.failure_diagnosis.output.read', ] as const); +export interface ClusterCopilotFailureDiagnosisReadCapability + extends ClusterCopilotFailureDiagnosisInspectionCapability, + ClusterCopilotFailureDiagnosisOutputReadCapability {} + export interface ProductionClusterControlAssemblyOptions { readonly createEventId?: ClusterRunCancellationEventIdFactory; readonly promptCatalog?: Readonly<{ @@ -116,6 +128,7 @@ export interface ProductionClusterControlAssemblyOptions { }>; readonly copilotFailureDiagnosis?: Readonly<{ readonly capability: ClusterCopilotFailureDiagnosisCapability; + readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability; }>; readonly workerIngress?: Readonly<{ readonly config: EnabledClusterWorkerIngressConfig; @@ -164,6 +177,7 @@ export interface ProductionClusterControlApplicationOptions }>; readonly copilotFailureDiagnosis?: Readonly<{ readonly capability: ClusterCopilotFailureDiagnosisCapability; + readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability; }>; readonly workerIngress?: ProductionClusterWorkerIngressOptions; } @@ -279,6 +293,16 @@ export function createProductionClusterControlApplicationStack( options.copilotFailureDiagnosis.capability, ), ]), + ...(options.copilotFailureDiagnosis?.readCapability === undefined + ? [] + : [ + createClusterControlCopilotFailureDiagnosisInspectionRoute( + options.copilotFailureDiagnosis.readCapability, + ), + createClusterControlCopilotFailureDiagnosisOutputReadRoute( + options.copilotFailureDiagnosis.readCapability, + ), + ]), ]; const routes = createClusterControlRouteRegistry(routeDefinitions); const expectedRouteCount = @@ -288,7 +312,8 @@ export function createProductionClusterControlApplicationStack( (options.promptExecutionInspection === undefined ? 0 : 1) + (options.promptOutputRead === undefined ? 0 : 1) + (options.promptExecutionOutputRead === undefined ? 0 : 1) + - (options.copilotFailureDiagnosis === undefined ? 0 : 1); + (options.copilotFailureDiagnosis === undefined ? 0 : 1) + + (options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2); if (routes.size !== expectedRouteCount) { throw new Error('Production cluster-control route allowlist is incomplete'); } diff --git a/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisReadRoutes.ts b/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisReadRoutes.ts new file mode 100644 index 00000000..41389842 --- /dev/null +++ b/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisReadRoutes.ts @@ -0,0 +1,436 @@ +// Cluster Copilot exposes separate low-sensitive status and protected output reads. +import type { SecurityPrincipal } from '@qinglong/runtime-core/security'; + +import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface'; +import type { + ClusterControlAuthorizedOperationRequest, + ClusterControlRouteDefinition, + ClusterControlRouteParameters, +} from '../../transport/routeRegistry'; + +export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESPONSE_SCHEMA = + 'qinglong/cluster-copilot-failure-diagnosis-inspection-response@v1' as const; +export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESPONSE_SCHEMA = + 'qinglong/cluster-copilot-failure-diagnosis-output-read-response@v1' as const; + +export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_ROUTE = + Object.freeze({ + method: 'GET' as const, + path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}', + operationId: 'copilot.failure_diagnosis.read', + permission: 'run.read', + projectParameter: 'projectId', + }); + +export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_ROUTE = + Object.freeze({ + method: 'GET' as const, + path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/output', + operationId: 'copilot.failure_diagnosis.output.read', + permission: 'artifact.read', + projectParameter: 'projectId', + }); + +interface ClusterCopilotFailureDiagnosisReadCommand { + readonly principal: Readonly; + readonly projectId: string; + readonly sourceRunId: string; + readonly requestId: string; +} + +export interface ClusterCopilotFailureDiagnosisInspectionCapability { + inspect( + command: Readonly, + ): Promise; +} + +export interface ClusterCopilotFailureDiagnosisOutputReadCapability { + readOutput( + command: Readonly, + ): Promise; +} + +const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/; +const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/; +const DIGEST = /^[0-9a-f]{64}$/; +const FINISH_REASONS = new Set([ + 'stop', + 'length', + 'content_filter', + 'tool_call', + 'unknown', +]); +const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']); +const STAGES = new Set(['model', 'tool', 'log', 'deadline', 'cancellation']); +const REASONS = new Set([ + 'tool_failed', + 'tool_timed_out', + 'log_not_found', + 'log_pending', + 'log_missing', + 'log_retired', + 'tool_budget_exhausted', + 'deadline_exceeded', + 'cancellation_requested', +]); + +function response( + statusCode: number, + body: Readonly>, +): ClusterControlAdmissionResponse { + return Object.freeze({ statusCode, body: Object.freeze(body) }); +} + +function exactRecord( + value: unknown, + required: readonly string[], + optional: readonly string[] = [], +): Record | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const record = value as Record; + const actual = Object.keys(record).sort(); + const expected = [ + ...required, + ...optional.filter((key) => key in record), + ].sort(); + return actual.length === expected.length && + actual.every((key, index) => key === expected[index]) + ? record + : null; +} + +function nonNegativeInteger(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} + +function target( + authorized: ClusterControlAuthorizedOperationRequest, + parameters: ClusterControlRouteParameters, +): Readonly | null { + if ( + authorized.request.body !== null || + authorized.projectId === null || + typeof parameters.runId !== 'string' || + !RUN_ID.test(parameters.runId) || + typeof parameters.requestId !== 'string' || + !IDENTITY.test(parameters.requestId) + ) { + return null; + } + return Object.freeze({ + principal: authorized.principal, + projectId: authorized.projectId, + sourceRunId: parameters.runId, + requestId: parameters.requestId, + }); +} + +function exactTarget( + value: Record, + expected: Readonly, +): boolean { + return ( + value.projectId === expected.projectId && + value.sourceRunId === expected.sourceRunId && + value.requestId === expected.requestId + ); +} + +function notFound( + value: unknown, + expected: Readonly, + schema: string, +): boolean { + const candidate = exactRecord(value, [ + 'projectId', + 'requestId', + 'schema', + 'sourceRunId', + 'status', + ]); + return ( + !!candidate && + candidate.schema === schema && + candidate.status === 'not_found' && + exactTarget(candidate, expected) + ); +} + +function usageView(value: unknown): Readonly> | null { + const usage = exactRecord(value, [ + 'costMicros', + 'currency', + 'inputTokens', + 'outputTokens', + 'totalTokens', + ]); + if ( + !usage || + !nonNegativeInteger(usage.inputTokens) || + !nonNegativeInteger(usage.outputTokens) || + !nonNegativeInteger(usage.totalTokens) || + usage.totalTokens !== usage.inputTokens + usage.outputTokens || + !( + (usage.currency === null && usage.costMicros === null) || + (usage.currency === 'USD' && nonNegativeInteger(usage.costMicros)) + ) + ) { + return null; + } + return Object.freeze({ ...usage }); +} + +function inspectionView( + value: unknown, + expected: Readonly, +): Readonly> | null { + const candidate = exactRecord(value, [ + 'admittedAtMs', + 'diagnosisRunId', + 'finalizedAtMs', + 'outcome', + 'outputAvailable', + 'projectId', + 'reason', + 'requestId', + 'schema', + 'sourceRunId', + 'stage', + 'status', + 'usage', + ]); + if ( + !candidate || + candidate.schema !== + 'qinglong/copilot-failure-diagnosis-inspection-result@v1' || + !exactTarget(candidate, expected) || + typeof candidate.diagnosisRunId !== 'string' || + !RUN_ID.test(candidate.diagnosisRunId) || + !nonNegativeInteger(candidate.admittedAtMs) || + typeof candidate.outputAvailable !== 'boolean' + ) { + return null; + } + if (candidate.status === 'running') { + if ( + candidate.outcome !== null || + candidate.stage !== null || + candidate.reason !== null || + candidate.outputAvailable !== false || + candidate.finalizedAtMs !== null || + candidate.usage !== null + ) { + return null; + } + } else if (candidate.status === 'terminal') { + if ( + typeof candidate.outcome !== 'string' || + !OUTCOMES.has(candidate.outcome) || + typeof candidate.stage !== 'string' || + !STAGES.has(candidate.stage) || + !nonNegativeInteger(candidate.finalizedAtMs) || + candidate.finalizedAtMs < candidate.admittedAtMs || + (candidate.stage === 'model') !== (candidate.reason === null) || + (candidate.reason !== null && + (typeof candidate.reason !== 'string' || + !REASONS.has(candidate.reason))) || + candidate.outputAvailable !== + (candidate.stage === 'model' && candidate.outcome === 'succeeded') + ) { + return null; + } + if (candidate.usage !== null && !usageView(candidate.usage)) return null; + } else { + return null; + } + return Object.freeze({ + schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESPONSE_SCHEMA, + status: candidate.status, + projectId: expected.projectId, + sourceRunId: expected.sourceRunId, + requestId: expected.requestId, + diagnosisRunId: candidate.diagnosisRunId, + outcome: candidate.outcome, + stage: candidate.stage, + reason: candidate.reason, + outputAvailable: candidate.outputAvailable, + admittedAtMs: candidate.admittedAtMs, + finalizedAtMs: candidate.finalizedAtMs, + usage: candidate.usage === null ? null : usageView(candidate.usage), + }); +} + +function outputView( + value: unknown, + expected: Readonly, +): Readonly> | null { + const candidate = exactRecord(value, [ + 'diagnosisRunId', + 'projectId', + 'reference', + 'requestId', + 'result', + 'schema', + 'sourceRunId', + 'status', + ]); + const reference = candidate + ? exactRecord(candidate.reference, [ + 'artifactDigest', + 'artifactId', + 'contentDigest', + 'outputBytes', + 'sealedAtMs', + ]) + : null; + const result = candidate + ? exactRecord(candidate.result, ['finishReason', 'text', 'usage']) + : null; + const usage = result + ? exactRecord( + result.usage, + ['inputTokens', 'outputTokens', 'totalTokens'], + ['costMicros'], + ) + : null; + if ( + !candidate || + !reference || + !result || + !usage || + candidate.schema !== + 'qinglong/copilot-failure-diagnosis-output-read-result@v1' || + candidate.status !== 'available' || + !exactTarget(candidate, expected) || + typeof candidate.diagnosisRunId !== 'string' || + !RUN_ID.test(candidate.diagnosisRunId) || + typeof reference.artifactId !== 'string' || + !IDENTITY.test(reference.artifactId) || + typeof reference.artifactDigest !== 'string' || + !DIGEST.test(reference.artifactDigest) || + typeof reference.contentDigest !== 'string' || + !DIGEST.test(reference.contentDigest) || + !nonNegativeInteger(reference.outputBytes) || + reference.outputBytes > 1024 * 1024 || + !nonNegativeInteger(reference.sealedAtMs) || + typeof result.text !== 'string' || + Buffer.byteLength(result.text, 'utf8') !== reference.outputBytes || + typeof result.finishReason !== 'string' || + !FINISH_REASONS.has(result.finishReason) || + !nonNegativeInteger(usage.inputTokens) || + !nonNegativeInteger(usage.outputTokens) || + !nonNegativeInteger(usage.totalTokens) || + usage.totalTokens !== usage.inputTokens + usage.outputTokens || + (usage.costMicros !== undefined && !nonNegativeInteger(usage.costMicros)) + ) { + return null; + } + return Object.freeze({ + schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESPONSE_SCHEMA, + status: 'available', + projectId: expected.projectId, + sourceRunId: expected.sourceRunId, + requestId: expected.requestId, + diagnosisRunId: candidate.diagnosisRunId, + reference: Object.freeze({ ...reference }), + result: Object.freeze({ + text: result.text, + finishReason: result.finishReason, + usage: Object.freeze({ ...usage }), + }), + }); +} + +export function createClusterControlCopilotFailureDiagnosisInspectionRoute( + capability: ClusterCopilotFailureDiagnosisInspectionCapability, +): Readonly { + if (!capability || typeof capability.inspect !== 'function') { + throw new TypeError( + 'Cluster-control Copilot diagnosis inspection capability is invalid', + ); + } + return Object.freeze({ + ...CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_ROUTE, + async handle( + authorized: ClusterControlAuthorizedOperationRequest, + parameters: ClusterControlRouteParameters, + ) { + const command = target(authorized, parameters); + if (!command) { + return response(400, { + code: 'invalid_copilot_failure_diagnosis_read_request', + }); + } + try { + const result = await capability.inspect(command); + if ( + notFound( + result, + command, + 'qinglong/copilot-failure-diagnosis-inspection-result@v1', + ) + ) { + return response(404, { code: 'copilot_failure_diagnosis_not_found' }); + } + const view = inspectionView(result, command); + return view + ? response(200, view) + : response(503, { + code: 'copilot_failure_diagnosis_read_unavailable', + }); + } catch { + return response(503, { + code: 'copilot_failure_diagnosis_read_unavailable', + }); + } + }, + }); +} + +export function createClusterControlCopilotFailureDiagnosisOutputReadRoute( + capability: ClusterCopilotFailureDiagnosisOutputReadCapability, +): Readonly { + if (!capability || typeof capability.readOutput !== 'function') { + throw new TypeError( + 'Cluster-control Copilot diagnosis output read capability is invalid', + ); + } + return Object.freeze({ + ...CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_ROUTE, + async handle( + authorized: ClusterControlAuthorizedOperationRequest, + parameters: ClusterControlRouteParameters, + ) { + const command = target(authorized, parameters); + if (!command) { + return response(400, { + code: 'invalid_copilot_failure_diagnosis_output_read_request', + }); + } + try { + const result = await capability.readOutput(command); + if ( + notFound( + result, + command, + 'qinglong/copilot-failure-diagnosis-output-read-result@v1', + ) + ) { + return response(404, { + code: 'copilot_failure_diagnosis_output_not_found', + }); + } + const view = outputView(result, command); + return view + ? response(200, view) + : response(503, { + code: 'copilot_failure_diagnosis_output_read_unavailable', + }); + } catch { + return response(503, { + code: 'copilot_failure_diagnosis_output_read_unavailable', + }); + } + }, + }); +} diff --git a/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs b/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs index 2992c039..c572af99 100644 --- a/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs +++ b/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs @@ -29,7 +29,8 @@ const { function enabledEnvironment(overrides = {}) { return { QL3_CLUSTER_AI_ENABLED: 'true', - QL3_CLUSTER_AI_PROVIDER_AUTHORITY_FILE: '/var/run/qinglong/ai/providers.json', + QL3_CLUSTER_AI_PROVIDER_AUTHORITY_FILE: + '/var/run/qinglong/ai/providers.json', QL3_CLUSTER_AI_SECRET_ROOT: '/var/run/qinglong/ai/provider-secrets', ...overrides, }; @@ -120,26 +121,30 @@ async function projectedFile(root, name, bytes) { test('Copilot composition is explicit, shares the Prompt gateway and injects one route capability', async () => { const secretRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-secret-')); const configRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-config-')); - const invocationRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-invocation-')); + const invocationRoot = await mkdtemp( + join(tmpdir(), 'ql3-copilot-invocation-'), + ); const resultRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-result-')); const outputRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-output-')); const key = Buffer.alloc(32, 0x55).toString('base64url'); - const config = Buffer.from(`${JSON.stringify({ - schema: 'qinglong/cluster-copilot-failure-diagnosis-config@v1', - provider: 'provider-primary', - model: 'model-diagnosis', - modelBoundary: 'external', - responseLanguage: 'zh-CN', - maxOutputTokens: 512, - executionTimeoutMs: 60_000, - egressPolicy: { - schema: 'qinglong/copilot-model-egress-policy@v1', - revision: 'cluster-copilot-v1', - potentiallySensitiveDataBoundaries: ['external'], - maxInputBytes: 64 * 1024, - maxOutputTokens: 1024, - }, - })}\n`); + const config = Buffer.from( + `${JSON.stringify({ + schema: 'qinglong/cluster-copilot-failure-diagnosis-config@v1', + provider: 'provider-primary', + model: 'model-diagnosis', + modelBoundary: 'external', + responseLanguage: 'zh-CN', + maxOutputTokens: 512, + executionTimeoutMs: 60_000, + egressPolicy: { + schema: 'qinglong/copilot-model-egress-policy@v1', + revision: 'cluster-copilot-v1', + potentiallySensitiveDataBoundaries: ['external'], + maxInputBytes: 64 * 1024, + maxOutputTokens: 1024, + }, + })}\n`, + ); const invocation = canonicalClusterToolInvocationKeyringManifest({ schema: CLUSTER_TOOL_INVOCATION_KEYRING_MANIFEST_SCHEMA, activeKeyId: 'invocation-key-1', @@ -163,8 +168,10 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one const fakePool = { query() {}, connect() {} }; const artifactStore = { put() {}, inspect() {}, readLogRange() {} }; const copilot = Object.freeze({ execute() {} }); + const copilotRead = Object.freeze({ inspect() {}, readOutput() {} }); let registeredSink; let created; + let createdRead; let controlOptions; try { await Promise.all([ @@ -205,22 +212,41 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one async recordWithAtomicSuccess() {}, }); return { - status: 'active', profile: 'cluster', readiness: {}, capability: gateway, - prompts: {}, promptCatalog: {}, promptExecutions: {}, - promptExecutionInspections: {}, async stop() { return 'stopped'; }, + status: 'active', + profile: 'cluster', + readiness: {}, + capability: gateway, + prompts: {}, + promptCatalog: {}, + promptExecutions: {}, + promptExecutionInspections: {}, + async stop() { + return 'stopped'; + }, }; }, async createCopilot(options) { created = options; return copilot; }, + createCopilotRead(options) { + createdRead = options; + return copilotRead; + }, async startControl(options) { controlOptions = options; return { - status: 'active', address: { host: '127.0.0.1', port: 5800 }, - evidence: {}, recovery: { safe: true, remaining: 0, failed: 0 }, - unavailable: new Promise(() => {}), availabilityStatus() { return 'ready'; }, - async stop() { return 'stopped'; }, + status: 'active', + address: { host: '127.0.0.1', port: 5800 }, + evidence: {}, + recovery: { safe: true, remaining: 0, failed: 0 }, + unavailable: new Promise(() => {}), + availabilityStatus() { + return 'ready'; + }, + async stop() { + return 'stopped'; + }, }; }, }); @@ -229,10 +255,19 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one assert.equal(created.gateway, gateway); assert.equal(created.successfulCompletion, registeredSink); assert.equal(created.artifactStore, artifactStore); + assert.equal(createdRead.pool, fakePool); + assert.equal(typeof createdRead.prepared.outputKeys.resolve, 'function'); assert.equal(controlOptions.copilotFailureDiagnosis.capability, copilot); + assert.equal( + controlOptions.copilotFailureDiagnosis.readCapability, + copilotRead, + ); assert.equal(await application.stop(), 'stopped'); } finally { - config.fill(0); invocation.fill(0); result.fill(0); output.fill(0); + config.fill(0); + invocation.fill(0); + result.fill(0); + output.fill(0); await Promise.all([ rm(secretRoot, { recursive: true, force: true }), rm(configRoot, { recursive: true, force: true }), @@ -362,7 +397,9 @@ test('output-enabled AI composition wires exact and request-keyed protected read promptExecutionInspections: { inspectAuthorized() {} }, promptOutputs, promptExecutionOutputs, - async stop() { return 'stopped'; }, + async stop() { + return 'stopped'; + }, }; }, async startControl(options) { @@ -373,8 +410,12 @@ test('output-enabled AI composition wires exact and request-keyed protected read evidence: {}, recovery: { safe: true, remaining: 0, failed: 0 }, unavailable: new Promise(() => {}), - availabilityStatus() { return 'ready'; }, - async stop() { return 'stopped'; }, + availabilityStatus() { + return 'ready'; + }, + async stop() { + return 'stopped'; + }, }; }, }); diff --git a/packages/ql3-cluster-control/test/copilotFailureDiagnosisReadRoutes.test.cjs b/packages/ql3-cluster-control/test/copilotFailureDiagnosisReadRoutes.test.cjs new file mode 100644 index 00000000..5c34954f --- /dev/null +++ b/packages/ql3-cluster-control/test/copilotFailureDiagnosisReadRoutes.test.cjs @@ -0,0 +1,288 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + CLUSTER_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESPONSE_SCHEMA, + CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESPONSE_SCHEMA, + CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_ROUTE, + CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_ROUTE, + createClusterControlCopilotFailureDiagnosisInspectionRoute, + createClusterControlCopilotFailureDiagnosisOutputReadRoute, +} = require('@qinglong/cluster-control/copilot-read-routes'); + +function authorized(path, body = null) { + return { + request: { + requestId: 'transport-request-1', + method: 'GET', + path, + query: {}, + headers: {}, + signal: new AbortController().signal, + body, + }, + principal: { + subject: { type: 'api_app', id: 'app-1' }, + authenticationId: 'credential-1', + authenticatedAtMs: 1, + expiresAtMs: 10_000, + assurance: 'service', + }, + operationId: 'copilot.failure_diagnosis.read', + permission: 'run.read', + projectId: 'project-1', + policyFence: { projectVersion: 3, bindingVersion: 7 }, + }; +} + +const parameters = { + projectId: 'project-1', + runId: 'source-run-1', + requestId: 'diagnosis-request-1', +}; + +function running(overrides = {}) { + return { + schema: 'qinglong/copilot-failure-diagnosis-inspection-result@v1', + status: 'running', + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'diagnosis-request-1', + diagnosisRunId: 'diagnosis-run-1', + outcome: null, + stage: null, + reason: null, + outputAvailable: false, + admittedAtMs: 100, + finalizedAtMs: null, + usage: null, + ...overrides, + }; +} + +test('defines separate run.read inspection and artifact.read output routes', () => { + assert.deepEqual(CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_ROUTE, { + method: 'GET', + path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}', + operationId: 'copilot.failure_diagnosis.read', + permission: 'run.read', + projectParameter: 'projectId', + }); + assert.deepEqual( + CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_ROUTE, + { + method: 'GET', + path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/output', + operationId: 'copilot.failure_diagnosis.output.read', + permission: 'artifact.read', + projectParameter: 'projectId', + }, + ); +}); + +test('projects a request-keyed running inspection and passes only trusted target facts', async () => { + let command; + const route = createClusterControlCopilotFailureDiagnosisInspectionRoute({ + async inspect(value) { + command = value; + return running(); + }, + }); + const request = authorized( + '/api/v3/projects/project-1/runs/source-run-1/copilot/failure-diagnoses/diagnosis-request-1', + ); + const result = await route.handle(request, parameters); + assert.deepEqual(command, { + principal: request.principal, + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'diagnosis-request-1', + }); + assert.deepEqual(result, { + statusCode: 200, + body: { + schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESPONSE_SCHEMA, + status: 'running', + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'diagnosis-request-1', + diagnosisRunId: 'diagnosis-run-1', + outcome: null, + stage: null, + reason: null, + outputAvailable: false, + admittedAtMs: 100, + finalizedAtMs: null, + usage: null, + }, + }); +}); + +test('projects terminal cancellation and settled Model usage without private fields', async () => { + for (const [value, expected] of [ + [ + running({ + status: 'terminal', + outcome: 'cancelled', + stage: 'cancellation', + reason: 'cancellation_requested', + finalizedAtMs: 200, + usage: { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + currency: 'USD', + costMicros: 0, + }, + }), + 'cancellation', + ], + [ + running({ + status: 'terminal', + outcome: 'succeeded', + stage: 'model', + reason: null, + outputAvailable: true, + finalizedAtMs: 200, + usage: { + inputTokens: 11, + outputTokens: 7, + totalTokens: 18, + currency: 'USD', + costMicros: 29, + }, + }), + 'model', + ], + ]) { + const route = createClusterControlCopilotFailureDiagnosisInspectionRoute({ + async inspect() { + return value; + }, + }); + const result = await route.handle(authorized('/read'), parameters); + assert.equal(result.statusCode, 200); + assert.equal(result.body.stage, expected); + assert.equal(JSON.stringify(result).includes('provider'), false); + assert.equal(JSON.stringify(result).includes('modelId'), false); + } +}); + +test('masks absent reads and fails closed on invalid input or widened results', async () => { + let calls = 0; + const route = createClusterControlCopilotFailureDiagnosisInspectionRoute({ + async inspect() { + calls += 1; + return { + schema: 'qinglong/copilot-failure-diagnosis-inspection-result@v1', + status: 'not_found', + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'diagnosis-request-1', + }; + }, + }); + assert.equal( + (await route.handle(authorized('/read'), parameters)).statusCode, + 404, + ); + assert.equal( + (await route.handle(authorized('/read', {}), parameters)).statusCode, + 400, + ); + assert.equal( + ( + await route.handle(authorized('/read'), { + ...parameters, + requestId: '../private', + }) + ).statusCode, + 400, + ); + assert.equal(calls, 1); + + const widened = createClusterControlCopilotFailureDiagnosisInspectionRoute({ + async inspect() { + return running({ privateModel: 'must not cross' }); + }, + }); + assert.equal( + (await widened.handle(authorized('/read'), parameters)).statusCode, + 503, + ); +}); + +test('returns only decrypted diagnosis content and low-sensitive Artifact metadata', async () => { + let command; + const route = createClusterControlCopilotFailureDiagnosisOutputReadRoute({ + async readOutput(value) { + command = value; + return { + schema: 'qinglong/copilot-failure-diagnosis-output-read-result@v1', + status: 'available', + projectId: value.projectId, + sourceRunId: value.sourceRunId, + requestId: value.requestId, + diagnosisRunId: 'diagnosis-run-1', + reference: { + artifactId: 'cdo:artifact-1', + artifactDigest: 'a'.repeat(64), + contentDigest: 'b'.repeat(64), + outputBytes: Buffer.byteLength('diagnosis'), + sealedAtMs: 200, + }, + result: { + text: 'diagnosis', + finishReason: 'stop', + usage: { inputTokens: 3, outputTokens: 2, totalTokens: 5 }, + }, + }; + }, + }); + const request = authorized('/output'); + request.operationId = 'copilot.failure_diagnosis.output.read'; + request.permission = 'artifact.read'; + const result = await route.handle(request, parameters); + assert.equal(result.statusCode, 200); + assert.equal( + result.body.schema, + CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESPONSE_SCHEMA, + ); + assert.equal(result.body.result.text, 'diagnosis'); + assert.equal('provider' in result.body.result, false); + assert.equal('model' in result.body.result, false); + assert.equal(command.principal, request.principal); +}); + +test('masks absent output and maps dependency/cipher failures to one 503 code', async () => { + const absent = createClusterControlCopilotFailureDiagnosisOutputReadRoute({ + async readOutput(value) { + return { + schema: 'qinglong/copilot-failure-diagnosis-output-read-result@v1', + status: 'not_found', + projectId: value.projectId, + sourceRunId: value.sourceRunId, + requestId: value.requestId, + }; + }, + }); + assert.equal( + (await absent.handle(authorized('/output'), parameters)).statusCode, + 404, + ); + + const unavailable = + createClusterControlCopilotFailureDiagnosisOutputReadRoute({ + async readOutput() { + throw new Error('private key failure'); + }, + }); + assert.deepEqual( + await unavailable.handle(authorized('/output'), parameters), + { + statusCode: 503, + body: { code: 'copilot_failure_diagnosis_output_read_unavailable' }, + }, + ); +}); diff --git a/packages/ql3-cluster-control/test/productionApplication.test.cjs b/packages/ql3-cluster-control/test/productionApplication.test.cjs index aa1c2104..e576b5a3 100644 --- a/packages/ql3-cluster-control/test/productionApplication.test.cjs +++ b/packages/ql3-cluster-control/test/productionApplication.test.cjs @@ -730,6 +730,8 @@ test('optionally exposes Prompt execution behind shared admission and policy', a 'prompt.execution.output.read', 'prompt.output.read', 'copilot.failure_diagnosis.execute', + 'copilot.failure_diagnosis.read', + 'copilot.failure_diagnosis.output.read', ]); const response = await invoke( stack, @@ -783,9 +785,30 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy terminalizationRequired: false, }; }, + async inspect(value) { + return { + schema: 'qinglong/copilot-failure-diagnosis-inspection-result@v1', + status: 'not_found', + projectId: value.projectId, + sourceRunId: value.sourceRunId, + requestId: value.requestId, + }; + }, + async readOutput(value) { + return { + schema: 'qinglong/copilot-failure-diagnosis-output-read-result@v1', + status: 'not_found', + projectId: value.projectId, + sourceRunId: value.sourceRunId, + requestId: value.requestId, + }; + }, }; const stack = createProductionClusterControlApplicationStack(input, { - copilotFailureDiagnosis: { capability }, + copilotFailureDiagnosis: { + capability, + readCapability: capability, + }, }); const result = await invoke( stack, @@ -811,6 +834,29 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy 'audit:copilot.failure_diagnosis.execute:allowed', 'diagnose:run-1', ]); + + const inspection = await invoke( + stack, + metadata( + '/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1', + ), + ); + const output = await invoke( + stack, + metadata( + '/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/output', + ), + ); + assert.equal(inspection.statusCode, 404); + assert.equal(output.statusCode, 404); + assert.equal( + events.includes('audit:copilot.failure_diagnosis.read:allowed'), + true, + ); + assert.equal( + events.includes('audit:copilot.failure_diagnosis.output.read:allowed'), + true, + ); }); test('keeps the Copilot route absent by default and never invokes it after Policy denial', async () => { @@ -830,6 +876,15 @@ test('keeps the Copilot route absent by default and never invokes it after Polic defaultStack.admission.prepare(request), (error) => error?.statusCode === 404 && error?.code === 'route_not_found', ); + for (const path of [ + '/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1', + '/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/output', + ]) { + await assert.rejects( + defaultStack.admission.prepare(metadata(path)), + (error) => error?.statusCode === 404 && error?.code === 'route_not_found', + ); + } let calls = 0; const deniedFixture = fixture({ diff --git a/scripts/ql3-cluster-dependency-audit.cjs b/scripts/ql3-cluster-dependency-audit.cjs index 627aeeaa..486211a7 100644 --- a/scripts/ql3-cluster-dependency-audit.cjs +++ b/scripts/ql3-cluster-dependency-audit.cjs @@ -1186,6 +1186,13 @@ function auditSourceImports(root, packagePath, findings) { '@qinglong/ai/postgres-failure-diagnosis-tool-execution-storage', '@qinglong/ai/profile', ]), + 'src/application-runtime/copilot/failureDiagnosisReadComposition.ts': + Object.freeze([ + '@qinglong/ai/failure-diagnosis-pre-model-terminalization', + '@qinglong/ai/failure-diagnosis-read-model', + '@qinglong/ai/postgres-failure-diagnosis-admission-storage', + '@qinglong/ai/postgres-failure-diagnosis-model-execution-storage', + ]), }); const sourceRelative = path.relative(packageDirectory, filePath); if (!clusterAiImports[sourceRelative]?.includes(specifier)) { diff --git a/scripts/ql3-postgres-ha-contract.cjs b/scripts/ql3-postgres-ha-contract.cjs index b15d111e..6a7f7f8e 100644 --- a/scripts/ql3-postgres-ha-contract.cjs +++ b/scripts/ql3-postgres-ha-contract.cjs @@ -81,6 +81,13 @@ const { const { PostgresCopilotFailureDiagnosisModelRepository, } = require('../packages/ql3-ai/dist/copilot/failure-diagnosis/postgresModelExecutionRepository.js'); +const { + CopilotFailureDiagnosisModelCompletionCoordinator, + executeCopilotFailureDiagnosisModel, +} = require('../packages/ql3-ai/dist/copilot/failure-diagnosis/modelExecution.js'); +const { + CopilotFailureDiagnosisReadService, +} = require('../packages/ql3-ai/dist/copilot/failure-diagnosis/read-model/service.js'); const { BUILTIN_RUN_LOG_EXCERPT_TOOL, BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION, @@ -2285,6 +2292,254 @@ async function executeCopilotFailureDiagnosisToolHaEvidence(options) { } } +async function copilotFailureDiagnosisReadFacts(pool, fixture, outputKeyByte) { + const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository( + pool, + ); + const models = new PostgresCopilotFailureDiagnosisModelRepository(pool); + const terminalizations = + new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository(pool); + const permissions = []; + let resolvedKey; + const reader = new CopilotFailureDiagnosisReadService({ + admissions, + terminalizations, + finalizations: models, + models, + authorizer: { + async authorize(_principal, projectId, permission) { + assert.equal(projectId, fixture.plan.projectId); + permissions.push(permission); + return { + effect: 'allow', + reasons: ['role_grant'], + fence: { projectVersion: 1, bindingVersion: 1 }, + }; + }, + }, + keys: { + async active() { + throw new Error('read path must not select the active output key'); + }, + async resolve(keyId) { + if (keyId !== 'ha-copilot-output-key-1') return null; + resolvedKey = Buffer.alloc(32, outputKeyByte); + return { keyId, key: resolvedKey }; + }, + }, + now: () => fixture.baseTimeMs + 1_000, + }); + const target = { + principal: fixture.principal, + projectId: fixture.plan.projectId, + sourceRunId: fixture.plan.source.runId, + requestId: fixture.plan.requestId, + }; + const inspection = await reader.inspect(target); + const output = await reader.readOutput(target); + assert.equal(inspection.status, 'terminal'); + assert.equal(inspection.outcome, 'succeeded'); + assert.equal(inspection.stage, 'model'); + assert.equal(inspection.outputAvailable, true); + assert.deepEqual(inspection.usage, { + inputTokens: 17, + outputTokens: 9, + totalTokens: 26, + currency: null, + costMicros: null, + }); + assert.equal(output.status, 'available'); + assert.equal(output.result.text, 'HA encrypted diagnosis output'); + assert.equal('provider' in output.result, false); + assert.equal('model' in output.result, false); + assert.deepEqual(permissions, ['run.read', 'artifact.read']); + assert.ok(resolvedKey); + assert.equal( + resolvedKey.every((byte) => byte === 0), + true, + ); + return { + inspection, + output: { + status: output.status, + diagnosisRunId: output.diagnosisRunId, + reference: output.reference, + finishReason: output.result.finishReason, + usage: output.result.usage, + textDigest: createHash('sha256') + .update(output.result.text, 'utf8') + .digest('hex'), + providerAbsent: !('provider' in output.result), + modelAbsent: !('model' in output.result), + resolvedKeyWiped: true, + }, + }; +} + +async function executeCopilotFailureDiagnosisModelReadHaEvidence(options) { + const { + port, + report, + fixture, + toolExecution, + outputKeyByte, + expectedExisting = false, + } = options; + const runtimeDatabase = await databaseOpener( + 'runtime', + databaseUrl(RUNTIME_USER, RUNTIME_PASSWORD, port), + expectedExisting + ? 'ql3-ha-copilot-read-promoted' + : 'ql3-ha-copilot-read-primary', + )(); + const models = new PostgresCopilotFailureDiagnosisModelRepository( + runtimeDatabase.pool, + ); + const durable = new DurableModelInvocationCoordinator(models); + const outputKeys = { + async active() { + return { + keyId: 'ha-copilot-output-key-1', + key: Buffer.alloc(32, outputKeyByte), + }; + }, + async resolve(keyId) { + return keyId === 'ha-copilot-output-key-1' + ? { keyId, key: Buffer.alloc(32, outputKeyByte) } + : null; + }, + }; + const successfulCompletion = + new CopilotFailureDiagnosisModelCompletionCoordinator({ + coordinator: durable, + keys: outputKeys, + now: () => fixture.baseTimeMs + 700, + nonceFactory: () => Buffer.alloc(12, 0x73), + }); + const result = Object.freeze({ + provider: fixture.plan.model.provider, + model: fixture.plan.model.model, + text: 'HA encrypted diagnosis output', + finishReason: 'stop', + usage: Object.freeze({ + inputTokens: 17, + outputTokens: 9, + totalTokens: 26, + }), + }); + let providerCalls = 0; + const gateway = { + supportsSuccessfulCompletionSink(sink) { + return sink === successfulCompletion; + }, + async generate(request, context) { + providerCalls += 1; + assert.equal(request.provider, fixture.plan.model.provider); + assert.equal(request.model, fixture.plan.model.model); + assert.equal(context.requestId, fixture.plan.modelInvocationId); + const requestDigest = `sha256:${createHash('sha256') + .update(JSON.stringify(request), 'utf8') + .digest('hex')}`; + const common = { + projectId: fixture.plan.projectId, + runId: fixture.plan.runId, + stepRunId: fixture.plan.modelStepRunId, + traceId: fixture.plan.traceId, + requestId: fixture.plan.modelInvocationId, + provider: fixture.plan.model.provider, + model: fixture.plan.model.model, + policyRevision: 'ha-diagnosis-model-policy-v1', + requestDigest, + deadlineAtMs: fixture.plan.deadlineAtMs, + inputBytes: Buffer.byteLength(JSON.stringify(request), 'utf8'), + maxOutputTokens: fixture.plan.model.maxOutputTokens, + }; + await durable.record({ + ...common, + phase: 'admitted', + outputBytes: 0, + usage: null, + errorCode: null, + occurredAtMs: fixture.baseTimeMs + 500, + }); + const recorded = await successfulCompletion.record( + { + ...common, + phase: 'completed', + outputBytes: Buffer.byteLength(result.text, 'utf8'), + usage: result.usage, + errorCode: null, + occurredAtMs: fixture.baseTimeMs + 600, + }, + result, + ); + assert.equal(recorded.handled, true); + return result; + }, + }; + const dependencies = { + admissions: new PostgresCopilotFailureDiagnosisAdmissionRepository( + runtimeDatabase.pool, + ), + unlocks: new PostgresCopilotFailureDiagnosisToolUnlockRepository( + runtimeDatabase.pool, + ), + toolResults: { + async open() { + if (!toolExecution) { + throw new Error('promoted replay must not reopen Tool output'); + } + return { + status: toolExecution.completionStatus, + completion: toolExecution.completion, + output: toolExecution.output, + }; + }, + }, + modelInvocations: models, + outputs: models, + gateway, + successfulCompletion, + finalizations: models, + }; + try { + const first = await executeCopilotFailureDiagnosisModel( + report.requestId, + dependencies, + ); + assert.equal(first.outcome, 'succeeded'); + assert.ok(first.output); + if (!expectedExisting) { + const replay = await executeCopilotFailureDiagnosisModel( + report.requestId, + dependencies, + ); + assert.deepEqual(replay, first); + assert.equal(providerCalls, 1); + } else { + assert.equal(providerCalls, 0); + } + const facts = await copilotFailureDiagnosisReadFacts( + runtimeDatabase.pool, + fixture, + outputKeyByte, + ); + if (expectedExisting) { + report.afterPromotion = facts; + report.promotedProviderCalls = providerCalls; + report.survivedPromotion = true; + } else { + report.beforePromotion = facts; + report.providerCalls = providerCalls; + report.exactReplay = true; + report.encryptedOutputReadable = true; + } + return facts; + } finally { + await runtimeDatabase.close(); + } +} + async function terminalizeCopilotFailureDiagnosisLogUnavailableHaEvidence( options, ) { @@ -10976,6 +11231,8 @@ async function main(argv = process.argv.slice(2)) { let copilotFailureDiagnosisTerminalizationFixture; let copilotFailureDiagnosisModelResolution; let copilotFailureDiagnosisModelResolutionFixture; + let copilotFailureDiagnosisRead; + let copilotFailureDiagnosisReadFixture; let modelProviderCredentialCatalog; let modelProviderCredentialTestConnection; let runAttemptLogRetentionEvidence; @@ -11186,6 +11443,12 @@ async function main(argv = process.argv.slice(2)) { copilotFailureDiagnosisModelResolution = copilotModelResolution.report; copilotFailureDiagnosisModelResolutionFixture = copilotModelResolution.fixture; + const copilotRead = await runCopilotFailureDiagnosisAdmissionHaEvidence({ + primaryPort, + migrationPool: migrationDatabase.pool, + }); + copilotFailureDiagnosisRead = copilotRead.report; + copilotFailureDiagnosisReadFixture = copilotRead.fixture; } finally { await migrationDatabase.close(); } @@ -11590,6 +11853,35 @@ async function main(argv = process.argv.slice(2)) { state: 'copilot_failure_diagnosis_tool_completed', atMs: Number((performance.now() - startedAt).toFixed(3)), }); + const copilotReadTool = await executeCopilotFailureDiagnosisToolHaEvidence({ + port: primaryPort, + report: copilotFailureDiagnosisRead, + fixture: copilotFailureDiagnosisReadFixture, + resultKeyFixture: toolResultKeyFixture, + }); + await executeCopilotFailureDiagnosisModelReadHaEvidence({ + port: primaryPort, + report: copilotFailureDiagnosisRead, + fixture: copilotFailureDiagnosisReadFixture, + toolExecution: copilotReadTool.execution, + outputKeyByte: 0x74, + }); + await waitFor(async () => { + const replicated = await copilotFailureDiagnosisReadFacts( + standbyDatabase.pool, + copilotFailureDiagnosisReadFixture, + 0x74, + ); + return JSON.stringify(replicated) === + JSON.stringify(copilotFailureDiagnosisRead.beforePromotion) + ? replicated + : null; + }, 'Copilot failure diagnosis request-keyed read WAL replay'); + copilotFailureDiagnosisRead.replicatedBeforePromotion = true; + timeline.push({ + state: 'copilot_failure_diagnosis_output_readable', + atMs: Number((performance.now() - startedAt).toFixed(3)), + }); await terminalizeCopilotFailureDiagnosisLogUnavailableHaEvidence({ port: primaryPort, report: copilotFailureDiagnosisTerminalization, @@ -12546,6 +12838,14 @@ async function main(argv = process.argv.slice(2)) { resultKeyFixture: toolResultKeyFixture, expectedExisting: true, }); + await executeCopilotFailureDiagnosisModelReadHaEvidence({ + port: standbyPort, + report: copilotFailureDiagnosisRead, + fixture: copilotFailureDiagnosisReadFixture, + toolExecution: null, + outputKeyByte: 0x74, + expectedExisting: true, + }); await verifyModelProviderCredentialCatalogAfterPromotion({ promotedPort: standbyPort, promotedPool: promotedDatabase.pool, @@ -13296,11 +13596,27 @@ async function main(argv = process.argv.slice(2)) { `SELECT (SELECT count(*)::integer FROM "ql3"."runs") AS runs, (SELECT count(*)::integer FROM "ql3"."run_events") AS "runEvents", + (SELECT count(*)::integer + FROM "ql3"."runs" + WHERE id IN ($1, $2)) AS "diagnosisReadRuns", + (SELECT count(*)::integer + FROM "ql3"."run_events" + WHERE run_id = $2) AS "diagnosisReadRunEvents", (SELECT count(*)::integer FROM "ql3"."worker_credential_deliveries") AS "credentialDeliveries"`, + [ + copilotFailureDiagnosisRead.sourceRunId, + copilotFailureDiagnosisRead.runId, + ], ); assert.deepEqual(sideEffects.rows, [ - { runs: 82, runEvents: 200, credentialDeliveries: 4 }, + { + runs: 84, + runEvents: 209, + diagnosisReadRuns: 2, + diagnosisReadRunEvents: 9, + credentialDeliveries: 4, + }, ]); timeline.push({ state: 'two_fresh_control_replicas_ready', @@ -13417,6 +13733,7 @@ async function main(argv = process.argv.slice(2)) { copilotFailureDiagnosisAdmission, copilotFailureDiagnosisTerminalization, copilotFailureDiagnosisModelResolution, + copilotFailureDiagnosisRead, modelProviderCredentialCatalog, modelProviderCredentialTestConnection, runAttemptLogRetention, @@ -13509,6 +13826,23 @@ async function main(argv = process.argv.slice(2)) { JSON.stringify( copilotFailureDiagnosisModelResolution.beforePromotion, ), + copilotFailureDiagnosisRequestKeyedReadDecryptsAndExactlyReplays: + copilotFailureDiagnosisRead.exactReplay && + copilotFailureDiagnosisRead.encryptedOutputReadable && + copilotFailureDiagnosisRead.providerCalls === 1 && + copilotFailureDiagnosisRead.beforePromotion.inspection.status === + 'terminal' && + copilotFailureDiagnosisRead.beforePromotion.inspection.outcome === + 'succeeded' && + copilotFailureDiagnosisRead.beforePromotion.output.providerAbsent && + copilotFailureDiagnosisRead.beforePromotion.output.modelAbsent && + copilotFailureDiagnosisRead.beforePromotion.output.resolvedKeyWiped, + copilotFailureDiagnosisRequestKeyedReadReplicatesAndSurvivesPromotion: + copilotFailureDiagnosisRead.replicatedBeforePromotion && + copilotFailureDiagnosisRead.survivedPromotion && + copilotFailureDiagnosisRead.promotedProviderCalls === 0 && + JSON.stringify(copilotFailureDiagnosisRead.afterPromotion) === + JSON.stringify(copilotFailureDiagnosisRead.beforePromotion), modelProviderCredentialCatalogSurvivesPromotion: modelProviderCredentialCatalog.survivedPromotion, modelProviderCredentialManagementIdentityLedgerSurvivesPromotion: diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index 05065c3f..63948bf1 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -268,10 +268,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( { path: 'packages/ql3-ai', name: '@qinglong/ai', - sourceFiles: 192, + sourceFiles: 193, rootSourceFiles: 1, rootSourceLines: 16, - nestedSourceFiles: 191, + nestedSourceFiles: 192, rootSourceFileHardCap: 1, rootSourceLineHardCap: 16, rootSourceFileRoles: { @@ -385,10 +385,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: clusterControl.rootSourceFileRoles, }, { - sourceFiles: 60, + sourceFiles: 62, rootSourceFiles: 2, rootSourceLines: 195, - nestedSourceFiles: 58, + nestedSourceFiles: 60, rootSourceFileRoles: { 'aiCli.ts': 'binary_entry', 'cli.ts': 'binary_entry',