From 0163c94be77e0ec0aca5376a34160a395bf449e8 Mon Sep 17 00:00:00 2001 From: whyour Date: Sat, 15 Aug 2026 15:07:49 +0800 Subject: [PATCH] feat(ql3): admit copilot diagnosis runs --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 12 +- ...copilot-failure-diagnosis-run-admission.md | 87 ++ docs/adr/README.md | 1 + packages/ql3-ai/package.json | 10 + .../failure-diagnosis/admission/contracts.ts | 191 +++++ .../admission/durableEvidence.ts | 274 +++++++ .../failure-diagnosis/admission/plan.ts | 315 ++++++++ .../admission/postgresRepository.ts | 760 ++++++++++++++++++ .../failure-diagnosis/admission/validation.ts | 416 ++++++++++ .../failure-diagnosis/executionAdmission.ts | 8 + .../postgresAdmissionRepository.ts | 4 + .../migration/model-invocation/identities.ts | 2 + .../migration/model-invocation/postgres.ts | 9 +- .../model-invocation/postgres/copilot.ts | 220 +++++ ...ailureDiagnosisExecutionAdmission.test.cjs | 325 ++++++++ .../test/modelInvocationMigration.test.cjs | 38 +- ...redentialManagementIdentityLedger.test.cjs | 5 +- ...elProviderCredentialTestExecution.test.cjs | 2 +- .../src/run-recovery/clusterRecoverySource.ts | 5 +- .../test/clusterRecoverySource.test.cjs | 11 +- .../clusterControlActivation.ts | 5 +- .../test/clusterControlActivation.test.cjs | 2 +- scripts/ql3-postgres-ha-contract.cjs | 394 ++++++++- test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 24 files changed, 3078 insertions(+), 22 deletions(-) create mode 100644 docs/adr/ADR-0407-cluster-copilot-failure-diagnosis-run-admission.md create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/admission/contracts.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/admission/durableEvidence.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/admission/plan.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/admission/postgresRepository.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/admission/validation.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/executionAdmission.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/postgresAdmissionRepository.ts create mode 100644 packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts create mode 100644 packages/ql3-ai/test/failureDiagnosisExecutionAdmission.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 88448fde..56a2b53d 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -6,11 +6,12 @@ - 目标版本：QingLong 3.x - 作者：QingLong Maintainers - 创建日期：2026-07-17 -- 最后更新：2026-08-14 +- 最后更新：2026-08-15 - 讨论范围：架构与演进路线，不包含最终 UI 视觉方案 -最新增量证据（2026-08-14）： +最新增量证据（2026-08-15）： +- D-315/ADR-0407（已接受）：Cluster Copilot 故障诊断不修改或重新打开终态源 Run，而是用源 Run/version、最新已结束 Attempt/status 与日志 Artifact 建立 exact fence，原子创建独立 `copilot_failure_diagnosis` Run、admission event、`ready` 的 `qinglong.run.log.excerpt@1.0.0` Tool Step、以其为父节点的 `pending` Model Step 和不可变 receipt。计划只接受 `cluster-control` 的 reviewed snapshot/binding/Policy/subject authority；`approval_required`、Tool contract/输入漂移或未显式允许的模型出口均在写库前失败关闭。`ql3_ai` 新增有界 admission ledger 与仅授予 runtime 的 `SECURITY DEFINER` source snapshot，在 SERIALIZABLE 事务内重验 active Project/binding 和源 Run/Attempt；response-loss replay 对 JSONB 做结构 exact 比较而不依赖无语义的对象键序。Copilot/Plugin 这类 StepRun 聚合从通用 Task orphan recovery 排除，后续由各自状态机恢复，不能伪造顶层 Attempt 或被错误终态化。本阶段只 admission，不执行 Tool/模型、不授予行动权；Tool encrypted completion、Model 解锁和 Copilot encrypted model completion 是下一 Gate。最终 18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项边界审计零 finding且 workspace 仍为 18 package、无单文件/浅平 package；14 档 Local Profile artifact 全通过，默认 Edge/Standalone 不引入 Cluster admission。PostgreSQL 18.4 arm64 HA 128/128、timeline `1→2`，报告 SHA-256 为 `a4ed1edec783e3f5b42507c0f8e11b94c59dbe44a57e691017d1445ec9d115e2`，证据审计与 Docker 清理通过。 - D-314/ADR-0406（已接受）：Cluster Trusted Tool 的 encrypted completion 不再停留在 storage port。`@qinglong/cluster-control/trusted-tool-result-keyring` 新增只读 projected material authority：canonical v1 manifest 只含最多 16 个 canonical 32-byte key，不含 generation、active/state/retirement，provider 只有 `resolve(keyId)` 而没有 `active()`；因此 PostgreSQL `trusted-tool-results` catalog 仍是 active/decryptable 状态唯一 authority，completion 会以 catalog material proof 再次校验。runtime 每次调用重新执行 direct-root、in-root atomic symlink、single-link、64 KiB、只读/不可执行/other-inaccessible mode、dev/inode/size/mtime 与双 realpath fence，不持有 Kubernetes API、cache、watcher 或 timer。新增能力位于 Cluster Control 既有 `trusted-tool/key-management/`，并把 mounted Secret 与 keyring 的 projected-file/TOCTOU 逻辑收敛到 package-private `security/privateProjectedFile` 真源；公开 mounted Secret 行为不变，不新增 package、依赖、migration、连接、route 或默认 Profile importer。定向共享回归 7/7，Cluster Control 完整 234 pass/2 条条件 skip/0 fail；最终 18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，package/dependency/Edge import/Cluster deployment 四项审计零 finding。workspace 仍为 18 package、无单文件或浅平 package；Cluster Control 54 个源码中仅 2 个 binary entry 位于根层，52 个处于嵌套领域目录。14 档 Local Profile artifact 全部通过，默认 Edge/Standalone 保持 2,589,812/2,589,890 bytes，AI 保持 3,121,108/3,121,198 bytes，MCP 保持 7,315,930/7,316,038 bytes。PostgreSQL 18.4 arm64 HA 125/125、timeline `1→2`，报告 SHA-256 为 `26c817647ed984d8d4627a7cae1c95de06017a5d6d32dd3dfd01414ba029e542`，证据审计与 Docker 容器/网络/卷零残留。下一 Gate 是独立 diagnosis Run 的 Tool/Model Step admission 与 Copilot encrypted model completion，不能借用终态源 Run 或 Plugin Prompt plan。 - D-313/ADR-0405（已接受）：新增 `@qinglong/ai/failure-diagnosis-prompt`，把 ADR-0403 的潜在敏感日志投影收敛为固定 system instruction + canonical JSON data envelope；日志只存在于 `log.content` JSON string value，不能通过引号、换行、伪造 role/schema 或 delimiter 拼接出新 message。builder 重新校验完整 trust/redaction/profile byte 契约，拒绝伪造 `safe`、行动权、未知字段与 byte/signal drift；envelope 不带 Run/Attempt、Artifact、path、cursor 或 content digest。部署者必须通过 `qinglong/copilot-model-egress-policy@v1` 显式允许 `potentially_sensitive` 数据进入 `on_device|external` 边界并提供输入/output token 双预算，空 allowlist 与 external 未授权均在 Model Gateway/Provider I/O 前失败关闭。输出只含 content-free egress evidence，并固定要求模型 completion 继承潜在敏感、仅加密持久化、禁止明文审计且无行动权；真正 Cluster Trusted Tool/model completion 仍需后续组合门。能力以 `ql3-ai/src/copilot/failure-diagnosis/` 三个内聚文件和精确 subpath 交付，不新增 package、依赖、迁移、连接或常驻组件。定向 12/12、AI 221 pass/3 条件 skip/0 fail；最终 18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项结构/部署审计零 finding，14 档 Local Profile artifact 全部通过。默认 Edge/Standalone 保持 2,589,812/2,589,890 bytes、315 files、56 modules，Edge/Standalone AI 保持 3,121,108/3,121,198 bytes、368 files、61 modules，MCP 保持 7,315,930/7,316,038 bytes、801 files、226 modules，证明未装配 subpath 被完全裁掉。PostgreSQL 18.4 arm64 HA 125/125、timeline `1→2`，报告 SHA-256 为 `2bbc8bdd0d90e6ec9ce82d2afcaec817679dddb82860c5d405a09d5e5458bece`，证据审计与 Docker 零残留。 - D-312/ADR-0404（已接受）：`qinglong.run.log.excerpt@1.0.0` 进入显式可选的本机 `ql3-mcp` stdio 产品入口。每次调用固定经过 Owner credential authentication、exact `tool.call:qinglong.run.log.excerpt` + `artifact.read` Policy、durable audit、credential/Pepper fence confirm，再复用同一 SQLite authority 和私有 Artifact reader 完成 ADR-0403 的 Edge 4 KiB/Standalone 8 KiB 双读取安全投影。配置升级为 `qinglong/local-mcp-server@v2` 并要求显式 private `artifactRoot`，旧 v1 不猜测路径而是失败关闭。产物实证否决了 MCP 直接依赖 `local-execution` 的方案：该方案会带入 process/scheduler/croner，达到 7,469,105 bytes/816 files/228 modules；唯一 reader 实现因此归入既有 `local-command-file/artifact-read` 私有文件 authority，Execution 通过兼容 re-export 复用，workspace 仍为 18 package 且没有根层平铺。Local MCP 48/48、Local Execution 41/41、私有文件 3/3、依赖防火墙 54/54；最终 18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项结构/部署审计零 finding，14 个 Local Profile artifact 全部通过。默认 Edge/Standalone 保持 2,589,812/2,589,890 bytes、315 files、56 modules，Edge/Standalone MCP 为 7,315,930/7,316,038 bytes、801 files、226 modules、RSS 38,420,480/39,567,360 bytes，闭包不含 `local-execution`、`local-process` 或 `croner`。PostgreSQL 18.4 arm64 HA 125/125、timeline `1→2`，报告 SHA-256 为 `29cd77d80737a3b1ab686c998d05a78c52deffd8add3b31d8035756d5dfcc433`，证据审计与 Docker 零残留。 @@ -6937,6 +6938,13 @@ ADR-0406 已关闭其中的 Cluster result-key material 缺口：只读 projecte Copilot execution authority 已完成；下一 Gate 必须创建独立 diagnosis Run 的 Tool/Model Step admission， 再把 S3 Tool completion、ADR-0405 builder、Model Gateway 与 Copilot 专用 encrypted model completion 串成可恢复闭环。 +ADR-0407 已完成该 admission Gate：源失败/超时 Run 和最新已结束 Attempt 只作为不可变 source fence， +独立 diagnosis Run 在一个 SERIALIZABLE 事务内同时获得 admission event、ready Tool Step、依赖它的 +pending Model Step 与 receipt；Cluster runtime 的 source snapshot authority 会重验 active Project/binding +和日志 Artifact，response-loss replay 以结构语义比较 JSONB。该 Run 是 StepRun 编排聚合，不进入通用 +Task orphan recovery。当前阶段仍没有执行 Tool 或模型；下一 Gate 必须先形成可信 encrypted Tool +completion，再原子解锁 Model Step，并以 ADR-0405 egress policy 和 Copilot 专用 encrypted model +completion 收敛最终 Run。 ## 17. Tool Registry diff --git a/docs/adr/ADR-0407-cluster-copilot-failure-diagnosis-run-admission.md b/docs/adr/ADR-0407-cluster-copilot-failure-diagnosis-run-admission.md new file mode 100644 index 00000000..5c80cdd8 --- /dev/null +++ b/docs/adr/ADR-0407-cluster-copilot-failure-diagnosis-run-admission.md @@ -0,0 +1,87 @@ +# ADR-0407：Cluster Copilot Failure Diagnosis Run Admission + +- 状态：Accepted +- 日期：2026-08-15 +- 关联 RFC：QL-RFC-0001 D-315、Phase 2 +- 关联 ADR：ADR-0163、ADR-0226、ADR-0403、ADR-0404、ADR-0405、ADR-0406 + +## 问题 + +ADR-0403~0406 已经给出有界日志摘录 Tool、显式模型出口策略和 Cluster result-key material +authority，但仍缺少把一次故障诊断变成 durable execution 的入口。源 Run 已经处于 `failed` 或 +`timed_out` 终态，不能向它追加 Tool/Model Step，也不能复用 Plugin Package Prompt admission +冒充 Copilot 工作流。 + +若仅在 API 内顺序调用 Tool 和模型，进程崩溃、响应丢失或 PostgreSQL failover 会让系统无法证明 +已经接受了什么、下一步应该执行什么；若 admission 直接启动模型，模型又可能在 Tool 结果尚未形成 +可信 encrypted completion 时读取未经 fencing 的数据。通用 Task recovery 也不能把这种以 StepRun +编排、没有顶层 RunAttempt 的聚合 Run 当成普通孤儿任务。 + +## 决策 + +1. 故障诊断创建独立 `copilot_failure_diagnosis` Run，并以源 Run 作为 `parentRunId`。源 fence 必须精确 + 固定 Project、Run ID/version、`failed|timed_out` 状态、最新且已结束的 Attempt、兼容的 Attempt + 状态和非空日志 Artifact；admission 不修改或重新打开源 Run。 +2. 计划只接受 `cluster-control` Profile 中经过 snapshot、Policy、subject 与 binding fencing 的 + `qinglong.run.log.excerpt@1.0.0` 内建只读 Tool。`approval_required`、不同 Tool/version、不同 + adapter/redaction/audit contract、输入 Artifact 未绑定源 Attempt 或权限漂移全部在数据库写入前 + 失败关闭。 +3. 模型 intent 固定 provider/model、`on_device|external` 边界、响应语言、输出 token 上限和 ADR-0405 + egress policy digest。计划与 receipt 均有 domain-separated digest 和字节上限；它们声明最终模型 + completion 必须加密、审计不得保存明文且 `actionAuthority=none`，本阶段不授予命令或写 Tool 权限。 +4. 一个 SERIALIZABLE PostgreSQL 事务原子创建 diagnosis Run、admission event、`collect-log` Tool + StepRun、`diagnose` Model StepRun 和 admission receipt。Tool Step 初始为 `ready`；Model Step 以 Tool + Step 为父节点且初始为 `pending`。本阶段只 admission，不执行 Tool/模型；后续只有可信 Tool + completion 成功后才能解锁 Model Step。 +5. `ql3_ai.copilot_failure_diagnosis_admissions` 保存有界 plan/receipt JSONB 及关键列镜像。 + `SECURITY DEFINER` source snapshot 函数只授予 `ql3_runtime`，并在同一事务内重新确认 active Project、 + 最新 active subject binding、源 Run/Attempt 状态与日志 Artifact。request identity 支持 response-loss + replay；同 request 不同事实、任何 durable 镜像漂移或部分证据缺失均冲突或失败关闭。 +6. PostgreSQL JSONB 的对象键序不具有语义；replay 采用严格的结构深比较，而不是比较 + `JSON.stringify` 文本。数组顺序、值类型、缺失/新增字段仍保持 exact，不能借键序修复放宽证据。 +7. `copilot_failure_diagnosis` 与 `plugin_package_workflow` 都是 StepRun 编排的聚合 ownership domain， + 从通用 Task orphan recovery 查询中排除。它们必须由各自 StepRun 状态机恢复，不能伪造顶层 + RunAttempt，也不能因不存在 Attempt 而被自动终态化。Cluster activation 在 recovery 未收敛时输出 + remaining/failed 计数，便于 failover 诊断。 +8. 能力只通过 `@qinglong/ai` 精确 subpath 和既有 Cluster PostgreSQL migration stream 提供；不新增 + workspace package、进程、连接池、listener、timer、watcher 或 cache。默认 Edge/Standalone 及其 + AI/MCP 制品不导入 PostgreSQL admission，因此低配路由设备没有新增常驻成本。 + +## 被否决方案 + +1. **把 Step 追加到源失败 Run**：破坏终态 Run 不可变和 event/version 单调性。 +2. **复用 Plugin Package Prompt admission**：两者的 authority、Artifact 密文域、恢复与产品语义不同。 +3. **admission 后立即调用 Tool/模型**：无法在 response loss 与主库切换后证明执行边界，也会绕过 + Tool completion 对 Model Step 的依赖。 +4. **为 diagnosis Run 伪造顶层 Attempt**：把 StepRun 聚合误装成 Task 执行，并让两套恢复状态机争夺 + ownership。 +5. **用序列化字符串比较 JSONB**：PostgreSQL 会重排对象键，合法重放会被错误判为证据损坏。 +6. **为 admission 新建 package 或进入 Local Profile**：没有独立进程/制品边界，并向低配设备引入 + 无用的 Cluster/PostgreSQL 闭包。 + +## 当前验证 + +1. admission 定向测试 6/6，覆盖 exact Tool/subject/policy/source fencing、approval 拒绝、模型出口策略、 + 原子创建、response-loss replay、冲突/损坏证据和 JSONB 键序等价。 +2. recovery SQL 4/4、Cluster activation 6/6；HA 门额外断言 diagnosis 聚合不会进入通用 recovery + candidate。 +3. 18 个 QL3 package 的 clean build/test 全部通过；完整 backend 为 1,207 pass、2 条条件 skip、 + 0 fail。package boundary、Cluster dependency、Edge import 与 Cluster deployment 四项审计均为 + compatible 且零 finding；workspace 仍为 18 个 package，`singleSourcePackages=[]`、 + `shallowSourcePackages=[]`。 +4. 14 档 Local Profile artifact 全部通过，证明 Cluster-only admission 未进入默认 Edge/Standalone、 + AI 或 MCP 的本地部署闭包。 +5. PostgreSQL 18.4 arm64 physical HA 为 128/128 Gate、timeline `1→2`；报告 SHA-256 为 + `a4ed1edec783e3f5b42507c0f8e11b94c59dbe44a57e691017d1445ec9d115e2`，独立证据审计零 finding， + Docker 容器、网络与卷零残留。 + +## 后续门禁 + +1. 以 admission 中的 exact plan 启动 Tool Step，并复用 invocation Artifact、S3 reader、catalog 与 + ADR-0406 result-key provider 完成加密 Tool completion；response loss 不得重复执行 adapter。 +2. 只有受信 Tool Step 成功后才原子解锁 Model Step；用 ADR-0405 builder 生成 prompt，并由 Model + Gateway 执行 provider credential/配额/价格 fence。 +3. 建立 Copilot 专用 encrypted model completion、Run/Step terminalization、取消、deadline 和恢复协议， + 证明明文不进入数据库、审计、日志或普通 model completion。 +4. 最后开放默认关闭的 Cluster API/CLI/UI/MCP 产品入口，并补多副本并发、真实 S3、外部 Provider + fault injection 与 plaintext negative evidence。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 5b635fa2..dd0d7db3 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -410,6 +410,7 @@ | [ADR-0404](./ADR-0404-optional-local-mcp-run-log-excerpt-surface.md) | 可选本机 MCP Run 日志摘录产品入口与私有文件边界 | Accepted | | [ADR-0405](./ADR-0405-bounded-failure-diagnosis-prompt-and-model-egress-policy.md) | 有界故障诊断 Prompt 与显式模型出口策略 | Accepted | | [ADR-0406](./ADR-0406-cluster-projected-tool-result-key-authority.md) | Cluster Projected Tool Result Key Authority | Accepted | +| [ADR-0407](./ADR-0407-cluster-copilot-failure-diagnosis-run-admission.md) | Cluster Copilot Failure Diagnosis Run Admission | Accepted | ## 规则 diff --git a/packages/ql3-ai/package.json b/packages/ql3-ai/package.json index e02b67ba..825b2eb8 100644 --- a/packages/ql3-ai/package.json +++ b/packages/ql3-ai/package.json @@ -30,6 +30,16 @@ "require": "./dist/copilot/failure-diagnosis/prompt.js", "default": "./dist/copilot/failure-diagnosis/prompt.js" }, + "./failure-diagnosis-execution-admission": { + "types": "./dist/copilot/failure-diagnosis/executionAdmission.d.ts", + "require": "./dist/copilot/failure-diagnosis/executionAdmission.js", + "default": "./dist/copilot/failure-diagnosis/executionAdmission.js" + }, + "./postgres-failure-diagnosis-admission-storage": { + "types": "./dist/copilot/failure-diagnosis/postgresAdmissionRepository.d.ts", + "require": "./dist/copilot/failure-diagnosis/postgresAdmissionRepository.js", + "default": "./dist/copilot/failure-diagnosis/postgresAdmissionRepository.js" + }, "./model-invocation": { "types": "./dist/model-invocation/modelInvocation.d.ts", "require": "./dist/model-invocation/modelInvocation.js", diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/admission/contracts.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/admission/contracts.ts new file mode 100644 index 00000000..8bc9094d --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/admission/contracts.ts @@ -0,0 +1,191 @@ +import type { RunEventRecord, RunRecord } from '@qinglong/runtime-core'; +import type { + SecurityPolicyFence, + SecuritySubject, +} from '@qinglong/runtime-core/security'; +import type { StepRunMutation } from '@qinglong/runtime-core/step-run'; +import type { + ToolInvocationInputArtifactReference, + ToolInvocationPreviewArtifactReference, +} from '@qinglong/runtime-core/tool-invocation-artifact'; +import type { + TrustedToolHandlerBindingRegistry, + TrustedToolInvocationPlan, +} from '@qinglong/runtime-core/trusted-tool-invocation'; + +import type { + FailureDiagnosisModelBoundary, + FailureDiagnosisModelEgressPolicy, + FailureDiagnosisResponseLanguage, +} from '../contracts'; + +export const COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_SCHEMA = + 'qinglong/copilot-failure-diagnosis-execution-plan@v1' as const; +export const COPILOT_FAILURE_DIAGNOSIS_ADMISSION_RECEIPT_SCHEMA = + 'qinglong/copilot-failure-diagnosis-admission-receipt@v1' as const; +export const COPILOT_FAILURE_DIAGNOSIS_SOURCE_RUN_STATUSES = [ + 'failed', + 'timed_out', +] as const; +export const COPILOT_FAILURE_DIAGNOSIS_SOURCE_ATTEMPT_STATUSES = [ + 'failed', + 'timed_out', + 'lost', +] as const; +export const MAX_COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_BYTES = 32 * 1024; +export const MAX_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_RECEIPT_BYTES = 16 * 1024; + +export type CopilotFailureDiagnosisSourceRunStatus = + (typeof COPILOT_FAILURE_DIAGNOSIS_SOURCE_RUN_STATUSES)[number]; +export type CopilotFailureDiagnosisSourceAttemptStatus = + (typeof COPILOT_FAILURE_DIAGNOSIS_SOURCE_ATTEMPT_STATUSES)[number]; + +export interface CopilotFailureDiagnosisSourceFence { + readonly runId: string; + readonly runVersion: number; + readonly runStatus: CopilotFailureDiagnosisSourceRunStatus; + readonly attemptId: string; + readonly attemptStatus: CopilotFailureDiagnosisSourceAttemptStatus; + readonly attemptFinishedAtMs: number; + readonly logArtifactId: string; +} + +export interface PrepareCopilotFailureDiagnosisModelIntent { + readonly provider: string; + readonly model: string; + readonly modelBoundary: FailureDiagnosisModelBoundary; + readonly responseLanguage: FailureDiagnosisResponseLanguage; + readonly maxOutputTokens: number; + readonly egressPolicy: Readonly; +} + +export interface CopilotFailureDiagnosisModelIntent + extends PrepareCopilotFailureDiagnosisModelIntent { + readonly egressPolicyDigest: string; + readonly intentDigest: string; +} + +export interface CopilotFailureDiagnosisToolIntent { + readonly actionRef: string; + readonly planDigest: string; + readonly actionDigest: string; + readonly invocationActionDigest: string; + readonly snapshotDigest: string; + readonly definitionDigest: string; + readonly bindingDigest: string; + readonly invocationArtifact: Readonly; + readonly previewArtifact: Readonly; + readonly sealedAtMs: number; +} + +export interface PrepareCopilotFailureDiagnosisExecutionInput { + readonly requestId: string; + readonly traceId: string; + readonly source: Readonly; + readonly toolPlan: Readonly; + readonly bindings: TrustedToolHandlerBindingRegistry; + readonly model: Readonly; + readonly deadlineAtMs: number; + readonly plannedAtMs: number; +} + +export interface CopilotFailureDiagnosisExecutionPlan { + readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_SCHEMA; + readonly requestId: string; + readonly runId: string; + readonly toolStepRunId: string; + readonly modelStepRunId: string; + readonly modelInvocationId: string; + readonly traceId: string; + readonly projectId: string; + readonly requestedBySubject: Readonly; + readonly policyFence: Readonly; + readonly source: Readonly; + readonly tool: Readonly; + readonly model: Readonly; + readonly deadlineAtMs: number; + readonly plannedAtMs: number; + readonly planDigest: string; +} + +export interface CopilotFailureDiagnosisAdmissionReceipt { + readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_ADMISSION_RECEIPT_SCHEMA; + readonly requestId: string; + readonly planDigest: string; + readonly runId: string; + readonly sourceRunId: string; + readonly sourceRunVersion: number; + readonly sourceAttemptId: string; + readonly toolStepRunId: string; + readonly toolStepRunDigest: string; + readonly toolMutationId: string; + readonly toolEventId: string; + readonly modelStepRunId: string; + readonly modelStepRunDigest: string; + readonly modelMutationId: string; + readonly modelEventId: string; + readonly finalRunVersion: 3; + readonly finalRunEventSequence: 3; + readonly admittedAtMs: number; + readonly receiptDigest: string; +} + +export interface CopilotFailureDiagnosisAdmissionBundle { + readonly plan: Readonly; + readonly run: Readonly; + readonly admissionEvent: Readonly; + readonly toolStepMutation: Readonly; + readonly modelStepMutation: Readonly; + readonly receipt: Readonly; +} + +export interface CopilotFailureDiagnosisAdmissionRepository { + findByRequestId( + requestId: string, + ): Promise | null>; + findPlanByRequestId( + requestId: string, + ): Promise | null>; + admit(plan: Readonly): Promise< + Readonly<{ + status: 'created' | 'existing'; + receipt: Readonly; + }> + >; +} + +export class InvalidCopilotFailureDiagnosisExecutionPlanError extends TypeError { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_INVALID'; + + constructor(message: string) { + super(`Copilot failure diagnosis execution plan is invalid: ${message}`); + this.name = 'InvalidCopilotFailureDiagnosisExecutionPlanError'; + } +} + +export class CopilotFailureDiagnosisAdmissionConflictError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_ADMISSION_CONFLICT'; + + constructor(message = 'durable diagnosis admission identity changed') { + super(`Copilot failure diagnosis admission conflicts: ${message}`); + this.name = 'CopilotFailureDiagnosisAdmissionConflictError'; + } +} + +export class CopilotFailureDiagnosisAdmissionNotAllowedError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_ADMISSION_NOT_ALLOWED'; + + constructor() { + super('The source Run is not eligible for failure diagnosis'); + this.name = 'CopilotFailureDiagnosisAdmissionNotAllowedError'; + } +} + +export class CopilotFailureDiagnosisAdmissionUnavailableError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_ADMISSION_UNAVAILABLE'; + + constructor(options?: ErrorOptions) { + super('Copilot failure diagnosis admission is unavailable', options); + this.name = 'CopilotFailureDiagnosisAdmissionUnavailableError'; + } +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/admission/durableEvidence.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/admission/durableEvidence.ts new file mode 100644 index 00000000..b529ae08 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/admission/durableEvidence.ts @@ -0,0 +1,274 @@ +import { Buffer } from 'node:buffer'; + +import type { RunEventRecord, RunRecord } from '@qinglong/runtime-core'; +import { + createStepRunMutation, + normalizeStepRunMutation, +} from '@qinglong/runtime-core/step-run'; + +import { + COPILOT_FAILURE_DIAGNOSIS_ADMISSION_RECEIPT_SCHEMA, + MAX_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_RECEIPT_BYTES, + type CopilotFailureDiagnosisAdmissionBundle, + type CopilotFailureDiagnosisAdmissionReceipt, + type CopilotFailureDiagnosisExecutionPlan, +} from './contracts'; +import { normalizeCopilotFailureDiagnosisExecutionPlan } from './plan'; +import { + dataRecord, + digest, + exactKeys, + hash, + identity, + integer, + invalid, + runIdentity, + timestamp, +} from './validation'; + +const EVIDENCE_IDENTITY_DOMAIN = + 'qinglong/copilot-failure-diagnosis-admission-evidence-identity@v1\0'; +const RECEIPT_DIGEST_DOMAIN = + 'qinglong/copilot-failure-diagnosis-admission-receipt-digest@v1\0'; + +function evidenceIdentity( + prefix: 'cda' | 'cdtm' | 'cdte' | 'cdmm' | 'cdme', + planDigest: string, +): string { + const maximumDigestLength = 35 - prefix.length; + return `${prefix}:${hash(EVIDENCE_IDENTITY_DOMAIN, { + prefix, + planDigest, + }).slice(0, maximumDigestLength)}`; +} + +function receiptFields( + value: Omit, +): object { + return { ...value }; +} + +export function copilotFailureDiagnosisAdmissionReceiptDigest( + value: Omit, +): string { + return hash(RECEIPT_DIGEST_DOMAIN, receiptFields(value)); +} + +export function normalizeCopilotFailureDiagnosisAdmissionReceipt( + value: CopilotFailureDiagnosisAdmissionReceipt, +): Readonly { + const candidate = dataRecord(value, 'admission receipt'); + exactKeys( + candidate, + [ + 'admittedAtMs', + 'finalRunEventSequence', + 'finalRunVersion', + 'modelEventId', + 'modelMutationId', + 'modelStepRunDigest', + 'modelStepRunId', + 'planDigest', + 'receiptDigest', + 'requestId', + 'runId', + 'schema', + 'sourceAttemptId', + 'sourceRunId', + 'sourceRunVersion', + 'toolEventId', + 'toolMutationId', + 'toolStepRunDigest', + 'toolStepRunId', + ], + 'admission receipt', + ); + if (candidate.schema !== COPILOT_FAILURE_DIAGNOSIS_ADMISSION_RECEIPT_SCHEMA) { + return invalid('admission receipt schema is unsupported'); + } + const unsigned = Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_ADMISSION_RECEIPT_SCHEMA, + requestId: identity(candidate.requestId, 'request id'), + planDigest: digest(candidate.planDigest, 'plan digest'), + runId: runIdentity(candidate.runId, 'diagnosis Run id'), + sourceRunId: runIdentity(candidate.sourceRunId, 'source Run id'), + sourceRunVersion: integer( + candidate.sourceRunVersion, + 1, + 2_147_483_647, + 'source Run version', + ), + sourceAttemptId: runIdentity( + candidate.sourceAttemptId, + 'source Attempt id', + ), + toolStepRunId: identity(candidate.toolStepRunId, 'Tool StepRun id'), + toolStepRunDigest: digest( + candidate.toolStepRunDigest, + 'Tool StepRun digest', + ), + toolMutationId: identity(candidate.toolMutationId, 'Tool mutation id'), + toolEventId: identity(candidate.toolEventId, 'Tool event id'), + modelStepRunId: identity(candidate.modelStepRunId, 'model StepRun id'), + modelStepRunDigest: digest( + candidate.modelStepRunDigest, + 'model StepRun digest', + ), + modelMutationId: identity(candidate.modelMutationId, 'model mutation id'), + modelEventId: identity(candidate.modelEventId, 'model event id'), + finalRunVersion: integer( + candidate.finalRunVersion, + 3, + 3, + 'final Run version', + ) as 3, + finalRunEventSequence: integer( + candidate.finalRunEventSequence, + 3, + 3, + 'final Run event sequence', + ) as 3, + admittedAtMs: timestamp(candidate.admittedAtMs, 'admitted time'), + } satisfies Omit); + const receiptDigest = digest(candidate.receiptDigest, 'receipt digest'); + if ( + copilotFailureDiagnosisAdmissionReceiptDigest(unsigned) !== receiptDigest + ) { + return invalid('admission receipt digest does not match'); + } + const normalized = Object.freeze({ ...unsigned, receiptDigest }); + if ( + Buffer.byteLength(JSON.stringify(normalized), 'utf8') > + MAX_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_RECEIPT_BYTES + ) { + return invalid('admission receipt exceeds its byte budget'); + } + return normalized; +} + +export function createCopilotFailureDiagnosisAdmissionBundle( + planValue: CopilotFailureDiagnosisExecutionPlan, +): Readonly { + const plan = normalizeCopilotFailureDiagnosisExecutionPlan(planValue); + const admissionEvent = Object.freeze({ + id: evidenceIdentity('cda', plan.planDigest), + runId: plan.runId, + sequence: 1, + type: 'copilot.failure_diagnosis.admitted', + dedupeKey: evidenceIdentity('cda', plan.planDigest), + actorType: plan.requestedBySubject.type, + actorId: plan.requestedBySubject.id, + payload: Object.freeze({ + planDigest: plan.planDigest, + sourceRunId: plan.source.runId, + sourceRunVersion: plan.source.runVersion, + sourceAttemptId: plan.source.attemptId, + toolPlanDigest: plan.tool.planDigest, + modelIntentDigest: plan.model.intentDigest, + }), + createdAtMs: plan.plannedAtMs, + } satisfies RunEventRecord); + const toolStepMutation = normalizeStepRunMutation( + createStepRunMutation( + { + id: plan.toolStepRunId, + runId: plan.runId, + stepKey: 'collect-log', + kind: 'tool', + definitionRef: `trusted-tool-plan:${plan.tool.planDigest}`, + definitionDigest: plan.tool.definitionDigest, + required: true, + initialStatus: 'ready', + inputRef: `tool-invocation:${plan.tool.invocationArtifact.artifactId}`, + mutationId: evidenceIdentity('cdtm', plan.planDigest), + createdAtMs: plan.plannedAtMs, + }, + { + expectedRunVersion: 1, + expectedRunEventSequence: 1, + eventId: evidenceIdentity('cdte', plan.planDigest), + dedupeKey: evidenceIdentity('cdte', plan.planDigest), + actor: plan.requestedBySubject, + }, + ), + ); + const modelStepMutation = normalizeStepRunMutation( + createStepRunMutation( + { + id: plan.modelStepRunId, + runId: plan.runId, + parentStepRunId: plan.toolStepRunId, + stepKey: 'diagnose', + kind: 'model', + definitionRef: `copilot-failure-diagnosis:${plan.model.intentDigest}`, + definitionDigest: plan.model.intentDigest, + required: true, + initialStatus: 'pending', + inputRef: `tool-result-step:${plan.toolStepRunId}`, + mutationId: evidenceIdentity('cdmm', plan.planDigest), + createdAtMs: plan.plannedAtMs, + }, + { + expectedRunVersion: 2, + expectedRunEventSequence: 2, + eventId: evidenceIdentity('cdme', plan.planDigest), + dedupeKey: evidenceIdentity('cdme', plan.planDigest), + actor: plan.requestedBySubject, + }, + ), + ); + const run = Object.freeze({ + id: plan.runId, + projectId: plan.projectId, + taskId: 'copilot.failure-diagnosis', + taskRevision: 'qinglong/copilot-failure-diagnosis@v1', + taskName: 'Copilot failure diagnosis', + taskSnapshotRef: `copilot-failure-diagnosis:${plan.planDigest}`, + parentRunId: plan.source.runId, + triggerType: 'copilot_failure_diagnosis', + executionOrigin: 'manual' as const, + executionOwner: 'runtime' as const, + triggeredBy: plan.requestedBySubject.id, + requestId: plan.requestId, + status: 'running' as const, + version: 3, + eventSequence: 3, + priority: 0, + idempotencyKey: `copilot-diagnosis:${plan.planDigest.slice(0, 32)}`, + createdAtMs: plan.plannedAtMs, + startedAtMs: plan.plannedAtMs, + } satisfies RunRecord); + const receiptUnsigned = Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_ADMISSION_RECEIPT_SCHEMA, + requestId: plan.requestId, + planDigest: plan.planDigest, + runId: plan.runId, + sourceRunId: plan.source.runId, + sourceRunVersion: plan.source.runVersion, + sourceAttemptId: plan.source.attemptId, + toolStepRunId: plan.toolStepRunId, + toolStepRunDigest: toolStepMutation.stepRun.stepRunDigest, + toolMutationId: toolStepMutation.mutationId, + toolEventId: toolStepMutation.event.id, + modelStepRunId: plan.modelStepRunId, + modelStepRunDigest: modelStepMutation.stepRun.stepRunDigest, + modelMutationId: modelStepMutation.mutationId, + modelEventId: modelStepMutation.event.id, + finalRunVersion: 3 as const, + finalRunEventSequence: 3 as const, + admittedAtMs: plan.plannedAtMs, + }); + const receipt = normalizeCopilotFailureDiagnosisAdmissionReceipt({ + ...receiptUnsigned, + receiptDigest: + copilotFailureDiagnosisAdmissionReceiptDigest(receiptUnsigned), + }); + return Object.freeze({ + plan, + run, + admissionEvent, + toolStepMutation, + modelStepMutation, + receipt, + }); +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/admission/plan.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/admission/plan.ts new file mode 100644 index 00000000..f2960eeb --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/admission/plan.ts @@ -0,0 +1,315 @@ +import { + BUILTIN_RUN_LOG_EXCERPT_ADAPTER, + BUILTIN_RUN_LOG_EXCERPT_AUDIT_CONTRACT, + BUILTIN_RUN_LOG_EXCERPT_REDACTION_CONTRACT, + BUILTIN_RUN_LOG_EXCERPT_TIMEOUT_SECONDS, + BUILTIN_RUN_LOG_EXCERPT_TOOL, +} from '@qinglong/runtime-core/builtin-run-log-excerpt-tool'; +import { normalizeTrustedToolInvocationPlan } from '@qinglong/runtime-core/trusted-tool-invocation'; + +import { + COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_SCHEMA, + MAX_COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_BYTES, + type CopilotFailureDiagnosisExecutionPlan, + type CopilotFailureDiagnosisToolIntent, + type PrepareCopilotFailureDiagnosisExecutionInput, +} from './contracts'; +import { + assertDeadline, + assertJsonBudget, + dataRecord, + digest, + exactKeys, + failureDiagnosisToolInputDigest, + hash, + identity, + invalid, + normalizeFence, + normalizeModelIntent, + normalizeProjectPolicySubject, + normalizeSourceFence, + normalizeToolIntent, + prepareModelIntent, + runIdentity, + sameFence, + sameSubject, + timestamp, +} from './validation'; + +const PLAN_DIGEST_DOMAIN = + 'qinglong/copilot-failure-diagnosis-execution-plan-digest@v1\0'; +const IDENTITY_DOMAIN = + 'qinglong/copilot-failure-diagnosis-execution-identity@v1\0'; + +function executionIdentity( + prefix: 'cdr' | 'cdt' | 'cdm' | 'cdi', + value: Readonly<{ + requestId: string; + projectId: string; + sourceRunId: string; + sourceAttemptId: string; + toolPlanDigest: string; + }>, +): string { + return `${prefix}:${hash(IDENTITY_DOMAIN, { prefix, ...value }).slice( + 0, + 32, + )}`; +} + +function sameContract( + left: Readonly<{ id: string; version: string }>, + right: Readonly<{ id: string; version: string }>, +): boolean { + return left.id === right.id && left.version === right.version; +} + +function planFields( + value: Omit, +): object { + return { + schema: value.schema, + requestId: value.requestId, + runId: value.runId, + toolStepRunId: value.toolStepRunId, + modelStepRunId: value.modelStepRunId, + modelInvocationId: value.modelInvocationId, + traceId: value.traceId, + projectId: value.projectId, + requestedBySubject: value.requestedBySubject, + policyFence: value.policyFence, + source: value.source, + tool: value.tool, + model: value.model, + deadlineAtMs: value.deadlineAtMs, + plannedAtMs: value.plannedAtMs, + }; +} + +export function copilotFailureDiagnosisExecutionPlanDigest( + value: Omit, +): string { + return hash(PLAN_DIGEST_DOMAIN, planFields(value)); +} + +function expectedIdentities( + value: Readonly<{ + requestId: string; + projectId: string; + sourceRunId: string; + sourceAttemptId: string; + toolPlanDigest: string; + }>, +): Readonly<{ + runId: string; + toolStepRunId: string; + modelStepRunId: string; + modelInvocationId: string; +}> { + return Object.freeze({ + runId: executionIdentity('cdr', value), + toolStepRunId: executionIdentity('cdt', value), + modelStepRunId: executionIdentity('cdm', value), + modelInvocationId: executionIdentity('cdi', value), + }); +} + +export function normalizeCopilotFailureDiagnosisExecutionPlan( + value: CopilotFailureDiagnosisExecutionPlan, +): Readonly { + const candidate = dataRecord(value, 'execution plan'); + exactKeys( + candidate, + [ + 'deadlineAtMs', + 'model', + 'modelInvocationId', + 'modelStepRunId', + 'planDigest', + 'plannedAtMs', + 'policyFence', + 'projectId', + 'requestId', + 'requestedBySubject', + 'runId', + 'schema', + 'source', + 'tool', + 'toolStepRunId', + 'traceId', + ], + 'execution plan', + ); + if (candidate.schema !== COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_SCHEMA) { + return invalid('execution plan schema is unsupported'); + } + const requestId = identity(candidate.requestId, 'request id'); + const projectId = identity(candidate.projectId, 'project id'); + const source = normalizeSourceFence(candidate.source); + const tool = normalizeToolIntent(candidate.tool); + const model = normalizeModelIntent(candidate.model); + if ( + tool.invocationArtifact.inputDigest !== + failureDiagnosisToolInputDigest(source) + ) { + return invalid('Tool input is not bound to the source Run Attempt'); + } + const plannedAtMs = timestamp(candidate.plannedAtMs, 'planned time'); + const deadlineAtMs = timestamp(candidate.deadlineAtMs, 'deadline'); + assertDeadline(plannedAtMs, deadlineAtMs); + if (tool.sealedAtMs > plannedAtMs) { + return invalid('Tool plan was sealed after diagnosis planning'); + } + const identityInput = Object.freeze({ + requestId, + projectId, + sourceRunId: source.runId, + sourceAttemptId: source.attemptId, + toolPlanDigest: tool.planDigest, + }); + const identities = expectedIdentities(identityInput); + const unsigned = Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_SCHEMA, + requestId, + runId: runIdentity(candidate.runId, 'diagnosis Run id'), + toolStepRunId: identity(candidate.toolStepRunId, 'Tool StepRun id'), + modelStepRunId: identity(candidate.modelStepRunId, 'model StepRun id'), + modelInvocationId: identity( + candidate.modelInvocationId, + 'model invocation id', + ), + traceId: identity(candidate.traceId, 'trace id'), + projectId, + requestedBySubject: normalizeProjectPolicySubject( + candidate.requestedBySubject as never, + ), + policyFence: normalizeFence(candidate.policyFence), + source, + tool, + model, + deadlineAtMs, + plannedAtMs, + } satisfies Omit); + if ( + unsigned.runId !== identities.runId || + unsigned.toolStepRunId !== identities.toolStepRunId || + unsigned.modelStepRunId !== identities.modelStepRunId || + unsigned.modelInvocationId !== identities.modelInvocationId + ) { + return invalid('execution identities are invalid'); + } + const planDigest = digest(candidate.planDigest, 'plan digest'); + if (copilotFailureDiagnosisExecutionPlanDigest(unsigned) !== planDigest) { + return invalid('plan digest does not match'); + } + const normalized = Object.freeze({ ...unsigned, planDigest }); + assertJsonBudget( + normalized, + MAX_COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_BYTES, + 'execution plan', + ); + return normalized; +} + +export function prepareCopilotFailureDiagnosisExecution( + inputValue: PrepareCopilotFailureDiagnosisExecutionInput, +): Readonly { + const input = dataRecord(inputValue, 'execution input'); + exactKeys( + input, + [ + 'bindings', + 'deadlineAtMs', + 'model', + 'plannedAtMs', + 'requestId', + 'source', + 'toolPlan', + 'traceId', + ], + 'execution input', + ); + const source = normalizeSourceFence(input.source); + const toolPlan = normalizeTrustedToolInvocationPlan( + inputValue.toolPlan, + inputValue.bindings, + ); + const binding = toolPlan.binding; + if ( + toolPlan.status !== 'ready' || + toolPlan.profile !== 'cluster-control' || + toolPlan.tool.name !== BUILTIN_RUN_LOG_EXCERPT_TOOL.name || + toolPlan.tool.version !== BUILTIN_RUN_LOG_EXCERPT_TOOL.version || + toolPlan.effect !== 'read' || + toolPlan.risk !== 'medium' || + toolPlan.permission !== 'tool.call:qinglong.run.log.excerpt' || + toolPlan.requiredPermissions.length !== 1 || + toolPlan.requiredPermissions[0] !== 'artifact.read' || + toolPlan.invocationArtifact.inputDigest !== + failureDiagnosisToolInputDigest(source) || + binding.executionClass !== 'builtin_in_process' || + binding.timeoutSeconds !== BUILTIN_RUN_LOG_EXCERPT_TIMEOUT_SECONDS || + !sameContract(binding.adapter, BUILTIN_RUN_LOG_EXCERPT_ADAPTER) || + !sameContract( + binding.redactionContract, + BUILTIN_RUN_LOG_EXCERPT_REDACTION_CONTRACT, + ) || + !sameContract(binding.auditContract, BUILTIN_RUN_LOG_EXCERPT_AUDIT_CONTRACT) + ) { + return invalid('Tool plan is not the exact Cluster Run log excerpt plan'); + } + const requestId = identity(input.requestId, 'request id'); + const projectId = identity(toolPlan.projectId, 'project id'); + const plannedAtMs = timestamp(input.plannedAtMs, 'planned time'); + const deadlineAtMs = timestamp(input.deadlineAtMs, 'deadline'); + assertDeadline(plannedAtMs, deadlineAtMs); + if (toolPlan.sealedAtMs > plannedAtMs) { + return invalid('Tool plan was sealed after diagnosis planning'); + } + const tool = normalizeToolIntent({ + actionRef: toolPlan.actionRef, + planDigest: toolPlan.planDigest, + actionDigest: toolPlan.actionDigest, + invocationActionDigest: toolPlan.invocationActionDigest, + snapshotDigest: toolPlan.snapshotDigest, + definitionDigest: toolPlan.definitionDigest, + bindingDigest: binding.bindingDigest, + invocationArtifact: toolPlan.invocationArtifact, + previewArtifact: toolPlan.previewArtifact, + sealedAtMs: toolPlan.sealedAtMs, + } satisfies CopilotFailureDiagnosisToolIntent); + const model = prepareModelIntent(inputValue.model); + const identityInput = Object.freeze({ + requestId, + projectId, + sourceRunId: source.runId, + sourceAttemptId: source.attemptId, + toolPlanDigest: tool.planDigest, + }); + const identities = expectedIdentities(identityInput); + const unsigned = Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_SCHEMA, + requestId, + ...identities, + traceId: identity(input.traceId, 'trace id'), + projectId, + requestedBySubject: toolPlan.requestedBy, + policyFence: toolPlan.policyFence, + source, + tool, + model, + deadlineAtMs, + plannedAtMs, + } satisfies Omit); + const plan = normalizeCopilotFailureDiagnosisExecutionPlan({ + ...unsigned, + planDigest: copilotFailureDiagnosisExecutionPlanDigest(unsigned), + }); + if ( + !sameSubject(plan.requestedBySubject, toolPlan.requestedBy) || + !sameFence(plan.policyFence, toolPlan.policyFence) + ) { + return invalid('Tool authority does not match diagnosis authority'); + } + return plan; +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/admission/postgresRepository.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/admission/postgresRepository.ts new file mode 100644 index 00000000..03a50ee7 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/admission/postgresRepository.ts @@ -0,0 +1,760 @@ +import type { + PostgresClient, + PostgresPool, + PostgresQueryable, +} from '@qinglong/runtime-core'; +import { isDeepStrictEqual } from 'node:util'; +import { + normalizeStepRunRecord, + type StepRunMutation, + type StepRunRecord, +} from '@qinglong/runtime-core/step-run'; + +import { POSTGRES_MODEL_INVOCATION_SCHEMA } from '../../../migration/modelInvocationMigration'; +import { + CopilotFailureDiagnosisAdmissionConflictError, + CopilotFailureDiagnosisAdmissionNotAllowedError, + CopilotFailureDiagnosisAdmissionUnavailableError, + type CopilotFailureDiagnosisAdmissionBundle, + type CopilotFailureDiagnosisAdmissionReceipt, + type CopilotFailureDiagnosisAdmissionRepository, + type CopilotFailureDiagnosisExecutionPlan, +} from './contracts'; +import { + createCopilotFailureDiagnosisAdmissionBundle, + normalizeCopilotFailureDiagnosisAdmissionReceipt, +} from './durableEvidence'; +import { normalizeCopilotFailureDiagnosisExecutionPlan } from './plan'; +import { identity } from './validation'; + +const ADMISSION_TABLE = 'copilot_failure_diagnosis_admissions'; +const SOURCE_SNAPSHOT_FUNCTION = + 'copilot_failure_diagnosis_admission_source_snapshot'; +const RETRYABLE_SQL_STATES = new Set(['40001', '40P01']); +const MAX_TRANSACTION_ATTEMPTS = 3; +const DIAGNOSIS_RUN_STATUSES = new Set([ + 'running', + 'lost', + 'succeeded', + 'failed', + 'cancelled', + 'timed_out', +]); + +type Row = Readonly>; + +export interface PostgresCopilotFailureDiagnosisAdmissionMutationGuard { + confirm( + input: Readonly<{ + client: PostgresClient; + plan: Readonly; + replay: boolean; + }>, + ): void | Promise; +} + +function unavailable( + cause?: unknown, +): CopilotFailureDiagnosisAdmissionUnavailableError { + return new CopilotFailureDiagnosisAdmissionUnavailableError({ + cause: cause instanceof Error ? cause : undefined, + }); +} + +function sqlState(error: unknown): string | undefined { + if (!error || typeof error !== 'object') return undefined; + const value = (error as { code?: unknown }).code; + return typeof value === 'string' ? value : undefined; +} + +function mapStorageError(error: unknown): Error { + if ( + error instanceof CopilotFailureDiagnosisAdmissionConflictError || + error instanceof CopilotFailureDiagnosisAdmissionNotAllowedError || + error instanceof CopilotFailureDiagnosisAdmissionUnavailableError + ) { + return error; + } + if (sqlState(error) === '23505') { + return new CopilotFailureDiagnosisAdmissionConflictError( + 'a durable diagnosis identity is already bound', + ); + } + return unavailable(error); +} + +function text(row: Row, key: string): string { + const value = row[key]; + if (typeof value !== 'string' || value.length === 0) throw unavailable(); + return value; +} + +function nullableText(row: Row, key: string): string | null { + const value = row[key]; + if (value === null) return null; + return text(row, key); +} + +function integer(row: Row, key: string): number { + const value = row[key]; + if (typeof value === 'number' && Number.isSafeInteger(value)) return value; + if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) { + const parsed = Number(value); + if (Number.isSafeInteger(parsed)) return parsed; + } + throw unavailable(); +} + +function boolean(row: Row, key: string): boolean { + const value = row[key]; + if (typeof value !== 'boolean') throw unavailable(); + return value; +} + +function jsonObject(value: unknown): Record { + let parsed = value; + if (typeof value === 'string') { + try { + parsed = JSON.parse(value); + } catch (cause) { + throw unavailable(cause); + } + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw unavailable(); + } + return parsed as Record; +} + +function json(value: unknown): string { + return JSON.stringify(value); +} + +export function storedJsonEquals(left: unknown, right: unknown): boolean { + return isDeepStrictEqual(left, right); +} + +async function begin(client: PostgresClient): Promise { + await client.query('BEGIN ISOLATION LEVEL SERIALIZABLE'); + await client.query(`SELECT set_config('statement_timeout', $1, true)`, [ + '5s', + ]); + await client.query(`SELECT set_config('lock_timeout', $1, true)`, ['2s']); + await client.query( + `SELECT set_config('idle_in_transaction_session_timeout', $1, true)`, + ['5s'], + ); +} + +async function rollback(client: PostgresClient): Promise { + try { + await client.query('ROLLBACK'); + } catch { + // Preserve the original transaction failure. + } +} + +async function transaction( + pool: PostgresPool, + work: (client: PostgresClient) => Promise, +): Promise { + for (let attempt = 0; attempt < MAX_TRANSACTION_ATTEMPTS; attempt += 1) { + let client: PostgresClient; + try { + client = await pool.connect(); + } catch (cause) { + throw unavailable(cause); + } + let began = false; + try { + await begin(client); + began = true; + const result = await work(client); + await client.query('COMMIT'); + began = false; + return result; + } catch (error) { + if (began) await rollback(client); + if ( + RETRYABLE_SQL_STATES.has(sqlState(error) ?? '') && + attempt + 1 < MAX_TRANSACTION_ATTEMPTS + ) { + continue; + } + throw mapStorageError(error); + } finally { + client.release(); + } + } + throw unavailable(); +} + +async function assertRunEvidence( + queryable: PostgresQueryable, + bundle: Readonly, +): Promise { + const result = await queryable.query( + `SELECT project_id AS "projectId", task_id AS "taskId", + task_revision AS "taskRevision", task_name AS "taskName", + task_snapshot_ref AS "taskSnapshotRef", + parent_run_id AS "parentRunId", trigger_type AS "triggerType", + execution_origin AS "executionOrigin", + execution_owner AS "executionOwner", + triggered_by AS "triggeredBy", request_id AS "requestId", + status, version, event_sequence AS "eventSequence", priority, + idempotency_key AS "idempotencyKey", + created_at_ms AS "createdAtMs", + started_at_ms AS "startedAtMs" + FROM "ql3"."runs" WHERE id = $1`, + [bundle.run.id], + ); + const row = result.rows.length === 1 ? result.rows[0]! : null; + const run = bundle.run; + if ( + !row || + text(row, 'projectId') !== run.projectId || + text(row, 'taskId') !== run.taskId || + text(row, 'taskRevision') !== run.taskRevision || + nullableText(row, 'taskName') !== (run.taskName ?? null) || + nullableText(row, 'taskSnapshotRef') !== (run.taskSnapshotRef ?? null) || + nullableText(row, 'parentRunId') !== (run.parentRunId ?? null) || + text(row, 'triggerType') !== run.triggerType || + text(row, 'executionOrigin') !== run.executionOrigin || + text(row, 'executionOwner') !== run.executionOwner || + nullableText(row, 'triggeredBy') !== (run.triggeredBy ?? null) || + nullableText(row, 'requestId') !== (run.requestId ?? null) || + !DIAGNOSIS_RUN_STATUSES.has(text(row, 'status')) || + integer(row, 'version') < run.version || + integer(row, 'eventSequence') < run.eventSequence || + integer(row, 'version') !== integer(row, 'eventSequence') || + integer(row, 'priority') !== run.priority || + nullableText(row, 'idempotencyKey') !== (run.idempotencyKey ?? null) || + integer(row, 'createdAtMs') !== run.createdAtMs || + integer(row, 'startedAtMs') !== run.startedAtMs + ) { + throw unavailable(); + } +} + +async function assertEventEvidence( + queryable: PostgresQueryable, + bundle: Readonly, +): Promise { + const result = await queryable.query( + `SELECT id, sequence, type, dedupe_key AS "dedupeKey", + actor_type AS "actorType", actor_id AS "actorId", + step_run_id AS "stepRunId", payload, + created_at_ms AS "createdAtMs" + FROM "ql3"."run_events" + WHERE run_id = $1 AND sequence <= 3 ORDER BY sequence`, + [bundle.run.id], + ); + const expected = [ + bundle.admissionEvent, + bundle.toolStepMutation.event, + bundle.modelStepMutation.event, + ]; + if ( + result.rows.length !== expected.length || + result.rows.some((row, index) => { + const event = expected[index]!; + return ( + text(row, 'id') !== event.id || + integer(row, 'sequence') !== event.sequence || + text(row, 'type') !== event.type || + nullableText(row, 'dedupeKey') !== (event.dedupeKey ?? null) || + text(row, 'actorType') !== event.actorType || + nullableText(row, 'actorId') !== (event.actorId ?? null) || + nullableText(row, 'stepRunId') !== (event.stepRunId ?? null) || + !storedJsonEquals(row.payload, event.payload) || + integer(row, 'createdAtMs') !== event.createdAtMs + ); + }) + ) { + throw unavailable(); + } +} + +function assertStepRow(row: Row, mutation: Readonly): void { + let current: Readonly; + try { + current = normalizeStepRunRecord( + jsonObject(row.stepRunJson) as unknown as StepRunRecord, + ); + } catch (cause) { + throw unavailable(cause); + } + const initial = mutation.stepRun; + if ( + current.id !== initial.id || + current.runId !== initial.runId || + current.parentStepRunId !== initial.parentStepRunId || + current.stepKey !== initial.stepKey || + current.kind !== initial.kind || + current.definitionRef !== initial.definitionRef || + current.definitionDigest !== initial.definitionDigest || + current.required !== true || + current.inputRef !== initial.inputRef || + current.createdAtMs !== initial.createdAtMs || + current.version < initial.version || + text(row, 'stepKey') !== current.stepKey || + text(row, 'kind') !== current.kind || + nullableText(row, 'parentStepRunId') !== current.parentStepRunId || + text(row, 'definitionRef') !== current.definitionRef || + text(row, 'definitionDigest') !== current.definitionDigest || + boolean(row, 'required') !== true || + text(row, 'status') !== current.status || + integer(row, 'version') !== current.version || + text(row, 'lastMutationId') !== current.lastMutationId || + text(row, 'stepRunDigest') !== current.stepRunDigest || + text(row, 'mutationId') !== mutation.mutationId || + text(row, 'mutationDigest') !== mutation.mutationDigest || + text(row, 'eventId') !== mutation.event.id || + integer(row, 'eventSequence') !== mutation.event.sequence || + integer(row, 'runVersion') !== mutation.expectedRunVersion + 1 || + text(row, 'initialStepRunDigest') !== initial.stepRunDigest || + !storedJsonEquals(row.initialStepRunJson, initial) + ) { + throw unavailable(); + } +} + +async function assertStepEvidence( + queryable: PostgresQueryable, + bundle: Readonly, +): Promise { + const tool = bundle.toolStepMutation; + const model = bundle.modelStepMutation; + const result = await queryable.query( + `SELECT runtime.id, runtime.parent_step_run_id AS "parentStepRunId", + runtime.step_key AS "stepKey", runtime.kind, + runtime.definition_ref AS "definitionRef", + runtime.definition_digest AS "definitionDigest", + runtime.required, runtime.status, runtime.version, + runtime.last_mutation_id AS "lastMutationId", + runtime.step_run_digest AS "stepRunDigest", + runtime.step_run_json AS "stepRunJson", + mutation.mutation_id AS "mutationId", + mutation.mutation_digest AS "mutationDigest", + mutation.event_id AS "eventId", + mutation.event_sequence AS "eventSequence", + mutation.run_version AS "runVersion", + mutation.step_run_digest AS "initialStepRunDigest", + mutation.step_run_json AS "initialStepRunJson" + FROM "ql3"."step_runs" AS runtime + JOIN "ql3"."step_run_mutations" AS mutation + ON mutation.run_id = runtime.run_id + AND mutation.step_run_id = runtime.id + AND mutation.mutation_id = CASE + WHEN runtime.id = $2 THEN $4 ELSE $5 END + WHERE runtime.run_id = $1 AND runtime.id IN ($2, $3) + ORDER BY runtime.id`, + [ + bundle.run.id, + tool.stepRun.id, + model.stepRun.id, + tool.mutationId, + model.mutationId, + ], + ); + if (result.rows.length !== 2) throw unavailable(); + const rows = new Map(result.rows.map((row) => [text(row, 'id'), row])); + const toolRow = rows.get(tool.stepRun.id); + const modelRow = rows.get(model.stepRun.id); + if (!toolRow || !modelRow) throw unavailable(); + assertStepRow(toolRow, tool); + assertStepRow(modelRow, model); +} + +async function assertStoredEvidence( + queryable: PostgresQueryable, + bundle: Readonly, +): Promise { + await assertRunEvidence(queryable, bundle); + await assertEventEvidence(queryable, bundle); + await assertStepEvidence(queryable, bundle); +} + +type StoredAdmission = Readonly<{ + plan: Readonly; + receipt: Readonly; +}>; + +async function findStored( + queryable: PostgresQueryable, + requestId: string, +): Promise { + const result = await queryable.query( + `SELECT request_id AS "requestId", plan_digest AS "planDigest", + run_id AS "runId", project_id AS "projectId", + source_run_id AS "sourceRunId", + source_run_version AS "sourceRunVersion", + source_run_status AS "sourceRunStatus", + source_attempt_id AS "sourceAttemptId", + source_attempt_status AS "sourceAttemptStatus", + source_log_artifact_id AS "sourceLogArtifactId", + tool_plan_digest AS "toolPlanDigest", + tool_action_digest AS "toolActionDigest", + tool_step_run_id AS "toolStepRunId", + model_intent_digest AS "modelIntentDigest", + model_step_run_id AS "modelStepRunId", + admitted_at_ms AS "admittedAtMs", + receipt_digest AS "receiptDigest", + plan_json AS "planJson", receipt_json AS "receiptJson" + FROM "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" + WHERE request_id = $1 LIMIT 2`, + [requestId], + ); + if (result.rows.length > 1) throw unavailable(); + const row = result.rows[0]; + if (!row) return null; + try { + const plan = normalizeCopilotFailureDiagnosisExecutionPlan( + jsonObject( + row.planJson, + ) as unknown as CopilotFailureDiagnosisExecutionPlan, + ); + const receipt = normalizeCopilotFailureDiagnosisAdmissionReceipt( + jsonObject( + row.receiptJson, + ) as unknown as CopilotFailureDiagnosisAdmissionReceipt, + ); + const bundle = createCopilotFailureDiagnosisAdmissionBundle(plan); + if ( + text(row, 'requestId') !== plan.requestId || + text(row, 'planDigest') !== plan.planDigest || + text(row, 'runId') !== plan.runId || + text(row, 'projectId') !== plan.projectId || + text(row, 'sourceRunId') !== plan.source.runId || + integer(row, 'sourceRunVersion') !== plan.source.runVersion || + text(row, 'sourceRunStatus') !== plan.source.runStatus || + text(row, 'sourceAttemptId') !== plan.source.attemptId || + text(row, 'sourceAttemptStatus') !== plan.source.attemptStatus || + text(row, 'sourceLogArtifactId') !== plan.source.logArtifactId || + text(row, 'toolPlanDigest') !== plan.tool.planDigest || + text(row, 'toolActionDigest') !== plan.tool.actionDigest || + text(row, 'toolStepRunId') !== plan.toolStepRunId || + text(row, 'modelIntentDigest') !== plan.model.intentDigest || + text(row, 'modelStepRunId') !== plan.modelStepRunId || + integer(row, 'admittedAtMs') !== receipt.admittedAtMs || + text(row, 'receiptDigest') !== receipt.receiptDigest || + !storedJsonEquals(bundle.receipt, receipt) + ) { + throw unavailable(); + } + await assertStoredEvidence(queryable, bundle); + return Object.freeze({ plan, receipt }); + } catch (error) { + if (error instanceof CopilotFailureDiagnosisAdmissionUnavailableError) { + throw error; + } + throw unavailable(error); + } +} + +async function assertSourceSnapshot( + client: PostgresClient, + plan: Readonly, +): Promise { + const result = await client.query( + `SELECT run_id AS "runId", run_version AS "runVersion", + run_status AS "runStatus", attempt_id AS "attemptId", + attempt_status AS "attemptStatus", + attempt_finished_at_ms AS "attemptFinishedAtMs", + log_artifact_id AS "logArtifactId" + FROM "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${SOURCE_SNAPSHOT_FUNCTION}"( + $1, $2, $3, $4, $5, $6, $7 + )`, + [ + plan.projectId, + plan.requestedBySubject.type, + plan.requestedBySubject.id, + plan.policyFence.projectVersion, + plan.policyFence.bindingVersion, + plan.source.runId, + plan.source.attemptId, + ], + ); + const row = result.rows.length === 1 ? result.rows[0]! : null; + if (!row) throw new CopilotFailureDiagnosisAdmissionNotAllowedError(); + const source = plan.source; + if ( + text(row, 'runId') !== source.runId || + integer(row, 'runVersion') !== source.runVersion || + text(row, 'runStatus') !== source.runStatus || + text(row, 'attemptId') !== source.attemptId || + text(row, 'attemptStatus') !== source.attemptStatus || + integer(row, 'attemptFinishedAtMs') !== source.attemptFinishedAtMs || + text(row, 'logArtifactId') !== source.logArtifactId + ) { + throw new CopilotFailureDiagnosisAdmissionConflictError( + 'source Run or final Attempt changed', + ); + } +} + +async function insertRun( + client: PostgresClient, + bundle: Readonly, +): Promise { + const run = bundle.run; + await client.query( + `INSERT INTO "ql3"."runs" ( + id, project_id, task_id, task_revision, task_name, + task_snapshot_ref, parent_run_id, trigger_type, execution_origin, + execution_owner, triggered_by, request_id, status, version, + event_sequence, priority, idempotency_key, created_at_ms, started_at_ms + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, + $15, $16, $17, $18, $19 + )`, + [ + run.id, + run.projectId, + run.taskId, + run.taskRevision, + run.taskName ?? null, + run.taskSnapshotRef ?? null, + run.parentRunId ?? null, + run.triggerType, + run.executionOrigin, + run.executionOwner, + run.triggeredBy ?? null, + run.requestId ?? null, + run.status, + run.version, + run.eventSequence, + run.priority, + run.idempotencyKey ?? null, + run.createdAtMs, + run.startedAtMs ?? null, + ], + ); +} + +async function insertEvent( + client: PostgresClient, + event: Readonly, +): Promise { + await client.query( + `INSERT INTO "ql3"."run_events" ( + id, run_id, sequence, type, dedupe_key, actor_type, actor_id, + attempt_id, step_run_id, payload, created_at_ms + ) VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, $8, $9::jsonb, $10)`, + [ + event.id, + event.runId, + event.sequence, + event.type, + event.dedupeKey ?? null, + event.actorType, + event.actorId ?? null, + event.stepRunId ?? null, + json(event.payload), + event.createdAtMs, + ], + ); +} + +async function insertStepEvidence( + client: PostgresClient, + mutation: Readonly, + admittedAtMs: number, +): Promise { + const step = mutation.stepRun; + await client.query( + `INSERT INTO "ql3"."step_runs" ( + id, run_id, parent_step_run_id, step_key, kind, definition_ref, + definition_digest, required, status, version, attempt_count, + input_ref, output_ref, approval_request_id, ready_at_ms, + started_at_ms, finished_at_ms, result_code, error_summary, + created_at_ms, updated_at_ms, last_mutation_id, step_run_digest, + step_run_json + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, + $15, $16, $17, $18, $19, $20, $21, $22, $23, $24::jsonb + )`, + [ + step.id, + step.runId, + step.parentStepRunId, + step.stepKey, + step.kind, + step.definitionRef, + step.definitionDigest, + step.required, + step.status, + step.version, + step.attemptCount, + step.inputRef, + step.outputRef, + step.approvalRequestId, + step.readyAtMs, + step.startedAtMs, + step.finishedAtMs, + step.resultCode, + step.errorSummary, + step.createdAtMs, + step.updatedAtMs, + step.lastMutationId, + step.stepRunDigest, + json(step), + ], + ); + await insertEvent(client, mutation.event); + await client.query( + `INSERT INTO "ql3"."step_run_mutations" ( + mutation_id, mutation_digest, run_id, step_run_id, + step_run_digest, event_id, event_sequence, run_version, + step_run_json, committed_at_ms + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10)`, + [ + mutation.mutationId, + mutation.mutationDigest, + mutation.runId, + step.id, + step.stepRunDigest, + mutation.event.id, + mutation.event.sequence, + mutation.expectedRunVersion + 1, + json(step), + admittedAtMs, + ], + ); +} + +async function insertAdmission( + client: PostgresClient, + bundle: Readonly, +): Promise { + const { plan, receipt } = bundle; + await client.query( + `INSERT INTO "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" ( + request_id, plan_digest, run_id, project_id, + source_run_id, source_run_version, source_run_status, + source_attempt_id, source_attempt_status, source_log_artifact_id, + tool_plan_digest, tool_action_digest, tool_step_run_id, + model_intent_digest, model_step_run_id, admitted_at_ms, + receipt_digest, plan_json, receipt_json + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, + $15, $16, $17, $18::jsonb, $19::jsonb + )`, + [ + plan.requestId, + plan.planDigest, + plan.runId, + plan.projectId, + plan.source.runId, + plan.source.runVersion, + plan.source.runStatus, + plan.source.attemptId, + plan.source.attemptStatus, + plan.source.logArtifactId, + plan.tool.planDigest, + plan.tool.actionDigest, + plan.toolStepRunId, + plan.model.intentDigest, + plan.modelStepRunId, + receipt.admittedAtMs, + receipt.receiptDigest, + json(plan), + json(receipt), + ], + ); +} + +export class PostgresCopilotFailureDiagnosisAdmissionRepository + implements CopilotFailureDiagnosisAdmissionRepository +{ + readonly #pool: PostgresPool; + readonly #mutationGuard: + | PostgresCopilotFailureDiagnosisAdmissionMutationGuard + | undefined; + + constructor( + pool: PostgresPool, + mutationGuard?: PostgresCopilotFailureDiagnosisAdmissionMutationGuard, + ) { + if ( + !pool || + typeof pool.query !== 'function' || + typeof pool.connect !== 'function' || + (mutationGuard !== undefined && + (!mutationGuard || typeof mutationGuard.confirm !== 'function')) + ) { + throw unavailable(); + } + this.#pool = pool; + this.#mutationGuard = mutationGuard; + } + + async findByRequestId( + requestIdValue: string, + ): Promise | null> { + const requestId = identity(requestIdValue, 'request id'); + try { + return (await findStored(this.#pool, requestId))?.receipt ?? null; + } catch (error) { + throw mapStorageError(error); + } + } + + async findPlanByRequestId( + requestIdValue: string, + ): Promise | null> { + const requestId = identity(requestIdValue, 'request id'); + try { + return (await findStored(this.#pool, requestId))?.plan ?? null; + } catch (error) { + throw mapStorageError(error); + } + } + + admit(planValue: Readonly): Promise< + Readonly<{ + status: 'created' | 'existing'; + receipt: Readonly; + }> + > { + const plan = normalizeCopilotFailureDiagnosisExecutionPlan(planValue); + return transaction(this.#pool, async (client) => { + const existing = await findStored(client, plan.requestId); + if (existing) { + if ( + existing.plan.planDigest !== plan.planDigest || + !storedJsonEquals(existing.plan, plan) + ) { + throw new CopilotFailureDiagnosisAdmissionConflictError( + 'requestId is already bound to another diagnosis plan', + ); + } + await this.#mutationGuard?.confirm({ client, plan, replay: true }); + return Object.freeze({ + status: 'existing' as const, + receipt: existing.receipt, + }); + } + await this.#mutationGuard?.confirm({ client, plan, replay: false }); + await assertSourceSnapshot(client, plan); + const bundle = createCopilotFailureDiagnosisAdmissionBundle(plan); + await insertRun(client, bundle); + await insertEvent(client, bundle.admissionEvent); + await insertStepEvidence( + client, + bundle.toolStepMutation, + bundle.receipt.admittedAtMs, + ); + await insertStepEvidence( + client, + bundle.modelStepMutation, + bundle.receipt.admittedAtMs, + ); + await insertAdmission(client, bundle); + return Object.freeze({ + status: 'created' as const, + receipt: bundle.receipt, + }); + }); + } +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/admission/validation.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/admission/validation.ts new file mode 100644 index 00000000..73b39550 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/admission/validation.ts @@ -0,0 +1,416 @@ +import { Buffer } from 'node:buffer'; +import { createHash } from 'node:crypto'; + +import { normalizeProjectPolicySubject } from '@qinglong/runtime-core/project-policy'; +import type { + SecurityPolicyFence, + SecuritySubject, +} from '@qinglong/runtime-core/security'; +import { + normalizeToolInvocationInputArtifactReference, + normalizeToolInvocationPreviewArtifactReference, +} from '@qinglong/runtime-core/tool-invocation-artifact'; + +import { MAX_MODEL_INVOCATION_MS } from '../../../model-gateway/model'; +import { + FAILURE_DIAGNOSIS_PROMPT_PROTOCOL, + type FailureDiagnosisModelEgressPolicy, +} from '../contracts'; +import { + normalizeFailureDiagnosisModelBoundary, + normalizeFailureDiagnosisModelEgressPolicy, + normalizeFailureDiagnosisResponseLanguage, +} from '../validation'; +import { + COPILOT_FAILURE_DIAGNOSIS_SOURCE_ATTEMPT_STATUSES, + COPILOT_FAILURE_DIAGNOSIS_SOURCE_RUN_STATUSES, + InvalidCopilotFailureDiagnosisExecutionPlanError, + type CopilotFailureDiagnosisModelIntent, + type CopilotFailureDiagnosisSourceFence, + type CopilotFailureDiagnosisToolIntent, + type PrepareCopilotFailureDiagnosisModelIntent, +} from './contracts'; + +const IDENTITY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const MODEL_EGRESS_POLICY_DIGEST_DOMAIN = + 'qinglong/copilot-failure-diagnosis-egress-policy-digest@v1\0'; +const MODEL_INTENT_DIGEST_DOMAIN = + 'qinglong/copilot-failure-diagnosis-model-intent-digest@v1\0'; + +export function invalid(message: string): never { + throw new InvalidCopilotFailureDiagnosisExecutionPlanError(message); +} + +export function dataRecord( + value: unknown, + label: string, +): Record { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + Object.getPrototypeOf(value) !== Object.prototype + ) { + return invalid(`${label} must be a plain object`); + } + return value as Record; +} + +export function exactKeys( + value: object, + expected: readonly string[], + label: string, +): void { + const actual = Reflect.ownKeys(value); + const allowed = new Set(expected); + if ( + actual.length !== expected.length || + actual.some((key) => typeof key !== 'string' || !allowed.has(key)) || + expected.some((key) => !actual.includes(key)) + ) { + invalid(`${label} shape is invalid`); + } +} + +export function hash(domain: string, value: unknown): string { + return createHash('sha256') + .update(domain, 'utf8') + .update(JSON.stringify(value)) + .digest('hex'); +} + +export function identity(value: unknown, label: string): string { + if (typeof value !== 'string' || !IDENTITY_PATTERN.test(value)) { + return invalid(`${label} is invalid`); + } + return value; +} + +export function runIdentity(value: unknown, label: string): string { + if (typeof value !== 'string' || !RUN_ID_PATTERN.test(value)) { + return invalid(`${label} is invalid`); + } + return value; +} + +export function digest(value: unknown, label: string): string { + if (typeof value !== 'string' || !DIGEST_PATTERN.test(value)) { + return invalid(`${label} is invalid`); + } + return value; +} + +export function integer( + value: unknown, + minimum: number, + maximum: number, + label: string, +): number { + if ( + !Number.isSafeInteger(value) || + (value as number) < minimum || + (value as number) > maximum + ) { + return invalid(`${label} is invalid`); + } + return value as number; +} + +export function timestamp(value: unknown, label: string): number { + return integer(value, 0, Number.MAX_SAFE_INTEGER, label); +} + +export function normalizeFence(value: unknown): Readonly { + const fence = dataRecord(value, 'policy fence'); + exactKeys(fence, ['bindingVersion', 'projectVersion'], 'policy fence'); + return Object.freeze({ + projectVersion: integer( + fence.projectVersion, + 1, + 2_147_483_647, + 'project version', + ), + bindingVersion: integer( + fence.bindingVersion, + 1, + 2_147_483_647, + 'binding version', + ), + }); +} + +export function sameSubject( + left: Readonly, + right: Readonly, +): boolean { + return left.type === right.type && left.id === right.id; +} + +export function sameFence( + left: Readonly, + right: Readonly, +): boolean { + return ( + left.projectVersion === right.projectVersion && + left.bindingVersion === right.bindingVersion + ); +} + +export function normalizeSourceFence( + value: unknown, +): Readonly { + const source = dataRecord(value, 'source fence'); + exactKeys( + source, + [ + 'attemptFinishedAtMs', + 'attemptId', + 'attemptStatus', + 'logArtifactId', + 'runId', + 'runStatus', + 'runVersion', + ], + 'source fence', + ); + if ( + !COPILOT_FAILURE_DIAGNOSIS_SOURCE_RUN_STATUSES.includes( + source.runStatus as never, + ) || + !COPILOT_FAILURE_DIAGNOSIS_SOURCE_ATTEMPT_STATUSES.includes( + source.attemptStatus as never, + ) || + (source.runStatus === 'failed' && + source.attemptStatus !== 'failed' && + source.attemptStatus !== 'lost') || + (source.runStatus === 'timed_out' && source.attemptStatus !== 'timed_out') + ) { + return invalid('source terminal status binding is invalid'); + } + return Object.freeze({ + runId: runIdentity(source.runId, 'source Run id'), + runVersion: integer( + source.runVersion, + 1, + 2_147_483_647, + 'source Run version', + ), + runStatus: source.runStatus, + attemptId: runIdentity(source.attemptId, 'source Attempt id'), + attemptStatus: source.attemptStatus, + attemptFinishedAtMs: timestamp( + source.attemptFinishedAtMs, + 'source Attempt finish time', + ), + logArtifactId: runIdentity(source.logArtifactId, 'source log Artifact id'), + } as CopilotFailureDiagnosisSourceFence); +} + +export function failureDiagnosisToolInputDigest( + source: Readonly, +): string { + return createHash('sha256') + .update( + JSON.stringify({ attemptId: source.attemptId, runId: source.runId }), + ) + .digest('hex'); +} + +export function failureDiagnosisEgressPolicyDigest( + value: FailureDiagnosisModelEgressPolicy, +): string { + return hash( + MODEL_EGRESS_POLICY_DIGEST_DOMAIN, + normalizeFailureDiagnosisModelEgressPolicy(value), + ); +} + +function modelIntentFields( + value: Omit, +): object { + return { + promptProtocol: FAILURE_DIAGNOSIS_PROMPT_PROTOCOL, + provider: value.provider, + model: value.model, + modelBoundary: value.modelBoundary, + responseLanguage: value.responseLanguage, + maxOutputTokens: value.maxOutputTokens, + egressPolicy: value.egressPolicy, + egressPolicyDigest: value.egressPolicyDigest, + completion: { + residualSensitivity: 'potentially_sensitive', + persistence: 'encrypted_only', + plaintextAudit: 'forbidden', + actionAuthority: 'none', + }, + }; +} + +export function failureDiagnosisModelIntentDigest( + value: Omit, +): string { + return hash(MODEL_INTENT_DIGEST_DOMAIN, modelIntentFields(value)); +} + +export function prepareModelIntent( + value: PrepareCopilotFailureDiagnosisModelIntent, +): Readonly { + const candidate = dataRecord(value, 'model intent'); + exactKeys( + candidate, + [ + 'egressPolicy', + 'maxOutputTokens', + 'model', + 'modelBoundary', + 'provider', + 'responseLanguage', + ], + 'model intent', + ); + const egressPolicy = normalizeFailureDiagnosisModelEgressPolicy( + candidate.egressPolicy, + ); + const modelBoundary = normalizeFailureDiagnosisModelBoundary( + candidate.modelBoundary, + ); + const maxOutputTokens = integer( + candidate.maxOutputTokens, + 1, + egressPolicy.maxOutputTokens, + 'model max output tokens', + ); + if ( + !egressPolicy.potentiallySensitiveDataBoundaries.includes(modelBoundary) + ) { + return invalid('model boundary is not allowed by egress policy'); + } + const unsigned = Object.freeze({ + provider: identity(candidate.provider, 'model provider'), + model: identity(candidate.model, 'model'), + modelBoundary, + responseLanguage: normalizeFailureDiagnosisResponseLanguage( + candidate.responseLanguage, + ), + maxOutputTokens, + egressPolicy, + egressPolicyDigest: failureDiagnosisEgressPolicyDigest(egressPolicy), + }); + return Object.freeze({ + ...unsigned, + intentDigest: failureDiagnosisModelIntentDigest(unsigned), + }); +} + +export function normalizeModelIntent( + value: unknown, +): Readonly { + const candidate = dataRecord(value, 'model intent'); + exactKeys( + candidate, + [ + 'egressPolicy', + 'egressPolicyDigest', + 'intentDigest', + 'maxOutputTokens', + 'model', + 'modelBoundary', + 'provider', + 'responseLanguage', + ], + 'model intent', + ); + const normalized = prepareModelIntent({ + provider: candidate.provider as never, + model: candidate.model as never, + modelBoundary: candidate.modelBoundary as never, + responseLanguage: candidate.responseLanguage as never, + maxOutputTokens: candidate.maxOutputTokens as never, + egressPolicy: candidate.egressPolicy as never, + }); + if ( + digest(candidate.egressPolicyDigest, 'egress policy digest') !== + normalized.egressPolicyDigest || + digest(candidate.intentDigest, 'model intent digest') !== + normalized.intentDigest + ) { + return invalid('model intent digest is invalid'); + } + return normalized; +} + +export function normalizeToolIntent( + value: unknown, +): Readonly { + const candidate = dataRecord(value, 'Tool intent'); + exactKeys( + candidate, + [ + 'actionDigest', + 'actionRef', + 'bindingDigest', + 'definitionDigest', + 'invocationActionDigest', + 'invocationArtifact', + 'planDigest', + 'previewArtifact', + 'sealedAtMs', + 'snapshotDigest', + ], + 'Tool intent', + ); + const invocationArtifact = normalizeToolInvocationInputArtifactReference( + candidate.invocationArtifact as never, + ); + const previewArtifact = normalizeToolInvocationPreviewArtifactReference( + candidate.previewArtifact as never, + ); + const actionDigest = digest(candidate.actionDigest, 'Tool action digest'); + if (previewArtifact.actionDigest !== actionDigest) { + return invalid('Tool preview action binding is invalid'); + } + return Object.freeze({ + actionRef: identity(candidate.actionRef, 'Tool action reference'), + planDigest: digest(candidate.planDigest, 'Tool plan digest'), + actionDigest, + invocationActionDigest: digest( + candidate.invocationActionDigest, + 'Tool invocation action digest', + ), + snapshotDigest: digest(candidate.snapshotDigest, 'Tool snapshot digest'), + definitionDigest: digest( + candidate.definitionDigest, + 'Tool definition digest', + ), + bindingDigest: digest(candidate.bindingDigest, 'Tool binding digest'), + invocationArtifact, + previewArtifact, + sealedAtMs: timestamp(candidate.sealedAtMs, 'Tool plan seal time'), + }); +} + +export function assertDeadline( + plannedAtMs: number, + deadlineAtMs: number, +): void { + if ( + deadlineAtMs <= plannedAtMs || + deadlineAtMs - plannedAtMs > MAX_MODEL_INVOCATION_MS + ) { + invalid('diagnosis deadline is invalid'); + } +} + +export function assertJsonBudget( + value: unknown, + maximumBytes: number, + label: string, +): void { + if (Buffer.byteLength(JSON.stringify(value), 'utf8') > maximumBytes) { + invalid(`${label} exceeds its byte budget`); + } +} + +export { normalizeProjectPolicySubject }; diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/executionAdmission.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/executionAdmission.ts new file mode 100644 index 00000000..bcad9381 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/executionAdmission.ts @@ -0,0 +1,8 @@ +export * from './admission/contracts'; +export * from './admission/durableEvidence'; +export * from './admission/plan'; +export { + failureDiagnosisEgressPolicyDigest, + failureDiagnosisModelIntentDigest, + failureDiagnosisToolInputDigest, +} from './admission/validation'; diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/postgresAdmissionRepository.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/postgresAdmissionRepository.ts new file mode 100644 index 00000000..89a5017b --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/postgresAdmissionRepository.ts @@ -0,0 +1,4 @@ +export { + PostgresCopilotFailureDiagnosisAdmissionRepository, + type PostgresCopilotFailureDiagnosisAdmissionMutationGuard, +} from './admission/postgresRepository'; diff --git a/packages/ql3-ai/src/migration/model-invocation/identities.ts b/packages/ql3-ai/src/migration/model-invocation/identities.ts index 950a324b..6bd877cf 100644 --- a/packages/ql3-ai/src/migration/model-invocation/identities.ts +++ b/packages/ql3-ai/src/migration/model-invocation/identities.ts @@ -61,6 +61,8 @@ export const POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_MIGRATION_ID = 'pg-9016-ai-plugin-package-prompt-output-key-rotation'; export const POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID = 'pg-9017-ai-plugin-package-prompt-product-authorization'; +export const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID = + 'pg-9018-ai-copilot-failure-diagnosis-admissions'; export const POSTGRES_MODEL_INVOCATION_SCHEMA = 'ql3_ai'; export const LOCAL_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE = 'QingLong3AiSchemaMigrations'; diff --git a/packages/ql3-ai/src/migration/model-invocation/postgres.ts b/packages/ql3-ai/src/migration/model-invocation/postgres.ts index 5aee57f8..aab56723 100644 --- a/packages/ql3-ai/src/migration/model-invocation/postgres.ts +++ b/packages/ql3-ai/src/migration/model-invocation/postgres.ts @@ -1,7 +1,4 @@ -import type { - PostgresPool, - PostgresQueryable, -} from '@qinglong/runtime-core'; +import type { PostgresPool, PostgresQueryable } from '@qinglong/runtime-core'; import { MigrationStreamHistoryCorruptionError, runMigrationStream, @@ -30,11 +27,13 @@ import { POSTGRES_MODEL_PROVIDER_CREDENTIAL_TEST_CONNECTION_MIGRATION_ID, POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_MIGRATION_ID, POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_SCHEMA, POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE, } from './identities'; import { postgresCatalogMigrations } from './postgres/catalog'; +import { postgresCopilotMigrations } from './postgres/copilot'; import { postgresCoreMigrations } from './postgres/core'; import { postgresCredentialMigrations } from './postgres/credential'; import { @@ -63,6 +62,7 @@ const POSTGRES_HISTORY_IDENTITY = Object.freeze({ POSTGRES_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_IDENTITY_MIGRATION_ID, POSTGRES_MODEL_PROVIDER_CREDENTIAL_TEST_CONNECTION_MIGRATION_ID, POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, ]), streamId: POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID, dialect: 'postgresql' as const, @@ -246,6 +246,7 @@ export const postgresModelInvocationMigrationDefinition: MigrationStreamDefiniti ...postgresPromptBaseMigrations, ...postgresCredentialMigrations, ...postgresPromptExtensionMigrations, + ...postgresCopilotMigrations, ]), }); diff --git a/packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts b/packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts new file mode 100644 index 00000000..764b75d3 --- /dev/null +++ b/packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts @@ -0,0 +1,220 @@ +import type { PostgresQueryable } from '@qinglong/runtime-core'; + +import { + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, + POSTGRES_MODEL_INVOCATION_SCHEMA, +} from '../identities'; +import { defineSqlMigration } from '../shared'; + +const ADMISSION_TABLE = 'copilot_failure_diagnosis_admissions'; +const SOURCE_SNAPSHOT_FUNCTION = + 'copilot_failure_diagnosis_admission_source_snapshot'; + +const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_TABLE_SQL = ` +CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" ( + request_id varchar(128) PRIMARY KEY, + plan_digest char(64) NOT NULL UNIQUE, + run_id varchar(36) NOT NULL UNIQUE, + project_id varchar(128) NOT NULL, + source_run_id varchar(36) NOT NULL, + source_run_version integer NOT NULL, + source_run_status varchar(32) NOT NULL, + source_attempt_id varchar(36) NOT NULL, + source_attempt_status varchar(32) NOT NULL, + source_log_artifact_id varchar(36) NOT NULL, + tool_plan_digest char(64) NOT NULL, + tool_action_digest char(64) NOT NULL, + tool_step_run_id varchar(128) NOT NULL UNIQUE, + model_intent_digest char(64) NOT NULL, + model_step_run_id varchar(128) NOT NULL UNIQUE, + admitted_at_ms bigint NOT NULL, + receipt_digest char(64) NOT NULL UNIQUE, + plan_json jsonb NOT NULL, + receipt_json jsonb NOT NULL, + CONSTRAINT ql3_ai_copilot_diagnosis_admission_run_fk + FOREIGN KEY (run_id) REFERENCES "ql3"."runs" (id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_admission_source_run_fk + FOREIGN KEY (source_run_id) REFERENCES "ql3"."runs" (id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_admission_source_attempt_fk + FOREIGN KEY (source_attempt_id) + REFERENCES "ql3"."run_attempts" (id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_admission_tool_step_fk + FOREIGN KEY (run_id, tool_step_run_id) + REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_admission_model_step_fk + FOREIGN KEY (run_id, model_step_run_id) + REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_admission_identity_check CHECK ( + request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND + run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$' AND + project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND + source_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$' AND + source_attempt_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$' AND + source_log_artifact_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$' AND + tool_step_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND + model_step_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND + source_run_version BETWEEN 1 AND 2147483647 AND admitted_at_ms >= 0 + ), + CONSTRAINT ql3_ai_copilot_diagnosis_admission_status_check CHECK ( + source_run_status IN ('failed', 'timed_out') AND + source_attempt_status IN ('failed', 'timed_out', 'lost') AND + ((source_run_status = 'failed' AND source_attempt_status IN ('failed', 'lost')) OR + (source_run_status = 'timed_out' AND source_attempt_status = 'timed_out')) + ), + CONSTRAINT ql3_ai_copilot_diagnosis_admission_digest_check CHECK ( + plan_digest ~ '^[0-9a-f]{64}$' AND + tool_plan_digest ~ '^[0-9a-f]{64}$' AND + tool_action_digest ~ '^[0-9a-f]{64}$' AND + model_intent_digest ~ '^[0-9a-f]{64}$' AND + receipt_digest ~ '^[0-9a-f]{64}$' + ), + CONSTRAINT ql3_ai_copilot_diagnosis_admission_json_check CHECK ( + jsonb_typeof(plan_json) = 'object' AND + octet_length(plan_json::text) BETWEEN 2 AND 32768 AND + plan_json @> jsonb_build_object( + 'schema', 'qinglong/copilot-failure-diagnosis-execution-plan@v1', + 'requestId', request_id, 'planDigest', plan_digest, + 'runId', run_id, 'projectId', project_id, + 'toolStepRunId', tool_step_run_id, + 'modelStepRunId', model_step_run_id, + 'plannedAtMs', admitted_at_ms + ) AND + plan_json -> 'source' @> jsonb_build_object( + 'runId', source_run_id, 'runVersion', source_run_version, + 'runStatus', source_run_status, 'attemptId', source_attempt_id, + 'attemptStatus', source_attempt_status, + 'logArtifactId', source_log_artifact_id + ) AND + plan_json -> 'tool' @> jsonb_build_object( + 'planDigest', tool_plan_digest, 'actionDigest', tool_action_digest + ) AND + plan_json -> 'model' @> jsonb_build_object( + 'intentDigest', model_intent_digest + ) AND + jsonb_typeof(receipt_json) = 'object' AND + octet_length(receipt_json::text) BETWEEN 2 AND 16384 AND + receipt_json @> jsonb_build_object( + 'schema', 'qinglong/copilot-failure-diagnosis-admission-receipt@v1', + 'requestId', request_id, 'planDigest', plan_digest, + 'runId', run_id, 'sourceRunId', source_run_id, + 'sourceRunVersion', source_run_version, + 'sourceAttemptId', source_attempt_id, + 'toolStepRunId', tool_step_run_id, + 'modelStepRunId', model_step_run_id, + 'admittedAtMs', admitted_at_ms, 'receiptDigest', receipt_digest + ) + ) +)`; + +const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_SOURCE_SNAPSHOT_SQL = ` +CREATE FUNCTION + "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${SOURCE_SNAPSHOT_FUNCTION}"( + p_project_id varchar, + p_subject_type varchar, + p_subject_id varchar, + p_project_version integer, + p_binding_version integer, + p_source_run_id varchar, + p_source_attempt_id varchar + ) +RETURNS TABLE( + run_id varchar, + run_version integer, + run_status varchar, + attempt_id varchar, + attempt_status varchar, + attempt_finished_at_ms bigint, + log_artifact_id varchar +) +LANGUAGE plpgsql +VOLATILE +SECURITY DEFINER +SET search_path = pg_catalog, ql3, ql3_ai +AS $ql3_ai$ +BEGIN + IF NOT pg_has_role(session_user, 'ql3_runtime', 'member') THEN + RAISE EXCEPTION 'Runtime authority is required' + USING ERRCODE = 'insufficient_privilege'; + END IF; + + PERFORM 1 + FROM "ql3"."projects" AS project + JOIN "ql3"."project_role_bindings" AS binding + ON binding.project_id = project.id + AND binding.subject_type = p_subject_type + AND binding.subject_id = p_subject_id + AND binding.version = p_binding_version + WHERE project.id = p_project_id + AND project.status = 'active' + AND project.version = p_project_version + AND binding.state = 'active' + AND binding.version = ( + SELECT max(candidate.version) + FROM "ql3"."project_role_bindings" AS candidate + WHERE candidate.project_id = p_project_id + AND candidate.subject_type = p_subject_type + AND candidate.subject_id = p_subject_id + ) + FOR SHARE OF project, binding; + IF NOT FOUND THEN + RETURN; + END IF; + + RETURN QUERY + SELECT source_run.id, source_run.version, source_run.status, + source_attempt.id, source_attempt.status, + source_attempt.finished_at_ms, source_attempt.log_artifact_id + FROM "ql3"."runs" AS source_run + JOIN "ql3"."run_attempts" AS source_attempt + ON source_attempt.run_id = source_run.id + WHERE source_run.id = p_source_run_id + AND source_run.project_id = p_project_id + AND source_run.status IN ('failed', 'timed_out') + AND source_attempt.id = p_source_attempt_id + AND source_attempt.attempt = ( + SELECT max(candidate.attempt) + FROM "ql3"."run_attempts" AS candidate + WHERE candidate.run_id = source_run.id + ) + AND source_attempt.status IN ('failed', 'timed_out', 'lost') + AND ((source_run.status = 'failed' AND source_attempt.status IN ('failed', 'lost')) OR + (source_run.status = 'timed_out' AND source_attempt.status = 'timed_out')) + AND source_attempt.finished_at_ms IS NOT NULL + AND source_attempt.log_artifact_id IS NOT NULL + FOR SHARE OF source_run, source_attempt; +END +$ql3_ai$`; + +const postgresCopilotFailureDiagnosisAdmissionMigration = + defineSqlMigration( + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, + [ + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_TABLE_SQL, + `CREATE UNIQUE INDEX ql3_ai_copilot_diagnosis_run_steps_uidx + ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" + (run_id, tool_step_run_id, model_step_run_id)`, + `CREATE INDEX ql3_ai_copilot_diagnosis_source_idx + ON "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" + (project_id, source_run_id, admitted_at_ms, request_id)`, + `REVOKE ALL ON TABLE + "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" + FROM PUBLIC`, + `GRANT SELECT, INSERT ON TABLE + "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" + TO ql3_runtime`, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_SOURCE_SNAPSHOT_SQL, + `REVOKE ALL ON FUNCTION + "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${SOURCE_SNAPSHOT_FUNCTION}"( + varchar, varchar, varchar, integer, integer, varchar, varchar + ) FROM PUBLIC`, + `GRANT EXECUTE ON FUNCTION + "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${SOURCE_SNAPSHOT_FUNCTION}"( + varchar, varchar, varchar, integer, integer, varchar, varchar + ) TO ql3_runtime`, + ], + (context, statement) => context.query(statement).then(() => undefined), + ); + +export const postgresCopilotMigrations = Object.freeze([ + postgresCopilotFailureDiagnosisAdmissionMigration, +]); diff --git a/packages/ql3-ai/test/failureDiagnosisExecutionAdmission.test.cjs b/packages/ql3-ai/test/failureDiagnosisExecutionAdmission.test.cjs new file mode 100644 index 00000000..4c0cd9a8 --- /dev/null +++ b/packages/ql3-ai/test/failureDiagnosisExecutionAdmission.test.cjs @@ -0,0 +1,325 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + BUILTIN_RUN_LOG_EXCERPT_TOOL, + BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION, + createBuiltInRunLogExcerptToolHandlerBinding, +} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool'); +const { + createPluginPackageResourceGenerationFromReferences, +} = require('@qinglong/runtime-core/plugin-package-resource-generation'); +const { + createProjectToolDefinitionSnapshot, + projectToolDefinitionRegistry, +} = require('@qinglong/runtime-core/project-tool-definition-snapshot'); +const { + prepareToolInvocation, +} = require('@qinglong/runtime-core/tool-registry'); +const { + TrustedToolHandlerBindingRegistry, + createTrustedToolInvocationPlan, +} = require('@qinglong/runtime-core/trusted-tool-invocation'); +const { + COPILOT_FAILURE_DIAGNOSIS_ADMISSION_RECEIPT_SCHEMA, + COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_SCHEMA, + InvalidCopilotFailureDiagnosisExecutionPlanError, + createCopilotFailureDiagnosisAdmissionBundle, + failureDiagnosisToolInputDigest, + normalizeCopilotFailureDiagnosisAdmissionReceipt, + normalizeCopilotFailureDiagnosisExecutionPlan, + prepareCopilotFailureDiagnosisExecution, +} = require('../dist/copilot/failure-diagnosis/executionAdmission.js'); +const { + storedJsonEquals, +} = require('../dist/copilot/failure-diagnosis/admission/postgresRepository.js'); + +const DIGEST_A = 'a'.repeat(64); +const DIGEST_B = 'b'.repeat(64); +const DIGEST_C = 'c'.repeat(64); +const SUBJECT = Object.freeze({ type: 'user', id: 'usr-diagnosis-owner' }); +const FENCE = Object.freeze({ projectVersion: 3, bindingVersion: 7 }); +const SOURCE = Object.freeze({ + runId: 'run-source-failed', + runVersion: 8, + runStatus: 'failed', + attemptId: 'attempt-source-final', + attemptStatus: 'failed', + attemptFinishedAtMs: 1_900, + logArtifactId: 'artifact-source-log', +}); + +test('compares PostgreSQL jsonb evidence independent of object key order', () => { + assert.equal( + storedJsonEquals( + { outer: { beta: 2, alpha: 1 }, items: [{ right: 2, left: 1 }] }, + { items: [{ left: 1, right: 2 }], outer: { alpha: 1, beta: 2 } }, + ), + true, + ); + assert.equal( + storedJsonEquals({ outer: { alpha: 1 } }, { outer: { alpha: 2 } }), + false, + ); +}); + +function snapshot() { + const generation = createPluginPackageResourceGenerationFromReferences({ + installationId: 'install-qinglong-run-log-excerpt', + projectId: 'project-diagnosis', + packageName: 'qinglong', + lockDigest: DIGEST_A, + generation: 1, + previousActiveLockDigest: null, + contentDigest: DIGEST_B, + resources: [], + }); + return createProjectToolDefinitionSnapshot({ + projectId: 'project-diagnosis', + contributions: [ + { + generation, + revisionDigest: DIGEST_C, + definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION], + }, + ], + }); +} + +function principal() { + return { + subject: SUBJECT, + authenticationId: 'auth-diagnosis-owner', + authenticatedAtMs: 1_000, + expiresAtMs: 10_000, + assurance: 'multi_factor', + }; +} + +function authorizer(effect = 'allow') { + return { + async authorize() { + return { + effect, + reasons: + effect === 'allow' + ? ['role_grant'] + : ['agent_action_requires_approval'], + fence: FENCE, + }; + }, + }; +} + +async function toolHarness(options = {}) { + const currentSnapshot = snapshot(); + const binding = createBuiltInRunLogExcerptToolHandlerBinding( + currentSnapshot, + [options.profile ?? 'cluster-control'], + ); + const bindings = new TrustedToolHandlerBindingRegistry(currentSnapshot, [ + binding, + ]); + const source = options.source ?? SOURCE; + const invocation = await prepareToolInvocation( + projectToolDefinitionRegistry(currentSnapshot), + { + projectId: 'project-diagnosis', + principal: principal(), + nowMs: 1_200, + tool: BUILTIN_RUN_LOG_EXCERPT_TOOL, + input: { + runId: options.inputRunId ?? source.runId, + attemptId: options.inputAttemptId ?? source.attemptId, + }, + }, + authorizer(options.effect), + ); + const bundle = createTrustedToolInvocationPlan(bindings, invocation, { + actionRef: 'diagnosis-log-tool-plan', + inputArtifactId: 'diagnosis-tool-input', + previewArtifactId: 'diagnosis-tool-preview', + artifactKeyId: 'diagnosis-input-key', + artifactKey: Buffer.alloc(32, 0x21), + artifactNonce: Buffer.alloc(12, 0x31), + profile: options.profile ?? 'cluster-control', + preview: { + title: 'Read failed Run log', + summary: 'Reads one bounded redacted log excerpt', + fields: [ + { kind: 'identifier', label: 'Run', value: source.runId }, + { kind: 'identifier', label: 'Attempt', value: source.attemptId }, + ], + warnings: ['potentially_sensitive_output'], + }, + sealedAtMs: 2_000, + }); + return { bindings, toolPlan: bundle.plan, source }; +} + +function model(overrides = {}) { + return { + provider: 'provider-primary', + model: 'model-diagnosis', + modelBoundary: 'external', + responseLanguage: 'zh-CN', + maxOutputTokens: 512, + egressPolicy: { + schema: 'qinglong/copilot-model-egress-policy@v1', + revision: 'policy-diagnosis-v1', + potentiallySensitiveDataBoundaries: ['on_device', 'external'], + maxInputBytes: 64 * 1024, + maxOutputTokens: 1_024, + }, + ...overrides, + }; +} + +async function plan(options = {}) { + const harness = await toolHarness(options); + return prepareCopilotFailureDiagnosisExecution({ + requestId: 'request-diagnosis-001', + traceId: 'trace-diagnosis-001', + source: harness.source, + toolPlan: harness.toolPlan, + bindings: harness.bindings, + model: model(options.model), + deadlineAtMs: 8_000, + plannedAtMs: 2_100, + }); +} + +test('creates one independent diagnosis Run with ready Tool and pending Model Steps', async () => { + const current = await plan(); + assert.equal(current.schema, COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_SCHEMA); + assert.equal(current.runId.startsWith('cdr:'), true); + assert.equal(current.toolStepRunId.startsWith('cdt:'), true); + assert.equal(current.modelStepRunId.startsWith('cdm:'), true); + assert.equal(current.modelInvocationId.startsWith('cdi:'), true); + assert.equal( + current.tool.invocationArtifact.inputDigest, + failureDiagnosisToolInputDigest(SOURCE), + ); + assert.equal(current.model.egressPolicyDigest.length, 64); + assert.equal(current.model.intentDigest.length, 64); + + const bundle = createCopilotFailureDiagnosisAdmissionBundle(current); + assert.equal(bundle.run.parentRunId, SOURCE.runId); + assert.equal(bundle.run.status, 'running'); + assert.equal(bundle.run.version, 3); + assert.equal(bundle.run.eventSequence, 3); + assert.equal(bundle.admissionEvent.sequence, 1); + for (const event of [ + bundle.admissionEvent, + bundle.toolStepMutation.event, + bundle.modelStepMutation.event, + ]) { + assert.equal(Buffer.byteLength(event.id, 'utf8') <= 36, true); + } + assert.deepEqual( + { + kind: bundle.toolStepMutation.stepRun.kind, + status: bundle.toolStepMutation.stepRun.status, + sequence: bundle.toolStepMutation.event.sequence, + }, + { kind: 'tool', status: 'ready', sequence: 2 }, + ); + assert.deepEqual( + { + kind: bundle.modelStepMutation.stepRun.kind, + status: bundle.modelStepMutation.stepRun.status, + parentStepRunId: bundle.modelStepMutation.stepRun.parentStepRunId, + sequence: bundle.modelStepMutation.event.sequence, + }, + { + kind: 'model', + status: 'pending', + parentStepRunId: current.toolStepRunId, + sequence: 3, + }, + ); + assert.equal( + bundle.receipt.schema, + COPILOT_FAILURE_DIAGNOSIS_ADMISSION_RECEIPT_SCHEMA, + ); + assert.equal(bundle.receipt.finalRunVersion, 3); + assert.deepEqual( + normalizeCopilotFailureDiagnosisExecutionPlan( + JSON.parse(JSON.stringify(current)), + ), + current, + ); + assert.deepEqual( + normalizeCopilotFailureDiagnosisAdmissionReceipt( + JSON.parse(JSON.stringify(bundle.receipt)), + ), + bundle.receipt, + ); +}); + +test('rejects a Tool input detached from the source failure fence', async () => { + await assert.rejects( + () => plan({ inputAttemptId: 'attempt-unrelated' }), + InvalidCopilotFailureDiagnosisExecutionPlanError, + ); +}); + +test('rejects approval-pending or non-Cluster Tool authority before Run creation', async () => { + await assert.rejects( + () => plan({ effect: 'require_approval' }), + InvalidCopilotFailureDiagnosisExecutionPlanError, + ); + await assert.rejects( + () => plan({ profile: 'edge' }), + InvalidCopilotFailureDiagnosisExecutionPlanError, + ); +}); + +test('rejects inconsistent source terminal evidence and denied model egress', async () => { + await assert.rejects( + () => + plan({ + source: { + ...SOURCE, + runStatus: 'timed_out', + attemptStatus: 'failed', + }, + }), + InvalidCopilotFailureDiagnosisExecutionPlanError, + ); + await assert.rejects( + () => + plan({ + model: { + egressPolicy: { + ...model().egressPolicy, + potentiallySensitiveDataBoundaries: ['on_device'], + }, + }, + }), + InvalidCopilotFailureDiagnosisExecutionPlanError, + ); +}); + +test('fails closed on widened or digest-drifted durable plans', async () => { + const current = await plan(); + assert.throws( + () => + normalizeCopilotFailureDiagnosisExecutionPlan({ + ...current, + callerAuthority: 'execute', + }), + InvalidCopilotFailureDiagnosisExecutionPlanError, + ); + assert.throws( + () => + normalizeCopilotFailureDiagnosisExecutionPlan({ + ...current, + source: { + ...current.source, + runVersion: current.source.runVersion + 1, + }, + }), + InvalidCopilotFailureDiagnosisExecutionPlanError, + ); +}); diff --git a/packages/ql3-ai/test/modelInvocationMigration.test.cjs b/packages/ql3-ai/test/modelInvocationMigration.test.cjs index caf1329e..bc3fa5fa 100644 --- a/packages/ql3-ai/test/modelInvocationMigration.test.cjs +++ b/packages/ql3-ai/test/modelInvocationMigration.test.cjs @@ -19,6 +19,7 @@ const { LOCAL_MODEL_INVOCATION_USAGE_MIGRATION_ID, LOCAL_MODEL_PRICE_CATALOG_AUTHORIZATION_MIGRATION_ID, LOCAL_MODEL_PRICE_CATALOG_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE, POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID, @@ -421,6 +422,10 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async () POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID, 'pg-9017-ai-plugin-package-prompt-product-authorization', ); + assert.equal( + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, + 'pg-9018-ai-copilot-failure-diagnosis-admissions', + ); assert.equal( POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE, 'ai_schema_migrations', @@ -499,9 +504,38 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async () /^[0-9a-f]{64}$/, ); assert.equal( - postgresModelInvocationMigrationDefinition.migrations.length, - 17, + postgresModelInvocationMigrationDefinition.migrations[17].checksum, + 'cd0837c68ecc6c2bce58d048308d0239397b6347b4483f02372f966c05ae7ad6', ); + assert.equal( + postgresModelInvocationMigrationDefinition.migrations.length, + 18, + ); + + const diagnosisAdmissionStatements = []; + await postgresModelInvocationMigrationDefinition.migrations[17].up({ + async query(statement) { + diagnosisAdmissionStatements.push(statement); + return { rows: [] }; + }, + }); + const diagnosisAdmissionSql = diagnosisAdmissionStatements.join('\n'); + assert.match( + diagnosisAdmissionSql, + /CREATE TABLE "ql3_ai"\."copilot_failure_diagnosis_admissions"/, + ); + assert.match( + diagnosisAdmissionSql, + /CREATE FUNCTION[\s\S]*copilot_failure_diagnosis_admission_source_snapshot/, + ); + assert.match(diagnosisAdmissionSql, /SECURITY DEFINER/); + assert.match(diagnosisAdmissionSql, /FOR SHARE OF project, binding/); + assert.match( + diagnosisAdmissionSql, + /FOR SHARE OF source_run, source_attempt/, + ); + assert.match(diagnosisAdmissionSql, /TO ql3_runtime/); + assert.doesNotMatch(diagnosisAdmissionSql, /GRANT[^;]*(?:UPDATE|DELETE)/); const retirementStatements = []; await postgresModelInvocationMigrationDefinition.migrations[10].up({ diff --git a/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs b/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs index 52f936e3..2c75c65d 100644 --- a/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs +++ b/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs @@ -173,7 +173,10 @@ test('readiness binds exact migration history and least-privilege primary author }, }); assert.equal(report.ready, true); - assert.equal(report.migrationIds.at(-1).startsWith('pg-9017-'), true); + assert.equal( + report.migrationIds.at(-1), + 'pg-9018-ai-copilot-failure-diagnosis-admissions', + ); assert.match( queries[1], /model_provider_credential_management_identity_keyset_ledger/, diff --git a/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs b/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs index c9c030a4..94206a78 100644 --- a/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs +++ b/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs @@ -277,7 +277,7 @@ test('tester readiness freezes migration history and least privilege', async () assert.equal(ready.ready, true); assert.equal( ready.migrationIds.at(-1), - 'pg-9017-ai-plugin-package-prompt-product-authorization', + 'pg-9018-ai-copilot-failure-diagnosis-admissions', ); await assert.rejects( diff --git a/packages/ql3-cluster-postgres/src/run-recovery/clusterRecoverySource.ts b/packages/ql3-cluster-postgres/src/run-recovery/clusterRecoverySource.ts index 3c5559eb..e65989c0 100644 --- a/packages/ql3-cluster-postgres/src/run-recovery/clusterRecoverySource.ts +++ b/packages/ql3-cluster-postgres/src/run-recovery/clusterRecoverySource.ts @@ -34,7 +34,10 @@ WITH observation AS ( FROM "ql3"."runs" CROSS JOIN observation WHERE execution_owner = 'runtime' - AND trigger_type <> 'plugin_package_workflow' + AND trigger_type NOT IN ( + 'plugin_package_workflow', + 'copilot_failure_diagnosis' + ) AND ( status = 'created' OR ( diff --git a/packages/ql3-cluster-postgres/test/clusterRecoverySource.test.cjs b/packages/ql3-cluster-postgres/test/clusterRecoverySource.test.cjs index 5ce88a0e..2f0b5389 100644 --- a/packages/ql3-cluster-postgres/test/clusterRecoverySource.test.cjs +++ b/packages/ql3-cluster-postgres/test/clusterRecoverySource.test.cjs @@ -1,6 +1,8 @@ const assert = require('node:assert/strict'); const { test } = require('node:test'); -const { PostgresClusterControlRecoverySource } = require('../dist/entrypoints/runtime'); +const { + PostgresClusterControlRecoverySource, +} = require('../dist/entrypoints/runtime'); function sourceWith(rows, observations = []) { return new PostgresClusterControlRecoverySource({ @@ -70,7 +72,7 @@ test('reads Run and Attempt recovery candidates through one bounded query', asyn assert.match(observations[0].text, /execution_owner = 'runtime'/); assert.match( observations[0].text, - /trigger_type <> 'plugin_package_workflow'/, + /trigger_type NOT IN \([\s\S]*'plugin_package_workflow',[\s\S]*'copilot_failure_diagnosis'[\s\S]*\)/, ); assert.match(observations[0].text, /attempt_candidates/); assert.match( @@ -81,10 +83,7 @@ test('reads Run and Attempt recovery candidates through one bounded query', asyn observations[0].text, /attempt_run\.trigger_type = 'plugin_package_workflow'/, ); - assert.match( - observations[0].text, - /workflow_task\.attempt_id = attempt\.id/, - ); + assert.match(observations[0].text, /workflow_task\.attempt_id = attempt\.id/); assert.match( observations[0].text, /lease_expires_at_ms > observation\.observed_at_ms/, diff --git a/packages/ql3-runtime-core/src/cluster-control/clusterControlActivation.ts b/packages/ql3-runtime-core/src/cluster-control/clusterControlActivation.ts index c00f1312..52086811 100644 --- a/packages/ql3-runtime-core/src/cluster-control/clusterControlActivation.ts +++ b/packages/ql3-runtime-core/src/cluster-control/clusterControlActivation.ts @@ -89,7 +89,10 @@ function assertSafeRecovery( recovery: ClusterControlStartupRecoverySummary, ): void { if (!recovery.safe || recovery.remaining !== 0 || recovery.failed !== 0) { - throw new Error('Cluster-control startup recovery did not converge safely'); + throw new Error( + 'Cluster-control startup recovery did not converge safely ' + + `(remaining=${recovery.remaining}, failed=${recovery.failed})`, + ); } } diff --git a/packages/ql3-runtime-core/test/clusterControlActivation.test.cjs b/packages/ql3-runtime-core/test/clusterControlActivation.test.cjs index 96ba6d5a..718d5e45 100644 --- a/packages/ql3-runtime-core/test/clusterControlActivation.test.cjs +++ b/packages/ql3-runtime-core/test/clusterControlActivation.test.cjs @@ -127,7 +127,7 @@ test('unsafe recovery stops the stack before lifecycles and admission', async () }, }), ), - /did not converge safely/, + /did not converge safely \(remaining=1, failed=0\)/, ); assert.equal(events.includes('start-lifecycles'), false); assert.equal(events.includes('install-admission'), false); diff --git a/scripts/ql3-postgres-ha-contract.cjs b/scripts/ql3-postgres-ha-contract.cjs index 79eba3ae..fca0b515 100644 --- a/scripts/ql3-postgres-ha-contract.cjs +++ b/scripts/ql3-postgres-ha-contract.cjs @@ -21,6 +21,7 @@ const { createPostgresDatabaseOpener, PostgresClusterRunCancellationConvergenceRepository, PostgresClusterRunCancellationRepository, + PostgresClusterControlRecoverySource, PostgresClusterScheduleRepository, PostgresRemoteWorkerCompletionRepository, PostgresRemoteWorkerLeaseControlRepository, @@ -32,6 +33,7 @@ const { runPostgresMigrations, } = require('../packages/ql3-cluster-postgres/dist/migration/migration.js'); const { + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_PRICING_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_QUOTA_MIGRATION_ID, @@ -51,6 +53,31 @@ const { POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID, migratePostgresModelInvocationFeature, } = require('../packages/ql3-ai/dist/migration/modelInvocationMigration.js'); +const { + prepareCopilotFailureDiagnosisExecution, +} = require('../packages/ql3-ai/dist/copilot/failure-diagnosis/executionAdmission.js'); +const { + PostgresCopilotFailureDiagnosisAdmissionRepository, +} = require('../packages/ql3-ai/dist/copilot/failure-diagnosis/postgresAdmissionRepository.js'); +const { + BUILTIN_RUN_LOG_EXCERPT_TOOL, + BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION, + createBuiltInRunLogExcerptToolHandlerBinding, +} = require('../packages/ql3-runtime-core/dist/tool-execution/builtin-run-log-excerpt/builtInRunLogExcerptTool.js'); +const { + createPluginPackageResourceGenerationFromReferences, +} = require('../packages/ql3-runtime-core/dist/plugin-package/pluginPackageResourceGeneration.js'); +const { + createProjectToolDefinitionSnapshot, + projectToolDefinitionRegistry, +} = require('../packages/ql3-runtime-core/dist/tool-execution/tool-registry/projectToolDefinitionSnapshot.js'); +const { + prepareToolInvocation, +} = require('../packages/ql3-runtime-core/dist/tool-execution/tool-registry/toolRegistry.js'); +const { + TrustedToolHandlerBindingRegistry, + createTrustedToolInvocationPlan, +} = require('../packages/ql3-runtime-core/dist/tool-execution/trustedToolInvocation.js'); const { MODEL_PROVIDER_CREDENTIAL_TRANSITION_COMMAND_SCHEMA, ModelProviderCredentialCatalogUnavailableError, @@ -902,6 +929,8 @@ async function modelInvocationFeatureFacts(pool) { FROM pg_tables WHERE schemaname = 'ql3_ai' AND ( + tablename LIKE 'copilot_failure_diagnosis_%' + OR tablename LIKE 'model_invocation_%' OR tablename LIKE 'model_price_catalog_%' OR tablename LIKE 'model_provider_credential_%' @@ -1424,6 +1453,42 @@ async function modelInvocationFeatureFacts(pool) { 'ql3_ai.plugin_package_prompt_admission_snapshot(varchar,varchar,character,varchar,varchar,integer,integer)', 'EXECUTE' ) AS "promptSnapshotRuntimeOnly", + has_table_privilege( + 'ql3_runtime', + 'ql3_ai.copilot_failure_diagnosis_admissions', 'SELECT,INSERT' + ) AND NOT has_table_privilege( + 'ql3_runtime', + 'ql3_ai.copilot_failure_diagnosis_admissions', 'UPDATE,DELETE' + ) AND NOT EXISTS ( + SELECT 1 + FROM unnest(ARRAY[ + 'ql3_admin', 'ql3_package_manager', 'ql3_package_executor', + 'ql3_worker_ingress', 'ql3_ai_maintenance', + 'ql3_ai_credential_manager', 'ql3_ai_credential_tester' + ]::text[]) AS denied(role_name) + WHERE has_table_privilege( + denied.role_name, + 'ql3_ai.copilot_failure_diagnosis_admissions', + 'SELECT,INSERT,UPDATE,DELETE' + ) + ) AS "copilotDiagnosisAdmissionRuntimeOnly", + has_function_privilege( + 'ql3_runtime', + 'ql3_ai.copilot_failure_diagnosis_admission_source_snapshot(varchar,varchar,varchar,integer,integer,varchar,varchar)', + 'EXECUTE' + ) AND NOT EXISTS ( + SELECT 1 + FROM unnest(ARRAY[ + 'ql3_admin', 'ql3_package_manager', 'ql3_package_executor', + 'ql3_worker_ingress', 'ql3_ai_maintenance', + 'ql3_ai_credential_manager', 'ql3_ai_credential_tester' + ]::text[]) AS denied(role_name) + WHERE has_function_privilege( + denied.role_name, + 'ql3_ai.copilot_failure_diagnosis_admission_source_snapshot(varchar,varchar,varchar,integer,integer,varchar,varchar)', + 'EXECUTE' + ) + ) AS "copilotDiagnosisSnapshotRuntimeOnly", has_table_privilege( 'ql3_runtime', 'ql3_ai.ai_schema_migrations', 'SELECT' ) AND NOT has_table_privilege( @@ -1518,6 +1583,273 @@ async function modelInvocationFeatureFacts(pool) { }; } +async function copilotFailureDiagnosisAdmissionFacts(pool, requestId) { + const result = await pool.query( + `SELECT admission.request_id AS "requestId", + admission.plan_digest AS "planDigest", + admission.receipt_digest AS "receiptDigest", + admission.source_run_id AS "sourceRunId", + admission.source_attempt_id AS "sourceAttemptId", + admission.tool_step_run_id AS "toolStepRunId", + admission.model_step_run_id AS "modelStepRunId", + run.parent_run_id AS "parentRunId", run.status, + run.version, run.event_sequence AS "eventSequence", + (SELECT count(*)::integer FROM "ql3"."run_events" + WHERE run_id = admission.run_id) AS "eventCount", + (SELECT count(*)::integer FROM "ql3"."step_run_mutations" + WHERE run_id = admission.run_id) AS "mutationCount", + tool.status AS "toolStatus", tool.kind AS "toolKind", + model.status AS "modelStatus", model.kind AS "modelKind", + model.parent_step_run_id AS "modelParentStepRunId", + admission.plan_json::text AS "planJson", + admission.receipt_json::text AS "receiptJson" + FROM "ql3_ai"."copilot_failure_diagnosis_admissions" AS admission + JOIN "ql3"."runs" AS run ON run.id = admission.run_id + JOIN "ql3"."step_runs" AS tool + ON tool.run_id = admission.run_id + AND tool.id = admission.tool_step_run_id + JOIN "ql3"."step_runs" AS model + ON model.run_id = admission.run_id + AND model.id = admission.model_step_run_id + WHERE admission.request_id = $1`, + [requestId], + ); + assert.equal(result.rowCount, 1); + const { planJson, receiptJson, ...facts } = result.rows[0]; + assert.equal(typeof planJson, 'string'); + assert.equal(typeof receiptJson, 'string'); + return { + ...facts, + planJsonDigest: createHash('sha256').update(planJson).digest('hex'), + receiptJsonDigest: createHash('sha256').update(receiptJson).digest('hex'), + }; +} + +async function runCopilotFailureDiagnosisAdmissionHaEvidence(options) { + const { primaryPort, migrationPool } = options; + const baseTimeMs = Date.now(); + const suffix = `${process.pid}-${randomBytes(3).toString('hex')}`; + const projectId = `diagnosis-${suffix}`; + const subjectId = `diagnosis-owner-${suffix}`; + const sourceRunId = `src-${randomBytes(10).toString('hex')}`; + const sourceAttemptId = `att-${randomBytes(10).toString('hex')}`; + const logArtifactId = `wlog-${createHash('sha256') + .update(suffix) + .digest('hex') + .slice(0, 30)}`; + const requestId = `diagnosis-request-${suffix}`; + const source = Object.freeze({ + runId: sourceRunId, + runVersion: 8, + runStatus: 'failed', + attemptId: sourceAttemptId, + attemptStatus: 'failed', + attemptFinishedAtMs: baseTimeMs, + logArtifactId, + }); + await migrationPool.query( + `INSERT INTO "ql3"."projects" ( + id, name, slug, status, version, created_at_ms, updated_at_ms + ) VALUES ($1, $1, $2, 'active', 1, 1, 1)`, + [projectId, projectId], + ); + await migrationPool.query( + `INSERT INTO "ql3"."project_role_bindings" ( + project_id, subject_type, subject_id, version, state, role, + mutation_id, changed_by_type, changed_by_id, created_at_ms + ) VALUES ($1, 'user', $2, 1, 'active', 'owner', $3, + 'system', 'ha-contract', 1)`, + [projectId, subjectId, `diagnosis-binding-${suffix}`], + ); + await migrationPool.query( + `INSERT INTO "ql3"."runs" ( + id, project_id, task_id, task_revision, trigger_type, + execution_origin, execution_owner, status, version, event_sequence, + created_at_ms, started_at_ms, finished_at_ms + ) VALUES ($1, $2, 'source-task', 'source-v1', 'manual', 'manual', + 'runtime', 'failed', 8, 8, $3, $4, $5)`, + [ + sourceRunId, + projectId, + baseTimeMs - 2_000, + baseTimeMs - 1_000, + baseTimeMs, + ], + ); + await migrationPool.query( + `INSERT INTO "ql3"."run_attempts" ( + id, run_id, attempt, status, executor_type, log_artifact_id, + callback_sequence, created_at_ms, started_at_ms, finished_at_ms, + error_code, error_summary + ) VALUES ($1, $2, 1, 'failed', 'remote_worker', $3, 0, + $4, $5, $6, 'exit_nonzero', 'bounded failure')`, + [ + sourceAttemptId, + sourceRunId, + logArtifactId, + baseTimeMs - 2_000, + baseTimeMs - 1_000, + baseTimeMs, + ], + ); + + const generation = createPluginPackageResourceGenerationFromReferences({ + installationId: `diagnosis-tool-${suffix}`, + projectId, + packageName: 'qinglong', + lockDigest: 'a'.repeat(64), + generation: 1, + previousActiveLockDigest: null, + contentDigest: 'b'.repeat(64), + resources: [], + }); + const snapshot = createProjectToolDefinitionSnapshot({ + projectId, + contributions: [ + { + generation, + revisionDigest: 'c'.repeat(64), + definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION], + }, + ], + }); + const principal = Object.freeze({ + subject: { type: 'user', id: subjectId }, + authenticationId: `diagnosis-auth-${suffix}`, + authenticatedAtMs: baseTimeMs - 1_000, + expiresAtMs: baseTimeMs + 60_000, + assurance: 'multi_factor', + }); + const invocation = await prepareToolInvocation( + projectToolDefinitionRegistry(snapshot), + { + projectId, + principal, + nowMs: baseTimeMs + 100, + tool: BUILTIN_RUN_LOG_EXCERPT_TOOL, + input: { runId: sourceRunId, attemptId: sourceAttemptId }, + }, + { + async authorize() { + return { + effect: 'allow', + reasons: ['role_grant'], + fence: { projectVersion: 1, bindingVersion: 1 }, + }; + }, + }, + ); + const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [ + createBuiltInRunLogExcerptToolHandlerBinding(snapshot, ['cluster-control']), + ]); + const toolPlan = createTrustedToolInvocationPlan(bindings, invocation, { + actionRef: `diagnosis-log-${suffix}`, + inputArtifactId: `diagnosis-input-${suffix}`, + previewArtifactId: `diagnosis-preview-${suffix}`, + artifactKeyId: `diagnosis-key-${suffix}`, + artifactKey: Buffer.alloc(32, 0x21), + artifactNonce: Buffer.alloc(12, 0x31), + profile: 'cluster-control', + preview: { + title: 'Read failed Run log', + summary: 'Reads one bounded redacted log excerpt', + fields: [ + { kind: 'identifier', label: 'Run', value: sourceRunId }, + { kind: 'identifier', label: 'Attempt', value: sourceAttemptId }, + ], + warnings: ['potentially_sensitive_output'], + }, + sealedAtMs: baseTimeMs + 200, + }).plan; + const plan = prepareCopilotFailureDiagnosisExecution({ + requestId, + traceId: `diagnosis-trace-${suffix}`, + source, + toolPlan, + bindings, + model: { + provider: 'provider-primary', + model: 'model-diagnosis', + modelBoundary: 'external', + responseLanguage: 'zh-CN', + maxOutputTokens: 512, + egressPolicy: { + schema: 'qinglong/copilot-model-egress-policy@v1', + revision: 'ha-diagnosis-policy-v1', + potentiallySensitiveDataBoundaries: ['external'], + maxInputBytes: 64 * 1024, + maxOutputTokens: 1_024, + }, + }, + deadlineAtMs: baseTimeMs + 60_000, + plannedAtMs: baseTimeMs + 300, + }); + const runtimeDatabase = await databaseOpener( + 'runtime', + databaseUrl(RUNTIME_USER, RUNTIME_PASSWORD, primaryPort), + 'ql3-ha-copilot-diagnosis-admission', + )(); + try { + const repository = new PostgresCopilotFailureDiagnosisAdmissionRepository( + runtimeDatabase.pool, + ); + const first = await repository.admit(plan); + const replay = await repository.admit(plan); + assert.equal(first.status, 'created'); + assert.equal(replay.status, 'existing'); + assert.deepEqual(replay.receipt, first.receipt); + assert.deepEqual( + await repository.findByRequestId(requestId), + first.receipt, + ); + assert.deepEqual(await repository.findPlanByRequestId(requestId), plan); + const facts = await copilotFailureDiagnosisAdmissionFacts( + runtimeDatabase.pool, + requestId, + ); + assert.deepEqual( + { + parentRunId: facts.parentRunId, + status: facts.status, + version: facts.version, + eventSequence: facts.eventSequence, + eventCount: facts.eventCount, + mutationCount: facts.mutationCount, + toolStatus: facts.toolStatus, + toolKind: facts.toolKind, + modelStatus: facts.modelStatus, + modelKind: facts.modelKind, + modelParentStepRunId: facts.modelParentStepRunId, + }, + { + parentRunId: sourceRunId, + status: 'running', + version: 3, + eventSequence: 3, + eventCount: 3, + mutationCount: 2, + toolStatus: 'ready', + toolKind: 'tool', + modelStatus: 'pending', + modelKind: 'model', + modelParentStepRunId: plan.toolStepRunId, + }, + ); + return { + requestId, + sourceRunId, + sourceAttemptId, + runId: plan.runId, + toolStepRunId: plan.toolStepRunId, + modelStepRunId: plan.modelStepRunId, + exactReplay: true, + beforePromotion: facts, + }; + } finally { + await runtimeDatabase.close(); + } +} + async function runModelProviderCredentialCatalogMatrix(options) { const { primaryPort, migrationPool } = options; const suffix = `${process.pid}-${randomBytes(3).toString('hex')}`; @@ -9967,6 +10299,7 @@ async function main(argv = process.argv.slice(2)) { let networkPartition; let oldPrimaryRejoin; let modelInvocationFeaturePromotion; + let copilotFailureDiagnosisAdmission; let modelProviderCredentialCatalog; let modelProviderCredentialTestConnection; let runAttemptLogRetentionEvidence; @@ -10064,6 +10397,7 @@ async function main(argv = process.argv.slice(2)) { migrationDatabase.pool, ); assert.deepEqual(beforePromotion.tables, [ + 'copilot_failure_diagnosis_admissions', 'model_invocation_completions', 'model_invocation_price_quotes', 'model_invocation_price_settlements', @@ -10113,6 +10447,7 @@ async function main(argv = process.argv.slice(2)) { POSTGRES_MODEL_PROVIDER_CREDENTIAL_TEST_CONNECTION_MIGRATION_ID, POSTGRES_PLUGIN_PACKAGE_PROMPT_OUTPUT_KEY_ROTATION_MIGRATION_ID, POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, ], ); assert.deepEqual(beforePromotion.privileges, { @@ -10136,10 +10471,17 @@ async function main(argv = process.argv.slice(2)) { promptOutputKeyRetirementAuthoritySplit: true, promptOutputKeyRotationAuthoritySplit: true, promptSnapshotRuntimeOnly: true, + copilotDiagnosisAdmissionRuntimeOnly: true, + copilotDiagnosisSnapshotRuntimeOnly: true, migrationHistoryRuntimeReadOnly: true, modelProviderCredentialManagementAuthoritySplit: true, }); modelInvocationFeaturePromotion = { beforePromotion }; + copilotFailureDiagnosisAdmission = + await runCopilotFailureDiagnosisAdmissionHaEvidence({ + primaryPort, + migrationPool: migrationDatabase.pool, + }); } finally { await migrationDatabase.close(); } @@ -10420,6 +10762,21 @@ async function main(argv = process.argv.slice(2)) { state: 'synchronous_remote_apply_ready', atMs: Number((performance.now() - startedAt).toFixed(3)), }); + await waitFor(async () => { + const replicated = await copilotFailureDiagnosisAdmissionFacts( + standbyDatabase.pool, + copilotFailureDiagnosisAdmission.requestId, + ); + return JSON.stringify(replicated) === + JSON.stringify(copilotFailureDiagnosisAdmission.beforePromotion) + ? replicated + : null; + }, 'Copilot failure diagnosis admission WAL replay'); + copilotFailureDiagnosisAdmission.replicatedBeforePromotion = true; + timeline.push({ + state: 'copilot_failure_diagnosis_admission_replicated', + atMs: Number((performance.now() - startedAt).toFixed(3)), + }); manualRunRetry = await runManualRunRetryHaEvidence({ primaryPort, primaryDatabase, @@ -11013,6 +11370,16 @@ async function main(argv = process.argv.slice(2)) { String(schedulerDueAtMs), ); + const recoveryBeforeControlActivation = + await new PostgresClusterControlRecoverySource( + primaryDatabase.pool, + ).listOutstanding(8); + assert.deepEqual( + recoveryBeforeControlActivation.candidates, + [], + 'HA fixture must not leave generic Task recovery candidates before control activation', + ); + proxy = await startEndpointProxy(primaryPort); const stableDatabaseUrl = databaseUrl( RUNTIME_USER, @@ -11380,6 +11747,18 @@ async function main(argv = process.argv.slice(2)) { modelInvocationFeaturePromotion.afterPromotion = promotedModelInvocationFeature; modelInvocationFeaturePromotion.survivedPromotion = true; + const promotedCopilotFailureDiagnosisAdmission = + await copilotFailureDiagnosisAdmissionFacts( + promotedDatabase.pool, + copilotFailureDiagnosisAdmission.requestId, + ); + assert.deepEqual( + promotedCopilotFailureDiagnosisAdmission, + copilotFailureDiagnosisAdmission.beforePromotion, + ); + copilotFailureDiagnosisAdmission.afterPromotion = + promotedCopilotFailureDiagnosisAdmission; + copilotFailureDiagnosisAdmission.survivedPromotion = true; await verifyModelProviderCredentialCatalogAfterPromotion({ promotedPort: standbyPort, promotedPool: promotedDatabase.pool, @@ -12134,7 +12513,7 @@ async function main(argv = process.argv.slice(2)) { FROM "ql3"."worker_credential_deliveries") AS "credentialDeliveries"`, ); assert.deepEqual(sideEffects.rows, [ - { runs: 76, runEvents: 176, credentialDeliveries: 4 }, + { runs: 78, runEvents: 179, credentialDeliveries: 4 }, ]); timeline.push({ state: 'two_fresh_control_replicas_ready', @@ -12248,6 +12627,7 @@ async function main(argv = process.argv.slice(2)) { workerCredentialDeliveryCommitResponseLoss: credentialDelivery.report, runDomainCommitResponseLoss: domainCommitResponseLoss.report, modelInvocationFeaturePromotion, + copilotFailureDiagnosisAdmission, modelProviderCredentialCatalog, modelProviderCredentialTestConnection, runAttemptLogRetention, @@ -12262,6 +12642,18 @@ async function main(argv = process.argv.slice(2)) { runAttemptLogRetention.survivedPromotion, packageAuthoritySplitReadinessBeforeAndAfterPromotion: true, optionalAiFeatureSchemaSurvivesPromotion: true, + copilotFailureDiagnosisAdmissionExactlyReplays: + copilotFailureDiagnosisAdmission.exactReplay && + copilotFailureDiagnosisAdmission.beforePromotion.eventCount === 3 && + copilotFailureDiagnosisAdmission.beforePromotion.mutationCount === 2, + copilotFailureDiagnosisAdmissionReplicatesAndSurvivesPromotion: + copilotFailureDiagnosisAdmission.replicatedBeforePromotion && + copilotFailureDiagnosisAdmission.survivedPromotion, + copilotFailureDiagnosisAdmissionUsesLeastPrivilegeRuntime: + modelInvocationFeaturePromotion.beforePromotion.privileges + .copilotDiagnosisAdmissionRuntimeOnly && + modelInvocationFeaturePromotion.beforePromotion.privileges + .copilotDiagnosisSnapshotRuntimeOnly, modelProviderCredentialCatalogSurvivesPromotion: modelProviderCredentialCatalog.survivedPromotion, modelProviderCredentialManagementIdentityLedgerSurvivesPromotion: diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index cee9f270..451ae378 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -268,10 +268,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( { path: 'packages/ql3-ai', name: '@qinglong/ai', - sourceFiles: 160, + sourceFiles: 168, rootSourceFiles: 1, rootSourceLines: 16, - nestedSourceFiles: 159, + nestedSourceFiles: 167, rootSourceFileHardCap: 1, rootSourceLineHardCap: 16, rootSourceFileRoles: {