From 453a7679b271ae327f1537d1a44170a2abd2bb2b Mon Sep 17 00:00:00 2001 From: whyour Date: Sat, 15 Aug 2026 20:38:17 +0800 Subject: [PATCH] feat(ql3): execute copilot diagnosis models --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 1 + ...pilot-failure-diagnosis-model-execution.md | 96 +++ docs/adr/README.md | 1 + packages/ql3-ai/package.json | 10 + .../model-execution/completion.ts | 283 +++++++++ .../model-execution/coordinator.ts | 280 +++++++++ .../model-execution/finalization.ts | 226 +++++++ .../model-execution/outputArtifact.ts | 491 +++++++++++++++ .../model-execution/postgresRepository.ts | 577 ++++++++++++++++++ .../failure-diagnosis/modelExecution.ts | 4 + .../postgresModelExecutionRepository.ts | 1 + .../migration/model-invocation/identities.ts | 2 + .../migration/model-invocation/postgres.ts | 2 + .../model-invocation/postgres/copilot.ts | 162 +++++ .../durableModelInvocationCoordinator.ts | 128 ++-- .../modelInvocationAtomicSuccess.ts | 55 ++ .../completionOperations.ts | 79 ++- .../pluginPackagePromptOutputCompletion.ts | 63 +- ...ailureDiagnosisExecutionAdmission.test.cjs | 377 ++++++++++++ .../test/modelInvocationMigration.test.cjs | 34 +- ...redentialManagementIdentityLedger.test.cjs | 2 +- ...elProviderCredentialTestExecution.test.cjs | 2 +- .../tool-execution/trustedToolCompletion.ts | 6 + .../trustedToolSuccessCompletion.ts | 84 ++- scripts/ql3-postgres-ha-contract.cjs | 34 ++ test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 26 files changed, 2872 insertions(+), 132 deletions(-) create mode 100644 docs/adr/ADR-0409-cluster-copilot-failure-diagnosis-model-execution.md create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/completion.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/coordinator.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/finalization.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/outputArtifact.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/postgresRepository.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/modelExecution.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/postgresModelExecutionRepository.ts create mode 100644 packages/ql3-ai/src/model-invocation/modelInvocationAtomicSuccess.ts diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index d45a54e6..d4088d15 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -12,6 +12,7 @@ 最新增量证据（2026-08-15）： - D-316/ADR-0408（已接受）：Cluster Copilot 现在能够从 ADR-0407 的 durable admission 恢复 exact `qinglong.run.log.excerpt@1.0.0` authority，复用通用 Trusted Tool start barrier、加密 success/failure completion、result catalog/rekey 和内建日志 adapter；确定性 start/completion identity 让 response-loss replay 直接打开既有证据，不重复读取日志或执行 adapter。Cluster invocation Artifact 使用独立 projected keyring，提供 active+historical material，但每次读取均重新执行 canonical path/symlink/mode/inode/realpath fence，且不取得 PostgreSQL Tool result generation authority。只有 exact `succeeded` completion 才能通过 `pg-9019` 的 SERIALIZABLE 事务把 Model Step 从 `pending` 原子推进到 `ready`，同时写 RunEvent、StepRunMutation 和 append-only unlock receipt；`failed|timed_out` 不解锁。本 Gate 不执行模型、不终态化 diagnosis Run，下一 Gate 是 ADR-0405 builder + Model Gateway + Copilot encrypted model completion/terminalization。实现仍为 18 个 package，无单文件/浅平 package；AI 175 个源码中 174 个、Cluster Control 56 个中 54 个位于嵌套目录，不新增依赖、进程、连接、timer/watcher/cache 或默认 Edge 成本。18-package clean build/test 全绿，AI 229 pass/3 条件 skip、backend 1,207 pass/2 条件 skip，四项架构审计零 finding；14 档 Local artifact 全通过，默认 Edge/Standalone 为 2,589,890/2,589,968 bytes。PostgreSQL 18.4 arm64 HA 130/130、timeline `1→2`，首次执行只读两次日志、密文不含敏感 fixture，提升后 exact replay 零日志读取；报告 SHA-256 为 `d525a303696e178d777b021b376729bd2c5382fb5eb7bc98466a2b79d3940517`，独立审计与 Docker 清理通过。 +- D-317/ADR-0409（已接受）：已解锁的 Cluster failure-diagnosis Model Step 现在从 durable encrypted Tool completion 重开受信日志投影，经 ADR-0405 builder 和既有 Model Gateway 执行；只有显式安装的 Copilot success sink 可以接管成功返回。Copilot `GenerateResult` 使用独立 AES-256-GCM Artifact，绑定 plan、Tool completion、egress evidence 与 Model identity，公开 reference、ModelInvocation completion、RunEvent 和审计均不含明文。通用 `DurableModelInvocationCoordinator` 已改为领域无关的 `ModelInvocationAtomicSuccess`，消除对 Plugin Prompt Artifact 的反向依赖；Plugin Prompt 通过 adapter 保持兼容。`pg-9020` 在一个 SERIALIZABLE 事务中原子提交 ciphertext、Model completion、StepRun/Event 与 usage/pricing/quota settlement，再由可重放 finalization 事务把 diagnosis Run 推进为 `succeeded|failed|timed_out`；两事务间崩溃只补 finalization，existing start/completion/finalization replay 均不重复调用 Provider。当前仍不自动终态化 Tool failure、日志 missing/retired、Model admission 前 deadline/cancel，也不把 `outcome_unknown` 冒充失败；Cluster 专用 output projected keyring 与产品 composition 完成前该入口保持不可达。实现仍在既有 18 个 package 的嵌套领域目录内，不新增进程、队列、timer/watcher/cache 或默认 Edge 成本。最终 AI 233 pass/3 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计零 finding；workspace 无单文件或浅平 package，AI 183 个源码中 182 个位于嵌套目录。14 档 Local Profile artifact 全部通过，默认 Edge/Standalone 为 2,589,890/2,589,968 bytes，Edge/Standalone AI 为 3,061,009/3,061,099 bytes。PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`，报告 SHA-256 为 `8401634f30635b45bfb583b02e94ac41f023bf8a0bdbcfd9744ebf459ab0d8f8`，独立证据审计与 Docker 零残留。 - D-315/ADR-0407（已接受）：Cluster Copilot 故障诊断不修改或重新打开终态源 Run，而是用源 Run/version、最新已结束 Attempt/status 与日志 Artifact 建立 exact fence，原子创建独立 `copilot_failure_diagnosis` Run、admission event、`ready` 的 `qinglong.run.log.excerpt@1.0.0` Tool Step、以其为父节点的 `pending` Model Step 和不可变 receipt。计划只接受 `cluster-control` 的 reviewed snapshot/binding/Policy/subject authority；`approval_required`、Tool contract/输入漂移或未显式允许的模型出口均在写库前失败关闭。`ql3_ai` 新增有界 admission ledger 与仅授予 runtime 的 `SECURITY DEFINER` source snapshot，在 SERIALIZABLE 事务内重验 active Project/binding 和源 Run/Attempt；response-loss replay 对 JSONB 做结构 exact 比较而不依赖无语义的对象键序。Copilot/Plugin 这类 StepRun 聚合从通用 Task orphan recovery 排除，后续由各自状态机恢复，不能伪造顶层 Attempt 或被错误终态化。本阶段只 admission，不执行 Tool/模型、不授予行动权；Tool encrypted completion、Model 解锁和 Copilot encrypted model completion 是下一 Gate。最终 18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项边界审计零 finding且 workspace 仍为 18 package、无单文件/浅平 package；14 档 Local Profile artifact 全通过，默认 Edge/Standalone 不引入 Cluster admission。PostgreSQL 18.4 arm64 HA 128/128、timeline `1→2`，报告 SHA-256 为 `a4ed1edec783e3f5b42507c0f8e11b94c59dbe44a57e691017d1445ec9d115e2`，证据审计与 Docker 清理通过。 - D-314/ADR-0406（已接受）：Cluster Trusted Tool 的 encrypted completion 不再停留在 storage port。`@qinglong/cluster-control/trusted-tool-result-keyring` 新增只读 projected material authority：canonical v1 manifest 只含最多 16 个 canonical 32-byte key，不含 generation、active/state/retirement，provider 只有 `resolve(keyId)` 而没有 `active()`；因此 PostgreSQL `trusted-tool-results` catalog 仍是 active/decryptable 状态唯一 authority，completion 会以 catalog material proof 再次校验。runtime 每次调用重新执行 direct-root、in-root atomic symlink、single-link、64 KiB、只读/不可执行/other-inaccessible mode、dev/inode/size/mtime 与双 realpath fence，不持有 Kubernetes API、cache、watcher 或 timer。新增能力位于 Cluster Control 既有 `trusted-tool/key-management/`，并把 mounted Secret 与 keyring 的 projected-file/TOCTOU 逻辑收敛到 package-private `security/privateProjectedFile` 真源；公开 mounted Secret 行为不变，不新增 package、依赖、migration、连接、route 或默认 Profile importer。定向共享回归 7/7，Cluster Control 完整 234 pass/2 条条件 skip/0 fail；最终 18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，package/dependency/Edge import/Cluster deployment 四项审计零 finding。workspace 仍为 18 package、无单文件或浅平 package；Cluster Control 54 个源码中仅 2 个 binary entry 位于根层，52 个处于嵌套领域目录。14 档 Local Profile artifact 全部通过，默认 Edge/Standalone 保持 2,589,812/2,589,890 bytes，AI 保持 3,121,108/3,121,198 bytes，MCP 保持 7,315,930/7,316,038 bytes。PostgreSQL 18.4 arm64 HA 125/125、timeline `1→2`，报告 SHA-256 为 `26c817647ed984d8d4627a7cae1c95de06017a5d6d32dd3dfd01414ba029e542`，证据审计与 Docker 容器/网络/卷零残留。下一 Gate 是独立 diagnosis Run 的 Tool/Model Step admission 与 Copilot encrypted model completion，不能借用终态源 Run 或 Plugin Prompt plan。 - D-313/ADR-0405（已接受）：新增 `@qinglong/ai/failure-diagnosis-prompt`，把 ADR-0403 的潜在敏感日志投影收敛为固定 system instruction + canonical JSON data envelope；日志只存在于 `log.content` JSON string value，不能通过引号、换行、伪造 role/schema 或 delimiter 拼接出新 message。builder 重新校验完整 trust/redaction/profile byte 契约，拒绝伪造 `safe`、行动权、未知字段与 byte/signal drift；envelope 不带 Run/Attempt、Artifact、path、cursor 或 content digest。部署者必须通过 `qinglong/copilot-model-egress-policy@v1` 显式允许 `potentially_sensitive` 数据进入 `on_device|external` 边界并提供输入/output token 双预算，空 allowlist 与 external 未授权均在 Model Gateway/Provider I/O 前失败关闭。输出只含 content-free egress evidence，并固定要求模型 completion 继承潜在敏感、仅加密持久化、禁止明文审计且无行动权；真正 Cluster Trusted Tool/model completion 仍需后续组合门。能力以 `ql3-ai/src/copilot/failure-diagnosis/` 三个内聚文件和精确 subpath 交付，不新增 package、依赖、迁移、连接或常驻组件。定向 12/12、AI 221 pass/3 条件 skip/0 fail；最终 18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项结构/部署审计零 finding，14 档 Local Profile artifact 全部通过。默认 Edge/Standalone 保持 2,589,812/2,589,890 bytes、315 files、56 modules，Edge/Standalone AI 保持 3,121,108/3,121,198 bytes、368 files、61 modules，MCP 保持 7,315,930/7,316,038 bytes、801 files、226 modules，证明未装配 subpath 被完全裁掉。PostgreSQL 18.4 arm64 HA 125/125、timeline `1→2`，报告 SHA-256 为 `2bbc8bdd0d90e6ec9ce82d2afcaec817679dddb82860c5d405a09d5e5458bece`，证据审计与 Docker 零残留。 diff --git a/docs/adr/ADR-0409-cluster-copilot-failure-diagnosis-model-execution.md b/docs/adr/ADR-0409-cluster-copilot-failure-diagnosis-model-execution.md new file mode 100644 index 00000000..a03ab545 --- /dev/null +++ b/docs/adr/ADR-0409-cluster-copilot-failure-diagnosis-model-execution.md @@ -0,0 +1,96 @@ +# ADR-0409：Cluster Copilot Failure Diagnosis Model Execution + +- 状态：Accepted +- 日期：2026-08-15 +- 关联 RFC：QL-RFC-0001 D-317、Phase 2 +- 关联 ADR：ADR-0262、ADR-0405、ADR-0407、ADR-0408 + +## 问题 + +ADR-0408 只把成功的日志 Tool completion 原子解锁为 `ready` Model Step。若调用方直接把 +Tool 返回值留在进程内再调用 Provider，进程重启后无法恢复 prompt；若 Model completion 先于 +密文输出写入，COMMIT response loss 会留下“Step 已成功但结果不存在”的分裂状态；若把 Copilot +Artifact 继续塞进通用 Model coordinator，则每增加一个 AI 领域都会让核心层反向依赖一个新领域。 + +同时，diagnosis Run 不能在 Model Step 已终态后永久停留在 `running`。成功、Provider 失败和 +deadline 必须由耐久 ModelInvocation completion 驱动 Run 终态，而不能由一次内存返回值决定。 + +## 决策 + +1. Model executor 只接受 ADR-0407/0408 的 exact admission、unlock receipt 和加密 Tool success。 + 它通过 Runtime Core 的只读 `openTrustedToolSuccessCompletion` 重开 ciphertext，复验 barrier、 + result catalog/rekey、key material proof、Tool definition 与 completion digest；绝不重新执行 Tool。 +2. 只有 `available`、`cluster-control` 且 Run/Attempt identity 与 source fence 完全一致的内建日志 + projection 才能进入 ADR-0405 builder。其余 Tool 输出不会被转成任意 Prompt，也不会调用 Provider。 +3. executor 使用既有 `BoundedModelGateway`，因此 Provider/model allowlist、egress budget、并发、 + deadline、quota、price quote、usage settlement 和 content-free audit 继续由一个通用边界裁决。 + Gateway 必须显式安装当前 Copilot successful-completion sink，否则执行失败关闭。 +4. Copilot 输出使用独立 `qinglong/copilot-failure-diagnosis-output-artifact@v1`，AES-256-GCM 密封 + 完整 `GenerateResult`。AAD 与 digest 绑定 request/plan/Tool completion/Project/Run/Step/ + invocation/provider/model/egress evidence/key；公开 reference 不含 nonce、ciphertext、auth tag 或明文。 +5. 通用 `DurableModelInvocationCoordinator` 不再导入 Plugin Package Prompt Artifact。新增领域无关的 + `ModelInvocationAtomicSuccess` 端口，由领域负责 exact replay、冲突类型和方言事务； + 原有 Plugin Prompt 通过 adapter 保持行为兼容，Copilot 使用独立 adapter。PostgreSQL 的通用 + atomic-output helper 在同一 SERIALIZABLE 事务提交 StepRun/Event、Model completion、usage、 + pricing/quota settlement 和领域 ciphertext，任何一步失败均整体回滚。 +6. `pg-9020-ai-copilot-failure-diagnosis-model-executions` 新增两个 append-only ledger： + `copilot_failure_diagnosis_model_outputs` 保存密文 Artifact； + `copilot_failure_diagnosis_finalizations` 保存 content-free Run terminal receipt。`ql3_runtime` 只有 + `SELECT, INSERT`，其他运行/管理角色不得读写，PUBLIC 无权限。 +7. Model completion 提交后，独立可重放的 finalization 事务锁定 diagnosis Run 与 Model Step，复验 + plan、completion、成功 Artifact 和当前计数链，再把 Run 推进为 `succeeded|failed|timed_out`、 + 写一个 `copilot.diagnosis.` Event 和 receipt。两个事务之间崩溃时，恢复只执行 + finalization，不再调用 Provider；Run 的成功 `outputRef` 只指向密文 Artifact。 +8. 已存在 completion 或 finalization 的 replay 永不调用 Provider；只有 start、没有 completion 的 + invocation 视为结果未知，禁止自动重试。`outcome_unknown` 仍必须先走既有显式 resolution，当前 + finalizer 不把未知结果伪装成失败。Tool failure、Tool 输出退役/缺失、Model admission 前 deadline + 和用户取消的无 Model-completion 终态化，留给下一状态机 Gate。 +9. 本能力继续内聚于 `@qinglong/ai` 的嵌套领域目录，不新增 workspace package、Node 进程、Pod、 + 连接池、队列、timer、watcher 或 cache。Edge/Standalone 默认制品不导入 PostgreSQL executor； + Cluster 的 Copilot output key material 必须由显式 `active()/resolve()` provider 注入，不能复用 + Provider credential、Tool invocation/result 或 Plugin Prompt output 的密钥用途。 + +## 被否决方案 + +1. **Model 成功后再单独写 Artifact**：留下成功 Step 与缺失输出的不可恢复窗口。 +2. **把 Tool 明文保存在 executor registry**：崩溃后无法恢复，也扩大路由设备常驻内存与泄漏面。 +3. **把第二个领域 Artifact 导入通用 coordinator**：继续形成 Model core → product domain 的反向依赖。 +4. **自动重试只有 start 的 Model invocation**：外部 Provider 可能已完成，重试会产生重复费用和结论。 +5. **将 `outcome_unknown` 直接标为 failed**：丢失真实不确定性，破坏人工 resolution 语义。 +6. **为 Copilot Model execution 新建 package**：没有新的独立部署边界，只会再次制造单文件薄包。 + +## 验证标准 + +1. 单元/契约测试覆盖密文 round-trip、metadata/ciphertext/key tamper、content-free reference、 + finalization receipt、显式 subpath、一次 Provider 调用、completion/finalization replay 零 Provider 调用。 +2. 既有 Plugin Prompt exact replay、Artifact 与 settlement crash window 全部通过，证明通用原子端口 + 重构没有改变旧领域语义。 +3. PostgreSQL 18.4 physical HA 必须应用 20 条 AI migration，证明两张新表 append-only 角色隔离、 + timeline `1→2` 提升后 history/schema/ACL 一致，并由独立证据审计复核。 +4. 18-package clean build/test、backend、四项架构审计、14 档 Artifact 和 GitNexus staged/change + detection 全部通过后才允许阶段性提交；不得把 focused test 当完整性证明。 + +## 当前验证 + +1. `@qinglong/ai` 全量测试 233 pass、3 条 PostgreSQL 条件测试 skip、0 fail；新增覆盖密文 + round-trip、tamper、content-free reference、一次 Provider 调用及 completion/finalization replay。 +2. 18 个 QL3 package 从清空 `dist` 开始全部构建并通过各自全量测试；backend 1,207 pass、 + 2 条条件测试 skip、0 fail。 +3. package boundary、cluster dependency、Edge import、Cluster deployment 四项审计均为 + `compatible: true` 且零 finding。workspace 仍为 18 个 package，`singleSourcePackages=[]`、 + `shallowSourcePackages=[]`；AI 的 183 个源码中 182 个位于嵌套领域目录。 +4. 14 档 Local Profile Artifact 全部通过；默认 Edge/Standalone 分别为 + 2,589,890/2,589,968 bytes，Edge/Standalone AI 为 3,061,009/3,061,099 bytes，均在预算内。 +5. PostgreSQL 18.6 arm64 physical HA 130/130、timeline `1→2`，20 条 AI migration、schema、 + append-only ACL 与提升后一致性全部通过；报告 SHA-256 为 + `8401634f30635b45bfb583b02e94ac41f023bf8a0bdbcfd9744ebf459ab0d8f8`，独立证据审计零 finding， + Docker 容器、网络和卷零残留。 + +## 后续门禁 + +1. 为 Copilot output 建立 Cluster 专用 projected keyring manifest、active rotation、retirement、 + external custody 与 lost-key recovery；在产品 composition 完成前 Model executor 保持不可达。 +2. 终态化 Tool failure、日志 missing/retired、Model admission 前 deadline、用户 cancellation 与 + `outcome_unknown` resolution，并加入跨主库提升的 crash-window execution matrix。 +3. 最后开放默认关闭的 Cluster API/CLI/UI/MCP 入口，补真实外部 Provider、KMS/Secret projection、 + plaintext negative evidence、多副本并发和费用/取消可观测性。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 298d8d0a..77e9cefa 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -412,6 +412,7 @@ | [ADR-0406](./ADR-0406-cluster-projected-tool-result-key-authority.md) | Cluster Projected Tool Result Key Authority | Accepted | | [ADR-0407](./ADR-0407-cluster-copilot-failure-diagnosis-run-admission.md) | Cluster Copilot Failure Diagnosis Run Admission | Accepted | | [ADR-0408](./ADR-0408-cluster-copilot-failure-diagnosis-tool-execution.md) | Cluster Copilot Failure Diagnosis Tool Execution 与原子 Model 解锁 | Accepted | +| [ADR-0409](./ADR-0409-cluster-copilot-failure-diagnosis-model-execution.md) | Cluster Copilot Failure Diagnosis Model Execution、密文输出与 Run 终态化 | Accepted | ## 规则 diff --git a/packages/ql3-ai/package.json b/packages/ql3-ai/package.json index f82e1a95..5c08cc7d 100644 --- a/packages/ql3-ai/package.json +++ b/packages/ql3-ai/package.json @@ -50,6 +50,16 @@ "require": "./dist/copilot/failure-diagnosis/postgresToolExecutionRepository.js", "default": "./dist/copilot/failure-diagnosis/postgresToolExecutionRepository.js" }, + "./failure-diagnosis-model-execution": { + "types": "./dist/copilot/failure-diagnosis/modelExecution.d.ts", + "require": "./dist/copilot/failure-diagnosis/modelExecution.js", + "default": "./dist/copilot/failure-diagnosis/modelExecution.js" + }, + "./postgres-failure-diagnosis-model-execution-storage": { + "types": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.d.ts", + "require": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.js", + "default": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.js" + }, "./model-invocation": { "types": "./dist/model-invocation/modelInvocation.d.ts", "require": "./dist/model-invocation/modelInvocation.js", diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/completion.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/completion.ts new file mode 100644 index 00000000..0a4a0433 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/completion.ts @@ -0,0 +1,283 @@ +import type { + ModelInvocationCompletionCommand, + ModelInvocationRepository, +} from '../../../model-invocation/modelInvocation'; +import type { DurableModelInvocationCoordinator } from '../../../model-invocation/durableModelInvocationCoordinator'; +import type { ModelInvocationAtomicSuccess } from '../../../model-invocation/modelInvocationAtomicSuccess'; +import type { + GenerateResult, + ModelInvocationAuditRecord, +} from '../../../model-gateway/model'; +import type { ModelInvocationSuccessfulCompletionSink } from '../../../model-gateway/gateway'; +import type { CopilotFailureDiagnosisExecutionPlan } from '../admission/contracts'; +import { normalizeCopilotFailureDiagnosisExecutionPlan } from '../admission/plan'; +import type { FailureDiagnosisPromptPlan } from '../contracts'; +import { + CopilotFailureDiagnosisOutputArtifactConflictError, + CopilotFailureDiagnosisOutputArtifactUnavailableError, + copilotFailureDiagnosisOutputReference, + createCopilotFailureDiagnosisOutputArtifact, + normalizeCopilotFailureDiagnosisOutputArtifact, + type CopilotFailureDiagnosisOutputArtifact, + type CopilotFailureDiagnosisOutputKeyProvider, + type CopilotFailureDiagnosisOutputReference, +} from './outputArtifact'; + +export const MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_MODEL_COMPLETIONS = 64; + +export interface CommitCopilotFailureDiagnosisOutputResult { + readonly status: 'created' | 'existing'; + readonly reference: Readonly; +} + +export interface CopilotFailureDiagnosisOutputCompletionRepository { + findCopilotFailureDiagnosisOutput( + artifactId: string, + ): Promise | null>; + completeWithCopilotFailureDiagnosisOutput( + command: Readonly, + artifact: Readonly, + ): Promise>; +} + +export function isCopilotFailureDiagnosisOutputCompletionRepository( + value: ModelInvocationRepository, +): value is ModelInvocationRepository & + CopilotFailureDiagnosisOutputCompletionRepository { + return ( + typeof (value as Partial) + .findCopilotFailureDiagnosisOutput === 'function' && + typeof (value as Partial) + .completeWithCopilotFailureDiagnosisOutput === 'function' + ); +} + +export function assertCopilotFailureDiagnosisOutputCompletionBinding( + command: Readonly, + artifactValue: CopilotFailureDiagnosisOutputArtifact, +): Readonly<{ + artifact: Readonly; + reference: Readonly; +}> { + const artifact = normalizeCopilotFailureDiagnosisOutputArtifact(artifactValue); + if ( + command.completion.outcome !== 'succeeded' || + command.completion.errorCode !== null || + command.completion.invocationId !== artifact.invocationId || + command.completion.projectId !== artifact.projectId || + command.completion.runId !== artifact.runId || + command.completion.stepRunId !== artifact.stepRunId || + command.completion.outputBytes !== artifact.outputBytes || + command.start.provider !== artifact.provider || + command.start.model !== artifact.model || + command.stepRunMutation.stepRun.outputRef !== artifact.artifactId + ) { + throw new CopilotFailureDiagnosisOutputArtifactConflictError(); + } + return Object.freeze({ + artifact, + reference: copilotFailureDiagnosisOutputReference(artifact), + }); +} + +function atomicSuccess( + artifactValue: CopilotFailureDiagnosisOutputArtifact, +): ModelInvocationAtomicSuccess { + const artifact = normalizeCopilotFailureDiagnosisOutputArtifact(artifactValue); + const reference = copilotFailureDiagnosisOutputReference(artifact); + const conflict = (): Error => + new CopilotFailureDiagnosisOutputArtifactConflictError(); + const extension: ModelInvocationAtomicSuccess = { + outputRef: artifact.artifactId, + assertAudit(audit): void { + if ( + audit.phase !== 'completed' || + audit.requestId !== artifact.invocationId || + audit.projectId !== artifact.projectId || + audit.runId !== artifact.runId || + audit.stepRunId !== artifact.stepRunId || + audit.provider !== artifact.provider || + audit.model !== artifact.model || + audit.outputBytes !== artifact.outputBytes + ) { + throw conflict(); + } + }, + async find(repository) { + if (!isCopilotFailureDiagnosisOutputCompletionRepository(repository)) { + throw conflict(); + } + const stored = await repository.findCopilotFailureDiagnosisOutput( + artifact.artifactId, + ); + if (!stored) return null; + if (JSON.stringify(stored) !== JSON.stringify(artifact)) throw conflict(); + return copilotFailureDiagnosisOutputReference(stored); + }, + matches(stored): boolean { + return JSON.stringify(stored) === JSON.stringify(reference); + }, + async commit(repository, command) { + if (!isCopilotFailureDiagnosisOutputCompletionRepository(repository)) { + throw conflict(); + } + return repository.completeWithCopilotFailureDiagnosisOutput( + command, + artifact, + ); + }, + conflict, + }; + return Object.freeze(extension); +} + +export interface CopilotFailureDiagnosisModelCompletionLease { + readonly invocationId: string; +} + +interface ActiveCompletion { + readonly lease: Readonly; + readonly plan: Readonly; + readonly prompt: Readonly; + readonly toolCompletionDigest: string; + reference: Readonly | null; +} + +export class CopilotFailureDiagnosisModelCompletionCoordinator + implements ModelInvocationSuccessfulCompletionSink +{ + readonly #coordinator: DurableModelInvocationCoordinator; + readonly #keys: CopilotFailureDiagnosisOutputKeyProvider; + readonly #now: () => number; + readonly #nonceFactory: (() => Uint8Array) | undefined; + readonly #active = new Map(); + + constructor(options: Readonly<{ + coordinator: DurableModelInvocationCoordinator; + keys: CopilotFailureDiagnosisOutputKeyProvider; + now?: () => number; + nonceFactory?: () => Uint8Array; + }>) { + if ( + !options || + typeof options !== 'object' || + typeof options.coordinator?.recordWithAtomicSuccess !== 'function' || + typeof options.keys?.active !== 'function' || + typeof options.keys?.resolve !== 'function' || + (options.now !== undefined && typeof options.now !== 'function') || + (options.nonceFactory !== undefined && + typeof options.nonceFactory !== 'function') + ) { + throw new CopilotFailureDiagnosisOutputArtifactUnavailableError(); + } + this.#coordinator = options.coordinator; + this.#keys = options.keys; + this.#now = options.now ?? Date.now; + this.#nonceFactory = options.nonceFactory; + } + + begin(input: Readonly<{ + plan: CopilotFailureDiagnosisExecutionPlan; + prompt: FailureDiagnosisPromptPlan; + toolCompletionDigest: string; + }>): Readonly { + const plan = normalizeCopilotFailureDiagnosisExecutionPlan(input.plan); + if ( + input.prompt.request.provider !== plan.model.provider || + input.prompt.request.model !== plan.model.model || + input.prompt.request.maxOutputTokens !== plan.model.maxOutputTokens || + input.prompt.egressEvidence.modelBoundary !== plan.model.modelBoundary || + input.prompt.egressEvidence.policyRevision !== + plan.model.egressPolicy.revision || + input.prompt.egressEvidence.maxOutputTokens !== + plan.model.maxOutputTokens || + input.prompt.completionRequirements.persistence !== 'encrypted_only' || + input.prompt.completionRequirements.plaintextAudit !== 'forbidden' || + !/^[0-9a-f]{64}$/.test(input.toolCompletionDigest) || + this.#active.has(plan.modelInvocationId) || + this.#active.size >= + MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_MODEL_COMPLETIONS + ) { + throw new CopilotFailureDiagnosisOutputArtifactConflictError(); + } + const lease = Object.freeze({ invocationId: plan.modelInvocationId }); + this.#active.set(plan.modelInvocationId, { + lease, + plan, + prompt: input.prompt, + toolCompletionDigest: input.toolCompletionDigest, + reference: null, + }); + return lease; + } + + reference( + lease: Readonly, + ): Readonly | null { + const active = this.#active.get(lease.invocationId); + if (!active || active.lease !== lease) { + throw new CopilotFailureDiagnosisOutputArtifactUnavailableError(); + } + return active.reference; + } + + end(lease: Readonly): void { + const active = this.#active.get(lease.invocationId); + if (!active || active.lease !== lease) { + throw new CopilotFailureDiagnosisOutputArtifactUnavailableError(); + } + this.#active.delete(lease.invocationId); + } + + async record( + audit: Readonly, + result: Readonly, + ) { + const active = this.#active.get(audit.requestId); + if (!active) return Object.freeze({ handled: false as const }); + const { plan, prompt } = active; + if ( + audit.phase !== 'completed' || + audit.projectId !== plan.projectId || + audit.runId !== plan.runId || + audit.stepRunId !== plan.modelStepRunId || + audit.traceId !== plan.traceId || + audit.requestId !== plan.modelInvocationId || + result.provider !== plan.model.provider || + result.model !== plan.model.model + ) { + throw new CopilotFailureDiagnosisOutputArtifactConflictError(); + } + const material = await this.#keys.active(); + try { + const artifact = createCopilotFailureDiagnosisOutputArtifact( + { + requestId: plan.requestId, + planDigest: plan.planDigest, + toolCompletionDigest: active.toolCompletionDigest, + projectId: plan.projectId, + runId: plan.runId, + stepRunId: plan.modelStepRunId, + invocationId: plan.modelInvocationId, + result, + egressEvidence: prompt.egressEvidence, + keyId: material.keyId, + key: material.key, + sealedAtMs: this.#now(), + }, + this.#nonceFactory, + ); + const disposition = await this.#coordinator.recordWithAtomicSuccess( + audit, + atomicSuccess(artifact), + ); + active.reference = disposition.reference; + return Object.freeze({ + handled: true as const, + disposition: Object.freeze({ status: disposition.status }), + }); + } finally { + material.key.fill(0); + } + } +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/coordinator.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/coordinator.ts new file mode 100644 index 00000000..5756b010 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/coordinator.ts @@ -0,0 +1,280 @@ +import type { ToolJsonValue } from '@qinglong/runtime-core/tool-registry'; +import { + openTrustedToolSuccessCompletion, + type TrustedToolSuccessCompletionResult, + type TrustedToolSuccessCompletionReadDependencies, +} from '@qinglong/runtime-core/trusted-tool-completion'; + +import { BoundedModelGateway } from '../../../model-gateway/gateway'; +import type { ModelInvocationRepository } from '../../../model-invocation/modelInvocation'; +import type { CopilotFailureDiagnosisToolExecutionAdmissionReader } from '../tool-execution/contracts'; +import type { CopilotFailureDiagnosisToolUnlockRepository } from '../tool-execution/contracts'; +import { buildFailureDiagnosisPromptPlan } from '../prompt'; +import { normalizeFailureDiagnosisProjection } from '../validation'; +import type { + CopilotFailureDiagnosisModelCompletionCoordinator, + CopilotFailureDiagnosisOutputCompletionRepository, +} from './completion'; +import { + copilotFailureDiagnosisOutputReference, + type CopilotFailureDiagnosisOutputReference, +} from './outputArtifact'; +import type { + CopilotFailureDiagnosisFinalizationReceipt, + CopilotFailureDiagnosisFinalizationRepository, +} from './finalization'; + +export interface CopilotFailureDiagnosisModelExecutionDependencies { + readonly admissions: CopilotFailureDiagnosisToolExecutionAdmissionReader; + readonly unlocks: Pick; + readonly toolResults: CopilotFailureDiagnosisToolResultReader; + readonly modelInvocations: Pick< + ModelInvocationRepository, + 'findStart' | 'findCompletion' + >; + readonly outputs: Pick< + CopilotFailureDiagnosisOutputCompletionRepository, + 'findCopilotFailureDiagnosisOutput' + >; + readonly gateway: BoundedModelGateway; + readonly successfulCompletion: CopilotFailureDiagnosisModelCompletionCoordinator; + readonly finalizations: CopilotFailureDiagnosisFinalizationRepository; +} + +export interface CopilotFailureDiagnosisToolResultReader { + open(startId: string): Promise>; +} + +export function createCopilotFailureDiagnosisToolResultReader( + dependencies: TrustedToolSuccessCompletionReadDependencies, +): CopilotFailureDiagnosisToolResultReader { + return Object.freeze({ + open: (startId: string) => + openTrustedToolSuccessCompletion(startId, dependencies), + }); +} + +export interface CopilotFailureDiagnosisModelExecutionResult { + readonly outcome: 'succeeded' | 'failed' | 'timed_out' | 'cancelled'; + readonly output: Readonly | null; + readonly finalization: Readonly; +} + +export class InvalidCopilotFailureDiagnosisModelExecutionError extends TypeError { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_INVALID'; + constructor(message: string) { + super(`Copilot failure diagnosis Model execution is invalid: ${message}`); + this.name = 'InvalidCopilotFailureDiagnosisModelExecutionError'; + } +} + +export class CopilotFailureDiagnosisModelExecutionConflictError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_CONFLICT'; + constructor(message = 'durable Model execution facts changed') { + super(`Copilot failure diagnosis Model execution conflicts: ${message}`); + this.name = 'CopilotFailureDiagnosisModelExecutionConflictError'; + } +} + +export class CopilotFailureDiagnosisModelExecutionUnavailableError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_UNAVAILABLE'; + constructor(options?: ErrorOptions) { + super('Copilot failure diagnosis Model execution is unavailable', options); + this.name = 'CopilotFailureDiagnosisModelExecutionUnavailableError'; + } +} + +function assertDependencies( + value: CopilotFailureDiagnosisModelExecutionDependencies, +): void { + if ( + !value || + typeof value !== 'object' || + typeof value.admissions?.findPlanByRequestId !== 'function' || + typeof value.admissions?.findByRequestId !== 'function' || + typeof value.unlocks?.findByRequestId !== 'function' || + typeof value.toolResults?.open !== 'function' || + typeof value.modelInvocations?.findStart !== 'function' || + typeof value.modelInvocations?.findCompletion !== 'function' || + typeof value.outputs?.findCopilotFailureDiagnosisOutput !== 'function' || + !(value.gateway instanceof BoundedModelGateway) || + typeof value.successfulCompletion?.begin !== 'function' || + typeof value.successfulCompletion?.reference !== 'function' || + typeof value.successfulCompletion?.end !== 'function' || + typeof value.finalizations?.findFinalization !== 'function' || + typeof value.finalizations?.finalize !== 'function' + ) { + throw new InvalidCopilotFailureDiagnosisModelExecutionError( + 'dependencies are invalid', + ); + } +} + +function projection( + output: ToolJsonValue, + runId: string, + attemptId: string, +) { + if (!output || typeof output !== 'object' || Array.isArray(output)) { + throw new CopilotFailureDiagnosisModelExecutionConflictError( + 'Tool output is not an object', + ); + } + const record = output as Readonly>; + if ( + record.status !== 'available' || + record.runId !== runId || + record.attemptId !== attemptId || + record.profile !== 'cluster-control' + ) { + throw new CopilotFailureDiagnosisModelExecutionConflictError( + 'the admitted source log projection is unavailable', + ); + } + return normalizeFailureDiagnosisProjection( + { + content: record.content, + sourceBytes: record.sourceBytes, + modelTextBytes: record.modelTextBytes, + redaction: record.redaction, + normalization: record.normalization, + trust: record.trust, + }, + 'cluster-control', + ); +} + +async function finalize( + requestId: string, + dependencies: CopilotFailureDiagnosisModelExecutionDependencies, +): Promise> { + const result = await dependencies.finalizations.finalize(requestId); + let output: Readonly | null = null; + if (result.receipt.outputArtifactId !== null) { + const artifact = + await dependencies.outputs.findCopilotFailureDiagnosisOutput( + result.receipt.outputArtifactId, + ); + if (artifact) output = copilotFailureDiagnosisOutputReference(artifact); + if (!output) { + throw new CopilotFailureDiagnosisModelExecutionConflictError( + 'terminal output reference is unavailable', + ); + } + } + return Object.freeze({ + outcome: result.receipt.outcome, + output, + finalization: result.receipt, + }); +} + +export async function executeCopilotFailureDiagnosisModel( + requestId: string, + dependencies: CopilotFailureDiagnosisModelExecutionDependencies, +): Promise> { + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(requestId)) { + throw new InvalidCopilotFailureDiagnosisModelExecutionError( + 'request id is invalid', + ); + } + assertDependencies(dependencies); + const existingFinalization = await dependencies.finalizations.findFinalization( + requestId, + ); + if (existingFinalization) return finalize(requestId, dependencies); + + const [plan, admission, unlock] = await Promise.all([ + dependencies.admissions.findPlanByRequestId(requestId), + dependencies.admissions.findByRequestId(requestId), + dependencies.unlocks.findByRequestId(requestId), + ]); + if ( + !plan || + !admission || + !unlock || + admission.planDigest !== plan.planDigest || + unlock.planDigest !== plan.planDigest || + unlock.runId !== plan.runId || + unlock.modelStepRunId !== plan.modelStepRunId + ) { + throw new CopilotFailureDiagnosisModelExecutionConflictError( + 'admission or Tool unlock evidence is incomplete', + ); + } + const existingCompletion = await dependencies.modelInvocations.findCompletion( + plan.modelInvocationId, + ); + if (existingCompletion) return finalize(requestId, dependencies); + const existingStart = await dependencies.modelInvocations.findStart( + plan.modelInvocationId, + ); + if (existingStart) { + throw new CopilotFailureDiagnosisModelExecutionConflictError( + 'an incomplete Model invocation cannot be executed again automatically', + ); + } + + const tool = await dependencies.toolResults.open(unlock.startId); + if ( + tool.completion.completionDigest !== unlock.toolCompletionDigest || + tool.completion.runId !== plan.runId || + tool.completion.stepRunId !== plan.toolStepRunId + ) { + throw new CopilotFailureDiagnosisModelExecutionConflictError( + 'Tool completion evidence changed', + ); + } + const prompt = buildFailureDiagnosisPromptPlan({ + provider: plan.model.provider, + model: plan.model.model, + modelBoundary: plan.model.modelBoundary, + profile: 'cluster-control', + responseLanguage: plan.model.responseLanguage, + projection: projection( + tool.output, + plan.source.runId, + plan.source.attemptId, + ), + maxOutputTokens: plan.model.maxOutputTokens, + egressPolicy: plan.model.egressPolicy, + }); + if ( + !dependencies.gateway.supportsSuccessfulCompletionSink( + dependencies.successfulCompletion, + ) + ) { + throw new InvalidCopilotFailureDiagnosisModelExecutionError( + 'Gateway successful completion sink is not the Copilot sink', + ); + } + const lease = dependencies.successfulCompletion.begin({ + plan, + prompt, + toolCompletionDigest: unlock.toolCompletionDigest, + }); + try { + try { + await dependencies.gateway.generate(prompt.request, { + projectId: plan.projectId, + runId: plan.runId, + stepRunId: plan.modelStepRunId, + traceId: plan.traceId, + requestId: plan.modelInvocationId, + deadlineAtMs: plan.deadlineAtMs, + }); + } catch (cause) { + const completion = await dependencies.modelInvocations.findCompletion( + plan.modelInvocationId, + ); + if (!completion) { + throw new CopilotFailureDiagnosisModelExecutionUnavailableError({ + cause: cause instanceof Error ? cause : undefined, + }); + } + } + return finalize(requestId, dependencies); + } finally { + dependencies.successfulCompletion.end(lease); + } +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/finalization.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/finalization.ts new file mode 100644 index 00000000..a2841457 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/finalization.ts @@ -0,0 +1,226 @@ +import { createHash } from 'node:crypto'; + +export const COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_RECEIPT_SCHEMA = + 'qinglong/copilot-failure-diagnosis-finalization-receipt@v1' as const; +export const COPILOT_FAILURE_DIAGNOSIS_FINAL_OUTCOMES = [ + 'succeeded', + 'failed', + 'timed_out', + 'cancelled', +] as const; + +export type CopilotFailureDiagnosisFinalOutcome = + (typeof COPILOT_FAILURE_DIAGNOSIS_FINAL_OUTCOMES)[number]; + +export interface CopilotFailureDiagnosisFinalizationReceipt { + readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_RECEIPT_SCHEMA; + readonly requestId: string; + readonly planDigest: string; + readonly runId: string; + readonly modelStepRunId: string; + readonly invocationId: string; + readonly completionDigest: string; + readonly outcome: CopilotFailureDiagnosisFinalOutcome; + readonly outputArtifactId: string | null; + readonly finalRunVersion: number; + readonly finalRunEventSequence: number; + readonly runEventId: string; + readonly finalizedAtMs: number; + readonly receiptDigest: string; +} + +export interface CopilotFailureDiagnosisFinalizationRepository { + findFinalization( + requestId: string, + ): Promise | null>; + finalize(requestId: string): Promise; + }>>; +} + +export class CopilotFailureDiagnosisModelExecutionInProgressError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_IN_PROGRESS'; + constructor() { + super('Copilot failure diagnosis Model execution is in progress'); + this.name = 'CopilotFailureDiagnosisModelExecutionInProgressError'; + } +} + +export class CopilotFailureDiagnosisModelResolutionRequiredError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_MODEL_RESOLUTION_REQUIRED'; + constructor() { + super('Copilot failure diagnosis Model execution requires resolution'); + this.name = 'CopilotFailureDiagnosisModelResolutionRequiredError'; + } +} + +export class CopilotFailureDiagnosisFinalizationConflictError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_CONFLICT'; + constructor() { + super('Copilot failure diagnosis finalization conflicts'); + this.name = 'CopilotFailureDiagnosisFinalizationConflictError'; + } +} + +export class CopilotFailureDiagnosisFinalizationUnavailableError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_UNAVAILABLE'; + constructor(options?: ErrorOptions) { + super('Copilot failure diagnosis finalization is unavailable', options); + this.name = 'CopilotFailureDiagnosisFinalizationUnavailableError'; + } +} + +const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/; +const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const DIGEST_DOMAIN = + 'qinglong/copilot-failure-diagnosis-finalization-receipt-digest@v1\0'; +const EVENT_DOMAIN = + 'qinglong/copilot-failure-diagnosis-finalization-event-id@v1\0'; + +function invalid(): never { + throw new CopilotFailureDiagnosisFinalizationConflictError(); +} + +function text(value: unknown, pattern: RegExp): string { + if (typeof value !== 'string' || !pattern.test(value)) return invalid(); + return value; +} + +function integer(value: unknown): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) return invalid(); + return value as number; +} + +function unsigned( + value: Omit, +): object { + return { + schema: value.schema, + requestId: value.requestId, + planDigest: value.planDigest, + runId: value.runId, + modelStepRunId: value.modelStepRunId, + invocationId: value.invocationId, + completionDigest: value.completionDigest, + outcome: value.outcome, + outputArtifactId: value.outputArtifactId, + finalRunVersion: value.finalRunVersion, + finalRunEventSequence: value.finalRunEventSequence, + runEventId: value.runEventId, + finalizedAtMs: value.finalizedAtMs, + }; +} + +export function copilotFailureDiagnosisFinalizationReceiptDigest( + value: Omit, +): string { + return createHash('sha256') + .update(DIGEST_DOMAIN) + .update(JSON.stringify(unsigned(value))) + .digest('hex'); +} + +export function copilotFailureDiagnosisFinalizationEventIdentity( + invocationId: string, + completionDigest: string, +): string { + const hex = createHash('sha256') + .update(EVENT_DOMAIN) + .update(text(invocationId, ID_PATTERN)) + .update(text(completionDigest, DIGEST_PATTERN)) + .digest('hex') + .slice(0, 32) + .split(''); + hex[12] = '4'; + hex[16] = '8'; + const value = hex.join(''); + return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice( + 12, + 16, + )}-${value.slice(16, 20)}-${value.slice(20)}`; +} + +export function normalizeCopilotFailureDiagnosisFinalizationReceipt( + value: CopilotFailureDiagnosisFinalizationReceipt, +): Readonly { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return invalid(); + } + const expected = [ + 'completionDigest', 'finalRunEventSequence', 'finalRunVersion', + 'finalizedAtMs', 'invocationId', 'outcome', 'outputArtifactId', + 'modelStepRunId', 'planDigest', 'receiptDigest', 'requestId', 'runEventId', + 'runId', 'schema', + ]; + const keys = Reflect.ownKeys(value); + if ( + keys.length !== expected.length || + keys.some((item) => typeof item !== 'string' || !expected.includes(item)) || + value.schema !== COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_RECEIPT_SCHEMA || + !COPILOT_FAILURE_DIAGNOSIS_FINAL_OUTCOMES.includes(value.outcome) + ) { + return invalid(); + } + const outputArtifactId = + value.outputArtifactId === null + ? null + : text(value.outputArtifactId, ID_PATTERN); + if ((value.outcome === 'succeeded') !== (outputArtifactId !== null)) { + return invalid(); + } + const normalized = Object.freeze({ + schema: value.schema, + requestId: text(value.requestId, ID_PATTERN), + planDigest: text(value.planDigest, DIGEST_PATTERN), + runId: text(value.runId, RUN_ID_PATTERN), + modelStepRunId: text(value.modelStepRunId, ID_PATTERN), + invocationId: text(value.invocationId, ID_PATTERN), + completionDigest: text(value.completionDigest, DIGEST_PATTERN), + outcome: value.outcome, + outputArtifactId, + finalRunVersion: integer(value.finalRunVersion), + finalRunEventSequence: integer(value.finalRunEventSequence), + runEventId: text(value.runEventId, RUN_ID_PATTERN), + finalizedAtMs: integer(value.finalizedAtMs), + } satisfies Omit< + CopilotFailureDiagnosisFinalizationReceipt, + 'receiptDigest' + >); + if ( + normalized.finalRunVersion < 1 || + normalized.finalRunEventSequence !== normalized.finalRunVersion || + normalized.runEventId !== + copilotFailureDiagnosisFinalizationEventIdentity( + normalized.invocationId, + normalized.completionDigest, + ) || + text(value.receiptDigest, DIGEST_PATTERN) !== + copilotFailureDiagnosisFinalizationReceiptDigest(normalized) + ) { + return invalid(); + } + return Object.freeze({ ...normalized, receiptDigest: value.receiptDigest }); +} + +export function createCopilotFailureDiagnosisFinalizationReceipt( + value: Omit< + CopilotFailureDiagnosisFinalizationReceipt, + 'schema' | 'runEventId' | 'receiptDigest' + >, +): Readonly { + const unsignedReceipt = Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_RECEIPT_SCHEMA, + ...value, + runEventId: copilotFailureDiagnosisFinalizationEventIdentity( + value.invocationId, + value.completionDigest, + ), + }); + return normalizeCopilotFailureDiagnosisFinalizationReceipt({ + ...unsignedReceipt, + receiptDigest: + copilotFailureDiagnosisFinalizationReceiptDigest(unsignedReceipt), + }); +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/outputArtifact.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/outputArtifact.ts new file mode 100644 index 00000000..08089c70 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/outputArtifact.ts @@ -0,0 +1,491 @@ +import { Buffer } from 'node:buffer'; +import { + createCipheriv, + createDecipheriv, + createHash, + randomBytes, +} from 'node:crypto'; + +import type { GenerateResult } from '../../../model-gateway/model'; +import { normalizeGenerateResult } from '../../../model-gateway/validation'; +import type { FailureDiagnosisModelEgressEvidence } from '../contracts'; + +export const COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_SCHEMA = + 'qinglong/copilot-failure-diagnosis-output-artifact@v1' as const; +export const COPILOT_FAILURE_DIAGNOSIS_OUTPUT_REFERENCE_SCHEMA = + 'qinglong/copilot-failure-diagnosis-output-reference@v1' as const; +export const COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM = + 'aes-256-gcm' as const; +export const MAX_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_BYTES = + 1536 * 1024; + +export interface CopilotFailureDiagnosisOutputArtifact { + readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_SCHEMA; + readonly artifactId: string; + readonly requestId: string; + readonly planDigest: string; + readonly toolCompletionDigest: string; + readonly projectId: string; + readonly runId: string; + readonly stepRunId: string; + readonly invocationId: string; + readonly provider: string; + readonly model: string; + readonly egressEvidenceDigest: string; + readonly contentDigest: string; + readonly outputBytes: number; + readonly keyId: string; + readonly algorithm: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM; + readonly nonce: string; + readonly ciphertext: string; + readonly authTag: string; + readonly plaintextBytes: number; + readonly sealedAtMs: number; + readonly artifactDigest: string; +} + +export interface CopilotFailureDiagnosisOutputReference { + readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_REFERENCE_SCHEMA; + readonly artifactId: string; + readonly requestId: string; + readonly planDigest: string; + readonly toolCompletionDigest: string; + readonly projectId: string; + readonly runId: string; + readonly stepRunId: string; + readonly invocationId: string; + readonly provider: string; + readonly model: string; + readonly egressEvidenceDigest: string; + readonly contentDigest: string; + readonly outputBytes: number; + readonly keyId: string; + readonly algorithm: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM; + readonly sealedAtMs: number; + readonly artifactDigest: string; +} + +export interface CopilotFailureDiagnosisOutputKeyMaterial { + readonly keyId: string; + readonly key: Uint8Array; +} + +export interface CopilotFailureDiagnosisOutputKeyProvider { + active(): Promise; + resolve(keyId: string): Promise; +} + +export class InvalidCopilotFailureDiagnosisOutputArtifactError extends TypeError { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_INVALID'; + + constructor(message: string) { + super(`Copilot failure diagnosis output Artifact is invalid: ${message}`); + this.name = 'InvalidCopilotFailureDiagnosisOutputArtifactError'; + } +} + +export class CopilotFailureDiagnosisOutputArtifactConflictError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_CONFLICT'; + + constructor() { + super('Copilot failure diagnosis output Artifact conflicts'); + this.name = 'CopilotFailureDiagnosisOutputArtifactConflictError'; + } +} + +export class CopilotFailureDiagnosisOutputArtifactUnavailableError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_UNAVAILABLE'; + + constructor(options?: ErrorOptions) { + super('Copilot failure diagnosis output Artifact is unavailable', options); + this.name = 'CopilotFailureDiagnosisOutputArtifactUnavailableError'; + } +} + +const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/; +const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/; +const MODEL_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/; +const KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const BASE64URL_PATTERN = /^[A-Za-z0-9_-]+$/; +const ARTIFACT_ID_DOMAIN = Buffer.from( + 'qinglong/copilot-failure-diagnosis-output-artifact-id@v1\0', +); +const CONTENT_DIGEST_DOMAIN = Buffer.from( + 'qinglong/copilot-failure-diagnosis-output-content-digest@v1\0', +); +const EGRESS_DIGEST_DOMAIN = Buffer.from( + 'qinglong/copilot-failure-diagnosis-output-egress-digest@v1\0', +); +const ARTIFACT_DIGEST_DOMAIN = Buffer.from( + 'qinglong/copilot-failure-diagnosis-output-artifact-digest@v1\0', +); +const AAD_DOMAIN = Buffer.from( + 'qinglong/copilot-failure-diagnosis-output-artifact-aad@v1\0', +); + +function invalid(message: string): never { + throw new InvalidCopilotFailureDiagnosisOutputArtifactError(message); +} + +function hash(domain: Buffer, value: unknown): string { + return createHash('sha256') + .update(domain) + .update(JSON.stringify(value)) + .digest('hex'); +} + +function text(value: unknown, pattern: RegExp, label: string): string { + if (typeof value !== 'string' || !pattern.test(value)) { + return invalid(`${label} is invalid`); + } + return value; +} + +function integer(value: unknown, maximum: number, label: string): number { + if ( + !Number.isSafeInteger(value) || + (value as number) < 0 || + (value as number) > maximum + ) { + return invalid(`${label} is invalid`); + } + return value as number; +} + +function bytes(value: unknown, expected?: number): Buffer { + if ( + typeof value !== 'string' || + value.length < 1 || + !BASE64URL_PATTERN.test(value) + ) { + return invalid('encoded bytes are invalid'); + } + const decoded = Buffer.from(value, 'base64url'); + if ( + decoded.toString('base64url') !== value || + (expected !== undefined && decoded.length !== expected) + ) { + decoded.fill(0); + return invalid('encoded bytes are invalid'); + } + return decoded; +} + +function key(value: Uint8Array): Buffer { + if (!(value instanceof Uint8Array) || value.byteLength !== 32) { + throw new CopilotFailureDiagnosisOutputArtifactUnavailableError(); + } + return Buffer.from(value); +} + +function unsigned( + value: CopilotFailureDiagnosisOutputArtifact, +): Omit { + return { + schema: value.schema, + artifactId: value.artifactId, + requestId: value.requestId, + planDigest: value.planDigest, + toolCompletionDigest: value.toolCompletionDigest, + projectId: value.projectId, + runId: value.runId, + stepRunId: value.stepRunId, + invocationId: value.invocationId, + provider: value.provider, + model: value.model, + egressEvidenceDigest: value.egressEvidenceDigest, + contentDigest: value.contentDigest, + outputBytes: value.outputBytes, + keyId: value.keyId, + algorithm: value.algorithm, + nonce: value.nonce, + ciphertext: value.ciphertext, + authTag: value.authTag, + plaintextBytes: value.plaintextBytes, + sealedAtMs: value.sealedAtMs, + }; +} + +function metadata( + value: CopilotFailureDiagnosisOutputArtifact, +): Omit< + CopilotFailureDiagnosisOutputArtifact, + 'artifactDigest' | 'nonce' | 'ciphertext' | 'authTag' +> { + const { + artifactDigest: _artifactDigest, + nonce: _nonce, + ciphertext: _ciphertext, + authTag: _authTag, + ...record + } = value; + return record; +} + +function aad(value: ReturnType): Buffer { + return Buffer.concat([AAD_DOMAIN, Buffer.from(JSON.stringify(value))]); +} + +export function copilotFailureDiagnosisOutputArtifactIdentity( + invocationIdValue: string, +): string { + const invocationId = text(invocationIdValue, ID_PATTERN, 'invocation id'); + return `cdo:${hash(ARTIFACT_ID_DOMAIN, invocationId).slice(0, 32)}`; +} + +export function copilotFailureDiagnosisEgressEvidenceDigest( + value: Readonly, +): string { + return hash(EGRESS_DIGEST_DOMAIN, value); +} + +export function normalizeCopilotFailureDiagnosisOutputArtifact( + value: CopilotFailureDiagnosisOutputArtifact, +): Readonly { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return invalid('Artifact must be an object'); + } + const expected = [ + 'algorithm', 'artifactDigest', 'artifactId', 'authTag', 'ciphertext', + 'contentDigest', 'egressEvidenceDigest', 'invocationId', 'keyId', 'model', + 'nonce', 'outputBytes', 'plaintextBytes', 'planDigest', 'projectId', + 'provider', 'requestId', 'runId', 'schema', 'sealedAtMs', 'stepRunId', + 'toolCompletionDigest', + ]; + const keys = Reflect.ownKeys(value); + if ( + keys.length !== expected.length || + keys.some((item) => typeof item !== 'string' || !expected.includes(item)) + ) { + return invalid('Artifact shape is invalid'); + } + if ( + value.schema !== COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_SCHEMA || + value.algorithm !== COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM + ) { + return invalid('Artifact protocol is unsupported'); + } + const normalized = Object.freeze({ + schema: value.schema, + artifactId: text(value.artifactId, ID_PATTERN, 'Artifact id'), + requestId: text(value.requestId, ID_PATTERN, 'request id'), + planDigest: text(value.planDigest, DIGEST_PATTERN, 'plan digest'), + toolCompletionDigest: text( + value.toolCompletionDigest, + DIGEST_PATTERN, + 'Tool completion digest', + ), + projectId: text(value.projectId, ID_PATTERN, 'Project id'), + runId: text(value.runId, RUN_ID_PATTERN, 'Run id'), + stepRunId: text(value.stepRunId, ID_PATTERN, 'StepRun id'), + invocationId: text(value.invocationId, ID_PATTERN, 'invocation id'), + provider: text(value.provider, ID_PATTERN, 'provider'), + model: text(value.model, MODEL_PATTERN, 'model'), + egressEvidenceDigest: text( + value.egressEvidenceDigest, + DIGEST_PATTERN, + 'egress evidence digest', + ), + contentDigest: text(value.contentDigest, DIGEST_PATTERN, 'content digest'), + outputBytes: integer(value.outputBytes, 1024 * 1024, 'output bytes'), + keyId: text(value.keyId, KEY_PATTERN, 'key id'), + algorithm: value.algorithm, + nonce: bytes(value.nonce, 12).toString('base64url'), + ciphertext: bytes(value.ciphertext).toString('base64url'), + authTag: bytes(value.authTag, 16).toString('base64url'), + plaintextBytes: integer( + value.plaintextBytes, + 1024 * 1024 + 4096, + 'plaintext bytes', + ), + sealedAtMs: integer(value.sealedAtMs, Number.MAX_SAFE_INTEGER, 'seal time'), + artifactDigest: text( + value.artifactDigest, + DIGEST_PATTERN, + 'Artifact digest', + ), + } satisfies CopilotFailureDiagnosisOutputArtifact); + if ( + normalized.artifactId !== + copilotFailureDiagnosisOutputArtifactIdentity(normalized.invocationId) || + Buffer.byteLength(JSON.stringify(normalized)) > + MAX_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_BYTES || + hash(ARTIFACT_DIGEST_DOMAIN, unsigned(normalized)) !== + normalized.artifactDigest + ) { + return invalid('Artifact binding is invalid'); + } + return normalized; +} + +export function copilotFailureDiagnosisOutputReference( + value: CopilotFailureDiagnosisOutputArtifact, +): Readonly { + const artifact = normalizeCopilotFailureDiagnosisOutputArtifact(value); + return Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_OUTPUT_REFERENCE_SCHEMA, + artifactId: artifact.artifactId, + requestId: artifact.requestId, + planDigest: artifact.planDigest, + toolCompletionDigest: artifact.toolCompletionDigest, + projectId: artifact.projectId, + runId: artifact.runId, + stepRunId: artifact.stepRunId, + invocationId: artifact.invocationId, + provider: artifact.provider, + model: artifact.model, + egressEvidenceDigest: artifact.egressEvidenceDigest, + contentDigest: artifact.contentDigest, + outputBytes: artifact.outputBytes, + keyId: artifact.keyId, + algorithm: artifact.algorithm, + sealedAtMs: artifact.sealedAtMs, + artifactDigest: artifact.artifactDigest, + }); +} + +export function createCopilotFailureDiagnosisOutputArtifact( + input: Readonly<{ + requestId: string; + planDigest: string; + toolCompletionDigest: string; + projectId: string; + runId: string; + stepRunId: string; + invocationId: string; + result: Readonly; + egressEvidence: Readonly; + keyId: string; + key: Uint8Array; + sealedAtMs: number; + }>, + nonceFactory: () => Uint8Array = () => randomBytes(12), +): Readonly { + const result = normalizeGenerateResult(input.result); + const plaintext = Buffer.from(JSON.stringify(result)); + const ownedKey = key(input.key); + let nonce: Buffer | undefined; + try { + nonce = Buffer.from(nonceFactory()); + if (nonce.length !== 12) { + throw new CopilotFailureDiagnosisOutputArtifactUnavailableError(); + } + const base = { + schema: COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_SCHEMA, + artifactId: copilotFailureDiagnosisOutputArtifactIdentity( + input.invocationId, + ), + requestId: text(input.requestId, ID_PATTERN, 'request id'), + planDigest: text(input.planDigest, DIGEST_PATTERN, 'plan digest'), + toolCompletionDigest: text( + input.toolCompletionDigest, + DIGEST_PATTERN, + 'Tool completion digest', + ), + projectId: text(input.projectId, ID_PATTERN, 'Project id'), + runId: text(input.runId, RUN_ID_PATTERN, 'Run id'), + stepRunId: text(input.stepRunId, ID_PATTERN, 'StepRun id'), + invocationId: text(input.invocationId, ID_PATTERN, 'invocation id'), + provider: result.provider, + model: result.model, + egressEvidenceDigest: copilotFailureDiagnosisEgressEvidenceDigest( + input.egressEvidence, + ), + contentDigest: hash(CONTENT_DIGEST_DOMAIN, result), + outputBytes: Buffer.byteLength(result.text), + keyId: text(input.keyId, KEY_PATTERN, 'key id'), + algorithm: COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM, + plaintextBytes: plaintext.length, + sealedAtMs: integer( + input.sealedAtMs, + Number.MAX_SAFE_INTEGER, + 'seal time', + ), + } as const; + const associated = aad(base as ReturnType); + const cipher = createCipheriv( + COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM, + ownedKey, + nonce, + { authTagLength: 16 }, + ); + cipher.setAAD(associated); + const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]); + try { + const unsignedArtifact = { + ...base, + nonce: nonce.toString('base64url'), + ciphertext: ciphertext.toString('base64url'), + authTag: cipher.getAuthTag().toString('base64url'), + }; + const candidate = { + ...unsignedArtifact, + artifactDigest: '0'.repeat(64), + }; + return normalizeCopilotFailureDiagnosisOutputArtifact({ + ...candidate, + artifactDigest: hash(ARTIFACT_DIGEST_DOMAIN, unsigned(candidate)), + }); + } finally { + associated.fill(0); + ciphertext.fill(0); + } + } catch (cause) { + if (cause instanceof InvalidCopilotFailureDiagnosisOutputArtifactError) { + throw cause; + } + throw new CopilotFailureDiagnosisOutputArtifactUnavailableError({ cause }); + } finally { + plaintext.fill(0); + ownedKey.fill(0); + nonce?.fill(0); + } +} + +export function openCopilotFailureDiagnosisOutputArtifact( + value: CopilotFailureDiagnosisOutputArtifact, + keyValue: Uint8Array, +): Readonly { + const artifact = normalizeCopilotFailureDiagnosisOutputArtifact(value); + const ownedKey = key(keyValue); + const nonce = bytes(artifact.nonce, 12); + const ciphertext = bytes(artifact.ciphertext); + const authTag = bytes(artifact.authTag, 16); + const associated = aad(metadata(artifact)); + let plaintext: Buffer | undefined; + try { + const decipher = createDecipheriv( + COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM, + ownedKey, + nonce, + { authTagLength: 16 }, + ); + decipher.setAAD(associated); + decipher.setAuthTag(authTag); + plaintext = Buffer.concat([decipher.update(ciphertext), decipher.final()]); + const result = normalizeGenerateResult(JSON.parse(plaintext.toString())); + if ( + plaintext.length !== artifact.plaintextBytes || + result.provider !== artifact.provider || + result.model !== artifact.model || + Buffer.byteLength(result.text) !== artifact.outputBytes || + hash(CONTENT_DIGEST_DOMAIN, result) !== artifact.contentDigest + ) { + throw new CopilotFailureDiagnosisOutputArtifactUnavailableError(); + } + return result; + } catch (cause) { + if (cause instanceof CopilotFailureDiagnosisOutputArtifactUnavailableError) { + throw cause; + } + throw new CopilotFailureDiagnosisOutputArtifactUnavailableError({ cause }); + } finally { + ownedKey.fill(0); + nonce.fill(0); + ciphertext.fill(0); + authTag.fill(0); + associated.fill(0); + plaintext?.fill(0); + } +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/postgresRepository.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/postgresRepository.ts new file mode 100644 index 00000000..385a3d9f --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/postgresRepository.ts @@ -0,0 +1,577 @@ +import type { PostgresClient, PostgresPool } from '@qinglong/runtime-core'; + +import { PostgresModelInvocationRepository } from '../../../model-invocation/postgres-model-invocation-repository/repository'; +import { completeWithAtomicOutputOperation } from '../../../model-invocation/postgres-model-invocation-repository/completionOperations'; +import { + normalizeModelInvocationCompletionRecord, + type ModelInvocationCompletionCommand, + type ModelInvocationCompletionRecord, +} from '../../../model-invocation/modelInvocation'; +import { normalizeCopilotFailureDiagnosisExecutionPlan } from '../admission/plan'; +import type { CopilotFailureDiagnosisExecutionPlan } from '../admission/contracts'; +import { + assertCopilotFailureDiagnosisOutputCompletionBinding, + type CommitCopilotFailureDiagnosisOutputResult, + type CopilotFailureDiagnosisOutputCompletionRepository, +} from './completion'; +import { + CopilotFailureDiagnosisOutputArtifactConflictError, + CopilotFailureDiagnosisOutputArtifactUnavailableError, + copilotFailureDiagnosisOutputArtifactIdentity, + normalizeCopilotFailureDiagnosisOutputArtifact, + type CopilotFailureDiagnosisOutputArtifact, +} from './outputArtifact'; +import { + CopilotFailureDiagnosisFinalizationConflictError, + CopilotFailureDiagnosisFinalizationUnavailableError, + CopilotFailureDiagnosisModelExecutionInProgressError, + CopilotFailureDiagnosisModelResolutionRequiredError, + createCopilotFailureDiagnosisFinalizationReceipt, + normalizeCopilotFailureDiagnosisFinalizationReceipt, + type CopilotFailureDiagnosisFinalizationReceipt, + type CopilotFailureDiagnosisFinalizationRepository, + type CopilotFailureDiagnosisFinalOutcome, +} from './finalization'; + +const TABLE = '"ql3_ai"."copilot_failure_diagnosis_model_outputs"'; +const FINALIZATION_TABLE = + '"ql3_ai"."copilot_failure_diagnosis_finalizations"'; + +interface OutputRow extends Record { + readonly artifactJson: unknown; +} + +function integer(value: unknown): number { + if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) { + return value; + } + if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) { + const parsed = Number(value); + if (Number.isSafeInteger(parsed)) return parsed; + } + throw new CopilotFailureDiagnosisFinalizationConflictError(); +} + +function object(value: unknown): Record { + if (value && typeof value === 'object' && !Array.isArray(value)) { + return value as Record; + } + if (typeof value === 'string') { + try { + return JSON.parse(value) as Record; + } catch { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + } + throw new CopilotFailureDiagnosisFinalizationConflictError(); +} + +function string(value: unknown): string { + if (typeof value !== 'string') { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + return value; +} + +function unavailable(cause?: unknown): never { + throw new CopilotFailureDiagnosisOutputArtifactUnavailableError({ + cause: cause instanceof Error ? cause : undefined, + }); +} + +function parse(row: OutputRow): Readonly { + try { + return normalizeCopilotFailureDiagnosisOutputArtifact( + row.artifactJson as CopilotFailureDiagnosisOutputArtifact, + ); + } catch (cause) { + return unavailable(cause); + } +} + +async function read( + queryable: Pick | Pick, + artifactId: string, +): Promise | null> { + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(artifactId)) { + throw new CopilotFailureDiagnosisOutputArtifactConflictError(); + } + try { + const result = await queryable.query( + `SELECT artifact_json AS "artifactJson" + FROM ${TABLE} + WHERE artifact_id = $1`, + [artifactId], + ); + if (result.rows.length > 1) { + throw new CopilotFailureDiagnosisOutputArtifactConflictError(); + } + return result.rows[0] ? parse(result.rows[0]) : null; + } catch (cause) { + if (cause instanceof CopilotFailureDiagnosisOutputArtifactConflictError) { + throw cause; + } + return unavailable(cause); + } +} + +async function put( + client: PostgresClient, + artifactValue: CopilotFailureDiagnosisOutputArtifact, +): Promise> { + const artifact = normalizeCopilotFailureDiagnosisOutputArtifact(artifactValue); + const existing = await read(client, artifact.artifactId); + if (existing) { + if (JSON.stringify(existing) !== JSON.stringify(artifact)) { + throw new CopilotFailureDiagnosisOutputArtifactConflictError(); + } + return existing; + } + try { + await client.query( + `INSERT INTO ${TABLE} ( + artifact_id, request_id, plan_digest, tool_completion_digest, + project_id, run_id, step_run_id, invocation_id, provider, model, + egress_evidence_digest, content_digest, output_bytes, key_id, + algorithm, sealed_at_ms, artifact_digest, artifact_json + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, $16, $17, $18::jsonb + )`, + [ + artifact.artifactId, + artifact.requestId, + artifact.planDigest, + artifact.toolCompletionDigest, + artifact.projectId, + artifact.runId, + artifact.stepRunId, + artifact.invocationId, + artifact.provider, + artifact.model, + artifact.egressEvidenceDigest, + artifact.contentDigest, + artifact.outputBytes, + artifact.keyId, + artifact.algorithm, + artifact.sealedAtMs, + artifact.artifactDigest, + JSON.stringify(artifact), + ], + ); + return artifact; + } catch (cause) { + return unavailable(cause); + } +} + +export class PostgresCopilotFailureDiagnosisModelRepository + extends PostgresModelInvocationRepository + implements + CopilotFailureDiagnosisOutputCompletionRepository, + CopilotFailureDiagnosisFinalizationRepository +{ + readonly #pool: PostgresPool; + + constructor(pool: PostgresPool) { + super(pool); + this.#pool = pool; + } + + findCopilotFailureDiagnosisOutput( + artifactId: string, + ): Promise | null> { + return read(this.#pool, artifactId); + } + + async completeWithCopilotFailureDiagnosisOutput( + commandValue: Readonly, + artifactValue: Readonly, + ): Promise> { + const binding = assertCopilotFailureDiagnosisOutputCompletionBinding( + commandValue, + artifactValue, + ); + const result = await completeWithAtomicOutputOperation( + this.#pool, + commandValue, + { + artifact: binding.artifact, + reference: binding.reference, + read: (client) => read(client, binding.artifact.artifactId), + put: (client) => put(client, binding.artifact), + matches: (stored) => + JSON.stringify(stored) === JSON.stringify(binding.artifact), + }, + ); + return Object.freeze({ + status: result.status, + reference: result.reference, + }); + } + + async findFinalization( + requestId: string, + ): Promise | null> { + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(requestId)) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + try { + const result = await this.#pool.query>( + `SELECT request_id AS "requestId", plan_digest AS "planDigest", + run_id AS "runId", model_step_run_id AS "modelStepRunId", + invocation_id AS "invocationId", + completion_digest AS "completionDigest", outcome, + output_artifact_id AS "outputArtifactId", + final_run_version AS "finalRunVersion", + final_run_event_sequence AS "finalRunEventSequence", + run_event_id AS "runEventId", + finalized_at_ms AS "finalizedAtMs", + receipt_digest AS "receiptDigest", + receipt_json AS "receiptJson" + FROM ${FINALIZATION_TABLE} + WHERE request_id = $1`, + [requestId], + ); + if (result.rows.length > 1) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + if (!result.rows[0]) return null; + const row = result.rows[0]; + const receipt = normalizeCopilotFailureDiagnosisFinalizationReceipt( + object(row.receiptJson) as unknown as CopilotFailureDiagnosisFinalizationReceipt, + ); + if ( + receipt.requestId !== string(row.requestId) || + receipt.planDigest !== string(row.planDigest) || + receipt.runId !== string(row.runId) || + receipt.modelStepRunId !== string(row.modelStepRunId) || + receipt.invocationId !== string(row.invocationId) || + receipt.completionDigest !== string(row.completionDigest) || + receipt.outcome !== string(row.outcome) || + receipt.outputArtifactId !== row.outputArtifactId || + receipt.finalRunVersion !== integer(row.finalRunVersion) || + receipt.finalRunEventSequence !== integer(row.finalRunEventSequence) || + receipt.runEventId !== string(row.runEventId) || + receipt.finalizedAtMs !== integer(row.finalizedAtMs) || + receipt.receiptDigest !== string(row.receiptDigest) + ) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + const durable = await this.#pool.query>( + `SELECT run.status AS "runStatus", run.version AS "runVersion", + run.event_sequence AS "runEventSequence", + run.finished_at_ms AS "finishedAtMs", + run.output_ref AS "outputRef", + step.status AS "stepStatus", + event.type AS "eventType", event.dedupe_key AS "dedupeKey", + event.step_run_id AS "eventStepRunId", + event.payload, event.created_at_ms AS "eventCreatedAtMs", + completion.completion_digest AS "completionDigest", + completion.outcome AS "completionOutcome" + FROM "ql3"."runs" AS run + JOIN "ql3"."step_runs" AS step + ON step.run_id = run.id AND step.id = $1 + JOIN "ql3"."run_events" AS event + ON event.run_id = run.id AND event.id = $2 + JOIN "ql3_ai"."model_invocation_completions" AS completion + ON completion.invocation_id = $3 + WHERE run.id = $4`, + [ + receipt.modelStepRunId, + receipt.runEventId, + receipt.invocationId, + receipt.runId, + ], + ); + if (durable.rows.length !== 1) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + const proof = durable.rows[0]!; + const payload = object(proof.payload); + if ( + string(proof.runStatus) !== receipt.outcome || + integer(proof.runVersion) !== receipt.finalRunVersion || + integer(proof.runEventSequence) !== receipt.finalRunEventSequence || + integer(proof.finishedAtMs) !== receipt.finalizedAtMs || + proof.outputRef !== receipt.outputArtifactId || + string(proof.stepStatus) !== receipt.outcome || + string(proof.eventType) !== `copilot.diagnosis.${receipt.outcome}` || + string(proof.dedupeKey) !== receipt.runEventId || + string(proof.eventStepRunId) !== receipt.modelStepRunId || + integer(proof.eventCreatedAtMs) !== receipt.finalizedAtMs || + string(proof.completionDigest) !== receipt.completionDigest || + string(proof.completionOutcome) !== receipt.outcome || + payload.requestId !== receipt.requestId || + payload.planDigest !== receipt.planDigest || + payload.invocationId !== receipt.invocationId || + payload.completionDigest !== receipt.completionDigest || + payload.outcome !== receipt.outcome || + payload.outputArtifactId !== receipt.outputArtifactId + ) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + return receipt; + } catch (cause) { + if (cause instanceof CopilotFailureDiagnosisFinalizationConflictError) { + throw cause; + } + throw new CopilotFailureDiagnosisFinalizationUnavailableError({ cause }); + } + } + + async finalize(requestId: string): Promise; + }>> { + const existing = await this.findFinalization(requestId); + if (existing) { + return Object.freeze({ status: 'existing' as const, receipt: existing }); + } + for (let attempt = 0; attempt < 3; attempt += 1) { + let client: PostgresClient; + try { + client = await this.#pool.connect(); + } catch (cause) { + throw new CopilotFailureDiagnosisFinalizationUnavailableError({ cause }); + } + let began = false; + try { + await client.query('BEGIN ISOLATION LEVEL SERIALIZABLE'); + began = true; + await client.query(`SELECT set_config('statement_timeout', $1, true)`, [ + '5s', + ]); + await client.query(`SELECT set_config('lock_timeout', $1, true)`, [ + '2s', + ]); + const result = await this.#finalizeInTransaction(client, requestId); + await client.query('COMMIT'); + began = false; + return result; + } catch (cause) { + if (began) { + try { + await client.query('ROLLBACK'); + } catch { + // Preserve the original failure. + } + } + const state = + cause && typeof cause === 'object' && 'code' in cause + ? String(cause.code) + : ''; + if ((state === '40001' || state === '40P01') && attempt < 2) { + continue; + } + const recovered = await this.findFinalization(requestId); + if (recovered) { + return Object.freeze({ + status: 'existing' as const, + receipt: recovered, + }); + } + if ( + cause instanceof CopilotFailureDiagnosisFinalizationConflictError || + cause instanceof CopilotFailureDiagnosisModelExecutionInProgressError || + cause instanceof CopilotFailureDiagnosisModelResolutionRequiredError || + cause instanceof CopilotFailureDiagnosisFinalizationUnavailableError + ) { + throw cause; + } + throw new CopilotFailureDiagnosisFinalizationUnavailableError({ cause }); + } finally { + client.release(); + } + } + throw new CopilotFailureDiagnosisFinalizationUnavailableError(); + } + + async #finalizeInTransaction( + client: PostgresClient, + requestId: string, + ): Promise; + }>> { + const admission = await client.query>( + `SELECT plan_json AS "planJson" + FROM "ql3_ai"."copilot_failure_diagnosis_admissions" + WHERE request_id = $1`, + [requestId], + ); + if (admission.rows.length !== 1) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + const plan = normalizeCopilotFailureDiagnosisExecutionPlan( + object(admission.rows[0]!.planJson) as unknown as CopilotFailureDiagnosisExecutionPlan, + ); + const durable = await client.query>( + `SELECT run.status AS "runStatus", run.version AS "runVersion", + run.event_sequence AS "runEventSequence", + step.status AS "stepStatus", + step.step_run_digest AS "stepRunDigest", + completion.record_json AS "completionJson" + FROM "ql3"."runs" AS run + JOIN "ql3"."step_runs" AS step + ON step.run_id = run.id AND step.id = $1 + LEFT JOIN "ql3_ai"."model_invocation_completions" AS completion + ON completion.invocation_id = $2 + WHERE run.id = $3 + FOR UPDATE OF run, step`, + [plan.modelStepRunId, plan.modelInvocationId, plan.runId], + ); + if (durable.rows.length !== 1) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + const row = durable.rows[0]!; + if (row.completionJson === null || row.completionJson === undefined) { + throw new CopilotFailureDiagnosisModelExecutionInProgressError(); + } + const completion = normalizeModelInvocationCompletionRecord( + object(row.completionJson) as unknown as ModelInvocationCompletionRecord, + ); + if ( + completion.invocationId !== plan.modelInvocationId || + completion.projectId !== plan.projectId || + completion.runId !== plan.runId || + completion.stepRunId !== plan.modelStepRunId || + completion.traceId !== plan.traceId || + string(row.runStatus) !== 'running' || + string(row.stepRunDigest) !== completion.completedStepRunDigest + ) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + if (completion.outcome === 'outcome_unknown') { + throw new CopilotFailureDiagnosisModelResolutionRequiredError(); + } + const outcome: CopilotFailureDiagnosisFinalOutcome = completion.outcome; + if (string(row.stepStatus) !== outcome) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + const output = + outcome === 'succeeded' + ? await read( + client, + copilotFailureDiagnosisOutputArtifactIdentity( + plan.modelInvocationId, + ), + ) + : null; + if ( + (outcome === 'succeeded' && + (!output || + output.requestId !== plan.requestId || + output.planDigest !== plan.planDigest || + output.invocationId !== completion.invocationId || + output.outputBytes !== completion.outputBytes)) || + (outcome !== 'succeeded' && output !== null) + ) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + const runVersion = integer(row.runVersion); + const eventSequence = integer(row.runEventSequence); + if (runVersion !== eventSequence) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + const receipt = createCopilotFailureDiagnosisFinalizationReceipt({ + requestId: plan.requestId, + planDigest: plan.planDigest, + runId: plan.runId, + modelStepRunId: plan.modelStepRunId, + invocationId: plan.modelInvocationId, + completionDigest: completion.completionDigest, + outcome, + outputArtifactId: output?.artifactId ?? null, + finalRunVersion: runVersion + 1, + finalRunEventSequence: eventSequence + 1, + finalizedAtMs: completion.completedAtMs, + }); + const failure = outcome === 'succeeded' + ? { code: null, summary: null } + : outcome === 'timed_out' + ? { + code: 'COPILOT_FAILURE_DIAGNOSIS_TIMED_OUT', + summary: 'Copilot failure diagnosis timed out', + } + : { + code: 'COPILOT_FAILURE_DIAGNOSIS_FAILED', + summary: 'Copilot failure diagnosis failed', + }; + const updated = await client.query( + `UPDATE "ql3"."runs" + SET status = $1, version = $2, event_sequence = $3, + output_ref = $4, finished_at_ms = $5, + error_code = $6, error_summary = $7 + WHERE id = $8 AND status = 'running' + AND version = $9 AND event_sequence = $10`, + [ + outcome, + receipt.finalRunVersion, + receipt.finalRunEventSequence, + receipt.outputArtifactId, + receipt.finalizedAtMs, + failure.code, + failure.summary, + receipt.runId, + runVersion, + eventSequence, + ], + ); + if ((updated.rowCount ?? updated.rows.length) !== 1) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + const payload = JSON.stringify({ + requestId: receipt.requestId, + planDigest: receipt.planDigest, + invocationId: receipt.invocationId, + completionDigest: receipt.completionDigest, + outcome: receipt.outcome, + outputArtifactId: receipt.outputArtifactId, + }); + await client.query( + `INSERT INTO "ql3"."run_events" ( + id, run_id, sequence, type, dedupe_key, actor_type, actor_id, + attempt_id, step_run_id, payload, created_at_ms + ) VALUES ($1, $2, $3, $4, $1, 'system', NULL, NULL, $5, $6::jsonb, $7)`, + [ + receipt.runEventId, + receipt.runId, + receipt.finalRunEventSequence, + `copilot.diagnosis.${receipt.outcome}`, + plan.modelStepRunId, + payload, + receipt.finalizedAtMs, + ], + ); + await client.query( + `INSERT INTO ${FINALIZATION_TABLE} ( + request_id, plan_digest, run_id, model_step_run_id, invocation_id, + completion_digest, + outcome, output_artifact_id, final_run_version, + final_run_event_sequence, run_event_id, finalized_at_ms, + receipt_digest, receipt_json + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, + $14::jsonb + )`, + [ + receipt.requestId, + receipt.planDigest, + receipt.runId, + receipt.modelStepRunId, + receipt.invocationId, + receipt.completionDigest, + receipt.outcome, + receipt.outputArtifactId, + receipt.finalRunVersion, + receipt.finalRunEventSequence, + receipt.runEventId, + receipt.finalizedAtMs, + receipt.receiptDigest, + JSON.stringify(receipt), + ], + ); + return Object.freeze({ status: 'created' as const, receipt }); + } +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/modelExecution.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/modelExecution.ts new file mode 100644 index 00000000..652d8a22 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/modelExecution.ts @@ -0,0 +1,4 @@ +export * from './model-execution/completion'; +export * from './model-execution/coordinator'; +export * from './model-execution/finalization'; +export * from './model-execution/outputArtifact'; diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/postgresModelExecutionRepository.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/postgresModelExecutionRepository.ts new file mode 100644 index 00000000..7bd6dcdf --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/postgresModelExecutionRepository.ts @@ -0,0 +1 @@ +export { PostgresCopilotFailureDiagnosisModelRepository } from './model-execution/postgresRepository'; diff --git a/packages/ql3-ai/src/migration/model-invocation/identities.ts b/packages/ql3-ai/src/migration/model-invocation/identities.ts index 610a3cb8..8b77a1ca 100644 --- a/packages/ql3-ai/src/migration/model-invocation/identities.ts +++ b/packages/ql3-ai/src/migration/model-invocation/identities.ts @@ -65,6 +65,8 @@ export const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID = 'pg-9018-ai-copilot-failure-diagnosis-admissions'; export const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID = 'pg-9019-ai-copilot-failure-diagnosis-tool-unlocks'; +export const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID = + 'pg-9020-ai-copilot-failure-diagnosis-model-executions'; export const POSTGRES_MODEL_INVOCATION_SCHEMA = 'ql3_ai'; export const LOCAL_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE = 'QingLong3AiSchemaMigrations'; diff --git a/packages/ql3-ai/src/migration/model-invocation/postgres.ts b/packages/ql3-ai/src/migration/model-invocation/postgres.ts index 1b4202a2..3f38467e 100644 --- a/packages/ql3-ai/src/migration/model-invocation/postgres.ts +++ b/packages/ql3-ai/src/migration/model-invocation/postgres.ts @@ -29,6 +29,7 @@ import { POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_SCHEMA, POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE, } from './identities'; @@ -65,6 +66,7 @@ const POSTGRES_HISTORY_IDENTITY = Object.freeze({ POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, ]), streamId: POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID, dialect: 'postgresql' as const, diff --git a/packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts b/packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts index b37cbbb6..0b09beca 100644 --- a/packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts +++ b/packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts @@ -3,6 +3,7 @@ import type { PostgresQueryable } from '@qinglong/runtime-core'; import { POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_SCHEMA, } from '../identities'; import { defineSqlMigration } from '../shared'; @@ -11,6 +12,8 @@ const ADMISSION_TABLE = 'copilot_failure_diagnosis_admissions'; const SOURCE_SNAPSHOT_FUNCTION = 'copilot_failure_diagnosis_admission_source_snapshot'; const TOOL_UNLOCK_TABLE = 'copilot_failure_diagnosis_tool_unlocks'; +const MODEL_OUTPUT_TABLE = 'copilot_failure_diagnosis_model_outputs'; +const FINALIZATION_TABLE = 'copilot_failure_diagnosis_finalizations'; const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_TABLE_SQL = ` CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" ( @@ -319,7 +322,166 @@ const postgresCopilotFailureDiagnosisToolUnlockMigration = (context, statement) => context.query(statement).then(() => undefined), ); +const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_OUTPUT_TABLE_SQL = ` +CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${MODEL_OUTPUT_TABLE}" ( + artifact_id varchar(128) PRIMARY KEY, + request_id varchar(128) NOT NULL UNIQUE, + plan_digest char(64) NOT NULL UNIQUE, + tool_completion_digest char(64) NOT NULL, + project_id varchar(128) NOT NULL, + run_id varchar(36) NOT NULL, + step_run_id varchar(128) NOT NULL UNIQUE, + invocation_id varchar(128) NOT NULL UNIQUE, + provider varchar(128) NOT NULL, + model varchar(256) NOT NULL, + egress_evidence_digest char(64) NOT NULL, + content_digest char(64) NOT NULL, + output_bytes integer NOT NULL, + key_id varchar(128) NOT NULL, + algorithm varchar(32) NOT NULL, + sealed_at_ms bigint NOT NULL, + artifact_digest char(64) NOT NULL UNIQUE, + artifact_json jsonb NOT NULL, + CONSTRAINT ql3_ai_copilot_diagnosis_model_output_admission_fk + FOREIGN KEY (request_id) + REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" + (request_id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_model_output_unlock_fk + FOREIGN KEY (request_id) + REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${TOOL_UNLOCK_TABLE}" + (request_id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_model_output_completion_fk + FOREIGN KEY (invocation_id) + REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions" + (invocation_id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_model_output_step_fk + FOREIGN KEY (run_id, step_run_id) + REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_model_output_identity_check CHECK ( + artifact_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND + request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND + project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND + run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$' AND + step_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND + invocation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND + output_bytes BETWEEN 0 AND 1048576 AND sealed_at_ms >= 0 + ), + CONSTRAINT ql3_ai_copilot_diagnosis_model_output_digest_check CHECK ( + plan_digest ~ '^[0-9a-f]{64}$' AND + tool_completion_digest ~ '^[0-9a-f]{64}$' AND + egress_evidence_digest ~ '^[0-9a-f]{64}$' AND + content_digest ~ '^[0-9a-f]{64}$' AND + artifact_digest ~ '^[0-9a-f]{64}$' + ), + CONSTRAINT ql3_ai_copilot_diagnosis_model_output_json_check CHECK ( + jsonb_typeof(artifact_json) = 'object' AND + octet_length(artifact_json::text) BETWEEN 2 AND 1572864 AND + artifact_json @> jsonb_build_object( + 'schema', 'qinglong/copilot-failure-diagnosis-output-artifact@v1', + 'artifactId', artifact_id, 'requestId', request_id, + 'planDigest', plan_digest, + 'toolCompletionDigest', tool_completion_digest, + 'projectId', project_id, 'runId', run_id, + 'stepRunId', step_run_id, 'invocationId', invocation_id, + 'provider', provider, 'model', model, + 'egressEvidenceDigest', egress_evidence_digest, + 'contentDigest', content_digest, 'outputBytes', output_bytes, + 'keyId', key_id, 'algorithm', algorithm, + 'sealedAtMs', sealed_at_ms, 'artifactDigest', artifact_digest + ) + ) +)`; + +const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_TABLE_SQL = ` +CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${FINALIZATION_TABLE}" ( + request_id varchar(128) PRIMARY KEY, + plan_digest char(64) NOT NULL UNIQUE, + run_id varchar(36) NOT NULL UNIQUE, + model_step_run_id varchar(128) NOT NULL UNIQUE, + invocation_id varchar(128) NOT NULL UNIQUE, + completion_digest char(64) NOT NULL UNIQUE, + outcome varchar(32) NOT NULL, + output_artifact_id varchar(128), + final_run_version integer NOT NULL, + final_run_event_sequence integer NOT NULL, + run_event_id varchar(36) NOT NULL UNIQUE, + finalized_at_ms bigint NOT NULL, + receipt_digest char(64) NOT NULL UNIQUE, + receipt_json jsonb NOT NULL, + CONSTRAINT ql3_ai_copilot_diagnosis_finalization_admission_fk + FOREIGN KEY (request_id) + REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" + (request_id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_finalization_completion_fk + FOREIGN KEY (invocation_id) + REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions" + (invocation_id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_finalization_output_fk + FOREIGN KEY (output_artifact_id) + REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${MODEL_OUTPUT_TABLE}" + (artifact_id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_finalization_run_fk + FOREIGN KEY (run_id) REFERENCES "ql3"."runs" (id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_finalization_step_fk + FOREIGN KEY (run_id, model_step_run_id) + REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_finalization_event_fk + FOREIGN KEY (run_event_id) + REFERENCES "ql3"."run_events" (id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_diagnosis_finalization_state_check CHECK ( + outcome IN ('succeeded', 'failed', 'timed_out', 'cancelled') AND + ((outcome = 'succeeded' AND output_artifact_id IS NOT NULL) OR + (outcome <> 'succeeded' AND output_artifact_id IS NULL)) AND + final_run_version >= 1 AND + final_run_event_sequence = final_run_version AND + finalized_at_ms >= 0 + ), + CONSTRAINT ql3_ai_copilot_diagnosis_finalization_digest_check CHECK ( + plan_digest ~ '^[0-9a-f]{64}$' AND + completion_digest ~ '^[0-9a-f]{64}$' AND + receipt_digest ~ '^[0-9a-f]{64}$' + ), + CONSTRAINT ql3_ai_copilot_diagnosis_finalization_json_check CHECK ( + jsonb_typeof(receipt_json) = 'object' AND + octet_length(receipt_json::text) BETWEEN 2 AND 16384 AND + receipt_json @> jsonb_build_object( + 'schema', 'qinglong/copilot-failure-diagnosis-finalization-receipt@v1', + 'requestId', request_id, 'planDigest', plan_digest, + 'runId', run_id, 'modelStepRunId', model_step_run_id, + 'invocationId', invocation_id, + 'completionDigest', completion_digest, 'outcome', outcome, + 'finalRunVersion', final_run_version, + 'finalRunEventSequence', final_run_event_sequence, + 'runEventId', run_event_id, 'finalizedAtMs', finalized_at_ms, + 'receiptDigest', receipt_digest + ) + ) +)`; + +const postgresCopilotFailureDiagnosisModelExecutionMigration = + defineSqlMigration( + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, + [ + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_OUTPUT_TABLE_SQL, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_TABLE_SQL, + `REVOKE ALL ON TABLE + "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${MODEL_OUTPUT_TABLE}" + FROM PUBLIC`, + `GRANT SELECT, INSERT ON TABLE + "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${MODEL_OUTPUT_TABLE}" + TO ql3_runtime`, + `REVOKE ALL ON TABLE + "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${FINALIZATION_TABLE}" + FROM PUBLIC`, + `GRANT SELECT, INSERT ON TABLE + "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${FINALIZATION_TABLE}" + TO ql3_runtime`, + ], + (context, statement) => context.query(statement).then(() => undefined), + ); + export const postgresCopilotMigrations = Object.freeze([ postgresCopilotFailureDiagnosisAdmissionMigration, postgresCopilotFailureDiagnosisToolUnlockMigration, + postgresCopilotFailureDiagnosisModelExecutionMigration, ]); diff --git a/packages/ql3-ai/src/model-invocation/durableModelInvocationCoordinator.ts b/packages/ql3-ai/src/model-invocation/durableModelInvocationCoordinator.ts index bb51cc31..b19fbd3a 100644 --- a/packages/ql3-ai/src/model-invocation/durableModelInvocationCoordinator.ts +++ b/packages/ql3-ai/src/model-invocation/durableModelInvocationCoordinator.ts @@ -33,16 +33,9 @@ import { type ModelInvocationPriceQuote, } from '../pricing/pricing'; import { - isPluginPackagePromptOutputCompletionRepository, - type PluginPackagePromptOutputCompletionRepository, -} from '../prompt-output/pluginPackagePromptOutputCompletion'; -import { - PluginPackagePromptOutputArtifactConflictError, - normalizePluginPackagePromptOutputArtifact, - pluginPackagePromptOutputArtifactReference, - type PluginPackagePromptOutputArtifact, - type PluginPackagePromptOutputArtifactReference, -} from '../prompt-output/pluginPackagePromptOutputArtifact'; + normalizeModelInvocationAtomicSuccess, + type ModelInvocationAtomicSuccess, +} from './modelInvocationAtomicSuccess'; const MAX_COORDINATOR_ATTEMPTS = 3; @@ -242,32 +235,21 @@ export class DurableModelInvocationCoordinator return this.#admit(record, admission, quote); } - async recordWithPromptOutputArtifact( + async recordWithAtomicSuccess( record: Readonly, - artifactValue: PluginPackagePromptOutputArtifact, + extensionValue: ModelInvocationAtomicSuccess, ): Promise< Readonly<{ status: 'created' | 'existing'; - reference: Readonly; + reference: Readonly; }> > { - const artifact = normalizePluginPackagePromptOutputArtifact(artifactValue); - if ( - record.phase !== 'completed' || - record.requestId !== artifact.invocationId || - record.projectId !== artifact.projectId || - record.runId !== artifact.runId || - record.stepRunId !== artifact.stepRunId || - record.provider !== artifact.provider || - record.model !== artifact.model || - record.outputBytes !== artifact.outputBytes || - !isPluginPackagePromptOutputCompletionRepository(this.repository) - ) { - throw new PluginPackagePromptOutputArtifactConflictError(); - } - const result = await this.#complete(record, artifact); + const extension = normalizeModelInvocationAtomicSuccess(extensionValue); + if (record.phase !== 'completed') throw extension.conflict(); + extension.assertAudit(record); + const result = await this.#complete(record, extension); if (!result.reference) { - throw new PluginPackagePromptOutputArtifactConflictError(); + throw extension.conflict(); } return Object.freeze({ status: result.status, @@ -383,22 +365,18 @@ export class DurableModelInvocationCoordinator throw new ModelInvocationConflictError(); } - async #complete( + async #complete( audit: Readonly, - artifactValue?: Readonly, + extensionValue?: ModelInvocationAtomicSuccess, ): Promise< Readonly< ModelInvocationAuditDisposition & { - reference?: Readonly; + reference?: Readonly; } > > { - const artifact = artifactValue - ? normalizePluginPackagePromptOutputArtifact(artifactValue) - : undefined; - const artifactRepository = artifact - ? (this.repository as ModelInvocationRepository & - PluginPackagePromptOutputCompletionRepository) + const extension = extensionValue + ? normalizeModelInvocationAtomicSuccess(extensionValue) : undefined; const startValue = await this.repository.findStart(audit.requestId); if (!startValue) throw new ModelInvocationConflictError(); @@ -406,16 +384,14 @@ export class DurableModelInvocationCoordinator const existing = await this.repository.findCompletion(audit.requestId); if (existing) { assertCompletionMatchesAudit(existing, start, audit); - if (artifact && artifactRepository) { - const stored = await artifactRepository.findPromptOutputArtifact( - artifact.artifactId, - ); - if (!stored || JSON.stringify(stored) !== JSON.stringify(artifact)) { - throw new PluginPackagePromptOutputArtifactConflictError(); + if (extension) { + const stored = await extension.find(this.repository); + if (!stored || !extension.matches(stored)) { + throw extension.conflict(); } return Object.freeze({ status: 'existing' as const, - reference: pluginPackagePromptOutputArtifactReference(stored), + reference: stored, }); } return Object.freeze({ status: 'existing' as const }); @@ -430,7 +406,7 @@ export class DurableModelInvocationCoordinator ) { throw new ModelInvocationConflictError(); } - const transition = completionTransition(audit, artifact?.artifactId); + const transition = completionTransition(audit, extension?.outputRef); const mutationIdentity = createModelInvocationMutationIdentity( audit.requestId, 'completion', @@ -464,7 +440,7 @@ export class DurableModelInvocationCoordinator actor: { type: 'executor', id: 'model-gateway' }, }, ), - artifact?.artifactId, + extension?.outputRef, ); try { const pricingAware = isPricingAwareModelInvocationRepository( @@ -479,42 +455,32 @@ export class DurableModelInvocationCoordinator const reservation = quotaAware ? await this.repository.findQuotaReservation(audit.requestId) : null; - const result = - artifactRepository && artifact - ? await artifactRepository.completeWithPromptOutputArtifact( - command, - artifact, - ) - : pricingAware && quote + if (extension) { + const result = await extension.commit(this.repository, command); + return Object.freeze({ + status: result.status, + reference: result.reference, + }); + } + const result = pricingAware && quote ? await this.repository.completeWithPricing(command) : quotaAware && reservation ? await this.repository.completeWithQuota(command) : await this.repository.complete(command); return Object.freeze({ status: result.status, - ...(artifact && artifactRepository - ? { - reference: pluginPackagePromptOutputArtifactReference(artifact), - } - : {}), }); } catch (error) { const stored = await this.#completionAfterFailure( start, audit, error, - artifact, - artifactRepository, + extension, ); if (stored) { return Object.freeze({ status: 'existing' as const, - ...(artifact - ? { - reference: - pluginPackagePromptOutputArtifactReference(artifact), - } - : {}), + ...(extension ? { reference: stored.reference } : {}), }); } if ( @@ -569,31 +535,29 @@ export class DurableModelInvocationCoordinator } } - async #completionAfterFailure( + async #completionAfterFailure( start: Readonly, audit: Readonly, original: unknown, - artifact?: Readonly, - artifactRepository?: PluginPackagePromptOutputCompletionRepository, - ): Promise | null> { + extension?: ModelInvocationAtomicSuccess, + ): Promise< + Readonly<{ + completion: Readonly; + reference?: Readonly; + }> | null + > { try { const stored = await this.repository.findCompletion(audit.requestId); if (!stored) return null; const completion = assertCompletionMatchesAudit(stored, start, audit); - if (artifact) { - if (!artifactRepository) throw original; - const storedArtifact = - await artifactRepository.findPromptOutputArtifact( - artifact.artifactId, - ); - if ( - !storedArtifact || - JSON.stringify(storedArtifact) !== JSON.stringify(artifact) - ) { + if (extension) { + const reference = await extension.find(this.repository); + if (!reference || !extension.matches(reference)) { throw original; } + return Object.freeze({ completion, reference }); } - return completion; + return Object.freeze({ completion }); } catch { throw original; } diff --git a/packages/ql3-ai/src/model-invocation/modelInvocationAtomicSuccess.ts b/packages/ql3-ai/src/model-invocation/modelInvocationAtomicSuccess.ts new file mode 100644 index 00000000..45823e3e --- /dev/null +++ b/packages/ql3-ai/src/model-invocation/modelInvocationAtomicSuccess.ts @@ -0,0 +1,55 @@ +import type { + ModelInvocationAuditRecord, + ModelInvocationAuditDisposition, +} from '../model-gateway/model'; +import type { + ModelInvocationCompletionCommand, + ModelInvocationRepository, +} from './modelInvocation'; + +export interface ModelInvocationAtomicSuccessCommit { + readonly status: ModelInvocationAuditDisposition['status']; + readonly reference: Readonly; +} + +/** + * Domain-owned extension for one encrypted successful Model output. + * + * The generic Model coordinator owns the StepRun/usage/pricing protocol while + * this extension owns the output identity, exact replay check and dialect + * transaction that persists the encrypted output beside that protocol. + */ +export interface ModelInvocationAtomicSuccess { + readonly outputRef: string; + assertAudit(record: Readonly): void; + find( + repository: ModelInvocationRepository, + ): Promise | null>; + matches(reference: Readonly): boolean; + commit( + repository: ModelInvocationRepository, + command: Readonly, + ): Promise>>; + conflict(): Error; +} + +export function normalizeModelInvocationAtomicSuccess( + value: ModelInvocationAtomicSuccess, +): ModelInvocationAtomicSuccess { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + typeof value.outputRef !== 'string' || + value.outputRef.length < 1 || + value.outputRef.length > 512 || + typeof value.assertAudit !== 'function' || + typeof value.find !== 'function' || + typeof value.matches !== 'function' || + typeof value.commit !== 'function' || + typeof value.conflict !== 'function' + ) { + throw new TypeError('Model invocation atomic success is invalid'); + } + return value; +} diff --git a/packages/ql3-ai/src/model-invocation/postgres-model-invocation-repository/completionOperations.ts b/packages/ql3-ai/src/model-invocation/postgres-model-invocation-repository/completionOperations.ts index 08240b8f..d854c704 100644 --- a/packages/ql3-ai/src/model-invocation/postgres-model-invocation-repository/completionOperations.ts +++ b/packages/ql3-ai/src/model-invocation/postgres-model-invocation-repository/completionOperations.ts @@ -1,4 +1,4 @@ -import type { PostgresPool } from '@qinglong/runtime-core'; +import type { PostgresClient, PostgresPool } from '@qinglong/runtime-core'; import { createModelInvocationPriceSettlement } from '../../pricing/pricing'; import { type PluginPackagePromptOutputArtifact } from '../../prompt-output/pluginPackagePromptOutputArtifact'; @@ -315,17 +315,36 @@ export async function completeWithPricingOperation( }); } -export async function completeWithPromptOutputArtifactOperation( +export interface PostgresModelInvocationAtomicOutputBinding< + TArtifact, + TReference, +> { + readonly artifact: Readonly; + readonly reference: Readonly; + read(client: PostgresClient): Promise | null>; + put(client: PostgresClient): Promise>; + matches(stored: Readonly): boolean; +} + +export interface CommitPostgresModelInvocationAtomicOutputResult< + TArtifact, + TReference, +> extends CommitModelInvocationResult { + readonly artifact: Readonly; + readonly reference: Readonly; +} + +export async function completeWithAtomicOutputOperation( pool: PostgresPool, commandValue: ModelInvocationCompletionCommand, - artifactValue: PluginPackagePromptOutputArtifact, -): Promise> { + binding: PostgresModelInvocationAtomicOutputBinding, +): Promise< + Readonly< + CommitPostgresModelInvocationAtomicOutputResult + > +> { const command = normalizeModelInvocationCompletionCommand(commandValue); const completion = command.completion; - const binding = assertPluginPackagePromptOutputCompletionBinding( - command, - artifactValue, - ); const expectedUsage = createModelInvocationUsageLedgerRecord( command.start, completion, @@ -377,10 +396,7 @@ export async function completeWithPromptOutputArtifactOperation( if (existing[0]) { const [storedArtifact, usage, priceSettlements, quotaSettlements] = await Promise.all([ - readPostgresPluginPackagePromptOutputArtifactInTransaction( - client, - binding.artifact.artifactId, - ), + binding.read(client), usageRows(client, 'usage.invocation_id = $1', [ completion.invocationId, ]), @@ -395,7 +411,7 @@ export async function completeWithPromptOutputArtifactOperation( if ( JSON.stringify(stored) !== JSON.stringify(completion) || !storedArtifact || - JSON.stringify(storedArtifact) !== JSON.stringify(binding.artifact) || + !binding.matches(storedArtifact) || usage.length !== (expectedUsage ? 1 : 0) || (expectedUsage && JSON.stringify(parseUsage(usage[0]!)) !== @@ -434,12 +450,6 @@ export async function completeWithPromptOutputArtifactOperation( throw new ModelInvocationConflictError(); } await assertCurrent(client, command.stepRunMutation, completion.projectId); - const artifact = ( - await putPostgresPluginPackagePromptOutputArtifactInTransaction( - client, - binding.artifact, - ) - ).artifact; await applyMutation(client, command.stepRunMutation); await insertCompletion(client, completion); if (expectedUsage) await insertUsage(client, expectedUsage); @@ -449,6 +459,7 @@ export async function completeWithPromptOutputArtifactOperation( if (expectedQuotaSettlement) { await insertQuotaSettlement(client, expectedQuotaSettlement); } + const artifact = await binding.put(client); return Object.freeze({ status: 'created' as const, record: completion, @@ -457,3 +468,33 @@ export async function completeWithPromptOutputArtifactOperation( }); }); } + +export async function completeWithPromptOutputArtifactOperation( + pool: PostgresPool, + commandValue: ModelInvocationCompletionCommand, + artifactValue: PluginPackagePromptOutputArtifact, +): Promise> { + const command = normalizeModelInvocationCompletionCommand(commandValue); + const binding = assertPluginPackagePromptOutputCompletionBinding( + command, + artifactValue, + ); + return completeWithAtomicOutputOperation(pool, command, { + artifact: binding.artifact, + reference: binding.reference, + read: (client) => + readPostgresPluginPackagePromptOutputArtifactInTransaction( + client, + binding.artifact.artifactId, + ), + put: async (client) => + ( + await putPostgresPluginPackagePromptOutputArtifactInTransaction( + client, + binding.artifact, + ) + ).artifact, + matches: (stored) => + JSON.stringify(stored) === JSON.stringify(binding.artifact), + }); +} diff --git a/packages/ql3-ai/src/prompt-output/pluginPackagePromptOutputCompletion.ts b/packages/ql3-ai/src/prompt-output/pluginPackagePromptOutputCompletion.ts index 94480548..899efab3 100644 --- a/packages/ql3-ai/src/prompt-output/pluginPackagePromptOutputCompletion.ts +++ b/packages/ql3-ai/src/prompt-output/pluginPackagePromptOutputCompletion.ts @@ -9,6 +9,7 @@ import type { } from '../model-gateway/model'; import type { ModelInvocationSuccessfulCompletionSink } from '../model-gateway/gateway'; import type { DurableModelInvocationCoordinator } from '../model-invocation/durableModelInvocationCoordinator'; +import type { ModelInvocationAtomicSuccess } from '../model-invocation/modelInvocationAtomicSuccess'; import { normalizePluginPackagePromptExecutionPlan, type PluginPackagePromptExecutionPlan, @@ -113,6 +114,61 @@ export function assertPluginPackagePromptOutputCompletionBinding( }); } +function pluginPackagePromptOutputAtomicSuccess( + artifactValue: PluginPackagePromptOutputArtifact, +): ModelInvocationAtomicSuccess { + const artifact = normalizePluginPackagePromptOutputArtifact(artifactValue); + const reference = pluginPackagePromptOutputArtifactReference(artifact); + const conflict = (): Error => + new PluginPackagePromptOutputArtifactConflictError(); + const extension: ModelInvocationAtomicSuccess = { + outputRef: artifact.artifactId, + assertAudit(audit: Readonly): void { + if ( + audit.phase !== 'completed' || + audit.requestId !== artifact.invocationId || + audit.projectId !== artifact.projectId || + audit.runId !== artifact.runId || + audit.stepRunId !== artifact.stepRunId || + audit.provider !== artifact.provider || + audit.model !== artifact.model || + audit.outputBytes !== artifact.outputBytes + ) { + throw conflict(); + } + }, + async find(repository) { + if (!isPluginPackagePromptOutputCompletionRepository(repository)) { + throw conflict(); + } + const stored = await repository.findPromptOutputArtifact( + artifact.artifactId, + ); + if (!stored) return null; + if (JSON.stringify(stored) !== JSON.stringify(artifact)) throw conflict(); + return pluginPackagePromptOutputArtifactReference(stored); + }, + matches(stored): boolean { + return JSON.stringify(stored) === JSON.stringify(reference); + }, + async commit(repository, command) { + if (!isPluginPackagePromptOutputCompletionRepository(repository)) { + throw conflict(); + } + const result = await repository.completeWithPromptOutputArtifact( + command, + artifact, + ); + return Object.freeze({ + status: result.status, + reference: result.reference, + }); + }, + conflict, + }; + return Object.freeze(extension); +} + interface ActiveCompletion { readonly lease: Readonly; readonly plan: Readonly; @@ -149,7 +205,7 @@ export class PluginPackagePromptOutputCompletionCoordinator typeof options !== 'object' || Array.isArray(options) || !options.coordinator || - typeof options.coordinator.recordWithPromptOutputArtifact !== + typeof options.coordinator.recordWithAtomicSuccess !== 'function' || !options.keys || typeof options.keys.active !== 'function' || @@ -252,7 +308,10 @@ export class PluginPackagePromptOutputCompletionCoordinator this.#nonceFactory, ); const disposition = - await this.#coordinator.recordWithPromptOutputArtifact(audit, artifact); + await this.#coordinator.recordWithAtomicSuccess( + audit, + pluginPackagePromptOutputAtomicSuccess(artifact), + ); active.reference = disposition.reference; return Object.freeze({ handled: true as const, diff --git a/packages/ql3-ai/test/failureDiagnosisExecutionAdmission.test.cjs b/packages/ql3-ai/test/failureDiagnosisExecutionAdmission.test.cjs index c4a033fe..995155e6 100644 --- a/packages/ql3-ai/test/failureDiagnosisExecutionAdmission.test.cjs +++ b/packages/ql3-ai/test/failureDiagnosisExecutionAdmission.test.cjs @@ -40,6 +40,13 @@ const { normalizeCopilotFailureDiagnosisToolUnlockCommand, restoreCopilotFailureDiagnosisTrustedToolAuthority, } = require('../dist/copilot/failure-diagnosis/toolExecution.js'); +const { + CopilotFailureDiagnosisOutputArtifactUnavailableError, + copilotFailureDiagnosisOutputReference, + createCopilotFailureDiagnosisFinalizationReceipt, + createCopilotFailureDiagnosisOutputArtifact, + openCopilotFailureDiagnosisOutputArtifact, +} = require('../dist/copilot/failure-diagnosis/modelExecution.js'); const DIGEST_A = 'a'.repeat(64); const DIGEST_B = 'b'.repeat(64); @@ -425,3 +432,373 @@ test('fails closed on widened or digest-drifted durable plans', async () => { InvalidCopilotFailureDiagnosisExecutionPlanError, ); }); + +test('encrypts one Copilot diagnosis output and exposes only a content-free reference', async () => { + const current = await plan(); + const prompt = require('../dist/copilot/failure-diagnosis/prompt.js') + .buildFailureDiagnosisPromptPlan({ + provider: current.model.provider, + model: current.model.model, + modelBoundary: current.model.modelBoundary, + profile: 'cluster-control', + responseLanguage: current.model.responseLanguage, + projection: { + content: 'failure: connection refused', + sourceBytes: 27, + modelTextBytes: 27, + redaction: { + contract: 'recognized_credentials_v1', + residualSensitivity: 'potentially_sensitive', + replacements: 0, + categories: [], + }, + normalization: { + invalidUtf8: false, + unsafeCodePointsReplaced: 0, + }, + trust: { + classification: 'untrusted_execution_output', + instructionPolicy: 'data_only_never_execute', + actionAuthority: 'none', + suspectedPromptInjection: false, + signals: [], + }, + }, + maxOutputTokens: current.model.maxOutputTokens, + egressPolicy: current.model.egressPolicy, + }); + const result = { + provider: current.model.provider, + model: current.model.model, + text: 'Likely a refused upstream connection.', + finishReason: 'stop', + usage: { inputTokens: 120, outputTokens: 8, totalTokens: 128 }, + }; + const artifact = createCopilotFailureDiagnosisOutputArtifact( + { + requestId: current.requestId, + planDigest: current.planDigest, + toolCompletionDigest: DIGEST_A, + projectId: current.projectId, + runId: current.runId, + stepRunId: current.modelStepRunId, + invocationId: current.modelInvocationId, + result, + egressEvidence: prompt.egressEvidence, + keyId: 'copilot-output-key-1', + key: Buffer.alloc(32, 0x61), + sealedAtMs: 4_000, + }, + () => Buffer.alloc(12, 0x62), + ); + assert.equal(JSON.stringify(artifact).includes(result.text), false); + assert.deepEqual( + openCopilotFailureDiagnosisOutputArtifact( + artifact, + Buffer.alloc(32, 0x61), + ), + result, + ); + const reference = copilotFailureDiagnosisOutputReference(artifact); + assert.equal(reference.artifactId, artifact.artifactId); + assert.equal(JSON.stringify(reference).includes('ciphertext'), false); + assert.equal(JSON.stringify(reference).includes(result.text), false); + assert.throws( + () => + openCopilotFailureDiagnosisOutputArtifact( + { ...artifact, ciphertext: `${artifact.ciphertext.slice(0, -1)}A` }, + Buffer.alloc(32, 0x61), + ), + TypeError, + ); + assert.throws( + () => + openCopilotFailureDiagnosisOutputArtifact( + artifact, + Buffer.alloc(32, 0x63), + ), + CopilotFailureDiagnosisOutputArtifactUnavailableError, + ); +}); + +test('binds a content-free diagnosis Run finalization receipt', async () => { + const current = await plan(); + const receipt = createCopilotFailureDiagnosisFinalizationReceipt({ + requestId: current.requestId, + planDigest: current.planDigest, + runId: current.runId, + modelStepRunId: current.modelStepRunId, + invocationId: current.modelInvocationId, + completionDigest: DIGEST_B, + outcome: 'succeeded', + outputArtifactId: 'cdo:diagnosis-output', + finalRunVersion: 9, + finalRunEventSequence: 9, + finalizedAtMs: 4_100, + }); + assert.equal(receipt.runEventId.length, 36); + assert.equal(receipt.receiptDigest.length, 64); + assert.equal(JSON.stringify(receipt).includes('model output'), false); + assert.throws( + () => + createCopilotFailureDiagnosisFinalizationReceipt({ + ...receipt, + outcome: 'failed', + }), + /finalization conflicts/, + ); +}); + +test('publishes Model execution through explicit AI subpaths only', () => { + const root = require('../dist'); + const execution = require('@qinglong/ai/failure-diagnosis-model-execution'); + const storage = require('@qinglong/ai/postgres-failure-diagnosis-model-execution-storage'); + assert.equal(root.executeCopilotFailureDiagnosisModel, undefined); + assert.equal( + typeof execution.executeCopilotFailureDiagnosisModel, + 'function', + ); + assert.equal( + typeof storage.PostgresCopilotFailureDiagnosisModelRepository, + 'function', + ); +}); + +test('executes the unlocked Model once, commits ciphertext, and terminalizes replay', async () => { + const current = await plan(); + const admission = createCopilotFailureDiagnosisAdmissionBundle(current); + const unlockCommand = createCopilotFailureDiagnosisToolUnlockCommand({ + plan: current, + completion: successfulToolCompletion(current), + modelStepRun: admission.modelStepMutation.stepRun, + run: { + id: current.runId, + projectId: current.projectId, + status: 'running', + version: 5, + eventSequence: 5, + }, + }); + const { + DurableModelInvocationCoordinator, + } = require('../dist/model-invocation/durableModelInvocationCoordinator.js'); + const { BoundedModelGateway } = require('../dist/model-gateway/gateway.js'); + const { + CopilotFailureDiagnosisModelCompletionCoordinator, + assertCopilotFailureDiagnosisOutputCompletionBinding, + copilotFailureDiagnosisOutputReference, + executeCopilotFailureDiagnosisModel, + } = require('../dist/copilot/failure-diagnosis/modelExecution.js'); + + let stepRun = unlockCommand.modelStepRunMutation.stepRun; + let runVersion = unlockCommand.receipt.finalRunVersion; + let runEventSequence = unlockCommand.receipt.finalRunEventSequence; + let start = null; + let completion = null; + let outputArtifact = null; + let finalization = null; + const repository = { + async findStart() { + return start; + }, + async findCompletion() { + return completion; + }, + async readAuthority() { + return { + projectId: current.projectId, + runId: current.runId, + runVersion, + runEventSequence, + stepRun, + }; + }, + async listIncomplete() { + return { observedAtMs: 3_000, candidates: [], hasMore: false }; + }, + async admit(command) { + start = command.start; + stepRun = command.stepRunMutation.stepRun; + runVersion += 1; + runEventSequence += 1; + return { status: 'created', record: start }; + }, + async complete(command) { + completion = command.completion; + stepRun = command.stepRunMutation.stepRun; + runVersion += 1; + runEventSequence += 1; + return { status: 'created', record: completion }; + }, + async findCopilotFailureDiagnosisOutput() { + return outputArtifact; + }, + async completeWithCopilotFailureDiagnosisOutput(command, artifact) { + const binding = assertCopilotFailureDiagnosisOutputCompletionBinding( + command, + artifact, + ); + completion = command.completion; + outputArtifact = binding.artifact; + stepRun = command.stepRunMutation.stepRun; + runVersion += 1; + runEventSequence += 1; + return { status: 'created', reference: binding.reference }; + }, + }; + const durable = new DurableModelInvocationCoordinator(repository); + const successfulCompletion = + new CopilotFailureDiagnosisModelCompletionCoordinator({ + coordinator: durable, + keys: { + async active() { + return { keyId: 'copilot-output-key-1', key: Buffer.alloc(32, 7) }; + }, + async resolve(keyId) { + return keyId === 'copilot-output-key-1' + ? { keyId, key: Buffer.alloc(32, 7) } + : null; + }, + }, + now: () => 3_500, + nonceFactory: () => Buffer.alloc(12, 8), + }); + let providerCalls = 0; + const gateway = new BoundedModelGateway({ + providers: [ + { + type: current.model.provider, + async listModels() { + return [{ id: current.model.model }]; + }, + async generate() { + providerCalls += 1; + return { + provider: current.model.provider, + model: current.model.model, + text: 'The upstream service refused the connection.', + finishReason: 'stop', + usage: { inputTokens: 100, outputTokens: 9, totalTokens: 109 }, + }; + }, + async *stream() {}, + }, + ], + policies: { + async resolve() { + return { + revision: 'model-policy-1', + allowedProviders: [current.model.provider], + allowedModels: [current.model.model], + maxInputBytes: 64 * 1024, + maxOutputBytes: 64 * 1024, + maxOutputTokens: 1_024, + maxTotalTokens: 2_048, + maxCostMicros: null, + priceRevision: null, + }; + }, + }, + pricing: { async resolve() { return null; } }, + audit: durable, + successfulCompletion, + maxConcurrent: 1, + now: () => 3_100, + }); + const finalizations = { + async findFinalization() { + return finalization; + }, + async finalize() { + if (!finalization) { + finalization = createCopilotFailureDiagnosisFinalizationReceipt({ + requestId: current.requestId, + planDigest: current.planDigest, + runId: current.runId, + modelStepRunId: current.modelStepRunId, + invocationId: current.modelInvocationId, + completionDigest: completion.completionDigest, + outcome: completion.outcome, + outputArtifactId: outputArtifact.artifactId, + finalRunVersion: runVersion + 1, + finalRunEventSequence: runEventSequence + 1, + finalizedAtMs: completion.completedAtMs, + }); + return { status: 'created', receipt: finalization }; + } + return { status: 'existing', receipt: finalization }; + }, + }; + const dependencies = { + admissions: { + async findPlanByRequestId() { + return current; + }, + async findByRequestId() { + return admission.receipt; + }, + }, + unlocks: { + async findByRequestId() { + return unlockCommand.receipt; + }, + }, + toolResults: { + async open() { + return { + status: 'existing', + completion: successfulToolCompletion(current), + output: { + status: 'available', + runId: current.source.runId, + attemptId: current.source.attemptId, + profile: 'cluster-control', + sourceWindowBytes: 16 * 1024, + content: 'connect ECONNREFUSED 127.0.0.1:5432', + sourceBytes: 35, + modelTextBytes: 35, + redaction: { + contract: 'recognized_credentials_v1', + residualSensitivity: 'potentially_sensitive', + replacements: 0, + categories: [], + }, + normalization: { + invalidUtf8: false, + unsafeCodePointsReplaced: 0, + }, + trust: { + classification: 'untrusted_execution_output', + instructionPolicy: 'data_only_never_execute', + actionAuthority: 'none', + suspectedPromptInjection: false, + signals: [], + }, + }, + }; + }, + }, + modelInvocations: repository, + outputs: repository, + gateway, + successfulCompletion, + finalizations, + }; + const created = await executeCopilotFailureDiagnosisModel( + current.requestId, + dependencies, + ); + assert.equal(created.outcome, 'succeeded'); + assert.equal(providerCalls, 1); + assert.equal(JSON.stringify(outputArtifact).includes('refused'), false); + assert.deepEqual( + created.output, + copilotFailureDiagnosisOutputReference(outputArtifact), + ); + const replay = await executeCopilotFailureDiagnosisModel( + current.requestId, + dependencies, + ); + assert.equal(replay.finalization.receiptDigest, created.finalization.receiptDigest); + assert.equal(providerCalls, 1); +}); diff --git a/packages/ql3-ai/test/modelInvocationMigration.test.cjs b/packages/ql3-ai/test/modelInvocationMigration.test.cjs index 54c024dc..0db474e8 100644 --- a/packages/ql3-ai/test/modelInvocationMigration.test.cjs +++ b/packages/ql3-ai/test/modelInvocationMigration.test.cjs @@ -21,6 +21,7 @@ const { LOCAL_MODEL_PRICE_CATALOG_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE, POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID, @@ -431,6 +432,10 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async () POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, 'pg-9019-ai-copilot-failure-diagnosis-tool-unlocks', ); + assert.equal( + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, + 'pg-9020-ai-copilot-failure-diagnosis-model-executions', + ); assert.equal( POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE, 'ai_schema_migrations', @@ -518,7 +523,7 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async () ); assert.equal( postgresModelInvocationMigrationDefinition.migrations.length, - 19, + 20, ); const diagnosisAdmissionStatements = []; @@ -565,6 +570,33 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async () assert.match(diagnosisToolUnlockSql, /TO ql3_runtime/); assert.doesNotMatch(diagnosisToolUnlockSql, /GRANT[^;]*(?:UPDATE|DELETE)/); + const diagnosisModelExecutionStatements = []; + await postgresModelInvocationMigrationDefinition.migrations[19].up({ + async query(statement) { + diagnosisModelExecutionStatements.push(statement); + return { rows: [] }; + }, + }); + const diagnosisModelExecutionSql = + diagnosisModelExecutionStatements.join('\n'); + assert.match( + diagnosisModelExecutionSql, + /CREATE TABLE "ql3_ai"\."copilot_failure_diagnosis_model_outputs"/, + ); + assert.match( + diagnosisModelExecutionSql, + /CREATE TABLE "ql3_ai"\."copilot_failure_diagnosis_finalizations"/, + ); + assert.match( + diagnosisModelExecutionSql, + /FOREIGN KEY \(invocation_id\)[\s\S]*model_invocation_completions/, + ); + assert.match(diagnosisModelExecutionSql, /TO ql3_runtime/); + assert.doesNotMatch( + diagnosisModelExecutionSql, + /GRANT[^;]*(?:UPDATE|DELETE)/, + ); + const retirementStatements = []; await postgresModelInvocationMigrationDefinition.migrations[10].up({ async query(statement) { diff --git a/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs b/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs index eabe1595..0179936b 100644 --- a/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs +++ b/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs @@ -175,7 +175,7 @@ test('readiness binds exact migration history and least-privilege primary author assert.equal(report.ready, true); assert.equal( report.migrationIds.at(-1), - 'pg-9019-ai-copilot-failure-diagnosis-tool-unlocks', + 'pg-9020-ai-copilot-failure-diagnosis-model-executions', ); assert.match( queries[1], diff --git a/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs b/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs index 78a998de..bfd36896 100644 --- a/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs +++ b/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs @@ -277,7 +277,7 @@ test('tester readiness freezes migration history and least privilege', async () assert.equal(ready.ready, true); assert.equal( ready.migrationIds.at(-1), - 'pg-9019-ai-copilot-failure-diagnosis-tool-unlocks', + 'pg-9020-ai-copilot-failure-diagnosis-model-executions', ); await assert.rejects( diff --git a/packages/ql3-runtime-core/src/tool-execution/trustedToolCompletion.ts b/packages/ql3-runtime-core/src/tool-execution/trustedToolCompletion.ts index eff3fdb2..3ff9e80f 100644 --- a/packages/ql3-runtime-core/src/tool-execution/trustedToolCompletion.ts +++ b/packages/ql3-runtime-core/src/tool-execution/trustedToolCompletion.ts @@ -37,6 +37,12 @@ import { } from './trustedToolExecution'; import type { ToolJsonValue } from './tool-registry/toolRegistry'; +export { + openTrustedToolSuccessCompletion, + type TrustedToolSuccessCompletionReadDependencies, + type TrustedToolSuccessCompletionResult, +} from './trustedToolSuccessCompletion'; + export interface TrustedToolFailureCompletionIdentities { readonly mutationId: string; readonly eventId: string; diff --git a/packages/ql3-runtime-core/src/tool-execution/trustedToolSuccessCompletion.ts b/packages/ql3-runtime-core/src/tool-execution/trustedToolSuccessCompletion.ts index 79e0f532..6af58926 100644 --- a/packages/ql3-runtime-core/src/tool-execution/trustedToolSuccessCompletion.ts +++ b/packages/ql3-runtime-core/src/tool-execution/trustedToolSuccessCompletion.ts @@ -60,15 +60,24 @@ export interface TrustedToolSuccessCompletionIdentityFactory { } export interface TrustedToolSuccessCompletionDependencies - extends TrustedToolExecutionDependencies { - readonly completions: ToolExecutionCompletionRepository; + extends TrustedToolExecutionDependencies, + TrustedToolSuccessCompletionReadDependencies { readonly stepRuns: Pick; readonly runs: Pick; + readonly identities: TrustedToolSuccessCompletionIdentityFactory; + readonly nonceFactory?: () => Uint8Array; +} + +export interface TrustedToolSuccessCompletionReadDependencies { + readonly completions: ToolExecutionCompletionRepository; + readonly barriers: Pick< + TrustedToolExecutionDependencies['barriers'], + 'findByStartId' + >; readonly resultKeyCatalog: ToolResultKeyCatalogReader; readonly resultRekeys: ToolExecutionResultRekeyReader; readonly resultKeys: Pick; - readonly identities: TrustedToolSuccessCompletionIdentityFactory; - readonly nonceFactory?: () => Uint8Array; + readonly adapters: TrustedToolExecutionAdapterRegistry; } export interface TrustedToolSuccessCompletionResult { @@ -101,26 +110,15 @@ function sameValue(left: unknown, right: unknown): boolean { function validateDependencies( dependencies: TrustedToolSuccessCompletionDependencies, ): void { + validateReadDependencies(dependencies); if ( - !dependencies || - typeof dependencies !== 'object' || - !dependencies.completions || - typeof dependencies.completions.findByStartId !== 'function' || - typeof dependencies.completions.findResultArtifact !== 'function' || typeof dependencies.completions.commit !== 'function' || !dependencies.stepRuns || typeof dependencies.stepRuns.findById !== 'function' || !dependencies.runs || typeof dependencies.runs.findRunById !== 'function' || - !dependencies.resultKeys || - typeof dependencies.resultKeys.resolve !== 'function' || - !dependencies.resultKeyCatalog || - typeof dependencies.resultKeyCatalog.findCurrent !== 'function' || - !dependencies.resultRekeys || - typeof dependencies.resultRekeys.findHeadByArtifactId !== 'function' || !dependencies.identities || typeof dependencies.identities.create !== 'function' || - !(dependencies.adapters instanceof TrustedToolExecutionAdapterRegistry) || (dependencies.nonceFactory !== undefined && typeof dependencies.nonceFactory !== 'function') ) { @@ -128,9 +126,32 @@ function validateDependencies( } } +function validateReadDependencies( + dependencies: TrustedToolSuccessCompletionReadDependencies, +): void { + if ( + !dependencies || + typeof dependencies !== 'object' || + !dependencies.completions || + typeof dependencies.completions.findByStartId !== 'function' || + typeof dependencies.completions.findResultArtifact !== 'function' || + !dependencies.barriers || + typeof dependencies.barriers.findByStartId !== 'function' || + !dependencies.resultKeys || + typeof dependencies.resultKeys.resolve !== 'function' || + !dependencies.resultKeyCatalog || + typeof dependencies.resultKeyCatalog.findCurrent !== 'function' || + !dependencies.resultRekeys || + typeof dependencies.resultRekeys.findHeadByArtifactId !== 'function' || + !(dependencies.adapters instanceof TrustedToolExecutionAdapterRegistry) + ) { + unavailable(); + } +} + async function findResultRekeyHead( artifactId: string, - dependencies: TrustedToolSuccessCompletionDependencies, + dependencies: TrustedToolSuccessCompletionReadDependencies, ): Promise | null> { try { const value = await dependencies.resultRekeys.findHeadByArtifactId( @@ -145,7 +166,7 @@ async function findResultRekeyHead( } async function findResultKeyCatalog( - dependencies: TrustedToolSuccessCompletionDependencies, + dependencies: TrustedToolSuccessCompletionReadDependencies, ): Promise> { try { const value = await dependencies.resultKeyCatalog.findCurrent(); @@ -169,7 +190,7 @@ function validCatalogMaterial( async function findCompletion( startId: string, - dependencies: TrustedToolSuccessCompletionDependencies, + dependencies: TrustedToolSuccessCompletionReadDependencies, ): Promise | null> { try { const value = await dependencies.completions.findByStartId(startId); @@ -183,7 +204,7 @@ async function findCompletion( async function findBarrier( startId: string, - dependencies: TrustedToolSuccessCompletionDependencies, + dependencies: TrustedToolSuccessCompletionReadDependencies, ): Promise> { try { const value = await dependencies.barriers.findByStartId(startId); @@ -245,7 +266,7 @@ function completionMatches( async function openDurableCompletion( completion: Readonly, - dependencies: TrustedToolSuccessCompletionDependencies, + dependencies: TrustedToolSuccessCompletionReadDependencies, ): Promise> { const barrier = await findBarrier(completion.startId, dependencies); let artifact: Readonly; @@ -307,6 +328,17 @@ async function openDurableCompletion( } } +/** Reopens one exact encrypted Tool success without executing the Tool. */ +export async function openTrustedToolSuccessCompletion( + startId: string, + dependencies: TrustedToolSuccessCompletionReadDependencies, +): Promise> { + validateReadDependencies(dependencies); + const completion = await findCompletion(startId, dependencies); + if (!completion) return unavailable(); + return openDurableCompletion(completion, dependencies); +} + async function findStepRun( barrier: Readonly, dependencies: TrustedToolSuccessCompletionDependencies, @@ -351,7 +383,7 @@ export async function executeAndCompleteTrustedToolSuccess( validateDependencies(dependencies); const existing = await findCompletion(startId, dependencies); - if (existing) return openDurableCompletion(existing, dependencies); + if (existing) return openTrustedToolSuccessCompletion(startId, dependencies); const executionResult = await executeTrustedToolAfterStart( startId, @@ -359,7 +391,9 @@ export async function executeAndCompleteTrustedToolSuccess( ); const concurrent = await findCompletion(startId, dependencies); - if (concurrent) return openDurableCompletion(concurrent, dependencies); + if (concurrent) { + return openTrustedToolSuccessCompletion(startId, dependencies); + } const barrier = await findBarrier(startId, dependencies); const adapter = dependencies.adapters.resolve(barrier); @@ -469,7 +503,9 @@ export async function executeAndCompleteTrustedToolSuccess( }); } catch (cause) { const recovered = await findCompletion(startId, dependencies); - if (recovered) return openDurableCompletion(recovered, dependencies); + if (recovered) { + return openTrustedToolSuccessCompletion(startId, dependencies); + } throw cause; } } diff --git a/scripts/ql3-postgres-ha-contract.cjs b/scripts/ql3-postgres-ha-contract.cjs index 006c5e83..eaa15458 100644 --- a/scripts/ql3-postgres-ha-contract.cjs +++ b/scripts/ql3-postgres-ha-contract.cjs @@ -41,6 +41,7 @@ const { const { POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_PRICING_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_QUOTA_MIGRATION_ID, @@ -1504,6 +1505,35 @@ async function modelInvocationFeatureFacts(pool) { 'SELECT,INSERT,UPDATE,DELETE' ) ) AS "copilotDiagnosisToolUnlockRuntimeOnly", + has_table_privilege( + 'ql3_runtime', + 'ql3_ai.copilot_failure_diagnosis_model_outputs', 'SELECT,INSERT' + ) AND NOT has_table_privilege( + 'ql3_runtime', + 'ql3_ai.copilot_failure_diagnosis_model_outputs', 'UPDATE,DELETE' + ) AND has_table_privilege( + 'ql3_runtime', + 'ql3_ai.copilot_failure_diagnosis_finalizations', 'SELECT,INSERT' + ) AND NOT has_table_privilege( + 'ql3_runtime', + 'ql3_ai.copilot_failure_diagnosis_finalizations', 'UPDATE,DELETE' + ) AND NOT EXISTS ( + SELECT 1 + FROM unnest(ARRAY[ + 'ql3_admin', 'ql3_package_manager', 'ql3_package_executor', + 'ql3_worker_ingress', 'ql3_ai_maintenance', + 'ql3_ai_credential_manager', 'ql3_ai_credential_tester' + ]::text[]) AS denied(role_name) + WHERE has_table_privilege( + denied.role_name, + 'ql3_ai.copilot_failure_diagnosis_model_outputs', + 'SELECT,INSERT,UPDATE,DELETE' + ) OR has_table_privilege( + denied.role_name, + 'ql3_ai.copilot_failure_diagnosis_finalizations', + 'SELECT,INSERT,UPDATE,DELETE' + ) + ) AS "copilotDiagnosisModelExecutionRuntimeOnly", has_function_privilege( 'ql3_runtime', 'ql3_ai.copilot_failure_diagnosis_admission_source_snapshot(varchar,varchar,varchar,integer,integer,varchar,varchar)', @@ -10666,6 +10696,8 @@ async function main(argv = process.argv.slice(2)) { ); assert.deepEqual(beforePromotion.tables, [ 'copilot_failure_diagnosis_admissions', + 'copilot_failure_diagnosis_finalizations', + 'copilot_failure_diagnosis_model_outputs', 'copilot_failure_diagnosis_tool_unlocks', 'model_invocation_completions', 'model_invocation_price_quotes', @@ -10718,6 +10750,7 @@ async function main(argv = process.argv.slice(2)) { POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, ], ); assert.deepEqual(beforePromotion.privileges, { @@ -10743,6 +10776,7 @@ async function main(argv = process.argv.slice(2)) { promptSnapshotRuntimeOnly: true, copilotDiagnosisAdmissionRuntimeOnly: true, copilotDiagnosisToolUnlockRuntimeOnly: true, + copilotDiagnosisModelExecutionRuntimeOnly: true, copilotDiagnosisSnapshotRuntimeOnly: true, migrationHistoryRuntimeReadOnly: true, modelProviderCredentialManagementAuthoritySplit: true, diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index c13099f4..e416597e 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -268,10 +268,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( { path: 'packages/ql3-ai', name: '@qinglong/ai', - sourceFiles: 175, + sourceFiles: 183, rootSourceFiles: 1, rootSourceLines: 16, - nestedSourceFiles: 174, + nestedSourceFiles: 182, rootSourceFileHardCap: 1, rootSourceLineHardCap: 16, rootSourceFileRoles: {