From 0a31c3364b8f4f1fd6f5a46cfb633f6f89922bd2 Mon Sep 17 00:00:00 2001 From: whyour Date: Sat, 15 Aug 2026 23:16:26 +0800 Subject: [PATCH] feat(ql3): terminalize copilot diagnosis failures --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 1 + ...-terminalization-and-unknown-resolution.md | 59 ++ docs/adr/README.md | 1 + packages/ql3-ai/package.json | 5 + .../application/contracts.ts | 4 +- .../failure-diagnosis/application/service.ts | 191 ++++-- .../model-execution/postgresRepository.ts | 141 +++-- .../preModelTerminalization.ts | 4 + .../terminalization/contracts.ts | 160 +++++ .../terminalization/coordinator.ts | 405 ++++++++++++ .../terminalization/postgresRepository.ts | 585 ++++++++++++++++++ .../terminalization/protocol.ts | 471 ++++++++++++++ .../tool-execution/contracts.ts | 4 + .../tool-execution/coordinator.ts | 2 + .../migration/model-invocation/identities.ts | 2 + .../migration/model-invocation/postgres.ts | 2 + .../model-invocation/postgres/copilot.ts | 97 +++ .../failure-diagnosis-application.test.cjs | 159 ++++- ...agnosis-pre-model-terminalization.test.cjs | 442 +++++++++++++ .../test/modelInvocationMigration.test.cjs | 30 +- ...redentialManagementIdentityLedger.test.cjs | 2 +- ...elProviderCredentialTestExecution.test.cjs | 2 +- .../copilot/failureDiagnosisComposition.ts | 46 +- scripts/ql3-cluster-dependency-audit.cjs | 1 + scripts/ql3-postgres-ha-contract.cjs | 524 +++++++++++++++- test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 26 files changed, 3217 insertions(+), 127 deletions(-) create mode 100644 docs/adr/ADR-0412-durable-copilot-pre-model-terminalization-and-unknown-resolution.md create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/preModelTerminalization.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/contracts.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/coordinator.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/postgresRepository.ts create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/protocol.ts create mode 100644 packages/ql3-ai/test/failure-diagnosis-pre-model-terminalization.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 29c6faed..842c15d9 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,7 @@ 最新增量证据（2026-08-15）： +- D-320/ADR-0412（已接受）：Cluster Copilot failure diagnosis 已补齐产品入口前的非成功收敛边界。新增的 pre-Model terminalizer 只从 durable Tool failure、受审日志 projection 或数据库观察到的 deadline/cancellation 派生封闭 reason，在 Model start 不存在时以一个 SERIALIZABLE 事务原子提交 StepRun mutations、RunEvents、父 Run 终态和 `pg-9021` append-only content-free receipt；Tool `failed|timed_out`、日志 `not_found|pending|missing|retired`、五秒 Tool budget 不足、deadline 和 cancellation 均可 exact replay。Model 已开始后的 `outcome_unknown` 继续要求强 User 显式 `fail|cancel|retry` resolution；Copilot finalizer 现在对 `fail|cancel` 精确校验 resolution mutation 的 resolved Step digest，既不伪造失败也不自动重试 Provider。实现复用既有 AI package、Cluster AI 进程、Pool 与 ledger，不增加 package、进程、连接、timer、watcher、队列、cache 或产品 route；workspace 仍为 18 package、无单文件/浅平 package，AI 192 个源码中 191 个位于嵌套领域目录。AI 完整测试 244 pass/3 条件 skip，18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计和 14 档 Local artifact 全部通过。PostgreSQL 18.6 arm64 HA 137/137、timeline `1→2`，覆盖日志不可用 pre-Model terminalization、unknown completion→人工 fail resolution→Run finalization 及晋升后零外部副作用 exact replay；报告 SHA-256 为 `6eaeb20615a62d153c5a69687344f41f31351c6ecf111cfb9cbafad115538c83`，离线审计零 finding。下一 Gate 才增加认证、Policy、audit 与 source fence 保护的 Cluster API，并由 CLI/UI/MCP 复用，仍不得建立旁路执行器。 - D-319/ADR-0411（已接受）：Cluster Copilot failure diagnosis 已在既有 `ql3-cluster-control-ai` 进程内完成默认关闭的 production composition。Prompt 与 Copilot 共享同一个 PostgreSQL AI Pool、Model Gateway、Provider client、恢复扫描、quota/pricing ledger 与 `maxConcurrent` 预算；有界 successful-completion router 只向声明 exact invocation 的 durable sink 分发，不复制 Gateway、连接或隐藏队列。application service 只接受 source Run/request identity，从数据库当前 Run、latest Attempt、Project Tool snapshot、Policy 和 canonical read-only egress config 派生计划，先 admission，再以 durable plan、historical key 与确定性 nonce 修复 admission→Artifact crash window，随后复用 Trusted Tool、Worker Artifact range reader、Model Gateway 及独立的 invocation/result/model-output 三域 keyring。能力保持 caller-driven，不增加 timer、watcher、队列、HTTP/CLI/UI/MCP route 或 Kubernetes API 权限；Kubernetes 独立可选 component 仅投影 config 与三个 0440 keyring。最终 AI 238 pass/3 条件 skip、Cluster Control 240 pass/2 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计、14 档 Local artifact 全部通过；workspace 仍为 18 package、无单文件/浅平 package，AI 187 个源码中 186 个、Cluster Control 59 个中 57 个位于嵌套领域目录。默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes，Edge/Standalone AI 为 3,064,454/3,064,544 bytes，证明 Cluster-only composition 未进入小设备闭包。PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`，报告 SHA-256 为 `981299b454dce5541e9596450b85816dc40559cba8dc42adf3d5fea571c3d3a6`；本 Gate 不改 migration/schema/role/SQL/HA 拓扑。下一 Gate 是 Tool failure、日志 missing/retired/pending、deadline/cancel 与 Model outcome-unknown 的 durable terminalization/recovery，完成前不开放产品入口。 - D-318/ADR-0410（已接受）：Cluster Copilot diagnosis Model output 获得独立的只读 projected key authority。canonical `qinglong/copilot-failure-diagnosis-output-projected-keyring@v1` manifest 只允许一个 active key 与最多 16 个 historical 32-byte key；每次 `active()`/`resolve()` 都重新执行 Cluster 私有投影文件的 direct-root、根内 atomic symlink、single-link、mode、dev/inode/size/mtime 与双 realpath fence，不使用 cache、watcher、timer 或 Kubernetes API。该 authority 以结构兼容的本地窄端口位于既有 `cluster-control/copilot/failure-diagnosis/`，只通过 `failure-diagnosis-output-keyring` subpath 发布，避免 Cluster Control 默认源码反向依赖 AI；它不复用 Prompt output、Tool invocation/result 或 Provider credential key domain，也不提前声称 Copilot 产品入口已可达。定向回归 13/13、Cluster Control 239 pass/2 条件 skip、backend 1,207 pass/2 条件 skip、18-package clean build/test、四项架构审计与 14 档 Local artifact 全部通过；workspace 仍无单文件/浅平 package，Cluster Control 58 个源码中 56 个位于嵌套领域目录。默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes，Edge/Standalone AI 保持 3,061,009/3,061,099 bytes，证明 Cluster-only subpath 被裁掉。本 Gate 不改 migration/schema/role/SQL/连接/HA 拓扑，因此数据库基线继续引用 ADR-0409 的 PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`；下一 Gate 是默认关闭的完整 Cluster Copilot composition，随后补齐 Tool failure、日志 missing/retired、Model admission 前 deadline/cancel 与 outcome-unknown 的 durable terminalization/recovery。 - D-316/ADR-0408（已接受）：Cluster Copilot 现在能够从 ADR-0407 的 durable admission 恢复 exact `qinglong.run.log.excerpt@1.0.0` authority，复用通用 Trusted Tool start barrier、加密 success/failure completion、result catalog/rekey 和内建日志 adapter；确定性 start/completion identity 让 response-loss replay 直接打开既有证据，不重复读取日志或执行 adapter。Cluster invocation Artifact 使用独立 projected keyring，提供 active+historical material，但每次读取均重新执行 canonical path/symlink/mode/inode/realpath fence，且不取得 PostgreSQL Tool result generation authority。只有 exact `succeeded` completion 才能通过 `pg-9019` 的 SERIALIZABLE 事务把 Model Step 从 `pending` 原子推进到 `ready`，同时写 RunEvent、StepRunMutation 和 append-only unlock receipt；`failed|timed_out` 不解锁。本 Gate 不执行模型、不终态化 diagnosis Run，下一 Gate 是 ADR-0405 builder + Model Gateway + Copilot encrypted model completion/terminalization。实现仍为 18 个 package，无单文件/浅平 package；AI 175 个源码中 174 个、Cluster Control 56 个中 54 个位于嵌套目录，不新增依赖、进程、连接、timer/watcher/cache 或默认 Edge 成本。18-package clean build/test 全绿，AI 229 pass/3 条件 skip、backend 1,207 pass/2 条件 skip，四项架构审计零 finding；14 档 Local artifact 全通过，默认 Edge/Standalone 为 2,589,890/2,589,968 bytes。PostgreSQL 18.4 arm64 HA 130/130、timeline `1→2`，首次执行只读两次日志、密文不含敏感 fixture，提升后 exact replay 零日志读取；报告 SHA-256 为 `d525a303696e178d777b021b376729bd2c5382fb5eb7bc98466a2b79d3940517`，独立审计与 Docker 清理通过。 diff --git a/docs/adr/ADR-0412-durable-copilot-pre-model-terminalization-and-unknown-resolution.md b/docs/adr/ADR-0412-durable-copilot-pre-model-terminalization-and-unknown-resolution.md new file mode 100644 index 00000000..eca521c4 --- /dev/null +++ b/docs/adr/ADR-0412-durable-copilot-pre-model-terminalization-and-unknown-resolution.md @@ -0,0 +1,59 @@ +# ADR-0412：耐久 Copilot Pre-Model 终态化与 Model Unknown Resolution + +- 状态：Accepted +- 日期：2026-08-15 +- 关联 RFC：QL-RFC-0001 D-320、Phase 2 +- 关联 ADR：ADR-0087、ADR-0407、ADR-0408、ADR-0409、ADR-0411 + +## 问题 + +ADR-0411 已把 Cluster Copilot failure diagnosis 组合到既有、默认关闭的 Cluster AI 进程，但成功链之外仍有两类不收敛窗口： + +1. Tool `failed|timed_out`、日志 `not_found|pending|missing|retired`、执行 Tool 所需的固定预算不足、数据库观察到 deadline 或 cancellation 时，Model 尚未开始，diagnosis Run 可能永久停留在 `running`； +2. Provider 已开始但结果不可确认时，通用 ModelInvocation 会耐久记录 `outcome_unknown` 和 `lost` Step。人工 `fail|cancel` resolution 会产生一个新的 resolved Step digest，原 Copilot finalizer 却仍只接受 unknown completion 时的 `lost` digest，因此正确 resolution 也无法终态化父 Run。 + +不能由调用者直接提交 outcome/reason，也不能用进程内异常或定时器伪造耐久终态。路由器和低配设备部署还要求该补强不能增加 package、进程、连接、队列、watcher、timer 或默认 Profile 闭包。 + +## 决策 + +1. 在 `@qinglong/ai` 的既有 `copilot/failure-diagnosis/terminalization` 领域目录建立一个 pre-Model terminalization capability，并仅通过 `failure-diagnosis-pre-model-terminalization` 精确 subpath 发布。它不是新 package，也不从 package `src` 根平铺实现文件。 +2. terminalizer 只接受三种受信触发：耐久 Tool failure completion、耐久 Tool success 加受审日志 projection、或无调用者事实的 boundary observation。repository 再从数据库读取 exact admission、Run、Tool/Model Step、Model start existence 和数据库时间；调用者不能指定 reason、outcome、Step mutation、Run version 或时间。 +3. reason 与结果采用封闭映射： + - `tool_failed` → Run `failed`，取消尚未运行的 Model Step； + - `tool_timed_out` → Run `timed_out`，取消 Model Step； + - `log_not_found|log_pending|log_missing|log_retired` → Run/Model Step `failed`； + - `tool_budget_exhausted|deadline_exceeded` → Run `timed_out`，按 Step 当前状态写 `timed_out|cancelled`； + - `cancellation_requested` → Run `cancelled`，取消未开始的两个 Step。 +4. 任何 Model start 已存在时都拒绝 pre-Model terminalization。Model 已经跨过外部副作用边界后，只能使用 Model completion/recovery/resolution 语义，不能把“不知道”改写为“没有执行”。 +5. `pg-9021-ai-copilot-failure-diagnosis-pre-model-terminalizations` 增加一个 append-only content-free receipt ledger。`ql3_runtime` 只有 `SELECT, INSERT`，PUBLIC 和其他角色没有权限。一个 `SERIALIZABLE` 事务锁定 Run/Step，复验 admission、reason evidence、version/digest 和 Model-start absence，原子提交 StepRun mutations、RunEvents、父 Run 终态和 receipt；COMMIT response loss 通过 receipt exact replay 收敛，不重复 Tool、日志读取或 Model 调用。 +6. application service 在 Tool 执行前先检查 boundary terminalization；Tool failure 和非 available 日志 projection 直接进入 terminalizer；只有 available projection 才继续 unlock/Model。Tool 的固定五秒预算必须完整落在 plan deadline 内，否则在 start barrier 前终态化。并发调用继续由既有 request coalescing 收敛。 +7. `outcome_unknown` 不自动重试、不自动假定失败。强认证 User 仍须通过通用 `DurableModelInvocationResolutionCoordinator` 显式选择 `fail|cancel|retry`。Copilot finalizer 对普通 completion 校验 completion 后 Step digest；对 unknown 的 `fail|cancel` 校验 resolution completion binding 和 resolution mutation 的 resolved Step digest，再终态化父 Run。`retry` 保持 in-progress，且确定性 invocation identity 不允许偷偷再调用 Provider。 +8. 该 capability 继续 caller-driven，并复用既有 PostgreSQL AI Pool、Run/Step ledger、Tool completion、ModelInvocation resolution 与 Cluster composition；不新增 daemon、scan、timer、watcher、队列、cache、HTTP/CLI/UI/MCP route、Pod 或 Kubernetes API 权限。默认 Edge/Standalone 及其 AI 制品不导入 Cluster-only composition，低配部署没有新增常驻工作。 + +## 被否决方案 + +1. **由 application catch 后直接返回失败**：没有耐久 Run/Step/Event 事实，崩溃和 replay 会漂移。 +2. **调用者提交 reason/outcome**：会把数据库时间、取消事实和 Tool completion authority 暴露给不可信边界。 +3. **后台轮询扫描所有 diagnosis Run**：增加每副本 timer/数据库负载，并与 caller-driven 组合重复竞争。 +4. **Model start 后仍走 pre-Model terminalizer**：可能把已经产生费用或外部结果的调用伪装为未执行。 +5. **将 unknown completion 直接映射为 failed**：抹掉真实歧义并绕过显式人工 resolution。 +6. **新增 terminalization package**：没有独立部署、依赖、权限或多 consumer 边界，只会制造薄 package。 + +## 验证标准 + +1. 单元测试覆盖 Tool failure/timed-out、四种日志状态、deadline、Tool budget、cancellation、exact replay、并发 coalescing、过早触发和 Model-start 后拒绝。 +2. PostgreSQL migration 测试固定 migration ID、顺序、表约束和 `SELECT, INSERT` 最小权限；package/dependency/import/deployment 审计必须保持零 finding。 +3. PostgreSQL 18 physical HA 以三个独立 diagnosis Run 证明：成功基线、日志不可用 pre-Model terminalization、unknown completion→人工 fail resolution→finalization；晋升后全部 exact replay，不重复日志读取、Tool 或 Model 外部执行。 +4. 18-package clean build/test、完整 backend、14 档 Local artifact、GitNexus staged/change detection 全通过后才允许阶段性提交。 + +## 当前验证 + +1. `@qinglong/ai` 完整测试 244 pass/3 条件 skip/0 fail；新增 9 条 pre-Model 场景覆盖 Tool failure、四种日志状态、deadline、固定 Tool budget、cancellation 和 Model-start fence。 +2. 18 个 QL3 package 从清空全部 `dist` 开始完成拓扑构建和全包测试，非沙箱 TLS loopback 回归全绿；backend 1,207 pass/2 条件 skip/0 fail。 +3. workspace 保持 18 个 package，`singleSourcePackages=[]`、`shallowSourcePackages=[]`。AI 192 个源码中 191 个位于嵌套领域目录；新增五个源码全部位于既有 failure-diagnosis 领域目录，package 根仍只有一个 16 行 public entrypoint。`local-command-file` 虽小但保留多 consumer 的独立 POSIX 文件安全边界，两个 migration 密集目录继续作为有顺序上限的 ledger 接受审计。 +4. Edge import、Cluster dependency、package boundary 与 Cluster deployment 审计兼容且零 finding；14 档 Local artifact 通过，说明 Cluster-only terminalizer 未进入默认路由设备闭包。 +5. PostgreSQL 18.6 arm64 physical HA 137/137、timeline `1→2`；21 条 AI migration、新 append-only ledger、最小权限、两条新增恢复链和晋升后 exact replay 全部通过。私有报告 SHA-256 为 `6eaeb20615a62d153c5a69687344f41f31351c6ecf111cfb9cbafad115538c83`，独立离线审计 `compatible=true`、零 finding。 + +## 后续门禁 + +下一 Gate 才能增加认证、Policy、audit、request identity 与 source fence 保护的 Cluster 产品 API；CLI/UI/MCP 必须复用同一 capability，不得建立旁路执行器。产品入口还需补多副本并发、真实外部 Provider、费用/取消可观测性和明文负证据后再评估默认策略。 diff --git a/docs/adr/README.md b/docs/adr/README.md index ad59d386..79726fcf 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -415,6 +415,7 @@ | [ADR-0409](./ADR-0409-cluster-copilot-failure-diagnosis-model-execution.md) | Cluster Copilot Failure Diagnosis Model Execution、密文输出与 Run 终态化 | Accepted | | [ADR-0410](./ADR-0410-cluster-copilot-failure-diagnosis-output-key-authority.md) | Cluster Copilot Failure Diagnosis Output Projected Key Authority | Accepted | | [ADR-0411](./ADR-0411-default-off-cluster-copilot-composition.md) | 默认关闭的 Cluster Copilot Failure Diagnosis Composition | Accepted | +| [ADR-0412](./ADR-0412-durable-copilot-pre-model-terminalization-and-unknown-resolution.md) | 耐久 Copilot Pre-Model 终态化与 Model Unknown Resolution | Accepted | ## 规则 diff --git a/packages/ql3-ai/package.json b/packages/ql3-ai/package.json index ca0d989c..d1858fe5 100644 --- a/packages/ql3-ai/package.json +++ b/packages/ql3-ai/package.json @@ -60,6 +60,11 @@ "require": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js", "default": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js" }, + "./failure-diagnosis-pre-model-terminalization": { + "types": "./dist/copilot/failure-diagnosis/preModelTerminalization.d.ts", + "require": "./dist/copilot/failure-diagnosis/preModelTerminalization.js", + "default": "./dist/copilot/failure-diagnosis/preModelTerminalization.js" + }, "./postgres-failure-diagnosis-model-execution-storage": { "types": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.d.ts", "require": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.js", diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/application/contracts.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/application/contracts.ts index 151ea73c..91541438 100644 --- a/packages/ql3-ai/src/copilot/failure-diagnosis/application/contracts.ts +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/application/contracts.ts @@ -3,6 +3,7 @@ import type { SecurityPrincipal } from '@qinglong/runtime-core/security'; import type { CopilotFailureDiagnosisAdmissionReceipt } from '../admission/contracts'; import type { CopilotFailureDiagnosisModelExecutionResult } from '../model-execution/coordinator'; import type { CopilotFailureDiagnosisToolExecutionResult } from '../tool-execution/contracts'; +import type { CopilotFailureDiagnosisPreModelTerminalizationReceipt } from '../terminalization/contracts'; export const MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS = 64; @@ -17,8 +18,9 @@ export interface ExecuteCopilotFailureDiagnosisApplicationCommand { export interface ExecuteCopilotFailureDiagnosisApplicationResult { readonly admissionStatus: 'created' | 'existing'; readonly admission: Readonly; - readonly tool: Readonly; + readonly tool: Readonly | null; readonly model: Readonly | null; + readonly terminalization: Readonly | null; readonly terminalizationRequired: boolean; } diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/application/service.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/application/service.ts index 087c899a..a2e869c5 100644 --- a/packages/ql3-ai/src/copilot/failure-diagnosis/application/service.ts +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/application/service.ts @@ -7,9 +7,7 @@ import { } from '@qinglong/runtime-core/builtin-run-log-excerpt-tool'; import type { RunRepositoryReader } from '@qinglong/runtime-core/run-repository'; import type { ToolPolicyAuthorizer } from '@qinglong/runtime-core/tool-registry'; -import { - prepareToolInvocation, -} from '@qinglong/runtime-core/tool-registry'; +import { prepareToolInvocation } from '@qinglong/runtime-core/tool-registry'; import type { ProjectToolDefinitionSnapshotRepository } from '@qinglong/runtime-core/project-tool-definition-snapshot'; import { projectToolDefinitionRegistry } from '@qinglong/runtime-core/project-tool-definition-snapshot'; import { @@ -25,9 +23,7 @@ import { } from '@qinglong/runtime-core/trusted-tool-invocation'; import { MAX_MODEL_INVOCATION_MS } from '../../../model-gateway/model'; -import { - prepareCopilotFailureDiagnosisExecution, -} from '../admission/plan'; +import { prepareCopilotFailureDiagnosisExecution } from '../admission/plan'; import type { CopilotFailureDiagnosisAdmissionRepository, CopilotFailureDiagnosisExecutionPlan, @@ -37,10 +33,20 @@ import { executeCopilotFailureDiagnosisTool, type CopilotFailureDiagnosisToolExecutionDependencies, } from '../tool-execution/coordinator'; +import { + CopilotFailureDiagnosisToolExecutionDeadlineExceededError, + type CopilotFailureDiagnosisToolExecutionResult, +} from '../tool-execution/contracts'; import { executeCopilotFailureDiagnosisModel, type CopilotFailureDiagnosisModelExecutionDependencies, } from '../model-execution/coordinator'; +import { + terminalizeCopilotFailureDiagnosisBeforeModel, + type CopilotFailureDiagnosisPreModelTerminalizationDependencies, + type CopilotFailureDiagnosisPreModelTerminalizationTrigger, +} from '../terminalization/coordinator'; +import { CopilotFailureDiagnosisPreModelTerminalizationNotReadyError } from '../terminalization/contracts'; import { CopilotFailureDiagnosisApplicationBusyError, CopilotFailureDiagnosisApplicationConflictError, @@ -64,7 +70,10 @@ const IDENTITY_DOMAIN = Buffer.from( export interface CopilotFailureDiagnosisApplicationDependencies { readonly admissions: CopilotFailureDiagnosisAdmissionRepository; - readonly snapshots: Pick; + readonly snapshots: Pick< + ProjectToolDefinitionSnapshotRepository, + 'findCurrent' + >; readonly runs: Pick< RunRepositoryReader, 'findRunById' | 'findLatestAttemptByRunId' @@ -77,19 +86,23 @@ export interface CopilotFailureDiagnosisApplicationDependencies { readonly authorizer: ToolPolicyAuthorizer; readonly tool: CopilotFailureDiagnosisToolExecutionDependencies; readonly model: CopilotFailureDiagnosisModelExecutionDependencies; + readonly terminalizations: CopilotFailureDiagnosisPreModelTerminalizationDependencies; readonly executeTool: typeof executeCopilotFailureDiagnosisTool; readonly executeModel: typeof executeCopilotFailureDiagnosisModel; + readonly terminalizeBeforeModel: typeof terminalizeCopilotFailureDiagnosisBeforeModel; readonly modelIntent: Readonly; readonly executionTimeoutMs: number; readonly now?: () => number; - readonly nonceFactory?: (input: Readonly<{ - key: Uint8Array; - keyId: string; - requestId: string; - projectId: string; - sourceRunId: string; - invocationActionDigest: string; - }>) => Uint8Array; + readonly nonceFactory?: ( + input: Readonly<{ + key: Uint8Array; + keyId: string; + requestId: string; + projectId: string; + sourceRunId: string; + invocationActionDigest: string; + }>, + ) => Uint8Array; } interface ActiveRequest { @@ -146,14 +159,16 @@ function preview( }); } -function defaultNonce(input: Readonly<{ - key: Uint8Array; - keyId: string; - requestId: string; - projectId: string; - sourceRunId: string; - invocationActionDigest: string; -}>): Uint8Array { +function defaultNonce( + input: Readonly<{ + key: Uint8Array; + keyId: string; + requestId: string; + projectId: string; + sourceRunId: string; + invocationActionDigest: string; + }>, +): Uint8Array { const derived = createHmac('sha256', Buffer.from(input.key)) .update(NONCE_DOMAIN) .update( @@ -224,6 +239,10 @@ function assertDependencies( typeof value.executeModel !== 'function' || !value.tool || !value.model || + typeof value.terminalizations?.repository?.findByRequestId !== 'function' || + typeof value.terminalizations?.repository?.readAuthority !== 'function' || + typeof value.terminalizations?.repository?.commit !== 'function' || + typeof value.terminalizeBeforeModel !== 'function' || !value.modelIntent || typeof value.modelIntent !== 'object' || !Number.isSafeInteger(value.executionTimeoutMs) || @@ -359,21 +378,91 @@ export class CopilotFailureDiagnosisApplicationService { }); } } - const tool = await this.#dependencies.executeTool( - { - requestId: plan.requestId, - principal: command.principal, - authorizer: this.#dependencies.authorizer, - }, - this.#dependencies.tool, - ); + const boundary = await this.#preModel(plan.requestId, { kind: 'boundary' }); + if (boundary) { + return Object.freeze({ + admissionStatus, + admission, + tool: null, + model: null, + terminalization: boundary, + terminalizationRequired: false, + }); + } + let tool: Readonly; + try { + tool = await this.#dependencies.executeTool( + { + requestId: plan.requestId, + principal: command.principal, + authorizer: this.#dependencies.authorizer, + }, + this.#dependencies.tool, + ); + } catch (cause) { + if ( + !( + cause instanceof + CopilotFailureDiagnosisToolExecutionDeadlineExceededError + ) + ) { + throw cause; + } + const terminalization = await this.#dependencies.terminalizeBeforeModel( + plan.requestId, + { kind: 'boundary' }, + this.#dependencies.terminalizations, + ); + return Object.freeze({ + admissionStatus, + admission, + tool: null, + model: null, + terminalization: terminalization.receipt, + terminalizationRequired: false, + }); + } if (tool.outcome !== 'succeeded') { + const terminalization = await this.#dependencies.terminalizeBeforeModel( + plan.requestId, + { kind: 'tool_failure', completion: tool.completion }, + this.#dependencies.terminalizations, + ); return Object.freeze({ admissionStatus, admission, tool, model: null, - terminalizationRequired: true, + terminalization: terminalization.receipt, + terminalizationRequired: false, + }); + } + const projection = await this.#preModel(plan.requestId, { + kind: 'tool_projection', + completion: tool.completion, + output: tool.output, + }); + if (projection) { + return Object.freeze({ + admissionStatus, + admission, + tool, + model: null, + terminalization: projection, + terminalizationRequired: false, + }); + } + const afterToolBoundary = await this.#preModel(plan.requestId, { + kind: 'boundary', + }); + if (afterToolBoundary) { + return Object.freeze({ + admissionStatus, + admission, + tool, + model: null, + terminalization: afterToolBoundary, + terminalizationRequired: false, }); } const model = await this.#dependencies.executeModel( @@ -385,10 +474,33 @@ export class CopilotFailureDiagnosisApplicationService { admission, tool, model, + terminalization: null, terminalizationRequired: false, }); } + async #preModel( + requestId: string, + trigger: CopilotFailureDiagnosisPreModelTerminalizationTrigger, + ) { + try { + const result = await this.#dependencies.terminalizeBeforeModel( + requestId, + trigger, + this.#dependencies.terminalizations, + ); + return result.receipt; + } catch (cause) { + if ( + cause instanceof + CopilotFailureDiagnosisPreModelTerminalizationNotReadyError + ) { + return null; + } + throw cause; + } + } + async #prepare( command: Readonly, ) { @@ -447,9 +559,7 @@ export class CopilotFailureDiagnosisApplicationService { const key = await this.#dependencies.invocationKeys.active(); try { const baseIdentity = identity('cda', command.requestId); - const nonce = ( - this.#dependencies.nonceFactory ?? defaultNonce - )({ + const nonce = (this.#dependencies.nonceFactory ?? defaultNonce)({ key: key.key, keyId: key.keyId, requestId: command.requestId, @@ -507,9 +617,9 @@ export class CopilotFailureDiagnosisApplicationService { return unavailable(); } try { - const nonce = ( - this.#dependencies.nonceFactory ?? defaultNonce - )(nonceInput(plan, material.key)); + const nonce = (this.#dependencies.nonceFactory ?? defaultNonce)( + nonceInput(plan, material.key), + ); const inputArtifact = createToolInvocationInputArtifact( { artifactId: plan.tool.invocationArtifact.artifactId, @@ -564,7 +674,10 @@ export class CopilotFailureDiagnosisApplicationService { return unavailable(cause); } if (!Number.isSafeInteger(value) || value < 0) return invalid('clock'); - if (value + this.#dependencies.executionTimeoutMs > Number.MAX_SAFE_INTEGER) { + if ( + value + this.#dependencies.executionTimeoutMs > + Number.MAX_SAFE_INTEGER + ) { return invalid('deadline overflows'); } return value; diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/postgresRepository.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/postgresRepository.ts index 385a3d9f..536b530a 100644 --- a/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/postgresRepository.ts +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/postgresRepository.ts @@ -34,8 +34,7 @@ import { } from './finalization'; const TABLE = '"ql3_ai"."copilot_failure_diagnosis_model_outputs"'; -const FINALIZATION_TABLE = - '"ql3_ai"."copilot_failure_diagnosis_finalizations"'; +const FINALIZATION_TABLE = '"ql3_ai"."copilot_failure_diagnosis_finalizations"'; interface OutputRow extends Record { readonly artifactJson: unknown; @@ -79,7 +78,9 @@ function unavailable(cause?: unknown): never { }); } -function parse(row: OutputRow): Readonly { +function parse( + row: OutputRow, +): Readonly { try { return normalizeCopilotFailureDiagnosisOutputArtifact( row.artifactJson as CopilotFailureDiagnosisOutputArtifact, @@ -119,7 +120,8 @@ async function put( client: PostgresClient, artifactValue: CopilotFailureDiagnosisOutputArtifact, ): Promise> { - const artifact = normalizeCopilotFailureDiagnosisOutputArtifact(artifactValue); + const artifact = + normalizeCopilotFailureDiagnosisOutputArtifact(artifactValue); const existing = await read(client, artifact.artifactId); if (existing) { if (JSON.stringify(existing) !== JSON.stringify(artifact)) { @@ -239,7 +241,9 @@ export class PostgresCopilotFailureDiagnosisModelRepository if (!result.rows[0]) return null; const row = result.rows[0]; const receipt = normalizeCopilotFailureDiagnosisFinalizationReceipt( - object(row.receiptJson) as unknown as CopilotFailureDiagnosisFinalizationReceipt, + object( + row.receiptJson, + ) as unknown as CopilotFailureDiagnosisFinalizationReceipt, ); if ( receipt.requestId !== string(row.requestId) || @@ -268,7 +272,9 @@ export class PostgresCopilotFailureDiagnosisModelRepository event.step_run_id AS "eventStepRunId", event.payload, event.created_at_ms AS "eventCreatedAtMs", completion.completion_digest AS "completionDigest", - completion.outcome AS "completionOutcome" + completion.outcome AS "completionOutcome", + resolution.decision AS "resolutionDecision", + resolution.resolved_at_ms AS "resolvedAtMs" FROM "ql3"."runs" AS run JOIN "ql3"."step_runs" AS step ON step.run_id = run.id AND step.id = $1 @@ -276,6 +282,8 @@ export class PostgresCopilotFailureDiagnosisModelRepository ON event.run_id = run.id AND event.id = $2 JOIN "ql3_ai"."model_invocation_completions" AS completion ON completion.invocation_id = $3 + LEFT JOIN "ql3_ai"."model_invocation_resolutions" AS resolution + ON resolution.invocation_id = completion.invocation_id WHERE run.id = $4`, [ receipt.modelStepRunId, @@ -301,7 +309,15 @@ export class PostgresCopilotFailureDiagnosisModelRepository string(proof.eventStepRunId) !== receipt.modelStepRunId || integer(proof.eventCreatedAtMs) !== receipt.finalizedAtMs || string(proof.completionDigest) !== receipt.completionDigest || - string(proof.completionOutcome) !== receipt.outcome || + !( + string(proof.completionOutcome) === receipt.outcome || + (string(proof.completionOutcome) === 'outcome_unknown' && + ((proof.resolutionDecision === 'fail' && + receipt.outcome === 'failed') || + (proof.resolutionDecision === 'cancel' && + receipt.outcome === 'cancelled')) && + integer(proof.resolvedAtMs) === receipt.finalizedAtMs) + ) || payload.requestId !== receipt.requestId || payload.planDigest !== receipt.planDigest || payload.invocationId !== receipt.invocationId || @@ -320,10 +336,12 @@ export class PostgresCopilotFailureDiagnosisModelRepository } } - async finalize(requestId: string): Promise; - }>> { + async finalize(requestId: string): Promise< + Readonly<{ + status: 'created' | 'existing'; + receipt: Readonly; + }> + > { const existing = await this.findFinalization(requestId); if (existing) { return Object.freeze({ status: 'existing' as const, receipt: existing }); @@ -333,7 +351,9 @@ export class PostgresCopilotFailureDiagnosisModelRepository try { client = await this.#pool.connect(); } catch (cause) { - throw new CopilotFailureDiagnosisFinalizationUnavailableError({ cause }); + throw new CopilotFailureDiagnosisFinalizationUnavailableError({ + cause, + }); } let began = false; try { @@ -373,13 +393,17 @@ export class PostgresCopilotFailureDiagnosisModelRepository } if ( cause instanceof CopilotFailureDiagnosisFinalizationConflictError || - cause instanceof CopilotFailureDiagnosisModelExecutionInProgressError || - cause instanceof CopilotFailureDiagnosisModelResolutionRequiredError || + cause instanceof + CopilotFailureDiagnosisModelExecutionInProgressError || + cause instanceof + CopilotFailureDiagnosisModelResolutionRequiredError || cause instanceof CopilotFailureDiagnosisFinalizationUnavailableError ) { throw cause; } - throw new CopilotFailureDiagnosisFinalizationUnavailableError({ cause }); + throw new CopilotFailureDiagnosisFinalizationUnavailableError({ + cause, + }); } finally { client.release(); } @@ -390,10 +414,12 @@ export class PostgresCopilotFailureDiagnosisModelRepository async #finalizeInTransaction( client: PostgresClient, requestId: string, - ): Promise; - }>> { + ): Promise< + Readonly<{ + status: 'created' | 'existing'; + receipt: Readonly; + }> + > { const admission = await client.query>( `SELECT plan_json AS "planJson" FROM "ql3_ai"."copilot_failure_diagnosis_admissions" @@ -404,19 +430,29 @@ export class PostgresCopilotFailureDiagnosisModelRepository throw new CopilotFailureDiagnosisFinalizationConflictError(); } const plan = normalizeCopilotFailureDiagnosisExecutionPlan( - object(admission.rows[0]!.planJson) as unknown as CopilotFailureDiagnosisExecutionPlan, + object( + admission.rows[0]!.planJson, + ) as unknown as CopilotFailureDiagnosisExecutionPlan, ); const durable = await client.query>( `SELECT run.status AS "runStatus", run.version AS "runVersion", run.event_sequence AS "runEventSequence", step.status AS "stepStatus", step.step_run_digest AS "stepRunDigest", - completion.record_json AS "completionJson" + completion.record_json AS "completionJson", + resolution.decision AS "resolutionDecision", + resolution.completion_digest AS "resolutionCompletionDigest", + resolution.resolved_at_ms AS "resolvedAtMs", + resolution_mutation.step_run_digest AS "resolutionStepRunDigest" FROM "ql3"."runs" AS run JOIN "ql3"."step_runs" AS step ON step.run_id = run.id AND step.id = $1 LEFT JOIN "ql3_ai"."model_invocation_completions" AS completion ON completion.invocation_id = $2 + LEFT JOIN "ql3_ai"."model_invocation_resolutions" AS resolution + ON resolution.invocation_id = $2 + LEFT JOIN "ql3"."step_run_mutations" AS resolution_mutation + ON resolution_mutation.mutation_id = resolution.mutation_id WHERE run.id = $3 FOR UPDATE OF run, step`, [plan.modelStepRunId, plan.modelInvocationId, plan.runId], @@ -438,14 +474,41 @@ export class PostgresCopilotFailureDiagnosisModelRepository completion.stepRunId !== plan.modelStepRunId || completion.traceId !== plan.traceId || string(row.runStatus) !== 'running' || - string(row.stepRunDigest) !== completion.completedStepRunDigest + (completion.outcome !== 'outcome_unknown' && + string(row.stepRunDigest) !== completion.completedStepRunDigest) ) { throw new CopilotFailureDiagnosisFinalizationConflictError(); } + let outcome: CopilotFailureDiagnosisFinalOutcome; + let finalizedAtMs = completion.completedAtMs; if (completion.outcome === 'outcome_unknown') { - throw new CopilotFailureDiagnosisModelResolutionRequiredError(); + if ( + row.resolutionDecision === null || + row.resolutionDecision === undefined + ) { + throw new CopilotFailureDiagnosisModelResolutionRequiredError(); + } + if (row.resolutionDecision === 'retry') { + throw new CopilotFailureDiagnosisModelExecutionInProgressError(); + } + if ( + row.resolutionDecision !== 'fail' && + row.resolutionDecision !== 'cancel' + ) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + if ( + string(row.resolutionCompletionDigest) !== + completion.completionDigest || + string(row.resolutionStepRunDigest) !== string(row.stepRunDigest) + ) { + throw new CopilotFailureDiagnosisFinalizationConflictError(); + } + outcome = row.resolutionDecision === 'fail' ? 'failed' : 'cancelled'; + finalizedAtMs = integer(row.resolvedAtMs); + } else { + outcome = completion.outcome; } - const outcome: CopilotFailureDiagnosisFinalOutcome = completion.outcome; if (string(row.stepStatus) !== outcome) { throw new CopilotFailureDiagnosisFinalizationConflictError(); } @@ -485,19 +548,25 @@ export class PostgresCopilotFailureDiagnosisModelRepository outputArtifactId: output?.artifactId ?? null, finalRunVersion: runVersion + 1, finalRunEventSequence: eventSequence + 1, - finalizedAtMs: completion.completedAtMs, + finalizedAtMs, }); - const failure = outcome === 'succeeded' - ? { code: null, summary: null } - : outcome === 'timed_out' - ? { - code: 'COPILOT_FAILURE_DIAGNOSIS_TIMED_OUT', - summary: 'Copilot failure diagnosis timed out', - } - : { - code: 'COPILOT_FAILURE_DIAGNOSIS_FAILED', - summary: 'Copilot failure diagnosis failed', - }; + const failure = + outcome === 'succeeded' + ? { code: null, summary: null } + : outcome === 'cancelled' + ? { + code: 'COPILOT_FAILURE_DIAGNOSIS_CANCELLED', + summary: 'Copilot failure diagnosis was cancelled', + } + : outcome === 'timed_out' + ? { + code: 'COPILOT_FAILURE_DIAGNOSIS_TIMED_OUT', + summary: 'Copilot failure diagnosis timed out', + } + : { + code: 'COPILOT_FAILURE_DIAGNOSIS_FAILED', + summary: 'Copilot failure diagnosis failed', + }; const updated = await client.query( `UPDATE "ql3"."runs" SET status = $1, version = $2, event_sequence = $3, diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/preModelTerminalization.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/preModelTerminalization.ts new file mode 100644 index 00000000..0929dc57 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/preModelTerminalization.ts @@ -0,0 +1,4 @@ +export * from './terminalization/contracts'; +export * from './terminalization/protocol'; +export * from './terminalization/coordinator'; +export * from './terminalization/postgresRepository'; diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/contracts.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/contracts.ts new file mode 100644 index 00000000..b7a7d155 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/contracts.ts @@ -0,0 +1,160 @@ +import type { RunRecord } from '@qinglong/runtime-core'; +import type { + StepRunMutation, + StepRunRecord, +} from '@qinglong/runtime-core/step-run'; + +import type { CopilotFailureDiagnosisExecutionPlan } from '../admission/contracts'; + +export const COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_SCHEMA = + 'qinglong/copilot-failure-diagnosis-pre-model-terminalization@v1' as const; +export const COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_COMMAND_SCHEMA = + 'qinglong/copilot-failure-diagnosis-pre-model-terminalization-command@v1' as const; + +export const COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_REASONS = [ + 'tool_failed', + 'tool_timed_out', + 'log_not_found', + 'log_pending', + 'log_missing', + 'log_retired', + 'tool_budget_exhausted', + 'deadline_exceeded', + 'cancellation_requested', +] as const; + +export type CopilotFailureDiagnosisPreModelTerminalizationReason = + (typeof COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_REASONS)[number]; + +export type CopilotFailureDiagnosisPreModelTerminalizationStage = + | 'tool' + | 'log' + | 'deadline' + | 'cancellation'; + +export type CopilotFailureDiagnosisPreModelTerminalizationOutcome = + | 'failed' + | 'timed_out' + | 'cancelled'; + +export interface CopilotFailureDiagnosisTerminalStepReference { + readonly stepRunId: string; + readonly status: 'failed' | 'timed_out' | 'cancelled'; + readonly version: number; + readonly mutationId: string; + readonly mutationDigest: string; + readonly eventId: string; +} + +export interface CopilotFailureDiagnosisPreModelTerminalizationReceipt { + readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_SCHEMA; + readonly requestId: string; + readonly planDigest: string; + readonly runId: string; + readonly stage: CopilotFailureDiagnosisPreModelTerminalizationStage; + readonly reason: CopilotFailureDiagnosisPreModelTerminalizationReason; + readonly outcome: CopilotFailureDiagnosisPreModelTerminalizationOutcome; + readonly evidenceDigest: string; + readonly toolStartId: string | null; + readonly toolCompletionDigest: string | null; + readonly terminalSteps: readonly Readonly[]; + readonly finalRunVersion: number; + readonly finalRunEventSequence: number; + readonly runEventId: string; + readonly finalizedAtMs: number; + readonly receiptDigest: string; +} + +export interface CopilotFailureDiagnosisPreModelTerminalizationCommand { + readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_COMMAND_SCHEMA; + readonly plan: Readonly; + readonly expectedRunVersion: number; + readonly expectedRunEventSequence: number; + readonly stepMutations: readonly Readonly[]; + readonly receipt: Readonly; + readonly commandDigest: string; +} + +export interface CopilotFailureDiagnosisPreModelTerminalizationAuthority { + readonly plan: Readonly; + readonly run: Readonly< + Pick< + RunRecord, + | 'id' + | 'projectId' + | 'status' + | 'version' + | 'eventSequence' + | 'cancelRequestedAtMs' + | 'cancelReason' + > + >; + readonly toolStep: Readonly; + readonly modelStep: Readonly; + readonly modelStartExists: boolean; + readonly observedAtMs: number; +} + +export interface CopilotFailureDiagnosisPreModelTerminalizationRepository { + findByRequestId( + requestId: string, + ): Promise | null>; + readAuthority( + requestId: string, + ): Promise>; + commit( + command: Readonly, + ): Promise< + Readonly<{ + status: 'created' | 'existing'; + receipt: Readonly; + }> + >; +} + +export class InvalidCopilotFailureDiagnosisPreModelTerminalizationError extends TypeError { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_INVALID'; + + constructor(message: string) { + super( + `Copilot failure diagnosis pre-Model terminalization is invalid: ${message}`, + ); + this.name = 'InvalidCopilotFailureDiagnosisPreModelTerminalizationError'; + } +} + +export class CopilotFailureDiagnosisPreModelTerminalizationConflictError extends Error { + readonly code = + 'COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_CONFLICT'; + + constructor(message = 'durable terminalization authority changed') { + super( + `Copilot failure diagnosis pre-Model terminalization conflicts: ${message}`, + ); + this.name = 'CopilotFailureDiagnosisPreModelTerminalizationConflictError'; + } +} + +export class CopilotFailureDiagnosisPreModelTerminalizationNotReadyError extends Error { + readonly code = + 'COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_NOT_READY'; + + constructor() { + super('Copilot failure diagnosis has no terminal pre-Model condition'); + this.name = 'CopilotFailureDiagnosisPreModelTerminalizationNotReadyError'; + } +} + +export class CopilotFailureDiagnosisPreModelTerminalizationUnavailableError extends Error { + readonly code = + 'COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_UNAVAILABLE'; + + constructor(options?: ErrorOptions) { + super( + 'Copilot failure diagnosis pre-Model terminalization is unavailable', + options, + ); + this.name = + 'CopilotFailureDiagnosisPreModelTerminalizationUnavailableError'; + } +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/coordinator.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/coordinator.ts new file mode 100644 index 00000000..c793244f --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/coordinator.ts @@ -0,0 +1,405 @@ +import type { ToolJsonValue } from '@qinglong/runtime-core/tool-registry'; +import type { ToolExecutionCompletionRecord } from '@qinglong/runtime-core/tool-execution-completion'; +import type { ToolExecutionFailureCompletionRecord } from '@qinglong/runtime-core/tool-execution-failure-completion'; +import { BUILTIN_RUN_LOG_EXCERPT_TIMEOUT_SECONDS } from '@qinglong/runtime-core/builtin-run-log-excerpt-tool'; +import { + STEP_RUN_TERMINAL_STATUSES, + transitionStepRunMutation, + type StepRunMutation, + type StepRunRecord, + type StepRunStatus, +} from '@qinglong/runtime-core/step-run'; + +import { + CopilotFailureDiagnosisPreModelTerminalizationConflictError, + CopilotFailureDiagnosisPreModelTerminalizationNotReadyError, + CopilotFailureDiagnosisPreModelTerminalizationUnavailableError, + InvalidCopilotFailureDiagnosisPreModelTerminalizationError, + type CopilotFailureDiagnosisPreModelTerminalizationReason, + type CopilotFailureDiagnosisPreModelTerminalizationRepository, + type CopilotFailureDiagnosisPreModelTerminalizationReceipt, +} from './contracts'; +import { + copilotFailureDiagnosisPreModelEvidenceDigest, + copilotFailureDiagnosisPreModelTerminalizationMapping, + copilotFailureDiagnosisTerminalizationIdentity, + createCopilotFailureDiagnosisPreModelTerminalizationCommand, + createCopilotFailureDiagnosisPreModelTerminalizationReceipt, + terminalStepReference, +} from './protocol'; + +export type CopilotFailureDiagnosisPreModelTerminalizationTrigger = + | Readonly<{ + kind: 'tool_failure'; + completion: Readonly; + }> + | Readonly<{ + kind: 'tool_projection'; + completion: Readonly; + output: ToolJsonValue; + }> + | Readonly<{ kind: 'boundary' }>; + +export interface CopilotFailureDiagnosisPreModelTerminalizationDependencies { + readonly repository: CopilotFailureDiagnosisPreModelTerminalizationRepository; +} + +export interface CopilotFailureDiagnosisPreModelTerminalizationResult { + readonly status: 'created' | 'existing'; + readonly receipt: Readonly; +} + +function invalid(message: string): never { + throw new InvalidCopilotFailureDiagnosisPreModelTerminalizationError(message); +} + +function conflict(message: string): never { + throw new CopilotFailureDiagnosisPreModelTerminalizationConflictError( + message, + ); +} + +function unavailable(cause?: unknown): never { + throw new CopilotFailureDiagnosisPreModelTerminalizationUnavailableError({ + cause: cause instanceof Error ? cause : undefined, + }); +} + +function assertDependencies( + value: CopilotFailureDiagnosisPreModelTerminalizationDependencies, +): void { + if ( + !value || + typeof value !== 'object' || + typeof value.repository?.findByRequestId !== 'function' || + typeof value.repository?.readAuthority !== 'function' || + typeof value.repository?.commit !== 'function' + ) { + return invalid('dependencies are invalid'); + } +} + +function statusFromProjection( + output: ToolJsonValue, + runId: string, + attemptId: string, +): 'not_found' | 'pending' | 'missing' | 'retired' | null { + if (!output || typeof output !== 'object' || Array.isArray(output)) { + return conflict('Tool projection is not an object'); + } + const record = output as Readonly>; + if ( + record.runId !== runId || + record.attemptId !== attemptId || + record.profile !== 'cluster-control' || + typeof record.status !== 'string' || + !['available', 'not_found', 'pending', 'missing', 'retired'].includes( + record.status, + ) + ) { + return conflict('Tool projection identity changed'); + } + return record.status === 'available' + ? null + : (record.status as 'not_found' | 'pending' | 'missing' | 'retired'); +} + +function triggerEvidence( + trigger: CopilotFailureDiagnosisPreModelTerminalizationTrigger, + authority: Awaited< + ReturnType< + CopilotFailureDiagnosisPreModelTerminalizationRepository['readAuthority'] + > + >, +): Readonly<{ + reason: CopilotFailureDiagnosisPreModelTerminalizationReason; + evidenceDigest: string; + toolStartId: string | null; + toolCompletionDigest: string | null; +}> { + const { plan, run, observedAtMs } = authority; + if (trigger.kind === 'tool_failure') { + const completion = trigger.completion; + if ( + completion.runId !== plan.runId || + completion.stepRunId !== plan.toolStepRunId || + completion.outcome !== authority.toolStep.status || + completion.completedStepRunDigest !== authority.toolStep.stepRunDigest + ) + return conflict('Tool failure completion changed'); + const reason = + completion.outcome === 'timed_out' ? 'tool_timed_out' : 'tool_failed'; + return Object.freeze({ + reason, + evidenceDigest: copilotFailureDiagnosisPreModelEvidenceDigest({ + reason, + planDigest: plan.planDigest, + toolCompletionDigest: completion.completionDigest, + }), + toolStartId: completion.startId, + toolCompletionDigest: completion.completionDigest, + }); + } + if (trigger.kind === 'tool_projection') { + const completion = trigger.completion; + if ( + completion.runId !== plan.runId || + completion.stepRunId !== plan.toolStepRunId || + authority.toolStep.status !== 'succeeded' || + completion.completedStepRunDigest !== authority.toolStep.stepRunDigest + ) + return conflict('Tool success completion changed'); + const status = statusFromProjection( + trigger.output, + plan.source.runId, + plan.source.attemptId, + ); + if (status === null) + throw new CopilotFailureDiagnosisPreModelTerminalizationNotReadyError(); + const reason = + `log_${status}` as CopilotFailureDiagnosisPreModelTerminalizationReason; + return Object.freeze({ + reason, + evidenceDigest: copilotFailureDiagnosisPreModelEvidenceDigest({ + reason, + planDigest: plan.planDigest, + toolCompletionDigest: completion.completionDigest, + sourceStatus: status, + }), + toolStartId: completion.startId, + toolCompletionDigest: completion.completionDigest, + }); + } + if (run.cancelRequestedAtMs !== undefined && run.cancelReason !== undefined) { + const reason = 'cancellation_requested' as const; + return Object.freeze({ + reason, + evidenceDigest: copilotFailureDiagnosisPreModelEvidenceDigest({ + reason, + planDigest: plan.planDigest, + cancelRequestedAtMs: run.cancelRequestedAtMs, + cancelReason: run.cancelReason, + }), + toolStartId: null, + toolCompletionDigest: null, + }); + } + if (observedAtMs >= plan.deadlineAtMs) { + const reason = 'deadline_exceeded' as const; + return Object.freeze({ + reason, + evidenceDigest: copilotFailureDiagnosisPreModelEvidenceDigest({ + reason, + planDigest: plan.planDigest, + deadlineAtMs: plan.deadlineAtMs, + }), + toolStartId: null, + toolCompletionDigest: null, + }); + } + const requiredToolBudgetMs = BUILTIN_RUN_LOG_EXCERPT_TIMEOUT_SECONDS * 1_000; + if ( + authority.toolStep.status === 'ready' && + observedAtMs + requiredToolBudgetMs > plan.deadlineAtMs + ) { + const reason = 'tool_budget_exhausted' as const; + return Object.freeze({ + reason, + evidenceDigest: copilotFailureDiagnosisPreModelEvidenceDigest({ + reason, + planDigest: plan.planDigest, + deadlineAtMs: plan.deadlineAtMs, + observedAtMs, + requiredToolBudgetMs, + }), + toolStartId: null, + toolCompletionDigest: null, + }); + } + throw new CopilotFailureDiagnosisPreModelTerminalizationNotReadyError(); +} + +function targetFor( + step: Readonly, + reason: CopilotFailureDiagnosisPreModelTerminalizationReason, + outcome: 'failed' | 'timed_out' | 'cancelled', +): StepRunStatus | null { + if (STEP_RUN_TERMINAL_STATUSES.includes(step.status as never)) return null; + if (reason === 'tool_failed' || reason === 'tool_timed_out') { + return step.kind === 'model' + ? 'cancelled' + : conflict('Tool Step is not terminal'); + } + if (reason.startsWith('log_')) { + return step.kind === 'model' + ? 'failed' + : conflict('log evidence Tool is not terminal'); + } + if (step.status === 'pending') return 'cancelled'; + return outcome; +} + +function transitionFacts( + target: StepRunStatus, + reason: CopilotFailureDiagnosisPreModelTerminalizationReason, +): Readonly<{ resultCode: string; errorSummary?: string }> { + if (target === 'failed') { + return Object.freeze({ + resultCode: 'copilot_log_unavailable', + errorSummary: 'Failure diagnosis source log is unavailable', + }); + } + if (target === 'timed_out') { + return Object.freeze({ + resultCode: 'copilot_deadline_exceeded', + errorSummary: 'Copilot failure diagnosis deadline exceeded', + }); + } + return Object.freeze({ + resultCode: reason.startsWith('tool_') + ? 'copilot_tool_dependency_failed' + : 'copilot_cancelled', + }); +} + +function mutations( + authority: Awaited< + ReturnType< + CopilotFailureDiagnosisPreModelTerminalizationRepository['readAuthority'] + > + >, + reason: CopilotFailureDiagnosisPreModelTerminalizationReason, + outcome: 'failed' | 'timed_out' | 'cancelled', +): readonly Readonly[] { + let runVersion = authority.run.version; + let eventSequence = authority.run.eventSequence; + const result: StepRunMutation[] = []; + for (const step of [authority.toolStep, authority.modelStep]) { + const target = targetFor(step, reason, outcome); + if (target === null) continue; + const facts = transitionFacts(target, reason); + const mutationId = copilotFailureDiagnosisTerminalizationIdentity( + 'mutation', + authority.plan.planDigest, + reason, + step.id, + ); + const eventId = copilotFailureDiagnosisTerminalizationIdentity( + 'step-event', + authority.plan.planDigest, + reason, + step.id, + ); + const mutation = transitionStepRunMutation( + step, + { + expectedVersion: step.version, + expectedDigest: step.stepRunDigest, + mutationId, + to: target, + atMs: authority.observedAtMs, + resultCode: facts.resultCode, + ...(facts.errorSummary === undefined + ? {} + : { errorSummary: facts.errorSummary }), + }, + { + expectedRunVersion: runVersion, + expectedRunEventSequence: eventSequence, + eventId, + dedupeKey: eventId, + actor: { type: 'system' }, + }, + ); + result.push(mutation); + runVersion += 1; + eventSequence += 1; + } + if (result.length < 1) return conflict('no non-terminal Step remains'); + return Object.freeze(result); +} + +export async function terminalizeCopilotFailureDiagnosisBeforeModel( + requestId: string, + trigger: CopilotFailureDiagnosisPreModelTerminalizationTrigger, + dependencies: CopilotFailureDiagnosisPreModelTerminalizationDependencies, +): Promise> { + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(requestId)) + return invalid('request id is invalid'); + if (!trigger || typeof trigger !== 'object' || Array.isArray(trigger)) + return invalid('trigger is invalid'); + assertDependencies(dependencies); + try { + const existing = await dependencies.repository.findByRequestId(requestId); + if (existing) + return Object.freeze({ status: 'existing' as const, receipt: existing }); + const authority = await dependencies.repository.readAuthority(requestId); + if ( + authority.plan.requestId !== requestId || + authority.run.id !== authority.plan.runId || + authority.run.status !== 'running' || + authority.run.version !== authority.run.eventSequence || + authority.toolStep.id !== authority.plan.toolStepRunId || + authority.toolStep.runId !== authority.plan.runId || + authority.toolStep.kind !== 'tool' || + authority.modelStep.id !== authority.plan.modelStepRunId || + authority.modelStep.runId !== authority.plan.runId || + authority.modelStep.kind !== 'model' || + authority.modelStartExists || + !Number.isSafeInteger(authority.observedAtMs) || + authority.observedAtMs < authority.plan.plannedAtMs + ) + return conflict('pre-Model authority is invalid'); + const evidence = triggerEvidence(trigger, authority); + const mapping = copilotFailureDiagnosisPreModelTerminalizationMapping( + evidence.reason, + authority.run.cancelReason, + ); + const stepMutations = mutations( + authority, + evidence.reason, + mapping.outcome, + ); + const receipt = createCopilotFailureDiagnosisPreModelTerminalizationReceipt( + { + requestId, + planDigest: authority.plan.planDigest, + runId: authority.plan.runId, + stage: mapping.stage, + reason: evidence.reason, + outcome: mapping.outcome, + evidenceDigest: evidence.evidenceDigest, + toolStartId: evidence.toolStartId, + toolCompletionDigest: evidence.toolCompletionDigest, + terminalSteps: stepMutations.map(terminalStepReference), + finalRunVersion: authority.run.version + stepMutations.length + 1, + finalRunEventSequence: + authority.run.eventSequence + stepMutations.length + 1, + finalizedAtMs: authority.observedAtMs, + }, + ); + const command = createCopilotFailureDiagnosisPreModelTerminalizationCommand( + { + plan: authority.plan, + expectedRunVersion: authority.run.version, + expectedRunEventSequence: authority.run.eventSequence, + stepMutations, + receipt, + }, + ); + return dependencies.repository.commit(command); + } catch (cause) { + if ( + cause instanceof + InvalidCopilotFailureDiagnosisPreModelTerminalizationError || + cause instanceof + CopilotFailureDiagnosisPreModelTerminalizationConflictError || + cause instanceof + CopilotFailureDiagnosisPreModelTerminalizationNotReadyError || + cause instanceof + CopilotFailureDiagnosisPreModelTerminalizationUnavailableError + ) + throw cause; + return unavailable(cause); + } +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/postgresRepository.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/postgresRepository.ts new file mode 100644 index 00000000..7d1bd07f --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/postgresRepository.ts @@ -0,0 +1,585 @@ +import type { PostgresClient, PostgresPool } from '@qinglong/runtime-core'; +import { BUILTIN_RUN_LOG_EXCERPT_TIMEOUT_SECONDS } from '@qinglong/runtime-core/builtin-run-log-excerpt-tool'; +import { + normalizeStepRunMutation, + normalizeStepRunRecord, + type StepRunMutation, + type StepRunRecord, +} from '@qinglong/runtime-core/step-run'; + +import { normalizeCopilotFailureDiagnosisExecutionPlan } from '../admission/plan'; +import type { CopilotFailureDiagnosisExecutionPlan } from '../admission/contracts'; +import { + CopilotFailureDiagnosisPreModelTerminalizationConflictError, + CopilotFailureDiagnosisPreModelTerminalizationUnavailableError, + type CopilotFailureDiagnosisPreModelTerminalizationAuthority, + type CopilotFailureDiagnosisPreModelTerminalizationCommand, + type CopilotFailureDiagnosisPreModelTerminalizationReceipt, + type CopilotFailureDiagnosisPreModelTerminalizationRepository, +} from './contracts'; +import { + normalizeCopilotFailureDiagnosisPreModelTerminalizationCommand, + normalizeCopilotFailureDiagnosisPreModelTerminalizationReceipt, +} from './protocol'; + +const TABLE = '"ql3_ai"."copilot_failure_diagnosis_pre_model_terminalizations"'; +const REQUEST_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; + +function conflict(message?: string): never { + throw new CopilotFailureDiagnosisPreModelTerminalizationConflictError( + message, + ); +} + +function unavailable(cause?: unknown): never { + throw new CopilotFailureDiagnosisPreModelTerminalizationUnavailableError({ + cause: cause instanceof Error ? cause : undefined, + }); +} + +function object(value: unknown): Record { + if (value && typeof value === 'object' && !Array.isArray(value)) { + return value as Record; + } + if (typeof value === 'string') { + try { + const parsed = JSON.parse(value) as unknown; + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as Record; + } + } catch { + // Mapped to the generic durable conflict below. + } + } + return conflict(); +} + +function integer(value: unknown): number { + if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) { + return value; + } + if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) { + const parsed = Number(value); + if (Number.isSafeInteger(parsed)) return parsed; + } + return conflict(); +} + +function string(value: unknown): string { + return typeof value === 'string' ? value : conflict(); +} + +function optionalString(value: unknown): string | undefined { + return value === null || value === undefined ? undefined : string(value); +} + +function same(left: unknown, right: unknown): boolean { + return JSON.stringify(left) === JSON.stringify(right); +} + +async function authority( + queryable: Pick | Pick, + requestId: string, + lock: boolean, +): Promise> { + const result = await queryable.query>( + `SELECT admission.plan_json AS "planJson", + run.id AS "runId", run.project_id AS "projectId", + run.status AS "runStatus", run.version AS "runVersion", + run.event_sequence AS "runEventSequence", + run.cancel_requested_at_ms AS "cancelRequestedAtMs", + run.cancel_reason AS "cancelReason", + tool_step.step_run_json AS "toolStepJson", + model_step.step_run_json AS "modelStepJson", + (model_start.invocation_id IS NOT NULL) AS "modelStartExists", + floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint + AS "observedAtMs" + FROM "ql3_ai"."copilot_failure_diagnosis_admissions" AS admission + JOIN "ql3"."runs" AS run ON run.id = admission.run_id + JOIN "ql3"."step_runs" AS tool_step + ON tool_step.run_id = run.id + AND tool_step.id = admission.tool_step_run_id + JOIN "ql3"."step_runs" AS model_step + ON model_step.run_id = run.id + AND model_step.id = admission.model_step_run_id + LEFT JOIN "ql3_ai"."model_invocation_starts" AS model_start + ON model_start.invocation_id = admission.plan_json->>'modelInvocationId' + WHERE admission.request_id = $1 + ${lock ? 'FOR UPDATE OF run, tool_step, model_step' : ''}`, + [requestId], + ); + if (result.rows.length !== 1) + return conflict('terminalization authority is absent'); + const row = result.rows[0]!; + const plan = normalizeCopilotFailureDiagnosisExecutionPlan( + object(row.planJson) as unknown as CopilotFailureDiagnosisExecutionPlan, + ); + const toolStep = normalizeStepRunRecord( + object(row.toolStepJson) as unknown as StepRunRecord, + ); + const modelStep = normalizeStepRunRecord( + object(row.modelStepJson) as unknown as StepRunRecord, + ); + return Object.freeze({ + plan, + run: Object.freeze({ + id: string(row.runId), + projectId: string(row.projectId), + status: string(row.runStatus) as never, + version: integer(row.runVersion), + eventSequence: integer(row.runEventSequence), + ...(row.cancelRequestedAtMs === null + ? {} + : { cancelRequestedAtMs: integer(row.cancelRequestedAtMs) }), + ...(row.cancelReason === null + ? {} + : { cancelReason: string(row.cancelReason) as never }), + }), + toolStep, + modelStep, + modelStartExists: row.modelStartExists === true, + observedAtMs: integer(row.observedAtMs), + }); +} + +async function updateStep( + client: PostgresClient, + mutationValue: Readonly, +): Promise { + const mutation = normalizeStepRunMutation(mutationValue); + const step = mutation.stepRun; + const updated = await client.query( + `UPDATE "ql3"."step_runs" + SET status = $1, version = $2, attempt_count = $3, + output_ref = $4, approval_request_id = $5, ready_at_ms = $6, + started_at_ms = $7, finished_at_ms = $8, result_code = $9, + error_summary = $10, updated_at_ms = $11, + last_mutation_id = $12, step_run_digest = $13, + step_run_json = $14::jsonb + WHERE id = $15 AND run_id = $16 AND version = $17 + AND step_run_digest = $18 AND status = $19`, + [ + step.status, + step.version, + step.attemptCount, + step.outputRef, + step.approvalRequestId, + step.readyAtMs, + step.startedAtMs, + step.finishedAtMs, + step.resultCode, + step.errorSummary, + step.updatedAtMs, + step.lastMutationId, + step.stepRunDigest, + JSON.stringify(step), + step.id, + step.runId, + mutation.expectedStepRunVersion, + mutation.expectedStepRunDigest, + mutation.previousStatus, + ], + ); + if ((updated.rowCount ?? updated.rows.length) !== 1) return conflict(); + const run = await client.query( + `UPDATE "ql3"."runs" + SET version = version + 1, event_sequence = event_sequence + 1 + WHERE id = $1 AND status = 'running' + AND version = $2 AND event_sequence = $3`, + [ + mutation.runId, + mutation.expectedRunVersion, + mutation.expectedRunEventSequence, + ], + ); + if ((run.rowCount ?? run.rows.length) !== 1) return conflict(); + const event = mutation.event; + await client.query( + `INSERT INTO "ql3"."run_events" ( + id, run_id, sequence, type, dedupe_key, actor_type, actor_id, + attempt_id, step_run_id, payload, created_at_ms + ) VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, $8, $9::jsonb, $10)`, + [ + event.id, + event.runId, + event.sequence, + event.type, + event.dedupeKey, + event.actorType, + event.actorId ?? null, + step.id, + JSON.stringify(event.payload), + event.createdAtMs, + ], + ); + await client.query( + `INSERT INTO "ql3"."step_run_mutations" ( + mutation_id, mutation_digest, run_id, step_run_id, step_run_digest, + event_id, event_sequence, run_version, step_run_json, committed_at_ms + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, + floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint)`, + [ + mutation.mutationId, + mutation.mutationDigest, + mutation.runId, + step.id, + step.stepRunDigest, + event.id, + event.sequence, + mutation.expectedRunVersion + 1, + JSON.stringify(step), + ], + ); +} + +function failure( + receipt: Readonly, +): Readonly<{ + code: string; + summary: string; +}> { + if (receipt.reason === 'tool_failed') { + return Object.freeze({ + code: 'COPILOT_DIAGNOSIS_TOOL_FAILED', + summary: 'Copilot diagnosis Tool failed', + }); + } + if (receipt.reason === 'tool_timed_out' || receipt.outcome === 'timed_out') { + return Object.freeze({ + code: 'COPILOT_DIAGNOSIS_TIMED_OUT', + summary: 'Copilot diagnosis timed out', + }); + } + if (receipt.stage === 'log') { + return Object.freeze({ + code: 'COPILOT_DIAGNOSIS_LOG_UNAVAILABLE', + summary: 'Copilot diagnosis log is unavailable', + }); + } + return Object.freeze({ + code: 'COPILOT_DIAGNOSIS_CANCELLED', + summary: 'Copilot diagnosis was cancelled', + }); +} + +export class PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository + implements CopilotFailureDiagnosisPreModelTerminalizationRepository +{ + constructor(private readonly pool: PostgresPool) { + if ( + !pool || + typeof pool.query !== 'function' || + typeof pool.connect !== 'function' + ) { + return unavailable(); + } + } + + async findByRequestId( + requestId: string, + ): Promise | null> { + if (!REQUEST_ID_PATTERN.test(requestId)) + return conflict('request id is invalid'); + try { + const result = await this.pool.query>( + `SELECT receipt_json AS "receiptJson" FROM ${TABLE} WHERE request_id = $1`, + [requestId], + ); + if (result.rows.length > 1) return conflict(); + if (!result.rows[0]) return null; + const receipt = + normalizeCopilotFailureDiagnosisPreModelTerminalizationReceipt( + object( + result.rows[0].receiptJson, + ) as unknown as CopilotFailureDiagnosisPreModelTerminalizationReceipt, + ); + const durable = await this.pool.query>( + `SELECT run.status AS "runStatus", run.version AS "runVersion", + run.event_sequence AS "runEventSequence", + run.finished_at_ms AS "finishedAtMs", + event.type AS "eventType", event.payload, + event.created_at_ms AS "eventCreatedAtMs" + FROM "ql3"."runs" AS run + JOIN "ql3"."run_events" AS event + ON event.run_id = run.id AND event.id = $1 + WHERE run.id = $2`, + [receipt.runEventId, receipt.runId], + ); + if (durable.rows.length !== 1) return conflict(); + const proof = durable.rows[0]!; + const payload = object(proof.payload); + if ( + string(proof.runStatus) !== receipt.outcome || + integer(proof.runVersion) !== receipt.finalRunVersion || + integer(proof.runEventSequence) !== receipt.finalRunEventSequence || + integer(proof.finishedAtMs) !== receipt.finalizedAtMs || + string(proof.eventType) !== `copilot.diagnosis.${receipt.outcome}` || + integer(proof.eventCreatedAtMs) !== receipt.finalizedAtMs || + payload.requestId !== receipt.requestId || + payload.planDigest !== receipt.planDigest || + payload.reason !== receipt.reason || + payload.evidenceDigest !== receipt.evidenceDigest + ) + return conflict(); + for (const step of receipt.terminalSteps) { + const stepProof = await this.pool.query>( + `SELECT step.status, step.version, + mutation.mutation_digest AS "mutationDigest", + mutation.event_id AS "eventId" + FROM "ql3"."step_runs" AS step + JOIN "ql3"."step_run_mutations" AS mutation + ON mutation.mutation_id = $1 AND mutation.step_run_id = step.id + WHERE step.id = $2 AND step.run_id = $3`, + [step.mutationId, step.stepRunId, receipt.runId], + ); + if ( + stepProof.rows.length !== 1 || + string(stepProof.rows[0]!.status) !== step.status || + integer(stepProof.rows[0]!.version) !== step.version || + string(stepProof.rows[0]!.mutationDigest) !== step.mutationDigest || + string(stepProof.rows[0]!.eventId) !== step.eventId + ) + return conflict(); + } + return receipt; + } catch (cause) { + if ( + cause instanceof + CopilotFailureDiagnosisPreModelTerminalizationConflictError + ) + throw cause; + return unavailable(cause); + } + } + + async readAuthority( + requestId: string, + ): Promise< + Readonly + > { + if (!REQUEST_ID_PATTERN.test(requestId)) + return conflict('request id is invalid'); + try { + return await authority(this.pool, requestId, false); + } catch (cause) { + if ( + cause instanceof + CopilotFailureDiagnosisPreModelTerminalizationConflictError + ) + throw cause; + return unavailable(cause); + } + } + + async commit( + commandValue: Readonly, + ): Promise< + Readonly<{ + status: 'created' | 'existing'; + receipt: Readonly; + }> + > { + const command = + normalizeCopilotFailureDiagnosisPreModelTerminalizationCommand( + commandValue, + ); + const existing = await this.findByRequestId(command.plan.requestId); + if (existing) { + if (existing.receiptDigest !== command.receipt.receiptDigest) + return conflict(); + return Object.freeze({ status: 'existing' as const, receipt: existing }); + } + for (let attempt = 0; attempt < 3; attempt += 1) { + let client: PostgresClient; + try { + client = await this.pool.connect(); + } catch (cause) { + return unavailable(cause); + } + let began = false; + try { + await client.query('BEGIN ISOLATION LEVEL SERIALIZABLE'); + began = true; + await client.query(`SELECT set_config('statement_timeout', $1, true)`, [ + '5s', + ]); + await client.query(`SELECT set_config('lock_timeout', $1, true)`, [ + '2s', + ]); + const current = await authority(client, command.plan.requestId, true); + if ( + !same(current.plan, command.plan) || + current.run.status !== 'running' || + current.run.version !== command.expectedRunVersion || + current.run.eventSequence !== command.expectedRunEventSequence || + current.modelStartExists || + current.observedAtMs < command.receipt.finalizedAtMs || + (command.receipt.reason === 'deadline_exceeded' && + current.observedAtMs < current.plan.deadlineAtMs) || + (command.receipt.reason === 'tool_budget_exhausted' && + (current.toolStep.status !== 'ready' || + current.observedAtMs + + BUILTIN_RUN_LOG_EXCERPT_TIMEOUT_SECONDS * 1_000 <= + current.plan.deadlineAtMs)) || + (command.receipt.reason === 'cancellation_requested' && + (current.run.cancelRequestedAtMs === undefined || + current.run.cancelReason === undefined)) + ) + return conflict(); + if (command.receipt.stage === 'tool') { + const proof = await client.query>( + `SELECT outcome, completion_digest AS "completionDigest" + FROM "ql3"."tool_execution_failure_completions" + WHERE start_id = $1 AND run_id = $2 AND step_run_id = $3`, + [ + command.receipt.toolStartId, + command.plan.runId, + command.plan.toolStepRunId, + ], + ); + if ( + proof.rows.length !== 1 || + string(proof.rows[0]!.completionDigest) !== + command.receipt.toolCompletionDigest || + string(proof.rows[0]!.outcome) !== command.receipt.outcome + ) + return conflict(); + } else if (command.receipt.stage === 'log') { + const proof = await client.query>( + `SELECT completion.completion_digest AS "completionDigest", + unlock.tool_completion_digest AS "unlockDigest" + FROM "ql3"."tool_execution_completions" AS completion + JOIN "ql3_ai"."copilot_failure_diagnosis_tool_unlocks" AS unlock + ON unlock.request_id = $1 AND unlock.start_id = completion.start_id + WHERE completion.start_id = $2 AND completion.run_id = $3 + AND completion.step_run_id = $4`, + [ + command.plan.requestId, + command.receipt.toolStartId, + command.plan.runId, + command.plan.toolStepRunId, + ], + ); + if ( + proof.rows.length !== 1 || + string(proof.rows[0]!.completionDigest) !== + command.receipt.toolCompletionDigest || + string(proof.rows[0]!.unlockDigest) !== + command.receipt.toolCompletionDigest + ) + return conflict(); + } + for (const mutation of command.stepMutations) + await updateStep(client, mutation); + const failureFact = failure(command.receipt); + const updated = await client.query( + `UPDATE "ql3"."runs" + SET status = $1, version = $2, event_sequence = $3, + output_ref = NULL, finished_at_ms = $4, + error_code = $5, error_summary = $6 + WHERE id = $7 AND status = 'running' + AND version = $8 AND event_sequence = $9`, + [ + command.receipt.outcome, + command.receipt.finalRunVersion, + command.receipt.finalRunEventSequence, + command.receipt.finalizedAtMs, + failureFact.code, + failureFact.summary, + command.plan.runId, + command.receipt.finalRunVersion - 1, + command.receipt.finalRunEventSequence - 1, + ], + ); + if ((updated.rowCount ?? updated.rows.length) !== 1) return conflict(); + const payload = JSON.stringify({ + requestId: command.receipt.requestId, + planDigest: command.receipt.planDigest, + reason: command.receipt.reason, + evidenceDigest: command.receipt.evidenceDigest, + outcome: command.receipt.outcome, + }); + await client.query( + `INSERT INTO "ql3"."run_events" ( + id, run_id, sequence, type, dedupe_key, actor_type, actor_id, + attempt_id, step_run_id, payload, created_at_ms + ) VALUES ($1, $2, $3, $4, $1, 'system', NULL, NULL, NULL, + $5::jsonb, $6)`, + [ + command.receipt.runEventId, + command.receipt.runId, + command.receipt.finalRunEventSequence, + `copilot.diagnosis.${command.receipt.outcome}`, + payload, + command.receipt.finalizedAtMs, + ], + ); + await client.query( + `INSERT INTO ${TABLE} ( + request_id, plan_digest, run_id, stage, reason, outcome, + evidence_digest, tool_start_id, tool_completion_digest, + terminal_steps_json, final_run_version, + final_run_event_sequence, run_event_id, finalized_at_ms, + receipt_digest, receipt_json + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb, + $11, $12, $13, $14, $15, $16::jsonb)`, + [ + command.receipt.requestId, + command.receipt.planDigest, + command.receipt.runId, + command.receipt.stage, + command.receipt.reason, + command.receipt.outcome, + command.receipt.evidenceDigest, + command.receipt.toolStartId, + command.receipt.toolCompletionDigest, + JSON.stringify(command.receipt.terminalSteps), + command.receipt.finalRunVersion, + command.receipt.finalRunEventSequence, + command.receipt.runEventId, + command.receipt.finalizedAtMs, + command.receipt.receiptDigest, + JSON.stringify(command.receipt), + ], + ); + await client.query('COMMIT'); + began = false; + return Object.freeze({ + status: 'created' as const, + receipt: command.receipt, + }); + } catch (cause) { + if (began) { + try { + await client.query('ROLLBACK'); + } catch { + /* preserve */ + } + } + const state = + cause && typeof cause === 'object' && 'code' in cause + ? String(cause.code) + : ''; + if ((state === '40001' || state === '40P01') && attempt < 2) continue; + const recovered = await this.findByRequestId(command.plan.requestId); + if (recovered) { + if (recovered.receiptDigest !== command.receipt.receiptDigest) + return conflict(); + return Object.freeze({ + status: 'existing' as const, + receipt: recovered, + }); + } + if ( + cause instanceof + CopilotFailureDiagnosisPreModelTerminalizationConflictError + ) + throw cause; + return unavailable(cause); + } finally { + client.release(); + } + } + return unavailable(); + } +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/protocol.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/protocol.ts new file mode 100644 index 00000000..514468ef --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/terminalization/protocol.ts @@ -0,0 +1,471 @@ +import { createHash } from 'node:crypto'; + +import { + normalizeStepRunMutation, + type StepRunMutation, +} from '@qinglong/runtime-core/step-run'; + +import { normalizeCopilotFailureDiagnosisExecutionPlan } from '../admission/plan'; +import { + COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_COMMAND_SCHEMA, + COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_REASONS, + COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_SCHEMA, + InvalidCopilotFailureDiagnosisPreModelTerminalizationError, + type CopilotFailureDiagnosisPreModelTerminalizationCommand, + type CopilotFailureDiagnosisPreModelTerminalizationOutcome, + type CopilotFailureDiagnosisPreModelTerminalizationReason, + type CopilotFailureDiagnosisPreModelTerminalizationReceipt, + type CopilotFailureDiagnosisPreModelTerminalizationStage, + type CopilotFailureDiagnosisTerminalStepReference, +} from './contracts'; + +const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/; +const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const RECEIPT_DOMAIN = + 'qinglong/copilot-failure-diagnosis-pre-model-terminalization-receipt@v1\0'; +const COMMAND_DOMAIN = + 'qinglong/copilot-failure-diagnosis-pre-model-terminalization-command@v1\0'; +const IDENTITY_DOMAIN = + 'qinglong/copilot-failure-diagnosis-pre-model-terminalization-identity@v1\0'; + +function invalid(message: string): never { + throw new InvalidCopilotFailureDiagnosisPreModelTerminalizationError(message); +} + +function exact( + value: object, + expected: readonly string[], + label: string, +): void { + const keys = Reflect.ownKeys(value); + if ( + keys.length !== expected.length || + keys.some((key) => typeof key !== 'string' || !expected.includes(key)) + ) { + return invalid(`${label} shape is invalid`); + } +} + +function text(value: unknown, pattern: RegExp, label: string): string { + if (typeof value !== 'string' || !pattern.test(value)) { + return invalid(`${label} is invalid`); + } + return value; +} + +function integer(value: unknown, minimum: number, label: string): number { + if (!Number.isSafeInteger(value) || (value as number) < minimum) { + return invalid(`${label} is invalid`); + } + return value as number; +} + +function hash(domain: string, value: unknown): string { + return createHash('sha256') + .update(domain) + .update(JSON.stringify(value)) + .digest('hex'); +} + +export function copilotFailureDiagnosisPreModelTerminalizationMapping( + reason: CopilotFailureDiagnosisPreModelTerminalizationReason, + cancellationReason?: string, +): Readonly<{ + stage: CopilotFailureDiagnosisPreModelTerminalizationStage; + outcome: CopilotFailureDiagnosisPreModelTerminalizationOutcome; +}> { + if ( + !COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_REASONS.includes( + reason, + ) + ) { + return invalid('reason is invalid'); + } + if (reason === 'tool_failed') + return Object.freeze({ stage: 'tool', outcome: 'failed' }); + if (reason === 'tool_timed_out') { + return Object.freeze({ stage: 'tool', outcome: 'timed_out' }); + } + if (reason.startsWith('log_')) + return Object.freeze({ stage: 'log', outcome: 'failed' }); + if (reason === 'tool_budget_exhausted' || reason === 'deadline_exceeded') { + return Object.freeze({ stage: 'deadline', outcome: 'timed_out' }); + } + return Object.freeze({ + stage: 'cancellation', + outcome: cancellationReason === 'timeout' ? 'timed_out' : 'cancelled', + }); +} + +export function copilotFailureDiagnosisTerminalizationIdentity( + prefix: 'mutation' | 'step-event' | 'run-event', + planDigest: string, + reason: CopilotFailureDiagnosisPreModelTerminalizationReason, + target: string, +): string { + const digest = hash(IDENTITY_DOMAIN, { prefix, planDigest, reason, target }); + if (prefix !== 'run-event' && prefix !== 'step-event') { + return `cdx:${digest.slice(0, 31)}`; + } + const hex = digest.slice(0, 32).split(''); + hex[12] = '4'; + hex[16] = '8'; + const value = hex.join(''); + return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice( + 12, + 16, + )}-${value.slice(16, 20)}-${value.slice(20)}`; +} + +function terminalStep( + value: CopilotFailureDiagnosisTerminalStepReference, +): Readonly { + exact( + value, + [ + 'eventId', + 'mutationDigest', + 'mutationId', + 'status', + 'stepRunId', + 'version', + ], + 'terminal Step', + ); + if (!['failed', 'timed_out', 'cancelled'].includes(value.status)) { + return invalid('terminal Step status is invalid'); + } + return Object.freeze({ + stepRunId: text(value.stepRunId, ID_PATTERN, 'StepRun id'), + status: value.status, + version: integer(value.version, 1, 'StepRun version'), + mutationId: text(value.mutationId, ID_PATTERN, 'mutation id'), + mutationDigest: text( + value.mutationDigest, + DIGEST_PATTERN, + 'mutation digest', + ), + eventId: text(value.eventId, RUN_ID_PATTERN, 'event id'), + }); +} + +function unsignedReceipt( + value: Omit< + CopilotFailureDiagnosisPreModelTerminalizationReceipt, + 'receiptDigest' + >, +): object { + return { + schema: value.schema, + requestId: value.requestId, + planDigest: value.planDigest, + runId: value.runId, + stage: value.stage, + reason: value.reason, + outcome: value.outcome, + evidenceDigest: value.evidenceDigest, + toolStartId: value.toolStartId, + toolCompletionDigest: value.toolCompletionDigest, + terminalSteps: value.terminalSteps, + finalRunVersion: value.finalRunVersion, + finalRunEventSequence: value.finalRunEventSequence, + runEventId: value.runEventId, + finalizedAtMs: value.finalizedAtMs, + }; +} + +export function copilotFailureDiagnosisPreModelTerminalizationReceiptDigest( + value: Omit< + CopilotFailureDiagnosisPreModelTerminalizationReceipt, + 'receiptDigest' + >, +): string { + return hash(RECEIPT_DOMAIN, unsignedReceipt(value)); +} + +export function normalizeCopilotFailureDiagnosisPreModelTerminalizationReceipt( + value: CopilotFailureDiagnosisPreModelTerminalizationReceipt, +): Readonly { + if (!value || typeof value !== 'object' || Array.isArray(value)) + return invalid('receipt'); + exact( + value, + [ + 'evidenceDigest', + 'finalRunEventSequence', + 'finalRunVersion', + 'finalizedAtMs', + 'outcome', + 'planDigest', + 'reason', + 'receiptDigest', + 'requestId', + 'runEventId', + 'runId', + 'schema', + 'stage', + 'terminalSteps', + 'toolCompletionDigest', + 'toolStartId', + ], + 'receipt', + ); + if ( + value.schema !== COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_SCHEMA + ) { + return invalid('receipt schema is invalid'); + } + const mapping = copilotFailureDiagnosisPreModelTerminalizationMapping( + value.reason, + ); + if ( + value.stage !== mapping.stage || + (value.reason !== 'cancellation_requested' && + value.outcome !== mapping.outcome) || + (value.reason === 'cancellation_requested' && + !['cancelled', 'timed_out'].includes(value.outcome)) || + !Array.isArray(value.terminalSteps) || + value.terminalSteps.length < 1 || + value.terminalSteps.length > 2 + ) { + return invalid('receipt state is invalid'); + } + const terminalSteps = Object.freeze(value.terminalSteps.map(terminalStep)); + if ( + new Set(terminalSteps.map((item) => item.stepRunId)).size !== + terminalSteps.length + ) { + return invalid('terminal Step identities are duplicated'); + } + const unsigned = Object.freeze({ + schema: value.schema, + requestId: text(value.requestId, ID_PATTERN, 'request id'), + planDigest: text(value.planDigest, DIGEST_PATTERN, 'plan digest'), + runId: text(value.runId, RUN_ID_PATTERN, 'Run id'), + stage: value.stage, + reason: value.reason, + outcome: value.outcome, + evidenceDigest: text( + value.evidenceDigest, + DIGEST_PATTERN, + 'evidence digest', + ), + toolStartId: + value.toolStartId === null + ? null + : text(value.toolStartId, ID_PATTERN, 'Tool start id'), + toolCompletionDigest: + value.toolCompletionDigest === null + ? null + : text( + value.toolCompletionDigest, + DIGEST_PATTERN, + 'Tool completion digest', + ), + terminalSteps, + finalRunVersion: integer(value.finalRunVersion, 1, 'final Run version'), + finalRunEventSequence: integer( + value.finalRunEventSequence, + 1, + 'final Run event sequence', + ), + runEventId: text(value.runEventId, RUN_ID_PATTERN, 'Run event id'), + finalizedAtMs: integer(value.finalizedAtMs, 0, 'finalized time'), + } satisfies Omit); + if ( + unsigned.finalRunVersion !== unsigned.finalRunEventSequence || + unsigned.runEventId !== + copilotFailureDiagnosisTerminalizationIdentity( + 'run-event', + unsigned.planDigest, + unsigned.reason, + unsigned.runId, + ) || + (unsigned.stage === 'tool' || unsigned.stage === 'log') !== + (unsigned.toolStartId !== null && + unsigned.toolCompletionDigest !== null) || + text(value.receiptDigest, DIGEST_PATTERN, 'receipt digest') !== + copilotFailureDiagnosisPreModelTerminalizationReceiptDigest(unsigned) + ) { + return invalid('receipt evidence is invalid'); + } + return Object.freeze({ ...unsigned, receiptDigest: value.receiptDigest }); +} + +export function createCopilotFailureDiagnosisPreModelTerminalizationReceipt( + value: Omit< + CopilotFailureDiagnosisPreModelTerminalizationReceipt, + 'schema' | 'runEventId' | 'receiptDigest' + >, +): Readonly { + const unsigned = Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_SCHEMA, + ...value, + runEventId: copilotFailureDiagnosisTerminalizationIdentity( + 'run-event', + value.planDigest, + value.reason, + value.runId, + ), + }); + return normalizeCopilotFailureDiagnosisPreModelTerminalizationReceipt({ + ...unsigned, + receiptDigest: + copilotFailureDiagnosisPreModelTerminalizationReceiptDigest(unsigned), + }); +} + +function commandUnsigned( + value: Omit< + CopilotFailureDiagnosisPreModelTerminalizationCommand, + 'commandDigest' + >, +): object { + return { + schema: value.schema, + plan: value.plan, + expectedRunVersion: value.expectedRunVersion, + expectedRunEventSequence: value.expectedRunEventSequence, + stepMutations: value.stepMutations, + receipt: value.receipt, + }; +} + +export function normalizeCopilotFailureDiagnosisPreModelTerminalizationCommand( + value: CopilotFailureDiagnosisPreModelTerminalizationCommand, +): Readonly { + if (!value || typeof value !== 'object' || Array.isArray(value)) + return invalid('command'); + exact( + value, + [ + 'commandDigest', + 'expectedRunEventSequence', + 'expectedRunVersion', + 'plan', + 'receipt', + 'schema', + 'stepMutations', + ], + 'command', + ); + if ( + value.schema !== + COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_COMMAND_SCHEMA + ) { + return invalid('command schema is invalid'); + } + const plan = normalizeCopilotFailureDiagnosisExecutionPlan(value.plan); + const receipt = + normalizeCopilotFailureDiagnosisPreModelTerminalizationReceipt( + value.receipt, + ); + if ( + !Array.isArray(value.stepMutations) || + value.stepMutations.length !== receipt.terminalSteps.length + ) { + return invalid('Step mutation count is invalid'); + } + const stepMutations = Object.freeze( + value.stepMutations.map(normalizeStepRunMutation), + ); + for (let index = 0; index < stepMutations.length; index += 1) { + const mutation = stepMutations[index]!; + const reference = receipt.terminalSteps[index]!; + if ( + mutation.runId !== plan.runId || + mutation.stepRun.id !== reference.stepRunId || + mutation.stepRun.status !== reference.status || + mutation.stepRun.version !== reference.version || + mutation.mutationId !== reference.mutationId || + mutation.mutationDigest !== reference.mutationDigest || + mutation.event.id !== reference.eventId + ) + return invalid('Step mutation evidence changed'); + } + const expectedRunVersion = integer( + value.expectedRunVersion, + 1, + 'expected Run version', + ); + const expectedRunEventSequence = integer( + value.expectedRunEventSequence, + 1, + 'expected Run event sequence', + ); + if ( + expectedRunVersion !== expectedRunEventSequence || + receipt.requestId !== plan.requestId || + receipt.planDigest !== plan.planDigest || + receipt.runId !== plan.runId || + receipt.finalRunVersion !== expectedRunVersion + stepMutations.length + 1 + ) + return invalid('command aggregate fence is invalid'); + const unsigned = Object.freeze({ + schema: value.schema, + plan, + expectedRunVersion, + expectedRunEventSequence, + stepMutations, + receipt, + }); + if ( + text(value.commandDigest, DIGEST_PATTERN, 'command digest') !== + hash(COMMAND_DOMAIN, commandUnsigned(unsigned)) + ) { + return invalid('command digest does not match'); + } + return Object.freeze({ ...unsigned, commandDigest: value.commandDigest }); +} + +export function createCopilotFailureDiagnosisPreModelTerminalizationCommand( + value: Omit< + CopilotFailureDiagnosisPreModelTerminalizationCommand, + 'schema' | 'commandDigest' + >, +): Readonly { + const unsigned = Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_COMMAND_SCHEMA, + ...value, + }); + return normalizeCopilotFailureDiagnosisPreModelTerminalizationCommand({ + ...unsigned, + commandDigest: hash(COMMAND_DOMAIN, commandUnsigned(unsigned)), + }); +} + +export function copilotFailureDiagnosisPreModelEvidenceDigest( + value: Readonly<{ + reason: CopilotFailureDiagnosisPreModelTerminalizationReason; + planDigest: string; + toolCompletionDigest?: string; + sourceStatus?: string; + deadlineAtMs?: number; + observedAtMs?: number; + requiredToolBudgetMs?: number; + cancelRequestedAtMs?: number; + cancelReason?: string; + }>, +): string { + return hash(`${IDENTITY_DOMAIN}evidence\0`, value); +} + +export function terminalStepReference( + mutation: Readonly, +): Readonly { + const normalized = normalizeStepRunMutation(mutation); + if ( + !['failed', 'timed_out', 'cancelled'].includes(normalized.stepRun.status) + ) { + return invalid('Step mutation is not terminal'); + } + return terminalStep({ + stepRunId: normalized.stepRun.id, + status: normalized.stepRun.status as 'failed' | 'timed_out' | 'cancelled', + version: normalized.stepRun.version, + mutationId: normalized.mutationId, + mutationDigest: normalized.mutationDigest, + eventId: normalized.event.id, + }); +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/tool-execution/contracts.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/tool-execution/contracts.ts index 7a740fdb..02f3a023 100644 --- a/packages/ql3-ai/src/copilot/failure-diagnosis/tool-execution/contracts.ts +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/tool-execution/contracts.ts @@ -5,6 +5,8 @@ import type { ToolExecutionCompletionRecord, ToolExecutionResultArtifactReference, } from '@qinglong/runtime-core/tool-execution-completion'; +import type { ToolJsonValue } from '@qinglong/runtime-core/tool-registry'; +import type { ToolExecutionFailureCompletionRecord } from '@qinglong/runtime-core/tool-execution-failure-completion'; import type { ToolPolicyAuthorizer } from '@qinglong/runtime-core/tool-registry'; import type { @@ -76,12 +78,14 @@ export type CopilotFailureDiagnosisToolExecutionResult = completionStatus: 'created' | 'existing'; unlockStatus: 'created' | 'existing'; completion: Readonly; + output: ToolJsonValue; unlock: Readonly; }> | Readonly<{ outcome: 'failed' | 'timed_out'; completionStatus: 'created' | 'existing'; unlockStatus: null; + completion: Readonly; }>; export class InvalidCopilotFailureDiagnosisToolExecutionError extends TypeError { diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/tool-execution/coordinator.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/tool-execution/coordinator.ts index c1fe559f..9be9b847 100644 --- a/packages/ql3-ai/src/copilot/failure-diagnosis/tool-execution/coordinator.ts +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/tool-execution/coordinator.ts @@ -586,6 +586,7 @@ export async function executeCopilotFailureDiagnosisTool( outcome: completed.outcome, completionStatus: completed.status, unlockStatus: null, + completion: completed.completion, }); } const unlock = await unlockModel(plan, completed.completion, dependencies); @@ -594,6 +595,7 @@ export async function executeCopilotFailureDiagnosisTool( completionStatus: completed.status, unlockStatus: unlock.status, completion: completed.completion, + output: completed.output, unlock: unlock.receipt, }); } diff --git a/packages/ql3-ai/src/migration/model-invocation/identities.ts b/packages/ql3-ai/src/migration/model-invocation/identities.ts index 8b77a1ca..a5d97f34 100644 --- a/packages/ql3-ai/src/migration/model-invocation/identities.ts +++ b/packages/ql3-ai/src/migration/model-invocation/identities.ts @@ -67,6 +67,8 @@ export const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID = 'pg-9019-ai-copilot-failure-diagnosis-tool-unlocks'; export const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID = 'pg-9020-ai-copilot-failure-diagnosis-model-executions'; +export const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID = + 'pg-9021-ai-copilot-failure-diagnosis-pre-model-terminalizations'; export const POSTGRES_MODEL_INVOCATION_SCHEMA = 'ql3_ai'; export const LOCAL_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE = 'QingLong3AiSchemaMigrations'; diff --git a/packages/ql3-ai/src/migration/model-invocation/postgres.ts b/packages/ql3-ai/src/migration/model-invocation/postgres.ts index 3f38467e..285d0193 100644 --- a/packages/ql3-ai/src/migration/model-invocation/postgres.ts +++ b/packages/ql3-ai/src/migration/model-invocation/postgres.ts @@ -30,6 +30,7 @@ import { POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_SCHEMA, POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE, } from './identities'; @@ -67,6 +68,7 @@ const POSTGRES_HISTORY_IDENTITY = Object.freeze({ POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID, ]), streamId: POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID, dialect: 'postgresql' as const, diff --git a/packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts b/packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts index 0b09beca..fb8436dc 100644 --- a/packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts +++ b/packages/ql3-ai/src/migration/model-invocation/postgres/copilot.ts @@ -4,6 +4,7 @@ import { POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_SCHEMA, } from '../identities'; import { defineSqlMigration } from '../shared'; @@ -14,6 +15,8 @@ const SOURCE_SNAPSHOT_FUNCTION = const TOOL_UNLOCK_TABLE = 'copilot_failure_diagnosis_tool_unlocks'; const MODEL_OUTPUT_TABLE = 'copilot_failure_diagnosis_model_outputs'; const FINALIZATION_TABLE = 'copilot_failure_diagnosis_finalizations'; +const PRE_MODEL_TERMINALIZATION_TABLE = + 'copilot_failure_diagnosis_pre_model_terminalizations'; const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_TABLE_SQL = ` CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" ( @@ -480,8 +483,102 @@ const postgresCopilotFailureDiagnosisModelExecutionMigration = (context, statement) => context.query(statement).then(() => undefined), ); +const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_TABLE_SQL = ` +CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${PRE_MODEL_TERMINALIZATION_TABLE}" ( + request_id varchar(128) PRIMARY KEY, + plan_digest char(64) NOT NULL UNIQUE, + run_id varchar(36) NOT NULL UNIQUE, + stage varchar(16) NOT NULL, + reason varchar(32) NOT NULL, + outcome varchar(16) NOT NULL, + evidence_digest char(64) NOT NULL, + tool_start_id varchar(128), + tool_completion_digest char(64), + terminal_steps_json jsonb NOT NULL, + final_run_version integer NOT NULL, + final_run_event_sequence integer NOT NULL, + run_event_id varchar(36) NOT NULL UNIQUE, + finalized_at_ms bigint NOT NULL, + receipt_digest char(64) NOT NULL UNIQUE, + receipt_json jsonb NOT NULL, + CONSTRAINT ql3_ai_copilot_pre_model_terminalization_admission_fk + FOREIGN KEY (request_id) + REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" + (request_id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_pre_model_terminalization_run_fk + FOREIGN KEY (run_id) REFERENCES "ql3"."runs" (id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_pre_model_terminalization_event_fk + FOREIGN KEY (run_event_id) REFERENCES "ql3"."run_events" (id) ON DELETE RESTRICT, + CONSTRAINT ql3_ai_copilot_pre_model_terminalization_state_check CHECK ( + stage IN ('tool', 'log', 'deadline', 'cancellation') AND + reason IN ( + 'tool_failed', 'tool_timed_out', 'log_not_found', 'log_pending', + 'log_missing', 'log_retired', 'tool_budget_exhausted', + 'deadline_exceeded', + 'cancellation_requested' + ) AND + outcome IN ('failed', 'timed_out', 'cancelled') AND + ((stage = 'tool' AND reason IN ('tool_failed', 'tool_timed_out')) OR + (stage = 'log' AND reason IN ( + 'log_not_found', 'log_pending', 'log_missing', 'log_retired' + )) OR + (stage = 'deadline' AND reason IN ( + 'tool_budget_exhausted', 'deadline_exceeded' + )) OR + (stage = 'cancellation' AND reason = 'cancellation_requested')) AND + ((stage IN ('tool', 'log') AND tool_start_id IS NOT NULL AND + tool_completion_digest IS NOT NULL) OR + (stage IN ('deadline', 'cancellation') AND tool_start_id IS NULL AND + tool_completion_digest IS NULL)) AND + final_run_version >= 1 AND + final_run_event_sequence = final_run_version AND + finalized_at_ms >= 0 + ), + CONSTRAINT ql3_ai_copilot_pre_model_terminalization_digest_check CHECK ( + plan_digest ~ '^[0-9a-f]{64}$' AND + evidence_digest ~ '^[0-9a-f]{64}$' AND + (tool_completion_digest IS NULL OR + tool_completion_digest ~ '^[0-9a-f]{64}$') AND + receipt_digest ~ '^[0-9a-f]{64}$' + ), + CONSTRAINT ql3_ai_copilot_pre_model_terminalization_json_check CHECK ( + jsonb_typeof(terminal_steps_json) = 'array' AND + jsonb_array_length(terminal_steps_json) BETWEEN 1 AND 2 AND + jsonb_typeof(receipt_json) = 'object' AND + octet_length(receipt_json::text) BETWEEN 2 AND 32768 AND + receipt_json @> jsonb_build_object( + 'schema', + 'qinglong/copilot-failure-diagnosis-pre-model-terminalization@v1', + 'requestId', request_id, 'planDigest', plan_digest, + 'runId', run_id, 'stage', stage, 'reason', reason, + 'outcome', outcome, 'evidenceDigest', evidence_digest, + 'terminalSteps', terminal_steps_json, + 'finalRunVersion', final_run_version, + 'finalRunEventSequence', final_run_event_sequence, + 'runEventId', run_event_id, 'finalizedAtMs', finalized_at_ms, + 'receiptDigest', receipt_digest + ) + ) +)`; + +const postgresCopilotFailureDiagnosisPreModelTerminalizationMigration = + defineSqlMigration( + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID, + [ + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_TABLE_SQL, + `REVOKE ALL ON TABLE + "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${PRE_MODEL_TERMINALIZATION_TABLE}" + FROM PUBLIC`, + `GRANT SELECT, INSERT ON TABLE + "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${PRE_MODEL_TERMINALIZATION_TABLE}" + TO ql3_runtime`, + ], + (context, statement) => context.query(statement).then(() => undefined), + ); + export const postgresCopilotMigrations = Object.freeze([ postgresCopilotFailureDiagnosisAdmissionMigration, postgresCopilotFailureDiagnosisToolUnlockMigration, postgresCopilotFailureDiagnosisModelExecutionMigration, + postgresCopilotFailureDiagnosisPreModelTerminalizationMigration, ]); diff --git a/packages/ql3-ai/test/failure-diagnosis-application.test.cjs b/packages/ql3-ai/test/failure-diagnosis-application.test.cjs index 04f30ac9..e295e7f2 100644 --- a/packages/ql3-ai/test/failure-diagnosis-application.test.cjs +++ b/packages/ql3-ai/test/failure-diagnosis-application.test.cjs @@ -5,6 +5,9 @@ const { CopilotFailureDiagnosisApplicationService, CopilotFailureDiagnosisApplicationUnavailableError, } = require('@qinglong/ai/failure-diagnosis-application'); +const { + CopilotFailureDiagnosisPreModelTerminalizationNotReadyError, +} = require('@qinglong/ai/failure-diagnosis-pre-model-terminalization'); const { BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION, } = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool'); @@ -80,9 +83,12 @@ function fixture(options = {}) { let resolvedKeyCopies = []; let toolCalls = 0; let modelCalls = 0; + let terminalizationCalls = 0; let releaseTool; const toolGate = options.blockTool - ? new Promise((resolve) => { releaseTool = resolve; }) + ? new Promise((resolve) => { + releaseTool = resolve; + }) : Promise.resolve(); const admissions = { async findByRequestId(requestId) { @@ -108,20 +114,30 @@ function fixture(options = {}) { }, }; const snapshots = { - async findCurrent() { return { snapshot: snapshot(), committedAtMs: NOW }; }, + async findCurrent() { + return { snapshot: snapshot(), committedAtMs: NOW }; + }, }; const runs = { async findRunById() { return { - id: 'source-run-1', projectId: 'project-1', status: 'failed', - version: 8, eventSequence: 8, + id: 'source-run-1', + projectId: 'project-1', + status: 'failed', + version: 8, + eventSequence: 8, }; }, async findLatestAttemptByRunId() { return { - id: 'source-attempt-1', runId: 'source-run-1', attempt: 1, - status: 'failed', executorType: 'remote_worker', callbackSequence: 0, - createdAtMs: NOW - 3000, finishedAtMs: NOW - 1000, + id: 'source-attempt-1', + runId: 'source-run-1', + attempt: 1, + status: 'failed', + executorType: 'remote_worker', + callbackSequence: 0, + createdAtMs: NOW - 3000, + finishedAtMs: NOW - 1000, logArtifactId: `wlog-${'d'.repeat(30)}`, }; }, @@ -140,8 +156,12 @@ function fixture(options = {}) { artifacts = { input, preview }; return { status: 'inserted' }; }, - async findInput() { return artifacts?.input ?? null; }, - async findPreview() { return artifacts?.preview ?? null; }, + async findInput() { + return artifacts?.input ?? null; + }, + async findPreview() { + return artifacts?.preview ?? null; + }, }; const invocationKeys = { async active() { @@ -156,23 +176,58 @@ function fixture(options = {}) { return { keyId, key: copy }; }, }; - const unlocks = { async findByRequestId() { return null; }, async commit() {} }; + const unlocks = { + async findByRequestId() { + return null; + }, + async commit() {}, + }; const tool = { admissions, snapshots, runs, artifacts: artifactRepository, invocationKeys, - resultKeys: { async resolve() { return null; } }, - stepRuns: { async findById() { return null; } }, - barriers: {}, completions: {}, failureCompletions: {}, - resultKeyCatalog: {}, resultRekeys: {}, logs: {}, unlocks, + resultKeys: { + async resolve() { + return null; + }, + }, + stepRuns: { + async findById() { + return null; + }, + }, + barriers: {}, + completions: {}, + failureCompletions: {}, + resultKeyCatalog: {}, + resultRekeys: {}, + logs: {}, + unlocks, }; const model = { admissions, unlocks, - toolResults: {}, modelInvocations: {}, outputs: {}, gateway: {}, - successfulCompletion: {}, finalizations: {}, + toolResults: {}, + modelInvocations: {}, + outputs: {}, + gateway: {}, + successfulCompletion: {}, + finalizations: {}, + }; + const terminalizations = { + repository: { + async findByRequestId() { + return null; + }, + async readAuthority() { + return {}; + }, + async commit() { + return {}; + }, + }, }; const service = new CopilotFailureDiagnosisApplicationService({ admissions, @@ -183,13 +238,15 @@ function fixture(options = {}) { authorizer: { async authorize() { return { - effect: 'allow', reasons: ['role_grant'], + effect: 'allow', + reasons: ['role_grant'], fence: { projectVersion: 1, bindingVersion: 1 }, }; }, }, tool, model, + terminalizations, modelIntent: MODEL, executionTimeoutMs: 60_000, now: () => NOW, @@ -197,10 +254,24 @@ function fixture(options = {}) { toolCalls += 1; await toolGate; return options.toolFailure - ? { outcome: 'failed', completionStatus: 'created', unlockStatus: null } + ? { + outcome: 'failed', + completionStatus: 'created', + unlockStatus: null, + completion: { completionDigest: 'f'.repeat(64) }, + } : { - outcome: 'succeeded', completionStatus: 'created', - unlockStatus: 'created', completion: {}, unlock: {}, + outcome: 'succeeded', + completionStatus: 'created', + unlockStatus: 'created', + completion: { completionDigest: 'e'.repeat(64) }, + output: { + status: 'available', + runId: 'source-run-1', + attemptId: 'source-attempt-1', + profile: 'cluster-control', + }, + unlock: {}, }; }, async executeModel() { @@ -211,13 +282,36 @@ function fixture(options = {}) { finalization: { requestId: 'diagnosis-request-1' }, }; }, + async terminalizeBeforeModel(_requestId, trigger) { + terminalizationCalls += 1; + if ( + trigger.kind === 'boundary' || + (trigger.kind === 'tool_projection' && + trigger.output.status === 'available') + ) { + throw new CopilotFailureDiagnosisPreModelTerminalizationNotReadyError(); + } + return { + status: 'created', + receipt: { + requestId: 'diagnosis-request-1', + reason: 'tool_failed', + outcome: 'failed', + }, + }; + }, }); return { service, releaseTool: () => releaseTool?.(), state: () => ({ - plan, artifacts, toolCalls, modelCalls, - activeKeyCopies, resolvedKeyCopies, + plan, + artifacts, + toolCalls, + modelCalls, + terminalizationCalls, + activeKeyCopies, + resolvedKeyCopies, }), }; } @@ -228,6 +322,7 @@ test('application derives, admits and executes one server-owned diagnosis with e assert.equal(first.admissionStatus, 'created'); assert.equal(first.tool.outcome, 'succeeded'); assert.equal(first.model.outcome, 'succeeded'); + assert.equal(first.terminalization, null); assert.equal(first.terminalizationRequired, false); const second = await testFixture.service.execute(command()); assert.equal(second.admissionStatus, 'existing'); @@ -235,9 +330,18 @@ test('application derives, admits and executes one server-owned diagnosis with e assert.equal(state.toolCalls, 2); assert.equal(state.modelCalls, 2); assert.equal(state.plan.source.attemptId, 'source-attempt-1'); - assert.equal(state.plan.tool.invocationArtifact.artifactId.startsWith('cdia:'), true); - assert.equal(state.activeKeyCopies[0].every((value) => value === 0), true); - assert.equal(state.resolvedKeyCopies[0].every((value) => value === 0), true); + assert.equal( + state.plan.tool.invocationArtifact.artifactId.startsWith('cdia:'), + true, + ); + assert.equal( + state.activeKeyCopies[0].every((value) => value === 0), + true, + ); + assert.equal( + state.resolvedKeyCopies[0].every((value) => value === 0), + true, + ); }); test('application repairs the durable admission-to-Artifact crash window', async () => { @@ -253,7 +357,7 @@ test('application repairs the durable admission-to-Artifact crash window', async assert.ok(testFixture.state().artifacts); }); -test('application coalesces exact callers and exposes Tool terminalization debt', async () => { +test('application coalesces exact callers and terminalizes Tool failure', async () => { const concurrent = fixture({ blockTool: true }); const first = concurrent.service.execute(command()); const second = concurrent.service.execute(command()); @@ -266,6 +370,7 @@ test('application coalesces exact callers and exposes Tool terminalization debt' const result = await failed.service.execute(command()); assert.equal(result.tool.outcome, 'failed'); assert.equal(result.model, null); - assert.equal(result.terminalizationRequired, true); + assert.equal(result.terminalization.reason, 'tool_failed'); + assert.equal(result.terminalizationRequired, false); assert.equal(failed.state().modelCalls, 0); }); diff --git a/packages/ql3-ai/test/failure-diagnosis-pre-model-terminalization.test.cjs b/packages/ql3-ai/test/failure-diagnosis-pre-model-terminalization.test.cjs new file mode 100644 index 00000000..5347cf65 --- /dev/null +++ b/packages/ql3-ai/test/failure-diagnosis-pre-model-terminalization.test.cjs @@ -0,0 +1,442 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + BUILTIN_RUN_LOG_EXCERPT_TOOL, + BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION, + createBuiltInRunLogExcerptToolHandlerBinding, +} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool'); +const { + createPluginPackageResourceGenerationFromReferences, +} = require('@qinglong/runtime-core/plugin-package-resource-generation'); +const { + createProjectToolDefinitionSnapshot, + projectToolDefinitionRegistry, +} = require('@qinglong/runtime-core/project-tool-definition-snapshot'); +const { + prepareToolInvocation, +} = require('@qinglong/runtime-core/tool-registry'); +const { + TrustedToolHandlerBindingRegistry, + createTrustedToolInvocationPlan, +} = require('@qinglong/runtime-core/trusted-tool-invocation'); +const { + transitionStepRunMutation, +} = require('@qinglong/runtime-core/step-run'); +const { + createCopilotFailureDiagnosisAdmissionBundle, + prepareCopilotFailureDiagnosisExecution, +} = require('@qinglong/ai/failure-diagnosis-execution-admission'); +const { + CopilotFailureDiagnosisPreModelTerminalizationNotReadyError, + normalizeCopilotFailureDiagnosisPreModelTerminalizationCommand, + terminalizeCopilotFailureDiagnosisBeforeModel, +} = require('@qinglong/ai/failure-diagnosis-pre-model-terminalization'); + +const NOW = 4_000; + +async function plan() { + const generation = createPluginPackageResourceGenerationFromReferences({ + installationId: 'install-terminalization-test', + projectId: 'project-terminalization', + packageName: 'qinglong', + lockDigest: 'a'.repeat(64), + generation: 1, + previousActiveLockDigest: null, + contentDigest: 'b'.repeat(64), + resources: [], + }); + const snapshot = createProjectToolDefinitionSnapshot({ + projectId: 'project-terminalization', + contributions: [ + { + generation, + revisionDigest: 'c'.repeat(64), + definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION], + }, + ], + }); + const binding = createBuiltInRunLogExcerptToolHandlerBinding(snapshot, [ + 'cluster-control', + ]); + const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]); + const principal = { + subject: { type: 'user', id: 'owner-terminalization' }, + authenticationId: 'auth-terminalization', + authenticatedAtMs: 1_000, + expiresAtMs: 10_000, + assurance: 'multi_factor', + }; + const invocation = await prepareToolInvocation( + projectToolDefinitionRegistry(snapshot), + { + projectId: 'project-terminalization', + principal, + nowMs: 1_200, + tool: BUILTIN_RUN_LOG_EXCERPT_TOOL, + input: { runId: 'source-run-failed', attemptId: 'source-attempt-failed' }, + }, + { + async authorize() { + return { + effect: 'allow', + reasons: ['role_grant'], + fence: { projectVersion: 1, bindingVersion: 1 }, + }; + }, + }, + ); + const tool = createTrustedToolInvocationPlan(bindings, invocation, { + actionRef: 'terminalization-log-tool', + inputArtifactId: 'terminalization-input', + previewArtifactId: 'terminalization-preview', + artifactKeyId: 'terminalization-key', + artifactKey: Buffer.alloc(32, 0x11), + artifactNonce: Buffer.alloc(12, 0x22), + profile: 'cluster-control', + preview: { + title: 'Read failed Run log', + summary: 'Read bounded evidence', + fields: [ + { kind: 'identifier', label: 'Run', value: 'source-run-failed' }, + { + kind: 'identifier', + label: 'Attempt', + value: 'source-attempt-failed', + }, + ], + warnings: ['potentially_sensitive_output'], + }, + sealedAtMs: 2_000, + }); + return prepareCopilotFailureDiagnosisExecution({ + requestId: 'terminalization-request-1', + traceId: 'terminalization-trace-1', + source: { + runId: 'source-run-failed', + runVersion: 4, + runStatus: 'failed', + attemptId: 'source-attempt-failed', + attemptStatus: 'failed', + attemptFinishedAtMs: 1_900, + logArtifactId: 'source-log-artifact', + }, + toolPlan: tool.plan, + bindings, + model: { + provider: 'provider-primary', + model: 'model-diagnosis', + modelBoundary: 'external', + responseLanguage: 'zh-CN', + maxOutputTokens: 256, + egressPolicy: { + schema: 'qinglong/copilot-model-egress-policy@v1', + revision: 'terminalization-policy-v1', + potentiallySensitiveDataBoundaries: ['external'], + maxInputBytes: 64 * 1024, + maxOutputTokens: 512, + }, + }, + plannedAtMs: 2_100, + deadlineAtMs: 8_000, + }); +} + +function transition(step, runVersion, to, atMs, suffix, outputRef) { + return transitionStepRunMutation( + step, + { + expectedVersion: step.version, + expectedDigest: step.stepRunDigest, + mutationId: `fixture-mutation-${suffix}`, + to, + atMs, + ...(outputRef ? { outputRef } : {}), + ...(['failed', 'timed_out', 'cancelled'].includes(to) + ? { resultCode: `fixture_${to}` } + : {}), + ...(['failed', 'timed_out'].includes(to) + ? { errorSummary: `Fixture ${to}` } + : {}), + }, + { + expectedRunVersion: runVersion, + expectedRunEventSequence: runVersion, + eventId: `fixture-event-${suffix}`, + dedupeKey: `fixture-event-${suffix}`, + actor: { type: 'system' }, + }, + ); +} + +function repository(authority) { + let command = null; + return { + value: { + async findByRequestId() { + return null; + }, + async readAuthority() { + return authority; + }, + async commit(value) { + command = + normalizeCopilotFailureDiagnosisPreModelTerminalizationCommand(value); + return { status: 'created', receipt: command.receipt }; + }, + }, + command: () => command, + }; +} + +test('terminalizes a durable Tool failure and cancels the pending Model Step', async () => { + const current = await plan(); + const admitted = createCopilotFailureDiagnosisAdmissionBundle(current); + const started = transition( + admitted.toolStepMutation.stepRun, + 3, + 'running', + 2_500, + 'tool-start', + ); + const failed = transition(started.stepRun, 4, 'failed', 3_000, 'tool-failed'); + const storage = repository({ + plan: current, + run: { + id: current.runId, + projectId: current.projectId, + status: 'running', + version: 5, + eventSequence: 5, + }, + toolStep: failed.stepRun, + modelStep: admitted.modelStepMutation.stepRun, + modelStartExists: false, + observedAtMs: NOW, + }); + const result = await terminalizeCopilotFailureDiagnosisBeforeModel( + current.requestId, + { + kind: 'tool_failure', + completion: { + startId: 'tool-start-1', + runId: current.runId, + stepRunId: current.toolStepRunId, + outcome: 'failed', + completedStepRunDigest: failed.stepRun.stepRunDigest, + completionDigest: 'd'.repeat(64), + }, + }, + { repository: storage.value }, + ); + assert.equal(result.receipt.reason, 'tool_failed'); + assert.equal(result.receipt.outcome, 'failed'); + assert.equal(result.receipt.terminalSteps.length, 1); + assert.equal( + result.receipt.terminalSteps[0].stepRunId, + current.modelStepRunId, + ); + assert.equal(result.receipt.terminalSteps[0].status, 'cancelled'); + assert.equal(storage.command().expectedRunVersion, 5); + assert.equal(result.receipt.finalRunVersion, 7); +}); + +test('terminalizes every non-available log status without exposing content', async () => { + for (const status of ['not_found', 'pending', 'missing', 'retired']) { + const current = await plan(); + const admitted = createCopilotFailureDiagnosisAdmissionBundle(current); + const toolStarted = transition( + admitted.toolStepMutation.stepRun, + 3, + 'running', + 2_500, + `start-${status}`, + ); + const toolSucceeded = transition( + toolStarted.stepRun, + 4, + 'succeeded', + 3_000, + `success-${status}`, + 'tool-result-artifact', + ); + const modelReady = transition( + admitted.modelStepMutation.stepRun, + 5, + 'ready', + 3_100, + `ready-${status}`, + ); + const storage = repository({ + plan: current, + run: { + id: current.runId, + projectId: current.projectId, + status: 'running', + version: 6, + eventSequence: 6, + }, + toolStep: toolSucceeded.stepRun, + modelStep: modelReady.stepRun, + modelStartExists: false, + observedAtMs: NOW, + }); + const result = await terminalizeCopilotFailureDiagnosisBeforeModel( + current.requestId, + { + kind: 'tool_projection', + completion: { + startId: 'tool-start-1', + runId: current.runId, + stepRunId: current.toolStepRunId, + completedStepRunDigest: toolSucceeded.stepRun.stepRunDigest, + completionDigest: 'e'.repeat(64), + }, + output: { + status, + runId: current.source.runId, + attemptId: current.source.attemptId, + profile: 'cluster-control', + }, + }, + { repository: storage.value }, + ); + assert.equal(result.receipt.reason, `log_${status}`); + assert.equal(result.receipt.outcome, 'failed'); + assert.equal(result.receipt.terminalSteps[0].status, 'failed'); + assert.equal(JSON.stringify(result.receipt).includes('log content'), false); + } +}); + +test('database-observed deadline terminalizes both unstarted Steps', async () => { + const current = await plan(); + const admitted = createCopilotFailureDiagnosisAdmissionBundle(current); + const storage = repository({ + plan: current, + run: { + id: current.runId, + projectId: current.projectId, + status: 'running', + version: 3, + eventSequence: 3, + }, + toolStep: admitted.toolStepMutation.stepRun, + modelStep: admitted.modelStepMutation.stepRun, + modelStartExists: false, + observedAtMs: current.deadlineAtMs, + }); + const result = await terminalizeCopilotFailureDiagnosisBeforeModel( + current.requestId, + { kind: 'boundary' }, + { repository: storage.value }, + ); + assert.equal(result.receipt.reason, 'deadline_exceeded'); + assert.equal(result.receipt.outcome, 'timed_out'); + assert.deepEqual( + result.receipt.terminalSteps.map(({ status }) => status), + ['timed_out', 'cancelled'], + ); + assert.equal(result.receipt.finalRunVersion, 6); +}); + +test('insufficient durable Tool budget terminalizes before starting the Tool', async () => { + const current = await plan(); + const admitted = createCopilotFailureDiagnosisAdmissionBundle(current); + const storage = repository({ + plan: current, + run: { + id: current.runId, + projectId: current.projectId, + status: 'running', + version: 3, + eventSequence: 3, + }, + toolStep: admitted.toolStepMutation.stepRun, + modelStep: admitted.modelStepMutation.stepRun, + modelStartExists: false, + observedAtMs: current.deadlineAtMs - 4_999, + }); + const result = await terminalizeCopilotFailureDiagnosisBeforeModel( + current.requestId, + { kind: 'boundary' }, + { repository: storage.value }, + ); + assert.equal(result.receipt.reason, 'tool_budget_exhausted'); + assert.equal(result.receipt.outcome, 'timed_out'); + assert.deepEqual( + result.receipt.terminalSteps.map(({ status }) => status), + ['timed_out', 'cancelled'], + ); +}); + +test('durable cancellation terminalizes both unstarted Steps', async () => { + const current = await plan(); + const admitted = createCopilotFailureDiagnosisAdmissionBundle(current); + const storage = repository({ + plan: current, + run: { + id: current.runId, + projectId: current.projectId, + status: 'running', + version: 3, + eventSequence: 3, + cancelRequestedAtMs: NOW - 1, + cancelReason: 'user', + }, + toolStep: admitted.toolStepMutation.stepRun, + modelStep: admitted.modelStepMutation.stepRun, + modelStartExists: false, + observedAtMs: NOW, + }); + const result = await terminalizeCopilotFailureDiagnosisBeforeModel( + current.requestId, + { kind: 'boundary' }, + { repository: storage.value }, + ); + assert.equal(result.receipt.reason, 'cancellation_requested'); + assert.equal(result.receipt.outcome, 'cancelled'); + assert.deepEqual( + result.receipt.terminalSteps.map(({ status }) => status), + ['cancelled', 'cancelled'], + ); +}); + +test('refuses early boundary settlement and any pre-Model lie after Model start', async () => { + const current = await plan(); + const admitted = createCopilotFailureDiagnosisAdmissionBundle(current); + const base = { + plan: current, + run: { + id: current.runId, + projectId: current.projectId, + status: 'running', + version: 3, + eventSequence: 3, + }, + toolStep: admitted.toolStepMutation.stepRun, + modelStep: admitted.modelStepMutation.stepRun, + observedAtMs: current.deadlineAtMs - 5_001, + }; + await assert.rejects( + terminalizeCopilotFailureDiagnosisBeforeModel( + current.requestId, + { kind: 'boundary' }, + { repository: repository({ ...base, modelStartExists: false }).value }, + ), + CopilotFailureDiagnosisPreModelTerminalizationNotReadyError, + ); + await assert.rejects( + terminalizeCopilotFailureDiagnosisBeforeModel( + current.requestId, + { kind: 'boundary' }, + { + repository: repository({ + ...base, + modelStartExists: true, + observedAtMs: current.deadlineAtMs, + }).value, + }, + ), + /pre-Model authority is invalid/, + ); +}); diff --git a/packages/ql3-ai/test/modelInvocationMigration.test.cjs b/packages/ql3-ai/test/modelInvocationMigration.test.cjs index 0db474e8..aa901724 100644 --- a/packages/ql3-ai/test/modelInvocationMigration.test.cjs +++ b/packages/ql3-ai/test/modelInvocationMigration.test.cjs @@ -22,6 +22,7 @@ const { POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE, POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID, @@ -436,6 +437,10 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async () POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, 'pg-9020-ai-copilot-failure-diagnosis-model-executions', ); + assert.equal( + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID, + 'pg-9021-ai-copilot-failure-diagnosis-pre-model-terminalizations', + ); assert.equal( POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE, 'ai_schema_migrations', @@ -523,7 +528,7 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async () ); assert.equal( postgresModelInvocationMigrationDefinition.migrations.length, - 20, + 21, ); const diagnosisAdmissionStatements = []; @@ -597,6 +602,29 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async () /GRANT[^;]*(?:UPDATE|DELETE)/, ); + const diagnosisPreModelTerminalizationStatements = []; + await postgresModelInvocationMigrationDefinition.migrations[20].up({ + async query(statement) { + diagnosisPreModelTerminalizationStatements.push(statement); + return { rows: [] }; + }, + }); + const diagnosisPreModelTerminalizationSql = + diagnosisPreModelTerminalizationStatements.join('\n'); + assert.match( + diagnosisPreModelTerminalizationSql, + /CREATE TABLE "ql3_ai"\."copilot_failure_diagnosis_pre_model_terminalizations"/, + ); + assert.match( + diagnosisPreModelTerminalizationSql, + /jsonb_array_length\(terminal_steps_json\) BETWEEN 1 AND 2/, + ); + assert.match(diagnosisPreModelTerminalizationSql, /TO ql3_runtime/); + assert.doesNotMatch( + diagnosisPreModelTerminalizationSql, + /GRANT[^;]*(?:UPDATE|DELETE)/, + ); + const retirementStatements = []; await postgresModelInvocationMigrationDefinition.migrations[10].up({ async query(statement) { diff --git a/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs b/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs index 0179936b..253b5a94 100644 --- a/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs +++ b/packages/ql3-ai/test/postgresModelProviderCredentialManagementIdentityLedger.test.cjs @@ -175,7 +175,7 @@ test('readiness binds exact migration history and least-privilege primary author assert.equal(report.ready, true); assert.equal( report.migrationIds.at(-1), - 'pg-9020-ai-copilot-failure-diagnosis-model-executions', + 'pg-9021-ai-copilot-failure-diagnosis-pre-model-terminalizations', ); assert.match( queries[1], diff --git a/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs b/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs index bfd36896..7125d088 100644 --- a/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs +++ b/packages/ql3-ai/test/postgresModelProviderCredentialTestExecution.test.cjs @@ -277,7 +277,7 @@ test('tester readiness freezes migration history and least privilege', async () assert.equal(ready.ready, true); assert.equal( ready.migrationIds.at(-1), - 'pg-9020-ai-copilot-failure-diagnosis-model-executions', + 'pg-9021-ai-copilot-failure-diagnosis-pre-model-terminalizations', ); await assert.rejects( diff --git a/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisComposition.ts b/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisComposition.ts index 1b0d55f3..eafc250a 100644 --- a/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisComposition.ts +++ b/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisComposition.ts @@ -18,6 +18,10 @@ import { import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage'; import { PostgresCopilotFailureDiagnosisModelRepository } from '@qinglong/ai/postgres-failure-diagnosis-model-execution-storage'; import { PostgresCopilotFailureDiagnosisToolUnlockRepository } from '@qinglong/ai/postgres-failure-diagnosis-tool-execution-storage'; +import { + PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository, + terminalizeCopilotFailureDiagnosisBeforeModel, +} from '@qinglong/ai/failure-diagnosis-pre-model-terminalization'; import type { ActiveModelGatewayCapability } from '@qinglong/ai/profile'; import { PostgresProjectPolicyRepository, @@ -33,9 +37,7 @@ import { PostgresToolResultRekeyReader, type QingLongPostgresPool, } from '@qinglong/cluster-postgres/runtime'; -import { - BuiltInRunLogExcerptToolAdapter, -} from '@qinglong/runtime-core/builtin-run-log-excerpt-tool'; +import { BuiltInRunLogExcerptToolAdapter } from '@qinglong/runtime-core/builtin-run-log-excerpt-tool'; import type { RunAttemptLogReadPort } from '@qinglong/runtime-core/builtin-run-log-excerpt-projection'; import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy'; import { RunAttemptLogReadService } from '@qinglong/runtime-core/run-attempt-log-read'; @@ -92,7 +94,9 @@ export interface PreparedClusterCopilotFailureDiagnosisProjection { ReturnType >; readonly outputKeys: Awaited< - ReturnType + ReturnType< + typeof createClusterCopilotFailureDiagnosisOutputProjectedKeyring + > >; } @@ -100,7 +104,10 @@ export class ClusterCopilotFailureDiagnosisCompositionError extends Error { readonly code = 'QL3_CLUSTER_COPILOT_FAILURE_DIAGNOSIS_COMPOSITION_INVALID'; constructor(message: string, options?: ErrorOptions) { - super(`Cluster Copilot failure diagnosis composition is invalid: ${message}`, options); + super( + `Cluster Copilot failure diagnosis composition is invalid: ${message}`, + options, + ); this.name = 'ClusterCopilotFailureDiagnosisCompositionError'; } } @@ -196,7 +203,8 @@ export function normalizeClusterCopilotFailureDiagnosisConfig( ], 'egress policy', ); - if (egress.schema !== EGRESS_SCHEMA) return invalid('egress schema is invalid'); + if (egress.schema !== EGRESS_SCHEMA) + return invalid('egress schema is invalid'); const selected = egress.potentiallySensitiveDataBoundaries; if ( !Array.isArray(selected) || @@ -307,7 +315,11 @@ function modelIntent( export async function prepareProductionClusterCopilotFailureDiagnosisProjection( projection: ClusterCopilotFailureDiagnosisProjection, ): Promise> { - if (!projection || typeof projection !== 'object' || Array.isArray(projection)) { + if ( + !projection || + typeof projection !== 'object' || + Array.isArray(projection) + ) { return invalid('projection is invalid'); } const [config, invocationKeys, resultKeys, outputKeys] = await Promise.all([ @@ -360,8 +372,12 @@ export async function createProductionClusterCopilotFailureDiagnosis( const runs = new PostgresRunRepository(options.pool); const artifacts = new PostgresToolInvocationArtifactRepository(options.pool); const stepRuns = new PostgresStepRunRepository(options.pool); - const barriers = new PostgresToolExecutionStartBarrierRepository(options.pool); - const completions = new PostgresToolExecutionCompletionRepository(options.pool); + const barriers = new PostgresToolExecutionStartBarrierRepository( + options.pool, + ); + const completions = new PostgresToolExecutionCompletionRepository( + options.pool, + ); const failureCompletions = new PostgresToolExecutionFailureCompletionRepository(options.pool); const resultKeyCatalog = new PostgresToolResultKeyCatalogReader(options.pool); @@ -369,7 +385,13 @@ export async function createProductionClusterCopilotFailureDiagnosis( const unlocks = new PostgresCopilotFailureDiagnosisToolUnlockRepository( options.pool, ); - const models = new PostgresCopilotFailureDiagnosisModelRepository(options.pool); + const models = new PostgresCopilotFailureDiagnosisModelRepository( + options.pool, + ); + const preModelTerminalizations = + new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository( + options.pool, + ); const logReader = new RunAttemptLogReadService( runs, Object.freeze({ @@ -457,8 +479,12 @@ export async function createProductionClusterCopilotFailureDiagnosis( authorizer: policy, tool, model, + terminalizations: Object.freeze({ + repository: preModelTerminalizations, + }), executeTool: executeCopilotFailureDiagnosisTool, executeModel: executeCopilotFailureDiagnosisModel, + terminalizeBeforeModel: terminalizeCopilotFailureDiagnosisBeforeModel, modelIntent: modelIntent(config), executionTimeoutMs: config.executionTimeoutMs, }; diff --git a/scripts/ql3-cluster-dependency-audit.cjs b/scripts/ql3-cluster-dependency-audit.cjs index 204e9447..627aeeaa 100644 --- a/scripts/ql3-cluster-dependency-audit.cjs +++ b/scripts/ql3-cluster-dependency-audit.cjs @@ -1179,6 +1179,7 @@ function auditSourceImports(root, packagePath, findings) { '@qinglong/ai/failure-diagnosis-application', '@qinglong/ai/failure-diagnosis-execution-admission', '@qinglong/ai/failure-diagnosis-model-execution', + '@qinglong/ai/failure-diagnosis-pre-model-terminalization', '@qinglong/ai/failure-diagnosis-tool-execution', '@qinglong/ai/postgres-failure-diagnosis-admission-storage', '@qinglong/ai/postgres-failure-diagnosis-model-execution-storage', diff --git a/scripts/ql3-postgres-ha-contract.cjs b/scripts/ql3-postgres-ha-contract.cjs index eaa15458..b15d111e 100644 --- a/scripts/ql3-postgres-ha-contract.cjs +++ b/scripts/ql3-postgres-ha-contract.cjs @@ -40,6 +40,7 @@ const { } = require('../packages/ql3-cluster-postgres/dist/migration/migration.js'); const { POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, POSTGRES_MODEL_INVOCATION_MIGRATION_ID, @@ -73,6 +74,13 @@ const { const { PostgresCopilotFailureDiagnosisToolUnlockRepository, } = require('../packages/ql3-ai/dist/copilot/failure-diagnosis/postgresToolExecutionRepository.js'); +const { + PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository, + terminalizeCopilotFailureDiagnosisBeforeModel, +} = require('../packages/ql3-ai/dist/copilot/failure-diagnosis/preModelTerminalization.js'); +const { + PostgresCopilotFailureDiagnosisModelRepository, +} = require('../packages/ql3-ai/dist/copilot/failure-diagnosis/postgresModelExecutionRepository.js'); const { BUILTIN_RUN_LOG_EXCERPT_TOOL, BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION, @@ -137,6 +145,9 @@ const { const { DurableModelInvocationCoordinator, } = require('../packages/ql3-ai/dist/model-invocation/durableModelInvocationCoordinator.js'); +const { + DurableModelInvocationResolutionCoordinator, +} = require('../packages/ql3-ai/dist/model-invocation/modelInvocationResolution.js'); const { PostgresModelInvocationRepository, } = require('../packages/ql3-ai/dist/model-invocation/postgresModelInvocationRepository.js'); @@ -1534,6 +1545,27 @@ async function modelInvocationFeatureFacts(pool) { 'SELECT,INSERT,UPDATE,DELETE' ) ) AS "copilotDiagnosisModelExecutionRuntimeOnly", + has_table_privilege( + 'ql3_runtime', + 'ql3_ai.copilot_failure_diagnosis_pre_model_terminalizations', + 'SELECT,INSERT' + ) AND NOT has_table_privilege( + 'ql3_runtime', + 'ql3_ai.copilot_failure_diagnosis_pre_model_terminalizations', + 'UPDATE,DELETE' + ) AND NOT EXISTS ( + SELECT 1 + FROM unnest(ARRAY[ + 'ql3_admin', 'ql3_package_manager', 'ql3_package_executor', + 'ql3_worker_ingress', 'ql3_ai_maintenance', + 'ql3_ai_credential_manager', 'ql3_ai_credential_tester' + ]::text[]) AS denied(role_name) + WHERE has_table_privilege( + denied.role_name, + 'ql3_ai.copilot_failure_diagnosis_pre_model_terminalizations', + 'SELECT,INSERT,UPDATE,DELETE' + ) + ) AS "copilotDiagnosisPreModelTerminalizationRuntimeOnly", has_function_privilege( 'ql3_runtime', 'ql3_ai.copilot_failure_diagnosis_admission_source_snapshot(varchar,varchar,varchar,integer,integer,varchar,varchar)', @@ -1714,6 +1746,108 @@ async function copilotFailureDiagnosisAdmissionFacts(pool, requestId) { }; } +async function copilotFailureDiagnosisTerminalizationFacts(pool, requestId) { + const result = await pool.query( + `SELECT terminalization.request_id AS "requestId", + terminalization.plan_digest AS "planDigest", + terminalization.stage, terminalization.reason, + terminalization.outcome, + terminalization.evidence_digest AS "evidenceDigest", + terminalization.receipt_digest AS "receiptDigest", + terminalization.final_run_version AS "finalRunVersion", + terminalization.final_run_event_sequence AS "finalRunEventSequence", + run.status AS "runStatus", run.version AS "runVersion", + run.event_sequence AS "runEventSequence", + tool.status AS "toolStatus", model.status AS "modelStatus", + (SELECT count(*)::integer FROM "ql3"."run_events" + WHERE run_id = terminalization.run_id) AS "eventCount", + (SELECT count(*)::integer FROM "ql3"."step_run_mutations" + WHERE run_id = terminalization.run_id) AS "mutationCount", + (SELECT count(*)::integer + FROM "ql3_ai"."copilot_failure_diagnosis_pre_model_terminalizations" + WHERE request_id = terminalization.request_id) + AS "terminalizationCount", + terminalization.receipt_json::text AS "receiptJson" + FROM "ql3_ai"."copilot_failure_diagnosis_pre_model_terminalizations" + AS terminalization + JOIN "ql3"."runs" AS run ON run.id = terminalization.run_id + JOIN "ql3"."step_runs" AS tool + ON tool.run_id = terminalization.run_id + AND tool.id = ( + SELECT tool_step_run_id + FROM "ql3_ai"."copilot_failure_diagnosis_admissions" + WHERE request_id = terminalization.request_id + ) + JOIN "ql3"."step_runs" AS model + ON model.run_id = terminalization.run_id + AND model.id = ( + SELECT model_step_run_id + FROM "ql3_ai"."copilot_failure_diagnosis_admissions" + WHERE request_id = terminalization.request_id + ) + WHERE terminalization.request_id = $1`, + [requestId], + ); + assert.equal(result.rowCount, 1); + const { receiptJson, ...facts } = result.rows[0]; + assert.equal(typeof receiptJson, 'string'); + return { + ...facts, + receiptJsonDigest: createHash('sha256').update(receiptJson).digest('hex'), + receiptContainsLogContent: + receiptJson.includes('diagnosis-sensitive-line') || + receiptJson.includes('must-be-redacted'), + }; +} + +async function copilotFailureDiagnosisModelResolutionFacts(pool, requestId) { + const result = await pool.query( + `SELECT admission.request_id AS "requestId", + run.status AS "runStatus", run.version AS "runVersion", + run.event_sequence AS "runEventSequence", + model.status AS "modelStatus", model.version AS "modelVersion", + completion.outcome AS "completionOutcome", + resolution.decision AS "resolutionDecision", + finalization.outcome AS "finalizationOutcome", + finalization.receipt_digest AS "finalizationReceiptDigest", + (SELECT count(*)::integer FROM "ql3"."run_events" + WHERE run_id = admission.run_id) AS "eventCount", + (SELECT count(*)::integer FROM "ql3"."step_run_mutations" + WHERE run_id = admission.run_id) AS "mutationCount", + (SELECT count(*)::integer + FROM "ql3_ai"."model_invocation_resolutions" + WHERE invocation_id = admission.plan_json->>'modelInvocationId') + AS "resolutionCount", + (SELECT count(*)::integer + FROM "ql3_ai"."copilot_failure_diagnosis_finalizations" + WHERE request_id = admission.request_id) AS "finalizationCount", + finalization.receipt_json::text AS "receiptJson" + FROM "ql3_ai"."copilot_failure_diagnosis_admissions" AS admission + JOIN "ql3"."runs" AS run ON run.id = admission.run_id + JOIN "ql3"."step_runs" AS model + ON model.run_id = admission.run_id + AND model.id = admission.model_step_run_id + JOIN "ql3_ai"."model_invocation_completions" AS completion + ON completion.invocation_id = admission.plan_json->>'modelInvocationId' + JOIN "ql3_ai"."model_invocation_resolutions" AS resolution + ON resolution.invocation_id = completion.invocation_id + JOIN "ql3_ai"."copilot_failure_diagnosis_finalizations" AS finalization + ON finalization.request_id = admission.request_id + WHERE admission.request_id = $1`, + [requestId], + ); + assert.equal(result.rowCount, 1); + const { receiptJson, ...facts } = result.rows[0]; + assert.equal(typeof receiptJson, 'string'); + return { + ...facts, + receiptJsonDigest: createHash('sha256').update(receiptJson).digest('hex'), + receiptContainsModelContent: + receiptJson.includes('diagnosis-sensitive-line') || + receiptJson.includes('must-be-redacted'), + }; +} + async function runCopilotFailureDiagnosisAdmissionHaEvidence(options) { const { primaryPort, migrationPool } = options; const baseTimeMs = Date.now(); @@ -1965,6 +2099,8 @@ async function executeCopilotFailureDiagnosisToolHaEvidence(options) { fixture, resultKeyFixture, expectedExisting = false, + expectedTerminalized = false, + logStatus = 'available', } = options; const runtimeDatabase = await databaseOpener( 'runtime', @@ -2030,6 +2166,8 @@ async function executeCopilotFailureDiagnosisToolHaEvidence(options) { assert.equal(request.projectId, fixture.plan.projectId); assert.equal(request.runId, report.sourceRunId); assert.equal(request.attemptId, report.sourceAttemptId); + if (logStatus === 'not_found') return { status: 'not_found' }; + assert.equal(logStatus, 'available'); const totalBytes = logContent.byteLength; const start = Math.min(request.range.offset, totalBytes); const endExclusive = Math.min(start + request.range.length, totalBytes); @@ -2092,7 +2230,7 @@ async function executeCopilotFailureDiagnosisToolHaEvidence(options) { assert.equal(replay.unlockStatus, 'existing'); assert.deepEqual(replay.completion, first.completion); assert.deepEqual(replay.unlock, first.unlock); - assert.equal(logReads, 2); + assert.equal(logReads, logStatus === 'not_found' ? 1 : 2); } else { assert.equal(logReads, 0); } @@ -2116,16 +2254,16 @@ async function executeCopilotFailureDiagnosisToolHaEvidence(options) { resultArtifactContainsPlaintext: facts.resultArtifactContainsPlaintext, }, { - status: 'running', - version: 6, - eventSequence: 6, - eventCount: 6, - mutationCount: 5, + status: expectedTerminalized ? 'failed' : 'running', + version: expectedTerminalized ? 8 : 6, + eventSequence: expectedTerminalized ? 8 : 6, + eventCount: expectedTerminalized ? 8 : 6, + mutationCount: expectedTerminalized ? 6 : 5, startCount: 1, completionCount: 1, unlockCount: 1, toolStatus: 'succeeded', - modelStatus: 'ready', + modelStatus: expectedTerminalized ? 'failed' : 'ready', resultArtifactPresent: true, resultArtifactContainsPlaintext: false, }, @@ -2139,7 +2277,7 @@ async function executeCopilotFailureDiagnosisToolHaEvidence(options) { report.promotedToolReplayReads = logReads; report.toolExecutionSurvivedPromotion = true; } - return facts; + return { facts, execution: first }; } finally { fixture.invocationKey.fill(0); logContent.fill(0); @@ -2147,6 +2285,242 @@ async function executeCopilotFailureDiagnosisToolHaEvidence(options) { } } +async function terminalizeCopilotFailureDiagnosisLogUnavailableHaEvidence( + options, +) { + const { + port, + report, + fixture, + resultKeyFixture, + expectedExisting = false, + } = options; + const tool = await executeCopilotFailureDiagnosisToolHaEvidence({ + port, + report, + fixture, + resultKeyFixture, + expectedExisting, + expectedTerminalized: expectedExisting, + logStatus: 'not_found', + }); + const runtimeDatabase = await databaseOpener( + 'runtime', + databaseUrl(RUNTIME_USER, RUNTIME_PASSWORD, port), + expectedExisting + ? 'ql3-ha-copilot-terminalization-promoted' + : 'ql3-ha-copilot-terminalization-primary', + )(); + try { + const repository = + new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository( + runtimeDatabase.pool, + ); + const trigger = { + kind: 'tool_projection', + completion: tool.execution.completion, + output: tool.execution.output, + }; + const first = await terminalizeCopilotFailureDiagnosisBeforeModel( + report.requestId, + trigger, + { repository }, + ); + const replay = await terminalizeCopilotFailureDiagnosisBeforeModel( + report.requestId, + trigger, + { repository }, + ); + assert.equal(first.status, expectedExisting ? 'existing' : 'created'); + assert.equal(replay.status, 'existing'); + assert.deepEqual(replay.receipt, first.receipt); + assert.equal(first.receipt.reason, 'log_not_found'); + assert.equal(first.receipt.outcome, 'failed'); + assert.equal(first.receipt.terminalSteps.length, 1); + assert.equal( + first.receipt.terminalSteps[0].stepRunId, + report.modelStepRunId, + ); + assert.equal(first.receipt.terminalSteps[0].status, 'failed'); + const facts = await copilotFailureDiagnosisTerminalizationFacts( + runtimeDatabase.pool, + report.requestId, + ); + assert.deepEqual( + { + stage: facts.stage, + reason: facts.reason, + outcome: facts.outcome, + runStatus: facts.runStatus, + runVersion: facts.runVersion, + runEventSequence: facts.runEventSequence, + eventCount: facts.eventCount, + mutationCount: facts.mutationCount, + terminalizationCount: facts.terminalizationCount, + toolStatus: facts.toolStatus, + modelStatus: facts.modelStatus, + receiptContainsLogContent: facts.receiptContainsLogContent, + }, + { + stage: 'log', + reason: 'log_not_found', + outcome: 'failed', + runStatus: 'failed', + runVersion: 8, + runEventSequence: 8, + eventCount: 8, + mutationCount: 6, + terminalizationCount: 1, + toolStatus: 'succeeded', + modelStatus: 'failed', + receiptContainsLogContent: false, + }, + ); + if (expectedExisting) { + report.afterPromotion = facts; + report.survivedPromotion = true; + report.promotedExactReplay = true; + } else { + report.beforePromotion = facts; + report.exactReplay = true; + report.contentFree = true; + } + return facts; + } finally { + await runtimeDatabase.close(); + } +} + +async function resolveCopilotFailureDiagnosisUnknownModelHaEvidence(options) { + const { + port, + report, + fixture, + resultKeyFixture, + expectedExisting = false, + } = options; + if (!expectedExisting) { + await executeCopilotFailureDiagnosisToolHaEvidence({ + port, + report, + fixture, + resultKeyFixture, + }); + } + const runtimeDatabase = await databaseOpener( + 'runtime', + databaseUrl(RUNTIME_USER, RUNTIME_PASSWORD, port), + expectedExisting + ? 'ql3-ha-copilot-model-resolution-promoted' + : 'ql3-ha-copilot-model-resolution-primary', + )(); + try { + const repository = new PostgresCopilotFailureDiagnosisModelRepository( + runtimeDatabase.pool, + ); + const coordinator = new DurableModelInvocationCoordinator(repository); + const auditBase = { + projectId: fixture.plan.projectId, + runId: fixture.plan.runId, + stepRunId: fixture.plan.modelStepRunId, + traceId: fixture.plan.traceId, + requestId: fixture.plan.modelInvocationId, + provider: fixture.plan.model.provider, + model: fixture.plan.model.model, + policyRevision: 'ha-diagnosis-resolution-policy-v1', + requestDigest: `sha256:${'f'.repeat(64)}`, + deadlineAtMs: fixture.plan.deadlineAtMs, + inputBytes: 128, + maxOutputTokens: fixture.plan.model.maxOutputTokens, + }; + const admitted = await coordinator.record({ + ...auditBase, + phase: 'admitted', + outputBytes: 0, + usage: null, + errorCode: null, + occurredAtMs: fixture.baseTimeMs + 500, + }); + const completed = await coordinator.record({ + ...auditBase, + phase: 'failed', + outputBytes: 0, + usage: null, + errorCode: 'MODEL_INVOCATION_OUTCOME_UNKNOWN', + occurredAtMs: fixture.baseTimeMs + 600, + }); + const resolution = await new DurableModelInvocationResolutionCoordinator( + repository, + ).resolve({ + invocationId: fixture.plan.modelInvocationId, + decision: 'fail', + resolvedByUserId: fixture.plan.requestedBySubject.id, + resolvedAtMs: fixture.baseTimeMs + 700, + }); + const finalization = await repository.finalize(report.requestId); + const replay = await repository.finalize(report.requestId); + assert.equal(admitted.status, expectedExisting ? 'existing' : 'created'); + assert.equal(completed.status, expectedExisting ? 'existing' : 'created'); + assert.equal(resolution.status, expectedExisting ? 'existing' : 'created'); + assert.equal( + finalization.status, + expectedExisting ? 'existing' : 'created', + ); + assert.equal(replay.status, 'existing'); + assert.deepEqual(replay.receipt, finalization.receipt); + assert.equal(finalization.receipt.outcome, 'failed'); + assert.equal(finalization.receipt.outputArtifactId, null); + const facts = await copilotFailureDiagnosisModelResolutionFacts( + runtimeDatabase.pool, + report.requestId, + ); + assert.deepEqual( + { + runStatus: facts.runStatus, + runVersion: facts.runVersion, + runEventSequence: facts.runEventSequence, + modelStatus: facts.modelStatus, + modelVersion: facts.modelVersion, + completionOutcome: facts.completionOutcome, + resolutionDecision: facts.resolutionDecision, + finalizationOutcome: facts.finalizationOutcome, + eventCount: facts.eventCount, + mutationCount: facts.mutationCount, + resolutionCount: facts.resolutionCount, + finalizationCount: facts.finalizationCount, + receiptContainsModelContent: facts.receiptContainsModelContent, + }, + { + runStatus: 'failed', + runVersion: 10, + runEventSequence: 10, + modelStatus: 'failed', + modelVersion: 5, + completionOutcome: 'outcome_unknown', + resolutionDecision: 'fail', + finalizationOutcome: 'failed', + eventCount: 10, + mutationCount: 8, + resolutionCount: 1, + finalizationCount: 1, + receiptContainsModelContent: false, + }, + ); + if (expectedExisting) { + report.afterPromotion = facts; + report.survivedPromotion = true; + report.promotedExactReplay = true; + } else { + report.beforePromotion = facts; + report.exactReplay = true; + report.contentFree = true; + } + return facts; + } finally { + await runtimeDatabase.close(); + } +} + async function runModelProviderCredentialCatalogMatrix(options) { const { primaryPort, migrationPool } = options; const suffix = `${process.pid}-${randomBytes(3).toString('hex')}`; @@ -10598,6 +10972,10 @@ async function main(argv = process.argv.slice(2)) { let modelInvocationFeaturePromotion; let copilotFailureDiagnosisAdmission; let copilotFailureDiagnosisToolFixture; + let copilotFailureDiagnosisTerminalization; + let copilotFailureDiagnosisTerminalizationFixture; + let copilotFailureDiagnosisModelResolution; + let copilotFailureDiagnosisModelResolutionFixture; let modelProviderCredentialCatalog; let modelProviderCredentialTestConnection; let runAttemptLogRetentionEvidence; @@ -10698,6 +11076,7 @@ async function main(argv = process.argv.slice(2)) { 'copilot_failure_diagnosis_admissions', 'copilot_failure_diagnosis_finalizations', 'copilot_failure_diagnosis_model_outputs', + 'copilot_failure_diagnosis_pre_model_terminalizations', 'copilot_failure_diagnosis_tool_unlocks', 'model_invocation_completions', 'model_invocation_price_quotes', @@ -10751,6 +11130,7 @@ async function main(argv = process.argv.slice(2)) { POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID, POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID, + POSTGRES_COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_MIGRATION_ID, ], ); assert.deepEqual(beforePromotion.privileges, { @@ -10777,6 +11157,7 @@ async function main(argv = process.argv.slice(2)) { copilotDiagnosisAdmissionRuntimeOnly: true, copilotDiagnosisToolUnlockRuntimeOnly: true, copilotDiagnosisModelExecutionRuntimeOnly: true, + copilotDiagnosisPreModelTerminalizationRuntimeOnly: true, copilotDiagnosisSnapshotRuntimeOnly: true, migrationHistoryRuntimeReadOnly: true, modelProviderCredentialManagementAuthoritySplit: true, @@ -10789,6 +11170,22 @@ async function main(argv = process.argv.slice(2)) { }); copilotFailureDiagnosisAdmission = copilotDiagnosis.report; copilotFailureDiagnosisToolFixture = copilotDiagnosis.fixture; + const copilotTerminalization = + await runCopilotFailureDiagnosisAdmissionHaEvidence({ + primaryPort, + migrationPool: migrationDatabase.pool, + }); + copilotFailureDiagnosisTerminalization = copilotTerminalization.report; + copilotFailureDiagnosisTerminalizationFixture = + copilotTerminalization.fixture; + const copilotModelResolution = + await runCopilotFailureDiagnosisAdmissionHaEvidence({ + primaryPort, + migrationPool: migrationDatabase.pool, + }); + copilotFailureDiagnosisModelResolution = copilotModelResolution.report; + copilotFailureDiagnosisModelResolutionFixture = + copilotModelResolution.fixture; } finally { await migrationDatabase.close(); } @@ -11193,6 +11590,48 @@ async function main(argv = process.argv.slice(2)) { state: 'copilot_failure_diagnosis_tool_completed', atMs: Number((performance.now() - startedAt).toFixed(3)), }); + await terminalizeCopilotFailureDiagnosisLogUnavailableHaEvidence({ + port: primaryPort, + report: copilotFailureDiagnosisTerminalization, + fixture: copilotFailureDiagnosisTerminalizationFixture, + resultKeyFixture: toolResultKeyFixture, + }); + await waitFor(async () => { + const replicated = await copilotFailureDiagnosisTerminalizationFacts( + standbyDatabase.pool, + copilotFailureDiagnosisTerminalization.requestId, + ); + return JSON.stringify(replicated) === + JSON.stringify(copilotFailureDiagnosisTerminalization.beforePromotion) + ? replicated + : null; + }, 'Copilot failure diagnosis pre-Model terminalization WAL replay'); + copilotFailureDiagnosisTerminalization.replicatedBeforePromotion = true; + timeline.push({ + state: 'copilot_failure_diagnosis_pre_model_terminalized', + atMs: Number((performance.now() - startedAt).toFixed(3)), + }); + await resolveCopilotFailureDiagnosisUnknownModelHaEvidence({ + port: primaryPort, + report: copilotFailureDiagnosisModelResolution, + fixture: copilotFailureDiagnosisModelResolutionFixture, + resultKeyFixture: toolResultKeyFixture, + }); + await waitFor(async () => { + const replicated = await copilotFailureDiagnosisModelResolutionFacts( + standbyDatabase.pool, + copilotFailureDiagnosisModelResolution.requestId, + ); + return JSON.stringify(replicated) === + JSON.stringify(copilotFailureDiagnosisModelResolution.beforePromotion) + ? replicated + : null; + }, 'Copilot failure diagnosis Model resolution WAL replay'); + copilotFailureDiagnosisModelResolution.replicatedBeforePromotion = true; + timeline.push({ + state: 'copilot_failure_diagnosis_model_resolution_finalized', + atMs: Number((performance.now() - startedAt).toFixed(3)), + }); await waitFor(async () => { const facts = await standbyDatabase.pool.query( `SELECT @@ -12093,6 +12532,20 @@ async function main(argv = process.argv.slice(2)) { resultKeyFixture: toolResultKeyFixture, expectedExisting: true, }); + await terminalizeCopilotFailureDiagnosisLogUnavailableHaEvidence({ + port: standbyPort, + report: copilotFailureDiagnosisTerminalization, + fixture: copilotFailureDiagnosisTerminalizationFixture, + resultKeyFixture: toolResultKeyFixture, + expectedExisting: true, + }); + await resolveCopilotFailureDiagnosisUnknownModelHaEvidence({ + port: standbyPort, + report: copilotFailureDiagnosisModelResolution, + fixture: copilotFailureDiagnosisModelResolutionFixture, + resultKeyFixture: toolResultKeyFixture, + expectedExisting: true, + }); await verifyModelProviderCredentialCatalogAfterPromotion({ promotedPort: standbyPort, promotedPool: promotedDatabase.pool, @@ -12847,7 +13300,7 @@ async function main(argv = process.argv.slice(2)) { FROM "ql3"."worker_credential_deliveries") AS "credentialDeliveries"`, ); assert.deepEqual(sideEffects.rows, [ - { runs: 78, runEvents: 182, credentialDeliveries: 4 }, + { runs: 82, runEvents: 200, credentialDeliveries: 4 }, ]); timeline.push({ state: 'two_fresh_control_replicas_ready', @@ -12962,6 +13415,8 @@ async function main(argv = process.argv.slice(2)) { runDomainCommitResponseLoss: domainCommitResponseLoss.report, modelInvocationFeaturePromotion, copilotFailureDiagnosisAdmission, + copilotFailureDiagnosisTerminalization, + copilotFailureDiagnosisModelResolution, modelProviderCredentialCatalog, modelProviderCredentialTestConnection, runAttemptLogRetention, @@ -12996,6 +13451,30 @@ async function main(argv = process.argv.slice(2)) { copilotFailureDiagnosisAdmissionReplicatesAndSurvivesPromotion: copilotFailureDiagnosisAdmission.replicatedBeforePromotion && copilotFailureDiagnosisAdmission.survivedPromotion, + copilotFailureDiagnosisPreModelTerminalizationExactlyReplays: + copilotFailureDiagnosisTerminalization.exactReplay && + copilotFailureDiagnosisTerminalization.beforePromotion + .terminalizationCount === 1 && + copilotFailureDiagnosisTerminalization.beforePromotion.reason === + 'log_not_found' && + copilotFailureDiagnosisTerminalization.beforePromotion.runStatus === + 'failed' && + copilotFailureDiagnosisTerminalization.beforePromotion.modelStatus === + 'failed', + copilotFailureDiagnosisPreModelTerminalizationIsContentFree: + copilotFailureDiagnosisTerminalization.contentFree && + copilotFailureDiagnosisTerminalization.beforePromotion + .receiptContainsLogContent === false, + copilotFailureDiagnosisPreModelTerminalizationReplicatesAndSurvivesPromotion: + copilotFailureDiagnosisTerminalization.replicatedBeforePromotion && + copilotFailureDiagnosisTerminalization.survivedPromotion && + copilotFailureDiagnosisTerminalization.promotedExactReplay && + JSON.stringify( + copilotFailureDiagnosisTerminalization.afterPromotion, + ) === + JSON.stringify( + copilotFailureDiagnosisTerminalization.beforePromotion, + ), copilotFailureDiagnosisAdmissionUsesLeastPrivilegeRuntime: modelInvocationFeaturePromotion.beforePromotion.privileges .copilotDiagnosisAdmissionRuntimeOnly && @@ -13003,6 +13482,33 @@ async function main(argv = process.argv.slice(2)) { .copilotDiagnosisToolUnlockRuntimeOnly && modelInvocationFeaturePromotion.beforePromotion.privileges .copilotDiagnosisSnapshotRuntimeOnly, + copilotFailureDiagnosisTerminalizationUsesLeastPrivilegeRuntime: + modelInvocationFeaturePromotion.beforePromotion.privileges + .copilotDiagnosisPreModelTerminalizationRuntimeOnly, + copilotFailureDiagnosisUnknownModelResolutionExactlyReplays: + copilotFailureDiagnosisModelResolution.exactReplay && + copilotFailureDiagnosisModelResolution.beforePromotion + .completionOutcome === 'outcome_unknown' && + copilotFailureDiagnosisModelResolution.beforePromotion + .resolutionDecision === 'fail' && + copilotFailureDiagnosisModelResolution.beforePromotion + .finalizationOutcome === 'failed' && + copilotFailureDiagnosisModelResolution.beforePromotion + .finalizationCount === 1, + copilotFailureDiagnosisUnknownModelResolutionIsContentFree: + copilotFailureDiagnosisModelResolution.contentFree && + copilotFailureDiagnosisModelResolution.beforePromotion + .receiptContainsModelContent === false, + copilotFailureDiagnosisUnknownModelResolutionReplicatesAndSurvivesPromotion: + copilotFailureDiagnosisModelResolution.replicatedBeforePromotion && + copilotFailureDiagnosisModelResolution.survivedPromotion && + copilotFailureDiagnosisModelResolution.promotedExactReplay && + JSON.stringify( + copilotFailureDiagnosisModelResolution.afterPromotion, + ) === + JSON.stringify( + copilotFailureDiagnosisModelResolution.beforePromotion, + ), modelProviderCredentialCatalogSurvivesPromotion: modelProviderCredentialCatalog.survivedPromotion, modelProviderCredentialManagementIdentityLedgerSurvivesPromotion: diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index 9456c7d1..f40d8749 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -268,10 +268,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( { path: 'packages/ql3-ai', name: '@qinglong/ai', - sourceFiles: 187, + sourceFiles: 192, rootSourceFiles: 1, rootSourceLines: 16, - nestedSourceFiles: 186, + nestedSourceFiles: 191, rootSourceFileHardCap: 1, rootSourceLineHardCap: 16, rootSourceFileRoles: {