From e5a3d83a8df3503f71cdd6253141929c5c293dfd Mon Sep 17 00:00:00 2001 From: whyour Date: Sun, 16 Aug 2026 01:34:55 +0800 Subject: [PATCH] feat(ql3): add fenced copilot diagnosis cancellation --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 1 + ...-copilot-failure-diagnosis-cancellation.md | 39 ++ docs/adr/README.md | 3 + packages/ql3-ai/package.json | 5 + .../failure-diagnosis/cancellation/service.ts | 415 ++++++++++++++++++ .../failure-diagnosis-cancellation.test.cjs | 341 ++++++++++++++ packages/ql3-cluster-control/package.json | 10 + .../aiProductionApplication.ts | 22 +- ...failureDiagnosisCancellationComposition.ts | 47 ++ .../productionApplication.ts | 20 +- .../failureDiagnosisCancellationRoute.ts | 261 +++++++++++ .../test/aiProductionApplication.test.cjs | 11 + ...FailureDiagnosisCancellationRoute.test.cjs | 232 ++++++++++ .../test/productionApplication.test.cjs | 45 ++ scripts/ql3-cluster-dependency-audit.cjs | 9 + scripts/ql3-postgres-ha-contract.cjs | 224 +++++++++- test/back/ql3PackageBoundaryAudit.test.cjs | 8 +- 17 files changed, 1685 insertions(+), 8 deletions(-) create mode 100644 docs/adr/ADR-0415-fenced-copilot-failure-diagnosis-cancellation.md create mode 100644 packages/ql3-ai/src/copilot/failure-diagnosis/cancellation/service.ts create mode 100644 packages/ql3-ai/test/failure-diagnosis-cancellation.test.cjs create mode 100644 packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisCancellationComposition.ts create mode 100644 packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts create mode 100644 packages/ql3-cluster-control/test/copilotFailureDiagnosisCancellationRoute.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 53890df5..c09c099c 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,7 @@ 最新增量证据（2026-08-16）： +- D-323/ADR-0415（已接受）：Cluster Copilot failure diagnosis 已补齐受围栏的 request-keyed 取消 mutation。`POST /api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/cancellation` 复用 exact `qinglong/run-cancellation@v1` body、`run.stop` 当前 Policy、durable audit 与通用 PostgreSQL Run cancellation transaction；调用方只能提供 `mutationId`，diagnosis Run、Event、reason、Provider 与终态均由服务端从 admission plan/receipt 和 durable authority 派生。pre-Model 取消立即与 Model start 通过 Run/Step/version fence 竞争并原子 terminalize；Model start 获胜时只返回 `model_in_flight` durable intent，不伪造 Provider abort、终态、usage 或 cost，真实 completion/finalization 仍可成为最终 winner。相同或不同 mutation 都不能追加第二个 intent/Event 或覆盖首次 reason/time。能力只注入默认关闭的 Cluster AI profile，复用既有 AI Pool、repository、Policy pipeline 与进程，不新增 package、schema、Pool、连接、timer、watcher、queue、cache、端口、Pod、Service 或 Kubernetes 权限，Edge/Standalone 与普通 Cluster Control 均无该 route。目录审计曾拒绝 `failure-diagnosis` 达到 12 个直属源码文件；最终删除仅做 re-export 的单文件 façade，让公开 package subpath 直接指向 `cancellation/service`，而不是再拆 package 或放宽阈值。workspace 保持 18 package，`singleSourcePackages=[]`、`shallowSourcePackages=[]`，AI 194 个源码中 193 个、Cluster Control 64 个源码中 62 个位于嵌套职责目录。AI 254 pass/3 条件 skip、Cluster Control 261 pass/2 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail；四项架构审计和 14 档 Local artifact 全部通过。默认 Edge/Standalone 为 2,589,890/2,589,968 bytes，Edge/Standalone AI application 为 4,493,043/4,493,175 bytes，证明 Cluster-only mutation 未进入低配闭包。PostgreSQL 18.6 arm64 physical HA 142/142、timeline `1→2`，覆盖 intent/Event/terminal receipt 同步复制、promotion 后 exact replay 与 in-flight 不伪造终态；报告 SHA-256 为 `5dbcffb74a3181aabee66a8f68ecfa7a65e0491a6f2ba24e2bc903c83da9d766`，离线审计零 finding。下一独立 Gate 可选择 CLI/UI/MCP 客户端或带 Provider abort acknowledgement 的取消语义，不得在没有 durable acknowledgement 时宣称外部调用已停止。 - D-322/ADR-0414（已接受）：Cluster Copilot failure diagnosis 已补齐 request-keyed 产品读模型。`GET /api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}` 只需 `run.read`，投影 running/terminal、取消/失败 stage/reason、authoritative admission/finalization time 与 durable usage/settled cost；同路径 `/output` 独立要求 `artifact.read`，只在 admission plan/receipt、Project/source Run、finalization、invocation 和 encrypted Artifact 全部 exact binding 且 current Project Policy 再授权后解析 historical key。调用者不能提交 Artifact、diagnosis Run、Model invocation、provider/model、价格、usage、outcome 或 key identity；deny/approval/absent/cross-target 统一 404，存储/Policy/key/decrypt/cipher 冲突统一 503。写 `capability` 与 `readCapability` 接口隔离，production 复用既有 AI PostgreSQL Pool、repository、Project Policy 与 projected output keyring，不新增 package、进程、端口、Pool、timer、watcher、queue、cache、Pod、Service 或 Kubernetes 权限；普通 Cluster Control、Edge/Standalone 均无这些 route。workspace 保持 18 package、`singleSourcePackages=[]`、`shallowSourcePackages=[]`，AI 193 个源码中 192 个、Cluster Control 62 个源码中 60 个位于嵌套领域目录。AI 249 pass/3 条件 skip、Cluster Control 256 pass/2 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计和 14 档 Local artifact 全部通过；默认 Edge/Standalone 仍为 2,589,890/2,589,968 bytes，Edge/Standalone AI 为 3,069,143/3,069,233 bytes。PostgreSQL 18.6 arm64 physical HA 139/139、timeline `1→2`，成功密文输出在 standby 可读，提升后 request-keyed exact replay 且 provider 调用为零；报告 SHA-256 为 `22decb54cfb8735bf787fe0665c877c201fc7b44d3c3de16fdbfdab31b7ac2cd`，离线审计零 finding。下一独立 Gate 可评审取消 mutation、CLI/UI/MCP 客户端或真实 Provider，不得把它们混入读边界。 - D-321/ADR-0413（已接受）：Cluster Copilot failure diagnosis 已开放唯一、默认关闭的产品写入口 `POST /api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses`。该 route 只在显式 AI 进程且 `QL3_CLUSTER_AI_COPILOT_ENABLED=true` 时注入既有 Cluster Control route registry，完整复用同一认证器、`model.invoke` Project Policy、fail-closed 同步安全审计、HTTP/TLS body/response/concurrency 上限和生命周期；内部只读 Tool 仍独立复验其 exact `tool.call:*` Policy。耐久幂等身份直接绑定 `x-request-id`，body 只允许 schema 与 `traceId`，Project/source Run/principal 来自 canonical path 和认证，Attempt、日志范围、Tool、provider/model、预算、deadline、Policy fence、reason/outcome 均由服务端从数据库与只读配置派生。响应只投影 created/existing、source/diagnosis Run、终态 stage/reason/outcome 和加密输出 Artifact id/digest，不返回日志、Tool/Model plaintext、prompt、模型信息、密钥或内部异常。普通 Cluster Control、Edge、Standalone 和未启用 Copilot 的 AI 进程没有 route，也没有新增 package、进程、监听器、Pool/连接、timer、watcher、队列、cache、Pod、Service 或 Kubernetes 权限；workspace 仍为 18 package、无单文件/浅平 package，新增源码位于既有嵌套 Copilot 领域目录。Cluster Control 250 pass/2 条件 skip，18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计和 14 档 Local artifact 全部通过；默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes。PostgreSQL 18.6 arm64 HA 137/137、timeline `1→2`，报告 SHA-256 为 `0a12b5c1102555823d43b5a93dd7868b98b194b491840a5242bab6fa2da26123`，离线审计零 finding。本 Gate 不改 migration/schema/role/SQL/HA 拓扑。下一 Gate 优先补同一 Policy 下的加密诊断输出读取与费用/取消可观测性；CLI/UI/MCP 只能复用该 API，不得直连 application capability。 - D-320/ADR-0412（已接受）：Cluster Copilot failure diagnosis 已补齐产品入口前的非成功收敛边界。新增的 pre-Model terminalizer 只从 durable Tool failure、受审日志 projection 或数据库观察到的 deadline/cancellation 派生封闭 reason，在 Model start 不存在时以一个 SERIALIZABLE 事务原子提交 StepRun mutations、RunEvents、父 Run 终态和 `pg-9021` append-only content-free receipt；Tool `failed|timed_out`、日志 `not_found|pending|missing|retired`、五秒 Tool budget 不足、deadline 和 cancellation 均可 exact replay。Model 已开始后的 `outcome_unknown` 继续要求强 User 显式 `fail|cancel|retry` resolution；Copilot finalizer 现在对 `fail|cancel` 精确校验 resolution mutation 的 resolved Step digest，既不伪造失败也不自动重试 Provider。实现复用既有 AI package、Cluster AI 进程、Pool 与 ledger，不增加 package、进程、连接、timer、watcher、队列、cache 或产品 route；workspace 仍为 18 package、无单文件/浅平 package，AI 192 个源码中 191 个位于嵌套领域目录。AI 完整测试 244 pass/3 条件 skip，18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计和 14 档 Local artifact 全部通过。PostgreSQL 18.6 arm64 HA 137/137、timeline `1→2`，覆盖日志不可用 pre-Model terminalization、unknown completion→人工 fail resolution→Run finalization 及晋升后零外部副作用 exact replay；报告 SHA-256 为 `6eaeb20615a62d153c5a69687344f41f31351c6ecf111cfb9cbafad115538c83`，离线审计零 finding。下一 Gate 才增加认证、Policy、audit 与 source fence 保护的 Cluster API，并由 CLI/UI/MCP 复用，仍不得建立旁路执行器。 diff --git a/docs/adr/ADR-0415-fenced-copilot-failure-diagnosis-cancellation.md b/docs/adr/ADR-0415-fenced-copilot-failure-diagnosis-cancellation.md new file mode 100644 index 00000000..83fc5403 --- /dev/null +++ b/docs/adr/ADR-0415-fenced-copilot-failure-diagnosis-cancellation.md @@ -0,0 +1,39 @@ +# ADR-0415：受围栏的 Copilot 故障诊断取消入口 + +- 状态：Accepted +- 日期：2026-08-16 +- 关联 RFC：QL-RFC-0001 D-323、Phase 2 + +## 背景 + +D-321 已提供受认证的故障诊断执行入口，D-322 已提供 request-keyed 状态与密文输出读取，但用户还不能按原始诊断 `requestId` 停止不再需要的诊断。直接要求调用者取消内部 diagnosis Run 会泄露实现 identity；直接复用普通 Run cancellation route 又无法验证 source Run、原始 request 与 diagnosis Run 的三方绑定。 + +现有 durable authority 已经足够：admission plan/receipt 唯一绑定 Project、source Run、request 与 diagnosis Run；通用 Run cancellation transaction 会重验当前 Project/RoleBinding Policy fence，并以数据库时间写唯一 cancellation intent/Event；pre-Model terminalizer 能在 Model start 不存在时原子终结 Tool/Model Step、父 Run 与 receipt。Model start 之后目前没有 Provider abort acknowledgement，因而不能把“已写停止意图”伪装成“Provider 已停止”。 + +## 决策 + +1. Cluster AI profile 新增： + `POST /api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/cancellation`，固定 operation `copilot.failure_diagnosis.cancel`、permission `run.stop`，不接受 query。 +2. 请求复用 profile-neutral exact body `qinglong/run-cancellation@v1`，仅含 `schema` 与调用方生成的 `mutationId`。调用方不能提交 diagnosis Run、Event ID、取消原因、Model invocation、Provider 或终态；Event ID 由服务端生成。 +3. admission plan 与 receipt 必须同时存在，并 exact 绑定 path 中的 Project、source Run 和 request。服务端只使用 plan 中的 diagnosis Run ID 调用既有 PostgreSQL Run cancellation repository；不存在、跨 Project、跨 source Run 或漂移统一返回 404。 +4. route admission 负责 Bearer authentication、`run.stop` 当前 Policy 判定与 durable audit；Run cancellation transaction 在写 intent/Event 的同一 SERIALIZABLE 事务中再次锁定并验证当前 Project/RoleBinding fence。撤权竞态返回 409，存储或 durable evidence 冲突返回 503。 +5. Model start 尚不存在时，取消 intent 提交后立即调用既有 pre-Model terminalizer。terminalizer 与 Model start 通过 PostgreSQL Run/Step/version fence 竞争：terminalizer 获胜则 Run/未完成 Step 原子进入 `cancelled`；Model start 获胜则进入 `model_in_flight`。 +6. Model 已开始时只承诺 durable cancellation intent，不伪造 Provider abort、usage、cost 或终态。真实 Provider completion 可以成为最终结果；`outcome_unknown` 仍必须走既有强 User resolution。后续若增加 Provider abort，必须另立包含 abort acknowledgement、late completion 与计费语义的 ADR。 +7. 响应 schema 为 `qinglong/cluster-copilot-failure-diagnosis-cancellation-response@v1`，投影 `accepted|already_requested|already_terminal`、`terminal|model_in_flight`、exact target、diagnosis Run、当前/终态 outcome 与 cancellation fact。首次 intent 为 202，其余幂等重放为 200。 +8. 相同 mutation 的响应丢失重放不得追加 Event 或再次递增版本；不同 mutation 也不得覆盖已有 reason/time。取消与 Model start/finalization 的任一竞态只能有一个合法 durable winner。 +9. 实现留在既有 `@qinglong/ai` 和 `@qinglong/cluster-control` 的嵌套 Copilot 领域目录，复用同一 AI PostgreSQL Pool、repositories、Policy pipeline 与进程；不新增 package、schema、Pool、连接、timer、watcher、队列、cache、端口、Pod、Service 或 Kubernetes 权限。普通 Cluster Control、Edge 与 Standalone 不注册该 route。 + +## 不选择 + +- **让客户端直接调用 diagnosis Run 的通用 cancellation route**：暴露内部 Run identity，且无法证明 source/request binding。 +- **Model start 后直接写 cancelled**：没有 Provider abort acknowledgement，会伪造外部副作用与计费事实，并允许迟到 completion 覆盖假终态。 +- **为 Copilot cancellation 新建 package、表或 supervisor**：现有 admission、Run intent、terminalization 与 Model recovery ledger 已覆盖所需 authority；新资源只会复制状态机并增加低配设备成本。 +- **在 MCP/UI/CLI 中同时实现写入口**：客户端需要独立的强认证、确认交互与错误展示门，不能与服务端 authority 混成一个不可审计改动。 + +## 验收 + +1. 单元/集成测试覆盖 invalid body、权限、跨 Project/source/request、Policy 撤权、首次/相同 mutation/不同 mutation、终态重放、pre-Model 原子 terminalization、Model-start winner、finalization race 与故障映射。 +2. PostgreSQL HA 覆盖 intent/Event/terminal receipt 复制、promotion 后 exact replay、pre-Model winner 和 in-flight 不伪造终态。 +3. `@qinglong/ai`、`@qinglong/cluster-control`、18-package clean build/test、backend、四项架构审计、14 档 Local artifact 与离线 HA 报告审计全部通过后才允许 D-323 阶段提交。 + +最终验收（2026-08-16）：AI 254 pass/3 条件 skip，Cluster Control 261 pass/2 条件 skip，18-package clean build/test 退出 0，backend 1,207 pass/2 条件 skip/0 fail；workspace 保持 18 package，`singleSourcePackages=[]`、`shallowSourcePackages=[]`。目录密度门拒绝新增的第 12 个 failure-diagnosis 直属文件后，删除了仅做 re-export 的单文件 façade，公开 subpath 直接指向内聚的 `cancellation/service`，未新增 package 或放宽目录阈值。四项架构审计、14 档 Edge/Standalone artifact 与 PostgreSQL HA 离线证据审计均通过；PostgreSQL 18.6 arm64 physical HA 为 142/142、timeline `1→2`，报告 SHA-256 为 `5dbcffb74a3181aabee66a8f68ecfa7a65e0491a6f2ba24e2bc903c83da9d766`。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 79726fcf..ed1dc46a 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -416,6 +416,9 @@ | [ADR-0410](./ADR-0410-cluster-copilot-failure-diagnosis-output-key-authority.md) | Cluster Copilot Failure Diagnosis Output Projected Key Authority | Accepted | | [ADR-0411](./ADR-0411-default-off-cluster-copilot-composition.md) | 默认关闭的 Cluster Copilot Failure Diagnosis Composition | Accepted | | [ADR-0412](./ADR-0412-durable-copilot-pre-model-terminalization-and-unknown-resolution.md) | 耐久 Copilot Pre-Model 终态化与 Model Unknown Resolution | Accepted | +| [ADR-0413](./ADR-0413-default-off-cluster-copilot-failure-diagnosis-api.md) | 默认关闭的 Cluster Copilot Failure Diagnosis API | Accepted | +| [ADR-0414](./ADR-0414-copilot-failure-diagnosis-read-model.md) | Copilot 故障诊断请求键读模型 | Accepted | +| [ADR-0415](./ADR-0415-fenced-copilot-failure-diagnosis-cancellation.md) | 受围栏的 Copilot 故障诊断取消入口 | Accepted | ## 规则 diff --git a/packages/ql3-ai/package.json b/packages/ql3-ai/package.json index 732b0248..50516739 100644 --- a/packages/ql3-ai/package.json +++ b/packages/ql3-ai/package.json @@ -65,6 +65,11 @@ "require": "./dist/copilot/failure-diagnosis/read-model/service.js", "default": "./dist/copilot/failure-diagnosis/read-model/service.js" }, + "./failure-diagnosis-cancellation": { + "types": "./dist/copilot/failure-diagnosis/cancellation/service.d.ts", + "require": "./dist/copilot/failure-diagnosis/cancellation/service.js", + "default": "./dist/copilot/failure-diagnosis/cancellation/service.js" + }, "./failure-diagnosis-pre-model-terminalization": { "types": "./dist/copilot/failure-diagnosis/preModelTerminalization.d.ts", "require": "./dist/copilot/failure-diagnosis/preModelTerminalization.js", diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/cancellation/service.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/cancellation/service.ts new file mode 100644 index 00000000..94a44187 --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/cancellation/service.ts @@ -0,0 +1,415 @@ +import { RUN_STATUSES, type RunStatus } from '@qinglong/runtime-core'; +import type { + ClusterRunCancellationRepository, + ClusterRunCancellationResult, +} from '@qinglong/runtime-core/cluster-run-cancellation'; +import type { + SecurityPolicyFence, + SecuritySubject, +} from '@qinglong/runtime-core/security'; + +import type { + CopilotFailureDiagnosisAdmissionRepository, + CopilotFailureDiagnosisAdmissionReceipt, + CopilotFailureDiagnosisExecutionPlan, +} from '../admission/contracts'; +import { normalizeCopilotFailureDiagnosisAdmissionReceipt } from '../admission/durableEvidence'; +import { normalizeCopilotFailureDiagnosisExecutionPlan } from '../admission/plan'; +import { + terminalizeCopilotFailureDiagnosisBeforeModel, + type CopilotFailureDiagnosisPreModelTerminalizationDependencies, +} from '../terminalization/coordinator'; +import { + CopilotFailureDiagnosisPreModelTerminalizationConflictError, + CopilotFailureDiagnosisPreModelTerminalizationNotReadyError, + CopilotFailureDiagnosisPreModelTerminalizationUnavailableError, +} from '../terminalization/contracts'; + +export const COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA = + 'qinglong/copilot-failure-diagnosis-cancellation-result@v1' as const; + +export interface CopilotFailureDiagnosisCancellationCommand { + readonly projectId: string; + readonly sourceRunId: string; + readonly requestId: string; + readonly mutationId: string; + readonly eventId: string; + readonly subject: Readonly; + readonly policyFence: Readonly; +} + +export interface CopilotFailureDiagnosisCancellationResult { + readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA; + readonly status: ClusterRunCancellationResult['status']; + readonly convergence: 'terminal' | 'model_in_flight'; + readonly projectId: string; + readonly sourceRunId: string; + readonly requestId: string; + readonly diagnosisRunId: string; + readonly runStatus: RunStatus; + readonly outcome: 'succeeded' | 'failed' | 'timed_out' | 'cancelled' | null; + readonly runVersion: number; + readonly eventSequence: number; + readonly cancelRequestedAtMs: number | null; + readonly cancelReason: + | 'user' + | 'policy' + | 'shutdown' + | 'reconcile' + | 'timeout' + | null; +} + +export interface CopilotFailureDiagnosisCancellationDependencies { + readonly admissions: Pick< + CopilotFailureDiagnosisAdmissionRepository, + 'findByRequestId' | 'findPlanByRequestId' + >; + readonly cancellations: ClusterRunCancellationRepository; + readonly terminalizations: CopilotFailureDiagnosisPreModelTerminalizationDependencies; + readonly terminalizeBeforeModel?: typeof terminalizeCopilotFailureDiagnosisBeforeModel; +} + +export class InvalidCopilotFailureDiagnosisCancellationError extends TypeError { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_INVALID'; + + constructor() { + super('Copilot failure diagnosis cancellation is invalid'); + this.name = 'InvalidCopilotFailureDiagnosisCancellationError'; + } +} + +export class CopilotFailureDiagnosisCancellationNotFoundError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_NOT_FOUND'; + + constructor() { + super('Copilot failure diagnosis cancellation target does not exist'); + this.name = 'CopilotFailureDiagnosisCancellationNotFoundError'; + } +} + +export class CopilotFailureDiagnosisCancellationUnavailableError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_UNAVAILABLE'; + + constructor(options?: ErrorOptions) { + super('Copilot failure diagnosis cancellation is unavailable', options); + this.name = 'CopilotFailureDiagnosisCancellationUnavailableError'; + } +} + +const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/; +const TERMINAL = new Set([ + 'succeeded', + 'failed', + 'cancelled', + 'timed_out', +]); + +function unavailable(cause?: unknown): never { + throw new CopilotFailureDiagnosisCancellationUnavailableError({ + cause: cause instanceof Error ? cause : undefined, + }); +} + +function exactKeys(value: object, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + const canonical = [...expected].sort(); + return ( + actual.length === canonical.length && + actual.every((key, index) => key === canonical[index]) + ); +} + +function command( + value: CopilotFailureDiagnosisCancellationCommand, +): Readonly { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + !exactKeys(value, [ + 'eventId', + 'mutationId', + 'policyFence', + 'projectId', + 'requestId', + 'sourceRunId', + 'subject', + ]) || + !IDENTITY.test(value.projectId) || + !RUN_ID.test(value.sourceRunId) || + !IDENTITY.test(value.requestId) || + !IDENTITY.test(value.mutationId) || + !IDENTITY.test(value.eventId) || + !value.subject || + typeof value.subject !== 'object' || + !value.policyFence || + typeof value.policyFence !== 'object' + ) { + throw new InvalidCopilotFailureDiagnosisCancellationError(); + } + return Object.freeze({ ...value }); +} + +function evidenceMatches( + plan: Readonly, + receipt: Readonly, +): boolean { + return ( + receipt.requestId === plan.requestId && + receipt.planDigest === plan.planDigest && + receipt.runId === plan.runId && + receipt.sourceRunId === plan.source.runId && + receipt.sourceRunVersion === plan.source.runVersion && + receipt.sourceAttemptId === plan.source.attemptId && + receipt.toolStepRunId === plan.toolStepRunId && + receipt.modelStepRunId === plan.modelStepRunId + ); +} + +function targetMatches( + plan: Readonly, + value: Readonly, +): boolean { + return ( + plan.projectId === value.projectId && + plan.source.runId === value.sourceRunId && + plan.requestId === value.requestId + ); +} + +function result( + target: Readonly, + cancellation: Readonly, + state: Readonly<{ + convergence: CopilotFailureDiagnosisCancellationResult['convergence']; + runStatus: RunStatus; + runVersion: number; + eventSequence: number; + cancelRequestedAtMs?: number; + cancelReason?: NonNullable< + CopilotFailureDiagnosisCancellationResult['cancelReason'] + >; + }>, +): Readonly { + if ( + cancellation.projectId !== target.projectId || + !RUN_STATUSES.includes(state.runStatus) || + !Number.isSafeInteger(state.runVersion) || + state.runVersion < 0 || + !Number.isSafeInteger(state.eventSequence) || + state.eventSequence < 0 || + (state.convergence === 'terminal') !== TERMINAL.has(state.runStatus) || + (state.convergence === 'model_in_flight' && state.runStatus !== 'running') + ) { + return unavailable(); + } + return Object.freeze({ + schema: COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA, + status: cancellation.status, + convergence: state.convergence, + projectId: target.projectId, + sourceRunId: target.sourceRunId, + requestId: target.requestId, + diagnosisRunId: cancellation.runId, + runStatus: state.runStatus, + outcome: TERMINAL.has(state.runStatus) + ? (state.runStatus as 'succeeded' | 'failed' | 'timed_out' | 'cancelled') + : null, + runVersion: state.runVersion, + eventSequence: state.eventSequence, + cancelRequestedAtMs: state.cancelRequestedAtMs ?? null, + cancelReason: state.cancelReason ?? null, + }); +} + +/** + * Resolves an external request key to its server-owned diagnosis Run, writes + * one ordinary Run cancellation intent, then converges only while no Model + * invocation start exists. + */ +export class CopilotFailureDiagnosisCancellationService { + readonly #dependencies: Readonly; + readonly #terminalize: typeof terminalizeCopilotFailureDiagnosisBeforeModel; + + constructor(dependencies: CopilotFailureDiagnosisCancellationDependencies) { + if ( + !dependencies || + typeof dependencies !== 'object' || + Array.isArray(dependencies) || + typeof dependencies.admissions?.findByRequestId !== 'function' || + typeof dependencies.admissions?.findPlanByRequestId !== 'function' || + typeof dependencies.cancellations?.requestUserCancellation !== + 'function' || + typeof dependencies.terminalizations?.repository?.findByRequestId !== + 'function' || + typeof dependencies.terminalizations?.repository?.readAuthority !== + 'function' || + typeof dependencies.terminalizations?.repository?.commit !== 'function' || + (dependencies.terminalizeBeforeModel !== undefined && + typeof dependencies.terminalizeBeforeModel !== 'function') + ) { + throw new InvalidCopilotFailureDiagnosisCancellationError(); + } + this.#dependencies = Object.freeze({ ...dependencies }); + this.#terminalize = + dependencies.terminalizeBeforeModel ?? + terminalizeCopilotFailureDiagnosisBeforeModel; + } + + async cancel( + value: CopilotFailureDiagnosisCancellationCommand, + ): Promise> { + const target = command(value); + let plan: Readonly | null; + let receipt: Readonly | null; + try { + const located = await Promise.all([ + this.#dependencies.admissions.findPlanByRequestId(target.requestId), + this.#dependencies.admissions.findByRequestId(target.requestId), + ]); + plan = located[0] + ? normalizeCopilotFailureDiagnosisExecutionPlan(located[0]) + : null; + receipt = located[1] + ? normalizeCopilotFailureDiagnosisAdmissionReceipt(located[1]) + : null; + } catch (cause) { + return unavailable(cause); + } + if (!plan || !receipt || !targetMatches(plan, target)) { + throw new CopilotFailureDiagnosisCancellationNotFoundError(); + } + if (!evidenceMatches(plan, receipt)) return unavailable(); + + const cancellation = + await this.#dependencies.cancellations.requestUserCancellation({ + projectId: target.projectId, + runId: plan.runId, + mutationId: target.mutationId, + eventId: target.eventId, + subject: target.subject, + policyFence: target.policyFence, + }); + if ( + cancellation.projectId !== target.projectId || + cancellation.runId !== plan.runId + ) { + return unavailable(); + } + if (TERMINAL.has(cancellation.runStatus)) { + return result(target, cancellation, { + convergence: 'terminal', + runStatus: cancellation.runStatus, + runVersion: cancellation.runVersion, + eventSequence: cancellation.eventSequence, + ...(cancellation.cancelRequestedAtMs === undefined + ? {} + : { cancelRequestedAtMs: cancellation.cancelRequestedAtMs }), + ...(cancellation.cancelReason === undefined + ? {} + : { cancelReason: cancellation.cancelReason }), + }); + } + + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + const terminalized = await this.#terminalize( + target.requestId, + { kind: 'boundary' }, + this.#dependencies.terminalizations, + ); + return result(target, cancellation, { + convergence: 'terminal', + runStatus: terminalized.receipt.outcome, + runVersion: terminalized.receipt.finalRunVersion, + eventSequence: terminalized.receipt.finalRunEventSequence, + ...(cancellation.cancelRequestedAtMs === undefined + ? {} + : { cancelRequestedAtMs: cancellation.cancelRequestedAtMs }), + ...(cancellation.cancelReason === undefined + ? {} + : { cancelReason: cancellation.cancelReason }), + }); + } catch (cause) { + if ( + !( + cause instanceof + CopilotFailureDiagnosisPreModelTerminalizationConflictError + ) && + !( + cause instanceof + CopilotFailureDiagnosisPreModelTerminalizationNotReadyError + ) + ) { + if ( + cause instanceof + CopilotFailureDiagnosisPreModelTerminalizationUnavailableError + ) { + return unavailable(cause); + } + return unavailable(cause); + } + } + + try { + const authority = + await this.#dependencies.terminalizations.repository.readAuthority( + target.requestId, + ); + if ( + !targetMatches(authority.plan, target) || + authority.plan.planDigest !== plan.planDigest || + authority.run.id !== plan.runId || + authority.run.projectId !== target.projectId || + authority.run.version !== authority.run.eventSequence + ) { + return unavailable(); + } + if (TERMINAL.has(authority.run.status)) { + return result(target, cancellation, { + convergence: 'terminal', + runStatus: authority.run.status, + runVersion: authority.run.version, + eventSequence: authority.run.eventSequence, + ...(authority.run.cancelRequestedAtMs === undefined + ? {} + : { cancelRequestedAtMs: authority.run.cancelRequestedAtMs }), + ...(authority.run.cancelReason === undefined + ? {} + : { cancelReason: authority.run.cancelReason }), + }); + } + if ( + authority.run.status === 'running' && + authority.modelStartExists && + authority.run.cancelRequestedAtMs !== undefined && + authority.run.cancelReason !== undefined + ) { + return result(target, cancellation, { + convergence: 'model_in_flight', + runStatus: 'running', + runVersion: authority.run.version, + eventSequence: authority.run.eventSequence, + cancelRequestedAtMs: authority.run.cancelRequestedAtMs, + cancelReason: authority.run.cancelReason, + }); + } + if ( + attempt === 0 && + authority.run.status === 'running' && + !authority.modelStartExists && + authority.run.cancelRequestedAtMs !== undefined && + authority.run.cancelReason !== undefined + ) { + continue; + } + return unavailable(); + } catch (cause) { + return unavailable(cause); + } + } + return unavailable(); + } +} diff --git a/packages/ql3-ai/test/failure-diagnosis-cancellation.test.cjs b/packages/ql3-ai/test/failure-diagnosis-cancellation.test.cjs new file mode 100644 index 00000000..ef9b9db7 --- /dev/null +++ b/packages/ql3-ai/test/failure-diagnosis-cancellation.test.cjs @@ -0,0 +1,341 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + BUILTIN_RUN_LOG_EXCERPT_TOOL, + BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION, + createBuiltInRunLogExcerptToolHandlerBinding, +} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool'); +const { + createPluginPackageResourceGenerationFromReferences, +} = require('@qinglong/runtime-core/plugin-package-resource-generation'); +const { + createProjectToolDefinitionSnapshot, + projectToolDefinitionRegistry, +} = require('@qinglong/runtime-core/project-tool-definition-snapshot'); +const { + prepareToolInvocation, +} = require('@qinglong/runtime-core/tool-registry'); +const { + TrustedToolHandlerBindingRegistry, + createTrustedToolInvocationPlan, +} = require('@qinglong/runtime-core/trusted-tool-invocation'); +const { + CopilotFailureDiagnosisCancellationNotFoundError, + CopilotFailureDiagnosisCancellationService, + CopilotFailureDiagnosisCancellationUnavailableError, +} = require('@qinglong/ai/failure-diagnosis-cancellation'); +const { + createCopilotFailureDiagnosisAdmissionBundle, + prepareCopilotFailureDiagnosisExecution, +} = require('@qinglong/ai/failure-diagnosis-execution-admission'); +const { + CopilotFailureDiagnosisPreModelTerminalizationConflictError, +} = require('@qinglong/ai/failure-diagnosis-pre-model-terminalization'); + +async function durablePlan() { + const generation = createPluginPackageResourceGenerationFromReferences({ + installationId: 'install-cancel-test', + projectId: 'project-cancel', + packageName: 'qinglong', + lockDigest: 'a'.repeat(64), + generation: 1, + previousActiveLockDigest: null, + contentDigest: 'b'.repeat(64), + resources: [], + }); + const snapshot = createProjectToolDefinitionSnapshot({ + projectId: 'project-cancel', + contributions: [ + { + generation, + revisionDigest: 'c'.repeat(64), + definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION], + }, + ], + }); + const binding = createBuiltInRunLogExcerptToolHandlerBinding(snapshot, [ + 'cluster-control', + ]); + const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]); + const principal = { + subject: { type: 'user', id: 'owner-cancel' }, + authenticationId: 'auth-cancel', + authenticatedAtMs: 100, + expiresAtMs: 10_000, + assurance: 'multi_factor', + }; + const invocation = await prepareToolInvocation( + projectToolDefinitionRegistry(snapshot), + { + projectId: 'project-cancel', + principal, + nowMs: 200, + tool: BUILTIN_RUN_LOG_EXCERPT_TOOL, + input: { runId: 'source-run-cancel', attemptId: 'source-attempt-cancel' }, + }, + { + async authorize() { + return { + effect: 'allow', + reasons: ['role_grant'], + fence: { projectVersion: 2, bindingVersion: 3 }, + }; + }, + }, + ); + const tool = createTrustedToolInvocationPlan(bindings, invocation, { + actionRef: 'cancel-log-tool', + inputArtifactId: 'cancel-input', + previewArtifactId: 'cancel-preview', + artifactKeyId: 'cancel-key', + artifactKey: Buffer.alloc(32, 0x11), + artifactNonce: Buffer.alloc(12, 0x22), + profile: 'cluster-control', + preview: { + title: 'Read failed Run log', + summary: 'Read bounded evidence', + fields: [ + { kind: 'identifier', label: 'Run', value: 'source-run-cancel' }, + { + kind: 'identifier', + label: 'Attempt', + value: 'source-attempt-cancel', + }, + ], + warnings: ['potentially_sensitive_output'], + }, + sealedAtMs: 300, + }); + return prepareCopilotFailureDiagnosisExecution({ + requestId: 'diagnosis-request-cancel', + traceId: 'diagnosis-trace-cancel', + source: { + runId: 'source-run-cancel', + runVersion: 4, + runStatus: 'failed', + attemptId: 'source-attempt-cancel', + attemptStatus: 'failed', + attemptFinishedAtMs: 250, + logArtifactId: 'source-log-cancel', + }, + toolPlan: tool.plan, + bindings, + model: { + provider: 'provider-primary', + model: 'model-diagnosis', + modelBoundary: 'external', + responseLanguage: 'zh-CN', + maxOutputTokens: 256, + egressPolicy: { + schema: 'qinglong/copilot-model-egress-policy@v1', + revision: 'cancel-policy-v1', + potentiallySensitiveDataBoundaries: ['external'], + maxInputBytes: 64 * 1024, + maxOutputTokens: 512, + }, + }, + plannedAtMs: 400, + deadlineAtMs: 8_000, + }); +} + +function command(overrides = {}) { + return { + projectId: 'project-cancel', + sourceRunId: 'source-run-cancel', + requestId: 'diagnosis-request-cancel', + mutationId: '11111111-1111-4111-8111-111111111111', + eventId: '22222222-2222-4222-8222-222222222222', + subject: { type: 'user', id: 'owner-cancel' }, + policyFence: { projectVersion: 2, bindingVersion: 3 }, + ...overrides, + }; +} + +function fixture(plan, overrides = {}) { + const receipt = createCopilotFailureDiagnosisAdmissionBundle(plan).receipt; + const state = { + cancellationCalls: [], + terminalizeCalls: 0, + cancellation: { + status: 'accepted', + projectId: plan.projectId, + runId: plan.runId, + runStatus: 'running', + runVersion: 4, + eventSequence: 4, + cancelRequestedAtMs: 500, + cancelReason: 'user', + }, + authority: { + plan, + run: { + id: plan.runId, + projectId: plan.projectId, + status: 'running', + version: 4, + eventSequence: 4, + cancelRequestedAtMs: 500, + cancelReason: 'user', + }, + toolStep: {}, + modelStep: {}, + modelStartExists: false, + observedAtMs: 501, + }, + ...overrides, + }; + const terminalizations = { + repository: { + async findByRequestId() { + return null; + }, + async readAuthority() { + return state.authority; + }, + async commit() { + throw new Error('not called by injected terminalizer'); + }, + }, + }; + const service = new CopilotFailureDiagnosisCancellationService({ + admissions: { + async findPlanByRequestId() { + return state.plan === undefined ? plan : state.plan; + }, + async findByRequestId() { + return state.receipt === undefined ? receipt : state.receipt; + }, + }, + cancellations: { + async requestUserCancellation(value) { + state.cancellationCalls.push(value); + return state.cancellation; + }, + }, + terminalizations, + async terminalizeBeforeModel(requestId, trigger) { + state.terminalizeCalls += 1; + if (state.terminalizeError) throw state.terminalizeError; + assert.equal(requestId, plan.requestId); + assert.deepEqual(trigger, { kind: 'boundary' }); + return { + status: 'created', + receipt: { + outcome: 'cancelled', + finalRunVersion: 7, + finalRunEventSequence: 7, + }, + }; + }, + }); + return { service, state, receipt }; +} + +test('resolves the request key to a server-owned Run and terminalizes pre-Model', async () => { + const plan = await durablePlan(); + const { service, state } = fixture(plan); + const result = await service.cancel(command()); + assert.deepEqual(state.cancellationCalls, [ + { + projectId: 'project-cancel', + runId: plan.runId, + mutationId: '11111111-1111-4111-8111-111111111111', + eventId: '22222222-2222-4222-8222-222222222222', + subject: { type: 'user', id: 'owner-cancel' }, + policyFence: { projectVersion: 2, bindingVersion: 3 }, + }, + ]); + assert.equal(state.terminalizeCalls, 1); + assert.equal(result.status, 'accepted'); + assert.equal(result.convergence, 'terminal'); + assert.equal(result.runStatus, 'cancelled'); + assert.equal(result.outcome, 'cancelled'); + assert.equal(result.diagnosisRunId, plan.runId); + assert.equal(result.cancelRequestedAtMs, 500); +}); + +test('masks cross-target requests before writing a cancellation intent', async () => { + const plan = await durablePlan(); + const { service, state } = fixture(plan); + await assert.rejects( + service.cancel(command({ sourceRunId: 'other-source-run' })), + CopilotFailureDiagnosisCancellationNotFoundError, + ); + assert.equal(state.cancellationCalls.length, 0); +}); + +test('keeps a durable intent pending when Model start wins the race', async () => { + const plan = await durablePlan(); + const conflict = + new CopilotFailureDiagnosisPreModelTerminalizationConflictError( + 'Model started', + ); + const { service, state } = fixture(plan, { + terminalizeError: conflict, + authority: { + plan, + run: { + id: plan.runId, + projectId: plan.projectId, + status: 'running', + version: 5, + eventSequence: 5, + cancelRequestedAtMs: 500, + cancelReason: 'user', + }, + toolStep: {}, + modelStep: {}, + modelStartExists: true, + observedAtMs: 501, + }, + }); + const result = await service.cancel(command()); + assert.equal(state.terminalizeCalls, 1); + assert.equal(result.convergence, 'model_in_flight'); + assert.equal(result.runStatus, 'running'); + assert.equal(result.outcome, null); + assert.equal(result.runVersion, 5); +}); + +test('reports the real terminal winner instead of forging cancellation', async () => { + const plan = await durablePlan(); + const { service } = fixture(plan, { + terminalizeError: + new CopilotFailureDiagnosisPreModelTerminalizationConflictError(), + authority: { + plan, + run: { + id: plan.runId, + projectId: plan.projectId, + status: 'succeeded', + version: 7, + eventSequence: 7, + cancelRequestedAtMs: 500, + cancelReason: 'user', + }, + toolStep: {}, + modelStep: {}, + modelStartExists: true, + observedAtMs: 600, + }, + }); + const result = await service.cancel(command()); + assert.equal(result.convergence, 'terminal'); + assert.equal(result.runStatus, 'succeeded'); + assert.equal(result.outcome, 'succeeded'); +}); + +test('fails closed when plan and admission receipt drift', async () => { + const plan = await durablePlan(); + const base = fixture(plan); + const { service, state } = fixture(plan, { + receipt: { ...base.receipt, runId: 'diagnosis-run-drift' }, + }); + await assert.rejects( + service.cancel(command()), + CopilotFailureDiagnosisCancellationUnavailableError, + ); + assert.equal(state.cancellationCalls.length, 0); +}); diff --git a/packages/ql3-cluster-control/package.json b/packages/ql3-cluster-control/package.json index f5d6301e..c953ca54 100644 --- a/packages/ql3-cluster-control/package.json +++ b/packages/ql3-cluster-control/package.json @@ -40,6 +40,11 @@ "require": "./dist/application-runtime/copilot/failureDiagnosisComposition.js", "default": "./dist/application-runtime/copilot/failureDiagnosisComposition.js" }, + "./copilot-cancellation-production": { + "types": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.d.ts", + "require": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.js", + "default": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.js" + }, "./failure-diagnosis-output-keyring": { "types": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.d.ts", "require": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js", @@ -55,6 +60,11 @@ "require": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js", "default": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js" }, + "./copilot-cancellation-route": { + "types": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.d.ts", + "require": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.js", + "default": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.js" + }, "./http": { "types": "./dist/transport/httpSurface.d.ts", "require": "./dist/transport/httpSurface.js", diff --git a/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts b/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts index c37d9481..4aeebe4a 100644 --- a/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts +++ b/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts @@ -2,6 +2,7 @@ import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile'; import type { DurableModelInvocationCoordinator } from '@qinglong/ai/durable-model-invocation'; import { CopilotFailureDiagnosisModelCompletionCoordinator } from '@qinglong/ai/failure-diagnosis-model-execution'; import type { CopilotFailureDiagnosisApplicationService } from '@qinglong/ai/failure-diagnosis-application'; +import type { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation'; import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential'; import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage'; import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority'; @@ -35,6 +36,10 @@ import { createProductionClusterCopilotFailureDiagnosisReadService, type CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions, } from './copilot/failureDiagnosisReadComposition'; +import { + createProductionClusterCopilotFailureDiagnosisCancellation, + type CreateProductionClusterCopilotFailureDiagnosisCancellationOptions, +} from './copilot/failureDiagnosisCancellationComposition'; export interface EnabledProductionClusterAiConfig { readonly enabled: true; @@ -63,6 +68,9 @@ export interface ProductionClusterAiControlApplicationOptions { ) => ReturnType< typeof createProductionClusterCopilotFailureDiagnosisReadService >; + readonly createCopilotCancellation?: ( + options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions, + ) => Readonly; readonly openAiDatabase?: ReturnType; } @@ -263,11 +271,15 @@ export async function startProductionClusterAiControlApplication( const createCopilotRead = options.createCopilotRead ?? createProductionClusterCopilotFailureDiagnosisReadService; + const createCopilotCancellation = + options.createCopilotCancellation ?? + createProductionClusterCopilotFailureDiagnosisCancellation; if ( typeof startControl !== 'function' || typeof bootstrapPrompt !== 'function' || typeof createCopilot !== 'function' || typeof createCopilotRead !== 'function' || + typeof createCopilotCancellation !== 'function' || (options.openAiDatabase !== undefined && typeof options.openAiDatabase !== 'function') ) { @@ -325,6 +337,9 @@ export async function startProductionClusterAiControlApplication( typeof createProductionClusterCopilotFailureDiagnosisReadService > | undefined; + let copilotCancellationApplication: + | Readonly + | undefined; let copilotSuccessfulCompletion: | CopilotFailureDiagnosisModelCompletionCoordinator | undefined; @@ -455,6 +470,9 @@ export async function startProductionClusterAiControlApplication( pool: aiDatabase.pool, prepared: preparedCopilot, }); + copilotCancellationApplication = createCopilotCancellation({ + pool: aiDatabase.pool, + }); } controlApplication = await startControl({ ...options.control, @@ -482,12 +500,14 @@ export async function startProductionClusterAiControlApplication( }, }), ...(copilotApplication === undefined || - copilotReadApplication === undefined + copilotReadApplication === undefined || + copilotCancellationApplication === undefined ? {} : { copilotFailureDiagnosis: { capability: copilotApplication, readCapability: copilotReadApplication, + cancellationCapability: copilotCancellationApplication, }, }), }); diff --git a/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisCancellationComposition.ts b/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisCancellationComposition.ts new file mode 100644 index 00000000..1228df12 --- /dev/null +++ b/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisCancellationComposition.ts @@ -0,0 +1,47 @@ +import { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation'; +import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage'; +import { + PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository, + terminalizeCopilotFailureDiagnosisBeforeModel, +} from '@qinglong/ai/failure-diagnosis-pre-model-terminalization'; +import { + PostgresClusterRunCancellationRepository, + type QingLongPostgresPool, +} from '@qinglong/cluster-postgres/runtime'; + +export interface CreateProductionClusterCopilotFailureDiagnosisCancellationOptions { + readonly pool: QingLongPostgresPool; +} + +/** + * Reuses the AI Pool and existing Run/admission ledgers. The cancellation + * capability owns no connection, timer, listener or background lifecycle. + */ +export function createProductionClusterCopilotFailureDiagnosisCancellation( + options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions, +): Readonly { + if ( + !options || + typeof options !== 'object' || + Array.isArray(options) || + typeof options.pool?.query !== 'function' || + typeof options.pool?.connect !== 'function' + ) { + throw new TypeError( + 'Production Cluster Copilot failure diagnosis cancellation dependencies are invalid', + ); + } + const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository( + options.pool, + ); + const terminalizations = + new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository( + options.pool, + ); + return new CopilotFailureDiagnosisCancellationService({ + admissions, + cancellations: new PostgresClusterRunCancellationRepository(options.pool), + terminalizations: Object.freeze({ repository: terminalizations }), + terminalizeBeforeModel: terminalizeCopilotFailureDiagnosisBeforeModel, + }); +} diff --git a/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts b/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts index b12a4863..9085e203 100644 --- a/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts +++ b/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts @@ -71,6 +71,10 @@ import { type ClusterCopilotFailureDiagnosisInspectionCapability, type ClusterCopilotFailureDiagnosisOutputReadCapability, } from '../copilot/failure-diagnosis/failureDiagnosisReadRoutes'; +import { + createClusterControlCopilotFailureDiagnosisCancellationRoute, + type ClusterCopilotFailureDiagnosisCancellationCapability, +} from '../copilot/failure-diagnosis/failureDiagnosisCancellationRoute'; export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([ 'task.get', @@ -100,6 +104,7 @@ export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS = 'copilot.failure_diagnosis.execute', 'copilot.failure_diagnosis.read', 'copilot.failure_diagnosis.output.read', + 'copilot.failure_diagnosis.cancel', ] as const); export interface ClusterCopilotFailureDiagnosisReadCapability @@ -129,6 +134,7 @@ export interface ProductionClusterControlAssemblyOptions { readonly copilotFailureDiagnosis?: Readonly<{ readonly capability: ClusterCopilotFailureDiagnosisCapability; readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability; + readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability; }>; readonly workerIngress?: Readonly<{ readonly config: EnabledClusterWorkerIngressConfig; @@ -178,6 +184,7 @@ export interface ProductionClusterControlApplicationOptions readonly copilotFailureDiagnosis?: Readonly<{ readonly capability: ClusterCopilotFailureDiagnosisCapability; readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability; + readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability; }>; readonly workerIngress?: ProductionClusterWorkerIngressOptions; } @@ -303,6 +310,14 @@ export function createProductionClusterControlApplicationStack( options.copilotFailureDiagnosis.readCapability, ), ]), + ...(options.copilotFailureDiagnosis?.cancellationCapability === undefined + ? [] + : [ + createClusterControlCopilotFailureDiagnosisCancellationRoute( + options.copilotFailureDiagnosis.cancellationCapability, + createEventId, + ), + ]), ]; const routes = createClusterControlRouteRegistry(routeDefinitions); const expectedRouteCount = @@ -313,7 +328,10 @@ export function createProductionClusterControlApplicationStack( (options.promptOutputRead === undefined ? 0 : 1) + (options.promptExecutionOutputRead === undefined ? 0 : 1) + (options.copilotFailureDiagnosis === undefined ? 0 : 1) + - (options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2); + (options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2) + + (options.copilotFailureDiagnosis?.cancellationCapability === undefined + ? 0 + : 1); if (routes.size !== expectedRouteCount) { throw new Error('Production cluster-control route allowlist is incomplete'); } diff --git a/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts b/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts new file mode 100644 index 00000000..05c80d71 --- /dev/null +++ b/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts @@ -0,0 +1,261 @@ +// Cluster Copilot resolves an external request key before cancelling its Run. +import { + CopilotFailureDiagnosisCancellationNotFoundError, + CopilotFailureDiagnosisCancellationUnavailableError, + InvalidCopilotFailureDiagnosisCancellationError, + type CopilotFailureDiagnosisCancellationCommand, +} from '@qinglong/ai/failure-diagnosis-cancellation'; +import { + CLUSTER_RUN_CANCELLATION_SCHEMA, + ClusterRunCancellationFenceRejectedError, + ClusterRunCancellationNotFoundError, + ClusterRunCancellationUnavailableError, + InvalidClusterRunCancellationError, + parseClusterRunCancellationRequestBody, +} from '@qinglong/runtime-core/cluster-run-cancellation'; + +import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface'; +import type { + ClusterControlAuthorizedOperationRequest, + ClusterControlRouteDefinition, + ClusterControlRouteParameters, +} from '../../transport/routeRegistry'; + +export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA = + 'qinglong/cluster-copilot-failure-diagnosis-cancellation-response@v1' as const; + +export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE = + Object.freeze({ + method: 'POST' as const, + path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/cancellation', + operationId: 'copilot.failure_diagnosis.cancel', + permission: 'run.stop', + projectParameter: 'projectId', + }); + +export interface ClusterCopilotFailureDiagnosisCancellationCapability { + cancel( + command: Readonly, + ): Promise; +} + +export type ClusterCopilotFailureDiagnosisCancellationEventIdFactory = + () => string; + +const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/; +const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']); +const STATUSES = new Set(['accepted', 'already_requested', 'already_terminal']); +const CANCEL_REASONS = new Set([ + 'user', + 'policy', + 'shutdown', + 'reconcile', + 'timeout', +]); + +function response( + statusCode: number, + body: Readonly>, +): ClusterControlAdmissionResponse { + return Object.freeze({ statusCode, body: Object.freeze(body) }); +} + +function exactRecord( + value: unknown, + expected: readonly string[], +): Record | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const record = value as Record; + const actual = Object.keys(record).sort(); + const canonical = [...expected].sort(); + return actual.length === canonical.length && + actual.every((key, index) => key === canonical[index]) + ? record + : null; +} + +function safeInteger(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} + +function projectResult( + value: unknown, + target: Readonly<{ + projectId: string; + sourceRunId: string; + requestId: string; + }>, +): Readonly> | null { + const candidate = exactRecord(value, [ + 'cancelReason', + 'cancelRequestedAtMs', + 'convergence', + 'diagnosisRunId', + 'eventSequence', + 'outcome', + 'projectId', + 'requestId', + 'runStatus', + 'runVersion', + 'schema', + 'sourceRunId', + 'status', + ]); + if ( + !candidate || + candidate.schema !== + 'qinglong/copilot-failure-diagnosis-cancellation-result@v1' || + candidate.projectId !== target.projectId || + candidate.sourceRunId !== target.sourceRunId || + candidate.requestId !== target.requestId || + typeof candidate.diagnosisRunId !== 'string' || + !RUN_ID.test(candidate.diagnosisRunId) || + typeof candidate.status !== 'string' || + !STATUSES.has(candidate.status) || + !safeInteger(candidate.runVersion) || + !safeInteger(candidate.eventSequence) || + candidate.runVersion !== candidate.eventSequence || + !( + (candidate.cancelRequestedAtMs === null && + candidate.cancelReason === null) || + (safeInteger(candidate.cancelRequestedAtMs) && + typeof candidate.cancelReason === 'string' && + CANCEL_REASONS.has(candidate.cancelReason)) + ) + ) { + return null; + } + if (candidate.convergence === 'model_in_flight') { + if ( + candidate.runStatus !== 'running' || + candidate.outcome !== null || + candidate.cancelRequestedAtMs === null + ) { + return null; + } + } else if (candidate.convergence === 'terminal') { + if ( + typeof candidate.runStatus !== 'string' || + !OUTCOMES.has(candidate.runStatus) || + candidate.outcome !== candidate.runStatus + ) { + return null; + } + } else { + return null; + } + return Object.freeze({ + schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA, + status: candidate.status, + convergence: candidate.convergence, + projectId: target.projectId, + sourceRunId: target.sourceRunId, + requestId: target.requestId, + diagnosisRunId: candidate.diagnosisRunId, + runStatus: candidate.runStatus, + outcome: candidate.outcome, + runVersion: candidate.runVersion, + eventSequence: candidate.eventSequence, + cancelRequestedAtMs: candidate.cancelRequestedAtMs, + cancelReason: candidate.cancelReason, + }); +} + +export function createClusterControlCopilotFailureDiagnosisCancellationRoute( + capability: ClusterCopilotFailureDiagnosisCancellationCapability, + createEventId: ClusterCopilotFailureDiagnosisCancellationEventIdFactory, +): Readonly { + if ( + !capability || + typeof capability.cancel !== 'function' || + typeof createEventId !== 'function' + ) { + throw new TypeError( + 'Cluster-control Copilot failure diagnosis cancellation route is invalid', + ); + } + return Object.freeze({ + ...CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE, + async handle( + authorized: ClusterControlAuthorizedOperationRequest, + parameters: ClusterControlRouteParameters, + ) { + let body; + try { + body = parseClusterRunCancellationRequestBody(authorized.request.body); + } catch (error) { + return error instanceof InvalidClusterRunCancellationError + ? response(400, { + code: 'invalid_copilot_failure_diagnosis_cancellation_request', + schema: CLUSTER_RUN_CANCELLATION_SCHEMA, + }) + : response(503, { + code: 'copilot_failure_diagnosis_cancellation_unavailable', + }); + } + const projectId = authorized.projectId; + const sourceRunId = parameters.runId; + const requestId = parameters.requestId; + if ( + projectId === null || + typeof sourceRunId !== 'string' || + !RUN_ID.test(sourceRunId) || + typeof requestId !== 'string' || + !IDENTITY.test(requestId) || + !authorized.policyFence || + authorized.policyFence.bindingVersion === null + ) { + return response(503, { + code: 'copilot_failure_diagnosis_cancellation_unavailable', + }); + } + const target = Object.freeze({ projectId, sourceRunId, requestId }); + try { + const result = await capability.cancel({ + ...target, + mutationId: body.mutationId, + eventId: createEventId(), + subject: authorized.principal.subject, + policyFence: authorized.policyFence, + }); + const view = projectResult(result, target); + if (!view) { + return response(503, { + code: 'copilot_failure_diagnosis_cancellation_unavailable', + }); + } + return response(view.status === 'accepted' ? 202 : 200, view); + } catch (error) { + if ( + error instanceof CopilotFailureDiagnosisCancellationNotFoundError || + error instanceof ClusterRunCancellationNotFoundError + ) { + return response(404, { + code: 'copilot_failure_diagnosis_not_found', + }); + } + if (error instanceof ClusterRunCancellationFenceRejectedError) { + return response(409, { + code: 'copilot_failure_diagnosis_cancellation_fence_rejected', + reason: error.reason, + }); + } + if ( + error instanceof InvalidCopilotFailureDiagnosisCancellationError || + error instanceof + CopilotFailureDiagnosisCancellationUnavailableError || + error instanceof InvalidClusterRunCancellationError || + error instanceof ClusterRunCancellationUnavailableError + ) { + return response(503, { + code: 'copilot_failure_diagnosis_cancellation_unavailable', + }); + } + return response(503, { + code: 'copilot_failure_diagnosis_cancellation_unavailable', + }); + } + }, + }); +} diff --git a/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs b/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs index c572af99..5aaad7d8 100644 --- a/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs +++ b/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs @@ -169,9 +169,11 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one const artifactStore = { put() {}, inspect() {}, readLogRange() {} }; const copilot = Object.freeze({ execute() {} }); const copilotRead = Object.freeze({ inspect() {}, readOutput() {} }); + const copilotCancellation = Object.freeze({ cancel() {} }); let registeredSink; let created; let createdRead; + let createdCancellation; let controlOptions; try { await Promise.all([ @@ -233,6 +235,10 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one createdRead = options; return copilotRead; }, + createCopilotCancellation(options) { + createdCancellation = options; + return copilotCancellation; + }, async startControl(options) { controlOptions = options; return { @@ -257,11 +263,16 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one assert.equal(created.artifactStore, artifactStore); assert.equal(createdRead.pool, fakePool); assert.equal(typeof createdRead.prepared.outputKeys.resolve, 'function'); + assert.equal(createdCancellation.pool, fakePool); assert.equal(controlOptions.copilotFailureDiagnosis.capability, copilot); assert.equal( controlOptions.copilotFailureDiagnosis.readCapability, copilotRead, ); + assert.equal( + controlOptions.copilotFailureDiagnosis.cancellationCapability, + copilotCancellation, + ); assert.equal(await application.stop(), 'stopped'); } finally { config.fill(0); diff --git a/packages/ql3-cluster-control/test/copilotFailureDiagnosisCancellationRoute.test.cjs b/packages/ql3-cluster-control/test/copilotFailureDiagnosisCancellationRoute.test.cjs new file mode 100644 index 00000000..02b877a9 --- /dev/null +++ b/packages/ql3-cluster-control/test/copilotFailureDiagnosisCancellationRoute.test.cjs @@ -0,0 +1,232 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + CopilotFailureDiagnosisCancellationNotFoundError, +} = require('@qinglong/ai/failure-diagnosis-cancellation'); +const { + CLUSTER_RUN_CANCELLATION_SCHEMA, + ClusterRunCancellationFenceRejectedError, +} = require('@qinglong/runtime-core/cluster-run-cancellation'); +const { + CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA, + CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE, + createClusterControlCopilotFailureDiagnosisCancellationRoute, +} = require('@qinglong/cluster-control/copilot-cancellation-route'); + +function authorized(body, overrides = {}) { + return { + request: { + requestId: 'transport-request-1', + method: 'POST', + path: '/api/v3/projects/project-1/runs/source-run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation', + query: {}, + headers: {}, + signal: new AbortController().signal, + body, + }, + principal: { + subject: { type: 'user', id: 'owner-1' }, + authenticationId: 'credential-1', + authenticatedAtMs: 1, + expiresAtMs: 10_000, + assurance: 'multi_factor', + }, + operationId: 'copilot.failure_diagnosis.cancel', + permission: 'run.stop', + projectId: 'project-1', + policyFence: { projectVersion: 3, bindingVersion: 7 }, + ...overrides, + }; +} + +const parameters = { + projectId: 'project-1', + runId: 'source-run-1', + requestId: 'diagnosis-request-1', +}; + +function body(overrides = {}) { + return { + schema: CLUSTER_RUN_CANCELLATION_SCHEMA, + mutationId: '11111111-1111-4111-8111-111111111111', + ...overrides, + }; +} + +function result(overrides = {}) { + return { + schema: 'qinglong/copilot-failure-diagnosis-cancellation-result@v1', + status: 'accepted', + convergence: 'terminal', + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'diagnosis-request-1', + diagnosisRunId: 'diagnosis-run-1', + runStatus: 'cancelled', + outcome: 'cancelled', + runVersion: 7, + eventSequence: 7, + cancelRequestedAtMs: 500, + cancelReason: 'user', + ...overrides, + }; +} + +test('defines an exact run.stop route and passes only fenced target facts', async () => { + let command; + const request = authorized(body()); + const route = createClusterControlCopilotFailureDiagnosisCancellationRoute( + { + async cancel(value) { + command = value; + return result(); + }, + }, + () => '22222222-2222-4222-8222-222222222222', + ); + const response = await route.handle(request, parameters); + assert.deepEqual( + CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE, + { + method: 'POST', + path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/cancellation', + operationId: 'copilot.failure_diagnosis.cancel', + permission: 'run.stop', + projectParameter: 'projectId', + }, + ); + assert.deepEqual(command, { + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'diagnosis-request-1', + mutationId: '11111111-1111-4111-8111-111111111111', + eventId: '22222222-2222-4222-8222-222222222222', + subject: request.principal.subject, + policyFence: request.policyFence, + }); + assert.deepEqual(response, { + statusCode: 202, + body: { + schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA, + status: 'accepted', + convergence: 'terminal', + projectId: 'project-1', + sourceRunId: 'source-run-1', + requestId: 'diagnosis-request-1', + diagnosisRunId: 'diagnosis-run-1', + runStatus: 'cancelled', + outcome: 'cancelled', + runVersion: 7, + eventSequence: 7, + cancelRequestedAtMs: 500, + cancelReason: 'user', + }, + }); +}); + +test('projects an in-flight durable intent without claiming Provider abort', async () => { + const route = createClusterControlCopilotFailureDiagnosisCancellationRoute( + { + async cancel() { + return result({ + status: 'already_requested', + convergence: 'model_in_flight', + runStatus: 'running', + outcome: null, + runVersion: 6, + eventSequence: 6, + }); + }, + }, + () => '22222222-2222-4222-8222-222222222222', + ); + const response = await route.handle(authorized(body()), parameters); + assert.equal(response.statusCode, 200); + assert.equal(response.body.convergence, 'model_in_flight'); + assert.equal(response.body.runStatus, 'running'); + assert.equal(response.body.outcome, null); + assert.equal('providerAborted' in response.body, false); +}); + +test('rejects non-exact bodies before invoking the capability', async () => { + let calls = 0; + const route = createClusterControlCopilotFailureDiagnosisCancellationRoute( + { + async cancel() { + calls += 1; + return result(); + }, + }, + () => '22222222-2222-4222-8222-222222222222', + ); + for (const value of [ + null, + {}, + body({ runId: 'caller-selected' }), + body({ reason: 'timeout' }), + body({ mutationId: '' }), + ]) { + const response = await route.handle(authorized(value), parameters); + assert.equal(response.statusCode, 400); + } + assert.equal(calls, 0); +}); + +test('fails closed on widened or identity-drifted capability results', async () => { + for (const value of [ + result({ sourceRunId: 'other' }), + result({ privateProvider: 'must-not-cross' }), + result({ runVersion: 8 }), + result({ convergence: 'model_in_flight' }), + ]) { + const route = createClusterControlCopilotFailureDiagnosisCancellationRoute( + { + async cancel() { + return value; + }, + }, + () => '22222222-2222-4222-8222-222222222222', + ); + const response = await route.handle(authorized(body()), parameters); + assert.deepEqual(response, { + statusCode: 503, + body: { + code: 'copilot_failure_diagnosis_cancellation_unavailable', + }, + }); + } +}); + +test('maps hidden targets, Policy races and storage failures to stable codes', async () => { + for (const [error, statusCode, code] of [ + [ + new CopilotFailureDiagnosisCancellationNotFoundError(), + 404, + 'copilot_failure_diagnosis_not_found', + ], + [ + new ClusterRunCancellationFenceRejectedError('authorization_changed'), + 409, + 'copilot_failure_diagnosis_cancellation_fence_rejected', + ], + [ + new Error('private storage detail'), + 503, + 'copilot_failure_diagnosis_cancellation_unavailable', + ], + ]) { + const route = createClusterControlCopilotFailureDiagnosisCancellationRoute( + { + async cancel() { + throw error; + }, + }, + () => '22222222-2222-4222-8222-222222222222', + ); + const response = await route.handle(authorized(body()), parameters); + assert.equal(response.statusCode, statusCode); + assert.equal(response.body.code, code); + assert.equal(JSON.stringify(response).includes('private'), false); + } +}); diff --git a/packages/ql3-cluster-control/test/productionApplication.test.cjs b/packages/ql3-cluster-control/test/productionApplication.test.cjs index e576b5a3..1a84b3d0 100644 --- a/packages/ql3-cluster-control/test/productionApplication.test.cjs +++ b/packages/ql3-cluster-control/test/productionApplication.test.cjs @@ -732,6 +732,7 @@ test('optionally exposes Prompt execution behind shared admission and policy', a 'copilot.failure_diagnosis.execute', 'copilot.failure_diagnosis.read', 'copilot.failure_diagnosis.output.read', + 'copilot.failure_diagnosis.cancel', ]); const response = await invoke( stack, @@ -762,6 +763,7 @@ test('optionally exposes Prompt execution behind shared admission and policy', a test('optionally exposes Copilot diagnosis behind shared authentication, Policy and audit', async () => { const { events, input } = fixture(); let command; + let cancellationCommand; const capability = { async execute(value) { command = value; @@ -803,11 +805,30 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy requestId: value.requestId, }; }, + async cancel(value) { + cancellationCommand = value; + return { + schema: 'qinglong/copilot-failure-diagnosis-cancellation-result@v1', + status: 'accepted', + convergence: 'terminal', + projectId: value.projectId, + sourceRunId: value.sourceRunId, + requestId: value.requestId, + diagnosisRunId: 'diagnosis-run-1', + runStatus: 'cancelled', + outcome: 'cancelled', + runVersion: 7, + eventSequence: 7, + cancelRequestedAtMs: 2_000, + cancelReason: 'user', + }; + }, }; const stack = createProductionClusterControlApplicationStack(input, { copilotFailureDiagnosis: { capability, readCapability: capability, + cancellationCapability: capability, }, }); const result = await invoke( @@ -849,6 +870,25 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy ); assert.equal(inspection.statusCode, 404); assert.equal(output.statusCode, 404); + const cancellation = await invoke( + stack, + metadata( + '/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation', + 'POST', + { + schema: 'qinglong/run-cancellation@v1', + mutationId: '00000000-0000-4000-8000-000000000099', + }, + ), + ); + assert.equal(cancellation.statusCode, 202); + assert.equal(cancellationCommand.projectId, 'project-1'); + assert.equal(cancellationCommand.sourceRunId, 'run-1'); + assert.equal(cancellationCommand.requestId, 'diagnosis-request-1'); + assert.deepEqual(cancellationCommand.policyFence, { + projectVersion: 3, + bindingVersion: 7, + }); assert.equal( events.includes('audit:copilot.failure_diagnosis.read:allowed'), true, @@ -857,6 +897,10 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy events.includes('audit:copilot.failure_diagnosis.output.read:allowed'), true, ); + assert.equal( + events.includes('audit:copilot.failure_diagnosis.cancel:allowed'), + true, + ); }); test('keeps the Copilot route absent by default and never invokes it after Policy denial', async () => { @@ -879,6 +923,7 @@ test('keeps the Copilot route absent by default and never invokes it after Polic for (const path of [ '/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1', '/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/output', + '/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation', ]) { await assert.rejects( defaultStack.admission.prepare(metadata(path)), diff --git a/scripts/ql3-cluster-dependency-audit.cjs b/scripts/ql3-cluster-dependency-audit.cjs index 486211a7..730a56c4 100644 --- a/scripts/ql3-cluster-dependency-audit.cjs +++ b/scripts/ql3-cluster-dependency-audit.cjs @@ -1165,6 +1165,7 @@ function auditSourceImports(root, packagePath, findings) { 'src/application-runtime/aiProductionApplication.ts': Object.freeze([ '@qinglong/ai/durable-model-invocation', '@qinglong/ai/failure-diagnosis-application', + '@qinglong/ai/failure-diagnosis-cancellation', '@qinglong/ai/failure-diagnosis-model-execution', '@qinglong/ai/plugin-package-prompt-output-projected-keyring', '@qinglong/ai/postgres-model-provider-credential-storage', @@ -1193,6 +1194,14 @@ function auditSourceImports(root, packagePath, findings) { '@qinglong/ai/postgres-failure-diagnosis-admission-storage', '@qinglong/ai/postgres-failure-diagnosis-model-execution-storage', ]), + 'src/application-runtime/copilot/failureDiagnosisCancellationComposition.ts': + Object.freeze([ + '@qinglong/ai/failure-diagnosis-cancellation', + '@qinglong/ai/failure-diagnosis-pre-model-terminalization', + '@qinglong/ai/postgres-failure-diagnosis-admission-storage', + ]), + 'src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts': + Object.freeze(['@qinglong/ai/failure-diagnosis-cancellation']), }); const sourceRelative = path.relative(packageDirectory, filePath); if (!clusterAiImports[sourceRelative]?.includes(specifier)) { diff --git a/scripts/ql3-postgres-ha-contract.cjs b/scripts/ql3-postgres-ha-contract.cjs index 6a7f7f8e..13084286 100644 --- a/scripts/ql3-postgres-ha-contract.cjs +++ b/scripts/ql3-postgres-ha-contract.cjs @@ -88,6 +88,9 @@ const { const { CopilotFailureDiagnosisReadService, } = require('../packages/ql3-ai/dist/copilot/failure-diagnosis/read-model/service.js'); +const { + CopilotFailureDiagnosisCancellationService, +} = require('../packages/ql3-ai/dist/copilot/failure-diagnosis/cancellation/service.js'); const { BUILTIN_RUN_LOG_EXCERPT_TOOL, BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION, @@ -1807,6 +1810,34 @@ async function copilotFailureDiagnosisTerminalizationFacts(pool, requestId) { }; } +async function copilotFailureDiagnosisCancellationFacts(pool, requestId) { + const terminalization = await copilotFailureDiagnosisTerminalizationFacts( + pool, + requestId, + ); + const result = await pool.query( + `SELECT run.cancel_requested_at_ms AS "cancelRequestedAtMs", + run.cancel_reason AS "cancelReason", + (SELECT count(*)::integer FROM "ql3"."run_events" + WHERE run_id = admission.run_id + AND type = 'run.cancel_requested') AS "cancelEventCount", + (SELECT count(*)::integer + FROM "ql3_ai"."model_invocation_starts" + WHERE invocation_id = admission.plan_json->>'modelInvocationId') + AS "modelStartCount", + (SELECT count(*)::integer + FROM "ql3_ai"."model_invocation_completions" + WHERE invocation_id = admission.plan_json->>'modelInvocationId') + AS "modelCompletionCount" + FROM "ql3_ai"."copilot_failure_diagnosis_admissions" AS admission + JOIN "ql3"."runs" AS run ON run.id = admission.run_id + WHERE admission.request_id = $1`, + [requestId], + ); + assert.equal(result.rowCount, 1); + return { ...terminalization, ...result.rows[0] }; +} + async function copilotFailureDiagnosisModelResolutionFacts(pool, requestId) { const result = await pool.query( `SELECT admission.request_id AS "requestId", @@ -2646,6 +2677,134 @@ async function terminalizeCopilotFailureDiagnosisLogUnavailableHaEvidence( } } +async function cancelCopilotFailureDiagnosisHaEvidence(options) { + const { port, report, fixture, expectedExisting = false } = options; + const runtimeDatabase = await databaseOpener( + 'runtime', + databaseUrl(RUNTIME_USER, RUNTIME_PASSWORD, port), + expectedExisting + ? 'ql3-ha-copilot-cancellation-promoted' + : 'ql3-ha-copilot-cancellation-primary', + )(); + let promotedClient; + try { + const pool = expectedExisting + ? await (async () => { + promotedClient = await runtimeDatabase.pool.connect(); + await promotedClient.query(`SET synchronous_commit = 'local'`); + return { + query: promotedClient.query.bind(promotedClient), + async connect() { + return { + query: promotedClient.query.bind(promotedClient), + release() {}, + }; + }, + }; + })() + : runtimeDatabase.pool; + const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository( + pool, + ); + const terminalizations = + new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository( + pool, + ); + const service = new CopilotFailureDiagnosisCancellationService({ + admissions, + cancellations: new PostgresClusterRunCancellationRepository(pool), + terminalizations: { repository: terminalizations }, + terminalizeBeforeModel: terminalizeCopilotFailureDiagnosisBeforeModel, + }); + const mutationId = report.cancellationMutationId ?? randomUUID(); + const command = { + projectId: fixture.plan.projectId, + sourceRunId: fixture.plan.source.runId, + requestId: fixture.plan.requestId, + mutationId, + eventId: randomUUID(), + subject: fixture.principal.subject, + policyFence: fixture.plan.policyFence, + }; + if (!expectedExisting) { + await assert.rejects( + service.cancel({ ...command, sourceRunId: 'other-source-run' }), + (error) => + error?.code === 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_NOT_FOUND', + ); + } + const first = await service.cancel(command); + const replay = await service.cancel({ ...command, eventId: randomUUID() }); + assert.equal( + first.status, + expectedExisting ? 'already_terminal' : 'accepted', + ); + assert.equal(first.convergence, 'terminal'); + assert.equal(first.runStatus, 'cancelled'); + assert.equal(first.outcome, 'cancelled'); + assert.equal(replay.status, 'already_terminal'); + assert.equal(replay.convergence, 'terminal'); + assert.equal(replay.runStatus, 'cancelled'); + const facts = await copilotFailureDiagnosisCancellationFacts( + runtimeDatabase.pool, + report.requestId, + ); + assert.deepEqual( + { + reason: facts.reason, + outcome: facts.outcome, + runStatus: facts.runStatus, + runVersion: facts.runVersion, + runEventSequence: facts.runEventSequence, + eventCount: facts.eventCount, + mutationCount: facts.mutationCount, + terminalizationCount: facts.terminalizationCount, + toolStatus: facts.toolStatus, + modelStatus: facts.modelStatus, + cancelReason: facts.cancelReason, + cancelEventCount: facts.cancelEventCount, + modelStartCount: facts.modelStartCount, + modelCompletionCount: facts.modelCompletionCount, + receiptContainsLogContent: facts.receiptContainsLogContent, + }, + { + reason: 'cancellation_requested', + outcome: 'cancelled', + runStatus: 'cancelled', + runVersion: 7, + runEventSequence: 7, + eventCount: 7, + mutationCount: 4, + terminalizationCount: 1, + toolStatus: 'cancelled', + modelStatus: 'cancelled', + cancelReason: 'user', + cancelEventCount: 1, + modelStartCount: 0, + modelCompletionCount: 0, + receiptContainsLogContent: false, + }, + ); + if (expectedExisting) { + report.afterPromotion = facts; + report.survivedPromotion = true; + report.promotedExactReplay = true; + report.promotedProviderCalls = 0; + } else { + report.cancellationMutationId = mutationId; + report.beforePromotion = facts; + report.cancellationExactReplay = true; + report.crossTargetHidden = true; + report.providerCalls = 0; + report.contentFree = true; + } + return facts; + } finally { + promotedClient?.release(); + await runtimeDatabase.close(); + } +} + async function resolveCopilotFailureDiagnosisUnknownModelHaEvidence(options) { const { port, @@ -11233,6 +11392,8 @@ async function main(argv = process.argv.slice(2)) { let copilotFailureDiagnosisModelResolutionFixture; let copilotFailureDiagnosisRead; let copilotFailureDiagnosisReadFixture; + let copilotFailureDiagnosisCancellation; + let copilotFailureDiagnosisCancellationFixture; let modelProviderCredentialCatalog; let modelProviderCredentialTestConnection; let runAttemptLogRetentionEvidence; @@ -11449,6 +11610,13 @@ async function main(argv = process.argv.slice(2)) { }); copilotFailureDiagnosisRead = copilotRead.report; copilotFailureDiagnosisReadFixture = copilotRead.fixture; + const copilotCancellation = + await runCopilotFailureDiagnosisAdmissionHaEvidence({ + primaryPort, + migrationPool: migrationDatabase.pool, + }); + copilotFailureDiagnosisCancellation = copilotCancellation.report; + copilotFailureDiagnosisCancellationFixture = copilotCancellation.fixture; } finally { await migrationDatabase.close(); } @@ -11903,6 +12071,26 @@ async function main(argv = process.argv.slice(2)) { state: 'copilot_failure_diagnosis_pre_model_terminalized', atMs: Number((performance.now() - startedAt).toFixed(3)), }); + await cancelCopilotFailureDiagnosisHaEvidence({ + port: primaryPort, + report: copilotFailureDiagnosisCancellation, + fixture: copilotFailureDiagnosisCancellationFixture, + }); + await waitFor(async () => { + const replicated = await copilotFailureDiagnosisCancellationFacts( + standbyDatabase.pool, + copilotFailureDiagnosisCancellation.requestId, + ); + return JSON.stringify(replicated) === + JSON.stringify(copilotFailureDiagnosisCancellation.beforePromotion) + ? replicated + : null; + }, 'Copilot failure diagnosis cancellation WAL replay'); + copilotFailureDiagnosisCancellation.replicatedBeforePromotion = true; + timeline.push({ + state: 'copilot_failure_diagnosis_cancellation_terminalized', + atMs: Number((performance.now() - startedAt).toFixed(3)), + }); await resolveCopilotFailureDiagnosisUnknownModelHaEvidence({ port: primaryPort, report: copilotFailureDiagnosisModelResolution, @@ -12846,6 +13034,12 @@ async function main(argv = process.argv.slice(2)) { outputKeyByte: 0x74, expectedExisting: true, }); + await cancelCopilotFailureDiagnosisHaEvidence({ + port: standbyPort, + report: copilotFailureDiagnosisCancellation, + fixture: copilotFailureDiagnosisCancellationFixture, + expectedExisting: true, + }); await verifyModelProviderCredentialCatalogAfterPromotion({ promotedPort: standbyPort, promotedPool: promotedDatabase.pool, @@ -13611,8 +13805,8 @@ async function main(argv = process.argv.slice(2)) { ); assert.deepEqual(sideEffects.rows, [ { - runs: 84, - runEvents: 209, + runs: 86, + runEvents: 216, diagnosisReadRuns: 2, diagnosisReadRunEvents: 9, credentialDeliveries: 4, @@ -13734,6 +13928,7 @@ async function main(argv = process.argv.slice(2)) { copilotFailureDiagnosisTerminalization, copilotFailureDiagnosisModelResolution, copilotFailureDiagnosisRead, + copilotFailureDiagnosisCancellation, modelProviderCredentialCatalog, modelProviderCredentialTestConnection, runAttemptLogRetention, @@ -13843,6 +14038,31 @@ async function main(argv = process.argv.slice(2)) { copilotFailureDiagnosisRead.promotedProviderCalls === 0 && JSON.stringify(copilotFailureDiagnosisRead.afterPromotion) === JSON.stringify(copilotFailureDiagnosisRead.beforePromotion), + copilotFailureDiagnosisCancellationExactlyReplaysWithoutProviderCall: + copilotFailureDiagnosisCancellation.cancellationExactReplay && + copilotFailureDiagnosisCancellation.crossTargetHidden && + copilotFailureDiagnosisCancellation.providerCalls === 0 && + copilotFailureDiagnosisCancellation.beforePromotion + .cancelEventCount === 1 && + copilotFailureDiagnosisCancellation.beforePromotion + .terminalizationCount === 1 && + copilotFailureDiagnosisCancellation.beforePromotion + .modelStartCount === 0 && + copilotFailureDiagnosisCancellation.beforePromotion + .modelCompletionCount === 0 && + copilotFailureDiagnosisCancellation.beforePromotion.runStatus === + 'cancelled', + copilotFailureDiagnosisCancellationIsContentFree: + copilotFailureDiagnosisCancellation.contentFree && + copilotFailureDiagnosisCancellation.beforePromotion + .receiptContainsLogContent === false, + copilotFailureDiagnosisCancellationReplicatesAndSurvivesPromotion: + copilotFailureDiagnosisCancellation.replicatedBeforePromotion && + copilotFailureDiagnosisCancellation.survivedPromotion && + copilotFailureDiagnosisCancellation.promotedExactReplay && + copilotFailureDiagnosisCancellation.promotedProviderCalls === 0 && + JSON.stringify(copilotFailureDiagnosisCancellation.afterPromotion) === + JSON.stringify(copilotFailureDiagnosisCancellation.beforePromotion), modelProviderCredentialCatalogSurvivesPromotion: modelProviderCredentialCatalog.survivedPromotion, modelProviderCredentialManagementIdentityLedgerSurvivesPromotion: diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index 63948bf1..f1f9f40d 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -268,10 +268,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( { path: 'packages/ql3-ai', name: '@qinglong/ai', - sourceFiles: 193, + sourceFiles: 194, rootSourceFiles: 1, rootSourceLines: 16, - nestedSourceFiles: 192, + nestedSourceFiles: 193, rootSourceFileHardCap: 1, rootSourceLineHardCap: 16, rootSourceFileRoles: { @@ -385,10 +385,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: clusterControl.rootSourceFileRoles, }, { - sourceFiles: 62, + sourceFiles: 64, rootSourceFiles: 2, rootSourceLines: 195, - nestedSourceFiles: 60, + nestedSourceFiles: 62, rootSourceFileRoles: { 'aiCli.ts': 'binary_entry', 'cli.ts': 'binary_entry',