diff --git a/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/README.md b/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/README.md new file mode 100644 index 00000000..0650ea44 --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/README.md @@ -0,0 +1,25 @@ +# Optional Cluster Copilot component + +Compose this component only with `../cluster-ai`. It enables the caller-driven +failure-diagnosis capability inside the existing Cluster AI process; it does +not add an HTTP route, queue, watcher, timer, PostgreSQL pool, provider, or +Model Gateway. + +Before production use: + +1. Replace the example provider, model and egress policy revision in + `copilot-configmap.yaml`. Keep `config.json` canonical, one-line JSON with + one trailing newline. +2. Provision these three Secrets out of band, each containing a canonical + `keyring.json`: `ql3-cluster-ai-copilot-invocation-keyring`, + `ql3-cluster-ai-copilot-result-keyring`, and + `ql3-cluster-ai-copilot-output-keyring`. +3. Keep old decryptable keys during rotation. Invocation, Tool result and + Copilot output keys are independent authorities and must never reuse + material. +4. Keep Worker ingress and its bounded S3 log-range reader enabled. Startup + fails closed if the log capability, canonical config, any keyring, or the + shared successful-completion sink is unavailable. + +All projections are read-only mode `0440`. The component adds no Kubernetes +API permission and does not change the default AI-free deployment. diff --git a/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/copilot-configmap.yaml b/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/copilot-configmap.yaml new file mode 100644 index 00000000..a0e3d014 --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/copilot-configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ql3-cluster-ai-copilot +data: + config.json: | + {"schema":"qinglong/cluster-copilot-failure-diagnosis-config@v1","provider":"provider-primary","model":"model-diagnosis","modelBoundary":"external","responseLanguage":"zh-CN","maxOutputTokens":512,"executionTimeoutMs":60000,"egressPolicy":{"schema":"qinglong/copilot-model-egress-policy@v1","revision":"replace-before-production","potentiallySensitiveDataBoundaries":["external"],"maxInputBytes":65536,"maxOutputTokens":1024}} diff --git a/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/deployment-patch.yaml b/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/deployment-patch.yaml new file mode 100644 index 00000000..db4e27bf --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/deployment-patch.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ql3-cluster-control +spec: + template: + spec: + containers: + - name: cluster-control + env: + - name: QL3_CLUSTER_AI_COPILOT_ENABLED + value: "true" + - name: QL3_CLUSTER_AI_COPILOT_CONFIG_FILE + value: /var/run/qinglong3/ai/copilot-config/config.json + - name: QL3_CLUSTER_AI_COPILOT_INVOCATION_KEYRING_ROOT + value: /var/run/secrets/qinglong3/ai/copilot-invocation-keyring + - name: QL3_CLUSTER_AI_COPILOT_RESULT_KEYRING_ROOT + value: /var/run/secrets/qinglong3/ai/copilot-result-keyring + - name: QL3_CLUSTER_AI_COPILOT_OUTPUT_KEYRING_ROOT + value: /var/run/secrets/qinglong3/ai/copilot-output-keyring + volumeMounts: + - name: cluster-ai-copilot-config + mountPath: /var/run/qinglong3/ai/copilot-config + readOnly: true + - name: cluster-ai-copilot-invocation-keyring + mountPath: /var/run/secrets/qinglong3/ai/copilot-invocation-keyring + readOnly: true + - name: cluster-ai-copilot-result-keyring + mountPath: /var/run/secrets/qinglong3/ai/copilot-result-keyring + readOnly: true + - name: cluster-ai-copilot-output-keyring + mountPath: /var/run/secrets/qinglong3/ai/copilot-output-keyring + readOnly: true + volumes: + - name: cluster-ai-copilot-config + configMap: + name: ql3-cluster-ai-copilot + defaultMode: 288 + items: + - key: config.json + path: config.json + - name: cluster-ai-copilot-invocation-keyring + secret: + secretName: ql3-cluster-ai-copilot-invocation-keyring + defaultMode: 288 + items: + - key: keyring.json + path: keyring.json + - name: cluster-ai-copilot-result-keyring + secret: + secretName: ql3-cluster-ai-copilot-result-keyring + defaultMode: 288 + items: + - key: keyring.json + path: keyring.json + - name: cluster-ai-copilot-output-keyring + secret: + secretName: ql3-cluster-ai-copilot-output-keyring + defaultMode: 288 + items: + - key: keyring.json + path: keyring.json diff --git a/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/kustomization.yaml b/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/kustomization.yaml new file mode 100644 index 00000000..1792e0e6 --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - copilot-configmap.yaml + +patches: + - path: deployment-patch.yaml diff --git a/deploy/kubernetes/ql3-cluster/overlays/cluster-ai-copilot-example/kustomization.yaml b/deploy/kubernetes/ql3-cluster/overlays/cluster-ai-copilot-example/kustomization.yaml new file mode 100644 index 00000000..8fd41a0e --- /dev/null +++ b/deploy/kubernetes/ql3-cluster/overlays/cluster-ai-copilot-example/kustomization.yaml @@ -0,0 +1,16 @@ +# Example only. Copy into a private overlay, replace all example model policy +# values and provision the three keyring Secrets out of band. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +components: + - ../../components/cluster-ai + - ../../components/cluster-ai-copilot + +images: + - name: qinglong3-cluster-control-ai + newName: registry.example.com/qinglong/qinglong3-cluster-control-ai + digest: sha256:0000000000000000000000000000000000000000000000000000000000000000 diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 2b6c7e06..29c6faed 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,7 @@ 最新增量证据(2026-08-15): +- D-319/ADR-0411(已接受):Cluster Copilot failure diagnosis 已在既有 `ql3-cluster-control-ai` 进程内完成默认关闭的 production composition。Prompt 与 Copilot 共享同一个 PostgreSQL AI Pool、Model Gateway、Provider client、恢复扫描、quota/pricing ledger 与 `maxConcurrent` 预算;有界 successful-completion router 只向声明 exact invocation 的 durable sink 分发,不复制 Gateway、连接或隐藏队列。application service 只接受 source Run/request identity,从数据库当前 Run、latest Attempt、Project Tool snapshot、Policy 和 canonical read-only egress config 派生计划,先 admission,再以 durable plan、historical key 与确定性 nonce 修复 admission→Artifact crash window,随后复用 Trusted Tool、Worker Artifact range reader、Model Gateway 及独立的 invocation/result/model-output 三域 keyring。能力保持 caller-driven,不增加 timer、watcher、队列、HTTP/CLI/UI/MCP route 或 Kubernetes API 权限;Kubernetes 独立可选 component 仅投影 config 与三个 0440 keyring。最终 AI 238 pass/3 条件 skip、Cluster Control 240 pass/2 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail,四项架构审计、14 档 Local artifact 全部通过;workspace 仍为 18 package、无单文件/浅平 package,AI 187 个源码中 186 个、Cluster Control 59 个中 57 个位于嵌套领域目录。默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes,Edge/Standalone AI 为 3,064,454/3,064,544 bytes,证明 Cluster-only composition 未进入小设备闭包。PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`,报告 SHA-256 为 `981299b454dce5541e9596450b85816dc40559cba8dc42adf3d5fea571c3d3a6`;本 Gate 不改 migration/schema/role/SQL/HA 拓扑。下一 Gate 是 Tool failure、日志 missing/retired/pending、deadline/cancel 与 Model outcome-unknown 的 durable terminalization/recovery,完成前不开放产品入口。 - D-318/ADR-0410(已接受):Cluster Copilot diagnosis Model output 获得独立的只读 projected key authority。canonical `qinglong/copilot-failure-diagnosis-output-projected-keyring@v1` manifest 只允许一个 active key 与最多 16 个 historical 32-byte key;每次 `active()`/`resolve()` 都重新执行 Cluster 私有投影文件的 direct-root、根内 atomic symlink、single-link、mode、dev/inode/size/mtime 与双 realpath fence,不使用 cache、watcher、timer 或 Kubernetes API。该 authority 以结构兼容的本地窄端口位于既有 `cluster-control/copilot/failure-diagnosis/`,只通过 `failure-diagnosis-output-keyring` subpath 发布,避免 Cluster Control 默认源码反向依赖 AI;它不复用 Prompt output、Tool invocation/result 或 Provider credential key domain,也不提前声称 Copilot 产品入口已可达。定向回归 13/13、Cluster Control 239 pass/2 条件 skip、backend 1,207 pass/2 条件 skip、18-package clean build/test、四项架构审计与 14 档 Local artifact 全部通过;workspace 仍无单文件/浅平 package,Cluster Control 58 个源码中 56 个位于嵌套领域目录。默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes,Edge/Standalone AI 保持 3,061,009/3,061,099 bytes,证明 Cluster-only subpath 被裁掉。本 Gate 不改 migration/schema/role/SQL/连接/HA 拓扑,因此数据库基线继续引用 ADR-0409 的 PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`;下一 Gate 是默认关闭的完整 Cluster Copilot composition,随后补齐 Tool failure、日志 missing/retired、Model admission 前 deadline/cancel 与 outcome-unknown 的 durable terminalization/recovery。 - D-316/ADR-0408(已接受):Cluster Copilot 现在能够从 ADR-0407 的 durable admission 恢复 exact `qinglong.run.log.excerpt@1.0.0` authority,复用通用 Trusted Tool start barrier、加密 success/failure completion、result catalog/rekey 和内建日志 adapter;确定性 start/completion identity 让 response-loss replay 直接打开既有证据,不重复读取日志或执行 adapter。Cluster invocation Artifact 使用独立 projected keyring,提供 active+historical material,但每次读取均重新执行 canonical path/symlink/mode/inode/realpath fence,且不取得 PostgreSQL Tool result generation authority。只有 exact `succeeded` completion 才能通过 `pg-9019` 的 SERIALIZABLE 事务把 Model Step 从 `pending` 原子推进到 `ready`,同时写 RunEvent、StepRunMutation 和 append-only unlock receipt;`failed|timed_out` 不解锁。本 Gate 不执行模型、不终态化 diagnosis Run,下一 Gate 是 ADR-0405 builder + Model Gateway + Copilot encrypted model completion/terminalization。实现仍为 18 个 package,无单文件/浅平 package;AI 175 个源码中 174 个、Cluster Control 56 个中 54 个位于嵌套目录,不新增依赖、进程、连接、timer/watcher/cache 或默认 Edge 成本。18-package clean build/test 全绿,AI 229 pass/3 条件 skip、backend 1,207 pass/2 条件 skip,四项架构审计零 finding;14 档 Local artifact 全通过,默认 Edge/Standalone 为 2,589,890/2,589,968 bytes。PostgreSQL 18.4 arm64 HA 130/130、timeline `1→2`,首次执行只读两次日志、密文不含敏感 fixture,提升后 exact replay 零日志读取;报告 SHA-256 为 `d525a303696e178d777b021b376729bd2c5382fb5eb7bc98466a2b79d3940517`,独立审计与 Docker 清理通过。 - D-317/ADR-0409(已接受):已解锁的 Cluster failure-diagnosis Model Step 现在从 durable encrypted Tool completion 重开受信日志投影,经 ADR-0405 builder 和既有 Model Gateway 执行;只有显式安装的 Copilot success sink 可以接管成功返回。Copilot `GenerateResult` 使用独立 AES-256-GCM Artifact,绑定 plan、Tool completion、egress evidence 与 Model identity,公开 reference、ModelInvocation completion、RunEvent 和审计均不含明文。通用 `DurableModelInvocationCoordinator` 已改为领域无关的 `ModelInvocationAtomicSuccess`,消除对 Plugin Prompt Artifact 的反向依赖;Plugin Prompt 通过 adapter 保持兼容。`pg-9020` 在一个 SERIALIZABLE 事务中原子提交 ciphertext、Model completion、StepRun/Event 与 usage/pricing/quota settlement,再由可重放 finalization 事务把 diagnosis Run 推进为 `succeeded|failed|timed_out`;两事务间崩溃只补 finalization,existing start/completion/finalization replay 均不重复调用 Provider。当前仍不自动终态化 Tool failure、日志 missing/retired、Model admission 前 deadline/cancel,也不把 `outcome_unknown` 冒充失败;Cluster 专用 output projected keyring 与产品 composition 完成前该入口保持不可达。实现仍在既有 18 个 package 的嵌套领域目录内,不新增进程、队列、timer/watcher/cache 或默认 Edge 成本。最终 AI 233 pass/3 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail,四项架构审计零 finding;workspace 无单文件或浅平 package,AI 183 个源码中 182 个位于嵌套目录。14 档 Local Profile artifact 全部通过,默认 Edge/Standalone 为 2,589,890/2,589,968 bytes,Edge/Standalone AI 为 3,061,009/3,061,099 bytes。PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`,报告 SHA-256 为 `8401634f30635b45bfb583b02e94ac41f023bf8a0bdbcfd9744ebf459ab0d8f8`,独立证据审计与 Docker 零残留。 diff --git a/docs/adr/ADR-0411-default-off-cluster-copilot-composition.md b/docs/adr/ADR-0411-default-off-cluster-copilot-composition.md new file mode 100644 index 00000000..340b622d --- /dev/null +++ b/docs/adr/ADR-0411-default-off-cluster-copilot-composition.md @@ -0,0 +1,83 @@ +# ADR-0411:默认关闭的 Cluster Copilot Failure Diagnosis Composition + +- 状态:Accepted +- 日期:2026-08-15 +- 关联 RFC:QL-RFC-0001 D-319、Phase 2 +- 关联 ADR:ADR-0407、ADR-0408、ADR-0409、ADR-0410 + +## 问题 + +ADR-0407 至 ADR-0410 已分别建立 diagnosis admission、Trusted Tool execution、Model execution +和独立 output key authority,但它们仍只在测试与 HA ceremony 中手工组装。若直接再启动一套 Model +Gateway,会为每个 replica 复制 PostgreSQL Pool、Provider client、恢复扫描和并发预算;这对小型集群 +不合理,也会让 Prompt 与 Copilot 绕过同一模型出口限额。若直接开放 HTTP route,则 Tool failure、日志 +missing/retired、Model admission 前取消以及 outcome-unknown 尚未终态化,产品会暴露无法收敛的 Run。 + +## 决策 + +1. Copilot 只在既有显式 `ql3-cluster-control-ai` 进程中装配,并由独立 + `QL3_CLUSTER_AI_COPILOT_ENABLED=true` 开关启用。默认 Cluster、Edge、Standalone 和仅 Prompt 的 + Cluster AI 均不读取 Copilot 配置、keyring 或 Artifact authority。 +2. Prompt 与 Copilot 共享一个 PostgreSQL runtime Pool、一个 Model Gateway、同一 Provider authority、 + recovery pass、pricing/quota ledger 和 `maxConcurrent` 预算。新增有界 successful-completion router, + 只把一次已开始的 invocation 交给声明该 invocation 的 durable sink;不得复制 Gateway 或隐藏队列。 +3. Copilot Model execution 依赖 `generate()` 与 `supportsSuccessfulCompletionSink()` 的窄结构端口,不依赖 + `BoundedModelGateway` 具体类,使 production Profile 的 drain/active-operation capability 可以直接注入, + 同时保留 exact sink identity 检查。 +4. Cluster Copilot application service 从当前 PostgreSQL Run、latest Attempt、Project Tool snapshot 和 + Policy 派生计划;调用者不能提供 Attempt、Artifact、Tool binding、Policy fence、key ID、nonce、 + `plannedAtMs` 或内部 Run/Step/ModelInvocation identity。模型与 egress policy 来自部署者的 canonical + read-only配置,而不是请求体。 +5. diagnosis admission 先持久化 exact plan/Run/Step ledger,再物化可重建的加密 Tool invocation Artifact。 + replay 使用 durable plan、historical invocation key 和域分离确定性 nonce 修复 admission→Artifact + crash window;只有 Artifact exact replay 成功后才允许 Tool execution。密钥副本使用后必须清零。 +6. Tool execution 复用同一 PostgreSQL repository、current snapshot、Project Policy 与 Worker Artifact + range reader;Model execution复用同一 Gateway,并由 ADR-0410 output keyring 原子落盘 ciphertext。 + invocation、Tool result 与 Model output keyring 继续是三个不可互换的域。 +7. composition 能力在进程内保持 caller-driven、无 timer、watcher、队列或后台扫描。当前 Gate 不注册 + HTTP/CLI/UI/MCP route;只有下一 Gate 完成所有非成功路径的 durable terminalization/recovery 后, + 才能把该能力接入认证产品面。 +8. Kubernetes 以独立可选 component 投影 canonical Copilot 配置和三个 read-only keyring;它必须与 + `cluster-ai` 以及可读 Worker Artifact storage 一起使用,不新增 Pod、ServiceAccount 权限或 sidecar。 + +## 被否决方案 + +1. **为 Copilot 再启动 Gateway/Pool**:资源翻倍并拆散全局并发、quota 和 Provider 出口约束。 +2. **让 Gateway 依赖具体业务 sink**:把通用模型边界反向耦合到 Prompt/Copilot 产品域。 +3. **由请求提交完整执行计划**:把 source fence、Tool binding、egress policy 和内部 identity 交给不可信边界。 +4. **先开放 route、以后补失败收敛**:会产生用户可见但永久 running/pending 的 diagnosis Run。 +5. **把三个 keyring 合并**:轮换、退役与泄漏半径跨越 Tool 输入、Tool 输出和模型输出域。 +6. **新增 workspace package**:composition 没有独立进程或依赖闭包,薄包会恶化已有 package 粒度。 + +## 验证标准 + +1. completion router 证明 Prompt/Copilot 精确分发、未知 invocation 不落盘、重复/扩展 sink 拒绝。 +2. 默认关闭时不读取 Copilot config、keyring、Worker Artifact 或新增 PostgreSQL authority。 +3. composition 测试覆盖 server-derived source/snapshot/Policy、admission→Artifact crash repair、Tool→Model + 成功链、exact replay、key rotation、配置/authority 缺失和有序 drain。 +4. AI、Cluster Control、18-package clean build/test、backend、四项架构审计和 14 档 Local artifact 全通过。 +5. PostgreSQL 组合证据覆盖真实 repository 上的 admission、Tool completion、Model ciphertext、finalization + 与 exact replay;本 Gate 不新增 migration、schema、role 或 SQL privilege。 + +## 当前验证 + +1. successful-completion router 与 Cluster Copilot application/composition 定向测试全部通过;AI 完整测试 + 238 pass/3 条件 skip,Cluster Control 完整测试 240 pass/2 条件 skip。 +2. 18-package clean build/test 全部通过;backend 1,207 pass/2 条件 skip/0 fail。package boundary、 + dependency、Edge import 与 Cluster deployment 四项审计均为 compatible、零 finding。 +3. workspace 仍为 18 个 package,无 single-source 或 shallow-source package;AI 187 个源码中 186 个、 + Cluster Control 59 个源码中 57 个位于嵌套领域目录。新增 composition 没有制造薄 package 或根层平铺。 +4. 14 档 Local Profile artifact 全部通过。默认 Edge/Standalone 保持 + 2,589,890/2,589,968 bytes;Edge/Standalone AI 为 3,064,454/3,064,544 bytes,均未引入 Cluster-only + composition,且分别保有 1,604,414/1,604,336 与 2,178,426/2,178,336 bytes 体积余量。 +5. PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`,报告 SHA-256 为 + `981299b454dce5541e9596450b85816dc40559cba8dc42adf3d5fea571c3d3a6`。本 Gate 不新增 + migration、schema、role、SQL privilege 或 HA 拓扑;真实 repository 的 admission、Tool completion、 + Model ciphertext、finalization 与 exact replay 沿用同一受审 authority,Docker 门禁与清理均通过。 + +## 后续门禁 + +1. 为 Tool `failed|timed_out`、日志 `missing|retired|pending`、deadline/cancel 和 Model + `outcome_unknown` 建立有界 durable terminalization/recovery。 +2. 完成后再增加经 authentication、Policy、audit、request identity 和 source fence 保护的 Cluster API, + 随后复用该 capability 提供 CLI/UI/MCP,而不是建立旁路执行器。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 33f0b337..ad59d386 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -414,6 +414,7 @@ | [ADR-0408](./ADR-0408-cluster-copilot-failure-diagnosis-tool-execution.md) | Cluster Copilot Failure Diagnosis Tool Execution 与原子 Model 解锁 | Accepted | | [ADR-0409](./ADR-0409-cluster-copilot-failure-diagnosis-model-execution.md) | Cluster Copilot Failure Diagnosis Model Execution、密文输出与 Run 终态化 | Accepted | | [ADR-0410](./ADR-0410-cluster-copilot-failure-diagnosis-output-key-authority.md) | Cluster Copilot Failure Diagnosis Output Projected Key Authority | Accepted | +| [ADR-0411](./ADR-0411-default-off-cluster-copilot-composition.md) | 默认关闭的 Cluster Copilot Failure Diagnosis Composition | Accepted | ## 规则 diff --git a/packages/ql3-ai/package.json b/packages/ql3-ai/package.json index 5c08cc7d..ca0d989c 100644 --- a/packages/ql3-ai/package.json +++ b/packages/ql3-ai/package.json @@ -55,6 +55,11 @@ "require": "./dist/copilot/failure-diagnosis/modelExecution.js", "default": "./dist/copilot/failure-diagnosis/modelExecution.js" }, + "./failure-diagnosis-application": { + "types": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.d.ts", + "require": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js", + "default": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js" + }, "./postgres-failure-diagnosis-model-execution-storage": { "types": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.d.ts", "require": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.js", diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/application/contracts.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/application/contracts.ts new file mode 100644 index 00000000..151ea73c --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/application/contracts.ts @@ -0,0 +1,59 @@ +import type { SecurityPrincipal } from '@qinglong/runtime-core/security'; + +import type { CopilotFailureDiagnosisAdmissionReceipt } from '../admission/contracts'; +import type { CopilotFailureDiagnosisModelExecutionResult } from '../model-execution/coordinator'; +import type { CopilotFailureDiagnosisToolExecutionResult } from '../tool-execution/contracts'; + +export const MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS = 64; + +export interface ExecuteCopilotFailureDiagnosisApplicationCommand { + readonly requestId: string; + readonly traceId: string; + readonly projectId: string; + readonly sourceRunId: string; + readonly principal: Readonly; +} + +export interface ExecuteCopilotFailureDiagnosisApplicationResult { + readonly admissionStatus: 'created' | 'existing'; + readonly admission: Readonly; + readonly tool: Readonly; + readonly model: Readonly | null; + readonly terminalizationRequired: boolean; +} + +export class InvalidCopilotFailureDiagnosisApplicationError extends TypeError { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_INVALID'; + + constructor(message: string) { + super(`Copilot failure diagnosis application is invalid: ${message}`); + this.name = 'InvalidCopilotFailureDiagnosisApplicationError'; + } +} + +export class CopilotFailureDiagnosisApplicationConflictError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_CONFLICT'; + + constructor(message = 'the durable diagnosis request changed') { + super(`Copilot failure diagnosis application conflicts: ${message}`); + this.name = 'CopilotFailureDiagnosisApplicationConflictError'; + } +} + +export class CopilotFailureDiagnosisApplicationUnavailableError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_UNAVAILABLE'; + + constructor(options?: ErrorOptions) { + super('Copilot failure diagnosis application is unavailable', options); + this.name = 'CopilotFailureDiagnosisApplicationUnavailableError'; + } +} + +export class CopilotFailureDiagnosisApplicationBusyError extends Error { + readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_BUSY'; + + constructor() { + super('Copilot failure diagnosis application request budget is exhausted'); + this.name = 'CopilotFailureDiagnosisApplicationBusyError'; + } +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/application/service.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/application/service.ts new file mode 100644 index 00000000..087c899a --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/application/service.ts @@ -0,0 +1,572 @@ +import { Buffer } from 'node:buffer'; +import { createHash, createHmac } from 'node:crypto'; + +import { + BUILTIN_RUN_LOG_EXCERPT_TOOL, + createBuiltInRunLogExcerptToolHandlerBinding, +} from '@qinglong/runtime-core/builtin-run-log-excerpt-tool'; +import type { RunRepositoryReader } from '@qinglong/runtime-core/run-repository'; +import type { ToolPolicyAuthorizer } from '@qinglong/runtime-core/tool-registry'; +import { + prepareToolInvocation, +} from '@qinglong/runtime-core/tool-registry'; +import type { ProjectToolDefinitionSnapshotRepository } from '@qinglong/runtime-core/project-tool-definition-snapshot'; +import { projectToolDefinitionRegistry } from '@qinglong/runtime-core/project-tool-definition-snapshot'; +import { + createToolInvocationInputArtifact, + createToolInvocationPreviewArtifact, + type ToolInvocationArtifactKeyProvider, + type ToolInvocationArtifactRepository, + type ToolInvocationPreviewDocument, +} from '@qinglong/runtime-core/tool-invocation-artifact'; +import { + TrustedToolHandlerBindingRegistry, + createTrustedToolInvocationPlan, +} from '@qinglong/runtime-core/trusted-tool-invocation'; + +import { MAX_MODEL_INVOCATION_MS } from '../../../model-gateway/model'; +import { + prepareCopilotFailureDiagnosisExecution, +} from '../admission/plan'; +import type { + CopilotFailureDiagnosisAdmissionRepository, + CopilotFailureDiagnosisExecutionPlan, + PrepareCopilotFailureDiagnosisModelIntent, +} from '../admission/contracts'; +import { + executeCopilotFailureDiagnosisTool, + type CopilotFailureDiagnosisToolExecutionDependencies, +} from '../tool-execution/coordinator'; +import { + executeCopilotFailureDiagnosisModel, + type CopilotFailureDiagnosisModelExecutionDependencies, +} from '../model-execution/coordinator'; +import { + CopilotFailureDiagnosisApplicationBusyError, + CopilotFailureDiagnosisApplicationConflictError, + CopilotFailureDiagnosisApplicationUnavailableError, + InvalidCopilotFailureDiagnosisApplicationError, + MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS, + type ExecuteCopilotFailureDiagnosisApplicationCommand, + type ExecuteCopilotFailureDiagnosisApplicationResult, +} from './contracts'; + +const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/; +const NONCE_DOMAIN = Buffer.from( + 'qinglong/copilot-failure-diagnosis-tool-invocation-nonce@v1\0', + 'utf8', +); +const IDENTITY_DOMAIN = Buffer.from( + 'qinglong/copilot-failure-diagnosis-application-identity@v1\0', + 'utf8', +); + +export interface CopilotFailureDiagnosisApplicationDependencies { + readonly admissions: CopilotFailureDiagnosisAdmissionRepository; + readonly snapshots: Pick; + readonly runs: Pick< + RunRepositoryReader, + 'findRunById' | 'findLatestAttemptByRunId' + >; + readonly artifacts: ToolInvocationArtifactRepository; + readonly invocationKeys: Pick< + ToolInvocationArtifactKeyProvider, + 'active' | 'resolve' + >; + readonly authorizer: ToolPolicyAuthorizer; + readonly tool: CopilotFailureDiagnosisToolExecutionDependencies; + readonly model: CopilotFailureDiagnosisModelExecutionDependencies; + readonly executeTool: typeof executeCopilotFailureDiagnosisTool; + readonly executeModel: typeof executeCopilotFailureDiagnosisModel; + readonly modelIntent: Readonly; + readonly executionTimeoutMs: number; + readonly now?: () => number; + readonly nonceFactory?: (input: Readonly<{ + key: Uint8Array; + keyId: string; + requestId: string; + projectId: string; + sourceRunId: string; + invocationActionDigest: string; + }>) => Uint8Array; +} + +interface ActiveRequest { + readonly digest: string; + readonly promise: Promise< + Readonly + >; +} + +function invalid(message: string): never { + throw new InvalidCopilotFailureDiagnosisApplicationError(message); +} + +function unavailable(cause?: unknown): never { + throw new CopilotFailureDiagnosisApplicationUnavailableError({ + cause: cause instanceof Error ? cause : undefined, + }); +} + +function digest(value: unknown): string { + return createHash('sha256').update(JSON.stringify(value)).digest('hex'); +} + +function identity(prefix: string, requestId: string): string { + return `${prefix}:${createHash('sha256') + .update(IDENTITY_DOMAIN) + .update(prefix) + .update('\0') + .update(requestId) + .digest('hex') + .slice(0, 32)}`; +} + +function preview( + sourceRunId: string, + sourceAttemptId: string, +): Readonly { + return Object.freeze({ + title: 'Diagnose failed Run', + summary: 'Read one bounded, redacted and untrusted execution log excerpt', + fields: Object.freeze([ + Object.freeze({ + kind: 'identifier' as const, + label: 'Run', + value: sourceRunId, + }), + Object.freeze({ + kind: 'identifier' as const, + label: 'Attempt', + value: sourceAttemptId, + }), + ]), + warnings: Object.freeze(['potentially_sensitive_output']), + }); +} + +function defaultNonce(input: Readonly<{ + key: Uint8Array; + keyId: string; + requestId: string; + projectId: string; + sourceRunId: string; + invocationActionDigest: string; +}>): Uint8Array { + const derived = createHmac('sha256', Buffer.from(input.key)) + .update(NONCE_DOMAIN) + .update( + JSON.stringify({ + keyId: input.keyId, + requestId: input.requestId, + projectId: input.projectId, + sourceRunId: input.sourceRunId, + invocationActionDigest: input.invocationActionDigest, + }), + ) + .digest(); + try { + return Buffer.from(derived.subarray(0, 12)); + } finally { + derived.fill(0); + } +} + +function normalizeCommand( + value: ExecuteCopilotFailureDiagnosisApplicationCommand, +): Readonly { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + Object.keys(value).sort().join('\0') !== + ['principal', 'projectId', 'requestId', 'sourceRunId', 'traceId'].join( + '\0', + ) || + !ID_PATTERN.test(value.requestId) || + !ID_PATTERN.test(value.traceId) || + !ID_PATTERN.test(value.projectId) || + !RUN_ID_PATTERN.test(value.sourceRunId) || + !value.principal || + typeof value.principal !== 'object' || + Array.isArray(value.principal) + ) { + return invalid('command is invalid'); + } + return Object.freeze({ + requestId: value.requestId, + traceId: value.traceId, + projectId: value.projectId, + sourceRunId: value.sourceRunId, + principal: value.principal, + }); +} + +function assertDependencies( + value: CopilotFailureDiagnosisApplicationDependencies, +): void { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + typeof value.admissions?.findByRequestId !== 'function' || + typeof value.admissions?.findPlanByRequestId !== 'function' || + typeof value.admissions?.admit !== 'function' || + typeof value.snapshots?.findCurrent !== 'function' || + typeof value.runs?.findRunById !== 'function' || + typeof value.runs?.findLatestAttemptByRunId !== 'function' || + typeof value.artifacts?.put !== 'function' || + typeof value.invocationKeys?.active !== 'function' || + typeof value.invocationKeys?.resolve !== 'function' || + typeof value.authorizer?.authorize !== 'function' || + typeof value.executeTool !== 'function' || + typeof value.executeModel !== 'function' || + !value.tool || + !value.model || + !value.modelIntent || + typeof value.modelIntent !== 'object' || + !Number.isSafeInteger(value.executionTimeoutMs) || + value.executionTimeoutMs < 1 || + value.executionTimeoutMs > MAX_MODEL_INVOCATION_MS || + (value.now !== undefined && typeof value.now !== 'function') || + (value.nonceFactory !== undefined && + typeof value.nonceFactory !== 'function') + ) { + return invalid('dependencies are invalid'); + } + if ( + value.tool.admissions !== value.admissions || + value.tool.snapshots !== value.snapshots || + value.tool.runs !== value.runs || + value.tool.artifacts !== value.artifacts || + value.tool.invocationKeys !== value.invocationKeys || + value.model.admissions !== value.admissions || + value.model.unlocks !== value.tool.unlocks + ) { + return invalid('dependency authorities are not shared'); + } +} + +function sameSubject( + left: Readonly<{ type: string; id: string }>, + right: Readonly<{ type: string; id: string }>, +): boolean { + return left.type === right.type && left.id === right.id; +} + +function sameModelIntent( + plan: Readonly, + configured: Readonly, +): boolean { + return ( + plan.model.provider === configured.provider && + plan.model.model === configured.model && + plan.model.modelBoundary === configured.modelBoundary && + plan.model.responseLanguage === configured.responseLanguage && + plan.model.maxOutputTokens === configured.maxOutputTokens && + JSON.stringify(plan.model.egressPolicy) === + JSON.stringify(configured.egressPolicy) + ); +} + +function nonceInput( + plan: Readonly, + key: Uint8Array, +) { + return Object.freeze({ + key, + keyId: plan.tool.invocationArtifact.keyId, + requestId: plan.requestId, + projectId: plan.projectId, + sourceRunId: plan.source.runId, + invocationActionDigest: plan.tool.invocationActionDigest, + }); +} + +export class CopilotFailureDiagnosisApplicationService { + readonly #dependencies: CopilotFailureDiagnosisApplicationDependencies; + readonly #active = new Map(); + + constructor(dependencies: CopilotFailureDiagnosisApplicationDependencies) { + assertDependencies(dependencies); + this.#dependencies = dependencies; + } + + execute( + commandValue: ExecuteCopilotFailureDiagnosisApplicationCommand, + ): Promise> { + const command = normalizeCommand(commandValue); + const commandDigest = digest(command); + const active = this.#active.get(command.requestId); + if (active) { + if (active.digest !== commandDigest) { + throw new CopilotFailureDiagnosisApplicationConflictError(); + } + return active.promise; + } + if ( + this.#active.size >= + MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS + ) { + throw new CopilotFailureDiagnosisApplicationBusyError(); + } + const promise = this.#execute(command).finally(() => { + this.#active.delete(command.requestId); + }); + this.#active.set(command.requestId, { digest: commandDigest, promise }); + return promise; + } + + async #execute( + command: Readonly, + ): Promise> { + const existing = await this.#dependencies.admissions.findPlanByRequestId( + command.requestId, + ); + let plan: Readonly; + let admissionStatus: 'created' | 'existing'; + let admission; + if (existing) { + if ( + existing.projectId !== command.projectId || + existing.source.runId !== command.sourceRunId || + existing.traceId !== command.traceId || + !sameSubject(existing.requestedBySubject, command.principal.subject) || + !sameModelIntent(existing, this.#dependencies.modelIntent) + ) { + throw new CopilotFailureDiagnosisApplicationConflictError(); + } + plan = existing; + const admitted = await this.#dependencies.admissions.admit(plan); + admissionStatus = admitted.status; + admission = admitted.receipt; + await this.#materializeArtifacts(plan); + } else { + const prepared = await this.#prepare(command); + plan = prepared.plan; + const admitted = await this.#dependencies.admissions.admit(plan); + admissionStatus = admitted.status; + admission = admitted.receipt; + try { + await this.#dependencies.artifacts.put( + prepared.inputArtifact, + prepared.previewArtifact, + ); + } catch (cause) { + throw new CopilotFailureDiagnosisApplicationUnavailableError({ + cause: cause instanceof Error ? cause : undefined, + }); + } + } + const tool = await this.#dependencies.executeTool( + { + requestId: plan.requestId, + principal: command.principal, + authorizer: this.#dependencies.authorizer, + }, + this.#dependencies.tool, + ); + if (tool.outcome !== 'succeeded') { + return Object.freeze({ + admissionStatus, + admission, + tool, + model: null, + terminalizationRequired: true, + }); + } + const model = await this.#dependencies.executeModel( + plan.requestId, + this.#dependencies.model, + ); + return Object.freeze({ + admissionStatus, + admission, + tool, + model, + terminalizationRequired: false, + }); + } + + async #prepare( + command: Readonly, + ) { + const now = this.#clock(); + let run; + let attempt; + let snapshotRecord; + try { + [run, attempt, snapshotRecord] = await Promise.all([ + this.#dependencies.runs.findRunById(command.sourceRunId), + this.#dependencies.runs.findLatestAttemptByRunId(command.sourceRunId), + this.#dependencies.snapshots.findCurrent(command.projectId), + ]); + } catch (cause) { + return unavailable(cause); + } + if ( + !run || + !attempt || + !snapshotRecord || + run.projectId !== command.projectId || + attempt.runId !== run.id || + !['failed', 'timed_out'].includes(run.status) || + !['failed', 'timed_out', 'lost'].includes(attempt.status) || + (run.status === 'failed' && + !['failed', 'lost'].includes(attempt.status)) || + (run.status === 'timed_out' && attempt.status !== 'timed_out') || + !Number.isSafeInteger(attempt.finishedAtMs) || + typeof attempt.logArtifactId !== 'string' + ) { + throw new CopilotFailureDiagnosisApplicationConflictError( + 'source Run is not an exact diagnosable terminal fence', + ); + } + const snapshot = snapshotRecord.snapshot; + const binding = createBuiltInRunLogExcerptToolHandlerBinding(snapshot, [ + 'cluster-control', + ]); + const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]); + const invocation = await prepareToolInvocation( + projectToolDefinitionRegistry(snapshot), + { + projectId: command.projectId, + principal: command.principal, + nowMs: now, + tool: BUILTIN_RUN_LOG_EXCERPT_TOOL, + input: { runId: run.id, attemptId: attempt.id }, + }, + this.#dependencies.authorizer, + ); + if (invocation.status !== 'ready') { + throw new CopilotFailureDiagnosisApplicationConflictError( + 'Tool admission is not ready', + ); + } + const key = await this.#dependencies.invocationKeys.active(); + try { + const baseIdentity = identity('cda', command.requestId); + const nonce = ( + this.#dependencies.nonceFactory ?? defaultNonce + )({ + key: key.key, + keyId: key.keyId, + requestId: command.requestId, + projectId: command.projectId, + sourceRunId: run.id, + invocationActionDigest: invocation.actionDigest, + }); + const tool = createTrustedToolInvocationPlan(bindings, invocation, { + actionRef: baseIdentity, + profile: 'cluster-control', + preview: preview(run.id, attempt.id), + inputArtifactId: identity('cdia', command.requestId), + previewArtifactId: identity('cdpa', command.requestId), + artifactKeyId: key.keyId, + artifactKey: key.key, + artifactNonce: nonce, + sealedAtMs: now, + }); + nonce.fill(0); + const plan = prepareCopilotFailureDiagnosisExecution({ + requestId: command.requestId, + traceId: command.traceId, + source: { + runId: run.id, + runVersion: run.version, + runStatus: run.status as 'failed' | 'timed_out', + attemptId: attempt.id, + attemptStatus: attempt.status as 'failed' | 'timed_out' | 'lost', + attemptFinishedAtMs: attempt.finishedAtMs!, + logArtifactId: attempt.logArtifactId, + }, + toolPlan: tool.plan, + bindings, + model: this.#dependencies.modelIntent, + deadlineAtMs: now + this.#dependencies.executionTimeoutMs, + plannedAtMs: now, + }); + return Object.freeze({ + plan, + inputArtifact: tool.inputArtifact, + previewArtifact: tool.previewArtifact, + }); + } finally { + key.key.fill(0); + } + } + + async #materializeArtifacts( + plan: Readonly, + ): Promise { + const material = await this.#dependencies.invocationKeys.resolve( + plan.tool.invocationArtifact.keyId, + ); + if (!material || material.keyId !== plan.tool.invocationArtifact.keyId) { + return unavailable(); + } + try { + const nonce = ( + this.#dependencies.nonceFactory ?? defaultNonce + )(nonceInput(plan, material.key)); + const inputArtifact = createToolInvocationInputArtifact( + { + artifactId: plan.tool.invocationArtifact.artifactId, + projectId: plan.projectId, + actionRef: plan.tool.actionRef, + requestedBy: plan.requestedBySubject, + tool: BUILTIN_RUN_LOG_EXCERPT_TOOL, + input: { + attemptId: plan.source.attemptId, + runId: plan.source.runId, + }, + inputDigest: plan.tool.invocationArtifact.inputDigest, + invocationActionDigest: plan.tool.invocationActionDigest, + keyId: material.keyId, + key: material.key, + sealedAtMs: plan.tool.sealedAtMs, + }, + () => nonce, + ); + nonce.fill(0); + const previewArtifact = createToolInvocationPreviewArtifact({ + artifactId: plan.tool.previewArtifact.artifactId, + projectId: plan.projectId, + actionRef: plan.tool.actionRef, + actionDigest: plan.tool.actionDigest, + redactionContractDigest: + plan.tool.previewArtifact.redactionContractDigest, + preview: preview(plan.source.runId, plan.source.attemptId), + sealedAtMs: plan.tool.sealedAtMs, + }); + if ( + inputArtifact.artifactDigest !== + plan.tool.invocationArtifact.artifactDigest || + previewArtifact.artifactDigest !== + plan.tool.previewArtifact.artifactDigest + ) { + throw new CopilotFailureDiagnosisApplicationConflictError( + 'durable Tool Artifact references cannot be reconstructed', + ); + } + await this.#dependencies.artifacts.put(inputArtifact, previewArtifact); + } finally { + material.key.fill(0); + } + } + + #clock(): number { + let value: number; + try { + value = (this.#dependencies.now ?? Date.now)(); + } catch (cause) { + return unavailable(cause); + } + if (!Number.isSafeInteger(value) || value < 0) return invalid('clock'); + if (value + this.#dependencies.executionTimeoutMs > Number.MAX_SAFE_INTEGER) { + return invalid('deadline overflows'); + } + return value; + } +} diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/failureDiagnosisApplication.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/failureDiagnosisApplication.ts new file mode 100644 index 00000000..f4329f5b --- /dev/null +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/failureDiagnosisApplication.ts @@ -0,0 +1,2 @@ +export * from './application/contracts'; +export * from './application/service'; diff --git a/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/coordinator.ts b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/coordinator.ts index 5756b010..977a0874 100644 --- a/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/coordinator.ts +++ b/packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/coordinator.ts @@ -5,7 +5,10 @@ import { type TrustedToolSuccessCompletionReadDependencies, } from '@qinglong/runtime-core/trusted-tool-completion'; -import { BoundedModelGateway } from '../../../model-gateway/gateway'; +import type { + BoundedModelGateway, + ModelInvocationSuccessfulCompletionSink, +} from '../../../model-gateway/gateway'; import type { ModelInvocationRepository } from '../../../model-invocation/modelInvocation'; import type { CopilotFailureDiagnosisToolExecutionAdmissionReader } from '../tool-execution/contracts'; import type { CopilotFailureDiagnosisToolUnlockRepository } from '../tool-execution/contracts'; @@ -36,20 +39,26 @@ export interface CopilotFailureDiagnosisModelExecutionDependencies { CopilotFailureDiagnosisOutputCompletionRepository, 'findCopilotFailureDiagnosisOutput' >; - readonly gateway: BoundedModelGateway; + readonly gateway: Pick< + BoundedModelGateway, + 'generate' | 'supportsSuccessfulCompletionSink' + >; readonly successfulCompletion: CopilotFailureDiagnosisModelCompletionCoordinator; readonly finalizations: CopilotFailureDiagnosisFinalizationRepository; } export interface CopilotFailureDiagnosisToolResultReader { - open(startId: string): Promise>; + open( + requestId: string, + startId: string, + ): Promise>; } export function createCopilotFailureDiagnosisToolResultReader( dependencies: TrustedToolSuccessCompletionReadDependencies, ): CopilotFailureDiagnosisToolResultReader { return Object.freeze({ - open: (startId: string) => + open: (_requestId: string, startId: string) => openTrustedToolSuccessCompletion(startId, dependencies), }); } @@ -97,7 +106,8 @@ function assertDependencies( typeof value.modelInvocations?.findStart !== 'function' || typeof value.modelInvocations?.findCompletion !== 'function' || typeof value.outputs?.findCopilotFailureDiagnosisOutput !== 'function' || - !(value.gateway instanceof BoundedModelGateway) || + typeof value.gateway?.generate !== 'function' || + typeof value.gateway?.supportsSuccessfulCompletionSink !== 'function' || typeof value.successfulCompletion?.begin !== 'function' || typeof value.successfulCompletion?.reference !== 'function' || typeof value.successfulCompletion?.end !== 'function' || @@ -215,7 +225,7 @@ export async function executeCopilotFailureDiagnosisModel( ); } - const tool = await dependencies.toolResults.open(unlock.startId); + const tool = await dependencies.toolResults.open(requestId, unlock.startId); if ( tool.completion.completionDigest !== unlock.toolCompletionDigest || tool.completion.runId !== plan.runId || @@ -241,7 +251,7 @@ export async function executeCopilotFailureDiagnosisModel( }); if ( !dependencies.gateway.supportsSuccessfulCompletionSink( - dependencies.successfulCompletion, + dependencies.successfulCompletion as ModelInvocationSuccessfulCompletionSink, ) ) { throw new InvalidCopilotFailureDiagnosisModelExecutionError( diff --git a/packages/ql3-ai/src/model-gateway/gateway.ts b/packages/ql3-ai/src/model-gateway/gateway.ts index cbe4188a..71cb770f 100644 --- a/packages/ql3-ai/src/model-gateway/gateway.ts +++ b/packages/ql3-ai/src/model-gateway/gateway.ts @@ -39,6 +39,12 @@ import { type ModelPriceCatalogResolver, } from '../pricing/pricing'; +export { + InvalidModelInvocationSuccessfulCompletionRouterError, + MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS, + ModelInvocationSuccessfulCompletionRouter, +} from './successfulCompletionRouter'; + export const MAX_MODEL_GATEWAY_CONCURRENCY = 64; export class ModelProviderUnavailableError extends Error { @@ -126,6 +132,9 @@ export interface BoundedModelGatewayOptions { } export interface ModelInvocationSuccessfulCompletionSink { + supportsSuccessfulCompletionSink?( + sink: ModelInvocationSuccessfulCompletionSink, + ): boolean; record( audit: Readonly, result: Readonly, @@ -372,7 +381,11 @@ export class BoundedModelGateway { supportsSuccessfulCompletionSink( sink: ModelInvocationSuccessfulCompletionSink, ): boolean { - return this.#successfulCompletion === sink; + const configured = this.#successfulCompletion; + return ( + configured === sink || + configured?.supportsSuccessfulCompletionSink?.(sink) === true + ); } async #prepare( diff --git a/packages/ql3-ai/src/model-gateway/successfulCompletionRouter.ts b/packages/ql3-ai/src/model-gateway/successfulCompletionRouter.ts new file mode 100644 index 00000000..70a30a56 --- /dev/null +++ b/packages/ql3-ai/src/model-gateway/successfulCompletionRouter.ts @@ -0,0 +1,80 @@ +import type { ModelInvocationSuccessfulCompletionSink } from './gateway'; +import type { + GenerateResult, + ModelInvocationAuditRecord, + ModelInvocationAuditResult, +} from './model'; + +export const MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS = 8; + +export class InvalidModelInvocationSuccessfulCompletionRouterError extends TypeError { + readonly code = 'MODEL_INVOCATION_SUCCESSFUL_COMPLETION_ROUTER_INVALID'; + + constructor() { + super('Model invocation successful completion router is invalid'); + this.name = 'InvalidModelInvocationSuccessfulCompletionRouterError'; + } +} + +/** Bounded dispatch for mutually exclusive durable output domains. */ +export class ModelInvocationSuccessfulCompletionRouter + implements ModelInvocationSuccessfulCompletionSink +{ + readonly #sinks: readonly ModelInvocationSuccessfulCompletionSink[]; + + constructor(sinks: readonly ModelInvocationSuccessfulCompletionSink[]) { + if ( + !Array.isArray(sinks) || + sinks.length < 2 || + sinks.length > MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS || + new Set(sinks).size !== sinks.length || + sinks.some( + (sink) => + !sink || + typeof sink !== 'object' || + typeof sink.record !== 'function', + ) + ) { + throw new InvalidModelInvocationSuccessfulCompletionRouterError(); + } + this.#sinks = Object.freeze([...sinks]); + Object.freeze(this); + } + + supportsSuccessfulCompletionSink( + sink: ModelInvocationSuccessfulCompletionSink, + ): boolean { + return this.#sinks.some( + (candidate) => + candidate === sink || + candidate.supportsSuccessfulCompletionSink?.(sink) === true, + ); + } + + async record( + audit: Readonly, + result: Readonly, + ): Promise< + Readonly< + | { handled: false } + | { handled: true; disposition: ModelInvocationAuditResult } + > + > { + for (const sink of this.#sinks) { + const routed = await sink.record(audit, result); + if ( + !routed || + typeof routed !== 'object' || + Array.isArray(routed) || + (routed.handled !== true && routed.handled !== false) + ) { + throw new InvalidModelInvocationSuccessfulCompletionRouterError(); + } + if (routed.handled) return routed; + if (Object.keys(routed).length !== 1) { + throw new InvalidModelInvocationSuccessfulCompletionRouterError(); + } + } + return Object.freeze({ handled: false as const }); + } +} diff --git a/packages/ql3-ai/src/prompt/postgres-plugin-package-prompt-application/bootstrap.ts b/packages/ql3-ai/src/prompt/postgres-plugin-package-prompt-application/bootstrap.ts index 46224adb..a765152c 100644 --- a/packages/ql3-ai/src/prompt/postgres-plugin-package-prompt-application/bootstrap.ts +++ b/packages/ql3-ai/src/prompt/postgres-plugin-package-prompt-application/bootstrap.ts @@ -15,6 +15,10 @@ import { import type { PluginPackagePromptOutputReadService } from '../../prompt-output/pluginPackagePromptOutputRead'; import type { PluginPackagePromptExecutionOutputReadService } from '../../prompt-output/pluginPackagePromptExecutionOutputRead'; import { bootstrapModelGatewayProfile } from '../../profile/profileComposition'; +import { + ModelInvocationSuccessfulCompletionRouter, + type ModelInvocationSuccessfulCompletionSink, +} from '../../model-gateway/gateway'; import { PostgresPluginPackagePromptApplicationUnavailableError, unavailable, @@ -49,6 +53,8 @@ function assertEnabledOptions( (options.confirmActive !== undefined && typeof options.confirmActive !== 'function') || (options.now !== undefined && typeof options.now !== 'function') || + (options.createAdditionalSuccessfulCompletion !== undefined && + typeof options.createAdditionalSuccessfulCompletion !== 'function') || (options.promptOutputKeys !== undefined && (!options.promptOutputKeys || typeof options.promptOutputKeys !== 'object' || @@ -118,17 +124,36 @@ export async function bootstrapPostgresPluginPackagePromptApplication( }); }, loadProviders: options.loadProviders, - ...(options.promptOutputKeys === undefined + ...(options.promptOutputKeys === undefined && + options.createAdditionalSuccessfulCompletion === undefined ? {} : { createSuccessfulCompletion: (coordinator) => { - durableOutput = - new PluginPackagePromptOutputCompletionCoordinator({ + const sinks: ModelInvocationSuccessfulCompletionSink[] = []; + if (options.promptOutputKeys !== undefined) { + durableOutput = + new PluginPackagePromptOutputCompletionCoordinator({ coordinator, keys: options.promptOutputKeys!, ...(options.now === undefined ? {} : { now: options.now }), }); - return durableOutput; + sinks.push(durableOutput); + } + if (options.createAdditionalSuccessfulCompletion !== undefined) { + const additionalSuccessfulCompletion = + options.createAdditionalSuccessfulCompletion(coordinator); + if ( + !additionalSuccessfulCompletion || + typeof additionalSuccessfulCompletion.record !== 'function' + ) { + throw new TypeError( + 'PostgreSQL Package Prompt additional completion is invalid', + ); + } + sinks.push(additionalSuccessfulCompletion); + } + if (sinks.length === 1) return sinks[0]!; + return new ModelInvocationSuccessfulCompletionRouter(sinks); }, }), audit: options.audit, diff --git a/packages/ql3-ai/src/prompt/postgres-plugin-package-prompt-application/contracts.ts b/packages/ql3-ai/src/prompt/postgres-plugin-package-prompt-application/contracts.ts index 47253b2e..38d3fadc 100644 --- a/packages/ql3-ai/src/prompt/postgres-plugin-package-prompt-application/contracts.ts +++ b/packages/ql3-ai/src/prompt/postgres-plugin-package-prompt-application/contracts.ts @@ -21,6 +21,8 @@ import type { ModelGatewayProfileAudit, ModelGatewayProviderAuthority, } from '../../profile/profileComposition'; +import type { DurableModelInvocationCoordinator } from '../../model-invocation/durableModelInvocationCoordinator'; +import type { ModelInvocationSuccessfulCompletionSink } from '../../model-gateway/gateway'; import type { PluginPackagePromptCatalogCapability } from '../pluginPackagePromptCatalog'; import type { PluginPackagePromptExecutionInspectionRepository } from '../pluginPackagePromptExecutionInspection'; @@ -80,6 +82,9 @@ export type BootstrapPostgresPluginPackagePromptApplicationOptions = maxConcurrent?: number; recoveryLimit?: number; now?: () => number; + createAdditionalSuccessfulCompletion?: ( + coordinator: DurableModelInvocationCoordinator, + ) => ModelInvocationSuccessfulCompletionSink; promptOutputKeys?: PluginPackagePromptOutputArtifactKeyProvider; promptOutputRead?: Readonly<{ authorizer: PluginPackagePromptOutputArtifactReadAuthorizer; diff --git a/packages/ql3-ai/test/failure-diagnosis-application.test.cjs b/packages/ql3-ai/test/failure-diagnosis-application.test.cjs new file mode 100644 index 00000000..04f30ac9 --- /dev/null +++ b/packages/ql3-ai/test/failure-diagnosis-application.test.cjs @@ -0,0 +1,271 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + CopilotFailureDiagnosisApplicationService, + CopilotFailureDiagnosisApplicationUnavailableError, +} = require('@qinglong/ai/failure-diagnosis-application'); +const { + BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION, +} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool'); +const { + createPluginPackageResourceGenerationFromReferences, +} = require('@qinglong/runtime-core/plugin-package-resource-generation'); +const { + createProjectToolDefinitionSnapshot, +} = require('@qinglong/runtime-core/project-tool-definition-snapshot'); + +const NOW = 1_800_000_000_000; +const KEY = Buffer.alloc(32, 0x42); +const MODEL = Object.freeze({ + provider: 'provider-primary', + model: 'model-diagnosis', + modelBoundary: 'external', + responseLanguage: 'zh-CN', + maxOutputTokens: 512, + egressPolicy: Object.freeze({ + schema: 'qinglong/copilot-model-egress-policy@v1', + revision: 'application-test-v1', + potentiallySensitiveDataBoundaries: Object.freeze(['external']), + maxInputBytes: 64 * 1024, + maxOutputTokens: 1024, + }), +}); + +function snapshot() { + const generation = createPluginPackageResourceGenerationFromReferences({ + installationId: 'installation-copilot-test', + projectId: 'project-1', + packageName: 'qinglong', + lockDigest: 'a'.repeat(64), + generation: 1, + previousActiveLockDigest: null, + contentDigest: 'b'.repeat(64), + resources: [], + }); + return createProjectToolDefinitionSnapshot({ + projectId: 'project-1', + contributions: [ + { + generation, + revisionDigest: 'c'.repeat(64), + definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION], + }, + ], + }); +} + +function command(overrides = {}) { + return { + requestId: 'diagnosis-request-1', + traceId: 'diagnosis-trace-1', + projectId: 'project-1', + sourceRunId: 'source-run-1', + principal: { + subject: { type: 'user', id: 'owner-1' }, + authenticationId: 'auth-1', + authenticatedAtMs: NOW - 1000, + expiresAtMs: NOW + 60_000, + assurance: 'multi_factor', + }, + ...overrides, + }; +} + +function fixture(options = {}) { + let plan = null; + let artifacts = null; + let failArtifactOnce = options.failArtifactOnce === true; + let activeKeyCopies = []; + let resolvedKeyCopies = []; + let toolCalls = 0; + let modelCalls = 0; + let releaseTool; + const toolGate = options.blockTool + ? new Promise((resolve) => { releaseTool = resolve; }) + : Promise.resolve(); + const admissions = { + async findByRequestId(requestId) { + return plan?.requestId === requestId + ? { requestId, planDigest: plan.planDigest } + : null; + }, + async findPlanByRequestId(requestId) { + return plan?.requestId === requestId ? plan : null; + }, + async admit(value) { + const status = plan ? 'existing' : 'created'; + plan ??= value; + assert.equal(plan.planDigest, value.planDigest); + return { + status, + receipt: { + requestId: plan.requestId, + planDigest: plan.planDigest, + runId: plan.runId, + }, + }; + }, + }; + const snapshots = { + async findCurrent() { return { snapshot: snapshot(), committedAtMs: NOW }; }, + }; + const runs = { + async findRunById() { + return { + id: 'source-run-1', projectId: 'project-1', status: 'failed', + version: 8, eventSequence: 8, + }; + }, + async findLatestAttemptByRunId() { + return { + id: 'source-attempt-1', runId: 'source-run-1', attempt: 1, + status: 'failed', executorType: 'remote_worker', callbackSequence: 0, + createdAtMs: NOW - 3000, finishedAtMs: NOW - 1000, + logArtifactId: `wlog-${'d'.repeat(30)}`, + }; + }, + }; + const artifactRepository = { + async put(input, preview) { + if (failArtifactOnce) { + failArtifactOnce = false; + throw new Error('simulated admission-to-artifact crash'); + } + if (artifacts) { + assert.deepEqual(input, artifacts.input); + assert.deepEqual(preview, artifacts.preview); + return { status: 'existing' }; + } + artifacts = { input, preview }; + return { status: 'inserted' }; + }, + async findInput() { return artifacts?.input ?? null; }, + async findPreview() { return artifacts?.preview ?? null; }, + }; + const invocationKeys = { + async active() { + const copy = Buffer.from(KEY); + activeKeyCopies.push(copy); + return { keyId: 'invocation-key-1', key: copy }; + }, + async resolve(keyId) { + assert.equal(keyId, 'invocation-key-1'); + const copy = Buffer.from(KEY); + resolvedKeyCopies.push(copy); + return { keyId, key: copy }; + }, + }; + const unlocks = { async findByRequestId() { return null; }, async commit() {} }; + const tool = { + admissions, + snapshots, + runs, + artifacts: artifactRepository, + invocationKeys, + resultKeys: { async resolve() { return null; } }, + stepRuns: { async findById() { return null; } }, + barriers: {}, completions: {}, failureCompletions: {}, + resultKeyCatalog: {}, resultRekeys: {}, logs: {}, unlocks, + }; + const model = { + admissions, + unlocks, + toolResults: {}, modelInvocations: {}, outputs: {}, gateway: {}, + successfulCompletion: {}, finalizations: {}, + }; + const service = new CopilotFailureDiagnosisApplicationService({ + admissions, + snapshots, + runs, + artifacts: artifactRepository, + invocationKeys, + authorizer: { + async authorize() { + return { + effect: 'allow', reasons: ['role_grant'], + fence: { projectVersion: 1, bindingVersion: 1 }, + }; + }, + }, + tool, + model, + modelIntent: MODEL, + executionTimeoutMs: 60_000, + now: () => NOW, + async executeTool() { + toolCalls += 1; + await toolGate; + return options.toolFailure + ? { outcome: 'failed', completionStatus: 'created', unlockStatus: null } + : { + outcome: 'succeeded', completionStatus: 'created', + unlockStatus: 'created', completion: {}, unlock: {}, + }; + }, + async executeModel() { + modelCalls += 1; + return { + outcome: 'succeeded', + output: { artifactId: 'output-1' }, + finalization: { requestId: 'diagnosis-request-1' }, + }; + }, + }); + return { + service, + releaseTool: () => releaseTool?.(), + state: () => ({ + plan, artifacts, toolCalls, modelCalls, + activeKeyCopies, resolvedKeyCopies, + }), + }; +} + +test('application derives, admits and executes one server-owned diagnosis with exact replay', async () => { + const testFixture = fixture(); + const first = await testFixture.service.execute(command()); + assert.equal(first.admissionStatus, 'created'); + assert.equal(first.tool.outcome, 'succeeded'); + assert.equal(first.model.outcome, 'succeeded'); + assert.equal(first.terminalizationRequired, false); + const second = await testFixture.service.execute(command()); + assert.equal(second.admissionStatus, 'existing'); + const state = testFixture.state(); + assert.equal(state.toolCalls, 2); + assert.equal(state.modelCalls, 2); + assert.equal(state.plan.source.attemptId, 'source-attempt-1'); + assert.equal(state.plan.tool.invocationArtifact.artifactId.startsWith('cdia:'), true); + assert.equal(state.activeKeyCopies[0].every((value) => value === 0), true); + assert.equal(state.resolvedKeyCopies[0].every((value) => value === 0), true); +}); + +test('application repairs the durable admission-to-Artifact crash window', async () => { + const testFixture = fixture({ failArtifactOnce: true }); + await assert.rejects( + testFixture.service.execute(command()), + CopilotFailureDiagnosisApplicationUnavailableError, + ); + assert.ok(testFixture.state().plan); + assert.equal(testFixture.state().artifacts, null); + const replay = await testFixture.service.execute(command()); + assert.equal(replay.admissionStatus, 'existing'); + assert.ok(testFixture.state().artifacts); +}); + +test('application coalesces exact callers and exposes Tool terminalization debt', async () => { + const concurrent = fixture({ blockTool: true }); + const first = concurrent.service.execute(command()); + const second = concurrent.service.execute(command()); + assert.equal(first, second); + concurrent.releaseTool(); + await first; + assert.equal(concurrent.state().toolCalls, 1); + + const failed = fixture({ toolFailure: true }); + const result = await failed.service.execute(command()); + assert.equal(result.tool.outcome, 'failed'); + assert.equal(result.model, null); + assert.equal(result.terminalizationRequired, true); + assert.equal(failed.state().modelCalls, 0); +}); diff --git a/packages/ql3-ai/test/model-successful-completion-router.test.cjs b/packages/ql3-ai/test/model-successful-completion-router.test.cjs new file mode 100644 index 00000000..b48ea2c2 --- /dev/null +++ b/packages/ql3-ai/test/model-successful-completion-router.test.cjs @@ -0,0 +1,81 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + BoundedModelGateway, + InvalidModelInvocationSuccessfulCompletionRouterError, + ModelInvocationSuccessfulCompletionRouter, +} = require('@qinglong/ai/gateway'); + +function sink(name, handled = false) { + return { + async record(audit) { + audit.order.push(name); + return handled + ? { handled: true, disposition: { status: 'created' } } + : { handled: false }; + }, + }; +} + +test('successful completion router exposes exact children and stops at the owning sink', async () => { + const first = sink('prompt'); + const second = sink('copilot', true); + const third = sink('unreachable', true); + const nested = new ModelInvocationSuccessfulCompletionRouter([ + second, + third, + ]); + const router = new ModelInvocationSuccessfulCompletionRouter([first, nested]); + const order = []; + assert.equal(router.supportsSuccessfulCompletionSink(first), true); + assert.equal(router.supportsSuccessfulCompletionSink(second), true); + assert.equal(router.supportsSuccessfulCompletionSink({ record() {} }), false); + assert.deepEqual(await router.record({ order }, {}), { + handled: true, + disposition: { status: 'created' }, + }); + assert.deepEqual(order, ['prompt', 'copilot']); + + const gateway = new BoundedModelGateway({ + providers: [ + { + type: 'test', + async generate() { throw new Error('unused'); }, + async *stream() { throw new Error('unused'); }, + async listModels() { return []; }, + }, + ], + policies: { async resolve() { throw new Error('unused'); } }, + pricing: { async resolve() { return null; } }, + audit: { async record() {} }, + successfulCompletion: router, + maxConcurrent: 1, + }); + assert.equal(gateway.supportsSuccessfulCompletionSink(router), true); + assert.equal(gateway.supportsSuccessfulCompletionSink(second), true); +}); + +test('successful completion router rejects unbounded, duplicate and malformed sinks', async () => { + const valid = sink('valid'); + assert.throws( + () => new ModelInvocationSuccessfulCompletionRouter([valid]), + InvalidModelInvocationSuccessfulCompletionRouterError, + ); + assert.throws( + () => new ModelInvocationSuccessfulCompletionRouter([valid, valid]), + InvalidModelInvocationSuccessfulCompletionRouterError, + ); + assert.throws( + () => new ModelInvocationSuccessfulCompletionRouter([valid, {}]), + InvalidModelInvocationSuccessfulCompletionRouterError, + ); + const malformed = new ModelInvocationSuccessfulCompletionRouter([ + valid, + { async record() { return { handled: false, widened: true }; } }, + ]); + await assert.rejects( + malformed.record({ order: [] }, {}), + InvalidModelInvocationSuccessfulCompletionRouterError, + ); +}); diff --git a/packages/ql3-cluster-control/package.json b/packages/ql3-cluster-control/package.json index 690b9375..275ddc72 100644 --- a/packages/ql3-cluster-control/package.json +++ b/packages/ql3-cluster-control/package.json @@ -35,6 +35,11 @@ "require": "./dist/application-runtime/aiProductionApplication.js", "default": "./dist/application-runtime/aiProductionApplication.js" }, + "./copilot-production": { + "types": "./dist/application-runtime/copilot/failureDiagnosisComposition.d.ts", + "require": "./dist/application-runtime/copilot/failureDiagnosisComposition.js", + "default": "./dist/application-runtime/copilot/failureDiagnosisComposition.js" + }, "./failure-diagnosis-output-keyring": { "types": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.d.ts", "require": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js", diff --git a/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts b/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts index c7362e76..2dd72370 100644 --- a/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts +++ b/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts @@ -1,4 +1,7 @@ import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile'; +import type { DurableModelInvocationCoordinator } from '@qinglong/ai/durable-model-invocation'; +import { CopilotFailureDiagnosisModelCompletionCoordinator } from '@qinglong/ai/failure-diagnosis-model-execution'; +import type { CopilotFailureDiagnosisApplicationService } from '@qinglong/ai/failure-diagnosis-application'; import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential'; import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage'; import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority'; @@ -22,12 +25,19 @@ import { startProductionClusterControlApplication, type ProductionClusterControlApplicationOptions, } from './productionApplication'; +import { + createProductionClusterCopilotFailureDiagnosis, + prepareProductionClusterCopilotFailureDiagnosisProjection, + type ClusterCopilotFailureDiagnosisProjection, + type CreateProductionClusterCopilotFailureDiagnosisOptions, +} from './copilot/failureDiagnosisComposition'; export interface EnabledProductionClusterAiConfig { readonly enabled: true; readonly providerAuthorityFile: string; readonly secretRootDirectory: string; readonly promptOutputKeyringRootDirectory?: string; + readonly copilot?: Readonly; readonly maxConcurrent: number; readonly recoveryLimit: number; readonly databaseMaxConnections: number; @@ -41,8 +51,18 @@ export interface ProductionClusterAiControlApplicationOptions { ) => void | Promise; readonly startControl?: typeof startProductionClusterControlApplication; readonly bootstrapPrompt?: typeof bootstrapPostgresPluginPackagePromptApplication; + readonly createCopilot?: ( + options: CreateProductionClusterCopilotFailureDiagnosisOptions, + ) => Promise>; + readonly openAiDatabase?: ReturnType; } +export type ProductionClusterAiControlApplicationResult = Extract< + ClusterControlApplicationResult, + { readonly status: 'active' } +> & + Readonly<{ copilot?: Readonly }>; + export class ProductionClusterAiConfigError extends TypeError { readonly code = 'QL3_CLUSTER_AI_CONFIG_INVALID'; @@ -121,6 +141,11 @@ export function loadProductionClusterAiConfig( 'QL3_CLUSTER_AI_PROMPT_OUTPUT_ENABLED', false, ); + const copilotEnabled = booleanValue( + environment, + 'QL3_CLUSTER_AI_COPILOT_ENABLED', + false, + ); return Object.freeze({ enabled: true, providerAuthorityFile: requiredPath( @@ -139,6 +164,28 @@ export function loadProductionClusterAiConfig( ), } : {}), + ...(copilotEnabled + ? { + copilot: Object.freeze({ + configFile: requiredPath( + environment, + 'QL3_CLUSTER_AI_COPILOT_CONFIG_FILE', + ), + invocationKeyringRootDirectory: requiredPath( + environment, + 'QL3_CLUSTER_AI_COPILOT_INVOCATION_KEYRING_ROOT', + ), + resultKeyringRootDirectory: requiredPath( + environment, + 'QL3_CLUSTER_AI_COPILOT_RESULT_KEYRING_ROOT', + ), + outputKeyringRootDirectory: requiredPath( + environment, + 'QL3_CLUSTER_AI_COPILOT_OUTPUT_KEYRING_ROOT', + ), + }), + } + : {}), maxConcurrent: boundedInteger( environment, 'QL3_CLUSTER_AI_MAX_CONCURRENT', @@ -187,7 +234,7 @@ function aiDatabaseOpener( */ export async function startProductionClusterAiControlApplication( options: ProductionClusterAiControlApplicationOptions, -): Promise { +): Promise { if ( !options || typeof options !== 'object' || @@ -200,9 +247,26 @@ export async function startProductionClusterAiControlApplication( options.startControl ?? startProductionClusterControlApplication; const bootstrapPrompt = options.bootstrapPrompt ?? bootstrapPostgresPluginPackagePromptApplication; - if (typeof startControl !== 'function' || typeof bootstrapPrompt !== 'function') { + const createCopilot = + options.createCopilot ?? createProductionClusterCopilotFailureDiagnosis; + if ( + typeof startControl !== 'function' || + typeof bootstrapPrompt !== 'function' || + typeof createCopilot !== 'function' || + (options.openAiDatabase !== undefined && + typeof options.openAiDatabase !== 'function') + ) { throw new TypeError('Production Cluster AI application factories are invalid'); } + const copilotArtifactStore = options.control.workerIngress?.artifactStore; + if ( + options.ai.copilot !== undefined && + typeof copilotArtifactStore?.readLogRange !== 'function' + ) { + throw new TypeError( + 'Cluster Copilot requires the bounded Worker log Artifact read capability', + ); + } const secretMaterial = await createProjectedModelProviderSecretMaterialProvider({ rootDirectory: options.ai.secretRootDirectory, @@ -213,6 +277,12 @@ export async function startProductionClusterAiControlApplication( : await createPluginPackagePromptOutputProjectedKeyring({ rootDirectory: options.ai.promptOutputKeyringRootDirectory, }); + const preparedCopilot = + options.ai.copilot === undefined + ? undefined + : await prepareProductionClusterCopilotFailureDiagnosisProjection( + options.ai.copilot, + ); let aiDatabase: | Awaited>> | undefined; @@ -230,6 +300,12 @@ export async function startProductionClusterAiControlApplication( | BootstrapPostgresPluginPackagePromptApplicationResult | undefined; let controlApplication: ClusterControlApplicationResult | undefined; + let copilotApplication: + | Readonly + | undefined; + let copilotSuccessfulCompletion: + | CopilotFailureDiagnosisModelCompletionCoordinator + | undefined; let stopPromise: Promise | undefined; let promptOutputPolicy: ProjectPolicyEngine | undefined; const promptOutputReadAuthorizer = Object.freeze({ @@ -271,11 +347,9 @@ export async function startProductionClusterAiControlApplication( return stopPromise; }; try { - const openDatabase = aiDatabaseOpener( - options.control.config, - options.ai, - onAiUnavailable, - ); + const openDatabase = + options.openAiDatabase ?? + aiDatabaseOpener(options.control.config, options.ai, onAiUnavailable); promptApplication = await bootstrapPrompt({ enabled: true, async openDatabase() { @@ -311,10 +385,41 @@ export async function startProductionClusterAiControlApplication( promptOutputKeys, promptOutputRead: { authorizer: promptOutputReadAuthorizer }, }), + ...(preparedCopilot === undefined + ? {} + : { + createAdditionalSuccessfulCompletion( + coordinator: DurableModelInvocationCoordinator, + ) { + if (copilotSuccessfulCompletion) { + throw new Error( + 'Cluster Copilot completion was created more than once', + ); + } + copilotSuccessfulCompletion = + new CopilotFailureDiagnosisModelCompletionCoordinator({ + coordinator, + keys: preparedCopilot.outputKeys, + }); + return copilotSuccessfulCompletion; + }, + }), }); if (promptApplication.status !== 'active') { throw new Error('Cluster AI Prompt application did not activate'); } + if (preparedCopilot !== undefined) { + if (!copilotSuccessfulCompletion || !aiDatabase || !copilotArtifactStore) { + throw new Error('Cluster Copilot shared authorities did not activate'); + } + copilotApplication = await createCopilot({ + pool: aiDatabase.pool, + gateway: promptApplication.capability, + prepared: preparedCopilot, + successfulCompletion: copilotSuccessfulCompletion, + artifactStore: copilotArtifactStore, + }); + } controlApplication = await startControl({ ...options.control, promptCatalog: { @@ -356,6 +461,9 @@ export async function startProductionClusterAiControlApplication( ? 'unavailable' : activeControl.availabilityStatus(); }, + ...(copilotApplication === undefined + ? {} + : { copilot: copilotApplication }), stop, }); } catch (error) { diff --git a/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisComposition.ts b/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisComposition.ts new file mode 100644 index 00000000..1b0d55f3 --- /dev/null +++ b/packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisComposition.ts @@ -0,0 +1,466 @@ +import { Buffer } from 'node:buffer'; +import { basename, dirname } from 'node:path'; + +import { + CopilotFailureDiagnosisApplicationService, + type CopilotFailureDiagnosisApplicationDependencies, +} from '@qinglong/ai/failure-diagnosis-application'; +import type { PrepareCopilotFailureDiagnosisModelIntent } from '@qinglong/ai/failure-diagnosis-execution-admission'; +import { + CopilotFailureDiagnosisModelCompletionCoordinator, + executeCopilotFailureDiagnosisModel, + type CopilotFailureDiagnosisToolResultReader, +} from '@qinglong/ai/failure-diagnosis-model-execution'; +import { + executeCopilotFailureDiagnosisTool, + restoreCopilotFailureDiagnosisTrustedToolAuthority, +} from '@qinglong/ai/failure-diagnosis-tool-execution'; +import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage'; +import { PostgresCopilotFailureDiagnosisModelRepository } from '@qinglong/ai/postgres-failure-diagnosis-model-execution-storage'; +import { PostgresCopilotFailureDiagnosisToolUnlockRepository } from '@qinglong/ai/postgres-failure-diagnosis-tool-execution-storage'; +import type { ActiveModelGatewayCapability } from '@qinglong/ai/profile'; +import { + PostgresProjectPolicyRepository, + PostgresProjectToolDefinitionSnapshotRepository, + PostgresRunAttemptLogRetentionClaimRepository, + PostgresRunRepository, + PostgresStepRunRepository, + PostgresToolExecutionCompletionRepository, + PostgresToolExecutionFailureCompletionRepository, + PostgresToolExecutionStartBarrierRepository, + PostgresToolInvocationArtifactRepository, + PostgresToolResultKeyCatalogReader, + PostgresToolResultRekeyReader, + type QingLongPostgresPool, +} from '@qinglong/cluster-postgres/runtime'; +import { + BuiltInRunLogExcerptToolAdapter, +} from '@qinglong/runtime-core/builtin-run-log-excerpt-tool'; +import type { RunAttemptLogReadPort } from '@qinglong/runtime-core/builtin-run-log-excerpt-projection'; +import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy'; +import { RunAttemptLogReadService } from '@qinglong/runtime-core/run-attempt-log-read'; +import { openTrustedToolSuccessCompletion } from '@qinglong/runtime-core/trusted-tool-completion'; +import { TrustedToolExecutionAdapterRegistry } from '@qinglong/runtime-core/trusted-tool-execution'; + +import type { ClusterRemoteWorkerArtifactStore } from '../../remote-execution/remoteWorkerCompletionService'; +import { PrivateProjectedFileReader } from '../../security/privateProjectedFile'; +import { createClusterToolInvocationProjectedKeyring } from '../../trusted-tool/key-management/toolInvocationProjectedKeyring'; +import { createClusterToolResultProjectedKeyring } from '../../trusted-tool/key-management/toolResultProjectedKeyring'; +import { createClusterCopilotFailureDiagnosisOutputProjectedKeyring } from '../../copilot/failure-diagnosis/outputProjectedKeyring'; + +export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA = + 'qinglong/cluster-copilot-failure-diagnosis-config@v1' as const; +export const MAX_CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_BYTES = 16 * 1024; + +const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const MODEL_BOUNDARIES = ['on_device', 'external'] as const; +const RESPONSE_LANGUAGES = ['en', 'zh-CN'] as const; +const EGRESS_SCHEMA = 'qinglong/copilot-model-egress-policy@v1' as const; + +export interface ClusterCopilotFailureDiagnosisConfig { + readonly schema: typeof CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA; + readonly provider: string; + readonly model: string; + readonly modelBoundary: 'on_device' | 'external'; + readonly responseLanguage: 'en' | 'zh-CN'; + readonly maxOutputTokens: number; + readonly executionTimeoutMs: number; + readonly egressPolicy: PrepareCopilotFailureDiagnosisModelIntent['egressPolicy']; +} + +export interface ClusterCopilotFailureDiagnosisProjection { + readonly configFile: string; + readonly invocationKeyringRootDirectory: string; + readonly resultKeyringRootDirectory: string; + readonly outputKeyringRootDirectory: string; +} + +export interface CreateProductionClusterCopilotFailureDiagnosisOptions { + readonly pool: QingLongPostgresPool; + readonly gateway: ActiveModelGatewayCapability; + readonly prepared: PreparedClusterCopilotFailureDiagnosisProjection; + readonly successfulCompletion: CopilotFailureDiagnosisModelCompletionCoordinator; + readonly artifactStore: ClusterRemoteWorkerArtifactStore; +} + +export interface PreparedClusterCopilotFailureDiagnosisProjection { + readonly config: Readonly; + readonly invocationKeys: Awaited< + ReturnType + >; + readonly resultKeys: Awaited< + ReturnType + >; + readonly outputKeys: Awaited< + ReturnType + >; +} + +export class ClusterCopilotFailureDiagnosisCompositionError extends Error { + readonly code = 'QL3_CLUSTER_COPILOT_FAILURE_DIAGNOSIS_COMPOSITION_INVALID'; + + constructor(message: string, options?: ErrorOptions) { + super(`Cluster Copilot failure diagnosis composition is invalid: ${message}`, options); + this.name = 'ClusterCopilotFailureDiagnosisCompositionError'; + } +} + +function invalid(message: string, cause?: unknown): never { + throw new ClusterCopilotFailureDiagnosisCompositionError(message, { + cause: cause instanceof Error ? cause : undefined, + }); +} + +function record(value: unknown, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return invalid(`${label} must be an object`); + } + return value as Record; +} + +function exactKeys( + value: Readonly>, + expected: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const canonical = [...expected].sort(); + if ( + actual.length !== canonical.length || + actual.some((key, index) => key !== canonical[index]) + ) { + return invalid(`${label} shape is invalid`); + } +} + +function identity(value: unknown, label: string): string { + if (typeof value !== 'string' || !IDENTITY.test(value)) { + return invalid(`${label} is invalid`); + } + return value; +} + +function integer( + value: unknown, + minimum: number, + maximum: number, + label: string, +): number { + if ( + typeof value !== 'number' || + !Number.isSafeInteger(value) || + value < minimum || + value > maximum + ) { + return invalid(`${label} is invalid`); + } + return value; +} + +export function normalizeClusterCopilotFailureDiagnosisConfig( + value: unknown, +): Readonly { + const candidate = record(value, 'configuration'); + exactKeys( + candidate, + [ + 'egressPolicy', + 'executionTimeoutMs', + 'maxOutputTokens', + 'model', + 'modelBoundary', + 'provider', + 'responseLanguage', + 'schema', + ], + 'configuration', + ); + if (candidate.schema !== CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA) { + return invalid('schema is invalid'); + } + if (!MODEL_BOUNDARIES.includes(candidate.modelBoundary as never)) { + return invalid('model boundary is invalid'); + } + if (!RESPONSE_LANGUAGES.includes(candidate.responseLanguage as never)) { + return invalid('response language is invalid'); + } + const egress = record(candidate.egressPolicy, 'egress policy'); + exactKeys( + egress, + [ + 'maxInputBytes', + 'maxOutputTokens', + 'potentiallySensitiveDataBoundaries', + 'revision', + 'schema', + ], + 'egress policy', + ); + if (egress.schema !== EGRESS_SCHEMA) return invalid('egress schema is invalid'); + const selected = egress.potentiallySensitiveDataBoundaries; + if ( + !Array.isArray(selected) || + selected.length < 1 || + selected.length > MODEL_BOUNDARIES.length || + selected.some((entry) => !MODEL_BOUNDARIES.includes(entry as never)) || + new Set(selected).size !== selected.length || + MODEL_BOUNDARIES.filter((entry) => selected.includes(entry)).some( + (entry, index) => entry !== selected[index], + ) || + !selected.includes(candidate.modelBoundary) + ) { + return invalid('egress model boundaries are invalid'); + } + const egressMaxOutputTokens = integer( + egress.maxOutputTokens, + 1, + 4_096, + 'egress max output tokens', + ); + const maxOutputTokens = integer( + candidate.maxOutputTokens, + 1, + egressMaxOutputTokens, + 'max output tokens', + ); + return Object.freeze({ + schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA, + provider: identity(candidate.provider, 'provider'), + model: identity(candidate.model, 'model'), + modelBoundary: candidate.modelBoundary as 'on_device' | 'external', + responseLanguage: candidate.responseLanguage as 'en' | 'zh-CN', + maxOutputTokens, + executionTimeoutMs: integer( + candidate.executionTimeoutMs, + 1, + 5 * 60_000, + 'execution timeout', + ), + egressPolicy: Object.freeze({ + schema: EGRESS_SCHEMA, + revision: identity(egress.revision, 'egress revision'), + potentiallySensitiveDataBoundaries: Object.freeze([...selected]) as ( + | 'on_device' + | 'external' + )[], + maxInputBytes: integer( + egress.maxInputBytes, + 1, + 64 * 1024, + 'egress max input bytes', + ), + maxOutputTokens: egressMaxOutputTokens, + }), + }); +} + +export function canonicalClusterCopilotFailureDiagnosisConfig( + value: unknown, +): Buffer { + return Buffer.from( + `${JSON.stringify(normalizeClusterCopilotFailureDiagnosisConfig(value))}\n`, + 'utf8', + ); +} + +export async function loadClusterCopilotFailureDiagnosisConfig( + configFile: string, +): Promise> { + let bytes: Buffer | undefined; + let canonical: Buffer | undefined; + try { + const reader = new PrivateProjectedFileReader({ + rootDirectory: dirname(configFile), + minimumBytes: 1, + maximumBytes: MAX_CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_BYTES, + access: 'read_only_keyring', + }); + bytes = await reader.read(basename(configFile)); + const parsed = JSON.parse(bytes.toString('utf8')) as unknown; + const config = normalizeClusterCopilotFailureDiagnosisConfig(parsed); + canonical = canonicalClusterCopilotFailureDiagnosisConfig(config); + if (!canonical.equals(bytes)) return invalid('file is not canonical'); + return config; + } catch (cause) { + return cause instanceof ClusterCopilotFailureDiagnosisCompositionError + ? invalid(cause.message, cause) + : invalid('projected configuration is unavailable', cause); + } finally { + bytes?.fill(0); + canonical?.fill(0); + } +} + +function modelIntent( + config: Readonly, +): Readonly { + return Object.freeze({ + provider: config.provider, + model: config.model, + modelBoundary: config.modelBoundary, + responseLanguage: config.responseLanguage, + maxOutputTokens: config.maxOutputTokens, + egressPolicy: config.egressPolicy, + }); +} + +export async function prepareProductionClusterCopilotFailureDiagnosisProjection( + projection: ClusterCopilotFailureDiagnosisProjection, +): Promise> { + if (!projection || typeof projection !== 'object' || Array.isArray(projection)) { + return invalid('projection is invalid'); + } + const [config, invocationKeys, resultKeys, outputKeys] = await Promise.all([ + loadClusterCopilotFailureDiagnosisConfig(projection.configFile), + createClusterToolInvocationProjectedKeyring({ + rootDirectory: projection.invocationKeyringRootDirectory, + }), + createClusterToolResultProjectedKeyring({ + rootDirectory: projection.resultKeyringRootDirectory, + }), + createClusterCopilotFailureDiagnosisOutputProjectedKeyring({ + rootDirectory: projection.outputKeyringRootDirectory, + }), + ]); + return Object.freeze({ config, invocationKeys, resultKeys, outputKeys }); +} + +export async function createProductionClusterCopilotFailureDiagnosis( + options: CreateProductionClusterCopilotFailureDiagnosisOptions, +): Promise> { + if ( + !options || + typeof options !== 'object' || + Array.isArray(options) || + typeof options.pool?.query !== 'function' || + typeof options.pool?.connect !== 'function' || + typeof options.gateway?.generate !== 'function' || + typeof options.gateway?.supportsSuccessfulCompletionSink !== 'function' || + typeof options.successfulCompletion?.begin !== 'function' || + typeof options.successfulCompletion?.record !== 'function' || + !options.prepared || + typeof options.artifactStore?.readLogRange !== 'function' + ) { + return invalid('dependencies are unavailable'); + } + if ( + !options.gateway.supportsSuccessfulCompletionSink( + options.successfulCompletion, + ) + ) { + return invalid('shared Model completion authority is unavailable'); + } + const { config, invocationKeys, resultKeys } = options.prepared; + const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository( + options.pool, + ); + const snapshots = new PostgresProjectToolDefinitionSnapshotRepository( + options.pool, + ); + const runs = new PostgresRunRepository(options.pool); + const artifacts = new PostgresToolInvocationArtifactRepository(options.pool); + const stepRuns = new PostgresStepRunRepository(options.pool); + const barriers = new PostgresToolExecutionStartBarrierRepository(options.pool); + const completions = new PostgresToolExecutionCompletionRepository(options.pool); + const failureCompletions = + new PostgresToolExecutionFailureCompletionRepository(options.pool); + const resultKeyCatalog = new PostgresToolResultKeyCatalogReader(options.pool); + const resultRekeys = new PostgresToolResultRekeyReader(options.pool); + const unlocks = new PostgresCopilotFailureDiagnosisToolUnlockRepository( + options.pool, + ); + const models = new PostgresCopilotFailureDiagnosisModelRepository(options.pool); + const logReader = new RunAttemptLogReadService( + runs, + Object.freeze({ + read: options.artifactStore.readLogRange.bind(options.artifactStore), + }), + { + executorType: 'remote_worker', + artifactIdPattern: /^wlog-[a-f0-9]{30}$/, + maximumReadBytes: 256 * 1024, + activeMissingIsPending: true, + }, + new PostgresRunAttemptLogRetentionClaimRepository(options.pool), + ); + const logs: RunAttemptLogReadPort = Object.freeze({ + read: logReader.read.bind(logReader), + }); + const successfulCompletion = options.successfulCompletion; + const toolResults: CopilotFailureDiagnosisToolResultReader = Object.freeze({ + async open(requestId: string, startId: string) { + const plan = await admissions.findPlanByRequestId(requestId); + if (!plan) return invalid('diagnosis plan is unavailable'); + const snapshot = await snapshots.findCurrent(plan.projectId); + if (!snapshot) return invalid('Tool snapshot is unavailable'); + const authority = restoreCopilotFailureDiagnosisTrustedToolAuthority( + plan, + snapshot.snapshot, + ); + const definitions = authority.bindings.definitionRegistry(); + const adapters = new TrustedToolExecutionAdapterRegistry( + authority.bindings, + [ + new BuiltInRunLogExcerptToolAdapter( + authority.binding, + 'cluster-control', + definitions, + logs, + ), + ], + ); + return openTrustedToolSuccessCompletion(startId, { + completions, + barriers, + resultKeyCatalog, + resultRekeys, + resultKeys, + adapters, + }); + }, + }); + const policy = new ProjectPolicyEngine( + new PostgresProjectPolicyRepository(options.pool), + ); + const tool = Object.freeze({ + admissions, + snapshots, + artifacts, + invocationKeys, + resultKeys, + stepRuns, + runs, + barriers, + completions, + failureCompletions, + resultKeyCatalog, + resultRekeys, + logs, + unlocks, + }); + const model = Object.freeze({ + admissions, + unlocks, + toolResults, + modelInvocations: models, + outputs: models, + gateway: options.gateway, + successfulCompletion, + finalizations: models, + }); + const dependencies: CopilotFailureDiagnosisApplicationDependencies = { + admissions, + snapshots, + runs, + artifacts, + invocationKeys, + authorizer: policy, + tool, + model, + executeTool: executeCopilotFailureDiagnosisTool, + executeModel: executeCopilotFailureDiagnosisModel, + modelIntent: modelIntent(config), + executionTimeoutMs: config.executionTimeoutMs, + }; + return new CopilotFailureDiagnosisApplicationService(dependencies); +} diff --git a/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs b/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs index 6f5b889b..9d1f0d8b 100644 --- a/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs +++ b/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs @@ -13,6 +13,18 @@ const { canonicalPluginPackagePromptOutputKeyringManifest, PLUGIN_PACKAGE_PROMPT_OUTPUT_KEYRING_MANIFEST_SCHEMA, } = require('@qinglong/ai/plugin-package-prompt-output-keyring-manifest'); +const { + canonicalClusterToolInvocationKeyringManifest, + CLUSTER_TOOL_INVOCATION_KEYRING_MANIFEST_SCHEMA, +} = require('@qinglong/cluster-control/trusted-tool-invocation-keyring'); +const { + canonicalClusterToolResultKeyringManifest, + CLUSTER_TOOL_RESULT_KEYRING_MANIFEST_SCHEMA, +} = require('@qinglong/cluster-control/trusted-tool-result-keyring'); +const { + canonicalClusterCopilotFailureDiagnosisOutputKeyringManifest, + CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_KEYRING_MANIFEST_SCHEMA, +} = require('@qinglong/cluster-control/failure-diagnosis-output-keyring'); function enabledEnvironment(overrides = {}) { return { @@ -74,6 +86,161 @@ test('AI config is fail-closed and bounded behind the explicit process flag', () ), /QL3_CLUSTER_AI_PROMPT_OUTPUT_KEYRING_ROOT is invalid/, ); + assert.throws( + () => + loadProductionClusterAiConfig( + enabledEnvironment({ QL3_CLUSTER_AI_COPILOT_ENABLED: 'true' }), + ), + /QL3_CLUSTER_AI_COPILOT_CONFIG_FILE is invalid/, + ); + assert.deepEqual( + loadProductionClusterAiConfig( + enabledEnvironment({ + QL3_CLUSTER_AI_COPILOT_ENABLED: 'true', + QL3_CLUSTER_AI_COPILOT_CONFIG_FILE: '/run/ql3/copilot/config.json', + QL3_CLUSTER_AI_COPILOT_INVOCATION_KEYRING_ROOT: '/run/ql3/invocation', + QL3_CLUSTER_AI_COPILOT_RESULT_KEYRING_ROOT: '/run/ql3/result', + QL3_CLUSTER_AI_COPILOT_OUTPUT_KEYRING_ROOT: '/run/ql3/output', + }), + ).copilot, + { + configFile: '/run/ql3/copilot/config.json', + invocationKeyringRootDirectory: '/run/ql3/invocation', + resultKeyringRootDirectory: '/run/ql3/result', + outputKeyringRootDirectory: '/run/ql3/output', + }, + ); +}); + +async function projectedFile(root, name, bytes) { + await writeFile(join(root, name), bytes, { mode: 0o440 }); + await chmod(join(root, name), 0o440); +} + +test('Copilot composition is explicit, shares the Prompt gateway and exposes no route', async () => { + const secretRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-secret-')); + const configRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-config-')); + const invocationRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-invocation-')); + const resultRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-result-')); + const outputRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-output-')); + const key = Buffer.alloc(32, 0x55).toString('base64url'); + const config = Buffer.from(`${JSON.stringify({ + schema: 'qinglong/cluster-copilot-failure-diagnosis-config@v1', + provider: 'provider-primary', + model: 'model-diagnosis', + modelBoundary: 'external', + responseLanguage: 'zh-CN', + maxOutputTokens: 512, + executionTimeoutMs: 60_000, + egressPolicy: { + schema: 'qinglong/copilot-model-egress-policy@v1', + revision: 'cluster-copilot-v1', + potentiallySensitiveDataBoundaries: ['external'], + maxInputBytes: 64 * 1024, + maxOutputTokens: 1024, + }, + })}\n`); + const invocation = canonicalClusterToolInvocationKeyringManifest({ + schema: CLUSTER_TOOL_INVOCATION_KEYRING_MANIFEST_SCHEMA, + activeKeyId: 'invocation-key-1', + keys: { 'invocation-key-1': key }, + }); + const result = canonicalClusterToolResultKeyringManifest({ + schema: CLUSTER_TOOL_RESULT_KEYRING_MANIFEST_SCHEMA, + keys: { 'result-key-1': key }, + }); + const output = canonicalClusterCopilotFailureDiagnosisOutputKeyringManifest({ + schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_KEYRING_MANIFEST_SCHEMA, + activeKeyId: 'output-key-1', + keys: { 'output-key-1': key }, + }); + const gateway = { + generate() {}, + supportsSuccessfulCompletionSink(sink) { + return sink === registeredSink; + }, + }; + const fakePool = { query() {}, connect() {} }; + const artifactStore = { put() {}, inspect() {}, readLogRange() {} }; + const copilot = Object.freeze({ execute() {} }); + let registeredSink; + let created; + let controlOptions; + try { + await Promise.all([ + projectedFile(configRoot, 'config.json', config), + projectedFile(invocationRoot, 'keyring.json', invocation), + projectedFile(resultRoot, 'keyring.json', result), + projectedFile(outputRoot, 'keyring.json', output), + ]); + const application = await startProductionClusterAiControlApplication({ + control: { + config: controlConfig(), + workerIngress: { + config: { enabled: true }, + artifactStore, + }, + }, + ai: { + enabled: true, + providerAuthorityFile: '/unused/providers.json', + secretRootDirectory: secretRoot, + copilot: { + configFile: join(configRoot, 'config.json'), + invocationKeyringRootDirectory: invocationRoot, + resultKeyringRootDirectory: resultRoot, + outputKeyringRootDirectory: outputRoot, + }, + maxConcurrent: 1, + recoveryLimit: 1, + databaseMaxConnections: 1, + }, + audit() {}, + async openAiDatabase() { + return { pool: fakePool, async close() {} }; + }, + async bootstrapPrompt(options) { + await options.openDatabase(); + registeredSink = options.createAdditionalSuccessfulCompletion({ + async recordWithAtomicSuccess() {}, + }); + return { + status: 'active', profile: 'cluster', readiness: {}, capability: gateway, + prompts: {}, promptCatalog: {}, promptExecutions: {}, + promptExecutionInspections: {}, async stop() { return 'stopped'; }, + }; + }, + async createCopilot(options) { + created = options; + return copilot; + }, + async startControl(options) { + controlOptions = options; + return { + status: 'active', address: { host: '127.0.0.1', port: 5800 }, + evidence: {}, recovery: { safe: true, remaining: 0, failed: 0 }, + unavailable: new Promise(() => {}), availabilityStatus() { return 'ready'; }, + async stop() { return 'stopped'; }, + }; + }, + }); + assert.equal(application.copilot, copilot); + assert.equal(created.pool, fakePool); + assert.equal(created.gateway, gateway); + assert.equal(created.successfulCompletion, registeredSink); + assert.equal(created.artifactStore, artifactStore); + assert.equal('copilot' in controlOptions, false); + assert.equal(await application.stop(), 'stopped'); + } finally { + config.fill(0); invocation.fill(0); result.fill(0); output.fill(0); + await Promise.all([ + rm(secretRoot, { recursive: true, force: true }), + rm(configRoot, { recursive: true, force: true }), + rm(invocationRoot, { recursive: true, force: true }), + rm(resultRoot, { recursive: true, force: true }), + rm(outputRoot, { recursive: true, force: true }), + ]); + } }); test('explicit AI composition injects one reviewed Prompt capability and drains it after HTTP control', async () => { diff --git a/packages/ql3-cluster-control/test/copilotFailureDiagnosisComposition.test.cjs b/packages/ql3-cluster-control/test/copilotFailureDiagnosisComposition.test.cjs new file mode 100644 index 00000000..652822cd --- /dev/null +++ b/packages/ql3-cluster-control/test/copilotFailureDiagnosisComposition.test.cjs @@ -0,0 +1,92 @@ +const assert = require('node:assert/strict'); +const { chmod, mkdtemp, rm, writeFile } = require('node:fs/promises'); +const { tmpdir } = require('node:os'); +const { join } = require('node:path'); +const { test } = require('node:test'); + +const { + CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA, + ClusterCopilotFailureDiagnosisCompositionError, + canonicalClusterCopilotFailureDiagnosisConfig, + loadClusterCopilotFailureDiagnosisConfig, + normalizeClusterCopilotFailureDiagnosisConfig, +} = require('@qinglong/cluster-control/copilot-production'); + +function config(overrides = {}) { + return { + schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA, + provider: 'provider-primary', + model: 'model-diagnosis', + modelBoundary: 'external', + responseLanguage: 'zh-CN', + maxOutputTokens: 512, + executionTimeoutMs: 60_000, + egressPolicy: { + schema: 'qinglong/copilot-model-egress-policy@v1', + revision: 'cluster-copilot-v1', + potentiallySensitiveDataBoundaries: ['external'], + maxInputBytes: 64 * 1024, + maxOutputTokens: 1024, + }, + ...overrides, + }; +} + +test('normalizes one bounded deployment-owned Copilot model intent', () => { + const normalized = normalizeClusterCopilotFailureDiagnosisConfig(config()); + assert.equal(normalized.provider, 'provider-primary'); + assert.equal(normalized.executionTimeoutMs, 60_000); + assert.equal(Object.isFrozen(normalized), true); + assert.equal(Object.isFrozen(normalized.egressPolicy), true); + assert.throws( + () => normalizeClusterCopilotFailureDiagnosisConfig(config({ extra: true })), + ClusterCopilotFailureDiagnosisCompositionError, + ); + assert.throws( + () => normalizeClusterCopilotFailureDiagnosisConfig(config({ + modelBoundary: 'on_device', + })), + /egress model boundaries/, + ); + assert.throws( + () => normalizeClusterCopilotFailureDiagnosisConfig(config({ + executionTimeoutMs: 300_001, + })), + /execution timeout/, + ); +}); + +test('loads only canonical read-only projected Copilot configuration', async () => { + const root = await mkdtemp(join(tmpdir(), 'ql3-copilot-config-test-')); + const file = join(root, 'config.json'); + const canonical = canonicalClusterCopilotFailureDiagnosisConfig(config()); + try { + await writeFile(file, canonical, { mode: 0o440 }); + await chmod(file, 0o440); + assert.deepEqual( + await loadClusterCopilotFailureDiagnosisConfig(file), + normalizeClusterCopilotFailureDiagnosisConfig(config()), + ); + + await chmod(file, 0o640); + await writeFile(file, Buffer.from(`${JSON.stringify(config(), null, 2)}\n`), { + mode: 0o440, + }); + await chmod(file, 0o440); + await assert.rejects( + loadClusterCopilotFailureDiagnosisConfig(file), + /not canonical/, + ); + + await chmod(file, 0o640); + await writeFile(file, canonical, { mode: 0o640 }); + await chmod(file, 0o640); + await assert.rejects( + loadClusterCopilotFailureDiagnosisConfig(file), + ClusterCopilotFailureDiagnosisCompositionError, + ); + } finally { + canonical.fill(0); + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/ql3-cluster-dependency-audit.cjs b/scripts/ql3-cluster-dependency-audit.cjs index aaed08d0..204e9447 100644 --- a/scripts/ql3-cluster-dependency-audit.cjs +++ b/scripts/ql3-cluster-dependency-audit.cjs @@ -1163,6 +1163,9 @@ function auditSourceImports(root, packagePath, findings) { const clusterAiImports = Object.freeze({ 'src/aiCli.ts': Object.freeze(['@qinglong/ai/profile']), 'src/application-runtime/aiProductionApplication.ts': Object.freeze([ + '@qinglong/ai/durable-model-invocation', + '@qinglong/ai/failure-diagnosis-application', + '@qinglong/ai/failure-diagnosis-model-execution', '@qinglong/ai/plugin-package-prompt-output-projected-keyring', '@qinglong/ai/postgres-model-provider-credential-storage', '@qinglong/ai/postgres-plugin-package-prompt-application', @@ -1171,6 +1174,17 @@ function auditSourceImports(root, packagePath, findings) { '@qinglong/ai/projected-model-provider-secret-material', '@qinglong/ai/provider-credential', ]), + 'src/application-runtime/copilot/failureDiagnosisComposition.ts': + Object.freeze([ + '@qinglong/ai/failure-diagnosis-application', + '@qinglong/ai/failure-diagnosis-execution-admission', + '@qinglong/ai/failure-diagnosis-model-execution', + '@qinglong/ai/failure-diagnosis-tool-execution', + '@qinglong/ai/postgres-failure-diagnosis-admission-storage', + '@qinglong/ai/postgres-failure-diagnosis-model-execution-storage', + '@qinglong/ai/postgres-failure-diagnosis-tool-execution-storage', + '@qinglong/ai/profile', + ]), }); const sourceRelative = path.relative(packageDirectory, filePath); if (!clusterAiImports[sourceRelative]?.includes(specifier)) { diff --git a/scripts/ql3-cluster-deployment-audit.cjs b/scripts/ql3-cluster-deployment-audit.cjs index 534d1465..8431655d 100644 --- a/scripts/ql3-cluster-deployment-audit.cjs +++ b/scripts/ql3-cluster-deployment-audit.cjs @@ -1311,6 +1311,28 @@ function assertClusterAiComponent(readFile, root, findings) { 'utf8', ), ); + const copilotComponentDirectory = path.join( + root, + 'deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot', + ); + const copilotComponent = yaml.load( + readFile(path.join(copilotComponentDirectory, 'kustomization.yaml'), 'utf8'), + ); + const copilotPatch = yaml.load( + readFile(path.join(copilotComponentDirectory, 'deployment-patch.yaml'), 'utf8'), + ); + const copilotConfig = yaml.load( + readFile(path.join(copilotComponentDirectory, 'copilot-configmap.yaml'), 'utf8'), + ); + const copilotOverlay = yaml.load( + readFile( + path.join( + root, + 'deploy/kubernetes/ql3-cluster/overlays/cluster-ai-copilot-example/kustomization.yaml', + ), + 'utf8', + ), + ); if ( component?.kind !== 'Component' || JSON.stringify(component?.resources) !== @@ -1326,6 +1348,156 @@ function assertClusterAiComponent(readFile, root, findings) { ); } + const copilotPod = objectAt(copilotPatch, ['spec', 'template', 'spec']); + const copilotContainer = namedEntry( + copilotPod?.containers, + 'cluster-control', + ); + const copilotEnv = environmentByName(copilotContainer); + const copilotProjections = [ + [ + 'cluster-ai-copilot-config', + '/var/run/qinglong3/ai/copilot-config', + 'configMap', + 'ql3-cluster-ai-copilot', + 'config.json', + ], + [ + 'cluster-ai-copilot-invocation-keyring', + '/var/run/secrets/qinglong3/ai/copilot-invocation-keyring', + 'secret', + 'ql3-cluster-ai-copilot-invocation-keyring', + 'keyring.json', + ], + [ + 'cluster-ai-copilot-result-keyring', + '/var/run/secrets/qinglong3/ai/copilot-result-keyring', + 'secret', + 'ql3-cluster-ai-copilot-result-keyring', + 'keyring.json', + ], + [ + 'cluster-ai-copilot-output-keyring', + '/var/run/secrets/qinglong3/ai/copilot-output-keyring', + 'secret', + 'ql3-cluster-ai-copilot-output-keyring', + 'keyring.json', + ], + ]; + const copilotProjectionInvalid = copilotProjections.some( + ([name, mountPath, kind, authorityName, key]) => { + const mount = namedEntry(copilotContainer?.volumeMounts, name); + const volume = namedEntry(copilotPod?.volumes, name); + const projection = volume?.[kind]; + return ( + mount?.mountPath !== mountPath || + mount?.readOnly !== true || + projection?.name !== authorityName && + projection?.secretName !== authorityName || + projection?.defaultMode !== 0o440 || + projection?.optional === true || + JSON.stringify(projection?.items) !== + JSON.stringify([{ key, path: key }]) + ); + }, + ); + if ( + copilotComponent?.kind !== 'Component' || + JSON.stringify(copilotComponent?.resources) !== + JSON.stringify(['copilot-configmap.yaml']) || + JSON.stringify(copilotComponent?.patches) !== + JSON.stringify([{ path: 'deployment-patch.yaml' }]) || + copilotPatch?.kind !== 'Deployment' || + copilotPatch?.metadata?.name !== 'ql3-cluster-control' || + copilotPod?.serviceAccountName !== undefined || + copilotPod?.automountServiceAccountToken !== undefined || + copilotPod?.containers?.length !== 1 || + copilotContainer?.image !== undefined || + copilotContainer?.env?.length !== 5 || + copilotContainer?.volumeMounts?.length !== 4 || + copilotPod?.volumes?.length !== 4 || + copilotProjectionInvalid + ) { + findings.push( + finding( + 'QL3_CLUSTER_AI_COPILOT_PROJECTION', + 'Cluster Copilot must remain an explicit component with one canonical ConfigMap and three independent required read-only 0440 keyring projections', + ), + ); + } + for (const [name, expected] of [ + ['QL3_CLUSTER_AI_COPILOT_ENABLED', 'true'], + [ + 'QL3_CLUSTER_AI_COPILOT_CONFIG_FILE', + '/var/run/qinglong3/ai/copilot-config/config.json', + ], + [ + 'QL3_CLUSTER_AI_COPILOT_INVOCATION_KEYRING_ROOT', + '/var/run/secrets/qinglong3/ai/copilot-invocation-keyring', + ], + [ + 'QL3_CLUSTER_AI_COPILOT_RESULT_KEYRING_ROOT', + '/var/run/secrets/qinglong3/ai/copilot-result-keyring', + ], + [ + 'QL3_CLUSTER_AI_COPILOT_OUTPUT_KEYRING_ROOT', + '/var/run/secrets/qinglong3/ai/copilot-output-keyring', + ], + ]) { + if (copilotEnv.get(name)?.value !== expected) { + findings.push( + finding( + 'QL3_CLUSTER_AI_COPILOT_ENVIRONMENT', + `${name} must be fixed to the reviewed projected authority ${expected}`, + ), + ); + } + } + const copilotConfigText = copilotConfig?.data?.['config.json']; + let parsedCopilotConfig; + try { + parsedCopilotConfig = JSON.parse(copilotConfigText); + } catch { + parsedCopilotConfig = undefined; + } + if ( + copilotConfig?.kind !== 'ConfigMap' || + copilotConfig?.metadata?.name !== 'ql3-cluster-ai-copilot' || + typeof copilotConfigText !== 'string' || + `${JSON.stringify(parsedCopilotConfig)}\n` !== copilotConfigText || + parsedCopilotConfig?.schema !== + 'qinglong/cluster-copilot-failure-diagnosis-config@v1' || + parsedCopilotConfig?.egressPolicy?.schema !== + 'qinglong/copilot-model-egress-policy@v1' + ) { + findings.push( + finding( + 'QL3_CLUSTER_AI_COPILOT_CONFIG', + 'Cluster Copilot configuration must remain canonical, bounded and explicitly versioned', + ), + ); + } + const copilotOverlayImage = copilotOverlay?.images?.[0]; + if ( + JSON.stringify(copilotOverlay?.components) !== + JSON.stringify([ + '../../components/cluster-ai', + '../../components/cluster-ai-copilot', + ]) || + copilotOverlayImage?.name !== 'qinglong3-cluster-control-ai' || + copilotOverlayImage?.newName !== + 'registry.example.com/qinglong/qinglong3-cluster-control-ai' || + !/^sha256:[0-9a-f]{64}$/.test(copilotOverlayImage?.digest ?? '') || + 'newTag' in (copilotOverlayImage ?? {}) + ) { + findings.push( + finding( + 'QL3_CLUSTER_AI_COPILOT_OVERLAY', + 'The Copilot overlay must compose explicit AI and Copilot components and independently pin the Cluster AI image digest', + ), + ); + } + const promptOutputPod = objectAt(promptOutputPatch, [ 'spec', 'template', @@ -1564,7 +1736,11 @@ function assertClusterAiComponent(readFile, root, findings) { ) || namedEntry(basePod?.volumes, 'cluster-ai-provider-authority') || namedEntry(basePod?.volumes, 'cluster-ai-provider-secrets') || - namedEntry(basePod?.volumes, 'cluster-ai-prompt-output-keyring') + namedEntry(basePod?.volumes, 'cluster-ai-prompt-output-keyring') || + [...copilotProjections].some(([name]) => + namedEntry(baseContainer?.volumeMounts, name) || + namedEntry(basePod?.volumes, name), + ) ) { findings.push( finding( diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index e2a82dc2..9456c7d1 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -268,10 +268,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( { path: 'packages/ql3-ai', name: '@qinglong/ai', - sourceFiles: 183, + sourceFiles: 187, rootSourceFiles: 1, rootSourceLines: 16, - nestedSourceFiles: 182, + nestedSourceFiles: 186, rootSourceFileHardCap: 1, rootSourceLineHardCap: 16, rootSourceFileRoles: { @@ -385,10 +385,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: clusterControl.rootSourceFileRoles, }, { - sourceFiles: 58, + sourceFiles: 59, rootSourceFiles: 2, rootSourceLines: 195, - nestedSourceFiles: 56, + nestedSourceFiles: 57, rootSourceFileRoles: { 'aiCli.ts': 'binary_entry', 'cli.ts': 'binary_entry',