mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-23 03:18:09 +08:00
feat(ql3): compose cluster copilot diagnosis
This commit is contained in:
@@ -0,0 +1,25 @@
|
|||||||
|
# Optional Cluster Copilot component
|
||||||
|
|
||||||
|
Compose this component only with `../cluster-ai`. It enables the caller-driven
|
||||||
|
failure-diagnosis capability inside the existing Cluster AI process; it does
|
||||||
|
not add an HTTP route, queue, watcher, timer, PostgreSQL pool, provider, or
|
||||||
|
Model Gateway.
|
||||||
|
|
||||||
|
Before production use:
|
||||||
|
|
||||||
|
1. Replace the example provider, model and egress policy revision in
|
||||||
|
`copilot-configmap.yaml`. Keep `config.json` canonical, one-line JSON with
|
||||||
|
one trailing newline.
|
||||||
|
2. Provision these three Secrets out of band, each containing a canonical
|
||||||
|
`keyring.json`: `ql3-cluster-ai-copilot-invocation-keyring`,
|
||||||
|
`ql3-cluster-ai-copilot-result-keyring`, and
|
||||||
|
`ql3-cluster-ai-copilot-output-keyring`.
|
||||||
|
3. Keep old decryptable keys during rotation. Invocation, Tool result and
|
||||||
|
Copilot output keys are independent authorities and must never reuse
|
||||||
|
material.
|
||||||
|
4. Keep Worker ingress and its bounded S3 log-range reader enabled. Startup
|
||||||
|
fails closed if the log capability, canonical config, any keyring, or the
|
||||||
|
shared successful-completion sink is unavailable.
|
||||||
|
|
||||||
|
All projections are read-only mode `0440`. The component adds no Kubernetes
|
||||||
|
API permission and does not change the default AI-free deployment.
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: ql3-cluster-ai-copilot
|
||||||
|
data:
|
||||||
|
config.json: |
|
||||||
|
{"schema":"qinglong/cluster-copilot-failure-diagnosis-config@v1","provider":"provider-primary","model":"model-diagnosis","modelBoundary":"external","responseLanguage":"zh-CN","maxOutputTokens":512,"executionTimeoutMs":60000,"egressPolicy":{"schema":"qinglong/copilot-model-egress-policy@v1","revision":"replace-before-production","potentiallySensitiveDataBoundaries":["external"],"maxInputBytes":65536,"maxOutputTokens":1024}}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: ql3-cluster-control
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: cluster-control
|
||||||
|
env:
|
||||||
|
- name: QL3_CLUSTER_AI_COPILOT_ENABLED
|
||||||
|
value: "true"
|
||||||
|
- name: QL3_CLUSTER_AI_COPILOT_CONFIG_FILE
|
||||||
|
value: /var/run/qinglong3/ai/copilot-config/config.json
|
||||||
|
- name: QL3_CLUSTER_AI_COPILOT_INVOCATION_KEYRING_ROOT
|
||||||
|
value: /var/run/secrets/qinglong3/ai/copilot-invocation-keyring
|
||||||
|
- name: QL3_CLUSTER_AI_COPILOT_RESULT_KEYRING_ROOT
|
||||||
|
value: /var/run/secrets/qinglong3/ai/copilot-result-keyring
|
||||||
|
- name: QL3_CLUSTER_AI_COPILOT_OUTPUT_KEYRING_ROOT
|
||||||
|
value: /var/run/secrets/qinglong3/ai/copilot-output-keyring
|
||||||
|
volumeMounts:
|
||||||
|
- name: cluster-ai-copilot-config
|
||||||
|
mountPath: /var/run/qinglong3/ai/copilot-config
|
||||||
|
readOnly: true
|
||||||
|
- name: cluster-ai-copilot-invocation-keyring
|
||||||
|
mountPath: /var/run/secrets/qinglong3/ai/copilot-invocation-keyring
|
||||||
|
readOnly: true
|
||||||
|
- name: cluster-ai-copilot-result-keyring
|
||||||
|
mountPath: /var/run/secrets/qinglong3/ai/copilot-result-keyring
|
||||||
|
readOnly: true
|
||||||
|
- name: cluster-ai-copilot-output-keyring
|
||||||
|
mountPath: /var/run/secrets/qinglong3/ai/copilot-output-keyring
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: cluster-ai-copilot-config
|
||||||
|
configMap:
|
||||||
|
name: ql3-cluster-ai-copilot
|
||||||
|
defaultMode: 288
|
||||||
|
items:
|
||||||
|
- key: config.json
|
||||||
|
path: config.json
|
||||||
|
- name: cluster-ai-copilot-invocation-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ql3-cluster-ai-copilot-invocation-keyring
|
||||||
|
defaultMode: 288
|
||||||
|
items:
|
||||||
|
- key: keyring.json
|
||||||
|
path: keyring.json
|
||||||
|
- name: cluster-ai-copilot-result-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ql3-cluster-ai-copilot-result-keyring
|
||||||
|
defaultMode: 288
|
||||||
|
items:
|
||||||
|
- key: keyring.json
|
||||||
|
path: keyring.json
|
||||||
|
- name: cluster-ai-copilot-output-keyring
|
||||||
|
secret:
|
||||||
|
secretName: ql3-cluster-ai-copilot-output-keyring
|
||||||
|
defaultMode: 288
|
||||||
|
items:
|
||||||
|
- key: keyring.json
|
||||||
|
path: keyring.json
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1alpha1
|
||||||
|
kind: Component
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- copilot-configmap.yaml
|
||||||
|
|
||||||
|
patches:
|
||||||
|
- path: deployment-patch.yaml
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# Example only. Copy into a private overlay, replace all example model policy
|
||||||
|
# values and provision the three keyring Secrets out of band.
|
||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
|
||||||
|
resources:
|
||||||
|
- ../../base
|
||||||
|
|
||||||
|
components:
|
||||||
|
- ../../components/cluster-ai
|
||||||
|
- ../../components/cluster-ai-copilot
|
||||||
|
|
||||||
|
images:
|
||||||
|
- name: qinglong3-cluster-control-ai
|
||||||
|
newName: registry.example.com/qinglong/qinglong3-cluster-control-ai
|
||||||
|
digest: sha256:0000000000000000000000000000000000000000000000000000000000000000
|
||||||
@@ -11,6 +11,7 @@
|
|||||||
|
|
||||||
最新增量证据(2026-08-15):
|
最新增量证据(2026-08-15):
|
||||||
|
|
||||||
|
- D-319/ADR-0411(已接受):Cluster Copilot failure diagnosis 已在既有 `ql3-cluster-control-ai` 进程内完成默认关闭的 production composition。Prompt 与 Copilot 共享同一个 PostgreSQL AI Pool、Model Gateway、Provider client、恢复扫描、quota/pricing ledger 与 `maxConcurrent` 预算;有界 successful-completion router 只向声明 exact invocation 的 durable sink 分发,不复制 Gateway、连接或隐藏队列。application service 只接受 source Run/request identity,从数据库当前 Run、latest Attempt、Project Tool snapshot、Policy 和 canonical read-only egress config 派生计划,先 admission,再以 durable plan、historical key 与确定性 nonce 修复 admission→Artifact crash window,随后复用 Trusted Tool、Worker Artifact range reader、Model Gateway 及独立的 invocation/result/model-output 三域 keyring。能力保持 caller-driven,不增加 timer、watcher、队列、HTTP/CLI/UI/MCP route 或 Kubernetes API 权限;Kubernetes 独立可选 component 仅投影 config 与三个 0440 keyring。最终 AI 238 pass/3 条件 skip、Cluster Control 240 pass/2 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail,四项架构审计、14 档 Local artifact 全部通过;workspace 仍为 18 package、无单文件/浅平 package,AI 187 个源码中 186 个、Cluster Control 59 个中 57 个位于嵌套领域目录。默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes,Edge/Standalone AI 为 3,064,454/3,064,544 bytes,证明 Cluster-only composition 未进入小设备闭包。PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`,报告 SHA-256 为 `981299b454dce5541e9596450b85816dc40559cba8dc42adf3d5fea571c3d3a6`;本 Gate 不改 migration/schema/role/SQL/HA 拓扑。下一 Gate 是 Tool failure、日志 missing/retired/pending、deadline/cancel 与 Model outcome-unknown 的 durable terminalization/recovery,完成前不开放产品入口。
|
||||||
- D-318/ADR-0410(已接受):Cluster Copilot diagnosis Model output 获得独立的只读 projected key authority。canonical `qinglong/copilot-failure-diagnosis-output-projected-keyring@v1` manifest 只允许一个 active key 与最多 16 个 historical 32-byte key;每次 `active()`/`resolve()` 都重新执行 Cluster 私有投影文件的 direct-root、根内 atomic symlink、single-link、mode、dev/inode/size/mtime 与双 realpath fence,不使用 cache、watcher、timer 或 Kubernetes API。该 authority 以结构兼容的本地窄端口位于既有 `cluster-control/copilot/failure-diagnosis/`,只通过 `failure-diagnosis-output-keyring` subpath 发布,避免 Cluster Control 默认源码反向依赖 AI;它不复用 Prompt output、Tool invocation/result 或 Provider credential key domain,也不提前声称 Copilot 产品入口已可达。定向回归 13/13、Cluster Control 239 pass/2 条件 skip、backend 1,207 pass/2 条件 skip、18-package clean build/test、四项架构审计与 14 档 Local artifact 全部通过;workspace 仍无单文件/浅平 package,Cluster Control 58 个源码中 56 个位于嵌套领域目录。默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes,Edge/Standalone AI 保持 3,061,009/3,061,099 bytes,证明 Cluster-only subpath 被裁掉。本 Gate 不改 migration/schema/role/SQL/连接/HA 拓扑,因此数据库基线继续引用 ADR-0409 的 PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`;下一 Gate 是默认关闭的完整 Cluster Copilot composition,随后补齐 Tool failure、日志 missing/retired、Model admission 前 deadline/cancel 与 outcome-unknown 的 durable terminalization/recovery。
|
- D-318/ADR-0410(已接受):Cluster Copilot diagnosis Model output 获得独立的只读 projected key authority。canonical `qinglong/copilot-failure-diagnosis-output-projected-keyring@v1` manifest 只允许一个 active key 与最多 16 个 historical 32-byte key;每次 `active()`/`resolve()` 都重新执行 Cluster 私有投影文件的 direct-root、根内 atomic symlink、single-link、mode、dev/inode/size/mtime 与双 realpath fence,不使用 cache、watcher、timer 或 Kubernetes API。该 authority 以结构兼容的本地窄端口位于既有 `cluster-control/copilot/failure-diagnosis/`,只通过 `failure-diagnosis-output-keyring` subpath 发布,避免 Cluster Control 默认源码反向依赖 AI;它不复用 Prompt output、Tool invocation/result 或 Provider credential key domain,也不提前声称 Copilot 产品入口已可达。定向回归 13/13、Cluster Control 239 pass/2 条件 skip、backend 1,207 pass/2 条件 skip、18-package clean build/test、四项架构审计与 14 档 Local artifact 全部通过;workspace 仍无单文件/浅平 package,Cluster Control 58 个源码中 56 个位于嵌套领域目录。默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes,Edge/Standalone AI 保持 3,061,009/3,061,099 bytes,证明 Cluster-only subpath 被裁掉。本 Gate 不改 migration/schema/role/SQL/连接/HA 拓扑,因此数据库基线继续引用 ADR-0409 的 PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`;下一 Gate 是默认关闭的完整 Cluster Copilot composition,随后补齐 Tool failure、日志 missing/retired、Model admission 前 deadline/cancel 与 outcome-unknown 的 durable terminalization/recovery。
|
||||||
- D-316/ADR-0408(已接受):Cluster Copilot 现在能够从 ADR-0407 的 durable admission 恢复 exact `qinglong.run.log.excerpt@1.0.0` authority,复用通用 Trusted Tool start barrier、加密 success/failure completion、result catalog/rekey 和内建日志 adapter;确定性 start/completion identity 让 response-loss replay 直接打开既有证据,不重复读取日志或执行 adapter。Cluster invocation Artifact 使用独立 projected keyring,提供 active+historical material,但每次读取均重新执行 canonical path/symlink/mode/inode/realpath fence,且不取得 PostgreSQL Tool result generation authority。只有 exact `succeeded` completion 才能通过 `pg-9019` 的 SERIALIZABLE 事务把 Model Step 从 `pending` 原子推进到 `ready`,同时写 RunEvent、StepRunMutation 和 append-only unlock receipt;`failed|timed_out` 不解锁。本 Gate 不执行模型、不终态化 diagnosis Run,下一 Gate 是 ADR-0405 builder + Model Gateway + Copilot encrypted model completion/terminalization。实现仍为 18 个 package,无单文件/浅平 package;AI 175 个源码中 174 个、Cluster Control 56 个中 54 个位于嵌套目录,不新增依赖、进程、连接、timer/watcher/cache 或默认 Edge 成本。18-package clean build/test 全绿,AI 229 pass/3 条件 skip、backend 1,207 pass/2 条件 skip,四项架构审计零 finding;14 档 Local artifact 全通过,默认 Edge/Standalone 为 2,589,890/2,589,968 bytes。PostgreSQL 18.4 arm64 HA 130/130、timeline `1→2`,首次执行只读两次日志、密文不含敏感 fixture,提升后 exact replay 零日志读取;报告 SHA-256 为 `d525a303696e178d777b021b376729bd2c5382fb5eb7bc98466a2b79d3940517`,独立审计与 Docker 清理通过。
|
- D-316/ADR-0408(已接受):Cluster Copilot 现在能够从 ADR-0407 的 durable admission 恢复 exact `qinglong.run.log.excerpt@1.0.0` authority,复用通用 Trusted Tool start barrier、加密 success/failure completion、result catalog/rekey 和内建日志 adapter;确定性 start/completion identity 让 response-loss replay 直接打开既有证据,不重复读取日志或执行 adapter。Cluster invocation Artifact 使用独立 projected keyring,提供 active+historical material,但每次读取均重新执行 canonical path/symlink/mode/inode/realpath fence,且不取得 PostgreSQL Tool result generation authority。只有 exact `succeeded` completion 才能通过 `pg-9019` 的 SERIALIZABLE 事务把 Model Step 从 `pending` 原子推进到 `ready`,同时写 RunEvent、StepRunMutation 和 append-only unlock receipt;`failed|timed_out` 不解锁。本 Gate 不执行模型、不终态化 diagnosis Run,下一 Gate 是 ADR-0405 builder + Model Gateway + Copilot encrypted model completion/terminalization。实现仍为 18 个 package,无单文件/浅平 package;AI 175 个源码中 174 个、Cluster Control 56 个中 54 个位于嵌套目录,不新增依赖、进程、连接、timer/watcher/cache 或默认 Edge 成本。18-package clean build/test 全绿,AI 229 pass/3 条件 skip、backend 1,207 pass/2 条件 skip,四项架构审计零 finding;14 档 Local artifact 全通过,默认 Edge/Standalone 为 2,589,890/2,589,968 bytes。PostgreSQL 18.4 arm64 HA 130/130、timeline `1→2`,首次执行只读两次日志、密文不含敏感 fixture,提升后 exact replay 零日志读取;报告 SHA-256 为 `d525a303696e178d777b021b376729bd2c5382fb5eb7bc98466a2b79d3940517`,独立审计与 Docker 清理通过。
|
||||||
- D-317/ADR-0409(已接受):已解锁的 Cluster failure-diagnosis Model Step 现在从 durable encrypted Tool completion 重开受信日志投影,经 ADR-0405 builder 和既有 Model Gateway 执行;只有显式安装的 Copilot success sink 可以接管成功返回。Copilot `GenerateResult` 使用独立 AES-256-GCM Artifact,绑定 plan、Tool completion、egress evidence 与 Model identity,公开 reference、ModelInvocation completion、RunEvent 和审计均不含明文。通用 `DurableModelInvocationCoordinator` 已改为领域无关的 `ModelInvocationAtomicSuccess<TReference>`,消除对 Plugin Prompt Artifact 的反向依赖;Plugin Prompt 通过 adapter 保持兼容。`pg-9020` 在一个 SERIALIZABLE 事务中原子提交 ciphertext、Model completion、StepRun/Event 与 usage/pricing/quota settlement,再由可重放 finalization 事务把 diagnosis Run 推进为 `succeeded|failed|timed_out`;两事务间崩溃只补 finalization,existing start/completion/finalization replay 均不重复调用 Provider。当前仍不自动终态化 Tool failure、日志 missing/retired、Model admission 前 deadline/cancel,也不把 `outcome_unknown` 冒充失败;Cluster 专用 output projected keyring 与产品 composition 完成前该入口保持不可达。实现仍在既有 18 个 package 的嵌套领域目录内,不新增进程、队列、timer/watcher/cache 或默认 Edge 成本。最终 AI 233 pass/3 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail,四项架构审计零 finding;workspace 无单文件或浅平 package,AI 183 个源码中 182 个位于嵌套目录。14 档 Local Profile artifact 全部通过,默认 Edge/Standalone 为 2,589,890/2,589,968 bytes,Edge/Standalone AI 为 3,061,009/3,061,099 bytes。PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`,报告 SHA-256 为 `8401634f30635b45bfb583b02e94ac41f023bf8a0bdbcfd9744ebf459ab0d8f8`,独立证据审计与 Docker 零残留。
|
- D-317/ADR-0409(已接受):已解锁的 Cluster failure-diagnosis Model Step 现在从 durable encrypted Tool completion 重开受信日志投影,经 ADR-0405 builder 和既有 Model Gateway 执行;只有显式安装的 Copilot success sink 可以接管成功返回。Copilot `GenerateResult` 使用独立 AES-256-GCM Artifact,绑定 plan、Tool completion、egress evidence 与 Model identity,公开 reference、ModelInvocation completion、RunEvent 和审计均不含明文。通用 `DurableModelInvocationCoordinator` 已改为领域无关的 `ModelInvocationAtomicSuccess<TReference>`,消除对 Plugin Prompt Artifact 的反向依赖;Plugin Prompt 通过 adapter 保持兼容。`pg-9020` 在一个 SERIALIZABLE 事务中原子提交 ciphertext、Model completion、StepRun/Event 与 usage/pricing/quota settlement,再由可重放 finalization 事务把 diagnosis Run 推进为 `succeeded|failed|timed_out`;两事务间崩溃只补 finalization,existing start/completion/finalization replay 均不重复调用 Provider。当前仍不自动终态化 Tool failure、日志 missing/retired、Model admission 前 deadline/cancel,也不把 `outcome_unknown` 冒充失败;Cluster 专用 output projected keyring 与产品 composition 完成前该入口保持不可达。实现仍在既有 18 个 package 的嵌套领域目录内,不新增进程、队列、timer/watcher/cache 或默认 Edge 成本。最终 AI 233 pass/3 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail,四项架构审计零 finding;workspace 无单文件或浅平 package,AI 183 个源码中 182 个位于嵌套目录。14 档 Local Profile artifact 全部通过,默认 Edge/Standalone 为 2,589,890/2,589,968 bytes,Edge/Standalone AI 为 3,061,009/3,061,099 bytes。PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`,报告 SHA-256 为 `8401634f30635b45bfb583b02e94ac41f023bf8a0bdbcfd9744ebf459ab0d8f8`,独立证据审计与 Docker 零残留。
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
# ADR-0411:默认关闭的 Cluster Copilot Failure Diagnosis Composition
|
||||||
|
|
||||||
|
- 状态:Accepted
|
||||||
|
- 日期:2026-08-15
|
||||||
|
- 关联 RFC:QL-RFC-0001 D-319、Phase 2
|
||||||
|
- 关联 ADR:ADR-0407、ADR-0408、ADR-0409、ADR-0410
|
||||||
|
|
||||||
|
## 问题
|
||||||
|
|
||||||
|
ADR-0407 至 ADR-0410 已分别建立 diagnosis admission、Trusted Tool execution、Model execution
|
||||||
|
和独立 output key authority,但它们仍只在测试与 HA ceremony 中手工组装。若直接再启动一套 Model
|
||||||
|
Gateway,会为每个 replica 复制 PostgreSQL Pool、Provider client、恢复扫描和并发预算;这对小型集群
|
||||||
|
不合理,也会让 Prompt 与 Copilot 绕过同一模型出口限额。若直接开放 HTTP route,则 Tool failure、日志
|
||||||
|
missing/retired、Model admission 前取消以及 outcome-unknown 尚未终态化,产品会暴露无法收敛的 Run。
|
||||||
|
|
||||||
|
## 决策
|
||||||
|
|
||||||
|
1. Copilot 只在既有显式 `ql3-cluster-control-ai` 进程中装配,并由独立
|
||||||
|
`QL3_CLUSTER_AI_COPILOT_ENABLED=true` 开关启用。默认 Cluster、Edge、Standalone 和仅 Prompt 的
|
||||||
|
Cluster AI 均不读取 Copilot 配置、keyring 或 Artifact authority。
|
||||||
|
2. Prompt 与 Copilot 共享一个 PostgreSQL runtime Pool、一个 Model Gateway、同一 Provider authority、
|
||||||
|
recovery pass、pricing/quota ledger 和 `maxConcurrent` 预算。新增有界 successful-completion router,
|
||||||
|
只把一次已开始的 invocation 交给声明该 invocation 的 durable sink;不得复制 Gateway 或隐藏队列。
|
||||||
|
3. Copilot Model execution 依赖 `generate()` 与 `supportsSuccessfulCompletionSink()` 的窄结构端口,不依赖
|
||||||
|
`BoundedModelGateway` 具体类,使 production Profile 的 drain/active-operation capability 可以直接注入,
|
||||||
|
同时保留 exact sink identity 检查。
|
||||||
|
4. Cluster Copilot application service 从当前 PostgreSQL Run、latest Attempt、Project Tool snapshot 和
|
||||||
|
Policy 派生计划;调用者不能提供 Attempt、Artifact、Tool binding、Policy fence、key ID、nonce、
|
||||||
|
`plannedAtMs` 或内部 Run/Step/ModelInvocation identity。模型与 egress policy 来自部署者的 canonical
|
||||||
|
read-only配置,而不是请求体。
|
||||||
|
5. diagnosis admission 先持久化 exact plan/Run/Step ledger,再物化可重建的加密 Tool invocation Artifact。
|
||||||
|
replay 使用 durable plan、historical invocation key 和域分离确定性 nonce 修复 admission→Artifact
|
||||||
|
crash window;只有 Artifact exact replay 成功后才允许 Tool execution。密钥副本使用后必须清零。
|
||||||
|
6. Tool execution 复用同一 PostgreSQL repository、current snapshot、Project Policy 与 Worker Artifact
|
||||||
|
range reader;Model execution复用同一 Gateway,并由 ADR-0410 output keyring 原子落盘 ciphertext。
|
||||||
|
invocation、Tool result 与 Model output keyring 继续是三个不可互换的域。
|
||||||
|
7. composition 能力在进程内保持 caller-driven、无 timer、watcher、队列或后台扫描。当前 Gate 不注册
|
||||||
|
HTTP/CLI/UI/MCP route;只有下一 Gate 完成所有非成功路径的 durable terminalization/recovery 后,
|
||||||
|
才能把该能力接入认证产品面。
|
||||||
|
8. Kubernetes 以独立可选 component 投影 canonical Copilot 配置和三个 read-only keyring;它必须与
|
||||||
|
`cluster-ai` 以及可读 Worker Artifact storage 一起使用,不新增 Pod、ServiceAccount 权限或 sidecar。
|
||||||
|
|
||||||
|
## 被否决方案
|
||||||
|
|
||||||
|
1. **为 Copilot 再启动 Gateway/Pool**:资源翻倍并拆散全局并发、quota 和 Provider 出口约束。
|
||||||
|
2. **让 Gateway 依赖具体业务 sink**:把通用模型边界反向耦合到 Prompt/Copilot 产品域。
|
||||||
|
3. **由请求提交完整执行计划**:把 source fence、Tool binding、egress policy 和内部 identity 交给不可信边界。
|
||||||
|
4. **先开放 route、以后补失败收敛**:会产生用户可见但永久 running/pending 的 diagnosis Run。
|
||||||
|
5. **把三个 keyring 合并**:轮换、退役与泄漏半径跨越 Tool 输入、Tool 输出和模型输出域。
|
||||||
|
6. **新增 workspace package**:composition 没有独立进程或依赖闭包,薄包会恶化已有 package 粒度。
|
||||||
|
|
||||||
|
## 验证标准
|
||||||
|
|
||||||
|
1. completion router 证明 Prompt/Copilot 精确分发、未知 invocation 不落盘、重复/扩展 sink 拒绝。
|
||||||
|
2. 默认关闭时不读取 Copilot config、keyring、Worker Artifact 或新增 PostgreSQL authority。
|
||||||
|
3. composition 测试覆盖 server-derived source/snapshot/Policy、admission→Artifact crash repair、Tool→Model
|
||||||
|
成功链、exact replay、key rotation、配置/authority 缺失和有序 drain。
|
||||||
|
4. AI、Cluster Control、18-package clean build/test、backend、四项架构审计和 14 档 Local artifact 全通过。
|
||||||
|
5. PostgreSQL 组合证据覆盖真实 repository 上的 admission、Tool completion、Model ciphertext、finalization
|
||||||
|
与 exact replay;本 Gate 不新增 migration、schema、role 或 SQL privilege。
|
||||||
|
|
||||||
|
## 当前验证
|
||||||
|
|
||||||
|
1. successful-completion router 与 Cluster Copilot application/composition 定向测试全部通过;AI 完整测试
|
||||||
|
238 pass/3 条件 skip,Cluster Control 完整测试 240 pass/2 条件 skip。
|
||||||
|
2. 18-package clean build/test 全部通过;backend 1,207 pass/2 条件 skip/0 fail。package boundary、
|
||||||
|
dependency、Edge import 与 Cluster deployment 四项审计均为 compatible、零 finding。
|
||||||
|
3. workspace 仍为 18 个 package,无 single-source 或 shallow-source package;AI 187 个源码中 186 个、
|
||||||
|
Cluster Control 59 个源码中 57 个位于嵌套领域目录。新增 composition 没有制造薄 package 或根层平铺。
|
||||||
|
4. 14 档 Local Profile artifact 全部通过。默认 Edge/Standalone 保持
|
||||||
|
2,589,890/2,589,968 bytes;Edge/Standalone AI 为 3,064,454/3,064,544 bytes,均未引入 Cluster-only
|
||||||
|
composition,且分别保有 1,604,414/1,604,336 与 2,178,426/2,178,336 bytes 体积余量。
|
||||||
|
5. PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`,报告 SHA-256 为
|
||||||
|
`981299b454dce5541e9596450b85816dc40559cba8dc42adf3d5fea571c3d3a6`。本 Gate 不新增
|
||||||
|
migration、schema、role、SQL privilege 或 HA 拓扑;真实 repository 的 admission、Tool completion、
|
||||||
|
Model ciphertext、finalization 与 exact replay 沿用同一受审 authority,Docker 门禁与清理均通过。
|
||||||
|
|
||||||
|
## 后续门禁
|
||||||
|
|
||||||
|
1. 为 Tool `failed|timed_out`、日志 `missing|retired|pending`、deadline/cancel 和 Model
|
||||||
|
`outcome_unknown` 建立有界 durable terminalization/recovery。
|
||||||
|
2. 完成后再增加经 authentication、Policy、audit、request identity 和 source fence 保护的 Cluster API,
|
||||||
|
随后复用该 capability 提供 CLI/UI/MCP,而不是建立旁路执行器。
|
||||||
@@ -414,6 +414,7 @@
|
|||||||
| [ADR-0408](./ADR-0408-cluster-copilot-failure-diagnosis-tool-execution.md) | Cluster Copilot Failure Diagnosis Tool Execution 与原子 Model 解锁 | Accepted |
|
| [ADR-0408](./ADR-0408-cluster-copilot-failure-diagnosis-tool-execution.md) | Cluster Copilot Failure Diagnosis Tool Execution 与原子 Model 解锁 | Accepted |
|
||||||
| [ADR-0409](./ADR-0409-cluster-copilot-failure-diagnosis-model-execution.md) | Cluster Copilot Failure Diagnosis Model Execution、密文输出与 Run 终态化 | Accepted |
|
| [ADR-0409](./ADR-0409-cluster-copilot-failure-diagnosis-model-execution.md) | Cluster Copilot Failure Diagnosis Model Execution、密文输出与 Run 终态化 | Accepted |
|
||||||
| [ADR-0410](./ADR-0410-cluster-copilot-failure-diagnosis-output-key-authority.md) | Cluster Copilot Failure Diagnosis Output Projected Key Authority | Accepted |
|
| [ADR-0410](./ADR-0410-cluster-copilot-failure-diagnosis-output-key-authority.md) | Cluster Copilot Failure Diagnosis Output Projected Key Authority | Accepted |
|
||||||
|
| [ADR-0411](./ADR-0411-default-off-cluster-copilot-composition.md) | 默认关闭的 Cluster Copilot Failure Diagnosis Composition | Accepted |
|
||||||
|
|
||||||
## 规则
|
## 规则
|
||||||
|
|
||||||
|
|||||||
@@ -55,6 +55,11 @@
|
|||||||
"require": "./dist/copilot/failure-diagnosis/modelExecution.js",
|
"require": "./dist/copilot/failure-diagnosis/modelExecution.js",
|
||||||
"default": "./dist/copilot/failure-diagnosis/modelExecution.js"
|
"default": "./dist/copilot/failure-diagnosis/modelExecution.js"
|
||||||
},
|
},
|
||||||
|
"./failure-diagnosis-application": {
|
||||||
|
"types": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.d.ts",
|
||||||
|
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js",
|
||||||
|
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js"
|
||||||
|
},
|
||||||
"./postgres-failure-diagnosis-model-execution-storage": {
|
"./postgres-failure-diagnosis-model-execution-storage": {
|
||||||
"types": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.d.ts",
|
"types": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.d.ts",
|
||||||
"require": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.js",
|
"require": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.js",
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
import type { SecurityPrincipal } from '@qinglong/runtime-core/security';
|
||||||
|
|
||||||
|
import type { CopilotFailureDiagnosisAdmissionReceipt } from '../admission/contracts';
|
||||||
|
import type { CopilotFailureDiagnosisModelExecutionResult } from '../model-execution/coordinator';
|
||||||
|
import type { CopilotFailureDiagnosisToolExecutionResult } from '../tool-execution/contracts';
|
||||||
|
|
||||||
|
export const MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS = 64;
|
||||||
|
|
||||||
|
export interface ExecuteCopilotFailureDiagnosisApplicationCommand {
|
||||||
|
readonly requestId: string;
|
||||||
|
readonly traceId: string;
|
||||||
|
readonly projectId: string;
|
||||||
|
readonly sourceRunId: string;
|
||||||
|
readonly principal: Readonly<SecurityPrincipal>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ExecuteCopilotFailureDiagnosisApplicationResult {
|
||||||
|
readonly admissionStatus: 'created' | 'existing';
|
||||||
|
readonly admission: Readonly<CopilotFailureDiagnosisAdmissionReceipt>;
|
||||||
|
readonly tool: Readonly<CopilotFailureDiagnosisToolExecutionResult>;
|
||||||
|
readonly model: Readonly<CopilotFailureDiagnosisModelExecutionResult> | null;
|
||||||
|
readonly terminalizationRequired: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class InvalidCopilotFailureDiagnosisApplicationError extends TypeError {
|
||||||
|
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_INVALID';
|
||||||
|
|
||||||
|
constructor(message: string) {
|
||||||
|
super(`Copilot failure diagnosis application is invalid: ${message}`);
|
||||||
|
this.name = 'InvalidCopilotFailureDiagnosisApplicationError';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class CopilotFailureDiagnosisApplicationConflictError extends Error {
|
||||||
|
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_CONFLICT';
|
||||||
|
|
||||||
|
constructor(message = 'the durable diagnosis request changed') {
|
||||||
|
super(`Copilot failure diagnosis application conflicts: ${message}`);
|
||||||
|
this.name = 'CopilotFailureDiagnosisApplicationConflictError';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class CopilotFailureDiagnosisApplicationUnavailableError extends Error {
|
||||||
|
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_UNAVAILABLE';
|
||||||
|
|
||||||
|
constructor(options?: ErrorOptions) {
|
||||||
|
super('Copilot failure diagnosis application is unavailable', options);
|
||||||
|
this.name = 'CopilotFailureDiagnosisApplicationUnavailableError';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class CopilotFailureDiagnosisApplicationBusyError extends Error {
|
||||||
|
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_BUSY';
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
super('Copilot failure diagnosis application request budget is exhausted');
|
||||||
|
this.name = 'CopilotFailureDiagnosisApplicationBusyError';
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,572 @@
|
|||||||
|
import { Buffer } from 'node:buffer';
|
||||||
|
import { createHash, createHmac } from 'node:crypto';
|
||||||
|
|
||||||
|
import {
|
||||||
|
BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||||
|
createBuiltInRunLogExcerptToolHandlerBinding,
|
||||||
|
} from '@qinglong/runtime-core/builtin-run-log-excerpt-tool';
|
||||||
|
import type { RunRepositoryReader } from '@qinglong/runtime-core/run-repository';
|
||||||
|
import type { ToolPolicyAuthorizer } from '@qinglong/runtime-core/tool-registry';
|
||||||
|
import {
|
||||||
|
prepareToolInvocation,
|
||||||
|
} from '@qinglong/runtime-core/tool-registry';
|
||||||
|
import type { ProjectToolDefinitionSnapshotRepository } from '@qinglong/runtime-core/project-tool-definition-snapshot';
|
||||||
|
import { projectToolDefinitionRegistry } from '@qinglong/runtime-core/project-tool-definition-snapshot';
|
||||||
|
import {
|
||||||
|
createToolInvocationInputArtifact,
|
||||||
|
createToolInvocationPreviewArtifact,
|
||||||
|
type ToolInvocationArtifactKeyProvider,
|
||||||
|
type ToolInvocationArtifactRepository,
|
||||||
|
type ToolInvocationPreviewDocument,
|
||||||
|
} from '@qinglong/runtime-core/tool-invocation-artifact';
|
||||||
|
import {
|
||||||
|
TrustedToolHandlerBindingRegistry,
|
||||||
|
createTrustedToolInvocationPlan,
|
||||||
|
} from '@qinglong/runtime-core/trusted-tool-invocation';
|
||||||
|
|
||||||
|
import { MAX_MODEL_INVOCATION_MS } from '../../../model-gateway/model';
|
||||||
|
import {
|
||||||
|
prepareCopilotFailureDiagnosisExecution,
|
||||||
|
} from '../admission/plan';
|
||||||
|
import type {
|
||||||
|
CopilotFailureDiagnosisAdmissionRepository,
|
||||||
|
CopilotFailureDiagnosisExecutionPlan,
|
||||||
|
PrepareCopilotFailureDiagnosisModelIntent,
|
||||||
|
} from '../admission/contracts';
|
||||||
|
import {
|
||||||
|
executeCopilotFailureDiagnosisTool,
|
||||||
|
type CopilotFailureDiagnosisToolExecutionDependencies,
|
||||||
|
} from '../tool-execution/coordinator';
|
||||||
|
import {
|
||||||
|
executeCopilotFailureDiagnosisModel,
|
||||||
|
type CopilotFailureDiagnosisModelExecutionDependencies,
|
||||||
|
} from '../model-execution/coordinator';
|
||||||
|
import {
|
||||||
|
CopilotFailureDiagnosisApplicationBusyError,
|
||||||
|
CopilotFailureDiagnosisApplicationConflictError,
|
||||||
|
CopilotFailureDiagnosisApplicationUnavailableError,
|
||||||
|
InvalidCopilotFailureDiagnosisApplicationError,
|
||||||
|
MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS,
|
||||||
|
type ExecuteCopilotFailureDiagnosisApplicationCommand,
|
||||||
|
type ExecuteCopilotFailureDiagnosisApplicationResult,
|
||||||
|
} from './contracts';
|
||||||
|
|
||||||
|
const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||||
|
const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
|
||||||
|
const NONCE_DOMAIN = Buffer.from(
|
||||||
|
'qinglong/copilot-failure-diagnosis-tool-invocation-nonce@v1\0',
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
const IDENTITY_DOMAIN = Buffer.from(
|
||||||
|
'qinglong/copilot-failure-diagnosis-application-identity@v1\0',
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
|
||||||
|
export interface CopilotFailureDiagnosisApplicationDependencies {
|
||||||
|
readonly admissions: CopilotFailureDiagnosisAdmissionRepository;
|
||||||
|
readonly snapshots: Pick<ProjectToolDefinitionSnapshotRepository, 'findCurrent'>;
|
||||||
|
readonly runs: Pick<
|
||||||
|
RunRepositoryReader,
|
||||||
|
'findRunById' | 'findLatestAttemptByRunId'
|
||||||
|
>;
|
||||||
|
readonly artifacts: ToolInvocationArtifactRepository;
|
||||||
|
readonly invocationKeys: Pick<
|
||||||
|
ToolInvocationArtifactKeyProvider,
|
||||||
|
'active' | 'resolve'
|
||||||
|
>;
|
||||||
|
readonly authorizer: ToolPolicyAuthorizer;
|
||||||
|
readonly tool: CopilotFailureDiagnosisToolExecutionDependencies;
|
||||||
|
readonly model: CopilotFailureDiagnosisModelExecutionDependencies;
|
||||||
|
readonly executeTool: typeof executeCopilotFailureDiagnosisTool;
|
||||||
|
readonly executeModel: typeof executeCopilotFailureDiagnosisModel;
|
||||||
|
readonly modelIntent: Readonly<PrepareCopilotFailureDiagnosisModelIntent>;
|
||||||
|
readonly executionTimeoutMs: number;
|
||||||
|
readonly now?: () => number;
|
||||||
|
readonly nonceFactory?: (input: Readonly<{
|
||||||
|
key: Uint8Array;
|
||||||
|
keyId: string;
|
||||||
|
requestId: string;
|
||||||
|
projectId: string;
|
||||||
|
sourceRunId: string;
|
||||||
|
invocationActionDigest: string;
|
||||||
|
}>) => Uint8Array;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ActiveRequest {
|
||||||
|
readonly digest: string;
|
||||||
|
readonly promise: Promise<
|
||||||
|
Readonly<ExecuteCopilotFailureDiagnosisApplicationResult>
|
||||||
|
>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function invalid(message: string): never {
|
||||||
|
throw new InvalidCopilotFailureDiagnosisApplicationError(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
function unavailable(cause?: unknown): never {
|
||||||
|
throw new CopilotFailureDiagnosisApplicationUnavailableError({
|
||||||
|
cause: cause instanceof Error ? cause : undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function digest(value: unknown): string {
|
||||||
|
return createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
function identity(prefix: string, requestId: string): string {
|
||||||
|
return `${prefix}:${createHash('sha256')
|
||||||
|
.update(IDENTITY_DOMAIN)
|
||||||
|
.update(prefix)
|
||||||
|
.update('\0')
|
||||||
|
.update(requestId)
|
||||||
|
.digest('hex')
|
||||||
|
.slice(0, 32)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function preview(
|
||||||
|
sourceRunId: string,
|
||||||
|
sourceAttemptId: string,
|
||||||
|
): Readonly<ToolInvocationPreviewDocument> {
|
||||||
|
return Object.freeze({
|
||||||
|
title: 'Diagnose failed Run',
|
||||||
|
summary: 'Read one bounded, redacted and untrusted execution log excerpt',
|
||||||
|
fields: Object.freeze([
|
||||||
|
Object.freeze({
|
||||||
|
kind: 'identifier' as const,
|
||||||
|
label: 'Run',
|
||||||
|
value: sourceRunId,
|
||||||
|
}),
|
||||||
|
Object.freeze({
|
||||||
|
kind: 'identifier' as const,
|
||||||
|
label: 'Attempt',
|
||||||
|
value: sourceAttemptId,
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
warnings: Object.freeze(['potentially_sensitive_output']),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultNonce(input: Readonly<{
|
||||||
|
key: Uint8Array;
|
||||||
|
keyId: string;
|
||||||
|
requestId: string;
|
||||||
|
projectId: string;
|
||||||
|
sourceRunId: string;
|
||||||
|
invocationActionDigest: string;
|
||||||
|
}>): Uint8Array {
|
||||||
|
const derived = createHmac('sha256', Buffer.from(input.key))
|
||||||
|
.update(NONCE_DOMAIN)
|
||||||
|
.update(
|
||||||
|
JSON.stringify({
|
||||||
|
keyId: input.keyId,
|
||||||
|
requestId: input.requestId,
|
||||||
|
projectId: input.projectId,
|
||||||
|
sourceRunId: input.sourceRunId,
|
||||||
|
invocationActionDigest: input.invocationActionDigest,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.digest();
|
||||||
|
try {
|
||||||
|
return Buffer.from(derived.subarray(0, 12));
|
||||||
|
} finally {
|
||||||
|
derived.fill(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeCommand(
|
||||||
|
value: ExecuteCopilotFailureDiagnosisApplicationCommand,
|
||||||
|
): Readonly<ExecuteCopilotFailureDiagnosisApplicationCommand> {
|
||||||
|
if (
|
||||||
|
!value ||
|
||||||
|
typeof value !== 'object' ||
|
||||||
|
Array.isArray(value) ||
|
||||||
|
Object.keys(value).sort().join('\0') !==
|
||||||
|
['principal', 'projectId', 'requestId', 'sourceRunId', 'traceId'].join(
|
||||||
|
'\0',
|
||||||
|
) ||
|
||||||
|
!ID_PATTERN.test(value.requestId) ||
|
||||||
|
!ID_PATTERN.test(value.traceId) ||
|
||||||
|
!ID_PATTERN.test(value.projectId) ||
|
||||||
|
!RUN_ID_PATTERN.test(value.sourceRunId) ||
|
||||||
|
!value.principal ||
|
||||||
|
typeof value.principal !== 'object' ||
|
||||||
|
Array.isArray(value.principal)
|
||||||
|
) {
|
||||||
|
return invalid('command is invalid');
|
||||||
|
}
|
||||||
|
return Object.freeze({
|
||||||
|
requestId: value.requestId,
|
||||||
|
traceId: value.traceId,
|
||||||
|
projectId: value.projectId,
|
||||||
|
sourceRunId: value.sourceRunId,
|
||||||
|
principal: value.principal,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertDependencies(
|
||||||
|
value: CopilotFailureDiagnosisApplicationDependencies,
|
||||||
|
): void {
|
||||||
|
if (
|
||||||
|
!value ||
|
||||||
|
typeof value !== 'object' ||
|
||||||
|
Array.isArray(value) ||
|
||||||
|
typeof value.admissions?.findByRequestId !== 'function' ||
|
||||||
|
typeof value.admissions?.findPlanByRequestId !== 'function' ||
|
||||||
|
typeof value.admissions?.admit !== 'function' ||
|
||||||
|
typeof value.snapshots?.findCurrent !== 'function' ||
|
||||||
|
typeof value.runs?.findRunById !== 'function' ||
|
||||||
|
typeof value.runs?.findLatestAttemptByRunId !== 'function' ||
|
||||||
|
typeof value.artifacts?.put !== 'function' ||
|
||||||
|
typeof value.invocationKeys?.active !== 'function' ||
|
||||||
|
typeof value.invocationKeys?.resolve !== 'function' ||
|
||||||
|
typeof value.authorizer?.authorize !== 'function' ||
|
||||||
|
typeof value.executeTool !== 'function' ||
|
||||||
|
typeof value.executeModel !== 'function' ||
|
||||||
|
!value.tool ||
|
||||||
|
!value.model ||
|
||||||
|
!value.modelIntent ||
|
||||||
|
typeof value.modelIntent !== 'object' ||
|
||||||
|
!Number.isSafeInteger(value.executionTimeoutMs) ||
|
||||||
|
value.executionTimeoutMs < 1 ||
|
||||||
|
value.executionTimeoutMs > MAX_MODEL_INVOCATION_MS ||
|
||||||
|
(value.now !== undefined && typeof value.now !== 'function') ||
|
||||||
|
(value.nonceFactory !== undefined &&
|
||||||
|
typeof value.nonceFactory !== 'function')
|
||||||
|
) {
|
||||||
|
return invalid('dependencies are invalid');
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
value.tool.admissions !== value.admissions ||
|
||||||
|
value.tool.snapshots !== value.snapshots ||
|
||||||
|
value.tool.runs !== value.runs ||
|
||||||
|
value.tool.artifacts !== value.artifacts ||
|
||||||
|
value.tool.invocationKeys !== value.invocationKeys ||
|
||||||
|
value.model.admissions !== value.admissions ||
|
||||||
|
value.model.unlocks !== value.tool.unlocks
|
||||||
|
) {
|
||||||
|
return invalid('dependency authorities are not shared');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameSubject(
|
||||||
|
left: Readonly<{ type: string; id: string }>,
|
||||||
|
right: Readonly<{ type: string; id: string }>,
|
||||||
|
): boolean {
|
||||||
|
return left.type === right.type && left.id === right.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameModelIntent(
|
||||||
|
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
|
||||||
|
configured: Readonly<PrepareCopilotFailureDiagnosisModelIntent>,
|
||||||
|
): boolean {
|
||||||
|
return (
|
||||||
|
plan.model.provider === configured.provider &&
|
||||||
|
plan.model.model === configured.model &&
|
||||||
|
plan.model.modelBoundary === configured.modelBoundary &&
|
||||||
|
plan.model.responseLanguage === configured.responseLanguage &&
|
||||||
|
plan.model.maxOutputTokens === configured.maxOutputTokens &&
|
||||||
|
JSON.stringify(plan.model.egressPolicy) ===
|
||||||
|
JSON.stringify(configured.egressPolicy)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function nonceInput(
|
||||||
|
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
|
||||||
|
key: Uint8Array,
|
||||||
|
) {
|
||||||
|
return Object.freeze({
|
||||||
|
key,
|
||||||
|
keyId: plan.tool.invocationArtifact.keyId,
|
||||||
|
requestId: plan.requestId,
|
||||||
|
projectId: plan.projectId,
|
||||||
|
sourceRunId: plan.source.runId,
|
||||||
|
invocationActionDigest: plan.tool.invocationActionDigest,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export class CopilotFailureDiagnosisApplicationService {
|
||||||
|
readonly #dependencies: CopilotFailureDiagnosisApplicationDependencies;
|
||||||
|
readonly #active = new Map<string, ActiveRequest>();
|
||||||
|
|
||||||
|
constructor(dependencies: CopilotFailureDiagnosisApplicationDependencies) {
|
||||||
|
assertDependencies(dependencies);
|
||||||
|
this.#dependencies = dependencies;
|
||||||
|
}
|
||||||
|
|
||||||
|
execute(
|
||||||
|
commandValue: ExecuteCopilotFailureDiagnosisApplicationCommand,
|
||||||
|
): Promise<Readonly<ExecuteCopilotFailureDiagnosisApplicationResult>> {
|
||||||
|
const command = normalizeCommand(commandValue);
|
||||||
|
const commandDigest = digest(command);
|
||||||
|
const active = this.#active.get(command.requestId);
|
||||||
|
if (active) {
|
||||||
|
if (active.digest !== commandDigest) {
|
||||||
|
throw new CopilotFailureDiagnosisApplicationConflictError();
|
||||||
|
}
|
||||||
|
return active.promise;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
this.#active.size >=
|
||||||
|
MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS
|
||||||
|
) {
|
||||||
|
throw new CopilotFailureDiagnosisApplicationBusyError();
|
||||||
|
}
|
||||||
|
const promise = this.#execute(command).finally(() => {
|
||||||
|
this.#active.delete(command.requestId);
|
||||||
|
});
|
||||||
|
this.#active.set(command.requestId, { digest: commandDigest, promise });
|
||||||
|
return promise;
|
||||||
|
}
|
||||||
|
|
||||||
|
async #execute(
|
||||||
|
command: Readonly<ExecuteCopilotFailureDiagnosisApplicationCommand>,
|
||||||
|
): Promise<Readonly<ExecuteCopilotFailureDiagnosisApplicationResult>> {
|
||||||
|
const existing = await this.#dependencies.admissions.findPlanByRequestId(
|
||||||
|
command.requestId,
|
||||||
|
);
|
||||||
|
let plan: Readonly<CopilotFailureDiagnosisExecutionPlan>;
|
||||||
|
let admissionStatus: 'created' | 'existing';
|
||||||
|
let admission;
|
||||||
|
if (existing) {
|
||||||
|
if (
|
||||||
|
existing.projectId !== command.projectId ||
|
||||||
|
existing.source.runId !== command.sourceRunId ||
|
||||||
|
existing.traceId !== command.traceId ||
|
||||||
|
!sameSubject(existing.requestedBySubject, command.principal.subject) ||
|
||||||
|
!sameModelIntent(existing, this.#dependencies.modelIntent)
|
||||||
|
) {
|
||||||
|
throw new CopilotFailureDiagnosisApplicationConflictError();
|
||||||
|
}
|
||||||
|
plan = existing;
|
||||||
|
const admitted = await this.#dependencies.admissions.admit(plan);
|
||||||
|
admissionStatus = admitted.status;
|
||||||
|
admission = admitted.receipt;
|
||||||
|
await this.#materializeArtifacts(plan);
|
||||||
|
} else {
|
||||||
|
const prepared = await this.#prepare(command);
|
||||||
|
plan = prepared.plan;
|
||||||
|
const admitted = await this.#dependencies.admissions.admit(plan);
|
||||||
|
admissionStatus = admitted.status;
|
||||||
|
admission = admitted.receipt;
|
||||||
|
try {
|
||||||
|
await this.#dependencies.artifacts.put(
|
||||||
|
prepared.inputArtifact,
|
||||||
|
prepared.previewArtifact,
|
||||||
|
);
|
||||||
|
} catch (cause) {
|
||||||
|
throw new CopilotFailureDiagnosisApplicationUnavailableError({
|
||||||
|
cause: cause instanceof Error ? cause : undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const tool = await this.#dependencies.executeTool(
|
||||||
|
{
|
||||||
|
requestId: plan.requestId,
|
||||||
|
principal: command.principal,
|
||||||
|
authorizer: this.#dependencies.authorizer,
|
||||||
|
},
|
||||||
|
this.#dependencies.tool,
|
||||||
|
);
|
||||||
|
if (tool.outcome !== 'succeeded') {
|
||||||
|
return Object.freeze({
|
||||||
|
admissionStatus,
|
||||||
|
admission,
|
||||||
|
tool,
|
||||||
|
model: null,
|
||||||
|
terminalizationRequired: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const model = await this.#dependencies.executeModel(
|
||||||
|
plan.requestId,
|
||||||
|
this.#dependencies.model,
|
||||||
|
);
|
||||||
|
return Object.freeze({
|
||||||
|
admissionStatus,
|
||||||
|
admission,
|
||||||
|
tool,
|
||||||
|
model,
|
||||||
|
terminalizationRequired: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async #prepare(
|
||||||
|
command: Readonly<ExecuteCopilotFailureDiagnosisApplicationCommand>,
|
||||||
|
) {
|
||||||
|
const now = this.#clock();
|
||||||
|
let run;
|
||||||
|
let attempt;
|
||||||
|
let snapshotRecord;
|
||||||
|
try {
|
||||||
|
[run, attempt, snapshotRecord] = await Promise.all([
|
||||||
|
this.#dependencies.runs.findRunById(command.sourceRunId),
|
||||||
|
this.#dependencies.runs.findLatestAttemptByRunId(command.sourceRunId),
|
||||||
|
this.#dependencies.snapshots.findCurrent(command.projectId),
|
||||||
|
]);
|
||||||
|
} catch (cause) {
|
||||||
|
return unavailable(cause);
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
!run ||
|
||||||
|
!attempt ||
|
||||||
|
!snapshotRecord ||
|
||||||
|
run.projectId !== command.projectId ||
|
||||||
|
attempt.runId !== run.id ||
|
||||||
|
!['failed', 'timed_out'].includes(run.status) ||
|
||||||
|
!['failed', 'timed_out', 'lost'].includes(attempt.status) ||
|
||||||
|
(run.status === 'failed' &&
|
||||||
|
!['failed', 'lost'].includes(attempt.status)) ||
|
||||||
|
(run.status === 'timed_out' && attempt.status !== 'timed_out') ||
|
||||||
|
!Number.isSafeInteger(attempt.finishedAtMs) ||
|
||||||
|
typeof attempt.logArtifactId !== 'string'
|
||||||
|
) {
|
||||||
|
throw new CopilotFailureDiagnosisApplicationConflictError(
|
||||||
|
'source Run is not an exact diagnosable terminal fence',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const snapshot = snapshotRecord.snapshot;
|
||||||
|
const binding = createBuiltInRunLogExcerptToolHandlerBinding(snapshot, [
|
||||||
|
'cluster-control',
|
||||||
|
]);
|
||||||
|
const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]);
|
||||||
|
const invocation = await prepareToolInvocation(
|
||||||
|
projectToolDefinitionRegistry(snapshot),
|
||||||
|
{
|
||||||
|
projectId: command.projectId,
|
||||||
|
principal: command.principal,
|
||||||
|
nowMs: now,
|
||||||
|
tool: BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||||
|
input: { runId: run.id, attemptId: attempt.id },
|
||||||
|
},
|
||||||
|
this.#dependencies.authorizer,
|
||||||
|
);
|
||||||
|
if (invocation.status !== 'ready') {
|
||||||
|
throw new CopilotFailureDiagnosisApplicationConflictError(
|
||||||
|
'Tool admission is not ready',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const key = await this.#dependencies.invocationKeys.active();
|
||||||
|
try {
|
||||||
|
const baseIdentity = identity('cda', command.requestId);
|
||||||
|
const nonce = (
|
||||||
|
this.#dependencies.nonceFactory ?? defaultNonce
|
||||||
|
)({
|
||||||
|
key: key.key,
|
||||||
|
keyId: key.keyId,
|
||||||
|
requestId: command.requestId,
|
||||||
|
projectId: command.projectId,
|
||||||
|
sourceRunId: run.id,
|
||||||
|
invocationActionDigest: invocation.actionDigest,
|
||||||
|
});
|
||||||
|
const tool = createTrustedToolInvocationPlan(bindings, invocation, {
|
||||||
|
actionRef: baseIdentity,
|
||||||
|
profile: 'cluster-control',
|
||||||
|
preview: preview(run.id, attempt.id),
|
||||||
|
inputArtifactId: identity('cdia', command.requestId),
|
||||||
|
previewArtifactId: identity('cdpa', command.requestId),
|
||||||
|
artifactKeyId: key.keyId,
|
||||||
|
artifactKey: key.key,
|
||||||
|
artifactNonce: nonce,
|
||||||
|
sealedAtMs: now,
|
||||||
|
});
|
||||||
|
nonce.fill(0);
|
||||||
|
const plan = prepareCopilotFailureDiagnosisExecution({
|
||||||
|
requestId: command.requestId,
|
||||||
|
traceId: command.traceId,
|
||||||
|
source: {
|
||||||
|
runId: run.id,
|
||||||
|
runVersion: run.version,
|
||||||
|
runStatus: run.status as 'failed' | 'timed_out',
|
||||||
|
attemptId: attempt.id,
|
||||||
|
attemptStatus: attempt.status as 'failed' | 'timed_out' | 'lost',
|
||||||
|
attemptFinishedAtMs: attempt.finishedAtMs!,
|
||||||
|
logArtifactId: attempt.logArtifactId,
|
||||||
|
},
|
||||||
|
toolPlan: tool.plan,
|
||||||
|
bindings,
|
||||||
|
model: this.#dependencies.modelIntent,
|
||||||
|
deadlineAtMs: now + this.#dependencies.executionTimeoutMs,
|
||||||
|
plannedAtMs: now,
|
||||||
|
});
|
||||||
|
return Object.freeze({
|
||||||
|
plan,
|
||||||
|
inputArtifact: tool.inputArtifact,
|
||||||
|
previewArtifact: tool.previewArtifact,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
key.key.fill(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async #materializeArtifacts(
|
||||||
|
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
|
||||||
|
): Promise<void> {
|
||||||
|
const material = await this.#dependencies.invocationKeys.resolve(
|
||||||
|
plan.tool.invocationArtifact.keyId,
|
||||||
|
);
|
||||||
|
if (!material || material.keyId !== plan.tool.invocationArtifact.keyId) {
|
||||||
|
return unavailable();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const nonce = (
|
||||||
|
this.#dependencies.nonceFactory ?? defaultNonce
|
||||||
|
)(nonceInput(plan, material.key));
|
||||||
|
const inputArtifact = createToolInvocationInputArtifact(
|
||||||
|
{
|
||||||
|
artifactId: plan.tool.invocationArtifact.artifactId,
|
||||||
|
projectId: plan.projectId,
|
||||||
|
actionRef: plan.tool.actionRef,
|
||||||
|
requestedBy: plan.requestedBySubject,
|
||||||
|
tool: BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||||
|
input: {
|
||||||
|
attemptId: plan.source.attemptId,
|
||||||
|
runId: plan.source.runId,
|
||||||
|
},
|
||||||
|
inputDigest: plan.tool.invocationArtifact.inputDigest,
|
||||||
|
invocationActionDigest: plan.tool.invocationActionDigest,
|
||||||
|
keyId: material.keyId,
|
||||||
|
key: material.key,
|
||||||
|
sealedAtMs: plan.tool.sealedAtMs,
|
||||||
|
},
|
||||||
|
() => nonce,
|
||||||
|
);
|
||||||
|
nonce.fill(0);
|
||||||
|
const previewArtifact = createToolInvocationPreviewArtifact({
|
||||||
|
artifactId: plan.tool.previewArtifact.artifactId,
|
||||||
|
projectId: plan.projectId,
|
||||||
|
actionRef: plan.tool.actionRef,
|
||||||
|
actionDigest: plan.tool.actionDigest,
|
||||||
|
redactionContractDigest:
|
||||||
|
plan.tool.previewArtifact.redactionContractDigest,
|
||||||
|
preview: preview(plan.source.runId, plan.source.attemptId),
|
||||||
|
sealedAtMs: plan.tool.sealedAtMs,
|
||||||
|
});
|
||||||
|
if (
|
||||||
|
inputArtifact.artifactDigest !==
|
||||||
|
plan.tool.invocationArtifact.artifactDigest ||
|
||||||
|
previewArtifact.artifactDigest !==
|
||||||
|
plan.tool.previewArtifact.artifactDigest
|
||||||
|
) {
|
||||||
|
throw new CopilotFailureDiagnosisApplicationConflictError(
|
||||||
|
'durable Tool Artifact references cannot be reconstructed',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await this.#dependencies.artifacts.put(inputArtifact, previewArtifact);
|
||||||
|
} finally {
|
||||||
|
material.key.fill(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#clock(): number {
|
||||||
|
let value: number;
|
||||||
|
try {
|
||||||
|
value = (this.#dependencies.now ?? Date.now)();
|
||||||
|
} catch (cause) {
|
||||||
|
return unavailable(cause);
|
||||||
|
}
|
||||||
|
if (!Number.isSafeInteger(value) || value < 0) return invalid('clock');
|
||||||
|
if (value + this.#dependencies.executionTimeoutMs > Number.MAX_SAFE_INTEGER) {
|
||||||
|
return invalid('deadline overflows');
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
export * from './application/contracts';
|
||||||
|
export * from './application/service';
|
||||||
@@ -5,7 +5,10 @@ import {
|
|||||||
type TrustedToolSuccessCompletionReadDependencies,
|
type TrustedToolSuccessCompletionReadDependencies,
|
||||||
} from '@qinglong/runtime-core/trusted-tool-completion';
|
} from '@qinglong/runtime-core/trusted-tool-completion';
|
||||||
|
|
||||||
import { BoundedModelGateway } from '../../../model-gateway/gateway';
|
import type {
|
||||||
|
BoundedModelGateway,
|
||||||
|
ModelInvocationSuccessfulCompletionSink,
|
||||||
|
} from '../../../model-gateway/gateway';
|
||||||
import type { ModelInvocationRepository } from '../../../model-invocation/modelInvocation';
|
import type { ModelInvocationRepository } from '../../../model-invocation/modelInvocation';
|
||||||
import type { CopilotFailureDiagnosisToolExecutionAdmissionReader } from '../tool-execution/contracts';
|
import type { CopilotFailureDiagnosisToolExecutionAdmissionReader } from '../tool-execution/contracts';
|
||||||
import type { CopilotFailureDiagnosisToolUnlockRepository } from '../tool-execution/contracts';
|
import type { CopilotFailureDiagnosisToolUnlockRepository } from '../tool-execution/contracts';
|
||||||
@@ -36,20 +39,26 @@ export interface CopilotFailureDiagnosisModelExecutionDependencies {
|
|||||||
CopilotFailureDiagnosisOutputCompletionRepository,
|
CopilotFailureDiagnosisOutputCompletionRepository,
|
||||||
'findCopilotFailureDiagnosisOutput'
|
'findCopilotFailureDiagnosisOutput'
|
||||||
>;
|
>;
|
||||||
readonly gateway: BoundedModelGateway;
|
readonly gateway: Pick<
|
||||||
|
BoundedModelGateway,
|
||||||
|
'generate' | 'supportsSuccessfulCompletionSink'
|
||||||
|
>;
|
||||||
readonly successfulCompletion: CopilotFailureDiagnosisModelCompletionCoordinator;
|
readonly successfulCompletion: CopilotFailureDiagnosisModelCompletionCoordinator;
|
||||||
readonly finalizations: CopilotFailureDiagnosisFinalizationRepository;
|
readonly finalizations: CopilotFailureDiagnosisFinalizationRepository;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CopilotFailureDiagnosisToolResultReader {
|
export interface CopilotFailureDiagnosisToolResultReader {
|
||||||
open(startId: string): Promise<Readonly<TrustedToolSuccessCompletionResult>>;
|
open(
|
||||||
|
requestId: string,
|
||||||
|
startId: string,
|
||||||
|
): Promise<Readonly<TrustedToolSuccessCompletionResult>>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createCopilotFailureDiagnosisToolResultReader(
|
export function createCopilotFailureDiagnosisToolResultReader(
|
||||||
dependencies: TrustedToolSuccessCompletionReadDependencies,
|
dependencies: TrustedToolSuccessCompletionReadDependencies,
|
||||||
): CopilotFailureDiagnosisToolResultReader {
|
): CopilotFailureDiagnosisToolResultReader {
|
||||||
return Object.freeze({
|
return Object.freeze({
|
||||||
open: (startId: string) =>
|
open: (_requestId: string, startId: string) =>
|
||||||
openTrustedToolSuccessCompletion(startId, dependencies),
|
openTrustedToolSuccessCompletion(startId, dependencies),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -97,7 +106,8 @@ function assertDependencies(
|
|||||||
typeof value.modelInvocations?.findStart !== 'function' ||
|
typeof value.modelInvocations?.findStart !== 'function' ||
|
||||||
typeof value.modelInvocations?.findCompletion !== 'function' ||
|
typeof value.modelInvocations?.findCompletion !== 'function' ||
|
||||||
typeof value.outputs?.findCopilotFailureDiagnosisOutput !== 'function' ||
|
typeof value.outputs?.findCopilotFailureDiagnosisOutput !== 'function' ||
|
||||||
!(value.gateway instanceof BoundedModelGateway) ||
|
typeof value.gateway?.generate !== 'function' ||
|
||||||
|
typeof value.gateway?.supportsSuccessfulCompletionSink !== 'function' ||
|
||||||
typeof value.successfulCompletion?.begin !== 'function' ||
|
typeof value.successfulCompletion?.begin !== 'function' ||
|
||||||
typeof value.successfulCompletion?.reference !== 'function' ||
|
typeof value.successfulCompletion?.reference !== 'function' ||
|
||||||
typeof value.successfulCompletion?.end !== 'function' ||
|
typeof value.successfulCompletion?.end !== 'function' ||
|
||||||
@@ -215,7 +225,7 @@ export async function executeCopilotFailureDiagnosisModel(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const tool = await dependencies.toolResults.open(unlock.startId);
|
const tool = await dependencies.toolResults.open(requestId, unlock.startId);
|
||||||
if (
|
if (
|
||||||
tool.completion.completionDigest !== unlock.toolCompletionDigest ||
|
tool.completion.completionDigest !== unlock.toolCompletionDigest ||
|
||||||
tool.completion.runId !== plan.runId ||
|
tool.completion.runId !== plan.runId ||
|
||||||
@@ -241,7 +251,7 @@ export async function executeCopilotFailureDiagnosisModel(
|
|||||||
});
|
});
|
||||||
if (
|
if (
|
||||||
!dependencies.gateway.supportsSuccessfulCompletionSink(
|
!dependencies.gateway.supportsSuccessfulCompletionSink(
|
||||||
dependencies.successfulCompletion,
|
dependencies.successfulCompletion as ModelInvocationSuccessfulCompletionSink,
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
throw new InvalidCopilotFailureDiagnosisModelExecutionError(
|
throw new InvalidCopilotFailureDiagnosisModelExecutionError(
|
||||||
|
|||||||
@@ -39,6 +39,12 @@ import {
|
|||||||
type ModelPriceCatalogResolver,
|
type ModelPriceCatalogResolver,
|
||||||
} from '../pricing/pricing';
|
} from '../pricing/pricing';
|
||||||
|
|
||||||
|
export {
|
||||||
|
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||||
|
MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS,
|
||||||
|
ModelInvocationSuccessfulCompletionRouter,
|
||||||
|
} from './successfulCompletionRouter';
|
||||||
|
|
||||||
export const MAX_MODEL_GATEWAY_CONCURRENCY = 64;
|
export const MAX_MODEL_GATEWAY_CONCURRENCY = 64;
|
||||||
|
|
||||||
export class ModelProviderUnavailableError extends Error {
|
export class ModelProviderUnavailableError extends Error {
|
||||||
@@ -126,6 +132,9 @@ export interface BoundedModelGatewayOptions {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface ModelInvocationSuccessfulCompletionSink {
|
export interface ModelInvocationSuccessfulCompletionSink {
|
||||||
|
supportsSuccessfulCompletionSink?(
|
||||||
|
sink: ModelInvocationSuccessfulCompletionSink,
|
||||||
|
): boolean;
|
||||||
record(
|
record(
|
||||||
audit: Readonly<ModelInvocationAuditRecord>,
|
audit: Readonly<ModelInvocationAuditRecord>,
|
||||||
result: Readonly<GenerateResult>,
|
result: Readonly<GenerateResult>,
|
||||||
@@ -372,7 +381,11 @@ export class BoundedModelGateway {
|
|||||||
supportsSuccessfulCompletionSink(
|
supportsSuccessfulCompletionSink(
|
||||||
sink: ModelInvocationSuccessfulCompletionSink,
|
sink: ModelInvocationSuccessfulCompletionSink,
|
||||||
): boolean {
|
): boolean {
|
||||||
return this.#successfulCompletion === sink;
|
const configured = this.#successfulCompletion;
|
||||||
|
return (
|
||||||
|
configured === sink ||
|
||||||
|
configured?.supportsSuccessfulCompletionSink?.(sink) === true
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async #prepare(
|
async #prepare(
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
import type { ModelInvocationSuccessfulCompletionSink } from './gateway';
|
||||||
|
import type {
|
||||||
|
GenerateResult,
|
||||||
|
ModelInvocationAuditRecord,
|
||||||
|
ModelInvocationAuditResult,
|
||||||
|
} from './model';
|
||||||
|
|
||||||
|
export const MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS = 8;
|
||||||
|
|
||||||
|
export class InvalidModelInvocationSuccessfulCompletionRouterError extends TypeError {
|
||||||
|
readonly code = 'MODEL_INVOCATION_SUCCESSFUL_COMPLETION_ROUTER_INVALID';
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
super('Model invocation successful completion router is invalid');
|
||||||
|
this.name = 'InvalidModelInvocationSuccessfulCompletionRouterError';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Bounded dispatch for mutually exclusive durable output domains. */
|
||||||
|
export class ModelInvocationSuccessfulCompletionRouter
|
||||||
|
implements ModelInvocationSuccessfulCompletionSink
|
||||||
|
{
|
||||||
|
readonly #sinks: readonly ModelInvocationSuccessfulCompletionSink[];
|
||||||
|
|
||||||
|
constructor(sinks: readonly ModelInvocationSuccessfulCompletionSink[]) {
|
||||||
|
if (
|
||||||
|
!Array.isArray(sinks) ||
|
||||||
|
sinks.length < 2 ||
|
||||||
|
sinks.length > MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS ||
|
||||||
|
new Set(sinks).size !== sinks.length ||
|
||||||
|
sinks.some(
|
||||||
|
(sink) =>
|
||||||
|
!sink ||
|
||||||
|
typeof sink !== 'object' ||
|
||||||
|
typeof sink.record !== 'function',
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
throw new InvalidModelInvocationSuccessfulCompletionRouterError();
|
||||||
|
}
|
||||||
|
this.#sinks = Object.freeze([...sinks]);
|
||||||
|
Object.freeze(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
supportsSuccessfulCompletionSink(
|
||||||
|
sink: ModelInvocationSuccessfulCompletionSink,
|
||||||
|
): boolean {
|
||||||
|
return this.#sinks.some(
|
||||||
|
(candidate) =>
|
||||||
|
candidate === sink ||
|
||||||
|
candidate.supportsSuccessfulCompletionSink?.(sink) === true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async record(
|
||||||
|
audit: Readonly<ModelInvocationAuditRecord>,
|
||||||
|
result: Readonly<GenerateResult>,
|
||||||
|
): Promise<
|
||||||
|
Readonly<
|
||||||
|
| { handled: false }
|
||||||
|
| { handled: true; disposition: ModelInvocationAuditResult }
|
||||||
|
>
|
||||||
|
> {
|
||||||
|
for (const sink of this.#sinks) {
|
||||||
|
const routed = await sink.record(audit, result);
|
||||||
|
if (
|
||||||
|
!routed ||
|
||||||
|
typeof routed !== 'object' ||
|
||||||
|
Array.isArray(routed) ||
|
||||||
|
(routed.handled !== true && routed.handled !== false)
|
||||||
|
) {
|
||||||
|
throw new InvalidModelInvocationSuccessfulCompletionRouterError();
|
||||||
|
}
|
||||||
|
if (routed.handled) return routed;
|
||||||
|
if (Object.keys(routed).length !== 1) {
|
||||||
|
throw new InvalidModelInvocationSuccessfulCompletionRouterError();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Object.freeze({ handled: false as const });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,6 +15,10 @@ import {
|
|||||||
import type { PluginPackagePromptOutputReadService } from '../../prompt-output/pluginPackagePromptOutputRead';
|
import type { PluginPackagePromptOutputReadService } from '../../prompt-output/pluginPackagePromptOutputRead';
|
||||||
import type { PluginPackagePromptExecutionOutputReadService } from '../../prompt-output/pluginPackagePromptExecutionOutputRead';
|
import type { PluginPackagePromptExecutionOutputReadService } from '../../prompt-output/pluginPackagePromptExecutionOutputRead';
|
||||||
import { bootstrapModelGatewayProfile } from '../../profile/profileComposition';
|
import { bootstrapModelGatewayProfile } from '../../profile/profileComposition';
|
||||||
|
import {
|
||||||
|
ModelInvocationSuccessfulCompletionRouter,
|
||||||
|
type ModelInvocationSuccessfulCompletionSink,
|
||||||
|
} from '../../model-gateway/gateway';
|
||||||
import {
|
import {
|
||||||
PostgresPluginPackagePromptApplicationUnavailableError,
|
PostgresPluginPackagePromptApplicationUnavailableError,
|
||||||
unavailable,
|
unavailable,
|
||||||
@@ -49,6 +53,8 @@ function assertEnabledOptions(
|
|||||||
(options.confirmActive !== undefined &&
|
(options.confirmActive !== undefined &&
|
||||||
typeof options.confirmActive !== 'function') ||
|
typeof options.confirmActive !== 'function') ||
|
||||||
(options.now !== undefined && typeof options.now !== 'function') ||
|
(options.now !== undefined && typeof options.now !== 'function') ||
|
||||||
|
(options.createAdditionalSuccessfulCompletion !== undefined &&
|
||||||
|
typeof options.createAdditionalSuccessfulCompletion !== 'function') ||
|
||||||
(options.promptOutputKeys !== undefined &&
|
(options.promptOutputKeys !== undefined &&
|
||||||
(!options.promptOutputKeys ||
|
(!options.promptOutputKeys ||
|
||||||
typeof options.promptOutputKeys !== 'object' ||
|
typeof options.promptOutputKeys !== 'object' ||
|
||||||
@@ -118,17 +124,36 @@ export async function bootstrapPostgresPluginPackagePromptApplication(
|
|||||||
});
|
});
|
||||||
},
|
},
|
||||||
loadProviders: options.loadProviders,
|
loadProviders: options.loadProviders,
|
||||||
...(options.promptOutputKeys === undefined
|
...(options.promptOutputKeys === undefined &&
|
||||||
|
options.createAdditionalSuccessfulCompletion === undefined
|
||||||
? {}
|
? {}
|
||||||
: {
|
: {
|
||||||
createSuccessfulCompletion: (coordinator) => {
|
createSuccessfulCompletion: (coordinator) => {
|
||||||
durableOutput =
|
const sinks: ModelInvocationSuccessfulCompletionSink[] = [];
|
||||||
new PluginPackagePromptOutputCompletionCoordinator({
|
if (options.promptOutputKeys !== undefined) {
|
||||||
|
durableOutput =
|
||||||
|
new PluginPackagePromptOutputCompletionCoordinator({
|
||||||
coordinator,
|
coordinator,
|
||||||
keys: options.promptOutputKeys!,
|
keys: options.promptOutputKeys!,
|
||||||
...(options.now === undefined ? {} : { now: options.now }),
|
...(options.now === undefined ? {} : { now: options.now }),
|
||||||
});
|
});
|
||||||
return durableOutput;
|
sinks.push(durableOutput);
|
||||||
|
}
|
||||||
|
if (options.createAdditionalSuccessfulCompletion !== undefined) {
|
||||||
|
const additionalSuccessfulCompletion =
|
||||||
|
options.createAdditionalSuccessfulCompletion(coordinator);
|
||||||
|
if (
|
||||||
|
!additionalSuccessfulCompletion ||
|
||||||
|
typeof additionalSuccessfulCompletion.record !== 'function'
|
||||||
|
) {
|
||||||
|
throw new TypeError(
|
||||||
|
'PostgreSQL Package Prompt additional completion is invalid',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
sinks.push(additionalSuccessfulCompletion);
|
||||||
|
}
|
||||||
|
if (sinks.length === 1) return sinks[0]!;
|
||||||
|
return new ModelInvocationSuccessfulCompletionRouter(sinks);
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
audit: options.audit,
|
audit: options.audit,
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ import type {
|
|||||||
ModelGatewayProfileAudit,
|
ModelGatewayProfileAudit,
|
||||||
ModelGatewayProviderAuthority,
|
ModelGatewayProviderAuthority,
|
||||||
} from '../../profile/profileComposition';
|
} from '../../profile/profileComposition';
|
||||||
|
import type { DurableModelInvocationCoordinator } from '../../model-invocation/durableModelInvocationCoordinator';
|
||||||
|
import type { ModelInvocationSuccessfulCompletionSink } from '../../model-gateway/gateway';
|
||||||
import type { PluginPackagePromptCatalogCapability } from '../pluginPackagePromptCatalog';
|
import type { PluginPackagePromptCatalogCapability } from '../pluginPackagePromptCatalog';
|
||||||
import type { PluginPackagePromptExecutionInspectionRepository } from '../pluginPackagePromptExecutionInspection';
|
import type { PluginPackagePromptExecutionInspectionRepository } from '../pluginPackagePromptExecutionInspection';
|
||||||
|
|
||||||
@@ -80,6 +82,9 @@ export type BootstrapPostgresPluginPackagePromptApplicationOptions =
|
|||||||
maxConcurrent?: number;
|
maxConcurrent?: number;
|
||||||
recoveryLimit?: number;
|
recoveryLimit?: number;
|
||||||
now?: () => number;
|
now?: () => number;
|
||||||
|
createAdditionalSuccessfulCompletion?: (
|
||||||
|
coordinator: DurableModelInvocationCoordinator,
|
||||||
|
) => ModelInvocationSuccessfulCompletionSink;
|
||||||
promptOutputKeys?: PluginPackagePromptOutputArtifactKeyProvider;
|
promptOutputKeys?: PluginPackagePromptOutputArtifactKeyProvider;
|
||||||
promptOutputRead?: Readonly<{
|
promptOutputRead?: Readonly<{
|
||||||
authorizer: PluginPackagePromptOutputArtifactReadAuthorizer;
|
authorizer: PluginPackagePromptOutputArtifactReadAuthorizer;
|
||||||
|
|||||||
@@ -0,0 +1,271 @@
|
|||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const { test } = require('node:test');
|
||||||
|
|
||||||
|
const {
|
||||||
|
CopilotFailureDiagnosisApplicationService,
|
||||||
|
CopilotFailureDiagnosisApplicationUnavailableError,
|
||||||
|
} = require('@qinglong/ai/failure-diagnosis-application');
|
||||||
|
const {
|
||||||
|
BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION,
|
||||||
|
} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool');
|
||||||
|
const {
|
||||||
|
createPluginPackageResourceGenerationFromReferences,
|
||||||
|
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
|
||||||
|
const {
|
||||||
|
createProjectToolDefinitionSnapshot,
|
||||||
|
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
|
||||||
|
|
||||||
|
const NOW = 1_800_000_000_000;
|
||||||
|
const KEY = Buffer.alloc(32, 0x42);
|
||||||
|
const MODEL = Object.freeze({
|
||||||
|
provider: 'provider-primary',
|
||||||
|
model: 'model-diagnosis',
|
||||||
|
modelBoundary: 'external',
|
||||||
|
responseLanguage: 'zh-CN',
|
||||||
|
maxOutputTokens: 512,
|
||||||
|
egressPolicy: Object.freeze({
|
||||||
|
schema: 'qinglong/copilot-model-egress-policy@v1',
|
||||||
|
revision: 'application-test-v1',
|
||||||
|
potentiallySensitiveDataBoundaries: Object.freeze(['external']),
|
||||||
|
maxInputBytes: 64 * 1024,
|
||||||
|
maxOutputTokens: 1024,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
function snapshot() {
|
||||||
|
const generation = createPluginPackageResourceGenerationFromReferences({
|
||||||
|
installationId: 'installation-copilot-test',
|
||||||
|
projectId: 'project-1',
|
||||||
|
packageName: 'qinglong',
|
||||||
|
lockDigest: 'a'.repeat(64),
|
||||||
|
generation: 1,
|
||||||
|
previousActiveLockDigest: null,
|
||||||
|
contentDigest: 'b'.repeat(64),
|
||||||
|
resources: [],
|
||||||
|
});
|
||||||
|
return createProjectToolDefinitionSnapshot({
|
||||||
|
projectId: 'project-1',
|
||||||
|
contributions: [
|
||||||
|
{
|
||||||
|
generation,
|
||||||
|
revisionDigest: 'c'.repeat(64),
|
||||||
|
definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function command(overrides = {}) {
|
||||||
|
return {
|
||||||
|
requestId: 'diagnosis-request-1',
|
||||||
|
traceId: 'diagnosis-trace-1',
|
||||||
|
projectId: 'project-1',
|
||||||
|
sourceRunId: 'source-run-1',
|
||||||
|
principal: {
|
||||||
|
subject: { type: 'user', id: 'owner-1' },
|
||||||
|
authenticationId: 'auth-1',
|
||||||
|
authenticatedAtMs: NOW - 1000,
|
||||||
|
expiresAtMs: NOW + 60_000,
|
||||||
|
assurance: 'multi_factor',
|
||||||
|
},
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function fixture(options = {}) {
|
||||||
|
let plan = null;
|
||||||
|
let artifacts = null;
|
||||||
|
let failArtifactOnce = options.failArtifactOnce === true;
|
||||||
|
let activeKeyCopies = [];
|
||||||
|
let resolvedKeyCopies = [];
|
||||||
|
let toolCalls = 0;
|
||||||
|
let modelCalls = 0;
|
||||||
|
let releaseTool;
|
||||||
|
const toolGate = options.blockTool
|
||||||
|
? new Promise((resolve) => { releaseTool = resolve; })
|
||||||
|
: Promise.resolve();
|
||||||
|
const admissions = {
|
||||||
|
async findByRequestId(requestId) {
|
||||||
|
return plan?.requestId === requestId
|
||||||
|
? { requestId, planDigest: plan.planDigest }
|
||||||
|
: null;
|
||||||
|
},
|
||||||
|
async findPlanByRequestId(requestId) {
|
||||||
|
return plan?.requestId === requestId ? plan : null;
|
||||||
|
},
|
||||||
|
async admit(value) {
|
||||||
|
const status = plan ? 'existing' : 'created';
|
||||||
|
plan ??= value;
|
||||||
|
assert.equal(plan.planDigest, value.planDigest);
|
||||||
|
return {
|
||||||
|
status,
|
||||||
|
receipt: {
|
||||||
|
requestId: plan.requestId,
|
||||||
|
planDigest: plan.planDigest,
|
||||||
|
runId: plan.runId,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const snapshots = {
|
||||||
|
async findCurrent() { return { snapshot: snapshot(), committedAtMs: NOW }; },
|
||||||
|
};
|
||||||
|
const runs = {
|
||||||
|
async findRunById() {
|
||||||
|
return {
|
||||||
|
id: 'source-run-1', projectId: 'project-1', status: 'failed',
|
||||||
|
version: 8, eventSequence: 8,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async findLatestAttemptByRunId() {
|
||||||
|
return {
|
||||||
|
id: 'source-attempt-1', runId: 'source-run-1', attempt: 1,
|
||||||
|
status: 'failed', executorType: 'remote_worker', callbackSequence: 0,
|
||||||
|
createdAtMs: NOW - 3000, finishedAtMs: NOW - 1000,
|
||||||
|
logArtifactId: `wlog-${'d'.repeat(30)}`,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const artifactRepository = {
|
||||||
|
async put(input, preview) {
|
||||||
|
if (failArtifactOnce) {
|
||||||
|
failArtifactOnce = false;
|
||||||
|
throw new Error('simulated admission-to-artifact crash');
|
||||||
|
}
|
||||||
|
if (artifacts) {
|
||||||
|
assert.deepEqual(input, artifacts.input);
|
||||||
|
assert.deepEqual(preview, artifacts.preview);
|
||||||
|
return { status: 'existing' };
|
||||||
|
}
|
||||||
|
artifacts = { input, preview };
|
||||||
|
return { status: 'inserted' };
|
||||||
|
},
|
||||||
|
async findInput() { return artifacts?.input ?? null; },
|
||||||
|
async findPreview() { return artifacts?.preview ?? null; },
|
||||||
|
};
|
||||||
|
const invocationKeys = {
|
||||||
|
async active() {
|
||||||
|
const copy = Buffer.from(KEY);
|
||||||
|
activeKeyCopies.push(copy);
|
||||||
|
return { keyId: 'invocation-key-1', key: copy };
|
||||||
|
},
|
||||||
|
async resolve(keyId) {
|
||||||
|
assert.equal(keyId, 'invocation-key-1');
|
||||||
|
const copy = Buffer.from(KEY);
|
||||||
|
resolvedKeyCopies.push(copy);
|
||||||
|
return { keyId, key: copy };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const unlocks = { async findByRequestId() { return null; }, async commit() {} };
|
||||||
|
const tool = {
|
||||||
|
admissions,
|
||||||
|
snapshots,
|
||||||
|
runs,
|
||||||
|
artifacts: artifactRepository,
|
||||||
|
invocationKeys,
|
||||||
|
resultKeys: { async resolve() { return null; } },
|
||||||
|
stepRuns: { async findById() { return null; } },
|
||||||
|
barriers: {}, completions: {}, failureCompletions: {},
|
||||||
|
resultKeyCatalog: {}, resultRekeys: {}, logs: {}, unlocks,
|
||||||
|
};
|
||||||
|
const model = {
|
||||||
|
admissions,
|
||||||
|
unlocks,
|
||||||
|
toolResults: {}, modelInvocations: {}, outputs: {}, gateway: {},
|
||||||
|
successfulCompletion: {}, finalizations: {},
|
||||||
|
};
|
||||||
|
const service = new CopilotFailureDiagnosisApplicationService({
|
||||||
|
admissions,
|
||||||
|
snapshots,
|
||||||
|
runs,
|
||||||
|
artifacts: artifactRepository,
|
||||||
|
invocationKeys,
|
||||||
|
authorizer: {
|
||||||
|
async authorize() {
|
||||||
|
return {
|
||||||
|
effect: 'allow', reasons: ['role_grant'],
|
||||||
|
fence: { projectVersion: 1, bindingVersion: 1 },
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
tool,
|
||||||
|
model,
|
||||||
|
modelIntent: MODEL,
|
||||||
|
executionTimeoutMs: 60_000,
|
||||||
|
now: () => NOW,
|
||||||
|
async executeTool() {
|
||||||
|
toolCalls += 1;
|
||||||
|
await toolGate;
|
||||||
|
return options.toolFailure
|
||||||
|
? { outcome: 'failed', completionStatus: 'created', unlockStatus: null }
|
||||||
|
: {
|
||||||
|
outcome: 'succeeded', completionStatus: 'created',
|
||||||
|
unlockStatus: 'created', completion: {}, unlock: {},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async executeModel() {
|
||||||
|
modelCalls += 1;
|
||||||
|
return {
|
||||||
|
outcome: 'succeeded',
|
||||||
|
output: { artifactId: 'output-1' },
|
||||||
|
finalization: { requestId: 'diagnosis-request-1' },
|
||||||
|
};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
service,
|
||||||
|
releaseTool: () => releaseTool?.(),
|
||||||
|
state: () => ({
|
||||||
|
plan, artifacts, toolCalls, modelCalls,
|
||||||
|
activeKeyCopies, resolvedKeyCopies,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('application derives, admits and executes one server-owned diagnosis with exact replay', async () => {
|
||||||
|
const testFixture = fixture();
|
||||||
|
const first = await testFixture.service.execute(command());
|
||||||
|
assert.equal(first.admissionStatus, 'created');
|
||||||
|
assert.equal(first.tool.outcome, 'succeeded');
|
||||||
|
assert.equal(first.model.outcome, 'succeeded');
|
||||||
|
assert.equal(first.terminalizationRequired, false);
|
||||||
|
const second = await testFixture.service.execute(command());
|
||||||
|
assert.equal(second.admissionStatus, 'existing');
|
||||||
|
const state = testFixture.state();
|
||||||
|
assert.equal(state.toolCalls, 2);
|
||||||
|
assert.equal(state.modelCalls, 2);
|
||||||
|
assert.equal(state.plan.source.attemptId, 'source-attempt-1');
|
||||||
|
assert.equal(state.plan.tool.invocationArtifact.artifactId.startsWith('cdia:'), true);
|
||||||
|
assert.equal(state.activeKeyCopies[0].every((value) => value === 0), true);
|
||||||
|
assert.equal(state.resolvedKeyCopies[0].every((value) => value === 0), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('application repairs the durable admission-to-Artifact crash window', async () => {
|
||||||
|
const testFixture = fixture({ failArtifactOnce: true });
|
||||||
|
await assert.rejects(
|
||||||
|
testFixture.service.execute(command()),
|
||||||
|
CopilotFailureDiagnosisApplicationUnavailableError,
|
||||||
|
);
|
||||||
|
assert.ok(testFixture.state().plan);
|
||||||
|
assert.equal(testFixture.state().artifacts, null);
|
||||||
|
const replay = await testFixture.service.execute(command());
|
||||||
|
assert.equal(replay.admissionStatus, 'existing');
|
||||||
|
assert.ok(testFixture.state().artifacts);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('application coalesces exact callers and exposes Tool terminalization debt', async () => {
|
||||||
|
const concurrent = fixture({ blockTool: true });
|
||||||
|
const first = concurrent.service.execute(command());
|
||||||
|
const second = concurrent.service.execute(command());
|
||||||
|
assert.equal(first, second);
|
||||||
|
concurrent.releaseTool();
|
||||||
|
await first;
|
||||||
|
assert.equal(concurrent.state().toolCalls, 1);
|
||||||
|
|
||||||
|
const failed = fixture({ toolFailure: true });
|
||||||
|
const result = await failed.service.execute(command());
|
||||||
|
assert.equal(result.tool.outcome, 'failed');
|
||||||
|
assert.equal(result.model, null);
|
||||||
|
assert.equal(result.terminalizationRequired, true);
|
||||||
|
assert.equal(failed.state().modelCalls, 0);
|
||||||
|
});
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const { test } = require('node:test');
|
||||||
|
|
||||||
|
const {
|
||||||
|
BoundedModelGateway,
|
||||||
|
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||||
|
ModelInvocationSuccessfulCompletionRouter,
|
||||||
|
} = require('@qinglong/ai/gateway');
|
||||||
|
|
||||||
|
function sink(name, handled = false) {
|
||||||
|
return {
|
||||||
|
async record(audit) {
|
||||||
|
audit.order.push(name);
|
||||||
|
return handled
|
||||||
|
? { handled: true, disposition: { status: 'created' } }
|
||||||
|
: { handled: false };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('successful completion router exposes exact children and stops at the owning sink', async () => {
|
||||||
|
const first = sink('prompt');
|
||||||
|
const second = sink('copilot', true);
|
||||||
|
const third = sink('unreachable', true);
|
||||||
|
const nested = new ModelInvocationSuccessfulCompletionRouter([
|
||||||
|
second,
|
||||||
|
third,
|
||||||
|
]);
|
||||||
|
const router = new ModelInvocationSuccessfulCompletionRouter([first, nested]);
|
||||||
|
const order = [];
|
||||||
|
assert.equal(router.supportsSuccessfulCompletionSink(first), true);
|
||||||
|
assert.equal(router.supportsSuccessfulCompletionSink(second), true);
|
||||||
|
assert.equal(router.supportsSuccessfulCompletionSink({ record() {} }), false);
|
||||||
|
assert.deepEqual(await router.record({ order }, {}), {
|
||||||
|
handled: true,
|
||||||
|
disposition: { status: 'created' },
|
||||||
|
});
|
||||||
|
assert.deepEqual(order, ['prompt', 'copilot']);
|
||||||
|
|
||||||
|
const gateway = new BoundedModelGateway({
|
||||||
|
providers: [
|
||||||
|
{
|
||||||
|
type: 'test',
|
||||||
|
async generate() { throw new Error('unused'); },
|
||||||
|
async *stream() { throw new Error('unused'); },
|
||||||
|
async listModels() { return []; },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
policies: { async resolve() { throw new Error('unused'); } },
|
||||||
|
pricing: { async resolve() { return null; } },
|
||||||
|
audit: { async record() {} },
|
||||||
|
successfulCompletion: router,
|
||||||
|
maxConcurrent: 1,
|
||||||
|
});
|
||||||
|
assert.equal(gateway.supportsSuccessfulCompletionSink(router), true);
|
||||||
|
assert.equal(gateway.supportsSuccessfulCompletionSink(second), true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('successful completion router rejects unbounded, duplicate and malformed sinks', async () => {
|
||||||
|
const valid = sink('valid');
|
||||||
|
assert.throws(
|
||||||
|
() => new ModelInvocationSuccessfulCompletionRouter([valid]),
|
||||||
|
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => new ModelInvocationSuccessfulCompletionRouter([valid, valid]),
|
||||||
|
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => new ModelInvocationSuccessfulCompletionRouter([valid, {}]),
|
||||||
|
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||||
|
);
|
||||||
|
const malformed = new ModelInvocationSuccessfulCompletionRouter([
|
||||||
|
valid,
|
||||||
|
{ async record() { return { handled: false, widened: true }; } },
|
||||||
|
]);
|
||||||
|
await assert.rejects(
|
||||||
|
malformed.record({ order: [] }, {}),
|
||||||
|
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||||
|
);
|
||||||
|
});
|
||||||
@@ -35,6 +35,11 @@
|
|||||||
"require": "./dist/application-runtime/aiProductionApplication.js",
|
"require": "./dist/application-runtime/aiProductionApplication.js",
|
||||||
"default": "./dist/application-runtime/aiProductionApplication.js"
|
"default": "./dist/application-runtime/aiProductionApplication.js"
|
||||||
},
|
},
|
||||||
|
"./copilot-production": {
|
||||||
|
"types": "./dist/application-runtime/copilot/failureDiagnosisComposition.d.ts",
|
||||||
|
"require": "./dist/application-runtime/copilot/failureDiagnosisComposition.js",
|
||||||
|
"default": "./dist/application-runtime/copilot/failureDiagnosisComposition.js"
|
||||||
|
},
|
||||||
"./failure-diagnosis-output-keyring": {
|
"./failure-diagnosis-output-keyring": {
|
||||||
"types": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.d.ts",
|
"types": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.d.ts",
|
||||||
"require": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js",
|
"require": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js",
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile';
|
import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile';
|
||||||
|
import type { DurableModelInvocationCoordinator } from '@qinglong/ai/durable-model-invocation';
|
||||||
|
import { CopilotFailureDiagnosisModelCompletionCoordinator } from '@qinglong/ai/failure-diagnosis-model-execution';
|
||||||
|
import type { CopilotFailureDiagnosisApplicationService } from '@qinglong/ai/failure-diagnosis-application';
|
||||||
import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential';
|
import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential';
|
||||||
import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage';
|
import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage';
|
||||||
import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority';
|
import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority';
|
||||||
@@ -22,12 +25,19 @@ import {
|
|||||||
startProductionClusterControlApplication,
|
startProductionClusterControlApplication,
|
||||||
type ProductionClusterControlApplicationOptions,
|
type ProductionClusterControlApplicationOptions,
|
||||||
} from './productionApplication';
|
} from './productionApplication';
|
||||||
|
import {
|
||||||
|
createProductionClusterCopilotFailureDiagnosis,
|
||||||
|
prepareProductionClusterCopilotFailureDiagnosisProjection,
|
||||||
|
type ClusterCopilotFailureDiagnosisProjection,
|
||||||
|
type CreateProductionClusterCopilotFailureDiagnosisOptions,
|
||||||
|
} from './copilot/failureDiagnosisComposition';
|
||||||
|
|
||||||
export interface EnabledProductionClusterAiConfig {
|
export interface EnabledProductionClusterAiConfig {
|
||||||
readonly enabled: true;
|
readonly enabled: true;
|
||||||
readonly providerAuthorityFile: string;
|
readonly providerAuthorityFile: string;
|
||||||
readonly secretRootDirectory: string;
|
readonly secretRootDirectory: string;
|
||||||
readonly promptOutputKeyringRootDirectory?: string;
|
readonly promptOutputKeyringRootDirectory?: string;
|
||||||
|
readonly copilot?: Readonly<ClusterCopilotFailureDiagnosisProjection>;
|
||||||
readonly maxConcurrent: number;
|
readonly maxConcurrent: number;
|
||||||
readonly recoveryLimit: number;
|
readonly recoveryLimit: number;
|
||||||
readonly databaseMaxConnections: number;
|
readonly databaseMaxConnections: number;
|
||||||
@@ -41,8 +51,18 @@ export interface ProductionClusterAiControlApplicationOptions {
|
|||||||
) => void | Promise<void>;
|
) => void | Promise<void>;
|
||||||
readonly startControl?: typeof startProductionClusterControlApplication;
|
readonly startControl?: typeof startProductionClusterControlApplication;
|
||||||
readonly bootstrapPrompt?: typeof bootstrapPostgresPluginPackagePromptApplication;
|
readonly bootstrapPrompt?: typeof bootstrapPostgresPluginPackagePromptApplication;
|
||||||
|
readonly createCopilot?: (
|
||||||
|
options: CreateProductionClusterCopilotFailureDiagnosisOptions,
|
||||||
|
) => Promise<Readonly<CopilotFailureDiagnosisApplicationService>>;
|
||||||
|
readonly openAiDatabase?: ReturnType<typeof createPostgresDatabaseOpener>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type ProductionClusterAiControlApplicationResult = Extract<
|
||||||
|
ClusterControlApplicationResult,
|
||||||
|
{ readonly status: 'active' }
|
||||||
|
> &
|
||||||
|
Readonly<{ copilot?: Readonly<CopilotFailureDiagnosisApplicationService> }>;
|
||||||
|
|
||||||
export class ProductionClusterAiConfigError extends TypeError {
|
export class ProductionClusterAiConfigError extends TypeError {
|
||||||
readonly code = 'QL3_CLUSTER_AI_CONFIG_INVALID';
|
readonly code = 'QL3_CLUSTER_AI_CONFIG_INVALID';
|
||||||
|
|
||||||
@@ -121,6 +141,11 @@ export function loadProductionClusterAiConfig(
|
|||||||
'QL3_CLUSTER_AI_PROMPT_OUTPUT_ENABLED',
|
'QL3_CLUSTER_AI_PROMPT_OUTPUT_ENABLED',
|
||||||
false,
|
false,
|
||||||
);
|
);
|
||||||
|
const copilotEnabled = booleanValue(
|
||||||
|
environment,
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_ENABLED',
|
||||||
|
false,
|
||||||
|
);
|
||||||
return Object.freeze({
|
return Object.freeze({
|
||||||
enabled: true,
|
enabled: true,
|
||||||
providerAuthorityFile: requiredPath(
|
providerAuthorityFile: requiredPath(
|
||||||
@@ -139,6 +164,28 @@ export function loadProductionClusterAiConfig(
|
|||||||
),
|
),
|
||||||
}
|
}
|
||||||
: {}),
|
: {}),
|
||||||
|
...(copilotEnabled
|
||||||
|
? {
|
||||||
|
copilot: Object.freeze({
|
||||||
|
configFile: requiredPath(
|
||||||
|
environment,
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_CONFIG_FILE',
|
||||||
|
),
|
||||||
|
invocationKeyringRootDirectory: requiredPath(
|
||||||
|
environment,
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_INVOCATION_KEYRING_ROOT',
|
||||||
|
),
|
||||||
|
resultKeyringRootDirectory: requiredPath(
|
||||||
|
environment,
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_RESULT_KEYRING_ROOT',
|
||||||
|
),
|
||||||
|
outputKeyringRootDirectory: requiredPath(
|
||||||
|
environment,
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_OUTPUT_KEYRING_ROOT',
|
||||||
|
),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
maxConcurrent: boundedInteger(
|
maxConcurrent: boundedInteger(
|
||||||
environment,
|
environment,
|
||||||
'QL3_CLUSTER_AI_MAX_CONCURRENT',
|
'QL3_CLUSTER_AI_MAX_CONCURRENT',
|
||||||
@@ -187,7 +234,7 @@ function aiDatabaseOpener(
|
|||||||
*/
|
*/
|
||||||
export async function startProductionClusterAiControlApplication(
|
export async function startProductionClusterAiControlApplication(
|
||||||
options: ProductionClusterAiControlApplicationOptions,
|
options: ProductionClusterAiControlApplicationOptions,
|
||||||
): Promise<ClusterControlApplicationResult> {
|
): Promise<ProductionClusterAiControlApplicationResult> {
|
||||||
if (
|
if (
|
||||||
!options ||
|
!options ||
|
||||||
typeof options !== 'object' ||
|
typeof options !== 'object' ||
|
||||||
@@ -200,9 +247,26 @@ export async function startProductionClusterAiControlApplication(
|
|||||||
options.startControl ?? startProductionClusterControlApplication;
|
options.startControl ?? startProductionClusterControlApplication;
|
||||||
const bootstrapPrompt =
|
const bootstrapPrompt =
|
||||||
options.bootstrapPrompt ?? bootstrapPostgresPluginPackagePromptApplication;
|
options.bootstrapPrompt ?? bootstrapPostgresPluginPackagePromptApplication;
|
||||||
if (typeof startControl !== 'function' || typeof bootstrapPrompt !== 'function') {
|
const createCopilot =
|
||||||
|
options.createCopilot ?? createProductionClusterCopilotFailureDiagnosis;
|
||||||
|
if (
|
||||||
|
typeof startControl !== 'function' ||
|
||||||
|
typeof bootstrapPrompt !== 'function' ||
|
||||||
|
typeof createCopilot !== 'function' ||
|
||||||
|
(options.openAiDatabase !== undefined &&
|
||||||
|
typeof options.openAiDatabase !== 'function')
|
||||||
|
) {
|
||||||
throw new TypeError('Production Cluster AI application factories are invalid');
|
throw new TypeError('Production Cluster AI application factories are invalid');
|
||||||
}
|
}
|
||||||
|
const copilotArtifactStore = options.control.workerIngress?.artifactStore;
|
||||||
|
if (
|
||||||
|
options.ai.copilot !== undefined &&
|
||||||
|
typeof copilotArtifactStore?.readLogRange !== 'function'
|
||||||
|
) {
|
||||||
|
throw new TypeError(
|
||||||
|
'Cluster Copilot requires the bounded Worker log Artifact read capability',
|
||||||
|
);
|
||||||
|
}
|
||||||
const secretMaterial =
|
const secretMaterial =
|
||||||
await createProjectedModelProviderSecretMaterialProvider({
|
await createProjectedModelProviderSecretMaterialProvider({
|
||||||
rootDirectory: options.ai.secretRootDirectory,
|
rootDirectory: options.ai.secretRootDirectory,
|
||||||
@@ -213,6 +277,12 @@ export async function startProductionClusterAiControlApplication(
|
|||||||
: await createPluginPackagePromptOutputProjectedKeyring({
|
: await createPluginPackagePromptOutputProjectedKeyring({
|
||||||
rootDirectory: options.ai.promptOutputKeyringRootDirectory,
|
rootDirectory: options.ai.promptOutputKeyringRootDirectory,
|
||||||
});
|
});
|
||||||
|
const preparedCopilot =
|
||||||
|
options.ai.copilot === undefined
|
||||||
|
? undefined
|
||||||
|
: await prepareProductionClusterCopilotFailureDiagnosisProjection(
|
||||||
|
options.ai.copilot,
|
||||||
|
);
|
||||||
let aiDatabase:
|
let aiDatabase:
|
||||||
| Awaited<ReturnType<ReturnType<typeof createPostgresDatabaseOpener>>>
|
| Awaited<ReturnType<ReturnType<typeof createPostgresDatabaseOpener>>>
|
||||||
| undefined;
|
| undefined;
|
||||||
@@ -230,6 +300,12 @@ export async function startProductionClusterAiControlApplication(
|
|||||||
| BootstrapPostgresPluginPackagePromptApplicationResult
|
| BootstrapPostgresPluginPackagePromptApplicationResult
|
||||||
| undefined;
|
| undefined;
|
||||||
let controlApplication: ClusterControlApplicationResult | undefined;
|
let controlApplication: ClusterControlApplicationResult | undefined;
|
||||||
|
let copilotApplication:
|
||||||
|
| Readonly<CopilotFailureDiagnosisApplicationService>
|
||||||
|
| undefined;
|
||||||
|
let copilotSuccessfulCompletion:
|
||||||
|
| CopilotFailureDiagnosisModelCompletionCoordinator
|
||||||
|
| undefined;
|
||||||
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||||
let promptOutputPolicy: ProjectPolicyEngine | undefined;
|
let promptOutputPolicy: ProjectPolicyEngine | undefined;
|
||||||
const promptOutputReadAuthorizer = Object.freeze({
|
const promptOutputReadAuthorizer = Object.freeze({
|
||||||
@@ -271,11 +347,9 @@ export async function startProductionClusterAiControlApplication(
|
|||||||
return stopPromise;
|
return stopPromise;
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
const openDatabase = aiDatabaseOpener(
|
const openDatabase =
|
||||||
options.control.config,
|
options.openAiDatabase ??
|
||||||
options.ai,
|
aiDatabaseOpener(options.control.config, options.ai, onAiUnavailable);
|
||||||
onAiUnavailable,
|
|
||||||
);
|
|
||||||
promptApplication = await bootstrapPrompt({
|
promptApplication = await bootstrapPrompt({
|
||||||
enabled: true,
|
enabled: true,
|
||||||
async openDatabase() {
|
async openDatabase() {
|
||||||
@@ -311,10 +385,41 @@ export async function startProductionClusterAiControlApplication(
|
|||||||
promptOutputKeys,
|
promptOutputKeys,
|
||||||
promptOutputRead: { authorizer: promptOutputReadAuthorizer },
|
promptOutputRead: { authorizer: promptOutputReadAuthorizer },
|
||||||
}),
|
}),
|
||||||
|
...(preparedCopilot === undefined
|
||||||
|
? {}
|
||||||
|
: {
|
||||||
|
createAdditionalSuccessfulCompletion(
|
||||||
|
coordinator: DurableModelInvocationCoordinator,
|
||||||
|
) {
|
||||||
|
if (copilotSuccessfulCompletion) {
|
||||||
|
throw new Error(
|
||||||
|
'Cluster Copilot completion was created more than once',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
copilotSuccessfulCompletion =
|
||||||
|
new CopilotFailureDiagnosisModelCompletionCoordinator({
|
||||||
|
coordinator,
|
||||||
|
keys: preparedCopilot.outputKeys,
|
||||||
|
});
|
||||||
|
return copilotSuccessfulCompletion;
|
||||||
|
},
|
||||||
|
}),
|
||||||
});
|
});
|
||||||
if (promptApplication.status !== 'active') {
|
if (promptApplication.status !== 'active') {
|
||||||
throw new Error('Cluster AI Prompt application did not activate');
|
throw new Error('Cluster AI Prompt application did not activate');
|
||||||
}
|
}
|
||||||
|
if (preparedCopilot !== undefined) {
|
||||||
|
if (!copilotSuccessfulCompletion || !aiDatabase || !copilotArtifactStore) {
|
||||||
|
throw new Error('Cluster Copilot shared authorities did not activate');
|
||||||
|
}
|
||||||
|
copilotApplication = await createCopilot({
|
||||||
|
pool: aiDatabase.pool,
|
||||||
|
gateway: promptApplication.capability,
|
||||||
|
prepared: preparedCopilot,
|
||||||
|
successfulCompletion: copilotSuccessfulCompletion,
|
||||||
|
artifactStore: copilotArtifactStore,
|
||||||
|
});
|
||||||
|
}
|
||||||
controlApplication = await startControl({
|
controlApplication = await startControl({
|
||||||
...options.control,
|
...options.control,
|
||||||
promptCatalog: {
|
promptCatalog: {
|
||||||
@@ -356,6 +461,9 @@ export async function startProductionClusterAiControlApplication(
|
|||||||
? 'unavailable'
|
? 'unavailable'
|
||||||
: activeControl.availabilityStatus();
|
: activeControl.availabilityStatus();
|
||||||
},
|
},
|
||||||
|
...(copilotApplication === undefined
|
||||||
|
? {}
|
||||||
|
: { copilot: copilotApplication }),
|
||||||
stop,
|
stop,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
+466
@@ -0,0 +1,466 @@
|
|||||||
|
import { Buffer } from 'node:buffer';
|
||||||
|
import { basename, dirname } from 'node:path';
|
||||||
|
|
||||||
|
import {
|
||||||
|
CopilotFailureDiagnosisApplicationService,
|
||||||
|
type CopilotFailureDiagnosisApplicationDependencies,
|
||||||
|
} from '@qinglong/ai/failure-diagnosis-application';
|
||||||
|
import type { PrepareCopilotFailureDiagnosisModelIntent } from '@qinglong/ai/failure-diagnosis-execution-admission';
|
||||||
|
import {
|
||||||
|
CopilotFailureDiagnosisModelCompletionCoordinator,
|
||||||
|
executeCopilotFailureDiagnosisModel,
|
||||||
|
type CopilotFailureDiagnosisToolResultReader,
|
||||||
|
} from '@qinglong/ai/failure-diagnosis-model-execution';
|
||||||
|
import {
|
||||||
|
executeCopilotFailureDiagnosisTool,
|
||||||
|
restoreCopilotFailureDiagnosisTrustedToolAuthority,
|
||||||
|
} from '@qinglong/ai/failure-diagnosis-tool-execution';
|
||||||
|
import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage';
|
||||||
|
import { PostgresCopilotFailureDiagnosisModelRepository } from '@qinglong/ai/postgres-failure-diagnosis-model-execution-storage';
|
||||||
|
import { PostgresCopilotFailureDiagnosisToolUnlockRepository } from '@qinglong/ai/postgres-failure-diagnosis-tool-execution-storage';
|
||||||
|
import type { ActiveModelGatewayCapability } from '@qinglong/ai/profile';
|
||||||
|
import {
|
||||||
|
PostgresProjectPolicyRepository,
|
||||||
|
PostgresProjectToolDefinitionSnapshotRepository,
|
||||||
|
PostgresRunAttemptLogRetentionClaimRepository,
|
||||||
|
PostgresRunRepository,
|
||||||
|
PostgresStepRunRepository,
|
||||||
|
PostgresToolExecutionCompletionRepository,
|
||||||
|
PostgresToolExecutionFailureCompletionRepository,
|
||||||
|
PostgresToolExecutionStartBarrierRepository,
|
||||||
|
PostgresToolInvocationArtifactRepository,
|
||||||
|
PostgresToolResultKeyCatalogReader,
|
||||||
|
PostgresToolResultRekeyReader,
|
||||||
|
type QingLongPostgresPool,
|
||||||
|
} from '@qinglong/cluster-postgres/runtime';
|
||||||
|
import {
|
||||||
|
BuiltInRunLogExcerptToolAdapter,
|
||||||
|
} from '@qinglong/runtime-core/builtin-run-log-excerpt-tool';
|
||||||
|
import type { RunAttemptLogReadPort } from '@qinglong/runtime-core/builtin-run-log-excerpt-projection';
|
||||||
|
import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
|
||||||
|
import { RunAttemptLogReadService } from '@qinglong/runtime-core/run-attempt-log-read';
|
||||||
|
import { openTrustedToolSuccessCompletion } from '@qinglong/runtime-core/trusted-tool-completion';
|
||||||
|
import { TrustedToolExecutionAdapterRegistry } from '@qinglong/runtime-core/trusted-tool-execution';
|
||||||
|
|
||||||
|
import type { ClusterRemoteWorkerArtifactStore } from '../../remote-execution/remoteWorkerCompletionService';
|
||||||
|
import { PrivateProjectedFileReader } from '../../security/privateProjectedFile';
|
||||||
|
import { createClusterToolInvocationProjectedKeyring } from '../../trusted-tool/key-management/toolInvocationProjectedKeyring';
|
||||||
|
import { createClusterToolResultProjectedKeyring } from '../../trusted-tool/key-management/toolResultProjectedKeyring';
|
||||||
|
import { createClusterCopilotFailureDiagnosisOutputProjectedKeyring } from '../../copilot/failure-diagnosis/outputProjectedKeyring';
|
||||||
|
|
||||||
|
export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA =
|
||||||
|
'qinglong/cluster-copilot-failure-diagnosis-config@v1' as const;
|
||||||
|
export const MAX_CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_BYTES = 16 * 1024;
|
||||||
|
|
||||||
|
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||||
|
const MODEL_BOUNDARIES = ['on_device', 'external'] as const;
|
||||||
|
const RESPONSE_LANGUAGES = ['en', 'zh-CN'] as const;
|
||||||
|
const EGRESS_SCHEMA = 'qinglong/copilot-model-egress-policy@v1' as const;
|
||||||
|
|
||||||
|
export interface ClusterCopilotFailureDiagnosisConfig {
|
||||||
|
readonly schema: typeof CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA;
|
||||||
|
readonly provider: string;
|
||||||
|
readonly model: string;
|
||||||
|
readonly modelBoundary: 'on_device' | 'external';
|
||||||
|
readonly responseLanguage: 'en' | 'zh-CN';
|
||||||
|
readonly maxOutputTokens: number;
|
||||||
|
readonly executionTimeoutMs: number;
|
||||||
|
readonly egressPolicy: PrepareCopilotFailureDiagnosisModelIntent['egressPolicy'];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ClusterCopilotFailureDiagnosisProjection {
|
||||||
|
readonly configFile: string;
|
||||||
|
readonly invocationKeyringRootDirectory: string;
|
||||||
|
readonly resultKeyringRootDirectory: string;
|
||||||
|
readonly outputKeyringRootDirectory: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CreateProductionClusterCopilotFailureDiagnosisOptions {
|
||||||
|
readonly pool: QingLongPostgresPool;
|
||||||
|
readonly gateway: ActiveModelGatewayCapability;
|
||||||
|
readonly prepared: PreparedClusterCopilotFailureDiagnosisProjection;
|
||||||
|
readonly successfulCompletion: CopilotFailureDiagnosisModelCompletionCoordinator;
|
||||||
|
readonly artifactStore: ClusterRemoteWorkerArtifactStore;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PreparedClusterCopilotFailureDiagnosisProjection {
|
||||||
|
readonly config: Readonly<ClusterCopilotFailureDiagnosisConfig>;
|
||||||
|
readonly invocationKeys: Awaited<
|
||||||
|
ReturnType<typeof createClusterToolInvocationProjectedKeyring>
|
||||||
|
>;
|
||||||
|
readonly resultKeys: Awaited<
|
||||||
|
ReturnType<typeof createClusterToolResultProjectedKeyring>
|
||||||
|
>;
|
||||||
|
readonly outputKeys: Awaited<
|
||||||
|
ReturnType<typeof createClusterCopilotFailureDiagnosisOutputProjectedKeyring>
|
||||||
|
>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ClusterCopilotFailureDiagnosisCompositionError extends Error {
|
||||||
|
readonly code = 'QL3_CLUSTER_COPILOT_FAILURE_DIAGNOSIS_COMPOSITION_INVALID';
|
||||||
|
|
||||||
|
constructor(message: string, options?: ErrorOptions) {
|
||||||
|
super(`Cluster Copilot failure diagnosis composition is invalid: ${message}`, options);
|
||||||
|
this.name = 'ClusterCopilotFailureDiagnosisCompositionError';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function invalid(message: string, cause?: unknown): never {
|
||||||
|
throw new ClusterCopilotFailureDiagnosisCompositionError(message, {
|
||||||
|
cause: cause instanceof Error ? cause : undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function record(value: unknown, label: string): Record<string, unknown> {
|
||||||
|
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||||
|
return invalid(`${label} must be an object`);
|
||||||
|
}
|
||||||
|
return value as Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function exactKeys(
|
||||||
|
value: Readonly<Record<string, unknown>>,
|
||||||
|
expected: readonly string[],
|
||||||
|
label: string,
|
||||||
|
): void {
|
||||||
|
const actual = Object.keys(value).sort();
|
||||||
|
const canonical = [...expected].sort();
|
||||||
|
if (
|
||||||
|
actual.length !== canonical.length ||
|
||||||
|
actual.some((key, index) => key !== canonical[index])
|
||||||
|
) {
|
||||||
|
return invalid(`${label} shape is invalid`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function identity(value: unknown, label: string): string {
|
||||||
|
if (typeof value !== 'string' || !IDENTITY.test(value)) {
|
||||||
|
return invalid(`${label} is invalid`);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function integer(
|
||||||
|
value: unknown,
|
||||||
|
minimum: number,
|
||||||
|
maximum: number,
|
||||||
|
label: string,
|
||||||
|
): number {
|
||||||
|
if (
|
||||||
|
typeof value !== 'number' ||
|
||||||
|
!Number.isSafeInteger(value) ||
|
||||||
|
value < minimum ||
|
||||||
|
value > maximum
|
||||||
|
) {
|
||||||
|
return invalid(`${label} is invalid`);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeClusterCopilotFailureDiagnosisConfig(
|
||||||
|
value: unknown,
|
||||||
|
): Readonly<ClusterCopilotFailureDiagnosisConfig> {
|
||||||
|
const candidate = record(value, 'configuration');
|
||||||
|
exactKeys(
|
||||||
|
candidate,
|
||||||
|
[
|
||||||
|
'egressPolicy',
|
||||||
|
'executionTimeoutMs',
|
||||||
|
'maxOutputTokens',
|
||||||
|
'model',
|
||||||
|
'modelBoundary',
|
||||||
|
'provider',
|
||||||
|
'responseLanguage',
|
||||||
|
'schema',
|
||||||
|
],
|
||||||
|
'configuration',
|
||||||
|
);
|
||||||
|
if (candidate.schema !== CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA) {
|
||||||
|
return invalid('schema is invalid');
|
||||||
|
}
|
||||||
|
if (!MODEL_BOUNDARIES.includes(candidate.modelBoundary as never)) {
|
||||||
|
return invalid('model boundary is invalid');
|
||||||
|
}
|
||||||
|
if (!RESPONSE_LANGUAGES.includes(candidate.responseLanguage as never)) {
|
||||||
|
return invalid('response language is invalid');
|
||||||
|
}
|
||||||
|
const egress = record(candidate.egressPolicy, 'egress policy');
|
||||||
|
exactKeys(
|
||||||
|
egress,
|
||||||
|
[
|
||||||
|
'maxInputBytes',
|
||||||
|
'maxOutputTokens',
|
||||||
|
'potentiallySensitiveDataBoundaries',
|
||||||
|
'revision',
|
||||||
|
'schema',
|
||||||
|
],
|
||||||
|
'egress policy',
|
||||||
|
);
|
||||||
|
if (egress.schema !== EGRESS_SCHEMA) return invalid('egress schema is invalid');
|
||||||
|
const selected = egress.potentiallySensitiveDataBoundaries;
|
||||||
|
if (
|
||||||
|
!Array.isArray(selected) ||
|
||||||
|
selected.length < 1 ||
|
||||||
|
selected.length > MODEL_BOUNDARIES.length ||
|
||||||
|
selected.some((entry) => !MODEL_BOUNDARIES.includes(entry as never)) ||
|
||||||
|
new Set(selected).size !== selected.length ||
|
||||||
|
MODEL_BOUNDARIES.filter((entry) => selected.includes(entry)).some(
|
||||||
|
(entry, index) => entry !== selected[index],
|
||||||
|
) ||
|
||||||
|
!selected.includes(candidate.modelBoundary)
|
||||||
|
) {
|
||||||
|
return invalid('egress model boundaries are invalid');
|
||||||
|
}
|
||||||
|
const egressMaxOutputTokens = integer(
|
||||||
|
egress.maxOutputTokens,
|
||||||
|
1,
|
||||||
|
4_096,
|
||||||
|
'egress max output tokens',
|
||||||
|
);
|
||||||
|
const maxOutputTokens = integer(
|
||||||
|
candidate.maxOutputTokens,
|
||||||
|
1,
|
||||||
|
egressMaxOutputTokens,
|
||||||
|
'max output tokens',
|
||||||
|
);
|
||||||
|
return Object.freeze({
|
||||||
|
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA,
|
||||||
|
provider: identity(candidate.provider, 'provider'),
|
||||||
|
model: identity(candidate.model, 'model'),
|
||||||
|
modelBoundary: candidate.modelBoundary as 'on_device' | 'external',
|
||||||
|
responseLanguage: candidate.responseLanguage as 'en' | 'zh-CN',
|
||||||
|
maxOutputTokens,
|
||||||
|
executionTimeoutMs: integer(
|
||||||
|
candidate.executionTimeoutMs,
|
||||||
|
1,
|
||||||
|
5 * 60_000,
|
||||||
|
'execution timeout',
|
||||||
|
),
|
||||||
|
egressPolicy: Object.freeze({
|
||||||
|
schema: EGRESS_SCHEMA,
|
||||||
|
revision: identity(egress.revision, 'egress revision'),
|
||||||
|
potentiallySensitiveDataBoundaries: Object.freeze([...selected]) as (
|
||||||
|
| 'on_device'
|
||||||
|
| 'external'
|
||||||
|
)[],
|
||||||
|
maxInputBytes: integer(
|
||||||
|
egress.maxInputBytes,
|
||||||
|
1,
|
||||||
|
64 * 1024,
|
||||||
|
'egress max input bytes',
|
||||||
|
),
|
||||||
|
maxOutputTokens: egressMaxOutputTokens,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function canonicalClusterCopilotFailureDiagnosisConfig(
|
||||||
|
value: unknown,
|
||||||
|
): Buffer {
|
||||||
|
return Buffer.from(
|
||||||
|
`${JSON.stringify(normalizeClusterCopilotFailureDiagnosisConfig(value))}\n`,
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function loadClusterCopilotFailureDiagnosisConfig(
|
||||||
|
configFile: string,
|
||||||
|
): Promise<Readonly<ClusterCopilotFailureDiagnosisConfig>> {
|
||||||
|
let bytes: Buffer | undefined;
|
||||||
|
let canonical: Buffer | undefined;
|
||||||
|
try {
|
||||||
|
const reader = new PrivateProjectedFileReader({
|
||||||
|
rootDirectory: dirname(configFile),
|
||||||
|
minimumBytes: 1,
|
||||||
|
maximumBytes: MAX_CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_BYTES,
|
||||||
|
access: 'read_only_keyring',
|
||||||
|
});
|
||||||
|
bytes = await reader.read(basename(configFile));
|
||||||
|
const parsed = JSON.parse(bytes.toString('utf8')) as unknown;
|
||||||
|
const config = normalizeClusterCopilotFailureDiagnosisConfig(parsed);
|
||||||
|
canonical = canonicalClusterCopilotFailureDiagnosisConfig(config);
|
||||||
|
if (!canonical.equals(bytes)) return invalid('file is not canonical');
|
||||||
|
return config;
|
||||||
|
} catch (cause) {
|
||||||
|
return cause instanceof ClusterCopilotFailureDiagnosisCompositionError
|
||||||
|
? invalid(cause.message, cause)
|
||||||
|
: invalid('projected configuration is unavailable', cause);
|
||||||
|
} finally {
|
||||||
|
bytes?.fill(0);
|
||||||
|
canonical?.fill(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function modelIntent(
|
||||||
|
config: Readonly<ClusterCopilotFailureDiagnosisConfig>,
|
||||||
|
): Readonly<PrepareCopilotFailureDiagnosisModelIntent> {
|
||||||
|
return Object.freeze({
|
||||||
|
provider: config.provider,
|
||||||
|
model: config.model,
|
||||||
|
modelBoundary: config.modelBoundary,
|
||||||
|
responseLanguage: config.responseLanguage,
|
||||||
|
maxOutputTokens: config.maxOutputTokens,
|
||||||
|
egressPolicy: config.egressPolicy,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function prepareProductionClusterCopilotFailureDiagnosisProjection(
|
||||||
|
projection: ClusterCopilotFailureDiagnosisProjection,
|
||||||
|
): Promise<Readonly<PreparedClusterCopilotFailureDiagnosisProjection>> {
|
||||||
|
if (!projection || typeof projection !== 'object' || Array.isArray(projection)) {
|
||||||
|
return invalid('projection is invalid');
|
||||||
|
}
|
||||||
|
const [config, invocationKeys, resultKeys, outputKeys] = await Promise.all([
|
||||||
|
loadClusterCopilotFailureDiagnosisConfig(projection.configFile),
|
||||||
|
createClusterToolInvocationProjectedKeyring({
|
||||||
|
rootDirectory: projection.invocationKeyringRootDirectory,
|
||||||
|
}),
|
||||||
|
createClusterToolResultProjectedKeyring({
|
||||||
|
rootDirectory: projection.resultKeyringRootDirectory,
|
||||||
|
}),
|
||||||
|
createClusterCopilotFailureDiagnosisOutputProjectedKeyring({
|
||||||
|
rootDirectory: projection.outputKeyringRootDirectory,
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
return Object.freeze({ config, invocationKeys, resultKeys, outputKeys });
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createProductionClusterCopilotFailureDiagnosis(
|
||||||
|
options: CreateProductionClusterCopilotFailureDiagnosisOptions,
|
||||||
|
): Promise<Readonly<CopilotFailureDiagnosisApplicationService>> {
|
||||||
|
if (
|
||||||
|
!options ||
|
||||||
|
typeof options !== 'object' ||
|
||||||
|
Array.isArray(options) ||
|
||||||
|
typeof options.pool?.query !== 'function' ||
|
||||||
|
typeof options.pool?.connect !== 'function' ||
|
||||||
|
typeof options.gateway?.generate !== 'function' ||
|
||||||
|
typeof options.gateway?.supportsSuccessfulCompletionSink !== 'function' ||
|
||||||
|
typeof options.successfulCompletion?.begin !== 'function' ||
|
||||||
|
typeof options.successfulCompletion?.record !== 'function' ||
|
||||||
|
!options.prepared ||
|
||||||
|
typeof options.artifactStore?.readLogRange !== 'function'
|
||||||
|
) {
|
||||||
|
return invalid('dependencies are unavailable');
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
!options.gateway.supportsSuccessfulCompletionSink(
|
||||||
|
options.successfulCompletion,
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
return invalid('shared Model completion authority is unavailable');
|
||||||
|
}
|
||||||
|
const { config, invocationKeys, resultKeys } = options.prepared;
|
||||||
|
const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository(
|
||||||
|
options.pool,
|
||||||
|
);
|
||||||
|
const snapshots = new PostgresProjectToolDefinitionSnapshotRepository(
|
||||||
|
options.pool,
|
||||||
|
);
|
||||||
|
const runs = new PostgresRunRepository(options.pool);
|
||||||
|
const artifacts = new PostgresToolInvocationArtifactRepository(options.pool);
|
||||||
|
const stepRuns = new PostgresStepRunRepository(options.pool);
|
||||||
|
const barriers = new PostgresToolExecutionStartBarrierRepository(options.pool);
|
||||||
|
const completions = new PostgresToolExecutionCompletionRepository(options.pool);
|
||||||
|
const failureCompletions =
|
||||||
|
new PostgresToolExecutionFailureCompletionRepository(options.pool);
|
||||||
|
const resultKeyCatalog = new PostgresToolResultKeyCatalogReader(options.pool);
|
||||||
|
const resultRekeys = new PostgresToolResultRekeyReader(options.pool);
|
||||||
|
const unlocks = new PostgresCopilotFailureDiagnosisToolUnlockRepository(
|
||||||
|
options.pool,
|
||||||
|
);
|
||||||
|
const models = new PostgresCopilotFailureDiagnosisModelRepository(options.pool);
|
||||||
|
const logReader = new RunAttemptLogReadService(
|
||||||
|
runs,
|
||||||
|
Object.freeze({
|
||||||
|
read: options.artifactStore.readLogRange.bind(options.artifactStore),
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
executorType: 'remote_worker',
|
||||||
|
artifactIdPattern: /^wlog-[a-f0-9]{30}$/,
|
||||||
|
maximumReadBytes: 256 * 1024,
|
||||||
|
activeMissingIsPending: true,
|
||||||
|
},
|
||||||
|
new PostgresRunAttemptLogRetentionClaimRepository(options.pool),
|
||||||
|
);
|
||||||
|
const logs: RunAttemptLogReadPort = Object.freeze({
|
||||||
|
read: logReader.read.bind(logReader),
|
||||||
|
});
|
||||||
|
const successfulCompletion = options.successfulCompletion;
|
||||||
|
const toolResults: CopilotFailureDiagnosisToolResultReader = Object.freeze({
|
||||||
|
async open(requestId: string, startId: string) {
|
||||||
|
const plan = await admissions.findPlanByRequestId(requestId);
|
||||||
|
if (!plan) return invalid('diagnosis plan is unavailable');
|
||||||
|
const snapshot = await snapshots.findCurrent(plan.projectId);
|
||||||
|
if (!snapshot) return invalid('Tool snapshot is unavailable');
|
||||||
|
const authority = restoreCopilotFailureDiagnosisTrustedToolAuthority(
|
||||||
|
plan,
|
||||||
|
snapshot.snapshot,
|
||||||
|
);
|
||||||
|
const definitions = authority.bindings.definitionRegistry();
|
||||||
|
const adapters = new TrustedToolExecutionAdapterRegistry(
|
||||||
|
authority.bindings,
|
||||||
|
[
|
||||||
|
new BuiltInRunLogExcerptToolAdapter(
|
||||||
|
authority.binding,
|
||||||
|
'cluster-control',
|
||||||
|
definitions,
|
||||||
|
logs,
|
||||||
|
),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
return openTrustedToolSuccessCompletion(startId, {
|
||||||
|
completions,
|
||||||
|
barriers,
|
||||||
|
resultKeyCatalog,
|
||||||
|
resultRekeys,
|
||||||
|
resultKeys,
|
||||||
|
adapters,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const policy = new ProjectPolicyEngine(
|
||||||
|
new PostgresProjectPolicyRepository(options.pool),
|
||||||
|
);
|
||||||
|
const tool = Object.freeze({
|
||||||
|
admissions,
|
||||||
|
snapshots,
|
||||||
|
artifacts,
|
||||||
|
invocationKeys,
|
||||||
|
resultKeys,
|
||||||
|
stepRuns,
|
||||||
|
runs,
|
||||||
|
barriers,
|
||||||
|
completions,
|
||||||
|
failureCompletions,
|
||||||
|
resultKeyCatalog,
|
||||||
|
resultRekeys,
|
||||||
|
logs,
|
||||||
|
unlocks,
|
||||||
|
});
|
||||||
|
const model = Object.freeze({
|
||||||
|
admissions,
|
||||||
|
unlocks,
|
||||||
|
toolResults,
|
||||||
|
modelInvocations: models,
|
||||||
|
outputs: models,
|
||||||
|
gateway: options.gateway,
|
||||||
|
successfulCompletion,
|
||||||
|
finalizations: models,
|
||||||
|
});
|
||||||
|
const dependencies: CopilotFailureDiagnosisApplicationDependencies = {
|
||||||
|
admissions,
|
||||||
|
snapshots,
|
||||||
|
runs,
|
||||||
|
artifacts,
|
||||||
|
invocationKeys,
|
||||||
|
authorizer: policy,
|
||||||
|
tool,
|
||||||
|
model,
|
||||||
|
executeTool: executeCopilotFailureDiagnosisTool,
|
||||||
|
executeModel: executeCopilotFailureDiagnosisModel,
|
||||||
|
modelIntent: modelIntent(config),
|
||||||
|
executionTimeoutMs: config.executionTimeoutMs,
|
||||||
|
};
|
||||||
|
return new CopilotFailureDiagnosisApplicationService(dependencies);
|
||||||
|
}
|
||||||
@@ -13,6 +13,18 @@ const {
|
|||||||
canonicalPluginPackagePromptOutputKeyringManifest,
|
canonicalPluginPackagePromptOutputKeyringManifest,
|
||||||
PLUGIN_PACKAGE_PROMPT_OUTPUT_KEYRING_MANIFEST_SCHEMA,
|
PLUGIN_PACKAGE_PROMPT_OUTPUT_KEYRING_MANIFEST_SCHEMA,
|
||||||
} = require('@qinglong/ai/plugin-package-prompt-output-keyring-manifest');
|
} = require('@qinglong/ai/plugin-package-prompt-output-keyring-manifest');
|
||||||
|
const {
|
||||||
|
canonicalClusterToolInvocationKeyringManifest,
|
||||||
|
CLUSTER_TOOL_INVOCATION_KEYRING_MANIFEST_SCHEMA,
|
||||||
|
} = require('@qinglong/cluster-control/trusted-tool-invocation-keyring');
|
||||||
|
const {
|
||||||
|
canonicalClusterToolResultKeyringManifest,
|
||||||
|
CLUSTER_TOOL_RESULT_KEYRING_MANIFEST_SCHEMA,
|
||||||
|
} = require('@qinglong/cluster-control/trusted-tool-result-keyring');
|
||||||
|
const {
|
||||||
|
canonicalClusterCopilotFailureDiagnosisOutputKeyringManifest,
|
||||||
|
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_KEYRING_MANIFEST_SCHEMA,
|
||||||
|
} = require('@qinglong/cluster-control/failure-diagnosis-output-keyring');
|
||||||
|
|
||||||
function enabledEnvironment(overrides = {}) {
|
function enabledEnvironment(overrides = {}) {
|
||||||
return {
|
return {
|
||||||
@@ -74,6 +86,161 @@ test('AI config is fail-closed and bounded behind the explicit process flag', ()
|
|||||||
),
|
),
|
||||||
/QL3_CLUSTER_AI_PROMPT_OUTPUT_KEYRING_ROOT is invalid/,
|
/QL3_CLUSTER_AI_PROMPT_OUTPUT_KEYRING_ROOT is invalid/,
|
||||||
);
|
);
|
||||||
|
assert.throws(
|
||||||
|
() =>
|
||||||
|
loadProductionClusterAiConfig(
|
||||||
|
enabledEnvironment({ QL3_CLUSTER_AI_COPILOT_ENABLED: 'true' }),
|
||||||
|
),
|
||||||
|
/QL3_CLUSTER_AI_COPILOT_CONFIG_FILE is invalid/,
|
||||||
|
);
|
||||||
|
assert.deepEqual(
|
||||||
|
loadProductionClusterAiConfig(
|
||||||
|
enabledEnvironment({
|
||||||
|
QL3_CLUSTER_AI_COPILOT_ENABLED: 'true',
|
||||||
|
QL3_CLUSTER_AI_COPILOT_CONFIG_FILE: '/run/ql3/copilot/config.json',
|
||||||
|
QL3_CLUSTER_AI_COPILOT_INVOCATION_KEYRING_ROOT: '/run/ql3/invocation',
|
||||||
|
QL3_CLUSTER_AI_COPILOT_RESULT_KEYRING_ROOT: '/run/ql3/result',
|
||||||
|
QL3_CLUSTER_AI_COPILOT_OUTPUT_KEYRING_ROOT: '/run/ql3/output',
|
||||||
|
}),
|
||||||
|
).copilot,
|
||||||
|
{
|
||||||
|
configFile: '/run/ql3/copilot/config.json',
|
||||||
|
invocationKeyringRootDirectory: '/run/ql3/invocation',
|
||||||
|
resultKeyringRootDirectory: '/run/ql3/result',
|
||||||
|
outputKeyringRootDirectory: '/run/ql3/output',
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
async function projectedFile(root, name, bytes) {
|
||||||
|
await writeFile(join(root, name), bytes, { mode: 0o440 });
|
||||||
|
await chmod(join(root, name), 0o440);
|
||||||
|
}
|
||||||
|
|
||||||
|
test('Copilot composition is explicit, shares the Prompt gateway and exposes no route', async () => {
|
||||||
|
const secretRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-secret-'));
|
||||||
|
const configRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-config-'));
|
||||||
|
const invocationRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-invocation-'));
|
||||||
|
const resultRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-result-'));
|
||||||
|
const outputRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-output-'));
|
||||||
|
const key = Buffer.alloc(32, 0x55).toString('base64url');
|
||||||
|
const config = Buffer.from(`${JSON.stringify({
|
||||||
|
schema: 'qinglong/cluster-copilot-failure-diagnosis-config@v1',
|
||||||
|
provider: 'provider-primary',
|
||||||
|
model: 'model-diagnosis',
|
||||||
|
modelBoundary: 'external',
|
||||||
|
responseLanguage: 'zh-CN',
|
||||||
|
maxOutputTokens: 512,
|
||||||
|
executionTimeoutMs: 60_000,
|
||||||
|
egressPolicy: {
|
||||||
|
schema: 'qinglong/copilot-model-egress-policy@v1',
|
||||||
|
revision: 'cluster-copilot-v1',
|
||||||
|
potentiallySensitiveDataBoundaries: ['external'],
|
||||||
|
maxInputBytes: 64 * 1024,
|
||||||
|
maxOutputTokens: 1024,
|
||||||
|
},
|
||||||
|
})}\n`);
|
||||||
|
const invocation = canonicalClusterToolInvocationKeyringManifest({
|
||||||
|
schema: CLUSTER_TOOL_INVOCATION_KEYRING_MANIFEST_SCHEMA,
|
||||||
|
activeKeyId: 'invocation-key-1',
|
||||||
|
keys: { 'invocation-key-1': key },
|
||||||
|
});
|
||||||
|
const result = canonicalClusterToolResultKeyringManifest({
|
||||||
|
schema: CLUSTER_TOOL_RESULT_KEYRING_MANIFEST_SCHEMA,
|
||||||
|
keys: { 'result-key-1': key },
|
||||||
|
});
|
||||||
|
const output = canonicalClusterCopilotFailureDiagnosisOutputKeyringManifest({
|
||||||
|
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_KEYRING_MANIFEST_SCHEMA,
|
||||||
|
activeKeyId: 'output-key-1',
|
||||||
|
keys: { 'output-key-1': key },
|
||||||
|
});
|
||||||
|
const gateway = {
|
||||||
|
generate() {},
|
||||||
|
supportsSuccessfulCompletionSink(sink) {
|
||||||
|
return sink === registeredSink;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const fakePool = { query() {}, connect() {} };
|
||||||
|
const artifactStore = { put() {}, inspect() {}, readLogRange() {} };
|
||||||
|
const copilot = Object.freeze({ execute() {} });
|
||||||
|
let registeredSink;
|
||||||
|
let created;
|
||||||
|
let controlOptions;
|
||||||
|
try {
|
||||||
|
await Promise.all([
|
||||||
|
projectedFile(configRoot, 'config.json', config),
|
||||||
|
projectedFile(invocationRoot, 'keyring.json', invocation),
|
||||||
|
projectedFile(resultRoot, 'keyring.json', result),
|
||||||
|
projectedFile(outputRoot, 'keyring.json', output),
|
||||||
|
]);
|
||||||
|
const application = await startProductionClusterAiControlApplication({
|
||||||
|
control: {
|
||||||
|
config: controlConfig(),
|
||||||
|
workerIngress: {
|
||||||
|
config: { enabled: true },
|
||||||
|
artifactStore,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ai: {
|
||||||
|
enabled: true,
|
||||||
|
providerAuthorityFile: '/unused/providers.json',
|
||||||
|
secretRootDirectory: secretRoot,
|
||||||
|
copilot: {
|
||||||
|
configFile: join(configRoot, 'config.json'),
|
||||||
|
invocationKeyringRootDirectory: invocationRoot,
|
||||||
|
resultKeyringRootDirectory: resultRoot,
|
||||||
|
outputKeyringRootDirectory: outputRoot,
|
||||||
|
},
|
||||||
|
maxConcurrent: 1,
|
||||||
|
recoveryLimit: 1,
|
||||||
|
databaseMaxConnections: 1,
|
||||||
|
},
|
||||||
|
audit() {},
|
||||||
|
async openAiDatabase() {
|
||||||
|
return { pool: fakePool, async close() {} };
|
||||||
|
},
|
||||||
|
async bootstrapPrompt(options) {
|
||||||
|
await options.openDatabase();
|
||||||
|
registeredSink = options.createAdditionalSuccessfulCompletion({
|
||||||
|
async recordWithAtomicSuccess() {},
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
status: 'active', profile: 'cluster', readiness: {}, capability: gateway,
|
||||||
|
prompts: {}, promptCatalog: {}, promptExecutions: {},
|
||||||
|
promptExecutionInspections: {}, async stop() { return 'stopped'; },
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async createCopilot(options) {
|
||||||
|
created = options;
|
||||||
|
return copilot;
|
||||||
|
},
|
||||||
|
async startControl(options) {
|
||||||
|
controlOptions = options;
|
||||||
|
return {
|
||||||
|
status: 'active', address: { host: '127.0.0.1', port: 5800 },
|
||||||
|
evidence: {}, recovery: { safe: true, remaining: 0, failed: 0 },
|
||||||
|
unavailable: new Promise(() => {}), availabilityStatus() { return 'ready'; },
|
||||||
|
async stop() { return 'stopped'; },
|
||||||
|
};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(application.copilot, copilot);
|
||||||
|
assert.equal(created.pool, fakePool);
|
||||||
|
assert.equal(created.gateway, gateway);
|
||||||
|
assert.equal(created.successfulCompletion, registeredSink);
|
||||||
|
assert.equal(created.artifactStore, artifactStore);
|
||||||
|
assert.equal('copilot' in controlOptions, false);
|
||||||
|
assert.equal(await application.stop(), 'stopped');
|
||||||
|
} finally {
|
||||||
|
config.fill(0); invocation.fill(0); result.fill(0); output.fill(0);
|
||||||
|
await Promise.all([
|
||||||
|
rm(secretRoot, { recursive: true, force: true }),
|
||||||
|
rm(configRoot, { recursive: true, force: true }),
|
||||||
|
rm(invocationRoot, { recursive: true, force: true }),
|
||||||
|
rm(resultRoot, { recursive: true, force: true }),
|
||||||
|
rm(outputRoot, { recursive: true, force: true }),
|
||||||
|
]);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('explicit AI composition injects one reviewed Prompt capability and drains it after HTTP control', async () => {
|
test('explicit AI composition injects one reviewed Prompt capability and drains it after HTTP control', async () => {
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const { chmod, mkdtemp, rm, writeFile } = require('node:fs/promises');
|
||||||
|
const { tmpdir } = require('node:os');
|
||||||
|
const { join } = require('node:path');
|
||||||
|
const { test } = require('node:test');
|
||||||
|
|
||||||
|
const {
|
||||||
|
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA,
|
||||||
|
ClusterCopilotFailureDiagnosisCompositionError,
|
||||||
|
canonicalClusterCopilotFailureDiagnosisConfig,
|
||||||
|
loadClusterCopilotFailureDiagnosisConfig,
|
||||||
|
normalizeClusterCopilotFailureDiagnosisConfig,
|
||||||
|
} = require('@qinglong/cluster-control/copilot-production');
|
||||||
|
|
||||||
|
function config(overrides = {}) {
|
||||||
|
return {
|
||||||
|
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA,
|
||||||
|
provider: 'provider-primary',
|
||||||
|
model: 'model-diagnosis',
|
||||||
|
modelBoundary: 'external',
|
||||||
|
responseLanguage: 'zh-CN',
|
||||||
|
maxOutputTokens: 512,
|
||||||
|
executionTimeoutMs: 60_000,
|
||||||
|
egressPolicy: {
|
||||||
|
schema: 'qinglong/copilot-model-egress-policy@v1',
|
||||||
|
revision: 'cluster-copilot-v1',
|
||||||
|
potentiallySensitiveDataBoundaries: ['external'],
|
||||||
|
maxInputBytes: 64 * 1024,
|
||||||
|
maxOutputTokens: 1024,
|
||||||
|
},
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('normalizes one bounded deployment-owned Copilot model intent', () => {
|
||||||
|
const normalized = normalizeClusterCopilotFailureDiagnosisConfig(config());
|
||||||
|
assert.equal(normalized.provider, 'provider-primary');
|
||||||
|
assert.equal(normalized.executionTimeoutMs, 60_000);
|
||||||
|
assert.equal(Object.isFrozen(normalized), true);
|
||||||
|
assert.equal(Object.isFrozen(normalized.egressPolicy), true);
|
||||||
|
assert.throws(
|
||||||
|
() => normalizeClusterCopilotFailureDiagnosisConfig(config({ extra: true })),
|
||||||
|
ClusterCopilotFailureDiagnosisCompositionError,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => normalizeClusterCopilotFailureDiagnosisConfig(config({
|
||||||
|
modelBoundary: 'on_device',
|
||||||
|
})),
|
||||||
|
/egress model boundaries/,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => normalizeClusterCopilotFailureDiagnosisConfig(config({
|
||||||
|
executionTimeoutMs: 300_001,
|
||||||
|
})),
|
||||||
|
/execution timeout/,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('loads only canonical read-only projected Copilot configuration', async () => {
|
||||||
|
const root = await mkdtemp(join(tmpdir(), 'ql3-copilot-config-test-'));
|
||||||
|
const file = join(root, 'config.json');
|
||||||
|
const canonical = canonicalClusterCopilotFailureDiagnosisConfig(config());
|
||||||
|
try {
|
||||||
|
await writeFile(file, canonical, { mode: 0o440 });
|
||||||
|
await chmod(file, 0o440);
|
||||||
|
assert.deepEqual(
|
||||||
|
await loadClusterCopilotFailureDiagnosisConfig(file),
|
||||||
|
normalizeClusterCopilotFailureDiagnosisConfig(config()),
|
||||||
|
);
|
||||||
|
|
||||||
|
await chmod(file, 0o640);
|
||||||
|
await writeFile(file, Buffer.from(`${JSON.stringify(config(), null, 2)}\n`), {
|
||||||
|
mode: 0o440,
|
||||||
|
});
|
||||||
|
await chmod(file, 0o440);
|
||||||
|
await assert.rejects(
|
||||||
|
loadClusterCopilotFailureDiagnosisConfig(file),
|
||||||
|
/not canonical/,
|
||||||
|
);
|
||||||
|
|
||||||
|
await chmod(file, 0o640);
|
||||||
|
await writeFile(file, canonical, { mode: 0o640 });
|
||||||
|
await chmod(file, 0o640);
|
||||||
|
await assert.rejects(
|
||||||
|
loadClusterCopilotFailureDiagnosisConfig(file),
|
||||||
|
ClusterCopilotFailureDiagnosisCompositionError,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
canonical.fill(0);
|
||||||
|
await rm(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -1163,6 +1163,9 @@ function auditSourceImports(root, packagePath, findings) {
|
|||||||
const clusterAiImports = Object.freeze({
|
const clusterAiImports = Object.freeze({
|
||||||
'src/aiCli.ts': Object.freeze(['@qinglong/ai/profile']),
|
'src/aiCli.ts': Object.freeze(['@qinglong/ai/profile']),
|
||||||
'src/application-runtime/aiProductionApplication.ts': Object.freeze([
|
'src/application-runtime/aiProductionApplication.ts': Object.freeze([
|
||||||
|
'@qinglong/ai/durable-model-invocation',
|
||||||
|
'@qinglong/ai/failure-diagnosis-application',
|
||||||
|
'@qinglong/ai/failure-diagnosis-model-execution',
|
||||||
'@qinglong/ai/plugin-package-prompt-output-projected-keyring',
|
'@qinglong/ai/plugin-package-prompt-output-projected-keyring',
|
||||||
'@qinglong/ai/postgres-model-provider-credential-storage',
|
'@qinglong/ai/postgres-model-provider-credential-storage',
|
||||||
'@qinglong/ai/postgres-plugin-package-prompt-application',
|
'@qinglong/ai/postgres-plugin-package-prompt-application',
|
||||||
@@ -1171,6 +1174,17 @@ function auditSourceImports(root, packagePath, findings) {
|
|||||||
'@qinglong/ai/projected-model-provider-secret-material',
|
'@qinglong/ai/projected-model-provider-secret-material',
|
||||||
'@qinglong/ai/provider-credential',
|
'@qinglong/ai/provider-credential',
|
||||||
]),
|
]),
|
||||||
|
'src/application-runtime/copilot/failureDiagnosisComposition.ts':
|
||||||
|
Object.freeze([
|
||||||
|
'@qinglong/ai/failure-diagnosis-application',
|
||||||
|
'@qinglong/ai/failure-diagnosis-execution-admission',
|
||||||
|
'@qinglong/ai/failure-diagnosis-model-execution',
|
||||||
|
'@qinglong/ai/failure-diagnosis-tool-execution',
|
||||||
|
'@qinglong/ai/postgres-failure-diagnosis-admission-storage',
|
||||||
|
'@qinglong/ai/postgres-failure-diagnosis-model-execution-storage',
|
||||||
|
'@qinglong/ai/postgres-failure-diagnosis-tool-execution-storage',
|
||||||
|
'@qinglong/ai/profile',
|
||||||
|
]),
|
||||||
});
|
});
|
||||||
const sourceRelative = path.relative(packageDirectory, filePath);
|
const sourceRelative = path.relative(packageDirectory, filePath);
|
||||||
if (!clusterAiImports[sourceRelative]?.includes(specifier)) {
|
if (!clusterAiImports[sourceRelative]?.includes(specifier)) {
|
||||||
|
|||||||
@@ -1311,6 +1311,28 @@ function assertClusterAiComponent(readFile, root, findings) {
|
|||||||
'utf8',
|
'utf8',
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
const copilotComponentDirectory = path.join(
|
||||||
|
root,
|
||||||
|
'deploy/kubernetes/ql3-cluster/components/cluster-ai-copilot',
|
||||||
|
);
|
||||||
|
const copilotComponent = yaml.load(
|
||||||
|
readFile(path.join(copilotComponentDirectory, 'kustomization.yaml'), 'utf8'),
|
||||||
|
);
|
||||||
|
const copilotPatch = yaml.load(
|
||||||
|
readFile(path.join(copilotComponentDirectory, 'deployment-patch.yaml'), 'utf8'),
|
||||||
|
);
|
||||||
|
const copilotConfig = yaml.load(
|
||||||
|
readFile(path.join(copilotComponentDirectory, 'copilot-configmap.yaml'), 'utf8'),
|
||||||
|
);
|
||||||
|
const copilotOverlay = yaml.load(
|
||||||
|
readFile(
|
||||||
|
path.join(
|
||||||
|
root,
|
||||||
|
'deploy/kubernetes/ql3-cluster/overlays/cluster-ai-copilot-example/kustomization.yaml',
|
||||||
|
),
|
||||||
|
'utf8',
|
||||||
|
),
|
||||||
|
);
|
||||||
if (
|
if (
|
||||||
component?.kind !== 'Component' ||
|
component?.kind !== 'Component' ||
|
||||||
JSON.stringify(component?.resources) !==
|
JSON.stringify(component?.resources) !==
|
||||||
@@ -1326,6 +1348,156 @@ function assertClusterAiComponent(readFile, root, findings) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const copilotPod = objectAt(copilotPatch, ['spec', 'template', 'spec']);
|
||||||
|
const copilotContainer = namedEntry(
|
||||||
|
copilotPod?.containers,
|
||||||
|
'cluster-control',
|
||||||
|
);
|
||||||
|
const copilotEnv = environmentByName(copilotContainer);
|
||||||
|
const copilotProjections = [
|
||||||
|
[
|
||||||
|
'cluster-ai-copilot-config',
|
||||||
|
'/var/run/qinglong3/ai/copilot-config',
|
||||||
|
'configMap',
|
||||||
|
'ql3-cluster-ai-copilot',
|
||||||
|
'config.json',
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'cluster-ai-copilot-invocation-keyring',
|
||||||
|
'/var/run/secrets/qinglong3/ai/copilot-invocation-keyring',
|
||||||
|
'secret',
|
||||||
|
'ql3-cluster-ai-copilot-invocation-keyring',
|
||||||
|
'keyring.json',
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'cluster-ai-copilot-result-keyring',
|
||||||
|
'/var/run/secrets/qinglong3/ai/copilot-result-keyring',
|
||||||
|
'secret',
|
||||||
|
'ql3-cluster-ai-copilot-result-keyring',
|
||||||
|
'keyring.json',
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'cluster-ai-copilot-output-keyring',
|
||||||
|
'/var/run/secrets/qinglong3/ai/copilot-output-keyring',
|
||||||
|
'secret',
|
||||||
|
'ql3-cluster-ai-copilot-output-keyring',
|
||||||
|
'keyring.json',
|
||||||
|
],
|
||||||
|
];
|
||||||
|
const copilotProjectionInvalid = copilotProjections.some(
|
||||||
|
([name, mountPath, kind, authorityName, key]) => {
|
||||||
|
const mount = namedEntry(copilotContainer?.volumeMounts, name);
|
||||||
|
const volume = namedEntry(copilotPod?.volumes, name);
|
||||||
|
const projection = volume?.[kind];
|
||||||
|
return (
|
||||||
|
mount?.mountPath !== mountPath ||
|
||||||
|
mount?.readOnly !== true ||
|
||||||
|
projection?.name !== authorityName &&
|
||||||
|
projection?.secretName !== authorityName ||
|
||||||
|
projection?.defaultMode !== 0o440 ||
|
||||||
|
projection?.optional === true ||
|
||||||
|
JSON.stringify(projection?.items) !==
|
||||||
|
JSON.stringify([{ key, path: key }])
|
||||||
|
);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
copilotComponent?.kind !== 'Component' ||
|
||||||
|
JSON.stringify(copilotComponent?.resources) !==
|
||||||
|
JSON.stringify(['copilot-configmap.yaml']) ||
|
||||||
|
JSON.stringify(copilotComponent?.patches) !==
|
||||||
|
JSON.stringify([{ path: 'deployment-patch.yaml' }]) ||
|
||||||
|
copilotPatch?.kind !== 'Deployment' ||
|
||||||
|
copilotPatch?.metadata?.name !== 'ql3-cluster-control' ||
|
||||||
|
copilotPod?.serviceAccountName !== undefined ||
|
||||||
|
copilotPod?.automountServiceAccountToken !== undefined ||
|
||||||
|
copilotPod?.containers?.length !== 1 ||
|
||||||
|
copilotContainer?.image !== undefined ||
|
||||||
|
copilotContainer?.env?.length !== 5 ||
|
||||||
|
copilotContainer?.volumeMounts?.length !== 4 ||
|
||||||
|
copilotPod?.volumes?.length !== 4 ||
|
||||||
|
copilotProjectionInvalid
|
||||||
|
) {
|
||||||
|
findings.push(
|
||||||
|
finding(
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_PROJECTION',
|
||||||
|
'Cluster Copilot must remain an explicit component with one canonical ConfigMap and three independent required read-only 0440 keyring projections',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (const [name, expected] of [
|
||||||
|
['QL3_CLUSTER_AI_COPILOT_ENABLED', 'true'],
|
||||||
|
[
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_CONFIG_FILE',
|
||||||
|
'/var/run/qinglong3/ai/copilot-config/config.json',
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_INVOCATION_KEYRING_ROOT',
|
||||||
|
'/var/run/secrets/qinglong3/ai/copilot-invocation-keyring',
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_RESULT_KEYRING_ROOT',
|
||||||
|
'/var/run/secrets/qinglong3/ai/copilot-result-keyring',
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_OUTPUT_KEYRING_ROOT',
|
||||||
|
'/var/run/secrets/qinglong3/ai/copilot-output-keyring',
|
||||||
|
],
|
||||||
|
]) {
|
||||||
|
if (copilotEnv.get(name)?.value !== expected) {
|
||||||
|
findings.push(
|
||||||
|
finding(
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_ENVIRONMENT',
|
||||||
|
`${name} must be fixed to the reviewed projected authority ${expected}`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const copilotConfigText = copilotConfig?.data?.['config.json'];
|
||||||
|
let parsedCopilotConfig;
|
||||||
|
try {
|
||||||
|
parsedCopilotConfig = JSON.parse(copilotConfigText);
|
||||||
|
} catch {
|
||||||
|
parsedCopilotConfig = undefined;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
copilotConfig?.kind !== 'ConfigMap' ||
|
||||||
|
copilotConfig?.metadata?.name !== 'ql3-cluster-ai-copilot' ||
|
||||||
|
typeof copilotConfigText !== 'string' ||
|
||||||
|
`${JSON.stringify(parsedCopilotConfig)}\n` !== copilotConfigText ||
|
||||||
|
parsedCopilotConfig?.schema !==
|
||||||
|
'qinglong/cluster-copilot-failure-diagnosis-config@v1' ||
|
||||||
|
parsedCopilotConfig?.egressPolicy?.schema !==
|
||||||
|
'qinglong/copilot-model-egress-policy@v1'
|
||||||
|
) {
|
||||||
|
findings.push(
|
||||||
|
finding(
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_CONFIG',
|
||||||
|
'Cluster Copilot configuration must remain canonical, bounded and explicitly versioned',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const copilotOverlayImage = copilotOverlay?.images?.[0];
|
||||||
|
if (
|
||||||
|
JSON.stringify(copilotOverlay?.components) !==
|
||||||
|
JSON.stringify([
|
||||||
|
'../../components/cluster-ai',
|
||||||
|
'../../components/cluster-ai-copilot',
|
||||||
|
]) ||
|
||||||
|
copilotOverlayImage?.name !== 'qinglong3-cluster-control-ai' ||
|
||||||
|
copilotOverlayImage?.newName !==
|
||||||
|
'registry.example.com/qinglong/qinglong3-cluster-control-ai' ||
|
||||||
|
!/^sha256:[0-9a-f]{64}$/.test(copilotOverlayImage?.digest ?? '') ||
|
||||||
|
'newTag' in (copilotOverlayImage ?? {})
|
||||||
|
) {
|
||||||
|
findings.push(
|
||||||
|
finding(
|
||||||
|
'QL3_CLUSTER_AI_COPILOT_OVERLAY',
|
||||||
|
'The Copilot overlay must compose explicit AI and Copilot components and independently pin the Cluster AI image digest',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const promptOutputPod = objectAt(promptOutputPatch, [
|
const promptOutputPod = objectAt(promptOutputPatch, [
|
||||||
'spec',
|
'spec',
|
||||||
'template',
|
'template',
|
||||||
@@ -1564,7 +1736,11 @@ function assertClusterAiComponent(readFile, root, findings) {
|
|||||||
) ||
|
) ||
|
||||||
namedEntry(basePod?.volumes, 'cluster-ai-provider-authority') ||
|
namedEntry(basePod?.volumes, 'cluster-ai-provider-authority') ||
|
||||||
namedEntry(basePod?.volumes, 'cluster-ai-provider-secrets') ||
|
namedEntry(basePod?.volumes, 'cluster-ai-provider-secrets') ||
|
||||||
namedEntry(basePod?.volumes, 'cluster-ai-prompt-output-keyring')
|
namedEntry(basePod?.volumes, 'cluster-ai-prompt-output-keyring') ||
|
||||||
|
[...copilotProjections].some(([name]) =>
|
||||||
|
namedEntry(baseContainer?.volumeMounts, name) ||
|
||||||
|
namedEntry(basePod?.volumes, name),
|
||||||
|
)
|
||||||
) {
|
) {
|
||||||
findings.push(
|
findings.push(
|
||||||
finding(
|
finding(
|
||||||
|
|||||||
@@ -268,10 +268,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', (
|
|||||||
{
|
{
|
||||||
path: 'packages/ql3-ai',
|
path: 'packages/ql3-ai',
|
||||||
name: '@qinglong/ai',
|
name: '@qinglong/ai',
|
||||||
sourceFiles: 183,
|
sourceFiles: 187,
|
||||||
rootSourceFiles: 1,
|
rootSourceFiles: 1,
|
||||||
rootSourceLines: 16,
|
rootSourceLines: 16,
|
||||||
nestedSourceFiles: 182,
|
nestedSourceFiles: 186,
|
||||||
rootSourceFileHardCap: 1,
|
rootSourceFileHardCap: 1,
|
||||||
rootSourceLineHardCap: 16,
|
rootSourceLineHardCap: 16,
|
||||||
rootSourceFileRoles: {
|
rootSourceFileRoles: {
|
||||||
@@ -385,10 +385,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', (
|
|||||||
rootSourceFileRoles: clusterControl.rootSourceFileRoles,
|
rootSourceFileRoles: clusterControl.rootSourceFileRoles,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
sourceFiles: 58,
|
sourceFiles: 59,
|
||||||
rootSourceFiles: 2,
|
rootSourceFiles: 2,
|
||||||
rootSourceLines: 195,
|
rootSourceLines: 195,
|
||||||
nestedSourceFiles: 56,
|
nestedSourceFiles: 57,
|
||||||
rootSourceFileRoles: {
|
rootSourceFileRoles: {
|
||||||
'aiCli.ts': 'binary_entry',
|
'aiCli.ts': 'binary_entry',
|
||||||
'cli.ts': 'binary_entry',
|
'cli.ts': 'binary_entry',
|
||||||
|
|||||||
Reference in New Issue
Block a user