From 774af4a4c957e3317c9b586cc685dfe4f46d232d Mon Sep 17 00:00:00 2001 From: whyour Date: Sat, 15 Aug 2026 23:45:13 +0800 Subject: [PATCH] feat(ql3): expose fenced copilot diagnosis API --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 1 + ...f-cluster-copilot-failure-diagnosis-api.md | 109 +++++++ packages/ql3-cluster-control/package.json | 5 + .../aiProductionApplication.ts | 7 + .../productionApplication.ts | 25 +- .../failureDiagnosisRoute.ts | 297 ++++++++++++++++++ .../test/aiProductionApplication.test.cjs | 4 +- .../copilotFailureDiagnosisRoute.test.cjs | 269 ++++++++++++++++ .../test/productionApplication.test.cjs | 129 ++++++++ test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 10 files changed, 845 insertions(+), 5 deletions(-) create mode 100644 docs/adr/ADR-0413-default-off-cluster-copilot-failure-diagnosis-api.md create mode 100644 packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisRoute.ts create mode 100644 packages/ql3-cluster-control/test/copilotFailureDiagnosisRoute.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 842c15d9..526f7360 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,7 @@ 最新增量证据（2026-08-15）： +- D-321/ADR-0413（已接受）：Cluster Copilot failure diagnosis 已开放唯一、默认关闭的产品写入口 `POST /api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses`。该 route 只在显式 AI 进程且 `QL3_CLUSTER_AI_COPILOT_ENABLED=true` 时注入既有 Cluster Control route registry，完整复用同一认证器、`model.invoke` Project Policy、fail-closed 同步安全审计、HTTP/TLS body/response/concurrency 上限和生命周期；内部只读 Tool 仍独立复验其 exact `tool.call:*` Policy。耐久幂等身份直接绑定 `x-request-id`，body 只允许 schema 与 `traceId`，Project/source Run/principal 来自 canonical path 和认证，Attempt、日志范围、Tool、provider/model、预算、deadline、Policy fence、reason/outcome 均由服务端从数据库与只读配置派生。响应只投影 created/existing、source/diagnosis Run、终态 stage/reason/outcome 和加密输出 Artifact id/digest，不返回日志、Tool/Model plaintext、prompt、模型信息、密钥或内部异常。普通 Cluster Control、Edge、Standalone 和未启用 Copilot 的 AI 进程没有 route，也没有新增 package、进程、监听器、Pool/连接、timer、watcher、队列、cache、Pod、Service 或 Kubernetes 权限；workspace 仍为 18 package、无单文件/浅平 package，新增源码位于既有嵌套 Copilot 领域目录。Cluster Control 250 pass/2 条件 skip，18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计和 14 档 Local artifact 全部通过；默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes。PostgreSQL 18.6 arm64 HA 137/137、timeline `1→2`，报告 SHA-256 为 `0a12b5c1102555823d43b5a93dd7868b98b194b491840a5242bab6fa2da26123`，离线审计零 finding。本 Gate 不改 migration/schema/role/SQL/HA 拓扑。下一 Gate 优先补同一 Policy 下的加密诊断输出读取与费用/取消可观测性；CLI/UI/MCP 只能复用该 API，不得直连 application capability。 - D-320/ADR-0412（已接受）：Cluster Copilot failure diagnosis 已补齐产品入口前的非成功收敛边界。新增的 pre-Model terminalizer 只从 durable Tool failure、受审日志 projection 或数据库观察到的 deadline/cancellation 派生封闭 reason，在 Model start 不存在时以一个 SERIALIZABLE 事务原子提交 StepRun mutations、RunEvents、父 Run 终态和 `pg-9021` append-only content-free receipt；Tool `failed|timed_out`、日志 `not_found|pending|missing|retired`、五秒 Tool budget 不足、deadline 和 cancellation 均可 exact replay。Model 已开始后的 `outcome_unknown` 继续要求强 User 显式 `fail|cancel|retry` resolution；Copilot finalizer 现在对 `fail|cancel` 精确校验 resolution mutation 的 resolved Step digest，既不伪造失败也不自动重试 Provider。实现复用既有 AI package、Cluster AI 进程、Pool 与 ledger，不增加 package、进程、连接、timer、watcher、队列、cache 或产品 route；workspace 仍为 18 package、无单文件/浅平 package，AI 192 个源码中 191 个位于嵌套领域目录。AI 完整测试 244 pass/3 条件 skip，18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计和 14 档 Local artifact 全部通过。PostgreSQL 18.6 arm64 HA 137/137、timeline `1→2`，覆盖日志不可用 pre-Model terminalization、unknown completion→人工 fail resolution→Run finalization 及晋升后零外部副作用 exact replay；报告 SHA-256 为 `6eaeb20615a62d153c5a69687344f41f31351c6ecf111cfb9cbafad115538c83`，离线审计零 finding。下一 Gate 才增加认证、Policy、audit 与 source fence 保护的 Cluster API，并由 CLI/UI/MCP 复用，仍不得建立旁路执行器。 - D-319/ADR-0411（已接受）：Cluster Copilot failure diagnosis 已在既有 `ql3-cluster-control-ai` 进程内完成默认关闭的 production composition。Prompt 与 Copilot 共享同一个 PostgreSQL AI Pool、Model Gateway、Provider client、恢复扫描、quota/pricing ledger 与 `maxConcurrent` 预算；有界 successful-completion router 只向声明 exact invocation 的 durable sink 分发，不复制 Gateway、连接或隐藏队列。application service 只接受 source Run/request identity，从数据库当前 Run、latest Attempt、Project Tool snapshot、Policy 和 canonical read-only egress config 派生计划，先 admission，再以 durable plan、historical key 与确定性 nonce 修复 admission→Artifact crash window，随后复用 Trusted Tool、Worker Artifact range reader、Model Gateway 及独立的 invocation/result/model-output 三域 keyring。能力保持 caller-driven，不增加 timer、watcher、队列、HTTP/CLI/UI/MCP route 或 Kubernetes API 权限；Kubernetes 独立可选 component 仅投影 config 与三个 0440 keyring。最终 AI 238 pass/3 条件 skip、Cluster Control 240 pass/2 条件 skip、18-package clean build/test 与 backend 1,207 pass/2 条件 skip/0 fail，四项架构审计、14 档 Local artifact 全部通过；workspace 仍为 18 package、无单文件/浅平 package，AI 187 个源码中 186 个、Cluster Control 59 个中 57 个位于嵌套领域目录。默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes，Edge/Standalone AI 为 3,064,454/3,064,544 bytes，证明 Cluster-only composition 未进入小设备闭包。PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`，报告 SHA-256 为 `981299b454dce5541e9596450b85816dc40559cba8dc42adf3d5fea571c3d3a6`；本 Gate 不改 migration/schema/role/SQL/HA 拓扑。下一 Gate 是 Tool failure、日志 missing/retired/pending、deadline/cancel 与 Model outcome-unknown 的 durable terminalization/recovery，完成前不开放产品入口。 - D-318/ADR-0410（已接受）：Cluster Copilot diagnosis Model output 获得独立的只读 projected key authority。canonical `qinglong/copilot-failure-diagnosis-output-projected-keyring@v1` manifest 只允许一个 active key 与最多 16 个 historical 32-byte key；每次 `active()`/`resolve()` 都重新执行 Cluster 私有投影文件的 direct-root、根内 atomic symlink、single-link、mode、dev/inode/size/mtime 与双 realpath fence，不使用 cache、watcher、timer 或 Kubernetes API。该 authority 以结构兼容的本地窄端口位于既有 `cluster-control/copilot/failure-diagnosis/`，只通过 `failure-diagnosis-output-keyring` subpath 发布，避免 Cluster Control 默认源码反向依赖 AI；它不复用 Prompt output、Tool invocation/result 或 Provider credential key domain，也不提前声称 Copilot 产品入口已可达。定向回归 13/13、Cluster Control 239 pass/2 条件 skip、backend 1,207 pass/2 条件 skip、18-package clean build/test、四项架构审计与 14 档 Local artifact 全部通过；workspace 仍无单文件/浅平 package，Cluster Control 58 个源码中 56 个位于嵌套领域目录。默认 Edge/Standalone 保持 2,589,890/2,589,968 bytes，Edge/Standalone AI 保持 3,061,009/3,061,099 bytes，证明 Cluster-only subpath 被裁掉。本 Gate 不改 migration/schema/role/SQL/连接/HA 拓扑，因此数据库基线继续引用 ADR-0409 的 PostgreSQL 18.6 arm64 HA 130/130、timeline `1→2`；下一 Gate 是默认关闭的完整 Cluster Copilot composition，随后补齐 Tool failure、日志 missing/retired、Model admission 前 deadline/cancel 与 outcome-unknown 的 durable terminalization/recovery。 diff --git a/docs/adr/ADR-0413-default-off-cluster-copilot-failure-diagnosis-api.md b/docs/adr/ADR-0413-default-off-cluster-copilot-failure-diagnosis-api.md new file mode 100644 index 00000000..292bbd7e --- /dev/null +++ b/docs/adr/ADR-0413-default-off-cluster-copilot-failure-diagnosis-api.md @@ -0,0 +1,109 @@ +# ADR-0413：默认关闭的 Cluster Copilot 故障诊断产品 API + +- 状态：Accepted +- 日期：2026-08-15 +- 关联 RFC：QL-RFC-0001 D-321、Phase 2 +- 关联 ADR：ADR-0087、ADR-0407、ADR-0408、ADR-0409、ADR-0411、ADR-0412 + +## 问题 + +ADR-0412 已使 Copilot failure diagnosis 的成功、Tool/日志失败和 Model unknown +路径都能耐久收敛，但 capability 仍没有产品入口。直接增加独立 AI HTTP 服务、让 CLI/UI/MCP +调用 application service，或允许调用者提交 Model、Tool、Attempt、日志范围和 outcome，都会绕过 +Cluster Control 已有的认证、Project Policy、同步安全审计、资源预算或 source fence。 + +路由器上的低配 Edge/Standalone 与集群节点还必须保持不同部署闭包：默认部署不能因为一个可选 +Copilot API 增加 AI 依赖、进程、监听器、连接池、队列或常驻任务；集群多副本则必须依靠耐久 +request identity 和数据库 fence 收敛，而不是依靠单进程锁。 + +## 决策 + +1. 唯一产品写入口为 + `POST /api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses`，operation 是 + `copilot.failure_diagnosis.execute`，Project permission 是 `model.invoke`。它只注入既有 + Cluster Control route registry，完整复用同一认证器、Project Policy 和 fail-closed 同步安全 + 审计；内部受信 Tool 仍独立复验其精确 `tool.call:*` permission。 +2. 路由只在显式 AI 进程且 `QL3_CLUSTER_AI_COPILOT_ENABLED=true` 时注入。普通 + `ql3-cluster-control`、Edge、Standalone 和未启用 Copilot 的 AI 进程都返回 `404`，并且不 + import/构造 Copilot capability。 +3. HTTP `x-request-id` 是耐久 diagnosis request identity；缺省时由既有 HTTP surface 生成并 + 回传。客户端重试必须复用它。JSON body 采用 exact-key schema，只包含 + `schema=qinglong/cluster-copilot-failure-diagnosis-request@v1` 与 `traceId`。调用者不能提交 + project、source Attempt、日志 Artifact/range、Tool、provider、model、预算、deadline、输出 + key、reason 或 outcome。 +4. `projectId` 与 `sourceRunId` 只从已匹配的 canonical path 获取，principal 只从认证结果获取。 + application service 再从当前 Run、latest Attempt、Project Tool snapshot、配置投影和数据库时间 + 派生 exact source fence、Tool plan、Model intent 与 deadline。HTTP Policy fence 负责产品入口 + admission；耐久执行计划仍记录 application 内部重新读取的当前 Policy fence，不能信任调用者 + 提交的 fence。 +5. 响应是 content-free 投影，只返回 schema、request identity、created/existing replay 状态、 + source/diagnosis Run identity、终态 outcome/stage/reason，以及成功时的加密输出 Artifact + `artifactId/artifactDigest`。不得返回日志、Tool output、Model plaintext、prompt、provider、 + model、密钥材料、内部异常、Policy reasons 或数据库细节。 +6. 请求体错误返回稳定 `400`；source/idempotency/durable conflict 返回 `409`；Tool approval/ + policy deny 返回低敏 `403`；容量耗尽返回 `429`；Model budget/egress policy 拒绝返回 `422`； + caller abort 返回 `408`，deadline 返回 `504`；其余依赖和存储错误统一为 `503`。任何响应都 + 不透传内部 message。 +7. 路由是 caller-driven 的薄 transport adapter，不增加 package。实现放入既有 + `cluster-control/src/copilot/failure-diagnosis` 领域目录；AI composition 只把已经创建的同一个 + application capability 注入 control composition。不得增加进程、监听器、数据库 Pool/连接、 + timer、watcher、队列、cache、Pod、Service 或 Kubernetes API 权限。 +8. 多副本并发继续以 application request coalescing 加 PostgreSQL exact replay 收敛。进程内 + coalescing 仅降低同实例重复工作，不是正确性边界；不同实例对同一 `x-request-id` 的相同请求 + 必须得到 `created|existing` 同一 diagnosis Run，对不同 source/trace 的复用必须 conflict。 +9. 后续 CLI、UI、MCP 只能调用该 HTTP API，不得直接导入 Copilot application/composition， + 也不得各自定义更宽的请求契约。 + +## 被否决方案 + +1. **独立 Copilot HTTP 进程或端口**：复制认证、审计、TLS、限流和生命周期，增加低配与集群 + 运维成本。 +2. **把路由放进默认 Cluster Control**：即使运行时返回 disabled，也会扩大默认制品和依赖闭包。 +3. **body 自带 requestId 且与 HTTP requestId 分离**：会产生两个重试/审计身份，难以可靠关联。 +4. **调用者选择 provider/model/Tool/日志范围**：扩大数据外发和权限边界，破坏 canonical config + 与 source fence。 +5. **只检查 `model.invoke`，跳过内部 Tool Policy**：把组合 capability 变成 Tool 权限旁路。 +6. **同步返回诊断正文**：使重放、响应大小、日志泄露和生命周期依赖于一次 HTTP 连接。 +7. **新增 Copilot API package**：没有独立部署或复用边界，只会制造薄 package。 + +## 验证标准 + +1. 路由单元测试覆盖 exact body、HTTP request identity 绑定、path-derived project/source Run、 + principal 传递、created/existing replay、成功/非成功 content-free projection、错误低敏映射和 + 非法输入零 capability call。 +2. production composition 测试证明默认 route 不存在；显式 capability 注入后只增加一个 exact + route，并先完成 authentication、`model.invoke` Policy 和同步 `allowed` audit 才读取 body/ + 调 capability。拒绝路径不得调用 capability。 +3. AI composition 测试证明只有显式 Copilot 配置才注入同一个 capability；Prompt 与 Copilot + 仍共享原 Pool/Gateway，stop 顺序和 availability ownership 不变。 +4. package/dependency/import/deployment 审计证明 package 数、默认 Edge/Standalone 闭包、进程、 + 端口、Pool、Pod、Service 与 Kubernetes 权限无增长。 +5. 18-package clean build/test、完整 backend、Local artifact、PostgreSQL 18 physical HA 与 + GitNexus staged/change detection 全通过后才允许阶段性提交。 + +## 当前验证 + +1. 路由与 composition 新增测试覆盖 HTTP request identity、path-derived Project/source Run、认证 + principal、exact body、created/existing replay、成功和 pre-Model content-free projection、内部 + 错误低敏映射、默认 route absence、`model.invoke` deny 零 capability call，以及 AI composition + 的同一 capability 注入。Cluster Control 全包 250 pass/2 条件 skip/0 fail；受影响测试格式化后 + 独立复跑 27/27。 +2. 18 个 QL3 package 从清空全部 `dist` 开始完成拓扑构建和全包测试；backend 1,207 pass/2 条件 + skip/0 fail。Edge import、Cluster dependency、package boundary 与 Cluster deployment 四项审计 + 全部兼容且零 finding。 +3. workspace 仍为 18 个 package，`singleSourcePackages=[]`、`shallowSourcePackages=[]`。Cluster + Control 60 个源码中 58 个位于嵌套领域目录；唯一新增源码位于既有 + `copilot/failure-diagnosis`，没有增加 package 或根部平铺文件。 +4. 14 档 Local artifact 全部 `compatible=true`；默认 Edge/Standalone 分别保持 + 2,589,890/2,589,968 bytes，Edge/Standalone AI 为 3,069,143/3,069,233 bytes，证明 + Cluster-only 产品路由未进入路由设备或 Local AI 闭包。 +5. PostgreSQL 18.6 arm64 physical HA 137/137、timeline `1→2`；既有成功、pre-Model + terminalization、unknown resolution 和晋升后 exact replay 全部通过。私有报告 SHA-256 为 + `0a12b5c1102555823d43b5a93dd7868b98b194b491840a5242bab6fa2da26123`，独立离线审计 + `compatible=true`、零 finding。本 Gate 未改 migration、schema、role、SQL 或 HA 拓扑。 + +## 后续门禁 + +本 Gate 不开放诊断正文读取、列表、取消、unknown resolution、CLI、UI 或 MCP。下一 Gate 应优先 +提供同一 Project Policy 下的加密输出 Artifact 读取与费用/取消可观测性；真实外部 Provider 和 +多副本并发证据完成前，Copilot 继续保持默认关闭。 diff --git a/packages/ql3-cluster-control/package.json b/packages/ql3-cluster-control/package.json index 275ddc72..0a0f0b45 100644 --- a/packages/ql3-cluster-control/package.json +++ b/packages/ql3-cluster-control/package.json @@ -45,6 +45,11 @@ "require": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js", "default": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js" }, + "./copilot-routes": { + "types": "./dist/copilot/failure-diagnosis/failureDiagnosisRoute.d.ts", + "require": "./dist/copilot/failure-diagnosis/failureDiagnosisRoute.js", + "default": "./dist/copilot/failure-diagnosis/failureDiagnosisRoute.js" + }, "./http": { "types": "./dist/transport/httpSurface.d.ts", "require": "./dist/transport/httpSurface.js", diff --git a/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts b/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts index 2dd72370..b8fb1ae5 100644 --- a/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts +++ b/packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts @@ -445,6 +445,13 @@ export async function startProductionClusterAiControlApplication( capability: promptApplication.promptExecutionOutputs, }, }), + ...(copilotApplication === undefined + ? {} + : { + copilotFailureDiagnosis: { + capability: copilotApplication, + }, + }), }); if (controlApplication.status !== 'active') { throw new Error('AI-enabled cluster-control did not activate'); diff --git a/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts b/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts index 916748c4..bb6437ec 100644 --- a/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts +++ b/packages/ql3-cluster-control/src/application-runtime/productionApplication.ts @@ -61,6 +61,10 @@ import { } from '../worker-ingress/productionWorkerIngress'; import type { ClusterWorkerIngressApplicationResult } from '../worker-ingress/workerIngressApplication'; import { createClusterControlPluginPackageWorkflowRoutes } from '../plugin-package/workflow/pluginPackageWorkflowRoute'; +import { + createClusterControlCopilotFailureDiagnosisRoute, + type ClusterCopilotFailureDiagnosisCapability, +} from '../copilot/failure-diagnosis/failureDiagnosisRoute'; export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([ 'task.get', @@ -87,6 +91,7 @@ export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS = 'prompt.execution.read', 'prompt.execution.output.read', 'prompt.output.read', + 'copilot.failure_diagnosis.execute', ] as const); export interface ProductionClusterControlAssemblyOptions { @@ -109,6 +114,9 @@ export interface ProductionClusterControlAssemblyOptions { readonly promptExecutionOutputRead?: Readonly<{ readonly capability: ClusterPluginPackagePromptExecutionOutputReadCapability; }>; + readonly copilotFailureDiagnosis?: Readonly<{ + readonly capability: ClusterCopilotFailureDiagnosisCapability; + }>; readonly workerIngress?: Readonly<{ readonly config: EnabledClusterWorkerIngressConfig; readonly onDiagnostic?: (error: unknown) => void | Promise; @@ -154,6 +162,9 @@ export interface ProductionClusterControlApplicationOptions readonly promptExecutionOutputRead?: Readonly<{ readonly capability: ClusterPluginPackagePromptExecutionOutputReadCapability; }>; + readonly copilotFailureDiagnosis?: Readonly<{ + readonly capability: ClusterCopilotFailureDiagnosisCapability; + }>; readonly workerIngress?: ProductionClusterWorkerIngressOptions; } @@ -261,6 +272,13 @@ export function createProductionClusterControlApplicationStack( options.promptExecutionOutputRead.capability, ), ]), + ...(options.copilotFailureDiagnosis === undefined + ? [] + : [ + createClusterControlCopilotFailureDiagnosisRoute( + options.copilotFailureDiagnosis.capability, + ), + ]), ]; const routes = createClusterControlRouteRegistry(routeDefinitions); const expectedRouteCount = @@ -269,7 +287,8 @@ export function createProductionClusterControlApplicationStack( (options.promptExecution === undefined ? 0 : 1) + (options.promptExecutionInspection === undefined ? 0 : 1) + (options.promptOutputRead === undefined ? 0 : 1) + - (options.promptExecutionOutputRead === undefined ? 0 : 1); + (options.promptExecutionOutputRead === undefined ? 0 : 1) + + (options.copilotFailureDiagnosis === undefined ? 0 : 1); if (routes.size !== expectedRouteCount) { throw new Error('Production cluster-control route allowlist is incomplete'); } @@ -359,6 +378,7 @@ export function startProductionClusterControlApplication( promptExecutionInspection, promptOutputRead, promptExecutionOutputRead, + copilotFailureDiagnosis, ...applicationOptions } = options; const database = createClusterControlDatabaseBinding(config); @@ -402,6 +422,9 @@ export function startProductionClusterControlApplication( ...(promptExecutionOutputRead === undefined ? {} : { promptExecutionOutputRead }), + ...(copilotFailureDiagnosis === undefined + ? {} + : { copilotFailureDiagnosis }), ...(workerIngress === undefined ? {} : { diff --git a/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisRoute.ts b/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisRoute.ts new file mode 100644 index 00000000..c625e4b6 --- /dev/null +++ b/packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisRoute.ts @@ -0,0 +1,297 @@ +// Cluster Copilot owns one bounded, Policy-fenced diagnosis admission route. +import type { SecurityPrincipal } from '@qinglong/runtime-core/security'; +import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface'; +import type { + ClusterControlAuthorizedOperationRequest, + ClusterControlRouteDefinition, + ClusterControlRouteParameters, +} from '../../transport/routeRegistry'; + +export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_REQUEST_SCHEMA = + 'qinglong/cluster-copilot-failure-diagnosis-request@v1' as const; +export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_RESPONSE_SCHEMA = + 'qinglong/cluster-copilot-failure-diagnosis-response@v1' as const; +export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_ROUTE = Object.freeze({ + method: 'POST' as const, + path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses', + operationId: 'copilot.failure_diagnosis.execute', + permission: 'model.invoke', + projectParameter: 'projectId', +}); + +export interface ClusterCopilotFailureDiagnosisCommand { + readonly requestId: string; + readonly traceId: string; + readonly projectId: string; + readonly sourceRunId: string; + readonly principal: Readonly; +} + +interface ClusterCopilotFailureDiagnosisOutputReference { + readonly artifactId: string; + readonly artifactDigest: string; +} + +export interface ClusterCopilotFailureDiagnosisCapability { + execute(command: Readonly): Promise< + Readonly<{ + readonly admissionStatus: 'created' | 'existing'; + readonly admission: Readonly<{ + readonly requestId: string; + readonly runId: string; + readonly sourceRunId: string; + }>; + readonly tool: Readonly<{ readonly outcome: string }> | null; + readonly model: Readonly<{ + readonly outcome: string; + readonly output: Readonly | null; + }> | null; + readonly terminalization: Readonly<{ + readonly stage: string; + readonly reason: string; + readonly outcome: string; + }> | null; + readonly terminalizationRequired: boolean; + }> + >; +} + +class InvalidClusterCopilotFailureDiagnosisRequestError extends TypeError {} + +const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']); +const TERMINAL_STAGES = new Set(['tool', 'log', 'deadline', 'cancellation']); +const TERMINAL_REASONS = new Set([ + 'tool_failed', + 'tool_timed_out', + 'log_not_found', + 'log_pending', + 'log_missing', + 'log_retired', + 'tool_budget_exhausted', + 'deadline_exceeded', + 'cancellation_requested', +]); + +function invalid(): never { + throw new InvalidClusterCopilotFailureDiagnosisRequestError(); +} + +function response( + statusCode: number, + body: Readonly>, +): ClusterControlAdmissionResponse { + return Object.freeze({ statusCode, body: Object.freeze(body) }); +} + +function parseBody(value: unknown): Readonly<{ traceId: string }> { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + Object.getPrototypeOf(value) !== Object.prototype || + Object.keys(value).sort().join('\0') !== ['schema', 'traceId'].join('\0') + ) { + return invalid(); + } + const body = value as Record; + if ( + body.schema !== CLUSTER_COPILOT_FAILURE_DIAGNOSIS_REQUEST_SCHEMA || + typeof body.traceId !== 'string' || + !ID_PATTERN.test(body.traceId) + ) { + return invalid(); + } + return Object.freeze({ traceId: body.traceId }); +} + +function errorCode(error: unknown): string | null { + if ( + !error || + typeof error !== 'object' || + !('code' in error) || + typeof error.code !== 'string' + ) { + return null; + } + return error.code; +} + +function executionError(error: unknown): ClusterControlAdmissionResponse { + const code = errorCode(error); + if ( + code === 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_CONFLICT' || + code === 'COPILOT_FAILURE_DIAGNOSIS_ADMISSION_CONFLICT' || + code === 'COPILOT_FAILURE_DIAGNOSIS_ADMISSION_NOT_ALLOWED' || + code === 'COPILOT_FAILURE_DIAGNOSIS_TOOL_EXECUTION_CONFLICT' || + code === 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_CONFLICT' || + code === 'COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_CONFLICT' || + code === 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_IN_PROGRESS' || + code === 'COPILOT_FAILURE_DIAGNOSIS_MODEL_RESOLUTION_REQUIRED' || + code === 'COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_CONFLICT' + ) { + return response(409, { code: 'copilot_failure_diagnosis_conflict' }); + } + if ( + code === 'TRUSTED_TOOL_EXECUTION_POLICY_DENIED' || + code === 'TRUSTED_TOOL_EXECUTION_APPROVAL_REQUIRED' + ) { + return response(403, { code: 'copilot_failure_diagnosis_forbidden' }); + } + if ( + code === 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_BUSY' || + code === 'MODEL_GATEWAY_BUSY' + ) { + return response(429, { + code: 'copilot_failure_diagnosis_capacity_exceeded', + }); + } + if ( + code === 'MODEL_POLICY_DENIED' || + code === 'MODEL_BUDGET_EXCEEDED' || + code === 'COPILOT_MODEL_EGRESS_DENIED' || + code === 'COPILOT_FAILURE_DIAGNOSIS_BUDGET_EXCEEDED' + ) { + return response(422, { code: 'copilot_failure_diagnosis_policy_rejected' }); + } + if ( + code === 'MODEL_INVOCATION_DEADLINE_EXCEEDED' || + code === 'COPILOT_FAILURE_DIAGNOSIS_TOOL_EXECUTION_DEADLINE_EXCEEDED' + ) { + return response(504, { + code: 'copilot_failure_diagnosis_deadline_exceeded', + }); + } + if (code === 'MODEL_INVOCATION_ABORTED') { + return response(408, { code: 'copilot_failure_diagnosis_aborted' }); + } + if ( + code === 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_INVALID' || + code === 'COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_INVALID' + ) { + return response(400, { code: 'invalid_copilot_failure_diagnosis_request' }); + } + return response(503, { code: 'copilot_failure_diagnosis_unavailable' }); +} + +function projectResult( + result: Awaited< + ReturnType + >, + requestId: string, + sourceRunId: string, +): ClusterControlAdmissionResponse { + const model = result?.model; + const terminalization = result?.terminalization; + const outcome = model?.outcome ?? terminalization?.outcome; + if ( + !result || + (result.admissionStatus !== 'created' && + result.admissionStatus !== 'existing') || + !result.admission || + result.admission.requestId !== requestId || + result.admission.sourceRunId !== sourceRunId || + !RUN_ID_PATTERN.test(result.admission.runId) || + result.terminalizationRequired !== false || + (model === null) === (terminalization === null) || + typeof outcome !== 'string' || + !OUTCOMES.has(outcome) + ) { + return response(503, { code: 'copilot_failure_diagnosis_unavailable' }); + } + let stage: string; + let reason: string | null; + let outputArtifact: Readonly | null; + if (model) { + if ( + !OUTCOMES.has(model.outcome) || + (model.outcome === 'succeeded') !== (model.output !== null) || + (model.output !== null && + (!ID_PATTERN.test(model.output.artifactId) || + !DIGEST_PATTERN.test(model.output.artifactDigest))) + ) { + return response(503, { code: 'copilot_failure_diagnosis_unavailable' }); + } + stage = 'model'; + reason = null; + outputArtifact = model.output; + } else { + if ( + !terminalization || + !TERMINAL_STAGES.has(terminalization.stage) || + !TERMINAL_REASONS.has(terminalization.reason) + ) { + return response(503, { code: 'copilot_failure_diagnosis_unavailable' }); + } + stage = terminalization.stage; + reason = terminalization.reason; + outputArtifact = null; + } + return response(result.admissionStatus === 'created' ? 201 : 200, { + schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_RESPONSE_SCHEMA, + requestId, + status: result.admissionStatus, + replayed: result.admissionStatus === 'existing', + sourceRunId, + diagnosisRunId: result.admission.runId, + outcome, + stage, + reason, + outputArtifact: + outputArtifact === null + ? null + : Object.freeze({ + artifactId: outputArtifact.artifactId, + artifactDigest: outputArtifact.artifactDigest, + }), + }); +} + +export function createClusterControlCopilotFailureDiagnosisRoute( + capability: ClusterCopilotFailureDiagnosisCapability, +): Readonly { + if (!capability || typeof capability.execute !== 'function') { + throw new TypeError( + 'Cluster-control Copilot failure diagnosis capability is invalid', + ); + } + return Object.freeze({ + ...CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_ROUTE, + async handle( + authorized: ClusterControlAuthorizedOperationRequest, + routeParameters: ClusterControlRouteParameters, + ) { + let body: Readonly<{ traceId: string }>; + try { + body = parseBody(authorized.request.body); + } catch { + return response(400, { + code: 'invalid_copilot_failure_diagnosis_request', + }); + } + const projectId = authorized.projectId; + const sourceRunId = routeParameters.runId; + if ( + projectId === null || + typeof sourceRunId !== 'string' || + !RUN_ID_PATTERN.test(sourceRunId) + ) { + return response(503, { code: 'copilot_failure_diagnosis_unavailable' }); + } + try { + const result = await capability.execute({ + requestId: authorized.request.requestId, + traceId: body.traceId, + projectId, + sourceRunId, + principal: authorized.principal, + }); + return projectResult(result, authorized.request.requestId, sourceRunId); + } catch (error) { + return executionError(error); + } + }, + }); +} diff --git a/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs b/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs index 9d1f0d8b..2992c039 100644 --- a/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs +++ b/packages/ql3-cluster-control/test/aiProductionApplication.test.cjs @@ -117,7 +117,7 @@ async function projectedFile(root, name, bytes) { await chmod(join(root, name), 0o440); } -test('Copilot composition is explicit, shares the Prompt gateway and exposes no route', async () => { +test('Copilot composition is explicit, shares the Prompt gateway and injects one route capability', async () => { const secretRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-secret-')); const configRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-config-')); const invocationRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-invocation-')); @@ -229,7 +229,7 @@ test('Copilot composition is explicit, shares the Prompt gateway and exposes no assert.equal(created.gateway, gateway); assert.equal(created.successfulCompletion, registeredSink); assert.equal(created.artifactStore, artifactStore); - assert.equal('copilot' in controlOptions, false); + assert.equal(controlOptions.copilotFailureDiagnosis.capability, copilot); assert.equal(await application.stop(), 'stopped'); } finally { config.fill(0); invocation.fill(0); result.fill(0); output.fill(0); diff --git a/packages/ql3-cluster-control/test/copilotFailureDiagnosisRoute.test.cjs b/packages/ql3-cluster-control/test/copilotFailureDiagnosisRoute.test.cjs new file mode 100644 index 00000000..2190267e --- /dev/null +++ b/packages/ql3-cluster-control/test/copilotFailureDiagnosisRoute.test.cjs @@ -0,0 +1,269 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const { + CLUSTER_COPILOT_FAILURE_DIAGNOSIS_REQUEST_SCHEMA, + CLUSTER_COPILOT_FAILURE_DIAGNOSIS_RESPONSE_SCHEMA, + CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_ROUTE, + createClusterControlCopilotFailureDiagnosisRoute, +} = require('@qinglong/cluster-control/copilot-routes'); + +function authorized(body, overrides = {}) { + return { + request: { + requestId: 'diagnosis-request-1', + method: 'POST', + path: '/api/v3/projects/project-1/runs/source-run-1/copilot/failure-diagnoses', + query: {}, + headers: {}, + signal: new AbortController().signal, + body, + }, + principal: { + subject: { type: 'api_app', id: 'app-1' }, + authenticationId: 'credential-1', + authenticatedAtMs: 1, + expiresAtMs: 10_000, + assurance: 'service', + }, + operationId: 'copilot.failure_diagnosis.execute', + permission: 'model.invoke', + projectId: 'project-1', + policyFence: { projectVersion: 3, bindingVersion: 7 }, + ...overrides, + }; +} + +function body(overrides = {}) { + return { + schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_REQUEST_SCHEMA, + traceId: 'trace-1', + ...overrides, + }; +} + +function succeeded(admissionStatus = 'created') { + return { + admissionStatus, + admission: { + requestId: 'diagnosis-request-1', + runId: 'diagnosis-run-1', + sourceRunId: 'source-run-1', + }, + tool: { outcome: 'succeeded', output: { private: 'must not cross' } }, + model: { + outcome: 'succeeded', + output: { + artifactId: 'cdo:artifact-1', + artifactDigest: 'a'.repeat(64), + provider: 'private-provider', + }, + plaintext: 'private diagnosis', + }, + terminalization: null, + terminalizationRequired: false, + }; +} + +test('defines one exact model.invoke route and binds HTTP request identity', async () => { + let command; + const route = createClusterControlCopilotFailureDiagnosisRoute({ + async execute(value) { + command = value; + return succeeded(); + }, + }); + const request = authorized(body()); + const result = await route.handle(request, { + projectId: 'project-1', + runId: 'source-run-1', + }); + + assert.deepEqual(CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_ROUTE, { + method: 'POST', + path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses', + operationId: 'copilot.failure_diagnosis.execute', + permission: 'model.invoke', + projectParameter: 'projectId', + }); + assert.deepEqual(command, { + requestId: 'diagnosis-request-1', + traceId: 'trace-1', + projectId: 'project-1', + sourceRunId: 'source-run-1', + principal: request.principal, + }); + assert.equal('policyFence' in command, false); + assert.equal('model' in command, false); + assert.equal('attemptId' in command, false); + assert.equal(result.statusCode, 201); + assert.deepEqual(result.body, { + schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_RESPONSE_SCHEMA, + requestId: 'diagnosis-request-1', + status: 'created', + replayed: false, + sourceRunId: 'source-run-1', + diagnosisRunId: 'diagnosis-run-1', + outcome: 'succeeded', + stage: 'model', + reason: null, + outputArtifact: { + artifactId: 'cdo:artifact-1', + artifactDigest: 'a'.repeat(64), + }, + }); + assert.equal(JSON.stringify(result).includes('private'), false); +}); + +test('returns a content-free existing receipt for exact replay', async () => { + const route = createClusterControlCopilotFailureDiagnosisRoute({ + async execute() { + return succeeded('existing'); + }, + }); + const result = await route.handle(authorized(body()), { + projectId: 'project-1', + runId: 'source-run-1', + }); + assert.equal(result.statusCode, 200); + assert.equal(result.body.status, 'existing'); + assert.equal(result.body.replayed, true); +}); + +test('projects pre-Model terminalization without Tool or log content', async () => { + const route = createClusterControlCopilotFailureDiagnosisRoute({ + async execute() { + return { + admissionStatus: 'created', + admission: { + requestId: 'diagnosis-request-1', + runId: 'diagnosis-run-1', + sourceRunId: 'source-run-1', + }, + tool: { outcome: 'failed', privateLog: 'must not cross' }, + model: null, + terminalization: { + stage: 'log', + reason: 'log_retired', + outcome: 'failed', + privateEvidence: 'must not cross', + }, + terminalizationRequired: false, + }; + }, + }); + const result = await route.handle(authorized(body()), { + projectId: 'project-1', + runId: 'source-run-1', + }); + assert.equal(result.statusCode, 201); + assert.equal(result.body.stage, 'log'); + assert.equal(result.body.reason, 'log_retired'); + assert.equal(result.body.outcome, 'failed'); + assert.equal(result.body.outputArtifact, null); + assert.equal(JSON.stringify(result).includes('private'), false); +}); + +test('rejects non-exact bodies before invoking the capability', async () => { + let calls = 0; + const route = createClusterControlCopilotFailureDiagnosisRoute({ + async execute() { + calls += 1; + return succeeded(); + }, + }); + for (const invalid of [ + null, + {}, + body({ requestId: 'body-request-must-not-exist' }), + body({ provider: 'caller-selected' }), + body({ traceId: '' }), + Object.assign(Object.create(null), body()), + ]) { + const result = await route.handle(authorized(invalid), { + projectId: 'project-1', + runId: 'source-run-1', + }); + assert.equal(result.statusCode, 400); + assert.deepEqual(result.body, { + code: 'invalid_copilot_failure_diagnosis_request', + }); + } + assert.equal(calls, 0); +}); + +test('fails closed on capability responses that do not bind the durable identity', async () => { + for (const mutation of [ + (value) => ({ + ...value, + admission: { ...value.admission, requestId: 'other' }, + }), + (value) => ({ + ...value, + admission: { ...value.admission, sourceRunId: 'other' }, + }), + (value) => ({ ...value, terminalizationRequired: true }), + (value) => ({ ...value, model: null }), + (value) => ({ ...value, model: { ...value.model, output: null } }), + ]) { + const route = createClusterControlCopilotFailureDiagnosisRoute({ + async execute() { + return mutation(succeeded()); + }, + }); + const result = await route.handle(authorized(body()), { + projectId: 'project-1', + runId: 'source-run-1', + }); + assert.deepEqual(result, { + statusCode: 503, + body: { code: 'copilot_failure_diagnosis_unavailable' }, + }); + } +}); + +test('maps internal failures to stable low-sensitive transport codes', async () => { + for (const [internal, statusCode, external] of [ + [ + 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_CONFLICT', + 409, + 'copilot_failure_diagnosis_conflict', + ], + [ + 'TRUSTED_TOOL_EXECUTION_POLICY_DENIED', + 403, + 'copilot_failure_diagnosis_forbidden', + ], + [ + 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_BUSY', + 429, + 'copilot_failure_diagnosis_capacity_exceeded', + ], + [ + 'COPILOT_MODEL_EGRESS_DENIED', + 422, + 'copilot_failure_diagnosis_policy_rejected', + ], + [ + 'MODEL_INVOCATION_DEADLINE_EXCEEDED', + 504, + 'copilot_failure_diagnosis_deadline_exceeded', + ], + ['MODEL_INVOCATION_ABORTED', 408, 'copilot_failure_diagnosis_aborted'], + ['PRIVATE_STORAGE_FAILURE', 503, 'copilot_failure_diagnosis_unavailable'], + ]) { + const route = createClusterControlCopilotFailureDiagnosisRoute({ + async execute() { + throw Object.assign(new Error('private internal detail'), { + code: internal, + }); + }, + }); + const result = await route.handle(authorized(body()), { + projectId: 'project-1', + runId: 'source-run-1', + }); + assert.equal(result.statusCode, statusCode, internal); + assert.deepEqual(result.body, { code: external }); + assert.equal(JSON.stringify(result).includes('private'), false); + } +}); diff --git a/packages/ql3-cluster-control/test/productionApplication.test.cjs b/packages/ql3-cluster-control/test/productionApplication.test.cjs index f29dddb2..aa1c2104 100644 --- a/packages/ql3-cluster-control/test/productionApplication.test.cjs +++ b/packages/ql3-cluster-control/test/productionApplication.test.cjs @@ -729,6 +729,7 @@ test('optionally exposes Prompt execution behind shared admission and policy', a 'prompt.execution.read', 'prompt.execution.output.read', 'prompt.output.read', + 'copilot.failure_diagnosis.execute', ]); const response = await invoke( stack, @@ -756,6 +757,134 @@ test('optionally exposes Prompt execution behind shared admission and policy', a assert.equal(events.includes('audit:prompt.execute:allowed'), true); }); +test('optionally exposes Copilot diagnosis behind shared authentication, Policy and audit', async () => { + const { events, input } = fixture(); + let command; + const capability = { + async execute(value) { + command = value; + events.push(`diagnose:${value.sourceRunId}`); + return { + admissionStatus: 'created', + admission: { + requestId: value.requestId, + runId: 'diagnosis-run-1', + sourceRunId: value.sourceRunId, + }, + tool: { outcome: 'succeeded' }, + model: { + outcome: 'succeeded', + output: { + artifactId: 'cdo:artifact-1', + artifactDigest: 'a'.repeat(64), + }, + }, + terminalization: null, + terminalizationRequired: false, + }; + }, + }; + const stack = createProductionClusterControlApplicationStack(input, { + copilotFailureDiagnosis: { capability }, + }); + const result = await invoke( + stack, + metadata( + '/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses', + 'POST', + { + schema: 'qinglong/cluster-copilot-failure-diagnosis-request@v1', + traceId: 'trace-production-1', + }, + ), + ); + + assert.equal(result.statusCode, 201); + assert.equal(command.requestId, 'request-production-1'); + assert.equal(command.projectId, 'project-1'); + assert.equal(command.sourceRunId, 'run-1'); + assert.equal(command.principal.subject.id, 'app-production'); + assert.equal('policyFence' in command, false); + assert.deepEqual(events.slice(-4), [ + 'authenticate', + 'authorize', + 'audit:copilot.failure_diagnosis.execute:allowed', + 'diagnose:run-1', + ]); +}); + +test('keeps the Copilot route absent by default and never invokes it after Policy denial', async () => { + const defaultFixture = fixture(); + const defaultStack = createProductionClusterControlApplicationStack( + defaultFixture.input, + ); + const request = metadata( + '/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses', + 'POST', + { + schema: 'qinglong/cluster-copilot-failure-diagnosis-request@v1', + traceId: 'trace-production-1', + }, + ); + await assert.rejects( + defaultStack.admission.prepare(request), + (error) => error?.statusCode === 404 && error?.code === 'route_not_found', + ); + + let calls = 0; + const deniedFixture = fixture({ + policies: { + async resolve() { + deniedFixture.events.push('authorize'); + return { + project: { + id: 'project-1', + name: 'Denied Project', + slug: 'denied-project', + status: 'active', + version: 3, + createdAtMs: 1, + updatedAtMs: 2, + }, + binding: { + projectId: 'project-1', + subject: { type: 'api_app', id: 'app-production' }, + state: 'active', + role: 'viewer', + version: 7, + mutationId: 'binding-denied-1', + changedBy: { type: 'system', id: 'bootstrap' }, + createdAtMs: 1, + }, + }; + }, + }, + }); + const deniedStack = createProductionClusterControlApplicationStack( + deniedFixture.input, + { + copilotFailureDiagnosis: { + capability: { + async execute() { + calls += 1; + }, + }, + }, + }, + ); + await assert.rejects( + deniedStack.admission.prepare(request), + (error) => error?.statusCode === 403 && error?.code === 'forbidden', + ); + assert.equal(calls, 0); + assert.equal( + deniedFixture.events.includes( + 'audit:copilot.failure_diagnosis.execute:denied', + ), + true, + ); +}); + test('optionally exposes the redacted Prompt catalog behind shared admission and policy', async () => { const { events, input } = fixture(); const stack = createProductionClusterControlApplicationStack(input, { diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index f40d8749..05065c3f 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -385,10 +385,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: clusterControl.rootSourceFileRoles, }, { - sourceFiles: 59, + sourceFiles: 60, rootSourceFiles: 2, rootSourceLines: 195, - nestedSourceFiles: 57, + nestedSourceFiles: 58, rootSourceFileRoles: { 'aiCli.ts': 'binary_entry', 'cli.ts': 'binary_entry',