From af5d5bfc0bdb963e201486a7d3033174d2979795 Mon Sep 17 00:00:00 2001 From: whyour Date: Thu, 20 Aug 2026 13:30:06 +0800 Subject: [PATCH] feat(ql3): add generic worker management entry --- .../ql3-cluster/operator-context.example.json | 3 + docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 24 +- ...generic-worker-management-product-entry.md | 93 +++++++ docs/adr/README.md | 1 + packages/ql3-cluster-admin/package.json | 11 + .../managementClientConfiguration.ts | 31 ++- .../pluginPackageManagementHttp.ts | 26 +- .../src/product-cli/productCommand.ts | 6 + .../src/product-cli/productContext.ts | 26 +- .../workerCredentialManagementHttp.ts | 11 +- .../workerManagementClient.ts | 81 ++++++ .../workerManagementClientCli.ts | 150 ++++++++++ .../workerManagementProduct.ts | 192 +++++++++++++ .../test/productCli.test.cjs | 16 +- .../workerCredentialManagementHttp.test.cjs | 48 +++- .../test/workerManagementClient.test.cjs | 240 ++++++++++++++++ .../test/workerManagementClientCli.test.cjs | 256 ++++++++++++++++++ scripts/ql3-cluster-deployment-audit.cjs | 8 +- test/back/ql3ClusterDeploymentAudit.test.cjs | 6 +- test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 20 files changed, 1191 insertions(+), 42 deletions(-) create mode 100644 docs/adr/ADR-0467-generic-worker-management-product-entry.md create mode 100644 packages/ql3-cluster-admin/src/worker-management/workerManagementClient.ts create mode 100644 packages/ql3-cluster-admin/src/worker-management/workerManagementClientCli.ts create mode 100644 packages/ql3-cluster-admin/src/worker-management/workerManagementProduct.ts create mode 100644 packages/ql3-cluster-admin/test/workerManagementClient.test.cjs create mode 100644 packages/ql3-cluster-admin/test/workerManagementClientCli.test.cjs diff --git a/deploy/kubernetes/ql3-cluster/operator-context.example.json b/deploy/kubernetes/ql3-cluster/operator-context.example.json index 983afe3c..95a93db1 100644 --- a/deploy/kubernetes/ql3-cluster/operator-context.example.json +++ b/deploy/kubernetes/ql3-cluster/operator-context.example.json @@ -11,6 +11,9 @@ "configFile": "/secure/qinglong3/plugin-package-client.json", "kubernetesFile": "/secure/qinglong3/plugin-package-kubernetes.json" }, + "worker": { + "configFile": "/secure/qinglong3/worker-client.json" + }, "worker-credential": { "configFile": "/secure/qinglong3/worker-credential-client.json" }, diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 48fe269d..9c1e1604 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,25 @@ 最新增量证据（2026-08-20）： +- D-374/ADR-0467（已接受）：完成 D-373 留下的 Worker 产品命名债务，同时保持零新增常驻服务。通用 canonical path 为 + `/api/v3/workers/management`；旧 `/api/v3/worker-credentials/management` 作为精确兼容 alias 继续由同一个 TLS 1.3/mTLS/ + OIDC listener、transport、quota、rate limiter 和连接集合处理。共享 host 只允许这一对 alias，任意跨 Run/Automation/ + Approval/Package 管理面的路径组合在监听前失败。新增 `ql3-worker-client` 与 `ql3-cluster-admin worker inspect|list`，只接受 + caller-driven point inspect 或固定 16 项 keyset page，不接受 command file、credential mutation、caller limit、filter、自动 + 翻页、重试、轮询或 cache；输出固定为 `qinglong/worker-session-inspection@v1` 或 + `qinglong/worker-session-list@v1`，不投影 transport request/inspection identity。旧 `worker-credential` binary、export、配置和 + Kubernetes ceremony 保持兼容。实现仍在现有 `@qinglong/cluster-admin` 的内聚 `worker-management/` 目录，不新增 workspace + package、dependency、端口、Deployment、数据库对象或权限。聚焦实现门 `87/87`；Cluster Admin 全量为 + `427 total / 424 pass / 3 conditional skip / 0 fail`，完整 backend 为 + `1,503 total / 1,501 pass / 2 conditional skip / 0 fail`（包含一条不进入本阶段提交的既有用户测试），18-package clean + build/逐包测试单次退出 0。package boundary、Cluster dependency、Edge import、Cluster deployment 与 Worker deployment + 审计全部 compatible；workspace 仍为 18 packages、`singleSourcePackages=[]`、`shallowSourcePackages=[]`，Cluster Admin 为 + `128 source / 127 nested`。14 档 Local artifact audit 全部 compatible；基础 Edge/Standalone 为 + `2,598,669 / 2,598,747` bytes、57 loaded modules，Application+AI 为 `4,501,822 / 4,501,954` bytes，MCP 为 + `7,324,601 / 7,324,709` bytes、227 loaded modules。本切片不改变 PostgreSQL schema、ACL、repository、role、Pool、连接或 + failover 语义，因此不重跑和不重新占有物理 HA 证明；D-373 的 PostgreSQL 18.6 arm64 HA `146/146`、timeline `1→2` + 仅作为相邻既有基线，后续任何数据库语义变化必须重新执行 HA 门。 + - D-373/ADR-0466（已接受）：完成 D-372 的首个 caller-driven Worker 管理只读面。在既有 Worker management service/transport/client 内增加 `worker-session.inspect|list`，继续复用同一个 TLS 1.3/mTLS/OIDC listener、强 User `worker.manage` 与耐久 `worker-session.observe` quota，不新增 package、服务、端口、连接池、timer、watcher、queue、 @@ -6480,8 +6499,9 @@ Maintainers 要求 ARMv6、ARMv7 或 386 成为 3.0 正式支持项，必须先 D-373/ADR-0466 已把该分层投影到 caller-driven Worker 管理只读面：operator 可以 point inspect 或显式读取固定 16 项 keyset page，区分 Session lifecycle 与 Placement compatibility，并查看有界 runtime/capacity；不能请求任意 filter、 limit、自动翻页或轮询。该能力复用现有 Worker manager 进程和 `worker.manage`/耐久 quota，PostgreSQL role 只新增 -`SELECT worker_sessions`，所以不会进入 Edge/Standalone 常驻闭包。alpha 的 credential-named URI/CLI 是待 Beta 迁移的 -命名债务，不得用新建第二套 listener、Deployment 或 package 的方式修复。 +`SELECT worker_sessions`，所以不会进入 Edge/Standalone 常驻闭包。D-374/ADR-0467 已用 canonical +`/api/v3/workers/management`、只读 `ql3-worker-client` 和同 listener 的精确 legacy alias 清偿 credential-named URI/CLI +债务；旧 credential ceremony 继续兼容，且没有新建第二套 listener、Deployment 或 package。 ### 7.10 资源基准与发布门禁 diff --git a/docs/adr/ADR-0467-generic-worker-management-product-entry.md b/docs/adr/ADR-0467-generic-worker-management-product-entry.md new file mode 100644 index 00000000..1781c0aa --- /dev/null +++ b/docs/adr/ADR-0467-generic-worker-management-product-entry.md @@ -0,0 +1,93 @@ +# ADR-0467：通用 Worker Management 产品入口与兼容路径 + +- 状态：Accepted +- 日期：2026-08-20 +- 关联 RFC：QL-RFC-0001 D-374、D-14、D-16、D-107 +- 关联 ADR：ADR-0059、ADR-0465、ADR-0466 +- Amends：ADR-0466 的 Beta 前命名债务与产品入口 + +## 上下文 + +ADR-0466 已提供 `worker-session.inspect|list`，但 alpha 期间它仍挂在 +`/api/v3/worker-credentials/management` 与 `ql3-worker-credential-client` 下。这个入口可以验证底层权限、配额、数据库 +投影和响应边界，却不是清晰的通用 Worker 产品面：只想判断 Worker 在线、兼容性或剩余 slot 的 operator 不应接触可提交 +credential mutation command file 的客户端。 + +直接新增服务、listener、Deployment 或 workspace package 会复制 TLS/OIDC、连接池、quota 和运维生命周期，也会让低资源 +路由设备与集群控制节点承担空闲开销。直接把旧入口改名则会破坏已经部署的 credential client、Job 和配置文件。 + +## 决策 + +1. `/api/v3/workers/management` 成为通用 Worker management canonical path；历史 + `/api/v3/worker-credentials/management` 保留为精确兼容 alias。两个路径由同一个 HTTPS server、端口、transport、身份、 + rate limiter、并发计数和连接集合处理，不创建第二个 listener 或进程。 +2. 共享 HTTP host 不接受任意 alias。只有 canonical Worker path 可以同时声明且只能声明一个 legacy credential path；把 Run、 + Automation、Approval、Package 或未知路径拼入 alias 列表必须在监听前失败。 +3. 新增 `worker` 管理客户端策略，固定 canonical path 和 required mTLS；既有 `worker-credential` 策略继续固定 legacy path。 + 旧 binary、export、Kubernetes Job 与配置保持可用，不要求原子迁移。 +4. `ql3-worker-client` 与 `ql3-cluster-admin worker` 只提供： + - `inspect --project=PROJECT --worker=WORKER`； + - `list --project=PROJECT [--after=WORKER]`。 + 客户端不接受 command file、credential operation、caller limit、filter、自动翻页、重试、轮询、cache 或后台刷新。每次调用 + 只产生一个内部 inspection ID 和一次 POST。 +5. 产品输出使用 `qinglong/worker-session-inspection@v1` 与 `qinglong/worker-session-list@v1` 固定 schema。它复用 ADR-0466 + 的严格 transport validator，再执行独立投影；不返回 HTTP request ID、inspection ID、assertion、credential、Secret、 + raw capability 或可扩展任意字段。文本卡片与 JSON 均由同一投影生成。 +6. generic client 与 credential-compatible client 位于现有 `@qinglong/cluster-admin` 内。`worker-management/` 以 client、product、 + CLI 三个职责形成内聚目录，不新增 workspace package，也不把单文件边界放进 `packages/`。 +7. operator context 可以同时声明 `worker` 和 `worker-credential` 配置。前者用于只读日常观察，后者只在显式凭据管理 ceremony + 中使用；context 仍只持有 owner-private 配置路径，assertion 每次调用显式提供。 +8. 该切片不修改 PostgreSQL schema、role、Session、Scheduler 或 Worker ingress。Edge/Standalone 与未启用 Cluster Worker + manager 的部署不会加载 Cluster Admin 客户端；小设备没有新 timer、socket、数据库连接、常驻模块或磁盘写入。 + +## 升级与回滚 + +- 先发布同时接受 canonical/legacy path 的 manager，再分发 generic client 配置；旧客户端可以在整个兼容窗口继续工作。 +- generic 配置必须精确指向 `/api/v3/workers/management`，legacy credential 配置不能被 generic client 接受。这样可防止一次配置 + 漂移重新暴露 mutation surface。 +- 回滚客户端不会影响 manager；回滚 manager 到 ADR-0466 时 generic path 暂不可用，但 legacy credential path 与数据库语义保持。 + 因本切片没有 schema/ACL 变化，不需要数据库降级或 HA promotion。 + +## 被拒绝的替代方案 + +### 新建 Worker observability 服务或 package + +拒绝。它会复制安全边界、增加镜像/部署/连接池和路由设备供应链成本，而底层只有两个 caller-driven read operation。 + +### 直接重命名旧 path 与 binary + +拒绝。已部署 Job、config 和 operator automation 会被无收益破坏。canonical-first 加精确 alias 能提供可迁移产品入口，同时保持 +旧 ceremony 的明确语义。 + +### 让新 CLI 继续接受任意 command file + +拒绝。即使 TypeScript 类型写成只读，JavaScript caller 仍可构造 credential mutation；generic boundary 必须在 runtime +normalizer 和 CLI parser 两层拒绝该词汇。 + +### 自动轮询或自动翻完所有 Worker + +拒绝。它会把一次低成本诊断变成不可预测数据库和网络负载。固定 16 项页面与显式 `--after` 让低配 manager 和 operator 都能 +控制每次成本。 + +## 验证与证据 + +- 聚焦实现门 `87/87`，覆盖 canonical/legacy 同 listener、mTLS/CRL、cross-plane alias 拒绝、通用策略、只读 normalizer、严格 + 产品投影、CLI 真进程单请求、mutation 词汇拒绝、产品 catalog/context、部署审计和 package 内部布局。 +- `@qinglong/cluster-admin` TypeScript build/check 通过；全量为 + `427 total / 424 pass / 3 conditional skip / 0 fail`。完整 backend 为 + `1,503 total / 1,501 pass / 2 conditional skip / 0 fail`（包含一条不进入本阶段提交的既有用户测试）。 +- 18-package clean build/逐包测试单次退出 0；package boundary、Cluster dependency、Edge import、Cluster deployment 与 + Worker deployment 审计全部 compatible。workspace 保持 18 packages、`singleSourcePackages=[]`、 + `shallowSourcePackages=[]`；Cluster Admin 为 `128 source / 127 nested`，没有新增外部 dependency。 +- 14 档 Local artifact audit 全部 compatible；基础 Edge/Standalone 为 `2,598,669 / 2,598,747` bytes、57 loaded modules， + Application+AI 为 `4,501,822 / 4,501,954` bytes，MCP 为 `7,324,601 / 7,324,709` bytes、227 loaded modules，证明 + generic Worker 客户端没有进入低配设备常驻闭包。 +- 本切片不触及 PostgreSQL schema、ACL、repository、role、Pool、连接或 failover 语义，因此不重跑和不重新占有物理 HA + 证明；复用 ADR-0466 的 PostgreSQL 18.6 arm64 HA `146/146`、timeline `1→2` 相邻基线。后续若改变数据库语义必须重新运行 + HA，不能沿用该豁免。 + +## 后续边界 + +- legacy credential path/binary 的移除必须另立版本化弃用 ADR、发布遥测与至少一个兼容窗口；本 ADR 不授权删除。 +- Console/UI 若接入只能显式点击读取，不得静默轮询、自动翻页或把 assertion 存入浏览器持久存储。 +- Worker 历史、指标、label/filter 或跨 Project inventory 仍需独立 ownership、索引、retention、隐私和资源预算决策。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 1d5bd89e..db93da32 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -470,6 +470,7 @@ | [ADR-0464](./ADR-0464-machine-enforced-node24-architecture-support-tiers.md) | 机器化 Node 24 架构支持分层 | Accepted | | [ADR-0465](./ADR-0465-versioned-worker-support-tier-admission.md) | 版本化 Worker 支持等级准入 | Accepted | | [ADR-0466](./ADR-0466-bounded-worker-session-compatibility-observation.md) | 有界 Worker Session 兼容性观察 | Accepted | +| [ADR-0467](./ADR-0467-generic-worker-management-product-entry.md) | 通用 Worker Management 产品入口与兼容路径 | Accepted | ## 规则 diff --git a/packages/ql3-cluster-admin/package.json b/packages/ql3-cluster-admin/package.json index 122be103..d4291c1c 100644 --- a/packages/ql3-cluster-admin/package.json +++ b/packages/ql3-cluster-admin/package.json @@ -180,6 +180,16 @@ "require": "./dist/worker-credential/workerCredentialManagementClient.js", "default": "./dist/worker-credential/workerCredentialManagementClient.js" }, + "./worker-management-client": { + "types": "./dist/worker-management/workerManagementClient.d.ts", + "require": "./dist/worker-management/workerManagementClient.js", + "default": "./dist/worker-management/workerManagementClient.js" + }, + "./worker-management-product": { + "types": "./dist/worker-management/workerManagementProduct.d.ts", + "require": "./dist/worker-management/workerManagementProduct.js", + "default": "./dist/worker-management/workerManagementProduct.js" + }, "./worker-credential-management-executor": { "types": "./dist/worker-credential/workerCredentialManagementExecutor.d.ts", "require": "./dist/worker-credential/workerCredentialManagementExecutor.js", @@ -420,6 +430,7 @@ "ql3-worker-credential-manage": "dist/worker-credential/management-server/workerCredentialManagementCli.js", "ql3-worker-credential-execute": "dist/worker-credential/workerCredentialExecutorCli.js", "ql3-worker-credential-client": "dist/worker-credential/workerCredentialManagementClientCli.js", + "ql3-worker-client": "dist/worker-management/workerManagementClientCli.js", "ql3-approval-manage": "dist/approval-management/approvalManagementCli.js", "ql3-approval-client": "dist/approval-management/approvalManagementClientCli.js", "ql3-run-manage": "dist/run-management/runManagementCli.js", diff --git a/packages/ql3-cluster-admin/src/management-support/managementClientConfiguration.ts b/packages/ql3-cluster-admin/src/management-support/managementClientConfiguration.ts index 1836d0c3..60aba4e7 100644 --- a/packages/ql3-cluster-admin/src/management-support/managementClientConfiguration.ts +++ b/packages/ql3-cluster-admin/src/management-support/managementClientConfiguration.ts @@ -15,6 +15,7 @@ import { TextDecoder } from 'node:util'; export type ClusterAuthenticatedManagementClientKind = | 'package' + | 'worker' | 'worker-credential' | 'automation' | 'approval' @@ -34,6 +35,10 @@ const MANAGEMENT_CLIENT_POLICIES: Readonly< managementPath: '/api/v3/plugin-packages/management', clientCertificate: 'forbidden', }), + worker: Object.freeze({ + managementPath: '/api/v3/workers/management', + clientCertificate: 'required', + }), 'worker-credential': Object.freeze({ managementPath: '/api/v3/worker-credentials/management', clientCertificate: 'required', @@ -147,7 +152,9 @@ export function readCanonicalFile( throw configurationFailure(); } } catch (error) { - if (error instanceof ClusterPluginPackageManagementClientConfigurationError) { + if ( + error instanceof ClusterPluginPackageManagementClientConfigurationError + ) { throw error; } throw configurationFailure(); @@ -209,7 +216,9 @@ export function readCanonicalFile( return bytes; } catch (error) { bytes?.fill(0); - if (error instanceof ClusterPluginPackageManagementClientConfigurationError) { + if ( + error instanceof ClusterPluginPackageManagementClientConfigurationError + ) { throw error; } throw configurationFailure(); @@ -349,7 +358,10 @@ export function prepareClusterAuthenticatedManagementClientConfiguration( throw configurationFailure(); } } catch (error) { - if (error instanceof ClusterPluginPackageManagementClientConfigurationError) { + if ( + error instanceof + ClusterPluginPackageManagementClientConfigurationError + ) { throw error; } throw configurationFailure(); @@ -380,7 +392,9 @@ export function prepareClusterAuthenticatedManagementClientConfiguration( caBytes?.fill(0); clientCertificateBytes?.fill(0); clientPrivateKeyBytes?.fill(0); - if (error instanceof ClusterPluginPackageManagementClientConfigurationError) { + if ( + error instanceof ClusterPluginPackageManagementClientConfigurationError + ) { throw error; } throw configurationFailure(); @@ -408,11 +422,10 @@ export function validateClusterAuthenticatedManagementClientConfiguration( ): Readonly { const policy = MANAGEMENT_CLIENT_POLICIES[kind]; if (policy === undefined) throw configurationFailure(); - const prepared = - prepareClusterAuthenticatedManagementClientKindConfiguration( - configFile, - kind, - ); + const prepared = prepareClusterAuthenticatedManagementClientKindConfiguration( + configFile, + kind, + ); try { return Object.freeze({ schemaVersion: 1, diff --git a/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts b/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts index 32a73986..594235d7 100644 --- a/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts +++ b/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementHttp.ts @@ -85,6 +85,7 @@ export const CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH = '/api/v3/plugin-packages/management'; export const CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH = '/api/v3/worker-credentials/management'; +export const CLUSTER_WORKER_MANAGEMENT_PATH = '/api/v3/workers/management'; export const CLUSTER_AUTOMATION_MANAGEMENT_PATH = '/api/v3/automations/management'; export const CLUSTER_APPROVAL_MANAGEMENT_PATH = '/api/v3/approvals/management'; @@ -94,6 +95,7 @@ export const CLUSTER_RUN_MANAGEMENT_PATH = '/api/v3/runs/management'; export type ClusterAuthenticatedManagementPath = | typeof CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH | typeof CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH + | typeof CLUSTER_WORKER_MANAGEMENT_PATH | typeof CLUSTER_AUTOMATION_MANAGEMENT_PATH | typeof CLUSTER_APPROVAL_MANAGEMENT_PATH | typeof CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH @@ -101,6 +103,7 @@ export type ClusterAuthenticatedManagementPath = const MANAGEMENT_PATHS = new Set([ CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH, CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH, + CLUSTER_WORKER_MANAGEMENT_PATH, CLUSTER_AUTOMATION_MANAGEMENT_PATH, CLUSTER_APPROVAL_MANAGEMENT_PATH, CLUSTER_MODEL_PROVIDER_CREDENTIAL_MANAGEMENT_PATH, @@ -143,6 +146,7 @@ export interface StartClusterPluginPackageManagementHttpOptions { readonly transport: ClusterAuthenticatedManagementTransport; readonly identities: ClusterPluginPackageIdentityKeysetFile; readonly managementPath?: ClusterAuthenticatedManagementPath; + readonly compatibleManagementPaths?: readonly ClusterAuthenticatedManagementPath[]; readonly limits?: ClusterPluginPackageManagementHttpLimits; readonly now?: () => number; readonly createRequestId?: () => string; @@ -547,7 +551,8 @@ function responseError(error: unknown): HttpRequestError { error instanceof ClusterApprovalManagementTransportAuthenticationError || error instanceof ClusterModelProviderCredentialManagementTransportAuthenticationError || - error instanceof ClusterModelProviderCredentialManagementAuthenticationError || + error instanceof + ClusterModelProviderCredentialManagementAuthenticationError || error instanceof ClusterRunManagementTransportAuthenticationError ) { return new HttpRequestError(401, 'authentication_required'); @@ -573,7 +578,8 @@ function responseError(error: unknown): HttpRequestError { error instanceof WorkerCredentialManagementAuthorizationError || error instanceof ClusterAutomationManagementAuthorizationError || error instanceof ClusterApprovalManagementTransportAuthorizationError || - error instanceof ClusterModelProviderCredentialManagementAuthorizationError || + error instanceof + ClusterModelProviderCredentialManagementAuthorizationError || error instanceof ClusterRunManagementAuthorizationError ) { return new HttpRequestError(403, 'forbidden'); @@ -665,6 +671,7 @@ export async function startClusterPluginPackageManagementHttp( 'transport', 'identities', 'managementPath', + 'compatibleManagementPaths', 'limits', 'now', 'createRequestId', @@ -711,6 +718,12 @@ export async function startClusterPluginPackageManagementHttp( typeof options.identities.reload !== 'function' || (options.managementPath !== undefined && !MANAGEMENT_PATHS.has(options.managementPath)) || + (options.compatibleManagementPaths !== undefined && + (!Array.isArray(options.compatibleManagementPaths) || + options.managementPath !== CLUSTER_WORKER_MANAGEMENT_PATH || + options.compatibleManagementPaths.length !== 1 || + options.compatibleManagementPaths[0] !== + CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH)) || (options.now !== undefined && typeof options.now !== 'function') || (options.createRequestId !== undefined && typeof options.createRequestId !== 'function') || @@ -721,6 +734,10 @@ export async function startClusterPluginPackageManagementHttp( const limits = reviewedLimits(options.limits); const managementPath = options.managementPath ?? CLUSTER_PLUGIN_PACKAGE_MANAGEMENT_PATH; + const managementPaths = new Set([ + managementPath, + ...(options.compatibleManagementPaths ?? []), + ]); const now = options.now ?? Date.now; const createRequestId = options.createRequestId ?? randomUUID; const clientCertificateRequired = @@ -824,7 +841,10 @@ export async function startClusterPluginPackageManagementHttp( ) { throw new HttpRequestError(401, 'client_certificate_required'); } - if (request.method !== 'POST' || url !== managementPath) { + if ( + request.method !== 'POST' || + !managementPaths.has(url as ClusterAuthenticatedManagementPath) + ) { throw new HttpRequestError(404, 'not_found'); } if (availability !== 'ready') { diff --git a/packages/ql3-cluster-admin/src/product-cli/productCommand.ts b/packages/ql3-cluster-admin/src/product-cli/productCommand.ts index 4e0573de..22e82894 100644 --- a/packages/ql3-cluster-admin/src/product-cli/productCommand.ts +++ b/packages/ql3-cluster-admin/src/product-cli/productCommand.ts @@ -71,6 +71,12 @@ export const QINGLONG3_CLUSTER_PRODUCT_COMMANDS: readonly QingLong3ClusterProduc 'plugin-package/management/pluginPackageManagementKubernetesClientCli.js', description: 'manage Plugin Packages through a bounded Kubernetes tunnel', }), + Object.freeze({ + name: 'worker', + binary: 'ql3-worker-client', + target: 'worker-management/workerManagementClientCli.js', + description: 'inspect bounded Worker session state', + }), Object.freeze({ name: 'worker-credential', binary: 'ql3-worker-credential-client', diff --git a/packages/ql3-cluster-admin/src/product-cli/productContext.ts b/packages/ql3-cluster-admin/src/product-cli/productContext.ts index 4e11bd7d..1be62004 100644 --- a/packages/ql3-cluster-admin/src/product-cli/productContext.ts +++ b/packages/ql3-cluster-admin/src/product-cli/productContext.ts @@ -14,9 +14,7 @@ import { probeClusterCopilotClientReadiness, validateClusterCopilotClientConfiguration, } from '../copilot-client/client'; -import { - validateClusterAuthenticatedManagementClientConfiguration, -} from '../management-support/pluginPackageManagementClient'; +import { validateClusterAuthenticatedManagementClientConfiguration } from '../management-support/pluginPackageManagementClient'; import type { ClusterAuthenticatedManagementClientKind } from '../management-support/pluginPackageManagementClient'; import { probeClusterAuthenticatedManagementClientReadiness } from '../management-support/managementReadinessProbe'; import { @@ -31,6 +29,7 @@ const CONTEXT_COMMANDS = Object.freeze([ 'copilot', 'package', 'package-kubernetes', + 'worker', 'worker-credential', 'approval', 'run', @@ -92,6 +91,7 @@ const CONTEXT_COMMAND_CLIENT_KINDS: Readonly< > = Object.freeze({ package: 'package', 'package-kubernetes': 'package', + worker: 'worker', 'worker-credential': 'worker-credential', approval: 'approval', run: 'run', @@ -337,11 +337,10 @@ export async function validateQingLong3ClusterProductContext( }), ); } else if (name === 'package-kubernetes') { - const https = - validateClusterAuthenticatedManagementClientConfiguration( - command.configFile, - CONTEXT_COMMAND_CLIENT_KINDS[name], - ); + const https = validateClusterAuthenticatedManagementClientConfiguration( + command.configFile, + CONTEXT_COMMAND_CLIENT_KINDS[name], + ); const kubernetes = await validateClusterPluginPackageManagementKubernetesConfiguration( command.kubernetesFile!, @@ -355,11 +354,10 @@ export async function validateQingLong3ClusterProductContext( }), ); } else { - const https = - validateClusterAuthenticatedManagementClientConfiguration( - command.configFile, - CONTEXT_COMMAND_CLIENT_KINDS[name], - ); + const https = validateClusterAuthenticatedManagementClientConfiguration( + command.configFile, + CONTEXT_COMMAND_CLIENT_KINDS[name], + ); commands.push( Object.freeze({ name, @@ -403,7 +401,7 @@ export async function probeQingLong3ClusterProductContext( command.kubernetesFile!, ) : name === 'copilot' - ? await probeClusterCopilotClientReadiness(command.configFile) + ? await probeClusterCopilotClientReadiness(command.configFile) : await probeClusterAuthenticatedManagementClientReadiness( command.configFile, CONTEXT_COMMAND_CLIENT_KINDS[name], diff --git a/packages/ql3-cluster-admin/src/worker-credential/management-server/workerCredentialManagementHttp.ts b/packages/ql3-cluster-admin/src/worker-credential/management-server/workerCredentialManagementHttp.ts index ec2d6034..dec45e38 100644 --- a/packages/ql3-cluster-admin/src/worker-credential/management-server/workerCredentialManagementHttp.ts +++ b/packages/ql3-cluster-admin/src/worker-credential/management-server/workerCredentialManagementHttp.ts @@ -1,6 +1,7 @@ /** TLS 1.3 Worker credential management HTTP adapter boundary. */ import { CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH, + CLUSTER_WORKER_MANAGEMENT_PATH, startClusterPluginPackageManagementHttp, type ClusterPluginPackageManagementHttpApplication, type ClusterPluginPackageManagementHttpLimits, @@ -32,15 +33,17 @@ export interface StartClusterWorkerCredentialManagementHttpOptions { } /** - * Starts the Worker credential management endpoint on the shared Cluster Admin - * TLS 1.3/OIDC boundary. The public manager process never receives credential - * delivery or Kubernetes execution capabilities. + * Starts the Worker management endpoint on the shared Cluster Admin TLS + * 1.3/OIDC boundary. The former credential-scoped path remains an exact + * compatibility alias on this listener. The public manager process never + * receives credential delivery or Kubernetes execution capabilities. */ export async function startClusterWorkerCredentialManagementHttp( options: StartClusterWorkerCredentialManagementHttpOptions, ): Promise> { return startClusterPluginPackageManagementHttp({ ...options, - managementPath: CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH, + managementPath: CLUSTER_WORKER_MANAGEMENT_PATH, + compatibleManagementPaths: [CLUSTER_WORKER_CREDENTIAL_MANAGEMENT_PATH], }); } diff --git a/packages/ql3-cluster-admin/src/worker-management/workerManagementClient.ts b/packages/ql3-cluster-admin/src/worker-management/workerManagementClient.ts new file mode 100644 index 00000000..11183c58 --- /dev/null +++ b/packages/ql3-cluster-admin/src/worker-management/workerManagementClient.ts @@ -0,0 +1,81 @@ +/** Read-only product client boundary for bounded Worker session observation. */ +import { + executeClusterAuthenticatedManagementClient, + type ClusterAuthenticatedManagementClientResult, + type ClusterAuthenticatedManagementCommandExecution, + type ClusterPluginPackageManagementClientConnectionOptions, +} from '../management-support/pluginPackageManagementClient'; +import { + ClusterWorkerCredentialManagementTransportRequestError, + normalizeClusterWorkerCredentialManagementCommand, + type ClusterWorkerCredentialManagementCommand, + type ClusterWorkerCredentialManagementTransportResult, +} from '../worker-credential/management-server/workerCredentialManagementTransport'; +import { validateClusterWorkerCredentialManagementClientResult } from '../worker-credential/workerCredentialManagementClient'; + +const MANAGEMENT_PATH = '/api/v3/workers/management'; + +export type ClusterWorkerManagementCommand = Extract< + ClusterWorkerCredentialManagementCommand, + { readonly operation: 'worker-session.inspect' | 'worker-session.list' } +>; +export type ClusterWorkerManagementTransportResult = Extract< + ClusterWorkerCredentialManagementTransportResult, + { readonly operation: 'worker-session.inspect' | 'worker-session.list' } +>; +export type ClusterWorkerManagementClientExecution = + ClusterAuthenticatedManagementCommandExecution; +export type ClusterWorkerManagementClientConnectionOptions = + ClusterPluginPackageManagementClientConnectionOptions; +export type ClusterWorkerManagementClientResult = + ClusterAuthenticatedManagementClientResult; + +export function normalizeClusterWorkerManagementCommand( + value: unknown, +): Readonly { + const command = normalizeClusterWorkerCredentialManagementCommand(value); + if ( + command.operation !== 'worker-session.inspect' && + command.operation !== 'worker-session.list' + ) { + throw new ClusterWorkerCredentialManagementTransportRequestError( + 'operation is not available through the read-only Worker client', + ); + } + return command; +} + +function validateClusterWorkerManagementResult( + value: unknown, + command: Readonly, +): Readonly { + const result = validateClusterWorkerCredentialManagementClientResult( + value, + command, + ); + if ( + result.operation !== 'worker-session.inspect' && + result.operation !== 'worker-session.list' + ) { + throw new Error('Worker management response is invalid'); + } + return result; +} + +const PROTOCOL = Object.freeze({ + managementPath: MANAGEMENT_PATH, + clientCertificate: 'required' as const, + normalizeCommand: normalizeClusterWorkerManagementCommand, + validateResult: validateClusterWorkerManagementResult, +}); + +export async function executeClusterWorkerManagementClient( + execution: ClusterWorkerManagementClientExecution, + connectionOptions?: ClusterWorkerManagementClientConnectionOptions, +): Promise> { + return executeClusterAuthenticatedManagementClient( + execution, + PROTOCOL, + connectionOptions, + ); +} diff --git a/packages/ql3-cluster-admin/src/worker-management/workerManagementClientCli.ts b/packages/ql3-cluster-admin/src/worker-management/workerManagementClientCli.ts new file mode 100644 index 00000000..f657c71b --- /dev/null +++ b/packages/ql3-cluster-admin/src/worker-management/workerManagementClientCli.ts @@ -0,0 +1,150 @@ +#!/usr/bin/env node + +import { ClusterPluginPackageManagementClientRemoteError } from '../management-support/pluginPackageManagementClient'; +import { executeClusterWorkerManagementClient } from './workerManagementClient'; +import { + createWorkerSessionInspectionCommand, + createWorkerSessionListCommand, + formatWorkerSessionInspectionCard, + formatWorkerSessionListCard, + projectWorkerSessionInspection, + projectWorkerSessionList, +} from './workerManagementProduct'; + +const USAGE = [ + 'Usage: ql3-worker-client inspect --config=/absolute/client.json --assertion=/absolute/assertion.jwt --project=PROJECT --worker=WORKER [--format=text|json]', + ' ql3-worker-client list --config=/absolute/client.json --assertion=/absolute/assertion.jwt --project=PROJECT [--after=WORKER] [--format=text|json]', + '', + 'One invocation performs one bounded read; it never retries, polls or auto-pages.', +].join('\n'); +const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; + +type WorkerClientArguments = Readonly<{ + kind: 'inspect' | 'list'; + configFile: string; + assertionFile: string; + projectId: string; + workerId?: string; + afterWorkerId?: string; + format: 'text' | 'json'; +}>; + +function argumentsFrom(argv: readonly string[]): WorkerClientArguments | null { + const modes = argv.filter( + (argument) => argument === 'inspect' || argument === 'list', + ); + if (modes.length !== 1 || argv.length < 4 || argv.length > 6) return null; + const kind = modes[0] as 'inspect' | 'list'; + const values = new Map(); + for (const argument of argv) { + if (argument === kind) continue; + const match = + /^--(config|assertion|project|worker|after|format)=(.+)$/.exec(argument); + if (!match || values.has(match[1]!)) return null; + values.set(match[1]!, match[2]!); + } + if ( + !values.get('config')?.startsWith('/') || + !values.get('assertion')?.startsWith('/') || + !IDENTIFIER.test(values.get('project') ?? '') || + (kind === 'inspect' && + (!IDENTIFIER.test(values.get('worker') ?? '') || values.has('after'))) || + (kind === 'list' && + (values.has('worker') || + (values.has('after') && !IDENTIFIER.test(values.get('after')!)))) || + (values.has('format') && + values.get('format') !== 'text' && + values.get('format') !== 'json') + ) { + return null; + } + return Object.freeze({ + kind, + configFile: values.get('config')!, + assertionFile: values.get('assertion')!, + projectId: values.get('project')!, + ...(kind === 'inspect' ? { workerId: values.get('worker')! } : {}), + ...(kind === 'list' && values.has('after') + ? { afterWorkerId: values.get('after')! } + : {}), + format: (values.get('format') ?? 'text') as 'text' | 'json', + }); +} + +function failureFact(error: unknown): Readonly> { + const candidate = error as { readonly code?: unknown }; + return Object.freeze({ + schemaVersion: 1, + component: 'qinglong3-worker-management-client', + event: 'inspection_failed', + code: + typeof candidate?.code === 'string' + ? candidate.code + : 'QL3_WORKER_MANAGEMENT_CLIENT_FAILED', + ...(error instanceof ClusterPluginPackageManagementClientRemoteError + ? { + statusCode: error.statusCode, + responseCode: error.responseCode, + requestId: error.requestId, + ...(error.retryAfterSeconds === null + ? {} + : { retryAfterSeconds: error.retryAfterSeconds }), + } + : {}), + }); +} + +async function run(argv: readonly string[]): Promise { + if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) { + process.stdout.write(`${USAGE}\n`); + return; + } + const argumentsValue = argumentsFrom(argv); + if (argumentsValue === null) { + process.stderr.write( + `${JSON.stringify({ + schemaVersion: 1, + component: 'qinglong3-worker-management-client', + event: 'usage_invalid', + code: 'QL3_WORKER_MANAGEMENT_CLIENT_USAGE_INVALID', + })}\n`, + ); + process.exitCode = 64; + return; + } + try { + const command = + argumentsValue.kind === 'inspect' + ? createWorkerSessionInspectionCommand( + argumentsValue.projectId, + argumentsValue.workerId!, + ) + : createWorkerSessionListCommand( + argumentsValue.projectId, + argumentsValue.afterWorkerId, + ); + const response = await executeClusterWorkerManagementClient({ + configFile: argumentsValue.configFile, + assertionFile: argumentsValue.assertionFile, + command, + }); + const projection = + argumentsValue.kind === 'inspect' + ? projectWorkerSessionInspection(argumentsValue.projectId, response) + : projectWorkerSessionList(argumentsValue.projectId, response); + process.stdout.write( + argumentsValue.format === 'json' + ? `${JSON.stringify(projection)}\n` + : `${ + projection.schema === 'qinglong/worker-session-inspection@v1' + ? formatWorkerSessionInspectionCard(projection) + : formatWorkerSessionListCard(projection) + }\n`, + ); + } catch (error) { + process.stderr.write(`${JSON.stringify(failureFact(error))}\n`); + process.exitCode = 1; + } +} + +void run(process.argv.slice(2)); diff --git a/packages/ql3-cluster-admin/src/worker-management/workerManagementProduct.ts b/packages/ql3-cluster-admin/src/worker-management/workerManagementProduct.ts new file mode 100644 index 00000000..7b48213b --- /dev/null +++ b/packages/ql3-cluster-admin/src/worker-management/workerManagementProduct.ts @@ -0,0 +1,192 @@ +/** Bounded commands and low-sensitive product projections for Worker sessions. */ +import { randomUUID } from 'node:crypto'; + +import type { + ClusterWorkerManagementClientResult, + ClusterWorkerManagementCommand, + ClusterWorkerManagementTransportResult, +} from './workerManagementClient'; + +const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; + +type InspectCommand = Extract< + ClusterWorkerManagementCommand, + { readonly operation: 'worker-session.inspect' } +>; +type ListCommand = Extract< + ClusterWorkerManagementCommand, + { readonly operation: 'worker-session.list' } +>; +type InspectResult = Extract< + ClusterWorkerManagementTransportResult, + { readonly operation: 'worker-session.inspect' } +>; +type ListResult = Extract< + ClusterWorkerManagementTransportResult, + { readonly operation: 'worker-session.list' } +>; + +export interface WorkerSessionInspection { + readonly schema: 'qinglong/worker-session-inspection@v1'; + readonly projectId: string; + readonly observedAtMs: number; + readonly found: boolean; + readonly worker: InspectResult['worker']; +} + +export interface WorkerSessionList { + readonly schema: 'qinglong/worker-session-list@v1'; + readonly projectId: string; + readonly observedAtMs: number; + readonly count: number; + readonly workers: ListResult['workers']; + readonly nextAfterWorkerId: string | null; +} + +export class ClusterWorkerManagementProductError extends TypeError { + readonly code = 'QL3_WORKER_MANAGEMENT_PRODUCT_INPUT_INVALID'; + + constructor() { + super('Worker management product input is invalid'); + this.name = 'ClusterWorkerManagementProductError'; + } +} + +function identifier(value: string): string { + if (!IDENTIFIER.test(value)) throw new ClusterWorkerManagementProductError(); + return value; +} + +function inspectionId(createId: () => string): string { + const value = createId(); + return identifier(value); +} + +export function createWorkerSessionInspectionCommand( + projectId: string, + workerId: string, + createId: () => string = randomUUID, +): Readonly { + return Object.freeze({ + schemaVersion: 1, + operation: 'worker-session.inspect', + request: Object.freeze({ + authorityProjectId: identifier(projectId), + workerId: identifier(workerId), + inspectionId: inspectionId(createId), + }), + }); +} + +export function createWorkerSessionListCommand( + projectId: string, + afterWorkerId?: string, + createId: () => string = randomUUID, +): Readonly { + return Object.freeze({ + schemaVersion: 1, + operation: 'worker-session.list', + request: Object.freeze({ + authorityProjectId: identifier(projectId), + afterWorkerId: + afterWorkerId === undefined ? null : identifier(afterWorkerId), + inspectionId: inspectionId(createId), + }), + }); +} + +function cloneWorker< + Worker extends + | NonNullable + | ListResult['workers'][number], +>(worker: Worker): Worker { + return Object.freeze({ + ...worker, + ...('runtimes' in worker + ? { + runtimes: Object.freeze( + worker.runtimes.map((runtime) => Object.freeze({ ...runtime })), + ), + declaredCapacity: Object.freeze({ ...worker.declaredCapacity }), + } + : {}), + }) as Worker; +} + +export function projectWorkerSessionInspection( + projectId: string, + response: Readonly, +): Readonly { + if (response.result.operation !== 'worker-session.inspect') { + throw new ClusterWorkerManagementProductError(); + } + return Object.freeze({ + schema: 'qinglong/worker-session-inspection@v1', + projectId: identifier(projectId), + observedAtMs: response.result.observedAtMs, + found: response.result.worker !== null, + worker: + response.result.worker === null + ? null + : cloneWorker(response.result.worker), + }); +} + +export function projectWorkerSessionList( + projectId: string, + response: Readonly, +): Readonly { + if (response.result.operation !== 'worker-session.list') { + throw new ClusterWorkerManagementProductError(); + } + const workers = Object.freeze(response.result.workers.map(cloneWorker)); + return Object.freeze({ + schema: 'qinglong/worker-session-list@v1', + projectId: identifier(projectId), + observedAtMs: response.result.observedAtMs, + count: workers.length, + workers, + nextAfterWorkerId: response.result.nextCursor, + }); +} + +export function formatWorkerSessionInspectionCard( + inspection: Readonly, +): string { + if (inspection.worker === null) { + return [ + `Worker session: not found`, + `Project: ${inspection.projectId}`, + `Observed: ${inspection.observedAtMs}`, + ].join('\n'); + } + const worker = inspection.worker; + return [ + `Worker session: ${worker.workerId}`, + `Project: ${inspection.projectId}`, + `State: ${worker.lifecycle} / ${worker.compatibility} / ${worker.supportTier}`, + `Platform: ${worker.operatingSystem ?? 'unknown'} ${ + worker.architecture + } / protocol ${worker.protocolVersion}`, + `Capacity: ${worker.availableSlots}/${worker.maxConcurrentRuns} slots available`, + `Heartbeat: ${worker.lastHeartbeatAtMs} / lease ${worker.leaseExpiresAtMs}`, + `Observed: ${inspection.observedAtMs}`, + ].join('\n'); +} + +export function formatWorkerSessionListCard( + page: Readonly, +): string { + const lines = [ + `Worker sessions: ${page.count}`, + `Project: ${page.projectId}`, + `Observed: ${page.observedAtMs}`, + ]; + for (const worker of page.workers) { + lines.push( + `${worker.workerId} ${worker.lifecycle} ${worker.supportTier} ${worker.architecture} slots ${worker.availableSlots}/${worker.maxConcurrentRuns}`, + ); + } + lines.push(`Next after: ${page.nextAfterWorkerId ?? '-'}`); + return lines.join('\n'); +} diff --git a/packages/ql3-cluster-admin/test/productCli.test.cjs b/packages/ql3-cluster-admin/test/productCli.test.cjs index 1f5a4c09..9b86f11b 100644 --- a/packages/ql3-cluster-admin/test/productCli.test.cjs +++ b/packages/ql3-cluster-admin/test/productCli.test.cjs @@ -284,6 +284,13 @@ function validContextFixture(t) { copilot: { configFile: copilotConfig }, package: { configFile: packageConfig }, 'package-kubernetes': { configFile: packageConfig, kubernetesFile }, + worker: { + configFile: config( + 'worker-observation-client', + '/api/v3/workers/management', + 'required', + ), + }, 'worker-credential': { configFile: config( 'worker-client', @@ -329,7 +336,7 @@ function validContextFixture(t) { test('catalog exposes only reviewed product entrypoints from the same package', () => { assert.equal(manifest.bin['ql3-cluster-admin'], 'dist/product-cli/cli.js'); - assert.equal(QINGLONG3_CLUSTER_PRODUCT_COMMANDS.length, 11); + assert.equal(QINGLONG3_CLUSTER_PRODUCT_COMMANDS.length, 12); assert.equal( new Set(QINGLONG3_CLUSTER_PRODUCT_COMMANDS.map(({ name }) => name)).size, QINGLONG3_CLUSTER_PRODUCT_COMMANDS.length, @@ -617,7 +624,7 @@ test('validates the complete operator context offline without operational author schemaVersion: 1, component: 'qinglong3-cluster-product-cli', event: 'context_valid', - commandCount: 8, + commandCount: 9, commands: [ { name: 'copilot', transport: 'https', clientCertificate: 'forbidden' }, { name: 'package', transport: 'https', clientCertificate: 'forbidden' }, @@ -627,6 +634,11 @@ test('validates the complete operator context offline without operational author clientCertificate: 'forbidden', kubernetesAuthentication: 'token', }, + { + name: 'worker', + transport: 'https', + clientCertificate: 'required', + }, { name: 'worker-credential', transport: 'https', diff --git a/packages/ql3-cluster-admin/test/workerCredentialManagementHttp.test.cjs b/packages/ql3-cluster-admin/test/workerCredentialManagementHttp.test.cjs index 7b094ba8..7d6c6b59 100644 --- a/packages/ql3-cluster-admin/test/workerCredentialManagementHttp.test.cjs +++ b/packages/ql3-cluster-admin/test/workerCredentialManagementHttp.test.cjs @@ -56,6 +56,7 @@ const NEXT_CLIENT_KEY = resolve( 'fixtures/management-service-key.pem', ); const WORKER_PATH = '/api/v3/worker-credentials/management'; +const CANONICAL_WORKER_PATH = '/api/v3/workers/management'; function command() { return { @@ -223,7 +224,7 @@ async function requestWithClientIdentity(application, certificate, key) { }); } -test('serves only the fixed Worker credential management route', async () => { +test('serves the canonical Worker route and exact credential compatibility alias', async () => { const calls = []; const application = await start(async (value, authentication) => { calls.push({ value, principal: await authentication.authenticate() }); @@ -245,12 +246,15 @@ test('serves only the fixed Worker credential management route', async () => { type: 'user', id: 'cluster-reviewer', }); + const canonical = await request(application, CANONICAL_WORKER_PATH); + assert.equal(canonical.statusCode, 200); + assert.equal(canonical.body.result.operation, 'worker-credential.inspect'); assert.equal( (await request(application, '/api/v3/plugin-packages/management')) .statusCode, 404, ); - assert.equal(calls.length, 1); + assert.equal(calls.length, 2); } finally { await application.close(); } @@ -312,10 +316,18 @@ test('rejects a CRL-revoked client certificate before OIDC', async () => { test('maps Worker credential management failures to stable HTTP errors', async () => { for (const [failure, statusCode, code] of [ - [new WorkerCredentialManagementRequestError('invalid'), 400, 'request_invalid'], + [ + new WorkerCredentialManagementRequestError('invalid'), + 400, + 'request_invalid', + ], [new WorkerCredentialManagementAuthorizationError(), 403, 'forbidden'], [new WorkerCredentialManagementConflictError('conflict'), 409, 'conflict'], - [new WorkerCredentialManagementQuotaExceededError(1_250), 429, 'quota_exceeded'], + [ + new WorkerCredentialManagementQuotaExceededError(1_250), + 429, + 'quota_exceeded', + ], [new WorkerCredentialManagementUnavailableError(), 503, 'unavailable'], ]) { const application = await start(async () => { @@ -353,6 +365,34 @@ test('rejects arbitrary management paths at configuration time', async () => { } }); +test('rejects arbitrary or cross-plane compatible route aliases', async () => { + for (const compatibleManagementPaths of [ + ['/api/v3/automations/management'], + ['/api/v3/worker-credentials/management', '/api/v3/runs/management'], + ]) { + const privateKey = Buffer.from(readFileSync(SERVER_KEY)); + try { + await assert.rejects( + startClusterPluginPackageManagementHttp({ + host: '127.0.0.1', + port: 0, + tls: { + privateKey, + certificate: Buffer.from(readFileSync(SERVER_CERT)), + }, + identities: identities(), + transport: { async execute() {} }, + managementPath: CANONICAL_WORKER_PATH, + compatibleManagementPaths, + }), + ClusterPluginPackageManagementHttpConfigurationError, + ); + } finally { + privateKey.fill(0); + } + } +}); + test('accepts both client CAs during overlap then rejects the retired CA', async () => { const execute = async () => ({ schemaVersion: 1, diff --git a/packages/ql3-cluster-admin/test/workerManagementClient.test.cjs b/packages/ql3-cluster-admin/test/workerManagementClient.test.cjs new file mode 100644 index 00000000..71a773b6 --- /dev/null +++ b/packages/ql3-cluster-admin/test/workerManagementClient.test.cjs @@ -0,0 +1,240 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); + +const { + executeClusterWorkerManagementClient, + normalizeClusterWorkerManagementCommand, +} = require('@qinglong/cluster-admin/worker-management-client'); +const { + createWorkerSessionInspectionCommand, + createWorkerSessionListCommand, + formatWorkerSessionInspectionCard, + formatWorkerSessionListCard, + projectWorkerSessionInspection, + projectWorkerSessionList, +} = require('@qinglong/cluster-admin/worker-management-product'); + +const fixtureRoot = path.resolve( + __dirname, + '../../ql3-cluster-control/test/fixtures/mtls', +); +const detailedWorker = Object.freeze({ + workerId: 'worker-a', + sessionId: 'session-a', + generation: 2, + sessionVersion: 5, + lifecycle: 'online', + compatibility: 'default_placement', + architecture: 'arm64', + supportTier: 'tier1', + protocolVersion: '1.0.0', + operatingSystem: 'linux', + maxConcurrentRuns: 2, + availableSlots: 1, + registeredAtMs: 900, + lastHeartbeatAtMs: 1_050, + leaseExpiresAtMs: 2_000, + updatedAtMs: 1_050, + observedAtMs: 1_100, + runtimes: Object.freeze([{ name: 'node', version: '24.18.0' }]), + declaredCapacity: Object.freeze({ + cpuCores: 1, + memoryBytes: 268_435_456, + diskBytes: 1_073_741_824, + gpuCount: 0, + }), +}); +const summaryWorker = Object.freeze( + Object.fromEntries( + Object.entries(detailedWorker).filter( + ([key]) => key !== 'runtimes' && key !== 'declaredCapacity', + ), + ), +); + +function response(result) { + return Object.freeze({ + schemaVersion: 1, + requestId: 'transport-request-must-not-project', + result: Object.freeze(result), + }); +} + +function privateFile(directory, name, value) { + const filePath = path.join(directory, name); + fs.writeFileSync(filePath, value, { mode: 0o600 }); + return fs.realpathSync(filePath); +} + +test('builds only immutable inspect and bounded list commands', () => { + const inspect = createWorkerSessionInspectionCommand( + 'project-a', + 'worker-a', + () => 'inspection-a', + ); + assert.deepEqual(inspect, { + schemaVersion: 1, + operation: 'worker-session.inspect', + request: { + authorityProjectId: 'project-a', + workerId: 'worker-a', + inspectionId: 'inspection-a', + }, + }); + assert.equal(Object.isFrozen(inspect), true); + assert.equal(Object.isFrozen(inspect.request), true); + + const page = createWorkerSessionListCommand( + 'project-a', + 'worker-a', + () => 'inspection-b', + ); + assert.deepEqual(page.request, { + authorityProjectId: 'project-a', + afterWorkerId: 'worker-a', + inspectionId: 'inspection-b', + }); + assert.equal(Object.hasOwn(page.request, 'limit'), false); + assert.throws(() => createWorkerSessionListCommand('../escape')); + assert.throws(() => + normalizeClusterWorkerManagementCommand({ + schemaVersion: 1, + operation: 'worker-credential.inspect', + request: { + actionRef: 'action-a', + authorityProjectId: 'project-a', + approvalRequestId: 'approval-a', + inspectionId: 'inspection-a', + }, + }), + ); +}); + +test('projects strict low-sensitive products without transport request identity', () => { + const inspection = projectWorkerSessionInspection( + 'project-a', + response({ + schemaVersion: 1, + operation: 'worker-session.inspect', + observedAtMs: 1_100, + worker: detailedWorker, + }), + ); + assert.equal(inspection.schema, 'qinglong/worker-session-inspection@v1'); + assert.equal(inspection.found, true); + assert.equal(inspection.worker.workerId, 'worker-a'); + assert.equal(Object.isFrozen(inspection.worker.runtimes), true); + assert.doesNotMatch( + JSON.stringify(inspection), + /transport-request|inspectionId/, + ); + assert.match( + formatWorkerSessionInspectionCard(inspection), + /slots available/, + ); + + const page = projectWorkerSessionList( + 'project-a', + response({ + schemaVersion: 1, + operation: 'worker-session.list', + observedAtMs: 1_100, + workers: [summaryWorker], + nextCursor: 'worker-a', + }), + ); + assert.deepEqual( + { + schema: page.schema, + count: page.count, + nextAfterWorkerId: page.nextAfterWorkerId, + }, + { + schema: 'qinglong/worker-session-list@v1', + count: 1, + nextAfterWorkerId: 'worker-a', + }, + ); + assert.match(formatWorkerSessionListCard(page), /worker-a online/); + assert.throws(() => + projectWorkerSessionInspection( + 'project-a', + response({ + schemaVersion: 1, + operation: 'worker-session.list', + observedAtMs: 1_100, + workers: [], + nextCursor: null, + }), + ), + ); +}); + +test('requires the canonical Worker endpoint before making a connection', async (t) => { + const directory = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-worker-product-client-')), + ); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const caFile = privateFile( + directory, + 'ca.pem', + fs.readFileSync(path.join(fixtureRoot, 'ca-cert.pem')), + ); + const clientCertificateFile = privateFile( + directory, + 'client.crt', + fs.readFileSync(path.join(fixtureRoot, 'client-cert.pem')), + ); + const clientPrivateKeyFile = privateFile( + directory, + 'client.key', + fs.readFileSync(path.join(fixtureRoot, 'client-key.pem')), + ); + const assertionFile = privateFile( + directory, + 'assertion.jwt', + 'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl', + ); + const command = createWorkerSessionListCommand( + 'project-a', + undefined, + () => 'inspection-a', + ); + + for (const [managementPath, expectedConnections] of [ + ['/api/v3/workers/management', 1], + ['/api/v3/worker-credentials/management', 0], + ]) { + const configFile = privateFile( + directory, + `client-${expectedConnections}.json`, + JSON.stringify({ + schemaVersion: 1, + endpoint: `https://manager.example.test:8443${managementPath}`, + servername: 'manager.example.test', + caFile, + clientCertificateFile, + clientPrivateKeyFile, + requestTimeoutMs: 1_000, + }), + ); + let connections = 0; + await assert.rejects( + executeClusterWorkerManagementClient( + { configFile, assertionFile, command }, + { + async connect() { + connections += 1; + throw new Error('stop-after-policy-validation'); + }, + }, + ), + ); + assert.equal(connections, expectedConnections); + } +}); diff --git a/packages/ql3-cluster-admin/test/workerManagementClientCli.test.cjs b/packages/ql3-cluster-admin/test/workerManagementClientCli.test.cjs new file mode 100644 index 00000000..9845801e --- /dev/null +++ b/packages/ql3-cluster-admin/test/workerManagementClientCli.test.cjs @@ -0,0 +1,256 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { spawn } = require('node:child_process'); +const fs = require('node:fs'); +const { createServer } = require('node:https'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); + +const packageRoot = path.resolve(__dirname, '..'); +const cliPath = path.join( + packageRoot, + 'dist', + 'worker-management', + 'workerManagementClientCli.js', +); +const fixtureRoot = path.resolve( + packageRoot, + '../ql3-cluster-control/test/fixtures/mtls', +); + +function privateFile(directory, name, value) { + const filePath = path.join(directory, name); + fs.writeFileSync(filePath, value, { mode: 0o600 }); + return fs.realpathSync(filePath); +} + +function runCli(args) { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [cliPath, ...args], { + cwd: packageRoot, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const stdout = []; + const stderr = []; + child.stdout.on('data', (chunk) => stdout.push(chunk)); + child.stderr.on('data', (chunk) => stderr.push(chunk)); + child.once('error', reject); + child.once('close', (status, signal) => { + resolve({ + status, + signal, + stdout: Buffer.concat(stdout).toString('utf8'), + stderr: Buffer.concat(stderr).toString('utf8'), + }); + }); + }); +} + +function summary(workerId) { + return { + workerId, + sessionId: `session-${workerId}`, + generation: 1, + sessionVersion: 1, + lifecycle: 'online', + compatibility: 'default_placement', + architecture: 'arm64', + supportTier: 'tier1', + protocolVersion: '1.0.0', + operatingSystem: 'linux', + maxConcurrentRuns: 1, + availableSlots: 1, + registeredAtMs: 900, + lastHeartbeatAtMs: 1_000, + leaseExpiresAtMs: 2_000, + updatedAtMs: 1_000, + observedAtMs: 1_100, + }; +} + +test('CLI performs one canonical read per invocation and rejects mutation vocabulary', async (t) => { + const directory = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-worker-cli-')), + ); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const requests = []; + const server = createServer( + { + key: fs.readFileSync(path.join(fixtureRoot, 'server-key.pem')), + cert: fs.readFileSync(path.join(fixtureRoot, 'server-cert.pem')), + ca: fs.readFileSync(path.join(fixtureRoot, 'ca-cert.pem')), + requestCert: true, + rejectUnauthorized: true, + minVersion: 'TLSv1.3', + maxVersion: 'TLSv1.3', + }, + (request, response) => { + const chunks = []; + request.on('data', (chunk) => chunks.push(chunk)); + request.on('end', () => { + const command = JSON.parse(Buffer.concat(chunks).toString('utf8')); + requests.push({ + method: request.method, + path: request.url, + authorized: request.socket.authorized, + command, + }); + const worker = summary('worker-a'); + const result = + command.operation === 'worker-session.inspect' + ? { + schemaVersion: 1, + operation: 'worker-session.inspect', + observedAtMs: 1_100, + worker: { + ...worker, + runtimes: [{ name: 'node', version: '24.18.0' }], + declaredCapacity: { + cpuCores: 1, + memoryBytes: 268_435_456, + diskBytes: 1_073_741_824, + gpuCount: 0, + }, + }, + } + : { + schemaVersion: 1, + operation: 'worker-session.list', + observedAtMs: 1_100, + workers: [worker], + nextCursor: null, + }; + const body = Buffer.from( + JSON.stringify({ + schemaVersion: 1, + requestId: 'server-request-hidden', + result, + }), + ); + response.writeHead(200, { + 'content-type': 'application/json; charset=utf-8', + 'content-length': String(body.length), + }); + response.end(body); + }); + }, + ); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + t.after( + () => + new Promise((resolve) => { + server.close(() => resolve()); + }), + ); + const address = server.address(); + assert.notEqual(address, null); + assert.notEqual(typeof address, 'string'); + const caFile = privateFile( + directory, + 'ca.pem', + fs.readFileSync(path.join(fixtureRoot, 'ca-cert.pem')), + ); + const clientCertificateFile = privateFile( + directory, + 'client.crt', + fs.readFileSync(path.join(fixtureRoot, 'client-cert.pem')), + ); + const clientPrivateKeyFile = privateFile( + directory, + 'client.key', + fs.readFileSync(path.join(fixtureRoot, 'client-key.pem')), + ); + const configFile = privateFile( + directory, + 'client.json', + JSON.stringify({ + schemaVersion: 1, + endpoint: `https://localhost:${address.port}/api/v3/workers/management`, + servername: 'localhost', + caFile, + clientCertificateFile, + clientPrivateKeyFile, + requestTimeoutMs: 2_000, + }), + ); + const assertionFile = privateFile( + directory, + 'assertion.jwt', + 'eyJhbGciOiJFZERTQSJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl', + ); + + const inspect = await runCli([ + 'inspect', + `--config=${configFile}`, + `--assertion=${assertionFile}`, + '--project=project-a', + '--worker=worker-a', + '--format=json', + ]); + assert.equal(inspect.status, 0, inspect.stderr); + assert.equal(inspect.stderr, ''); + const inspection = JSON.parse(inspect.stdout); + assert.equal(inspection.schema, 'qinglong/worker-session-inspection@v1'); + assert.equal(inspection.worker.workerId, 'worker-a'); + assert.equal(inspect.stdout.includes('server-request-hidden'), false); + + const list = await runCli([ + 'list', + `--config=${configFile}`, + `--assertion=${assertionFile}`, + '--project=project-a', + '--format=json', + ]); + assert.equal(list.status, 0, list.stderr); + assert.equal(JSON.parse(list.stdout).count, 1); + assert.equal(requests.length, 2); + assert.deepEqual( + requests.map(({ method, path, authorized, command }) => ({ + method, + path, + authorized, + operation: command.operation, + })), + [ + { + method: 'POST', + path: '/api/v3/workers/management', + authorized: true, + operation: 'worker-session.inspect', + }, + { + method: 'POST', + path: '/api/v3/workers/management', + authorized: true, + operation: 'worker-session.list', + }, + ], + ); + + const rejected = await runCli([ + 'inspect', + `--config=${configFile}`, + `--assertion=${assertionFile}`, + '--project=project-a', + '--worker=worker-a', + '--command=/private/mutation.json', + ]); + assert.equal(rejected.status, 64); + assert.equal(requests.length, 2); + assert.equal(rejected.stdout, ''); + assert.equal(JSON.parse(rejected.stderr).event, 'usage_invalid'); +}); + +test('CLI help states its bounded read contract', async () => { + const help = await runCli(['--help']); + assert.equal(help.status, 0); + assert.match(help.stdout, /ql3-worker-client inspect/); + assert.match(help.stdout, /ql3-worker-client list/); + assert.match(help.stdout, /never retries, polls or auto-pages/); + assert.doesNotMatch(help.stdout, /credential|secret|token/i); +}); diff --git a/scripts/ql3-cluster-deployment-audit.cjs b/scripts/ql3-cluster-deployment-audit.cjs index e7bc0923..d213b422 100644 --- a/scripts/ql3-cluster-deployment-audit.cjs +++ b/scripts/ql3-cluster-deployment-audit.cjs @@ -475,6 +475,8 @@ function assertExactExternalClosure(readFile, root, findings) { 'dist/worker-credential/management-server/workerCredentialManagementCli.js' || adminManifest.bin?.['ql3-worker-credential-client'] !== 'dist/worker-credential/workerCredentialManagementClientCli.js' || + adminManifest.bin?.['ql3-worker-client'] !== + 'dist/worker-management/workerManagementClientCli.js' || adminManifest.bin?.['ql3-worker-credential-execute'] !== 'dist/worker-credential/workerCredentialExecutorCli.js' || adminManifest.exports?.['./plugin-package-recovery-process']?.require !== @@ -498,13 +500,17 @@ function assertExactExternalClosure(readFile, root, findings) { adminManifest.exports?.['./worker-credential-management-client'] ?.require !== './dist/worker-credential/workerCredentialManagementClient.js' || + adminManifest.exports?.['./worker-management-client']?.require !== + './dist/worker-management/workerManagementClient.js' || + adminManifest.exports?.['./worker-management-product']?.require !== + './dist/worker-management/workerManagementProduct.js' || adminManifest.exports?.['./worker-credential-executor-process']?.require !== './dist/worker-credential/workerCredentialExecutorProcess.js' ) { findings.push( finding( 'QL3_CLUSTER_PLUGIN_RECOVERY_ENTRYPOINT_MISSING', - 'cluster-admin must publish the reviewed product facade, Package and Worker management and executor entrypoints', + 'cluster-admin must publish the reviewed product facade, Package and credential-compatible/read-only Worker management and executor entrypoints', ), ); } diff --git a/test/back/ql3ClusterDeploymentAudit.test.cjs b/test/back/ql3ClusterDeploymentAudit.test.cjs index 911704f1..a837e55c 100644 --- a/test/back/ql3ClusterDeploymentAudit.test.cjs +++ b/test/back/ql3ClusterDeploymentAudit.test.cjs @@ -211,6 +211,7 @@ test('ships a path-only Cluster operator context example without durable authori 'package', 'package-kubernetes', 'run', + 'worker', 'worker-credential', ]); for (const [name, command] of Object.entries(example.commands)) { @@ -753,13 +754,16 @@ test('rejects widened authority or lifecycle in the Worker management client', ( } }); -test('requires the production Worker credential management client export and binary', () => { +test('requires both read-only Worker and credential-compatible client entrypoints', () => { const report = auditClusterDeployment({ root: ROOT, readFile: intercept('packages/ql3-cluster-admin/package.json', (source) => { const manifest = JSON.parse(source); delete manifest.bin['ql3-worker-credential-client']; + delete manifest.bin['ql3-worker-client']; delete manifest.exports['./worker-credential-management-client']; + delete manifest.exports['./worker-management-client']; + delete manifest.exports['./worker-management-product']; return JSON.stringify(manifest); }), }); diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index fbfde0b8..933da833 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -340,10 +340,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: clusterAdmin.rootSourceFileRoles, }, { - sourceFiles: 125, + sourceFiles: 128, rootSourceFiles: 1, rootSourceLines: 61, - nestedSourceFiles: 124, + nestedSourceFiles: 127, rootSourceFileRoles: { 'modelInvocationMigrationCli.ts': 'binary_entry', },