From a83b4c5e8d94a6134ea934b184dddd0fb27837c8 Mon Sep 17 00:00:00 2001 From: whyour Date: Thu, 13 Aug 2026 01:19:23 +0800 Subject: [PATCH] feat(ql3): add offline cluster context preflight --- deploy/kubernetes/ql3-cluster/README.md | 15 + docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 12 + ...line-cluster-operator-context-preflight.md | 42 +++ docs/adr/README.md | 1 + .../pluginPackageManagementClient.ts | 276 +++++++++++++----- ...pluginPackageManagementKubernetesClient.ts | 134 +++++++-- .../ql3-cluster-admin/src/product-cli/cli.ts | 12 +- .../src/product-cli/productCommand.ts | 26 ++ .../src/product-cli/productContext.ts | 85 ++++++ .../test/productCli.test.cjs | 264 +++++++++++++++++ ...l3-cluster-admin-product-live-contract.cjs | 13 +- scripts/ql3-cluster-image-release-audit.cjs | 5 +- .../back/ql3ClusterImageReleaseAudit.test.cjs | 19 ++ 13 files changed, 805 insertions(+), 99 deletions(-) create mode 100644 docs/adr/ADR-0390-offline-cluster-operator-context-preflight.md diff --git a/deploy/kubernetes/ql3-cluster/README.md b/deploy/kubernetes/ql3-cluster/README.md index 76382625..680803ff 100644 --- a/deploy/kubernetes/ql3-cluster/README.md +++ b/deploy/kubernetes/ql3-cluster/README.md @@ -69,6 +69,21 @@ contents are deliberately forbidden. There is no home-directory, environment or ambient Kubernetes context discovery. Existing explicit `--config` calls remain supported when `--context` is absent. +Validate the complete local operator context before a maintenance window: + +```sh +ql3-cluster-admin context validate \ + --context=/secure/qinglong3/operator-context.json +``` + +This command reuses the production client parsers to validate every selected +endpoint shape, CA, matching client certificate/private key and the bounded +Kubernetes config. It also verifies that all seven facade targets exist in the +same installation. It does not read a command or assertion, open a network +connection, query Kubernetes or mutate the cluster. Its JSON summary contains +only command names and reviewed transport/authentication classes; paths, +endpoints, namespaces, context names and credentials are never emitted. + The runtime image installs only the 43 external packages reachable from the five exact production roots in its production-only lock plus `runtime-core`, `cluster-postgres` and `cluster-control`: 46 runtime components in total. The diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 89e8cc5c..eca2b602 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,18 @@ 最新增量证据（2026-08-13）： +- D-302/ADR-0390（已接受） + Cluster operator context 增加无网络、无 mutation 的内建 `ql3-cluster-admin context validate` 预检。它先复用 owner-private context + reader，再让每个 entry 经过与真实请求相同的 production HTTPS/Kubernetes configuration preparation，验证精确 route、hostname、CA、 + client certificate/private key 配对、embedded Kubernetes credential/config 与同安装七项 client target；只有真实命令才读取 command/ + assertion、创建 Pod client、PortForward、socket 或 deadline timer。成功结果只暴露固定 command、transport/authentication class 与 + `networkAccess:false/mutation:false`，不输出路径、endpoint、port、namespace、context、证书主体或 credential。实现仍内聚在既有 + Cluster Admin product/client 目录，不新增 package、依赖、binary、listener、timer、数据库或 workload，Local/Edge、Control、Worker 零导入。 + product/preflight 11/11、七类 client/tunnel 定向 45/45；Cluster Admin 完整 296 pass/2 条件 skip，18-package clean build/test 退出 0， + backend 1,187 pass/2 skip，五项边界审计零 finding。真实 arm64 Admin image 为 330,463,528 bytes，较 D-301 增加 10,219 bytes，并在 + `10001:10001`、network none、read-only root、drop ALL、128 MiB/32 PIDs 下报告 `contextPreflight=true`。14 个 Local Profile artifact + 字节数与 D-301 完全一致。PostgreSQL 18.4 arm64 HA 123 项 gate 全绿、timeline `1→2`，证据 SHA-256 为 + `339cd10e1da2428da6c099c52c2397d5f79f7cb32b64b7e1ae927d2803b8cfc0`，离线审计及 Docker 清理通过。完整证据见 ADR-0390。 - D-301/ADR-0389（已接受） `ql3-cluster-admin` 现在可通过显式 `--context=/absolute/operator-context.json` 复用七个远程 client 的稳定路径，同时保持每次 command 与短生命周期强 assertion 必须显式传入。schema v1 只允许 catalog command → `configFile`，Kubernetes tunnel 精确多一个 diff --git a/docs/adr/ADR-0390-offline-cluster-operator-context-preflight.md b/docs/adr/ADR-0390-offline-cluster-operator-context-preflight.md new file mode 100644 index 00000000..d783a2f5 --- /dev/null +++ b/docs/adr/ADR-0390-offline-cluster-operator-context-preflight.md @@ -0,0 +1,42 @@ +# ADR-0390：Offline Cluster Operator Context Preflight + +- 状态：Accepted +- 日期：2026-08-13 +- 关联 RFC：QL-RFC-0001 D-302 +- 前置决策：ADR-0250、ADR-0388、ADR-0389 + +## 上下文 + +ADR-0389 让操作者可以显式复用稳定 client 路径，但 context reader 只证明路径、owner、权限和 schema；endpoint route、CA、client certificate/private key 配对及 Kubernetes config 仍要到某次真实命令才由生产 client 发现。发布或维护窗口中用 mutation 命令做配置探针既迟又危险，另写一套宽松校验器则会与真实请求路径发生语义漂移。 + +## 决策 + +1. `ql3-cluster-admin` 增加内建 `context validate --context=/absolute/operator-context.json`。它是 facade 的本地命令，不进入七项远程 client catalog，不产生新的 binary、package、进程或部署 authority。 +2. context 仍先通过 ADR-0389 的 canonical/current-UID/`0600`/no-follow reader。随后每个 entry 必须调用与真实 client 相同的 production configuration preparation：精确 route、HTTPS hostname、timeout、CA、client certificate/private key 配对及 Kubernetes embedded credential/config 全部同语义验证。 +3. production HTTPS client 把配置读取、解析和密钥配对提炼为可显式 `dispose()` 的 preparation；真实请求与 preflight 共同调用。Kubernetes tunnel 同样提炼 config preparation；只有真实命令才创建 Pod client、PortForward、socket 和 deadline timer。 +4. preflight 不读取 per-operation command 或 assertion，不打开 socket、不访问 DNS/HTTP/Kubernetes、不创建 timer、不查询数据库且不执行 mutation。它还复验同一安装内七个受审 client target 均为 canonical non-symlink regular file。 +5. 成功结果只输出固定 command 名、`https|kubernetes-port-forward`、是否要求 client certificate、Kubernetes credential class，以及 `networkAccess:false/mutation:false`。禁止输出路径、endpoint、port、servername、namespace、Kubernetes context、证书主体、token 或其他 credential 内容。 +6. 任一 context、安装或引用配置失败均失败关闭。context/config 类错误统一返回低敏 `QL3_CLUSTER_PRODUCT_CONTEXT_INVALID` 与退出 78；CLI 语法错误返回既有 usage code 与退出 64；不暴露失败 entry 或路径。 +7. 能力继续只存在于短生命周期 Cluster Admin image。Local/Edge、Cluster Control 与 Worker 的依赖、文件、模块、RSS、listener、timer、连接和镜像闭包不得变化。 + +## 不采用方案 + +- **发送 health/read-only 请求验证连接**：仍会产生外部副作用、泄露使用时机，也不能证明 mutation 路径的全部本地材料。 +- **复用某个虚构 command/assertion 执行真实 client**：会混淆配置错误与业务拒绝，并可能进入审计、quota 或 mutation 前置流程。 +- **复制 JSON schema 到 facade**：两套 parser 会随 route、TLS 或 Kubernetes contract 演进而漂移。 +- **把 endpoint 或证书主体写入成功摘要**：这些事实对“本地材料可解析”结论没有必要，会扩大日志敏感面。 +- **为 preflight 新建 package**：它只有一个 Cluster Admin 产品消费者，不形成部署、authority 或供应链边界。 + +## 验收门 + +- 七命令完整 context、部分 context、mTLS/非 mTLS、Kubernetes token/certificate、错误 route、CA、key 配对、权限、symlink、未知字段与低敏失败； +- 生产 HTTPS/Kubernetes client 全量回归，证明提炼前后请求、TLS、PortForward 和错误语义不变； +- 真实 Admin image 在 network none、non-root、read-only root、drop ALL、no-new-privileges、128 MiB/32 PIDs 下完成 preflight； +- package/dependency/deployment/image release/Local image、18-package clean build/test、backend、14 Local Profile artifact 与 PostgreSQL HA 不回归。 + +## 当前证据 + +- product facade/preflight 11/11；七类 client 与 Kubernetes tunnel 定向回归 45/45。Cluster Admin 完整回归 298 tests、296 pass/2 条件 skip/0 fail；18-package clean build/test 退出 0；backend 1,189 tests、1,187 pass/2 skip/0 fail。 +- 真实 arm64 Admin image 为 330,463,528 bytes，较 D-301 增加 10,219 bytes；在 `10001:10001`、read-only root、network none、drop ALL、no-new-privileges、0.25 CPU、128 MiB/32 PIDs 下报告 `contextPreflight=true`。五项 package/dependency/deployment/image release/Local image 边界审计零 finding。 +- workspace 保持 18 package；Cluster Admin 97 source 中 96 nested/1 root，无 single-source/shallow package且未增加依赖。14 个 Local Profile artifact 全部 compatible，最小 Edge 为 2,467,343 bytes/295 files/53 modules，最大 Standalone MCP 为 7,168,978 bytes/778 files/213 modules；与 D-301 对应制品字节数一致。 +- PostgreSQL 18.4 arm64 HA 123 项 gate 全绿，timeline `1→2`；报告 SHA-256 为 `339cd10e1da2428da6c099c52c2397d5f79f7cb32b64b7e1ae927d2803b8cfc0`，离线审计零 finding，门禁容器、网络与卷均零残留。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 79695949..024800f1 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -393,6 +393,7 @@ | [ADR-0387](./ADR-0387-local-api-cancellation-linux-live-evidence.md) | Local API Cancellation Linux 组合实证门 | Accepted | | [ADR-0388](./ADR-0388-bounded-unified-cluster-product-cli.md) | 有界统一 Cluster Operator 产品 CLI | Accepted | | [ADR-0389](./ADR-0389-owner-private-cluster-operator-context.md) | Owner-private Cluster Operator Context | Accepted | +| [ADR-0390](./ADR-0390-offline-cluster-operator-context-preflight.md) | Offline Cluster Operator Context Preflight | Accepted | ## 规则 diff --git a/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementClient.ts b/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementClient.ts index 52bc3c6d..def78f09 100644 --- a/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementClient.ts +++ b/packages/ql3-cluster-admin/src/management-support/pluginPackageManagementClient.ts @@ -23,14 +23,48 @@ import { } from '../plugin-package/management/pluginPackageManagementTransport'; const MANAGEMENT_PATH = '/api/v3/plugin-packages/management'; -const ALLOWED_MANAGEMENT_PATHS = new Set([ - MANAGEMENT_PATH, - '/api/v3/worker-credentials/management', - '/api/v3/automations/management', - '/api/v3/approvals/management', - '/api/v3/provider-credentials/management', - '/api/v3/runs/management', -]); +export type ClusterAuthenticatedManagementClientKind = + | 'package' + | 'worker-credential' + | 'automation' + | 'approval' + | 'model-credential' + | 'run'; + +const MANAGEMENT_CLIENT_POLICIES: Readonly< + Record< + ClusterAuthenticatedManagementClientKind, + Readonly<{ + managementPath: string; + clientCertificate: 'forbidden' | 'required'; + }> + > +> = Object.freeze({ + package: Object.freeze({ + managementPath: MANAGEMENT_PATH, + clientCertificate: 'forbidden', + }), + 'worker-credential': Object.freeze({ + managementPath: '/api/v3/worker-credentials/management', + clientCertificate: 'required', + }), + automation: Object.freeze({ + managementPath: '/api/v3/automations/management', + clientCertificate: 'required', + }), + approval: Object.freeze({ + managementPath: '/api/v3/approvals/management', + clientCertificate: 'required', + }), + 'model-credential': Object.freeze({ + managementPath: '/api/v3/provider-credentials/management', + clientCertificate: 'required', + }), + run: Object.freeze({ + managementPath: '/api/v3/runs/management', + clientCertificate: 'required', + }), +}); const MAX_CONFIG_BYTES = 16 * 1024; const MAX_ASSERTION_BYTES = 16 * 1024; const MAX_COMMAND_BYTES = 256 * 1024; @@ -778,68 +812,48 @@ function rawHeaderCount(rawHeaders: readonly string[], name: string): number { return count; } -export async function executeClusterAuthenticatedManagementClient< - Command, - Result, ->( - paths: ClusterPluginPackageManagementClientPaths, - protocol: ClusterAuthenticatedManagementClientProtocol, - connectionOptions?: ClusterPluginPackageManagementClientConnectionOptions, -): Promise>> { - exactObject(paths, ['configFile', 'commandFile', 'assertionFile']); +export interface ClusterAuthenticatedManagementClientConfigurationSummary { + readonly schemaVersion: 1; + readonly managementPath: string; + readonly transport: 'https'; + readonly clientCertificate: 'forbidden' | 'required'; +} + +interface PreparedClusterAuthenticatedManagementClientConfiguration { + readonly endpoint: URL; + readonly servername: string; + readonly port: number; + readonly requestTimeoutMs: number; + readonly caBytes: Buffer; + readonly clientCertificateBytes?: Buffer; + readonly clientPrivateKeyBytes?: Buffer; + dispose(): void; +} + +function prepareClusterAuthenticatedManagementClientConfiguration( + configFile: string, + managementPath: string, + clientCertificate: 'forbidden' | 'required', +): PreparedClusterAuthenticatedManagementClientConfiguration { if ( - !protocol || - typeof protocol !== 'object' || - Array.isArray(protocol) || - Object.keys(protocol).length !== 4 || - Object.keys(protocol).some( - (key) => - ![ - 'managementPath', - 'clientCertificate', - 'normalizeCommand', - 'validateResult', - ].includes(key), - ) || - !ALLOWED_MANAGEMENT_PATHS.has(protocol.managementPath) || - !['forbidden', 'required'].includes(protocol.clientCertificate) || - typeof protocol.normalizeCommand !== 'function' || - typeof protocol.validateResult !== 'function' || - (connectionOptions !== undefined && - (!connectionOptions || - typeof connectionOptions !== 'object' || - Array.isArray(connectionOptions) || - Object.keys(connectionOptions).length !== 1 || - typeof connectionOptions.connect !== 'function')) + !Object.values(MANAGEMENT_CLIENT_POLICIES).some( + (policy) => + policy.managementPath === managementPath && + policy.clientCertificate === clientCertificate, + ) ) { throw configurationFailure(); } let configBytes: Buffer | undefined; - let commandBytes: Buffer | undefined; - let assertionBytes: Buffer | undefined; let caBytes: Buffer | undefined; let clientCertificateBytes: Buffer | undefined; let clientPrivateKeyBytes: Buffer | undefined; try { - configBytes = readCanonicalFile( - paths.configFile, - MAX_CONFIG_BYTES, - 'private', - ); - commandBytes = readCanonicalFile( - paths.commandFile, - MAX_COMMAND_BYTES, - 'private', - ); - assertionBytes = readCanonicalFile( - paths.assertionFile, - MAX_ASSERTION_BYTES, - 'private', - ); + configBytes = readCanonicalFile(configFile, MAX_CONFIG_BYTES, 'private'); const config = parseJson(configBytes); exactObject( config, - protocol.clientCertificate === 'required' + clientCertificate === 'required' ? [ 'schemaVersion', 'endpoint', @@ -864,7 +878,7 @@ export async function executeClusterAuthenticatedManagementClient< !DNS_NAME_PATTERN.test(config.servername) || isIP(config.servername) !== 0 || typeof config.caFile !== 'string' || - (protocol.clientCertificate === 'required' && + (clientCertificate === 'required' && (typeof config.clientCertificateFile !== 'string' || typeof config.clientPrivateKeyFile !== 'string')) || !Number.isSafeInteger(config.requestTimeoutMs) || @@ -874,7 +888,6 @@ export async function executeClusterAuthenticatedManagementClient< throw configurationFailure(); } const servername = config.servername; - const caFile = config.caFile; const requestTimeoutMs = config.requestTimeoutMs as number; let endpoint: URL; try { @@ -888,7 +901,7 @@ export async function executeClusterAuthenticatedManagementClient< endpoint.password !== '' || endpoint.search !== '' || endpoint.hash !== '' || - endpoint.pathname !== protocol.managementPath || + endpoint.pathname !== managementPath || endpoint.hostname !== servername || isIP(endpoint.hostname) !== 0 ) { @@ -898,13 +911,17 @@ export async function executeClusterAuthenticatedManagementClient< if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) { throw configurationFailure(); } - caBytes = readCanonicalFile(caFile, MAX_CA_BYTES, 'public-integrity'); + caBytes = readCanonicalFile( + config.caFile as string, + MAX_CA_BYTES, + 'public-integrity', + ); try { new X509Certificate(caBytes); } catch { throw configurationFailure(); } - if (protocol.clientCertificate === 'required') { + if (clientCertificate === 'required') { clientCertificateBytes = readCanonicalFile( config.clientCertificateFile as string, MAX_CLIENT_CERTIFICATE_BYTES, @@ -931,6 +948,134 @@ export async function executeClusterAuthenticatedManagementClient< throw configurationFailure(); } } + let disposed = false; + return Object.freeze({ + endpoint, + servername, + port, + requestTimeoutMs, + caBytes, + ...(clientCertificateBytes === undefined + ? {} + : { + clientCertificateBytes, + clientPrivateKeyBytes: clientPrivateKeyBytes!, + }), + dispose() { + if (disposed) return; + disposed = true; + caBytes?.fill(0); + clientCertificateBytes?.fill(0); + clientPrivateKeyBytes?.fill(0); + }, + }); + } catch (error) { + caBytes?.fill(0); + clientCertificateBytes?.fill(0); + clientPrivateKeyBytes?.fill(0); + if ( + error instanceof ClusterPluginPackageManagementClientConfigurationError + ) { + throw error; + } + throw configurationFailure(); + } finally { + configBytes?.fill(0); + } +} + +export function validateClusterAuthenticatedManagementClientConfiguration( + configFile: string, + kind: ClusterAuthenticatedManagementClientKind, +): Readonly { + const policy = MANAGEMENT_CLIENT_POLICIES[kind]; + if (policy === undefined) throw configurationFailure(); + const prepared = prepareClusterAuthenticatedManagementClientConfiguration( + configFile, + policy.managementPath, + policy.clientCertificate, + ); + try { + return Object.freeze({ + schemaVersion: 1, + managementPath: policy.managementPath, + transport: 'https', + clientCertificate: policy.clientCertificate, + }); + } finally { + prepared.dispose(); + } +} + +export async function executeClusterAuthenticatedManagementClient< + Command, + Result, +>( + paths: ClusterPluginPackageManagementClientPaths, + protocol: ClusterAuthenticatedManagementClientProtocol, + connectionOptions?: ClusterPluginPackageManagementClientConnectionOptions, +): Promise>> { + exactObject(paths, ['configFile', 'commandFile', 'assertionFile']); + if ( + !protocol || + typeof protocol !== 'object' || + Array.isArray(protocol) || + Object.keys(protocol).length !== 4 || + Object.keys(protocol).some( + (key) => + ![ + 'managementPath', + 'clientCertificate', + 'normalizeCommand', + 'validateResult', + ].includes(key), + ) || + !Object.values(MANAGEMENT_CLIENT_POLICIES).some( + (policy) => + policy.managementPath === protocol.managementPath && + policy.clientCertificate === protocol.clientCertificate, + ) || + typeof protocol.normalizeCommand !== 'function' || + typeof protocol.validateResult !== 'function' || + (connectionOptions !== undefined && + (!connectionOptions || + typeof connectionOptions !== 'object' || + Array.isArray(connectionOptions) || + Object.keys(connectionOptions).length !== 1 || + typeof connectionOptions.connect !== 'function')) + ) { + throw configurationFailure(); + } + let commandBytes: Buffer | undefined; + let assertionBytes: Buffer | undefined; + let prepared: + | PreparedClusterAuthenticatedManagementClientConfiguration + | undefined; + try { + prepared = prepareClusterAuthenticatedManagementClientConfiguration( + paths.configFile, + protocol.managementPath, + protocol.clientCertificate, + ); + commandBytes = readCanonicalFile( + paths.commandFile, + MAX_COMMAND_BYTES, + 'private', + ); + assertionBytes = readCanonicalFile( + paths.assertionFile, + MAX_ASSERTION_BYTES, + 'private', + ); + const { + endpoint, + servername, + port, + requestTimeoutMs, + caBytes, + clientCertificateBytes, + clientPrivateKeyBytes, + } = prepared; const command = protocol.normalizeCommand(parseJson(commandBytes)); const assertion = assertionBytes.toString('ascii'); if ( @@ -1186,12 +1331,9 @@ export async function executeClusterAuthenticatedManagementClient< } } } finally { - configBytes?.fill(0); commandBytes?.fill(0); assertionBytes?.fill(0); - caBytes?.fill(0); - clientCertificateBytes?.fill(0); - clientPrivateKeyBytes?.fill(0); + prepared?.dispose(); } } diff --git a/packages/ql3-cluster-admin/src/plugin-package/management/pluginPackageManagementKubernetesClient.ts b/packages/ql3-cluster-admin/src/plugin-package/management/pluginPackageManagementKubernetesClient.ts index 85c70d18..3501e776 100644 --- a/packages/ql3-cluster-admin/src/plugin-package/management/pluginPackageManagementKubernetesClient.ts +++ b/packages/ql3-cluster-admin/src/plugin-package/management/pluginPackageManagementKubernetesClient.ts @@ -49,6 +49,20 @@ interface ReviewedKubernetesClientConfig { readonly apiTimeoutMs: number; } +export interface ClusterPluginPackageManagementKubernetesConfigurationSummary { + readonly schemaVersion: 1; + readonly transport: 'kubernetes-port-forward'; + readonly authentication: 'token' | 'client-certificate'; +} + +interface PreparedKubernetesClientConfiguration { + readonly config: Readonly; + readonly kubeConfig: KubernetesConfig; + readonly kubernetes: KubernetesModule; + readonly authentication: 'token' | 'client-certificate'; + dispose(): void; +} + interface KubernetesPod { readonly metadata?: { readonly name?: string; @@ -450,6 +464,95 @@ function validateKubeConfig( } } +async function prepareKubernetesClientConfiguration( + kubernetesFile: string, +): Promise { + let kubernetesConfigBytes: Buffer | undefined; + let kubeconfigBytes: Buffer | undefined; + try { + kubernetesConfigBytes = readPrivateFile( + kubernetesFile, + MAX_KUBERNETES_CONFIG_BYTES, + ); + const config = normalizeConfig(parseJson(kubernetesConfigBytes)); + kubeconfigBytes = readPrivateFile( + config.kubeconfigFile, + MAX_KUBECONFIG_BYTES, + ); + const rawKubeconfig = parseJson(kubeconfigBytes); + validateRawKubeconfig(rawKubeconfig, config); + let kubernetes: KubernetesModule; + try { + kubernetes = await import('@kubernetes/client-node'); + } catch (error) { + throw new ClusterPluginPackageManagementKubernetesClientTunnelError( + error, + ); + } + const kubeConfig = new kubernetes.KubeConfig(); + try { + kubeConfig.loadFromString(decodeUtf8(kubeconfigBytes)); + validateKubeConfig(kubeConfig, config); + } catch (error) { + if ( + error instanceof + ClusterPluginPackageManagementKubernetesClientConfigurationError + ) { + throw error; + } + throw configurationFailure(); + } + const rawUser = (rawKubeconfig as JsonObject).users as readonly JsonObject[]; + const authentication = Object.hasOwn( + rawUser[0]!.user as object, + 'token', + ) + ? 'token' + : 'client-certificate'; + let disposed = false; + return Object.freeze({ + config, + kubeConfig, + kubernetes, + authentication, + dispose() { + if (disposed) return; + disposed = true; + kubernetesConfigBytes?.fill(0); + kubeconfigBytes?.fill(0); + }, + }); + } catch (error) { + kubernetesConfigBytes?.fill(0); + kubeconfigBytes?.fill(0); + if ( + error instanceof + ClusterPluginPackageManagementKubernetesClientConfigurationError || + error instanceof ClusterPluginPackageManagementKubernetesClientTunnelError + ) { + throw error; + } + throw configurationFailure(); + } +} + +export async function validateClusterPluginPackageManagementKubernetesConfiguration( + kubernetesFile: string, +): Promise< + Readonly +> { + const prepared = await prepareKubernetesClientConfiguration(kubernetesFile); + try { + return Object.freeze({ + schemaVersion: 1, + transport: 'kubernetes-port-forward', + authentication: prepared.authentication, + }); + } finally { + prepared.dispose(); + } +} + function isReviewedPod( value: KubernetesPod, namespace: string, @@ -681,34 +784,12 @@ export async function executeClusterPluginPackageManagementKubernetesClient( throw configurationFailure(); } - let kubernetesConfigBytes: Buffer | undefined; - let kubeconfigBytes: Buffer | undefined; + let prepared: PreparedKubernetesClientConfiguration | undefined; try { - kubernetesConfigBytes = readPrivateFile( + prepared = await prepareKubernetesClientConfiguration( paths.kubernetesFile, - MAX_KUBERNETES_CONFIG_BYTES, ); - const config = normalizeConfig(parseJson(kubernetesConfigBytes)); - kubeconfigBytes = readPrivateFile( - config.kubeconfigFile, - MAX_KUBECONFIG_BYTES, - ); - const kubernetes = await import('@kubernetes/client-node'); - const kubeConfig = new kubernetes.KubeConfig(); - try { - const kubeconfigText = decodeUtf8(kubeconfigBytes); - validateRawKubeconfig(parseJson(kubeconfigBytes), config); - kubeConfig.loadFromString(kubeconfigText); - validateKubeConfig(kubeConfig, config); - } catch (error) { - if ( - error instanceof - ClusterPluginPackageManagementKubernetesClientConfigurationError - ) { - throw error; - } - throw configurationFailure(); - } + const { config, kubeConfig, kubernetes } = prepared; const runtime = (options.createRuntime ?? productionRuntime)( kubeConfig, kubernetes, @@ -790,7 +871,6 @@ export async function executeClusterPluginPackageManagementKubernetesClient( error, ); } finally { - kubernetesConfigBytes?.fill(0); - kubeconfigBytes?.fill(0); + prepared?.dispose(); } } diff --git a/packages/ql3-cluster-admin/src/product-cli/cli.ts b/packages/ql3-cluster-admin/src/product-cli/cli.ts index 0c715b22..ae55a299 100644 --- a/packages/ql3-cluster-admin/src/product-cli/cli.ts +++ b/packages/ql3-cluster-admin/src/product-cli/cli.ts @@ -5,6 +5,7 @@ import { constants } from 'node:os'; import { resolveQingLong3ClusterProductCommand } from './productCommand'; import { QingLong3ClusterProductContextError } from './productContext'; +import { validateQingLong3ClusterProductContext } from './productContext'; const FORWARDED_SIGNALS = Object.freeze([ 'SIGINT', @@ -108,7 +109,7 @@ function invoke(targetFilePath: string, argv: readonly string[]): void { }); } -function main(argv: readonly string[]): void { +async function main(argv: readonly string[]): Promise { try { const resolution = resolveQingLong3ClusterProductCommand(argv, __dirname); if (resolution.kind === 'help' || resolution.kind === 'version') { @@ -124,6 +125,13 @@ function main(argv: readonly string[]): void { process.exitCode = 64; return; } + if (resolution.kind === 'context-validation') { + const result = await validateQingLong3ClusterProductContext( + resolution.contextFile, + ); + process.stdout.write(`${JSON.stringify(result)}\n`); + return; + } invoke(resolution.targetFilePath, resolution.argv); } catch (error) { if ( @@ -157,5 +165,5 @@ function main(argv: readonly string[]): void { } if (require.main === module) { - main(process.argv.slice(2)); + void main(process.argv.slice(2)); } diff --git a/packages/ql3-cluster-admin/src/product-cli/productCommand.ts b/packages/ql3-cluster-admin/src/product-cli/productCommand.ts index d0534c19..a86d6054 100644 --- a/packages/ql3-cluster-admin/src/product-cli/productCommand.ts +++ b/packages/ql3-cluster-admin/src/product-cli/productCommand.ts @@ -13,6 +13,7 @@ export interface QingLong3ClusterProductCommandDefinition { export type QingLong3ClusterProductCommandResolution = | Readonly<{ kind: 'help'; output: string }> | Readonly<{ kind: 'version'; output: string }> + | Readonly<{ kind: 'context-validation'; contextFile: string }> | Readonly<{ kind: 'invoke'; command: QingLong3ClusterProductCommandDefinition; @@ -172,6 +173,9 @@ export function qingLong3ClusterProductHelp(): string { 'Remote client commands:', commands, '', + 'Local operator commands:', + ' context validate --context=/absolute/operator-context.json', + '', 'Use `ql3-cluster-admin --help` for command-specific usage.', 'Use `--context=/absolute/operator-context.json` to inject only stable client paths.', 'Command and short-lived assertion files always remain explicit per invocation.', @@ -193,6 +197,28 @@ export function resolveQingLong3ClusterProductCommand( output: qingLong3ClusterProductHelp(), }); } + if (argv[0] === 'context') { + if ( + argv.length !== 3 || + argv[1] !== 'validate' || + !argv[2]!.startsWith('--context=') || + argv[2] === '--context=' + ) { + return Object.freeze({ + kind: 'invalid', + code: 'QL3_CLUSTER_PRODUCT_CLI_USAGE_INVALID', + message: 'QingLong 3.0 Cluster product context command is invalid', + }); + } + const { distRoot } = installationPaths(moduleDirectory); + for (const definition of QINGLONG3_CLUSTER_PRODUCT_COMMANDS) { + resolveInstalledTarget(distRoot, definition); + } + return Object.freeze({ + kind: 'context-validation', + contextFile: argv[2]!.slice('--context='.length), + }); + } if ( argv.length === 1 && (argv[0] === '--version' || argv[0] === '-V' || argv[0] === 'version') diff --git a/packages/ql3-cluster-admin/src/product-cli/productContext.ts b/packages/ql3-cluster-admin/src/product-cli/productContext.ts index e0e07f5e..3e902e85 100644 --- a/packages/ql3-cluster-admin/src/product-cli/productContext.ts +++ b/packages/ql3-cluster-admin/src/product-cli/productContext.ts @@ -10,6 +10,10 @@ import { import { isAbsolute } from 'node:path'; import { TextDecoder } from 'node:util'; +import { validateClusterAuthenticatedManagementClientConfiguration } from '../management-support/pluginPackageManagementClient'; +import type { ClusterAuthenticatedManagementClientKind } from '../management-support/pluginPackageManagementClient'; +import { validateClusterPluginPackageManagementKubernetesConfiguration } from '../plugin-package/management/pluginPackageManagementKubernetesClient'; + const MAXIMUM_CONTEXT_BYTES = 64 * 1024; const MAXIMUM_PATH_BYTES = 4_096; const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/u; @@ -38,6 +42,33 @@ export interface QingLong3ClusterProductContext { >; } +export interface QingLong3ClusterProductContextValidation { + readonly schemaVersion: 1; + readonly component: 'qinglong3-cluster-product-cli'; + readonly event: 'context_valid'; + readonly commandCount: number; + readonly commands: readonly Readonly<{ + name: ContextCommandName; + transport: 'https' | 'kubernetes-port-forward'; + clientCertificate: 'forbidden' | 'required'; + kubernetesAuthentication?: 'token' | 'client-certificate'; + }>[]; + readonly networkAccess: false; + readonly mutation: false; +} + +const CONTEXT_COMMAND_CLIENT_KINDS: Readonly< + Record +> = Object.freeze({ + package: 'package', + 'package-kubernetes': 'package', + 'worker-credential': 'worker-credential', + approval: 'approval', + run: 'run', + automation: 'automation', + 'model-credential': 'model-credential', +}); + export class QingLong3ClusterProductContextError extends TypeError { readonly code = 'QL3_CLUSTER_PRODUCT_CONTEXT_INVALID'; @@ -243,3 +274,57 @@ export function resolveQingLong3ClusterProductContextArguments( ...argv, ]); } + +export async function validateQingLong3ClusterProductContext( + contextFile: string, +): Promise> { + try { + const context = loadQingLong3ClusterProductContext(contextFile); + const commands: Array< + QingLong3ClusterProductContextValidation['commands'][number] + > = []; + for (const name of CONTEXT_COMMANDS) { + const command = context.commands[name]; + if (command === undefined) continue; + const clientKind = CONTEXT_COMMAND_CLIENT_KINDS[name]; + const https = validateClusterAuthenticatedManagementClientConfiguration( + command.configFile, + clientKind, + ); + if (name === 'package-kubernetes') { + const kubernetes = + await validateClusterPluginPackageManagementKubernetesConfiguration( + command.kubernetesFile!, + ); + commands.push( + Object.freeze({ + name, + transport: kubernetes.transport, + clientCertificate: https.clientCertificate, + kubernetesAuthentication: kubernetes.authentication, + }), + ); + } else { + commands.push( + Object.freeze({ + name, + transport: https.transport, + clientCertificate: https.clientCertificate, + }), + ); + } + } + return Object.freeze({ + schemaVersion: 1, + component: 'qinglong3-cluster-product-cli', + event: 'context_valid', + commandCount: commands.length, + commands: Object.freeze(commands), + networkAccess: false, + mutation: false, + }); + } catch (error) { + if (error instanceof QingLong3ClusterProductContextError) throw error; + throw new QingLong3ClusterProductContextError(); + } +} diff --git a/packages/ql3-cluster-admin/test/productCli.test.cjs b/packages/ql3-cluster-admin/test/productCli.test.cjs index 626677d0..88298b17 100644 --- a/packages/ql3-cluster-admin/test/productCli.test.cjs +++ b/packages/ql3-cluster-admin/test/productCli.test.cjs @@ -9,6 +9,18 @@ const { test } = require('node:test'); const packageRoot = path.resolve(__dirname, '..'); const moduleDirectory = path.join(packageRoot, 'dist', 'product-cli'); const cliPath = path.join(moduleDirectory, 'cli.js'); +const certificateFixture = path.join( + packageRoot, + 'test', + 'fixtures', + 'management-service-cert.pem', +); +const privateKeyFixture = path.join( + packageRoot, + 'test', + 'fixtures', + 'management-service-key.pem', +); const manifest = JSON.parse( fs.readFileSync(path.join(packageRoot, 'package.json'), 'utf8'), ); @@ -26,6 +38,9 @@ const { loadQingLong3ClusterProductContext, resolveQingLong3ClusterProductContextArguments, } = require('../dist/product-cli/productContext.js'); +const { + validateClusterAuthenticatedManagementClientConfiguration, +} = require('../dist/management-support/pluginPackageManagementClient.js'); function runCli(args) { return spawnSync(process.execPath, [cliPath, ...args], { @@ -65,6 +80,135 @@ function contextFixture(t) { return { directory, runConfig, packageConfig, kubernetes, contextFile }; } +function validContextFixture(t) { + const directory = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-cluster-valid-context-')), + ); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + const caFile = privateFile( + directory, + 'management-ca.pem', + fs.readFileSync(certificateFixture), + ); + const clientCertificateFile = privateFile( + directory, + 'operator-certificate.pem', + fs.readFileSync(certificateFixture), + ); + const clientPrivateKeyFile = privateFile( + directory, + 'operator-private-key.pem', + fs.readFileSync(privateKeyFixture), + ); + function config(name, managementPath, clientCertificate) { + return privateFile( + directory, + `${name}.json`, + JSON.stringify({ + schemaVersion: 1, + endpoint: `https://manager.example.test:8443${managementPath}`, + servername: 'manager.example.test', + caFile, + ...(clientCertificate === 'required' + ? { clientCertificateFile, clientPrivateKeyFile } + : {}), + requestTimeoutMs: 1_000, + }), + ); + } + const packageConfig = config( + 'package-client', + '/api/v3/plugin-packages/management', + 'forbidden', + ); + const kubeconfigFile = privateFile( + directory, + 'kubeconfig.json', + JSON.stringify({ + apiVersion: 'v1', + kind: 'Config', + clusters: [ + { + name: 'production', + cluster: { + server: 'https://kubernetes.example.test:6443', + 'certificate-authority-data': fs + .readFileSync(certificateFixture) + .toString('base64'), + }, + }, + ], + users: [{ name: 'operator', user: { token: 'bounded-token' } }], + contexts: [ + { + name: 'production', + context: { + cluster: 'production', + user: 'operator', + namespace: 'qinglong3', + }, + }, + ], + 'current-context': 'production', + }), + ); + const kubernetesFile = privateFile( + directory, + 'kubernetes-client.json', + JSON.stringify({ + schemaVersion: 1, + kubeconfigFile, + context: 'production', + namespace: 'qinglong3', + apiTimeoutMs: 1_000, + }), + ); + const commands = { + package: { configFile: packageConfig }, + 'package-kubernetes': { configFile: packageConfig, kubernetesFile }, + 'worker-credential': { + configFile: config( + 'worker-client', + '/api/v3/worker-credentials/management', + 'required', + ), + }, + approval: { + configFile: config( + 'approval-client', + '/api/v3/approvals/management', + 'required', + ), + }, + run: { + configFile: config('run-client', '/api/v3/runs/management', 'required'), + }, + automation: { + configFile: config( + 'automation-client', + '/api/v3/automations/management', + 'required', + ), + }, + 'model-credential': { + configFile: config( + 'provider-client', + '/api/v3/provider-credentials/management', + 'required', + ), + }, + }; + return { + directory, + contextFile: privateFile( + directory, + 'operator-context.json', + JSON.stringify({ schemaVersion: 1, commands }), + ), + commands, + }; +} + test('catalog exposes only reviewed remote clients from the same package', () => { assert.equal(manifest.bin['ql3-cluster-admin'], 'dist/product-cli/cli.js'); assert.equal(QINGLONG3_CLUSTER_PRODUCT_COMMANDS.length, 7); @@ -319,6 +463,126 @@ test('operator context rejects weak files, unknown or secret fields and argument ); }); +test('validates the complete operator context offline without operational authority', (t) => { + const fixture = validContextFixture(t); + const resolution = resolveQingLong3ClusterProductCommand( + ['context', 'validate', `--context=${fixture.contextFile}`], + moduleDirectory, + ); + assert.deepEqual(resolution, { + kind: 'context-validation', + contextFile: fixture.contextFile, + }); + + const validated = runCli([ + 'context', + 'validate', + `--context=${fixture.contextFile}`, + ]); + assert.equal(validated.status, 0); + assert.equal(validated.stderr, ''); + const fact = JSON.parse(validated.stdout); + assert.deepEqual(fact, { + schemaVersion: 1, + component: 'qinglong3-cluster-product-cli', + event: 'context_valid', + commandCount: 7, + commands: [ + { name: 'package', transport: 'https', clientCertificate: 'forbidden' }, + { + name: 'package-kubernetes', + transport: 'kubernetes-port-forward', + clientCertificate: 'forbidden', + kubernetesAuthentication: 'token', + }, + { + name: 'worker-credential', + transport: 'https', + clientCertificate: 'required', + }, + { name: 'approval', transport: 'https', clientCertificate: 'required' }, + { name: 'run', transport: 'https', clientCertificate: 'required' }, + { + name: 'automation', + transport: 'https', + clientCertificate: 'required', + }, + { + name: 'model-credential', + transport: 'https', + clientCertificate: 'required', + }, + ], + networkAccess: false, + mutation: false, + }); + assert.equal(validated.stdout.includes(fixture.directory), false); + assert.equal(validated.stdout.includes('manager.example.test'), false); + assert.equal(validated.stdout.includes('bounded-token'), false); +}); + +test('context validation fails closed for invalid client configuration and syntax', (t) => { + const fixture = validContextFixture(t); + fs.writeFileSync(fixture.commands.run.configFile, '{}', { mode: 0o600 }); + const invalid = runCli([ + 'context', + 'validate', + `--context=${fixture.contextFile}`, + ]); + assert.equal(invalid.status, 78); + assert.equal(invalid.stdout, ''); + assert.deepEqual(JSON.parse(invalid.stderr), { + schemaVersion: 1, + component: 'qinglong3-cluster-product-cli', + code: 'QL3_CLUSTER_PRODUCT_CONTEXT_INVALID', + message: 'QingLong 3.0 Cluster operator context is invalid', + }); + assert.equal(invalid.stderr.includes(fixture.directory), false); + + for (const argv of [ + ['context'], + ['context', 'validate'], + ['context', 'validate', '--context'], + ['context', 'validate', '--context='], + ['context', 'inspect', `--context=${fixture.contextFile}`], + ]) { + const rejected = resolveQingLong3ClusterProductCommand( + argv, + moduleDirectory, + ); + assert.equal(rejected.kind, 'invalid'); + assert.equal(rejected.code, 'QL3_CLUSTER_PRODUCT_CLI_USAGE_INVALID'); + } +}); + +test('configuration preflight fixes every management route to its reviewed authentication class', (t) => { + const fixture = validContextFixture(t); + assert.throws( + () => + validateClusterAuthenticatedManagementClientConfiguration( + fixture.commands.run.configFile, + 'package', + ), + /configuration is invalid/, + ); + assert.throws( + () => + validateClusterAuthenticatedManagementClientConfiguration( + fixture.commands.package.configFile, + 'run', + ), + /configuration is invalid/, + ); + assert.throws( + () => + validateClusterAuthenticatedManagementClientConfiguration( + fixture.commands.package.configFile, + 'unknown', + ), + /configuration is invalid/, + ); +}); + test('rejects symlink targets and package manifests', (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-cluster-product-')); t.after(() => fs.rmSync(root, { recursive: true, force: true })); diff --git a/scripts/ql3-cluster-admin-product-live-contract.cjs b/scripts/ql3-cluster-admin-product-live-contract.cjs index e28eca4c..244da915 100644 --- a/scripts/ql3-cluster-admin-product-live-contract.cjs +++ b/scripts/ql3-cluster-admin-product-live-contract.cjs @@ -80,11 +80,13 @@ function runOperatorContextContract(image) { const source = String.raw` const { spawnSync } = require('node:child_process'); const { writeFileSync } = require('node:fs'); +const { rootCertificates } = require('node:tls'); const facade = '/opt/qinglong/node_modules/@qinglong/cluster-admin/dist/product-cli/cli.js'; const config = '/tmp/run-client.json'; const command = '/tmp/command.json'; const assertion = '/tmp/assertion.jwt'; const context = '/tmp/operator-context.json'; +const ca = '/tmp/management-ca.pem'; for (const [file, contents] of [ [config, '{}'], [command, '{}'], @@ -95,12 +97,19 @@ const injected = spawnSync(process.execPath, [facade, 'run', '--context=' + cont let injectedFailure; try { injectedFailure = JSON.parse(injected.stderr); } catch { process.exit(21); } if (injected.status !== 1 || injectedFailure.code !== 'QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_CONFIG_INVALID' || injected.stdout !== '') process.exit(22); +writeFileSync(ca, rootCertificates[0], { mode: 0o600 }); +writeFileSync(config, JSON.stringify({ schemaVersion: 1, endpoint: 'https://manager.example.test:8443/api/v3/plugin-packages/management', servername: 'manager.example.test', caFile: ca, requestTimeoutMs: 1000 }), { mode: 0o600 }); +writeFileSync(context, JSON.stringify({ schemaVersion: 1, commands: { package: { configFile: config } } }), { mode: 0o600 }); +const validated = spawnSync(process.execPath, [facade, 'context', 'validate', '--context=' + context], { encoding: 'utf8' }); +let validationFact; +try { validationFact = JSON.parse(validated.stdout); } catch { process.exit(25); } +if (validated.status !== 0 || validated.stderr !== '' || validationFact.event !== 'context_valid' || validationFact.commandCount !== 1 || validationFact.networkAccess !== false || validationFact.mutation !== false || JSON.stringify(validationFact.commands) !== JSON.stringify([{ name: 'package', transport: 'https', clientCertificate: 'forbidden' }]) || validated.stdout.includes('/tmp/') || validated.stdout.includes('manager.example.test')) process.exit(26); writeFileSync(context, JSON.stringify({ schemaVersion: 1, commands: { run: { configFile: config, assertionFile: assertion } } }), { mode: 0o600 }); const rejected = spawnSync(process.execPath, [facade, 'run', '--context=' + context, '--command=' + command, '--assertion=' + assertion], { encoding: 'utf8' }); let rejectedFailure; try { rejectedFailure = JSON.parse(rejected.stderr); } catch { process.exit(23); } if (rejected.status !== 78 || rejectedFailure.code !== 'QL3_CLUSTER_PRODUCT_CONTEXT_INVALID' || rejected.stdout !== '' || rejected.stderr.includes('/tmp/') || rejected.stderr.includes('assertion.jwt')) process.exit(24); -process.stdout.write(JSON.stringify({ schemaVersion: 1, injected: true, secretFieldsRejected: true })); +process.stdout.write(JSON.stringify({ schemaVersion: 1, injected: true, contextPreflight: true, secretFieldsRejected: true })); `; const output = docker([ 'run', @@ -137,6 +146,7 @@ process.stdout.write(JSON.stringify({ schemaVersion: 1, injected: true, secretFi if ( result?.schemaVersion !== 1 || result?.injected !== true || + result?.contextPreflight !== true || result?.secretFieldsRejected !== true ) { fail('operator context contract drifted'); @@ -192,6 +202,7 @@ function main() { imageBytes: fact.Size, commandCount: COMMANDS.length, operatorContext: true, + contextPreflight: true, isolation: Object.freeze({ readOnlyRoot: true, network: 'none', diff --git a/scripts/ql3-cluster-image-release-audit.cjs b/scripts/ql3-cluster-image-release-audit.cjs index 52091361..14c687fe 100644 --- a/scripts/ql3-cluster-image-release-audit.cjs +++ b/scripts/ql3-cluster-image-release-audit.cjs @@ -289,8 +289,8 @@ function auditClusterImageCiWorkflow( ); requirePattern( adminProductLiveContract, - /runOperatorContextContract\(image\);[\s\S]*operatorContext: true/, - 'native admin image contract must verify owner-private operator context injection', + /runOperatorContextContract\(image\);[\s\S]*operatorContext: true,[\s\S]*contextPreflight: true/, + 'native admin image contract must verify owner-private operator context injection and offline preflight', ); requirePattern( source, @@ -334,6 +334,7 @@ function auditClusterImageCiWorkflow( runtimeInventory: true, clusterAdminProductFacade: true, clusterAdminOperatorContext: true, + clusterAdminContextPreflight: true, ociAttestations: true, osVulnerabilityScan: { scanner: 'trivy@0.70.0', diff --git a/test/back/ql3ClusterImageReleaseAudit.test.cjs b/test/back/ql3ClusterImageReleaseAudit.test.cjs index b9dcccc5..d259095e 100644 --- a/test/back/ql3ClusterImageReleaseAudit.test.cjs +++ b/test/back/ql3ClusterImageReleaseAudit.test.cjs @@ -28,6 +28,7 @@ test('accepts the reviewed native CI and digest release contracts', () => { runtimeInventory: true, clusterAdminProductFacade: true, clusterAdminOperatorContext: true, + clusterAdminContextPreflight: true, ociAttestations: true, osVulnerabilityScan: { scanner: 'trivy@0.70.0', @@ -126,6 +127,24 @@ test('rejects a Cluster Admin live gate that omits operator context injection', ); }); +test('rejects a Cluster Admin live gate that omits offline context preflight', () => { + const contract = fs.readFileSync( + path.join(root, 'scripts/ql3-cluster-admin-product-live-contract.cjs'), + 'utf8', + ); + assert.throws( + () => + auditClusterImageCiWorkflow( + ciSource, + contract.replace( + 'operatorContext: true,\n contextPreflight: true', + 'operatorContext: true,\n contextPreflight: false', + ), + ), + /offline preflight/, + ); +}); + test('rejects removal of the native cluster-admin image gate', () => { const mutated = ciSource.replace( 'image_arch: arm64\n image: admin',