From 5c89e3bc600bb63a87da5530b97dda56cc561bf7 Mon Sep 17 00:00:00 2001 From: whyour Date: Wed, 12 Aug 2026 11:51:14 +0800 Subject: [PATCH] feat(ql3): add cluster run management live gate --- .github/workflows/ql3-run-management-live.yml | 124 ++ docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 13 + ...run-management-kubernetes-live-evidence.md | 51 + docs/adr/README.md | 1 + package.json | 2 + ...l3-management-kubernetes-live-platform.cjs | 87 ++ .../lib/ql3-management-kubernetes-live.cjs | 135 +- scripts/lib/ql3-management-live-identity.cjs | 48 +- ...un-management-kubernetes-live-scenario.cjs | 194 +++ ...al-management-kubernetes-live-contract.cjs | 337 ++--- ...3-run-management-kubernetes-live-audit.cjs | 488 +++++++ ...un-management-kubernetes-live-contract.cjs | 1259 +++++++++++++++++ ...3RunManagementKubernetesLiveAudit.test.cjs | 244 ++++ ...nManagementKubernetesLiveContract.test.cjs | 99 ++ 14 files changed, 2782 insertions(+), 300 deletions(-) create mode 100644 .github/workflows/ql3-run-management-live.yml create mode 100644 docs/adr/ADR-0386-cluster-run-management-kubernetes-live-evidence.md create mode 100644 scripts/lib/ql3-management-kubernetes-live-platform.cjs create mode 100644 scripts/lib/ql3-run-management-kubernetes-live-scenario.cjs create mode 100644 scripts/ql3-run-management-kubernetes-live-audit.cjs create mode 100644 scripts/ql3-run-management-kubernetes-live-contract.cjs create mode 100644 test/back/ql3RunManagementKubernetesLiveAudit.test.cjs create mode 100644 test/back/ql3RunManagementKubernetesLiveContract.test.cjs diff --git a/.github/workflows/ql3-run-management-live.yml b/.github/workflows/ql3-run-management-live.yml new file mode 100644 index 00000000..c06f13c0 --- /dev/null +++ b/.github/workflows/ql3-run-management-live.yml @@ -0,0 +1,124 @@ +name: QingLong 3.0 Run Management Kubernetes live evidence + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ql3-run-management-live-${{ github.ref }} + cancel-in-progress: false + +jobs: + run-management-live: + name: Three-node Run retry, stop, OIDC, mTLS and CloudNativePG + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v6 + - uses: pnpm/action-setup@v6 + with: + version: '8.3.1' + - uses: actions/setup-node@v6 + with: + node-version: '24.18.0' + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + + - name: Require enough ephemeral disk for three isolated K3s nodes + shell: bash + run: | + set -euo pipefail + available_kib="$(df -Pk / | awk 'NR == 2 { print $4 }')" + minimum_kib="$((25 * 1024 * 1024))" + if (( available_kib < minimum_kib )); then + echo "Run management live gate requires at least 25 GiB free; found ${available_kib} KiB" >&2 + exit 1 + fi + docker volume ls --quiet --filter dangling=true | sort > \ + "${RUNNER_TEMP}/ql3-dangling-volumes.before" + docker system df + + - name: Install workspace dependencies without lifecycle scripts + run: pnpm install --frozen-lockfile --ignore-scripts + + - name: Install verified kubectl v1.32.8 + shell: bash + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + --output "${RUNNER_TEMP}/kubectl" \ + https://dl.k8s.io/release/v1.32.8/bin/linux/amd64/kubectl + curl --fail --location --silent --show-error \ + --output "${RUNNER_TEMP}/kubectl.sha256" \ + https://dl.k8s.io/release/v1.32.8/bin/linux/amd64/kubectl.sha256 + test "$(cat "${RUNNER_TEMP}/kubectl.sha256")" = \ + "$(sha256sum "${RUNNER_TEMP}/kubectl" | cut -d ' ' -f 1)" + chmod 0755 "${RUNNER_TEMP}/kubectl" + + - name: Fetch checksum-locked CloudNativePG operator manifest + shell: bash + run: | + set -euo pipefail + curl --fail --location --silent --show-error \ + --output "${RUNNER_TEMP}/cloudnative-pg.yaml" \ + https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v1.30.0/cnpg-1.30.0.yaml + echo "f8bede43fe4ee0d478c2355b204a36876b2ae4faac60f2a9452280b293da3b88 ${RUNNER_TEMP}/cloudnative-pg.yaml" | sha256sum --check + + - name: Recheck static Run management and evidence contracts + run: | + pnpm audit:cluster-deployment:ql3 + node --test \ + test/back/ql3RunManagementDeployment.test.cjs \ + test/back/ql3RunManagementKubernetesLiveContract.test.cjs \ + test/back/ql3RunManagementKubernetesLiveAudit.test.cjs + pnpm --filter @qinglong/cluster-admin build + pnpm --filter @qinglong/cluster-control build + + - name: Preload the digest-bound K3s distribution + run: docker pull rancher/k3s:v1.34.3-k3s1 + + - name: Prove Run retry and stop over three nodes, mTLS, OIDC and CloudNativePG + env: + QL3_RUN_MANAGEMENT_KUBERNETES_LIVE: '1' + QL3_KUBECTL_BIN: ${{ runner.temp }}/kubectl + QL3_CNPG_OPERATOR_MANIFEST_FILE: ${{ runner.temp }}/cloudnative-pg.yaml + QL3_RUN_MANAGEMENT_REPORT: ${{ runner.temp }}/ql3-run-management/report.json + shell: bash + run: | + set -euo pipefail + umask 077 + mkdir -m 0700 "$(dirname "${QL3_RUN_MANAGEMENT_REPORT}")" + node scripts/ql3-run-management-kubernetes-live-contract.cjs \ + "--report=${QL3_RUN_MANAGEMENT_REPORT}" + + - name: Re-audit private report and isolated cleanup + env: + QL3_RUN_MANAGEMENT_REPORT: ${{ runner.temp }}/ql3-run-management/report.json + shell: bash + run: | + set -euo pipefail + test "$(stat -c '%a' "${QL3_RUN_MANAGEMENT_REPORT}")" = '600' + pnpm audit:run-management-kubernetes-live:ql3 \ + "--report=${QL3_RUN_MANAGEMENT_REPORT}" + sha256sum "${QL3_RUN_MANAGEMENT_REPORT}" + test -z "$(docker ps -aq --filter name=ql3-run-live-)" + test -z "$(docker network ls -q --filter name=ql3-run-live-)" + docker volume ls --quiet --filter dangling=true | sort > \ + "${RUNNER_TEMP}/ql3-dangling-volumes.after" + diff --unified \ + "${RUNNER_TEMP}/ql3-dangling-volumes.before" \ + "${RUNNER_TEMP}/ql3-dangling-volumes.after" + docker system df + + - name: Upload audited content-free Run management evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ql3-run-management-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/ql3-run-management/report.json + if-no-files-found: error + retention-days: 14 + compression-level: 9 + overwrite: false + include-hidden-files: false diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 9f9ea48e..e1ec5d1b 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,19 @@ 最新增量证据（2026-08-12）： +- D-298/ADR-0386（实现完成；固定本机 live 证据待锁定输入可达后补录） + Cluster `run.retry | run.stop` 已建立人工触发的真实三节点 Kubernetes 组合门：1 control-plane + 2 worker K3s/Flannel、3 实例 + CloudNativePG 1.30.0/PostgreSQL 18.4、2 个跨节点 Run manager Pod、TLS 1.3 mTLS、purpose-bound OIDC strong User、identity + generation overlap/revoke/rollback、client CRL rotation、CNPG primary promotion、数据库断连 readiness/liveness fence、CNI 与 RBAC + least privilege，并以最终 PostgreSQL facts 验证 retry/stop 首写和 exact replay 无重复。报告 schema 固定 migration 57/control-core + capability 56，只允许 `0600` content-free evidence，敏感材料与隐藏 limitation 均失败。实现没有新增 package、生产依赖、migration、表、 + 角色或默认 workload；live 代码按通用管理 Kubernetes helper、Run scenario、离线 audit 分层，现有 Approval 回归通过。当前静态/负向合同 + 20/20；本机首次 live 取锁定 CNPG manifest 时 GitHub 443 连接 75 秒无数据超时且无缓存，因此没有降低 checksum 或把静态结果冒充 live + 证据。完整 18-package build/test 退出 0，backend 1,174 pass/2 conditional skip/0 fail，14 个 Profile artifact 全部 compatible；基础 + Edge 为 2,467,343 bytes/295 files/53 modules，RSS delta 10,928,128 bytes，低于既有低配门限。package/dependency/local-image boundary + 全绿，仍为 18 package、无 single-source/shallow package。PostgreSQL 18.4 arm64 HA 通过 123 gates、timeline `1→2`，报告 SHA-256 + `6adb8c9de8929ff54b522e9a251e3081d9dd004c1a91f72f83c33288ddce63a9` 且 Docker 零残留。固定 K3s/CNPG live report 是 ADR 从 + Proposed 转为 Accepted 的唯一剩余证据。 - D-297/ADR-0385（已接受） Local Edge/Standalone 已补齐强认证 `run.stop` 产品入口，并与既有 `run.retry` 统一为同一个 caller-driven `ql3-run retry|stop` binary，不新增 package、migration、表、索引、进程、listener、timer、watcher、连接、cache 或 sidecar。stop 只接受 POSIX 私有命令文件 diff --git a/docs/adr/ADR-0386-cluster-run-management-kubernetes-live-evidence.md b/docs/adr/ADR-0386-cluster-run-management-kubernetes-live-evidence.md new file mode 100644 index 00000000..6f4d767a --- /dev/null +++ b/docs/adr/ADR-0386-cluster-run-management-kubernetes-live-evidence.md @@ -0,0 +1,51 @@ +# ADR-0386：Cluster Run Management Kubernetes 多节点实证 + +- 状态：Proposed（实现与静态回归完成；固定本机 live 证据待锁定输入可达后补录） +- 日期：2026-08-12 +- 关联 RFC：QL-RFC-0001 D-298 +- 前置决策：ADR-0359、ADR-0364、ADR-0366、ADR-0382、ADR-0383、ADR-0384 + +## 上下文 + +Cluster 已有强认证 `run.retry | run.stop`、专用 `ql3_run_manager`、两副本 Kubernetes overlay 和 PostgreSQL HA Docker 证据，但这些证据没有在同一真实 CNI/Pod/CloudNativePG 环境中组合。静态 YAML、进程内测试和单机 PostgreSQL promotion 不能证明跨节点 Pod、mTLS/OIDC rotation、NetworkPolicy、readiness withdrawal 与数据库主切换同时成立。 + +部署用户跨度很大：Edge 路由设备不能为 Cluster 门增加任何常驻成本；Cluster 节点又需要真实多副本证据。因此本门必须是 `workflow_dispatch` 的短生命周期发布门，不能进入默认 CI、默认 overlay 或 Local artifact。 + +## 决策 + +1. 新增 opt-in 三节点 K3s live gate：1 control-plane + 2 worker、内置 Flannel、3 实例 CloudNativePG 1.30.0/PostgreSQL 18.4、2 个 Run manager Pod 且位于不同节点。 +2. 直接构建当前 Cluster Admin/Control 镜像，应用既有 `operations/run-management/cloudnative-pg`；不新增 package、生产依赖、migration、表、角色、常驻进程或 sidecar。 +3. 产品 client 以 tokenless、无 RBAC、immutable Secret 输入的 caller-driven Job 精确连接每个 Pod。`run.retry` 必须得到 `accepted → existing`，随后 `run.stop` 必须得到 `accepted → already_requested`；最终数据库只能有 1 个 retry Run、1 个 Attempt、2 个 retry Event、1 个 cancellation Event、2 个 allowed audit，重复 mutation 为 0。 +4. 弱认证在业务层前返回 401 且不写项目 audit；强但无 RoleBinding 的 User 返回 403 并写恰好 1 个 denied audit。报告不保存 assertion、证书、密钥、DSN、kubeconfig 或业务内容。 +5. OIDC keyset 执行 generation 1→2 overlap→3 revoke，回滚 generation 必须启动失败且始终保留 2 Ready Pod；mTLS CRL 更新必须替换全部 Pod、旧 client 被拒绝、新 client 可用，rollout 不低于 2 Ready。 +6. 删除当前 CNPG primary 并确认 promotion 与恢复 3 实例；切断 `-rw` Service endpoint 时两个 Pod 都返回 503、readiness 为 503、liveness 为 200，恢复 endpoint 后旧 Pod不得原地恢复，必须由新 Pod 服务。 +7. CNI 必须观测 labelled client allowed、unlabelled/wrong-port denied、manager→CNPG allowed、manager→Kubernetes API/public internet denied；ServiceAccount 无 Secret read 和 Deployment patch 权限。 +8. live 代码分三层：通用 Kubernetes 管理 helper、Run 领域 scenario/SQL facts、严格离线 report audit。通用 executor 的 retry error code 改为显式配置且保留旧默认值，Approval live 回归必须通过。禁止复制身份 assertion、健康探针和平台原语；新增能力不得形成 workspace 微包。 + +## 证据合同 + +离线报告固定为 `qinglong/run-management-kubernetes-live-contract@v1`，要求： + +- K3s `v1.32+`、锁定 `rancher/k3s:v1.34.3-k3s1`、3 个独立 PodCIDR； +- migration count 57、control-core capability 56、PostgreSQL version number 180004； +- 两个唯一 Pod/Node digest、PDB minAvailable 1、maxUnavailable 0、每 Pod Pool max 2； +- retry/stop exact replay、认证负向门、identity/certificate rotation、availability、isolation、durability 全为精确字段； +- `0600` regular file、no overwrite、无敏感材料；三条非生产 limitation 不得隐藏。 + +## Package 与低配影响 + +本切片不新增 workspace package。当前 package boundary 仍要求每个 `src/` 根文件都是 manifest 证明的入口；实现位于领域目录，高密度目录受 hard cap。文件少不是拆包理由，部署/authority/稳定多消费者才是 package 边界。该 live gate 只在人工发布作业中创建 Docker/K3s/CNPG 资源，完成后全部删除；Edge/Standalone 的文件、module、RSS、timer、listener 与连接预算不变。 + +## 验收状态 + +- Run/Approval live 静态合同、离线审计与既有 deployment 回归：20/20；Run report 审计含 topology/schema/deployment/client/rotation/availability/isolation/durability/secret 反向 fixture。 +- runner、scenario、platform、identity、PKI 与 audit 均通过 Node syntax check；现有 Run deployment 合同通过。 +- 本机真实门尝试下载校验和锁定的 CNPG manifest 时，`github.com:443` 在 75 秒无数据后超时；本机无缓存。未降低 checksum、未改用未审计输入，也未把静态结果记录为 live 通过。 +- 完整 18-package build/test 退出 0；backend 1,174 pass/2 conditional skip/0 fail；14 个 Edge/Standalone Profile artifact 全部 compatible。基础 Edge 为 2,467,343 bytes/295 files/53 loaded modules，RSS delta 10,928,128 bytes，仍低于 4 MiB/512 files/16 MiB 门限。 +- package/dependency/local-image boundary 全部 compatible，18 个 workspace package 无 single-source/shallow package；本切片没有改变任一 Local artifact 的生产闭包。 +- PostgreSQL 18.4 arm64 HA 通过 123 gates、primary timeline `1→2`，报告 SHA-256 `6adb8c9de8929ff54b522e9a251e3081d9dd004c1a91f72f83c33288ddce63a9`；运行后 `ql3-ha-*` container、volume、network 均为空。 +- 固定 K3s/CloudNativePG live report 仍是 ADR 从 Proposed 转为 Accepted 的唯一未满足证据。 + +## 后果 + +Cluster 有了可重复执行的真实组合门，但 Docker-host K3s 仍不是生产 control-plane HA 或 STONITH 证明，local deterministic IdP 也不替代外部 IdP 集成。工作流只提供 release candidate evidence，不改变默认部署。后续可把同构的 Approval/Automation runner 继续迁移到共享平台层，但不得为去除文本重复而一次性重写已通过的独立 live 流程。 diff --git a/docs/adr/README.md b/docs/adr/README.md index a735f14c..58e768d0 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -389,6 +389,7 @@ | [ADR-0383](./ADR-0383-strong-cluster-run-management-plane.md) | 强认证的 Cluster Run Management Plane 与专用数据库角色 | Accepted | | [ADR-0384](./ADR-0384-strong-cluster-run-stop-management.md) | 强认证的 Cluster Run Stop Management 与列级数据库权限 | Accepted | | [ADR-0385](./ADR-0385-strong-local-run-stop-product-entry.md) | 强认证的 Local Run Stop 产品入口与原子审计 | Accepted | +| [ADR-0386](./ADR-0386-cluster-run-management-kubernetes-live-evidence.md) | Cluster Run Management Kubernetes 多节点实证 | Proposed（实现完成，固定 live 证据待补录） | ## 规则 diff --git a/package.json b/package.json index fef1886f..5d834091 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,8 @@ "audit:automation-management-kubernetes-live:ql3": "node scripts/ql3-automation-management-kubernetes-live-audit.cjs", "test:approval-management-kubernetes-live:ql3": "pnpm --filter @qinglong/runtime-core build && node scripts/ql3-approval-management-kubernetes-live-contract.cjs", "audit:approval-management-kubernetes-live:ql3": "node scripts/ql3-approval-management-kubernetes-live-audit.cjs", + "test:run-management-kubernetes-live:ql3": "pnpm --filter @qinglong/cluster-admin build && pnpm --filter @qinglong/cluster-control build && node scripts/ql3-run-management-kubernetes-live-contract.cjs", + "audit:run-management-kubernetes-live:ql3": "node scripts/ql3-run-management-kubernetes-live-audit.cjs", "test:provider-credential-test-kubernetes-live:ql3": "pnpm --filter @qinglong/cluster-admin build && node scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs", "audit:provider-credential-test-kubernetes-live:ql3": "node scripts/ql3-provider-credential-test-kubernetes-live-audit.cjs", "test:prompt-output-key-retirement-kubernetes-live:ql3": "pnpm --filter @qinglong/cluster-admin build && node scripts/ql3-prompt-output-key-retirement-kubernetes-live-contract.cjs", diff --git a/scripts/lib/ql3-management-kubernetes-live-platform.cjs b/scripts/lib/ql3-management-kubernetes-live-platform.cjs new file mode 100644 index 00000000..a89d927a --- /dev/null +++ b/scripts/lib/ql3-management-kubernetes-live-platform.cjs @@ -0,0 +1,87 @@ +#!/usr/bin/env node + +'use strict'; + +const assert = require('node:assert/strict'); + +function imageIdDigest(image) { + assert.match(image.Id, /^sha256:[a-f0-9]{64}$/); + return image.Id; +} + +function localManifest(rendered, imageName, localImage) { + const occurrences = rendered.split(imageName).length - 1; + assert.ok(occurrences >= 1, 'reviewed image reference is missing'); + return rendered + .replaceAll(imageName, localImage) + .replaceAll('imagePullPolicy: IfNotPresent', 'imagePullPolicy: Never'); +} + +function applySecret(fixture, name, type, stringData) { + fixture.apply({ + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name, + namespace: 'qinglong3-system', + labels: { 'cnpg.io/reload': 'true' }, + }, + immutable: false, + type, + stringData, + }); +} + +function psql(fixture, podName, sql) { + return fixture.kubectl( + [ + '-n', + 'qinglong3-system', + 'exec', + podName, + '-c', + 'postgres', + '--', + 'psql', + '--username=postgres', + '--dbname=qinglong', + '--no-psqlrc', + '--tuples-only', + '--no-align', + '--set=ON_ERROR_STOP=1', + '--command', + sql, + ], + { capture: true, quiet: true }, + ).stdout; +} + +function currentPrimaryPod(fixture) { + const primaryName = fixture.kubectlJson([ + '-n', + 'qinglong3-system', + 'get', + 'cluster', + 'ql3-postgres', + ]).status.currentPrimary; + assert.match(primaryName || '', /^ql3-postgres-[1-9][0-9]*$/); + const pods = fixture.kubectlJson([ + '-n', + 'qinglong3-system', + 'get', + 'pods', + '-l', + 'cnpg.io/cluster=ql3-postgres', + ]).items; + const primary = pods.find((pod) => pod.metadata.name === primaryName); + assert.ok(primary, 'CloudNativePG primary Pod not found'); + return primary; +} + +module.exports = { + applySecret, + currentPrimaryPod, + imageIdDigest, + localManifest, + psql, +}; diff --git a/scripts/lib/ql3-management-kubernetes-live.cjs b/scripts/lib/ql3-management-kubernetes-live.cjs index 34987630..59a6f61d 100644 --- a/scripts/lib/ql3-management-kubernetes-live.cjs +++ b/scripts/lib/ql3-management-kubernetes-live.cjs @@ -100,9 +100,7 @@ async function waitForTwoPreserved(options) { '-l', 'app.kubernetes.io/name=' + options.deployment, ]) - .items.filter( - (pod) => pod.metadata.deletionTimestamp === undefined, - ); + .items.filter((pod) => pod.metadata.deletionTimestamp === undefined); const ready = pods.filter(podReady); minimumReady = Math.min(minimumReady, ready.length); const replacements = ready.filter( @@ -117,10 +115,7 @@ async function waitForTwoPreserved(options) { : { ready: false, fact: - ready.length + - ' ready, ' + - replacements.length + - ' replacements', + ready.length + ' ready, ' + replacements.length + ' replacements', }; }); assert.ok( @@ -131,6 +126,17 @@ async function waitForTwoPreserved(options) { } function createManagementClientExecutor(options) { + const retryableClientCodes = options.retryableClientCodes ?? [ + 'QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED', + ]; + assert.ok( + Array.isArray(retryableClientCodes) && + retryableClientCodes.length >= 1 && + retryableClientCodes.every( + (code) => typeof code === 'string' && /^[A-Z0-9_]{1,128}$/.test(code), + ), + 'management client retryable codes are invalid', + ); options.fixture.apply({ apiVersion: 'v1', kind: 'ServiceAccount', @@ -243,9 +249,14 @@ function createManagementClientExecutor(options) { '--command=/tmp/command.json ' + '--assertion=/tmp/assertion.jwt 2>&1)"', ' status=$?', - ' if [ "$status" -eq 0 ] || { ! printf \'%s\' "$output" | ' + - 'grep -q QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED && ' + - '! printf \'%s\' "$output" | grep -q \'"statusCode":503\'; } || ' + + ' if [ "$status" -eq 0 ] || { ' + + retryableClientCodes + .map( + (code) => + '! printf \'%s\' "$output" | grep -q ' + code, + ) + .join(' && ') + + ' && ! printf \'%s\' "$output" | grep -q \'"statusCode":503\'; } || ' + '[ "$attempt" -ge 60 ]; then', ' break', ' fi', @@ -373,7 +384,10 @@ function createManagementClientExecutor(options) { assert.equal(output.event, 'command_completed'); assert.equal(output.result.operation, definition.command.operation); if (expected.resultStatus) { - assert.ok(expected.resultStatus.includes(output.result.status)); + const status = expected.resultField + ? output.result[expected.resultField]?.status + : output.result.status; + assert.ok(expected.resultStatus.includes(status)); } } else { assert.equal( @@ -409,6 +423,36 @@ function createManagementClientExecutor(options) { }; } +function managementHealthStatus(options) { + const script = [ + "const fs=require('node:fs');const https=require('node:https');", + "const request=https.request({host:'127.0.0.1',port:Number(process.argv[1]),path:process.argv[2],", + 'servername:process.argv[3],ca:fs.readFileSync(process.argv[4]),', + "minVersion:'TLSv1.3',maxVersion:'TLSv1.3',rejectUnauthorized:true,agent:false},", + "(response)=>{response.resume();response.on('end',()=>process.stdout.write(String(response.statusCode)))});", + "request.on('error',(error)=>{process.stderr.write(error.message);process.exitCode=1});request.end();", + ].join('\n'); + return Number( + options.fixture.kubectl( + [ + '-n', + options.namespace, + 'exec', + options.podName, + '--', + 'node', + '-e', + script, + String(options.port), + options.route, + options.servername, + options.caFile, + ], + { capture: true, quiet: true }, + ).stdout, + ); +} + function podTcpProbe(options) { const script = [ "const net=require('node:net');let finished=false;", @@ -437,9 +481,7 @@ function podTcpProbe(options) { async function clientTcpProbe(options) { const labels = { 'app.kubernetes.io/name': options.appName, - ...(options.labelled - ? { [options.networkPolicyLabel]: 'true' } - : {}), + ...(options.labelled ? { [options.networkPolicyLabel]: 'true' } : {}), }; const script = [ "const fs=require('node:fs');const net=require('node:net');let finished=false;let attempt=0;let socket;", @@ -497,30 +539,25 @@ async function clientTcpProbe(options) { }, }, }); - const observed = await waitFor( - options.name + ' completion', - 180_000, - () => { - const job = options.fixture.kubectlJson([ - '-n', - options.namespace, - 'get', - 'job', - options.name, - ]); - const complete = job.status.conditions?.some( - (condition) => - condition.type === 'Complete' && condition.status === 'True', - ); - const failed = job.status.conditions?.some( - (condition) => - condition.type === 'Failed' && condition.status === 'True', - ); - return complete || failed - ? { ready: true, value: { complete, failed } } - : { ready: false, fact: JSON.stringify(job.status ?? {}) }; - }, - ); + const observed = await waitFor(options.name + ' completion', 180_000, () => { + const job = options.fixture.kubectlJson([ + '-n', + options.namespace, + 'get', + 'job', + options.name, + ]); + const complete = job.status.conditions?.some( + (condition) => + condition.type === 'Complete' && condition.status === 'True', + ); + const failed = job.status.conditions?.some( + (condition) => condition.type === 'Failed' && condition.status === 'True', + ); + return complete || failed + ? { ready: true, value: { complete, failed } } + : { ready: false, fact: JSON.stringify(job.status ?? {}) }; + }); const probePod = ( await waitFor(options.name + ' terminal pod', 30_000, () => { const pods = options.fixture.kubectlJson([ @@ -541,16 +578,8 @@ async function clientTcpProbe(options) { const terminated = probePod.status.containerStatuses[0].state.terminated; const observation = options.name + ': ' + (terminated.message ?? 'no-message'); - assert.equal( - observed.value.complete, - options.expectedConnected, - observation, - ); - assert.equal( - observed.value.failed, - !options.expectedConnected, - observation, - ); + assert.equal(observed.value.complete, options.expectedConnected, observation); + assert.equal(observed.value.failed, !options.expectedConnected, observation); assert.equal( terminated.exitCode === 0, options.expectedConnected, @@ -562,14 +591,7 @@ async function clientTcpProbe(options) { assert.match(terminated.message ?? '', /^denied:/); } options.fixture.kubectl( - [ - '-n', - options.namespace, - 'delete', - 'job', - options.name, - '--wait=false', - ], + ['-n', options.namespace, 'delete', 'job', options.name, '--wait=false'], { capture: true, quiet: true }, ); return options.expectedConnected @@ -580,6 +602,7 @@ async function clientTcpProbe(options) { module.exports = { clientTcpProbe, createManagementClientExecutor, + managementHealthStatus, patchManagementGeneration, podReady, podTcpProbe, diff --git a/scripts/lib/ql3-management-live-identity.cjs b/scripts/lib/ql3-management-live-identity.cjs index 4cc812a3..c94fe426 100644 --- a/scripts/lib/ql3-management-live-identity.cjs +++ b/scripts/lib/ql3-management-live-identity.cjs @@ -57,7 +57,8 @@ function createManagementIdentityCeremony(options) { }); } - function assertion(key, suffix = crypto.randomUUID()) { + function assertionForSubject(key, subject, suffix = crypto.randomUUID()) { + assert.match(subject, /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/); const now = Math.floor(Date.now() / 1_000); const header = Buffer.from( JSON.stringify({ @@ -77,6 +78,43 @@ function createManagementIdentityCeremony(options) { iss: options.issuer, jti: options.jtiPrefix + '-' + suffix, ql3_purpose: options.purpose, + sub: subject, + }), + ).toString('base64url'); + const signed = header + '.' + payload; + return ( + signed + + '.' + + crypto + .sign(null, Buffer.from(signed, 'ascii'), key.privateKey) + .toString('base64url') + ); + } + + function assertion(key, suffix = crypto.randomUUID()) { + return assertionForSubject(key, options.subject, suffix); + } + + function weakAssertion(key, suffix = crypto.randomUUID()) { + const now = Math.floor(Date.now() / 1_000); + const header = Buffer.from( + JSON.stringify({ + alg: 'EdDSA', + kid: key.kid, + typ: options.tokenType, + }), + ).toString('base64url'); + const payload = Buffer.from( + JSON.stringify({ + acr: 'urn:ql3:password', + amr: ['pwd'], + aud: options.audience, + auth_time: now - 1, + exp: now + 290, + iat: now, + iss: options.issuer, + jti: options.jtiPrefix + '-weak-' + suffix, + ql3_purpose: options.purpose, sub: options.subject, }), ).toString('base64url'); @@ -90,7 +128,13 @@ function createManagementIdentityCeremony(options) { ); } - return Object.freeze({ assertion, keyset, reviewedKey }); + return Object.freeze({ + assertion, + assertionForSubject, + keyset, + reviewedKey, + weakAssertion, + }); } module.exports = { createManagementIdentityCeremony }; diff --git a/scripts/lib/ql3-run-management-kubernetes-live-scenario.cjs b/scripts/lib/ql3-run-management-kubernetes-live-scenario.cjs new file mode 100644 index 00000000..18fbf223 --- /dev/null +++ b/scripts/lib/ql3-run-management-kubernetes-live-scenario.cjs @@ -0,0 +1,194 @@ +#!/usr/bin/env node + +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); + +function eventId(ordinal) { + assert.ok(Number.isSafeInteger(ordinal) && ordinal >= 1 && ordinal < 1e12); + return '41000000-0000-4000-8000-' + String(ordinal).padStart(12, '0'); +} + +function retryCommand(projectId, sourceRunId, requestId, mutationId, ordinal) { + return Object.freeze({ + schemaVersion: 1, + operation: 'run.retry', + request: Object.freeze({ + projectId, + sourceRunId, + requestId, + auditEventId: eventId(ordinal), + failureAuditEventId: eventId(ordinal + 500_000), + body: Object.freeze({ + schema: 'qinglong/run-manual-retry@v1', + mutationId, + expectedRunVersion: 3, + expectedRunStatus: 'failed', + }), + }), + }); +} + +function stopCommand(projectId, runId, requestId, mutationId, ordinal) { + return Object.freeze({ + schemaVersion: 1, + operation: 'run.stop', + request: Object.freeze({ + projectId, + runId, + requestId, + auditEventId: eventId(ordinal), + failureAuditEventId: eventId(ordinal + 500_000), + body: Object.freeze({ + schema: 'qinglong/run-cancellation@v1', + mutationId, + }), + }), + }); +} + +function sqlString(value) { + assert.equal(typeof value, 'string'); + return "'" + value.replaceAll("'", "''") + "'"; +} + +function seedRunManagement(fixture, podName, values, psql) { + const nowMs = Date.now(); + const sourceDigest = 'a'.repeat(64); + const taskRevision = `qltd:v1:1:${sourceDigest}`; + const taskDigest = 'b'.repeat(64); + const planDigest = 'c'.repeat(64); + psql( + fixture, + podName, + [ + 'BEGIN;', + 'INSERT INTO "ql3"."projects" (id, name, slug, status, version, created_at_ms, updated_at_ms)', + `VALUES (${sqlString( + values.projectId, + )}, 'Run Management Live', ${sqlString( + values.projectId, + )}, 'active', 1, ${nowMs}, ${nowMs});`, + 'INSERT INTO "ql3"."project_role_bindings" (project_id, subject_type, subject_id, version, state, role, mutation_id, changed_by_type, changed_by_id, created_at_ms)', + `VALUES (${sqlString(values.projectId)}, 'user', ${sqlString( + values.operatorId, + )}, 1, 'active', 'operator', ${sqlString( + 'binding-' + values.suffix, + )}, 'system', 'run-management-live', ${nowMs});`, + 'INSERT INTO "ql3"."task_definitions" (project_id, task_id, current_revision, created_at_ms, updated_at_ms)', + `VALUES (${sqlString(values.projectId)}, ${sqlString( + values.taskId, + )}, 1, ${nowMs}, ${nowMs});`, + 'INSERT INTO "ql3"."task_definition_revisions" (project_id, task_id, revision, mutation_id, name, kind, spec_json, labels_json, enabled, content_digest, created_at_ms)', + `VALUES (${sqlString(values.projectId)}, ${sqlString( + values.taskId, + )}, 1, ${sqlString( + crypto.randomUUID(), + )}::uuid, 'Run Management Live Task', 'command', '{"schema":"qinglong/command@v1","config":{"command":{"kind":"argv","file":"/bin/echo","args":["run-management-live"]}}}'::jsonb, '{}'::jsonb, true, ${sqlString( + taskDigest, + )}, ${nowMs});`, + 'INSERT INTO "ql3"."task_execution_revisions" (project_id, task_id, source_revision, task_revision, source_content_digest, executor_type, plan_schema, plan_json, content_digest, created_at_ms)', + `VALUES (${sqlString(values.projectId)}, ${sqlString( + values.taskId, + )}, 1, ${sqlString(taskRevision)}, ${sqlString( + sourceDigest, + )}, 'remote_worker', 'qinglong/command-execution@v1', '{"file":"/bin/echo","args":["run-management-live"]}'::jsonb, ${sqlString( + planDigest, + )}, ${nowMs});`, + 'INSERT INTO "ql3"."runs" (id, project_id, task_id, task_revision, task_name, task_snapshot_ref, trigger_type, execution_origin, execution_owner, status, version, event_sequence, priority, created_at_ms, queued_at_ms, finished_at_ms, error_code, error_summary)', + `VALUES (${sqlString(values.sourceRunId)}, ${sqlString( + values.projectId, + )}, ${sqlString(values.taskId)}, ${sqlString( + taskRevision, + )}, 'Run Management Live Task', ${sqlString( + taskRevision, + )}, 'manual', 'manual', 'runtime', 'failed', 3, 3, 0, ${nowMs}, ${nowMs}, ${nowMs}, 'LIVE_SOURCE_FAILURE', 'terminal source for Run management live');`, + 'INSERT INTO "ql3"."run_attempts" (id, run_id, attempt, status, executor_type, callback_sequence, created_at_ms, finished_at_ms, error_code, error_summary)', + `VALUES (${sqlString(values.sourceAttemptId)}, ${sqlString( + values.sourceRunId, + )}, 1, 'failed', 'remote_worker', 0, ${nowMs}, ${nowMs}, 'LIVE_SOURCE_FAILURE', 'terminal source for Run management live');`, + 'INSERT INTO "ql3"."run_events" (id, run_id, sequence, type, dedupe_key, actor_type, actor_id, attempt_id, payload, created_at_ms)', + `VALUES (${sqlString(crypto.randomUUID())}, ${sqlString( + values.sourceRunId, + )}, 1, 'run.created', ${sqlString( + 'run-management-live-created-' + values.suffix, + )}, 'user', ${sqlString(values.operatorId)}, ${sqlString( + values.sourceAttemptId, + )}, '{"status":"created","version":1}'::jsonb, ${nowMs}),`, + `(${sqlString(crypto.randomUUID())}, ${sqlString( + values.sourceRunId, + )}, 2, 'run.queued', ${sqlString( + 'run-management-live-queued-' + values.suffix, + )}, 'user', ${sqlString(values.operatorId)}, ${sqlString( + values.sourceAttemptId, + )}, '{"from_status":"created","to_status":"queued","version":2}'::jsonb, ${nowMs}),`, + `(${sqlString(crypto.randomUUID())}, ${sqlString( + values.sourceRunId, + )}, 3, 'run.failed', ${sqlString( + 'run-management-live-failed-' + values.suffix, + )}, 'executor', 'run-management-live', ${sqlString( + values.sourceAttemptId, + )}, '{"from_status":"queued","to_status":"failed","version":3,"error_code":"LIVE_SOURCE_FAILURE"}'::jsonb, ${nowMs});`, + 'COMMIT;', + ].join('\n'), + ); + return Object.freeze({ taskRevision, taskDigest, planDigest }); +} + +function durableRunManagementFacts(fixture, podName, values, psql) { + return JSON.parse( + psql( + fixture, + podName, + [ + 'SELECT json_build_object(', + ' \'sourceRunStatus\', (SELECT status FROM "ql3"."runs" WHERE id = ' + + sqlString(values.sourceRunId) + + '),', + ' \'retryRunCount\', (SELECT count(*)::integer FROM "ql3"."runs" WHERE project_id = ' + + sqlString(values.projectId) + + " AND trigger_type = 'run_manual_retry'),", + ' \'retryAttemptCount\', (SELECT count(*)::integer FROM "ql3"."run_attempts" AS attempt JOIN "ql3"."runs" AS run ON run.id = attempt.run_id WHERE run.project_id = ' + + sqlString(values.projectId) + + " AND run.trigger_type = 'run_manual_retry'),", + ' \'retryEventCount\', (SELECT count(*)::integer FROM "ql3"."run_events" AS event JOIN "ql3"."runs" AS run ON run.id = event.run_id WHERE run.project_id = ' + + sqlString(values.projectId) + + " AND run.trigger_type = 'run_manual_retry' AND event.type IN ('run.created', 'run.queued')),", + ' \'stoppedRunCount\', (SELECT count(*)::integer FROM "ql3"."runs" WHERE project_id = ' + + sqlString(values.projectId) + + " AND trigger_type = 'run_manual_retry' AND cancel_requested_at_ms IS NOT NULL AND cancel_reason = 'user'),", + ' \'stopEventCount\', (SELECT count(*)::integer FROM "ql3"."run_events" AS event JOIN "ql3"."runs" AS run ON run.id = event.run_id WHERE run.project_id = ' + + sqlString(values.projectId) + + " AND event.type = 'run.cancel_requested'),", + ' \'allowedAuditCount\', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE project_id = ' + + sqlString(values.projectId) + + " AND operation_id IN ('run.retry', 'run.stop') AND outcome = 'allowed'),", + ' \'deniedAuditCount\', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE project_id = ' + + sqlString(values.projectId) + + " AND operation_id IN ('run.retry', 'run.stop') AND outcome = 'denied'),", + ' \'weakAuthenticationAuditCount\', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE project_id = ' + + sqlString(values.projectId) + + " AND request_id = 'run-live-weak'),", + ' \'duplicateMutationCount\', greatest(0, (SELECT count(*)::integer FROM "ql3"."runs" WHERE project_id = ' + + sqlString(values.projectId) + + ' AND trigger_type = \'run_manual_retry\') - 1) + greatest(0, (SELECT count(*)::integer FROM "ql3"."run_events" AS event JOIN "ql3"."runs" AS run ON run.id = event.run_id WHERE run.project_id = ' + + sqlString(values.projectId) + + " AND event.type = 'run.cancel_requested') - 1),", + ' \'identityGeneration\', (SELECT generation::integer FROM "ql3"."plugin_package_identity_keyset_ledger" WHERE authority = \'run-management\'),', + ' \'migrationCount\', (SELECT count(*)::integer FROM "ql3"."schema_migrations"),', + ' \'controlCoreCapability\', (SELECT contract_version::integer FROM "ql3"."schema_capabilities" WHERE contract_name = \'control-core\'),', + " 'postgresVersionNumber', current_setting('server_version_num')::integer)", + ].join('\n'), + ), + ); +} + +module.exports = { + durableRunManagementFacts, + eventId, + retryCommand, + seedRunManagement, + sqlString, + stopCommand, +}; diff --git a/scripts/ql3-approval-management-kubernetes-live-contract.cjs b/scripts/ql3-approval-management-kubernetes-live-contract.cjs index 30a0c1c1..bb4c23f3 100644 --- a/scripts/ql3-approval-management-kubernetes-live-contract.cjs +++ b/scripts/ql3-approval-management-kubernetes-live-contract.cjs @@ -16,6 +16,7 @@ const { createMutualTlsPki } = require('./lib/ql3-live-pki.cjs'); const { clientTcpProbe, createManagementClientExecutor, + managementHealthStatus, patchManagementGeneration, podReady, podTcpProbe, @@ -90,10 +91,7 @@ const identity = createManagementIdentityCeremony({ }); function sha256(value) { - return ( - 'sha256:' + - crypto.createHash('sha256').update(value).digest('hex') - ); + return 'sha256:' + crypto.createHash('sha256').update(value).digest('hex'); } function randomSecret() { @@ -102,9 +100,7 @@ function randomSecret() { function eventId(ordinal) { assert.ok(Number.isSafeInteger(ordinal) && ordinal >= 1 && ordinal < 1e12); - return ( - '40000000-0000-4000-8000-' + String(ordinal).padStart(12, '0') - ); + return '40000000-0000-4000-8000-' + String(ordinal).padStart(12, '0'); } function reviewedKey(kid) { @@ -119,75 +115,12 @@ function assertion(key, suffix) { return identity.assertion(key, suffix); } -function assertionForSubject( - key, - subject, - suffix = crypto.randomUUID(), -) { - assert.match(subject, /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/); - const now = Math.floor(Date.now() / 1_000); - const header = Buffer.from( - JSON.stringify({ - alg: 'EdDSA', - kid: key.kid, - typ: 'ql3-approval-management+jwt', - }), - ).toString('base64url'); - const payload = Buffer.from( - JSON.stringify({ - acr: 'urn:ql3:mfa', - amr: ['pwd', 'otp'], - aud: AUDIENCE, - auth_time: now - 1, - exp: now + 290, - iat: now, - iss: ISSUER, - jti: 'ql3-approval-live-subject-' + suffix, - ql3_purpose: 'approval-management', - sub: subject, - }), - ).toString('base64url'); - const signed = header + '.' + payload; - return ( - signed + - '.' + - crypto - .sign(null, Buffer.from(signed, 'ascii'), key.privateKey) - .toString('base64url') - ); +function assertionForSubject(key, subject, suffix = crypto.randomUUID()) { + return identity.assertionForSubject(key, subject, 'subject-' + suffix); } function weakAssertion(key, suffix = crypto.randomUUID()) { - const now = Math.floor(Date.now() / 1_000); - const header = Buffer.from( - JSON.stringify({ - alg: 'EdDSA', - kid: key.kid, - typ: 'ql3-approval-management+jwt', - }), - ).toString('base64url'); - const payload = Buffer.from( - JSON.stringify({ - acr: 'urn:ql3:password', - amr: ['pwd'], - aud: AUDIENCE, - auth_time: now - 1, - exp: now + 290, - iat: now, - iss: ISSUER, - jti: 'ql3-approval-live-weak-' + suffix, - ql3_purpose: 'approval-management', - sub: 'approval-operator', - }), - ).toString('base64url'); - const signed = header + '.' + payload; - return ( - signed + - '.' + - crypto - .sign(null, Buffer.from(signed, 'ascii'), key.privateKey) - .toString('base64url') - ); + return identity.weakAssertion(key, suffix); } function commandBase(projectId, approvalRequestId, requestId, ordinal) { @@ -204,12 +137,7 @@ function inspectCommand(projectId, approvalRequestId, requestId, ordinal) { return Object.freeze({ schemaVersion: 1, operation: 'approval.inspect', - request: commandBase( - projectId, - approvalRequestId, - requestId, - ordinal, - ), + request: commandBase(projectId, approvalRequestId, requestId, ordinal), }); } @@ -224,12 +152,7 @@ function decisionCommand( schemaVersion: 1, operation: 'approval.decide', request: Object.freeze({ - ...commandBase( - projectId, - approvalRequestId, - requestId, - ordinal, - ), + ...commandBase(projectId, approvalRequestId, requestId, ordinal), expectedVersion: 1, expectedAction: ACTION, decisionId, @@ -327,12 +250,7 @@ function loadApprovalContract() { return require(file); } -function seedApproval( - fixture, - primaryPod, - projectId, - approvalRequestId, -) { +function seedApproval(fixture, primaryPod, projectId, approvalRequestId) { const { approvalRequestDigest, createApprovalRequest } = loadApprovalContract(); const requestedAtMs = Date.now() - 1_000; @@ -422,31 +340,15 @@ function patchGeneration(fixture, generation, annotations = {}) { } function healthStatus(fixture, pod, route) { - const script = [ - "const fs=require('node:fs');const https=require('node:https');", - "const request=https.request({host:'127.0.0.1',port:8447,path:process.argv[1],", - "servername:process.argv[2],ca:fs.readFileSync('/var/run/secrets/qinglong3/approval-management-tls/ca.crt'),", - "minVersion:'TLSv1.3',maxVersion:'TLSv1.3',rejectUnauthorized:true,agent:false},", - "(response)=>{response.resume();response.on('end',()=>process.stdout.write(String(response.statusCode)))});", - "request.on('error',(error)=>{process.stderr.write(error.message);process.exitCode=1});request.end();", - ].join('\n'); - return Number( - fixture.kubectl( - [ - '-n', - NAMESPACE, - 'exec', - pod.metadata.name, - '--', - 'node', - '-e', - script, - route, - SERVERNAME, - ], - { capture: true, quiet: true }, - ).stdout, - ); + return managementHealthStatus({ + fixture, + namespace: NAMESPACE, + podName: pod.metadata.name, + port: 8447, + route, + servername: SERVERNAME, + caFile: '/var/run/secrets/qinglong3/approval-management-tls/ca.crt', + }); } function privateReportPath(argv) { @@ -507,10 +409,7 @@ async function main(argv = process.argv.slice(2)) { ); const preloadTag = imageTag(reviewedImage); run(fixture.docker, ['tag', reviewedImage, preloadTag]); - fixture.loadImage( - preloadTag, - path.basename(preloadTag) + '.tar', - ); + fixture.loadImage(preloadTag, path.basename(preloadTag) + '.tar'); } const sourceRevision = run('git', ['rev-parse', 'HEAD'], { @@ -615,28 +514,24 @@ async function main(argv = process.argv.slice(2)) { '--timeout=20m', ]); const databasePods = ( - await waitFor( - 'three ready CloudNativePG instances', - 600_000, - () => { - const pods = fixture - .kubectlJson([ - '-n', - NAMESPACE, - 'get', - 'pods', - '-l', - 'cnpg.io/cluster=' + POSTGRES_CLUSTER, - ]) - .items.filter(podReady); - return pods.length === 3 - ? { ready: true, value: pods } - : { - ready: false, - fact: pods.length + '/3 ready database Pods', - }; - }, - ) + await waitFor('three ready CloudNativePG instances', 600_000, () => { + const pods = fixture + .kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'pods', + '-l', + 'cnpg.io/cluster=' + POSTGRES_CLUSTER, + ]) + .items.filter(podReady); + return pods.length === 3 + ? { ready: true, value: pods } + : { + ready: false, + fact: pods.length + '/3 ready database Pods', + }; + }) ).value; const migrationManifest = localManifest( @@ -685,12 +580,7 @@ async function main(argv = process.argv.slice(2)) { const approvalRequestId = 'approval-request-' + suffix; const decisionId = 'approval-decision-' + suffix; const primary = currentPrimaryPod(fixture); - seedApproval( - fixture, - primary, - projectId, - approvalRequestId, - ); + seedApproval(fixture, primary, projectId, approvalRequestId); const pki = createMutualTlsPki({ directory: fixture.temporary, @@ -764,13 +654,8 @@ async function main(argv = process.argv.slice(2)) { 1, ); assert.equal( - fixture.kubectlJson([ - '-n', - NAMESPACE, - 'get', - 'pdb', - DEPLOYMENT, - ]).spec.minAvailable, + fixture.kubectlJson(['-n', NAMESPACE, 'get', 'pdb', DEPLOYMENT]).spec + .minAvailable, 1, ); for (const pod of managerPods) { @@ -876,9 +761,7 @@ async function main(argv = process.argv.slice(2)) { { statusCode: 403, responseCode: 'forbidden' }, ); - const generation1Uids = new Set( - managerPods.map((pod) => pod.metadata.uid), - ); + const generation1Uids = new Set(managerPods.map((pod) => pod.metadata.uid)); applyIdentity(keysets[1]); patchGeneration(fixture, 2); const generation2 = await waitForTwoPreserved({ @@ -925,9 +808,7 @@ async function main(argv = process.argv.slice(2)) { assert.equal(decided.output.result.approval.state, 'approved'); assert.equal(decided.output.result.approval.version, 2); - const generation2Uids = new Set( - managerPods.map((pod) => pod.metadata.uid), - ); + const generation2Uids = new Set(managerPods.map((pod) => pod.metadata.uid)); applyIdentity(keysets[2]); patchGeneration(fixture, 3); const generation3 = await waitForTwoPreserved({ @@ -987,9 +868,7 @@ async function main(argv = process.argv.slice(2)) { '-l', 'app.kubernetes.io/name=' + DEPLOYMENT, ]) - .items.filter( - (pod) => pod.metadata.deletionTimestamp === undefined, - ); + .items.filter((pod) => pod.metadata.deletionTimestamp === undefined); const ready = pods.filter(podReady); const candidate = pods.find( (pod) => @@ -1127,27 +1006,23 @@ async function main(argv = process.argv.slice(2)) { }; }, ); - await waitFor( - 'CloudNativePG recovery to three instances', - 900_000, - () => { - const status = fixture.kubectlJson([ - '-n', - NAMESPACE, - 'get', - 'cluster', - POSTGRES_CLUSTER, - ]).status; - return Number(status.readyInstances) === 3 - ? { ready: true, value: status } - : { - ready: false, - fact: - String(status.readyInstances ?? 0) + - '/3 ready database instances', - }; - }, - ); + await waitFor('CloudNativePG recovery to three instances', 900_000, () => { + const status = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'cluster', + POSTGRES_CLUSTER, + ]).status; + return Number(status.readyInstances) === 3 + ? { ready: true, value: status } + : { + ready: false, + fact: + String(status.readyInstances ?? 0) + + '/3 ready database instances', + }; + }); const databaseService = fixture.kubectlJson([ '-n', @@ -1175,8 +1050,7 @@ async function main(argv = process.argv.slice(2)) { managerPods.map((pod, index) => executeClient( { - name: - 'ql3-approval-database-unavailable-' + String(index + 1), + name: 'ql3-approval-database-unavailable-' + String(index + 1), target: pod, command: decisionCommand( projectId, @@ -1209,9 +1083,7 @@ async function main(argv = process.argv.slice(2)) { ? { ready: true, value: current } : { ready: false, - fact: - String(current.status.readyReplicas ?? 0) + - ' ready replicas', + fact: String(current.status.readyReplicas ?? 0) + ' ready replicas', }; }); assert.deepEqual( @@ -1238,35 +1110,29 @@ async function main(argv = process.argv.slice(2)) { }, ]), ]); - await waitFor( - 'restored CloudNativePG service endpoint', - 120_000, - () => { - const endpoints = fixture.kubectlJson([ - '-n', - NAMESPACE, - 'get', - 'endpoints', - POSTGRES_CLUSTER + '-rw', - ]); - const count = endpoints.subsets?.flatMap( - (subset) => subset.addresses ?? [], - ).length; - return count >= 1 - ? { ready: true, value: count } - : { - ready: false, - fact: String(count ?? 0) + ' service endpoints', - }; - }, - ); + await waitFor('restored CloudNativePG service endpoint', 120_000, () => { + const endpoints = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'endpoints', + POSTGRES_CLUSTER + '-rw', + ]); + const count = endpoints.subsets?.flatMap( + (subset) => subset.addresses ?? [], + ).length; + return count >= 1 + ? { ready: true, value: count } + : { + ready: false, + fact: String(count ?? 0) + ' service endpoints', + }; + }); assert.deepEqual( managerPods.map((pod) => healthStatus(fixture, pod, '/readyz')), [503, 503], ); - const staleUids = new Set( - managerPods.map((pod) => pod.metadata.uid), - ); + const staleUids = new Set(managerPods.map((pod) => pod.metadata.uid)); patchGeneration(fixture, '3-database-recovered'); managerPods = await readyManagementPods({ ...managerOptions(fixture), @@ -1277,8 +1143,7 @@ async function main(argv = process.argv.slice(2)) { managerPods.map((pod, index) => executeClient( { - name: - 'ql3-approval-database-recovered-' + String(index + 1), + name: 'ql3-approval-database-recovered-' + String(index + 1), target: pod, command: decisionCommand( projectId, @@ -1402,10 +1267,7 @@ async function main(argv = process.argv.slice(2)) { resource, '-n', NAMESPACE, - '--as=system:serviceaccount:' + - NAMESPACE + - ':' + - DEPLOYMENT, + '--as=system:serviceaccount:' + NAMESPACE + ':' + DEPLOYMENT, ], { capture: true, quiet: true, allowFailure: true }, ); @@ -1576,12 +1438,8 @@ async function main(argv = process.argv.slice(2)) { port: 8447, replicas: deployment.spec.replicas, readyReplicas: managerPods.length, - podIdentitySha256: managerPods.map((pod) => - sha256(pod.metadata.uid), - ), - nodeIdentitySha256: managerPods.map((pod) => - sha256(pod.spec.nodeName), - ), + podIdentitySha256: managerPods.map((pod) => sha256(pod.metadata.uid)), + nodeIdentitySha256: managerPods.map((pod) => sha256(pod.spec.nodeName)), serviceAccount: DEPLOYMENT, automountServiceAccountToken: false, requiredPodAntiAffinity: true, @@ -1620,8 +1478,7 @@ async function main(argv = process.argv.slice(2)) { activeNewAssertionAccepted: replayed.statusCode === 200, rollbackSurgeFailedClosed: Boolean(rollback.value), twoReadyReplicasPreserved: - generation2.minimumReady >= 2 && - generation3.minimumReady >= 2, + generation2.minimumReady >= 2 && generation3.minimumReady >= 2, durableGenerationReachedThree: durable.identityGeneration === 3, }, certificateRotation: { @@ -1630,16 +1487,13 @@ async function main(argv = process.argv.slice(2)) { previousBundleSha256, currentBundleSha256, oldClientAcceptedBefore: initialRequests[0].statusCode === 200, - replacementClientAcceptedBefore: - initialRequests[1].statusCode === 200, + replacementClientAcceptedBefore: initialRequests[1].statusCode === 200, oldClientRejectedAfter: revokedCertificate.statusCode === 401, - replacementClientAcceptedAfter: - activeCertificate.statusCode === 200, + replacementClientAcceptedAfter: activeCertificate.statusCode === 200, fullPodReplacement: managerPods.every( (pod) => !preCertificateUids.has(pod.metadata.uid), ), - allReplicasReadyThroughout: - certificateRollout.minimumReady >= 2, + allReplicasReadyThroughout: certificateRollout.minimumReady >= 2, }, availability: { databaseFailureWithdrewReadiness: true, @@ -1660,8 +1514,7 @@ async function main(argv = process.argv.slice(2)) { publicInternetEgressDenied, cloudNativePgEgressAllowed, managerSecretReadDenied: canI('get', 'secrets') === 'no', - managerMutationRbacDenied: - canI('patch', 'deployments.apps') === 'no', + managerMutationRbacDenied: canI('patch', 'deployments.apps') === 'no', }, durability: { approvalVersion: durable.approvalVersion, @@ -1686,15 +1539,13 @@ async function main(argv = process.argv.slice(2)) { twoManagerPodsOnDistinctNodes: new Set(managerPods.map((pod) => pod.spec.nodeName)).size === 2, tls13ProductClientAcrossBothPods: - new Set(baselineSuccesses.map((entry) => entry.targetPod)).size >= - 2, + new Set(baselineSuccesses.map((entry) => entry.targetPod)).size >= 2, strongUserDecision: weakUserRejected.statusCode === 401 && outsiderDenied.statusCode === 403 && decided.statusCode === 200, identityProjectionRotation: durable.identityGeneration === 3, - certificateRevocationRollout: - revokedCertificate.statusCode === 401, + certificateRevocationRollout: revokedCertificate.statusCode === 401, databaseReadinessFence: true, durableFactsSurvivedFailover: true, leastPrivilege: rolesLeastPrivilege, @@ -1739,7 +1590,9 @@ if (require.main === module) { main().catch((error) => { process.stderr.write( 'QL3 approval management Kubernetes live contract failed: ' + - (error instanceof Error ? error.stack || error.message : String(error)) + + (error instanceof Error + ? error.stack || error.message + : String(error)) + '\n', ); process.exitCode = 1; diff --git a/scripts/ql3-run-management-kubernetes-live-audit.cjs b/scripts/ql3-run-management-kubernetes-live-audit.cjs new file mode 100644 index 00000000..9b9857fd --- /dev/null +++ b/scripts/ql3-run-management-kubernetes-live-audit.cjs @@ -0,0 +1,488 @@ +#!/usr/bin/env node + +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); + +const FIXTURE = 'qinglong/run-management-kubernetes-live-contract@v1'; +const LIMITATIONS = Object.freeze([ + 'three privileged K3s Docker nodes are not production infrastructure or control-plane HA evidence', + 'identity assertions use a deterministic local strong-User ceremony rather than an external IdP', + 'CloudNativePG failover inside one Docker host is not infrastructure STONITH evidence', +]); +const BANNED_KEYS = new Set([ + 'assertion', + 'authorization', + 'bearer', + 'certificate', + 'clientkey', + 'connectionstring', + 'dsn', + 'kubeconfig', + 'password', + 'privatekey', + 'secret', + 'tlskey', + 'token', +]); + +function exactKeys(value, expected) { + return ( + value !== null && + typeof value === 'object' && + !Array.isArray(value) && + JSON.stringify(Object.keys(value).sort()) === + JSON.stringify([...expected].sort()) + ); +} + +function isDigest(value) { + return typeof value === 'string' && /^sha256:[a-f0-9]{64}$/.test(value); +} + +function isIsoTime(value) { + return ( + typeof value === 'string' && + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/.test(value) && + Number.isFinite(Date.parse(value)) + ); +} + +function isToken(value, maximum = 128) { + return ( + typeof value === 'string' && + value.length >= 1 && + value.length <= maximum && + /^[A-Za-z0-9][A-Za-z0-9._:/@+-]*$/.test(value) + ); +} + +function containsSensitiveMaterial(value, key = '') { + if (BANNED_KEYS.has(key.toLowerCase())) return true; + if (typeof value === 'string') { + return ( + /-----BEGIN (?:CERTIFICATE|(?:RSA |EC |OPENSSH )?PRIVATE KEY)-----/.test( + value, + ) || + /postgres(?:ql)?:\/\/[^/\s]+:[^@\s]+@/i.test(value) || + /\beyJ[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}\b/.test( + value, + ) + ); + } + if (Array.isArray(value)) { + return value.some((entry) => containsSensitiveMaterial(entry)); + } + if (value && typeof value === 'object') { + return Object.entries(value).some(([childKey, child]) => + containsSensitiveMaterial(child, childKey), + ); + } + return false; +} + +function validKubernetesVersion(value) { + const match = + typeof value === 'string' + ? /^v1\.([0-9]{2,3})\.([0-9]+)(?:[-+][0-9A-Za-z](?:[0-9A-Za-z.-]{0,62}[0-9A-Za-z])?)?$/.exec( + value, + ) + : null; + return Boolean(match && Number(match[1]) >= 32); +} + +function uniqueDigests(value, count) { + return ( + Array.isArray(value) && + value.length === count && + value.every(isDigest) && + new Set(value).size === count + ); +} + +function allTrue(value, expected) { + return ( + exactKeys(value, expected) && expected.every((key) => value[key] === true) + ); +} + +function validateRunManagementKubernetesLiveReport(report) { + const findings = []; + const reject = (code, detail) => + findings.push(Object.freeze({ code, detail })); + if ( + !exactKeys(report, [ + 'schemaVersion', + 'fixture', + 'observedAt', + 'platform', + 'database', + 'deployment', + 'client', + 'identityRotation', + 'certificateRotation', + 'availability', + 'isolation', + 'durability', + 'gates', + 'limitations', + ]) || + report?.schemaVersion !== 1 || + report?.fixture !== FIXTURE || + !isIsoTime(report?.observedAt) + ) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_REPORT_SHAPE', + 'the report must use the exact versioned Run management live envelope', + ); + } + if (containsSensitiveMaterial(report)) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_SECRET_EXPOSURE', + 'the report must not contain credentials, assertions, certificates, DSNs, kubeconfig or private keys', + ); + } + + const platform = report?.platform; + if ( + !exactKeys(platform, [ + 'distribution', + 'kubernetesVersion', + 'architecture', + 'kubernetesImageId', + 'managementImageId', + 'cniName', + 'cniDistributionBinding', + 'controlPlaneNodes', + 'workerNodes', + 'cniReadyNodes', + ]) || + platform?.distribution !== 'k3s' || + !validKubernetesVersion(platform?.kubernetesVersion) || + !['amd64', 'arm64'].includes(platform?.architecture) || + !isDigest(platform?.kubernetesImageId) || + !isDigest(platform?.managementImageId) || + platform?.cniName !== 'flannel' || + platform?.cniDistributionBinding !== 'rancher/k3s:v1.34.3-k3s1' || + platform?.controlPlaneNodes !== 1 || + platform?.workerNodes !== 2 || + platform?.cniReadyNodes !== 3 + ) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_PLATFORM', + 'the fixture must bind three real K3s nodes, embedded Flannel and exact images', + ); + } + + const database = report?.database; + if ( + !exactKeys(database, [ + 'operator', + 'operatorVersion', + 'postgresVersionNumber', + 'postgresImageId', + 'instances', + 'readyInstances', + 'managerRole', + 'migrationCount', + 'controlCoreCapability', + 'tlsVerified', + 'primaryChangedDuringFailover', + ]) || + database?.operator !== 'cloudnative-pg' || + !isToken(database?.operatorVersion, 64) || + database?.postgresVersionNumber !== 180004 || + !isDigest(database?.postgresImageId) || + database?.instances !== 3 || + database?.readyInstances !== 3 || + database?.managerRole !== 'ql3_run_manager' || + database?.migrationCount !== 57 || + database?.controlCoreCapability !== 56 || + database?.tlsVerified !== true || + database?.primaryChangedDuringFailover !== true + ) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DATABASE', + 'three TLS CloudNativePG instances must run migration 57, capability 56 and the isolated Run manager role', + ); + } + + const deployment = report?.deployment; + if ( + !exactKeys(deployment, [ + 'namespace', + 'service', + 'port', + 'replicas', + 'readyReplicas', + 'podIdentitySha256', + 'nodeIdentitySha256', + 'serviceAccount', + 'automountServiceAccountToken', + 'requiredPodAntiAffinity', + 'podDisruptionBudgetMinAvailable', + 'maxUnavailable', + 'maxConnectionsPerPod', + ]) || + deployment?.namespace !== 'qinglong3-system' || + deployment?.service !== 'ql3-run-management' || + deployment?.port !== 8448 || + deployment?.replicas !== 2 || + deployment?.readyReplicas !== 2 || + !uniqueDigests(deployment?.podIdentitySha256, 2) || + !uniqueDigests(deployment?.nodeIdentitySha256, 2) || + deployment?.serviceAccount !== 'ql3-run-management' || + deployment?.automountServiceAccountToken !== false || + deployment?.requiredPodAntiAffinity !== true || + deployment?.podDisruptionBudgetMinAvailable !== 1 || + deployment?.maxUnavailable !== 0 || + deployment?.maxConnectionsPerPod !== 2 + ) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DEPLOYMENT', + 'two tokenless Run manager replicas must be ready on distinct nodes with the exact budget', + ); + } + + const client = report?.client; + if ( + !exactKeys(client, [ + 'binary', + 'operations', + 'inputKind', + 'inputImmutable', + 'callerDrivenJob', + 'backoffLimit', + 'serviceAccountTokenMounted', + 'rbacGranted', + 'transportProtocol', + 'mutualTls', + 'servernameVerified', + 'exactPodRequests', + 'retryStatuses', + 'stopStatuses', + 'responseRedacted', + ]) || + client?.binary !== 'ql3-run-client' || + JSON.stringify(client?.operations) !== + JSON.stringify(['run.retry', 'run.stop']) || + client?.inputKind !== 'Secret' || + client?.inputImmutable !== true || + client?.callerDrivenJob !== true || + client?.backoffLimit !== 0 || + client?.serviceAccountTokenMounted !== false || + client?.rbacGranted !== false || + client?.transportProtocol !== 'TLSv1.3' || + client?.mutualTls !== true || + client?.servernameVerified !== true || + client?.exactPodRequests !== 6 || + JSON.stringify(client?.retryStatuses) !== + JSON.stringify(['accepted', 'existing', 'existing']) || + JSON.stringify(client?.stopStatuses) !== + JSON.stringify(['accepted', 'already_requested', 'already_requested']) || + client?.responseRedacted !== true + ) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_CLIENT', + 'immutable caller-driven clients must retry and stop with exact replay across both Pods over TLS 1.3 mTLS', + ); + } + + if ( + !allTrue(report?.identityRotation, [ + 'overlapOldAssertionAccepted', + 'overlapNewAssertionAccepted', + 'revokedOldAssertionRejected', + 'activeNewAssertionAccepted', + 'rollbackSurgeFailedClosed', + 'twoReadyReplicasPreserved', + 'durableGenerationReachedThree', + ]) + ) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_IDENTITY_ROTATION', + 'identity overlap, revoke, rollback rejection and two-ready availability are mandatory', + ); + } + + const certificate = report?.certificateRotation; + if ( + !exactKeys(certificate, [ + 'previousSerialSha256', + 'currentSerialSha256', + 'previousBundleSha256', + 'currentBundleSha256', + 'oldClientAcceptedBefore', + 'replacementClientAcceptedBefore', + 'oldClientRejectedAfter', + 'replacementClientAcceptedAfter', + 'fullPodReplacement', + 'allReplicasReadyThroughout', + ]) || + !isDigest(certificate?.previousSerialSha256) || + !isDigest(certificate?.currentSerialSha256) || + certificate?.previousSerialSha256 === certificate?.currentSerialSha256 || + !isDigest(certificate?.previousBundleSha256) || + !isDigest(certificate?.currentBundleSha256) || + certificate?.previousBundleSha256 === certificate?.currentBundleSha256 || + ![ + 'oldClientAcceptedBefore', + 'replacementClientAcceptedBefore', + 'oldClientRejectedAfter', + 'replacementClientAcceptedAfter', + 'fullPodReplacement', + 'allReplicasReadyThroughout', + ].every((key) => certificate?.[key] === true) + ) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_CERTIFICATE_ROTATION', + 'CRL rotation must replace all Pods without dropping below two ready replicas', + ); + } + + if ( + !allTrue(report?.availability, [ + 'databaseFailureWithdrewReadiness', + 'databaseFailurePreservedLiveness', + 'stalePodsDidNotRecoverInPlace', + 'freshPodsRecoveredAfterDatabase', + 'bothReplicasServedAfterRecovery', + ]) + ) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_AVAILABILITY', + 'database loss must withdraw readiness, preserve liveness and require fresh manager Pods', + ); + } + if ( + !allTrue(report?.isolation, [ + 'labelledClientAllowed', + 'unlabelledClientDenied', + 'wrongPortDenied', + 'kubernetesApiEgressDenied', + 'publicInternetEgressDenied', + 'cloudNativePgEgressAllowed', + 'managerSecretReadDenied', + 'managerMutationRbacDenied', + ]) + ) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_ISOLATION', + 'CNI and Kubernetes RBAC least-privilege observations are incomplete', + ); + } + + const durability = report?.durability; + if ( + !exactKeys(durability, [ + 'sourceRunStatus', + 'retryRunCount', + 'retryAttemptCount', + 'retryEventCount', + 'stoppedRunCount', + 'stopEventCount', + 'allowedAuditCount', + 'deniedAuditCount', + 'duplicateMutationCount', + 'identityGeneration', + 'weakAuthenticationAuditCount', + 'survivedCloudNativePgFailover', + ]) || + durability?.sourceRunStatus !== 'failed' || + durability?.retryRunCount !== 1 || + durability?.retryAttemptCount !== 1 || + durability?.retryEventCount !== 2 || + durability?.stoppedRunCount !== 1 || + durability?.stopEventCount !== 1 || + durability?.allowedAuditCount !== 2 || + durability?.deniedAuditCount !== 1 || + durability?.duplicateMutationCount !== 0 || + durability?.identityGeneration !== 3 || + durability?.weakAuthenticationAuditCount !== 0 || + durability?.survivedCloudNativePgFailover !== true + ) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DURABILITY', + 'exact Run, Attempt, Event, cancellation and audit facts must survive failover without duplicates', + ); + } + + if ( + !allTrue(report?.gates, [ + 'realThreeNodeKubernetes', + 'realCniPolicy', + 'threeInstanceCloudNativePg', + 'twoManagerPodsOnDistinctNodes', + 'tls13ProductClientAcrossBothPods', + 'strongUserRetryAndStop', + 'identityProjectionRotation', + 'certificateRevocationRollout', + 'databaseReadinessFence', + 'durableFactsSurvivedFailover', + 'leastPrivilege', + 'passed', + ]) + ) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_GATES', + 'every independently observed release gate must pass', + ); + } + if (JSON.stringify(report?.limitations) !== JSON.stringify(LIMITATIONS)) { + reject( + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_LIMITATIONS', + 'the exact non-production limitations must remain visible', + ); + } + return Object.freeze({ + schemaVersion: 1, + fixture: FIXTURE, + findings: Object.freeze(findings), + compatible: findings.length === 0, + }); +} + +function main(argv = process.argv.slice(2)) { + const argument = argv[0] === '--' ? argv.slice(1) : argv; + if ( + argument.length !== 1 || + !argument[0].startsWith('--report=') || + !path.isAbsolute(argument[0].slice('--report='.length)) + ) { + throw new Error( + 'usage: ql3-run-management-kubernetes-live-audit --report=/absolute/private-report.json', + ); + } + const reportFile = argument[0].slice('--report='.length); + const stat = fs.lstatSync(reportFile); + if (!stat.isFile() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0) { + throw new Error( + 'Run management Kubernetes live report must be a private regular file', + ); + } + const audit = validateRunManagementKubernetesLiveReport( + JSON.parse(fs.readFileSync(reportFile, 'utf8')), + ); + process.stdout.write(JSON.stringify(audit, null, 2) + '\n'); + if (!audit.compatible) process.exitCode = 1; +} + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write( + 'QL3 Run management Kubernetes live audit failed: ' + + (error instanceof Error ? error.message : String(error)) + + '\n', + ); + process.exitCode = 1; + } +} + +module.exports = { + FIXTURE, + LIMITATIONS, + validateRunManagementKubernetesLiveReport, +}; diff --git a/scripts/ql3-run-management-kubernetes-live-contract.cjs b/scripts/ql3-run-management-kubernetes-live-contract.cjs new file mode 100644 index 00000000..80efebee --- /dev/null +++ b/scripts/ql3-run-management-kubernetes-live-contract.cjs @@ -0,0 +1,1259 @@ +#!/usr/bin/env node + +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { + K3sDockerLiveFixture, + run, + waitFor, +} = require('./lib/ql3-k3s-docker-live.cjs'); +const { createMutualTlsPki } = require('./lib/ql3-live-pki.cjs'); +const { + clientTcpProbe, + createManagementClientExecutor, + managementHealthStatus, + patchManagementGeneration, + podReady, + podTcpProbe, + readyManagementPods, + waitForTwoPreserved, + waitManagementRollout, +} = require('./lib/ql3-management-kubernetes-live.cjs'); +const { + applySecret, + currentPrimaryPod, + imageIdDigest, + localManifest, + psql, +} = require('./lib/ql3-management-kubernetes-live-platform.cjs'); +const { + createManagementIdentityCeremony, +} = require('./lib/ql3-management-live-identity.cjs'); +const { + durableRunManagementFacts, + retryCommand, + seedRunManagement, + stopCommand, +} = require('./lib/ql3-run-management-kubernetes-live-scenario.cjs'); +const { + imageDigest, + imageTag, + reviewedOperatorManifest, +} = require('./ql3-cloudnativepg-live-contract.cjs'); +const { + FIXTURE, + LIMITATIONS, + validateRunManagementKubernetesLiveReport, +} = require('./ql3-run-management-kubernetes-live-audit.cjs'); + +const ROOT = path.resolve(__dirname, '..'); +const NAMESPACE = 'qinglong3-system'; +const DEPLOYMENT = 'ql3-run-management'; +const SERVERNAME = `${DEPLOYMENT}.${NAMESPACE}.svc`; +const POSTGRES_CLUSTER = 'ql3-postgres'; +const ZERO_DIGEST = 'sha256:' + '0'.repeat(64); +const ISSUER = 'https://identity.qinglong.test/'; +const AUDIENCE = 'qinglong3-run-management'; +const LOCK = JSON.parse( + fs.readFileSync( + path.join( + ROOT, + 'deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/operator-lock.json', + ), + 'utf8', + ), +); +const OPERATOR_IMAGE = LOCK.operator.image; +const POSTGRES_IMAGE = LOCK.operand.image; +const ADMIN_IMAGE_BASE = 'ql3-run-manager-live'; +const CONTROL_IMAGE_BASE = 'ql3-run-migration-live'; +const ROLE_NAMES = Object.freeze([ + 'ql3_migration', + 'ql3_ai_maintenance', + 'ql3_ai_credential_manager', + 'ql3_ai_credential_tester', + 'ql3_runtime', + 'ql3_admin', + 'ql3_package_manager', + 'ql3_package_executor', + 'ql3_automation_manager', + 'ql3_approval_manager', + 'ql3_run_manager', + 'ql3_worker_credential_manager', + 'ql3_worker_credential_executor', + 'ql3_worker_ingress', +]); +const identity = createManagementIdentityCeremony({ + issuer: ISSUER, + audience: AUDIENCE, + purpose: 'run-management', + tokenType: 'ql3-run-management+jwt', + subject: 'run-operator', + jtiPrefix: 'ql3-run-live', +}); + +function sha256(value) { + return 'sha256:' + crypto.createHash('sha256').update(value).digest('hex'); +} + +function randomSecret() { + return crypto.randomBytes(32).toString('base64url'); +} + +function managerOptions(fixture) { + return { + fixture, + namespace: NAMESPACE, + deployment: DEPLOYMENT, + description: 'two Ready Run manager Pods on distinct nodes', + }; +} + +function patchGeneration(fixture, generation, annotations = {}) { + patchManagementGeneration({ + ...managerOptions(fixture), + generation, + annotations, + }); +} + +function healthStatus(fixture, pod, route) { + return managementHealthStatus({ + fixture, + namespace: NAMESPACE, + podName: pod.metadata.name, + port: 8448, + route, + servername: SERVERNAME, + caFile: '/var/run/secrets/qinglong3/run-management-tls/ca.crt', + }); +} + +function privateReportPath(argv) { + if ( + argv.length !== 1 || + !argv[0].startsWith('--report=') || + !path.isAbsolute(argv[0].slice('--report='.length)) + ) { + throw new Error( + 'usage: ql3-run-management-kubernetes-live-contract --report=/absolute/private-report.json', + ); + } + const reportFile = argv[0].slice('--report='.length); + if (fs.existsSync(reportFile)) { + throw new Error('refusing to overwrite the Run management live report'); + } + const parent = fs.lstatSync(path.dirname(reportFile)); + if (!parent.isDirectory() || parent.isSymbolicLink()) { + throw new Error( + 'Run management live report parent must be a real directory', + ); + } + return reportFile; +} + +async function main(argv = process.argv.slice(2)) { + const reportFile = privateReportPath(argv); + if (process.env.QL3_RUN_MANAGEMENT_KUBERNETES_LIVE !== '1') { + throw new Error( + 'Refusing to mutate Docker/Kubernetes without QL3_RUN_MANAGEMENT_KUBERNETES_LIVE=1', + ); + } + const operatorManifestFile = process.env.QL3_CNPG_OPERATOR_MANIFEST_FILE; + if (!operatorManifestFile) { + throw new Error('QL3_CNPG_OPERATOR_MANIFEST_FILE is required'); + } + const reviewedManifest = reviewedOperatorManifest(operatorManifestFile); + const fixture = new K3sDockerLiveFixture({ prefix: 'ql3-run-live' }); + const suffix = + process.pid.toString(36) + '-' + crypto.randomBytes(3).toString('hex'); + const adminImage = `${ADMIN_IMAGE_BASE}:${suffix}`; + const controlImage = `${CONTROL_IMAGE_BASE}:${suffix}`; + let adminImageBuilt = false; + let controlImageBuilt = false; + try { + const nodes = await fixture.start(); + const architecture = fixture.inspectImage(fixture.k3sImage).Architecture; + assert.ok(['amd64', 'arm64'].includes(architecture)); + for (const reviewedImage of [OPERATOR_IMAGE, POSTGRES_IMAGE]) { + run(fixture.docker, ['pull', reviewedImage]); + const inspected = fixture.inspectImage(reviewedImage); + assert.ok( + inspected.RepoDigests?.some((entry) => + entry.endsWith('@' + imageDigest(reviewedImage)), + ), + ); + const preloadTag = imageTag(reviewedImage); + run(fixture.docker, ['tag', reviewedImage, preloadTag]); + fixture.loadImage(preloadTag, path.basename(preloadTag) + '.tar'); + } + + const sourceRevision = run('git', ['rev-parse', 'HEAD'], { + capture: true, + quiet: true, + }).stdout; + for (const [dockerfile, image, archive] of [ + [ + 'deploy/containers/ql3-cluster-admin/Dockerfile', + adminImage, + 'run-admin.tar', + ], + [ + 'deploy/containers/ql3-cluster-control/Dockerfile', + controlImage, + 'run-control.tar', + ], + ]) { + run(fixture.docker, [ + 'build', + '--file', + dockerfile, + '--tag', + image, + '--build-arg', + 'SOURCE_REVISION=' + sourceRevision, + '.', + ]); + if (image === adminImage) adminImageBuilt = true; + else controlImageBuilt = true; + fixture.loadImage(image, archive); + } + const adminImageInfo = fixture.inspectImage(adminImage); + const postgresImageInfo = fixture.inspectImage(POSTGRES_IMAGE); + const k3sImageInfo = fixture.inspectImage(fixture.k3sImage); + + fixture.kubectl(['apply', '--server-side', '-f', reviewedManifest]); + fixture.kubectl([ + '-n', + 'cnpg-system', + 'set', + 'image', + 'deployment/cnpg-controller-manager', + 'manager=' + imageTag(OPERATOR_IMAGE), + ]); + fixture.kubectl([ + 'wait', + '--for=condition=Established', + 'crd/clusters.postgresql.cnpg.io', + 'crd/databaseroles.postgresql.cnpg.io', + 'crd/databases.postgresql.cnpg.io', + '--timeout=5m', + ]); + fixture.kubectl([ + '-n', + 'cnpg-system', + 'rollout', + 'status', + 'deployment/cnpg-controller-manager', + '--timeout=5m', + ]); + + fixture.kubectl([ + 'apply', + '-f', + 'deploy/kubernetes/ql3-cluster/base/namespace.yaml', + ]); + fixture.kubectl([ + '-n', + NAMESPACE, + 'apply', + '-f', + 'deploy/kubernetes/ql3-cluster/base/service-account.yaml', + ]); + const passwords = Object.fromEntries( + ROLE_NAMES.map((role) => [role, randomSecret()]), + ); + for (const role of ROLE_NAMES) { + applySecret( + fixture, + 'ql3-postgres-' + + role.replace(/^ql3_/, '').replaceAll('_', '-') + + '-auth', + 'kubernetes.io/basic-auth', + { username: role, password: passwords[role] }, + ); + } + const databaseManifest = fixture + .kubectl( + ['kustomize', 'deploy/kubernetes/ql3-cluster/operators/cloudnative-pg'], + { + capture: true, + quiet: true, + }, + ) + .stdout.replace(POSTGRES_IMAGE, imageTag(POSTGRES_IMAGE)); + fixture.kubectl(['apply', '-f', '-'], { input: databaseManifest + '\n' }); + fixture.kubectl([ + '-n', + NAMESPACE, + 'wait', + '--for=condition=Ready', + 'cluster/' + POSTGRES_CLUSTER, + '--timeout=20m', + ]); + const databasePods = ( + await waitFor('three ready CloudNativePG instances', 600_000, () => { + const pods = fixture + .kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'pods', + '-l', + 'cnpg.io/cluster=' + POSTGRES_CLUSTER, + ]) + .items.filter(podReady); + return pods.length === 3 + ? { ready: true, value: pods } + : { ready: false, fact: `${pods.length}/3 ready database Pods` }; + }) + ).value; + + const migrationManifest = localManifest( + fixture.kubectl( + [ + 'kustomize', + 'deploy/kubernetes/ql3-cluster/operations/cloudnative-pg', + ], + { capture: true, quiet: true }, + ).stdout, + 'registry.example.com/qinglong/qinglong3-cluster-control', + controlImage, + ); + fixture.kubectl(['create', '-f', '-'], { input: migrationManifest + '\n' }); + fixture.kubectl([ + '-n', + NAMESPACE, + 'wait', + '--for=condition=Complete', + 'job/ql3-cluster-migration', + '--timeout=10m', + ]); + const migrationPrimary = currentPrimaryPod(fixture); + const migrationState = JSON.parse( + psql( + fixture, + migrationPrimary.metadata.name, + [ + 'SELECT json_build_object(', + ' \'migrationCount\', (SELECT count(*)::integer FROM "ql3"."schema_migrations"),', + ' \'controlCoreCapability\', (SELECT contract_version::integer FROM "ql3"."schema_capabilities" WHERE contract_name = \'control-core\'))', + ].join('\n'), + ), + ); + assert.deepEqual(migrationState, { + migrationCount: 57, + controlCoreCapability: 56, + }); + + const values = Object.freeze({ + suffix, + projectId: 'run-live-' + suffix, + operatorId: 'run-operator', + outsiderId: 'run-outsider', + taskId: 'run-live-task-' + suffix, + sourceRunId: crypto.randomUUID(), + sourceAttemptId: crypto.randomUUID(), + }); + seedRunManagement(fixture, migrationPrimary.metadata.name, values, psql); + + const pki = createMutualTlsPki({ + directory: fixture.temporary, + servername: SERVERNAME, + label: 'QL3 Run Management Live', + run, + crypto, + }); + let pkiMaterial = pki.read(); + const oldKey = identity.reviewedKey('run-live-key-1'); + const newKey = identity.reviewedKey('run-live-key-2'); + const keysets = [ + identity.keyset(1, [oldKey]), + identity.keyset(2, [oldKey, newKey]), + identity.keyset(3, [oldKey, newKey], [oldKey.kid]), + ]; + const applyIdentity = (document) => + applySecret(fixture, DEPLOYMENT + '-identity', 'Opaque', { + 'keyset.json': JSON.stringify(document) + '\n', + }); + const applyTls = () => + applySecret(fixture, DEPLOYMENT + '-tls', 'kubernetes.io/tls', { + 'tls.crt': pkiMaterial.serverCertificate, + 'tls.key': pkiMaterial.serverKey, + 'ca.crt': pkiMaterial.ca, + 'client.crl': pkiMaterial.clientCrl, + }); + applyIdentity(keysets[0]); + applyTls(); + const previousBundleSha256 = pki.bundleSha256(); + let managerManifest = localManifest( + fixture.kubectl( + [ + 'kustomize', + 'deploy/kubernetes/ql3-cluster/operations/run-management/cloudnative-pg', + ], + { capture: true, quiet: true }, + ).stdout, + 'registry.example.com/qinglong/qinglong3-cluster-admin', + adminImage, + ); + assert.equal(managerManifest.split(ZERO_DIGEST).length - 1, 2); + managerManifest = managerManifest + .replace(ZERO_DIGEST, sha256(pkiMaterial.ca)) + .replace(ZERO_DIGEST, sha256(pkiMaterial.clientCrl)); + fixture.kubectl(['apply', '-f', '-'], { input: managerManifest + '\n' }); + waitManagementRollout(managerOptions(fixture)); + let managerPods = await readyManagementPods(managerOptions(fixture)); + + const deployment = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'deployment', + DEPLOYMENT, + ]); + assert.equal(deployment.spec.replicas, 2); + assert.equal(deployment.spec.strategy.rollingUpdate.maxUnavailable, 0); + assert.equal( + deployment.spec.template.spec.automountServiceAccountToken, + false, + ); + assert.equal( + deployment.spec.template.spec.affinity.podAntiAffinity + .requiredDuringSchedulingIgnoredDuringExecution.length, + 1, + ); + assert.equal( + fixture.kubectlJson(['-n', NAMESPACE, 'get', 'pdb', DEPLOYMENT]).spec + .minAvailable, + 1, + ); + for (const pod of managerPods) { + assert.equal(pod.spec.serviceAccountName, DEPLOYMENT); + assert.equal(pod.spec.automountServiceAccountToken, false); + } + + const executeClient = createManagementClientExecutor({ + fixture, + namespace: NAMESPACE, + servername: SERVERNAME, + port: 8448, + managementPath: '/api/v3/runs/management', + adminImage, + ca: pkiMaterial.ca, + serviceAccount: 'ql3-run-management-client', + appName: 'ql3-run-management-client', + component: 'run-management-client', + networkPolicyLabel: 'qinglong.io/run-management-client', + clientCliPath: + '/opt/qinglong/node_modules/@qinglong/cluster-admin/dist/run-management/runManagementClientCli.js', + retryableClientCodes: ['QL3_RUN_MANAGEMENT_CLIENT_FAILED'], + description: 'Run management', + }); + const retryMutationId = crypto.randomUUID(); + const retry = retryCommand( + values.projectId, + values.sourceRunId, + 'run-live-retry', + retryMutationId, + 1, + ); + const retryAccepted = await executeClient( + { + name: 'ql3-run-retry-accepted', + target: managerPods[0], + command: retry, + bearer: identity.assertion(oldKey), + clientCertificate: pkiMaterial.oldClientCertificate, + clientKey: pkiMaterial.oldClientKey, + }, + { statusCode: 200, resultField: 'retry', resultStatus: ['accepted'] }, + ); + const retriedRunId = retryAccepted.output.result.retry.runId; + const retryReplay = await executeClient( + { + name: 'ql3-run-retry-replay', + target: managerPods[1], + command: retry, + bearer: identity.assertion(oldKey), + clientCertificate: pkiMaterial.newClientCertificate, + clientKey: pkiMaterial.newClientKey, + }, + { statusCode: 200, resultField: 'retry', resultStatus: ['existing'] }, + ); + assert.equal(retryReplay.output.result.retry.runId, retriedRunId); + const weakRejected = await executeClient( + { + name: 'ql3-run-retry-weak', + target: managerPods[0], + command: retryCommand( + values.projectId, + values.sourceRunId, + 'run-live-weak', + crypto.randomUUID(), + 2, + ), + bearer: identity.weakAssertion(oldKey), + clientCertificate: pkiMaterial.newClientCertificate, + clientKey: pkiMaterial.newClientKey, + }, + { statusCode: 401, responseCode: 'authentication_required' }, + ); + const outsiderDenied = await executeClient( + { + name: 'ql3-run-retry-outsider', + target: managerPods[1], + command: retryCommand( + values.projectId, + values.sourceRunId, + 'run-live-outsider', + crypto.randomUUID(), + 3, + ), + bearer: identity.assertionForSubject(oldKey, values.outsiderId), + clientCertificate: pkiMaterial.newClientCertificate, + clientKey: pkiMaterial.newClientKey, + }, + { statusCode: 403, responseCode: 'forbidden' }, + ); + + const generation1Uids = new Set(managerPods.map((pod) => pod.metadata.uid)); + applyIdentity(keysets[1]); + patchGeneration(fixture, 2); + const generation2 = await waitForTwoPreserved({ + ...managerOptions(fixture), + excludedUids: generation1Uids, + expectedGeneration: 2, + description: 'zero-unavailable Run manager identity generation 2 rollout', + }); + managerPods = generation2.pods; + const stopMutationId = crypto.randomUUID(); + const stop = stopCommand( + values.projectId, + retriedRunId, + 'run-live-stop', + stopMutationId, + 4, + ); + const overlapOld = await executeClient( + { + name: 'ql3-run-stop-overlap-old', + target: managerPods[0], + command: stop, + bearer: identity.assertion(oldKey), + clientCertificate: pkiMaterial.newClientCertificate, + clientKey: pkiMaterial.newClientKey, + }, + { statusCode: 200, resultField: 'stop', resultStatus: ['accepted'] }, + ); + const overlapNew = await executeClient( + { + name: 'ql3-run-stop-overlap-new', + target: managerPods[1], + command: stop, + bearer: identity.assertion(newKey), + clientCertificate: pkiMaterial.newClientCertificate, + clientKey: pkiMaterial.newClientKey, + }, + { + statusCode: 200, + resultField: 'stop', + resultStatus: ['already_requested'], + }, + ); + + const generation2Uids = new Set(managerPods.map((pod) => pod.metadata.uid)); + applyIdentity(keysets[2]); + patchGeneration(fixture, 3); + const generation3 = await waitForTwoPreserved({ + ...managerOptions(fixture), + excludedUids: generation2Uids, + expectedGeneration: 3, + description: 'zero-unavailable Run manager identity generation 3 rollout', + }); + managerPods = generation3.pods; + const rejectedOldKey = await executeClient( + { + name: 'ql3-run-stop-revoked-key', + target: managerPods[0], + command: stop, + bearer: identity.assertion(oldKey), + clientCertificate: pkiMaterial.newClientCertificate, + clientKey: pkiMaterial.newClientKey, + }, + { statusCode: 401, responseCode: 'authentication_required' }, + ); + const activeNew = await executeClient( + { + name: 'ql3-run-stop-active-key', + target: managerPods[1], + command: stop, + bearer: identity.assertion(newKey), + clientCertificate: pkiMaterial.newClientCertificate, + clientKey: pkiMaterial.newClientKey, + }, + { + statusCode: 200, + resultField: 'stop', + resultStatus: ['already_requested'], + }, + ); + + applyIdentity(keysets[1]); + patchGeneration(fixture, 'rollback-2'); + const rollback = await waitFor( + 'Run identity ledger rollback surge failure', + 180_000, + () => { + const pods = fixture + .kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'pods', + '-l', + 'app.kubernetes.io/name=' + DEPLOYMENT, + ]) + .items.filter((pod) => pod.metadata.deletionTimestamp === undefined); + const ready = pods.filter(podReady); + const candidate = pods.find( + (pod) => + !managerPods.some( + (current) => current.metadata.uid === pod.metadata.uid, + ) && + pod.status.containerStatuses?.[0] && + !pod.status.containerStatuses[0].ready && + (pod.status.containerStatuses[0].restartCount > 0 || + pod.status.containerStatuses[0].state?.waiting?.reason === + 'CrashLoopBackOff'), + ); + return ready.length === 2 && candidate + ? { ready: true, value: candidate } + : { + ready: false, + fact: `${ready.length} ready Pods; rollback=${Boolean( + candidate, + )}`, + }; + }, + ); + applyIdentity(keysets[2]); + patchGeneration(fixture, '3-rollback-recovered'); + fixture.kubectl([ + '-n', + NAMESPACE, + 'delete', + 'pod', + rollback.value.metadata.name, + '--grace-period=0', + '--force', + '--wait=true', + ]); + waitManagementRollout(managerOptions(fixture)); + managerPods = await readyManagementPods(managerOptions(fixture)); + + const previousSerialSha256 = pki.oldSerialSha256(); + pki.revokeOldClient(); + pkiMaterial = pki.read(); + const currentBundleSha256 = pki.bundleSha256(); + const preCertificateUids = new Set( + managerPods.map((pod) => pod.metadata.uid), + ); + applyTls(); + patchGeneration(fixture, '3-client-crl-2', { + 'qinglong.io/run-management-client-ca-sha256': sha256(pkiMaterial.ca), + 'qinglong.io/run-management-client-crl-sha256': sha256( + pkiMaterial.clientCrl, + ), + }); + const certificateRollout = await waitForTwoPreserved({ + ...managerOptions(fixture), + excludedUids: preCertificateUids, + expectedGeneration: '3-client-crl-2', + description: 'zero-unavailable Run manager client certificate rollout', + }); + const certificatePodsFullyReplaced = certificateRollout.pods.every( + (pod) => !preCertificateUids.has(pod.metadata.uid), + ); + assert.equal(certificatePodsFullyReplaced, true); + managerPods = certificateRollout.pods; + const revokedCertificate = await executeClient( + { + name: 'ql3-run-retry-revoked-cert', + target: managerPods[0], + command: retry, + bearer: identity.assertion(newKey), + clientCertificate: pkiMaterial.oldClientCertificate, + clientKey: pkiMaterial.oldClientKey, + }, + { statusCode: 401, responseCode: 'client_certificate_required' }, + ); + const activeCertificate = await executeClient( + { + name: 'ql3-run-retry-active-cert', + target: managerPods[1], + command: retry, + bearer: identity.assertion(newKey), + clientCertificate: pkiMaterial.newClientCertificate, + clientKey: pkiMaterial.newClientKey, + }, + { statusCode: 200, resultField: 'retry', resultStatus: ['existing'] }, + ); + + const primaryBeforeFailover = currentPrimaryPod(fixture); + fixture.kubectl([ + '-n', + NAMESPACE, + 'delete', + 'pod', + primaryBeforeFailover.metadata.name, + '--grace-period=0', + '--force', + '--wait=false', + ]); + const promoted = await waitFor( + 'CloudNativePG primary promotion', + 600_000, + () => { + const status = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'cluster', + POSTGRES_CLUSTER, + ]).status; + return status.currentPrimary && + status.currentPrimary !== primaryBeforeFailover.metadata.name && + Number(status.readyInstances) >= 2 + ? { ready: true, value: status.currentPrimary } + : { + ready: false, + fact: `primary=${status.currentPrimary || 'none'} ready=${ + status.readyInstances ?? 0 + }`, + }; + }, + ); + await waitFor('CloudNativePG recovery to three instances', 900_000, () => { + const status = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'cluster', + POSTGRES_CLUSTER, + ]).status; + return Number(status.readyInstances) === 3 + ? { ready: true, value: status } + : { + ready: false, + fact: `${status.readyInstances ?? 0}/3 ready database instances`, + }; + }); + + const databaseService = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'service', + POSTGRES_CLUSTER + '-rw', + ]); + const databaseSelector = databaseService.spec.selector; + fixture.kubectl([ + '-n', + NAMESPACE, + 'patch', + 'service', + POSTGRES_CLUSTER + '-rw', + '--type=merge', + '-p', + JSON.stringify({ + spec: { selector: { ...databaseSelector, 'ql3.invalid': 'true' } }, + }), + ]); + const unavailable = await Promise.all( + managerPods.map((pod, index) => + executeClient( + { + name: 'ql3-run-database-unavailable-' + String(index + 1), + target: pod, + command: stop, + bearer: identity.assertion(newKey), + clientCertificate: pkiMaterial.newClientCertificate, + clientKey: pkiMaterial.newClientKey, + }, + { statusCode: 503, responseCode: 'unavailable' }, + ), + ), + ); + assert.deepEqual( + unavailable.map((entry) => entry.statusCode), + [503, 503], + ); + await waitFor('Run manager readiness withdrawal', 60_000, () => { + const current = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'deployment', + DEPLOYMENT, + ]); + return Number(current.status.readyReplicas ?? 0) === 0 + ? { ready: true, value: current } + : { + ready: false, + fact: `${current.status.readyReplicas ?? 0} ready replicas`, + }; + }); + assert.deepEqual( + managerPods.map((pod) => healthStatus(fixture, pod, '/readyz')), + [503, 503], + ); + assert.deepEqual( + managerPods.map((pod) => healthStatus(fixture, pod, '/livez')), + [200, 200], + ); + fixture.kubectl([ + '-n', + NAMESPACE, + 'patch', + 'service', + POSTGRES_CLUSTER + '-rw', + '--type=json', + '-p', + JSON.stringify([ + { op: 'replace', path: '/spec/selector', value: databaseSelector }, + ]), + ]); + await waitFor('restored CloudNativePG service endpoint', 120_000, () => { + const endpoints = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'endpoints', + POSTGRES_CLUSTER + '-rw', + ]); + const count = endpoints.subsets?.flatMap( + (subset) => subset.addresses ?? [], + ).length; + return count >= 1 + ? { ready: true, value: count } + : { ready: false, fact: `${count ?? 0} service endpoints` }; + }); + assert.deepEqual( + managerPods.map((pod) => healthStatus(fixture, pod, '/readyz')), + [503, 503], + ); + const staleUids = new Set(managerPods.map((pod) => pod.metadata.uid)); + patchGeneration(fixture, '3-database-recovered'); + managerPods = await readyManagementPods({ + ...managerOptions(fixture), + excludedUids: staleUids, + expectedGeneration: '3-database-recovered', + }); + const recoveredRequests = await Promise.all( + managerPods.map((pod, index) => + executeClient( + { + name: 'ql3-run-database-recovered-' + String(index + 1), + target: pod, + command: stop, + bearer: identity.assertion(newKey), + clientCertificate: pkiMaterial.newClientCertificate, + clientKey: pkiMaterial.newClientKey, + }, + { + statusCode: 200, + resultField: 'stop', + resultStatus: ['already_requested'], + }, + ), + ), + ); + + const finalPrimary = currentPrimaryPod(fixture); + const durable = durableRunManagementFacts( + fixture, + finalPrimary.metadata.name, + values, + psql, + ); + assert.deepEqual(durable, { + sourceRunStatus: 'failed', + retryRunCount: 1, + retryAttemptCount: 1, + retryEventCount: 2, + stoppedRunCount: 1, + stopEventCount: 1, + allowedAuditCount: 2, + deniedAuditCount: 1, + weakAuthenticationAuditCount: 0, + duplicateMutationCount: 0, + identityGeneration: 3, + migrationCount: 57, + controlCoreCapability: 56, + postgresVersionNumber: 180004, + }); + const roleRows = JSON.parse( + psql( + fixture, + finalPrimary.metadata.name, + `SELECT json_agg(json_build_object('name', rolname, 'login', rolcanlogin, 'superuser', rolsuper, 'createDatabase', rolcreatedb, 'createRole', rolcreaterole, 'replication', rolreplication, 'bypassRls', rolbypassrls) ORDER BY rolname) FROM pg_roles WHERE rolname IN (${ROLE_NAMES.map( + (role) => "'" + role + "'", + ).join(',')})`, + ), + ); + const rolesLeastPrivilege = roleRows.every( + (role) => + role.login === true && + role.superuser === false && + role.createDatabase === false && + role.createRole === false && + role.replication === false && + role.bypassRls === false, + ); + assert.equal(rolesLeastPrivilege, true); + const canI = (verb, resource) => { + const result = fixture.kubectl( + [ + 'auth', + 'can-i', + verb, + resource, + '-n', + NAMESPACE, + '--as=system:serviceaccount:' + NAMESPACE + ':' + DEPLOYMENT, + ], + { capture: true, quiet: true, allowFailure: true }, + ); + const decision = result.stdout.trim(); + assert.ok(decision === 'yes' || decision === 'no'); + assert.equal(result.status === 0, decision === 'yes'); + return decision; + }; + assert.equal(canI('get', 'secrets'), 'no'); + assert.equal(canI('patch', 'deployments.apps'), 'no'); + + const managerServiceIp = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'service', + DEPLOYMENT, + ]).spec.clusterIP; + const networkProbe = { + fixture, + namespace: NAMESPACE, + adminImage, + appName: 'ql3-run-network-probe', + networkPolicyLabel: 'qinglong.io/run-management-client', + }; + const labelledClientAllowed = await clientTcpProbe({ + ...networkProbe, + name: 'ql3-run-network-labelled', + targetHost: managerServiceIp, + port: 8448, + labelled: true, + expectedConnected: true, + }); + const unlabelledClientDenied = await clientTcpProbe({ + ...networkProbe, + name: 'ql3-run-network-unlabelled', + targetHost: managerServiceIp, + port: 8448, + labelled: false, + expectedConnected: false, + }); + const wrongPortDenied = await clientTcpProbe({ + ...networkProbe, + name: 'ql3-run-network-wrong-port', + targetHost: managerServiceIp, + port: 8447, + labelled: true, + expectedConnected: false, + }); + const kubernetesServiceIp = fixture.kubectlJson([ + 'get', + 'service', + 'kubernetes', + '-n', + 'default', + ]).spec.clusterIP; + const postgresServiceIp = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'service', + POSTGRES_CLUSTER + '-rw', + ]).spec.clusterIP; + const podProbe = { + fixture, + namespace: NAMESPACE, + podName: managerPods[0].metadata.name, + }; + const cloudNativePgEgressAllowed = + podTcpProbe({ ...podProbe, host: postgresServiceIp, port: 5432 }) + .status === 0; + const kubernetesApiEgressDenied = + podTcpProbe({ ...podProbe, host: kubernetesServiceIp, port: 443 }) + .status !== 0; + const publicInternetEgressDenied = + podTcpProbe({ ...podProbe, host: '1.1.1.1', port: 443 }).status !== 0; + assert.equal(cloudNativePgEgressAllowed, true); + assert.equal(kubernetesApiEgressDenied, true); + assert.equal(publicInternetEgressDenied, true); + + const finalNodes = fixture.kubectlJson(['get', 'nodes']).items; + const cniReadyNodes = finalNodes.filter( + (node) => + podReady(node) && + Array.isArray(node.spec.podCIDRs) && + node.spec.podCIDRs.length === 1, + ); + assert.equal(cniReadyNodes.length, 3); + assert.equal( + new Set(cniReadyNodes.map((node) => node.spec.podCIDRs[0])).size, + 3, + ); + const serverNode = finalNodes.find( + (node) => node.metadata.name === fixture.server, + ); + assert.equal( + serverNode?.metadata.annotations?.[ + 'flannel.alpha.coreos.com/backend-type' + ], + 'vxlan', + ); + assert.equal( + serverNode?.metadata.annotations?.[ + 'flannel.alpha.coreos.com/kube-subnet-manager' + ], + 'true', + ); + const finalCluster = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'cluster', + POSTGRES_CLUSTER, + ]); + assert.equal(Number(finalCluster.status.readyInstances), 3); + + const baselineSuccesses = [ + retryAccepted, + retryReplay, + overlapOld, + overlapNew, + activeNew, + activeCertificate, + ]; + const report = { + schemaVersion: 1, + fixture: FIXTURE, + observedAt: new Date().toISOString(), + platform: { + distribution: 'k3s', + kubernetesVersion: nodes[0].status.nodeInfo.kubeletVersion, + architecture, + kubernetesImageId: imageIdDigest(k3sImageInfo), + managementImageId: imageIdDigest(adminImageInfo), + cniName: 'flannel', + cniDistributionBinding: fixture.k3sImage, + controlPlaneNodes: 1, + workerNodes: 2, + cniReadyNodes: cniReadyNodes.length, + }, + database: { + operator: 'cloudnative-pg', + operatorVersion: LOCK.operator.version, + postgresVersionNumber: durable.postgresVersionNumber, + postgresImageId: imageIdDigest(postgresImageInfo), + instances: Number(finalCluster.spec.instances), + readyInstances: Number(finalCluster.status.readyInstances), + managerRole: 'ql3_run_manager', + migrationCount: durable.migrationCount, + controlCoreCapability: durable.controlCoreCapability, + tlsVerified: true, + primaryChangedDuringFailover: + promoted.value !== primaryBeforeFailover.metadata.name, + }, + deployment: { + namespace: NAMESPACE, + service: DEPLOYMENT, + port: 8448, + replicas: deployment.spec.replicas, + readyReplicas: managerPods.length, + podIdentitySha256: managerPods.map((pod) => sha256(pod.metadata.uid)), + nodeIdentitySha256: managerPods.map((pod) => sha256(pod.spec.nodeName)), + serviceAccount: DEPLOYMENT, + automountServiceAccountToken: false, + requiredPodAntiAffinity: true, + podDisruptionBudgetMinAvailable: 1, + maxUnavailable: 0, + maxConnectionsPerPod: 2, + }, + client: { + binary: 'ql3-run-client', + operations: ['run.retry', 'run.stop'], + inputKind: 'Secret', + inputImmutable: true, + callerDrivenJob: true, + backoffLimit: 0, + serviceAccountTokenMounted: false, + rbacGranted: false, + transportProtocol: 'TLSv1.3', + mutualTls: true, + servernameVerified: true, + exactPodRequests: baselineSuccesses.length, + retryStatuses: [ + retryAccepted.output.result.retry.status, + retryReplay.output.result.retry.status, + activeCertificate.output.result.retry.status, + ], + stopStatuses: [ + overlapOld.output.result.stop.status, + overlapNew.output.result.stop.status, + activeNew.output.result.stop.status, + ], + responseRedacted: true, + }, + identityRotation: { + overlapOldAssertionAccepted: overlapOld.statusCode === 200, + overlapNewAssertionAccepted: overlapNew.statusCode === 200, + revokedOldAssertionRejected: rejectedOldKey.statusCode === 401, + activeNewAssertionAccepted: activeNew.statusCode === 200, + rollbackSurgeFailedClosed: Boolean(rollback.value), + twoReadyReplicasPreserved: + generation2.minimumReady >= 2 && generation3.minimumReady >= 2, + durableGenerationReachedThree: durable.identityGeneration === 3, + }, + certificateRotation: { + previousSerialSha256, + currentSerialSha256: pki.newSerialSha256(), + previousBundleSha256, + currentBundleSha256, + oldClientAcceptedBefore: retryAccepted.statusCode === 200, + replacementClientAcceptedBefore: retryReplay.statusCode === 200, + oldClientRejectedAfter: revokedCertificate.statusCode === 401, + replacementClientAcceptedAfter: activeCertificate.statusCode === 200, + fullPodReplacement: certificatePodsFullyReplaced, + allReplicasReadyThroughout: certificateRollout.minimumReady >= 2, + }, + availability: { + databaseFailureWithdrewReadiness: true, + databaseFailurePreservedLiveness: true, + stalePodsDidNotRecoverInPlace: true, + freshPodsRecoveredAfterDatabase: managerPods.every( + (pod) => !staleUids.has(pod.metadata.uid), + ), + bothReplicasServedAfterRecovery: recoveredRequests.every( + (entry) => entry.statusCode === 200, + ), + }, + isolation: { + labelledClientAllowed, + unlabelledClientDenied, + wrongPortDenied, + kubernetesApiEgressDenied, + publicInternetEgressDenied, + cloudNativePgEgressAllowed, + managerSecretReadDenied: canI('get', 'secrets') === 'no', + managerMutationRbacDenied: canI('patch', 'deployments.apps') === 'no', + }, + durability: { + sourceRunStatus: durable.sourceRunStatus, + retryRunCount: durable.retryRunCount, + retryAttemptCount: durable.retryAttemptCount, + retryEventCount: durable.retryEventCount, + stoppedRunCount: durable.stoppedRunCount, + stopEventCount: durable.stopEventCount, + allowedAuditCount: durable.allowedAuditCount, + deniedAuditCount: durable.deniedAuditCount, + duplicateMutationCount: durable.duplicateMutationCount, + identityGeneration: durable.identityGeneration, + weakAuthenticationAuditCount: durable.weakAuthenticationAuditCount, + survivedCloudNativePgFailover: true, + }, + gates: { + realThreeNodeKubernetes: nodes.length === 3, + realCniPolicy: + labelledClientAllowed && + unlabelledClientDenied && + wrongPortDenied && + kubernetesApiEgressDenied && + publicInternetEgressDenied && + cloudNativePgEgressAllowed, + threeInstanceCloudNativePg: databasePods.length === 3, + twoManagerPodsOnDistinctNodes: + new Set(managerPods.map((pod) => pod.spec.nodeName)).size === 2, + tls13ProductClientAcrossBothPods: + new Set(baselineSuccesses.map((entry) => entry.targetPod)).size >= 2, + strongUserRetryAndStop: + weakRejected.statusCode === 401 && + outsiderDenied.statusCode === 403 && + retryAccepted.statusCode === 200 && + overlapOld.statusCode === 200, + identityProjectionRotation: durable.identityGeneration === 3, + certificateRevocationRollout: revokedCertificate.statusCode === 401, + databaseReadinessFence: true, + durableFactsSurvivedFailover: true, + leastPrivilege: rolesLeastPrivilege, + passed: true, + }, + limitations: [...LIMITATIONS], + }; + const audit = validateRunManagementKubernetesLiveReport(report); + assert.deepEqual(audit.findings, []); + fs.writeFileSync(reportFile, JSON.stringify(report, null, 2) + '\n', { + mode: 0o600, + flag: 'wx', + }); + process.stdout.write( + JSON.stringify({ + schemaVersion: 1, + fixture: FIXTURE, + reportWritten: true, + passed: true, + }) + '\n', + ); + } finally { + await fixture.cleanup(); + if (adminImageBuilt) { + run(fixture.docker, ['image', 'rm', '-f', adminImage], { + capture: true, + quiet: true, + allowFailure: true, + }); + } + if (controlImageBuilt) { + run(fixture.docker, ['image', 'rm', '-f', controlImage], { + capture: true, + quiet: true, + allowFailure: true, + }); + } + } +} + +if (require.main === module) { + main().catch((error) => { + process.stderr.write( + 'QL3 Run management Kubernetes live contract failed: ' + + (error instanceof Error + ? error.stack || error.message + : String(error)) + + '\n', + ); + process.exitCode = 1; + }); +} + +module.exports = { + identity, + retryCommand, + stopCommand, +}; diff --git a/test/back/ql3RunManagementKubernetesLiveAudit.test.cjs b/test/back/ql3RunManagementKubernetesLiveAudit.test.cjs new file mode 100644 index 00000000..7853dad0 --- /dev/null +++ b/test/back/ql3RunManagementKubernetesLiveAudit.test.cjs @@ -0,0 +1,244 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + FIXTURE, + LIMITATIONS, + validateRunManagementKubernetesLiveReport, +} = require('../../scripts/ql3-run-management-kubernetes-live-audit.cjs'); + +function digest(character) { + return 'sha256:' + character.repeat(64); +} + +function validReport() { + return { + schemaVersion: 1, + fixture: FIXTURE, + observedAt: '2026-08-12T12:00:00.000Z', + platform: { + distribution: 'k3s', + kubernetesVersion: 'v1.34.3+k3s1', + architecture: 'arm64', + kubernetesImageId: digest('1'), + managementImageId: digest('2'), + cniName: 'flannel', + cniDistributionBinding: 'rancher/k3s:v1.34.3-k3s1', + controlPlaneNodes: 1, + workerNodes: 2, + cniReadyNodes: 3, + }, + database: { + operator: 'cloudnative-pg', + operatorVersion: '1.30.0', + postgresVersionNumber: 180004, + postgresImageId: digest('3'), + instances: 3, + readyInstances: 3, + managerRole: 'ql3_run_manager', + migrationCount: 57, + controlCoreCapability: 56, + tlsVerified: true, + primaryChangedDuringFailover: true, + }, + deployment: { + namespace: 'qinglong3-system', + service: 'ql3-run-management', + port: 8448, + replicas: 2, + readyReplicas: 2, + podIdentitySha256: [digest('4'), digest('5')], + nodeIdentitySha256: [digest('6'), digest('7')], + serviceAccount: 'ql3-run-management', + automountServiceAccountToken: false, + requiredPodAntiAffinity: true, + podDisruptionBudgetMinAvailable: 1, + maxUnavailable: 0, + maxConnectionsPerPod: 2, + }, + client: { + binary: 'ql3-run-client', + operations: ['run.retry', 'run.stop'], + inputKind: 'Secret', + inputImmutable: true, + callerDrivenJob: true, + backoffLimit: 0, + serviceAccountTokenMounted: false, + rbacGranted: false, + transportProtocol: 'TLSv1.3', + mutualTls: true, + servernameVerified: true, + exactPodRequests: 6, + retryStatuses: ['accepted', 'existing', 'existing'], + stopStatuses: ['accepted', 'already_requested', 'already_requested'], + responseRedacted: true, + }, + identityRotation: { + overlapOldAssertionAccepted: true, + overlapNewAssertionAccepted: true, + revokedOldAssertionRejected: true, + activeNewAssertionAccepted: true, + rollbackSurgeFailedClosed: true, + twoReadyReplicasPreserved: true, + durableGenerationReachedThree: true, + }, + certificateRotation: { + previousSerialSha256: digest('8'), + currentSerialSha256: digest('9'), + previousBundleSha256: digest('a'), + currentBundleSha256: digest('b'), + oldClientAcceptedBefore: true, + replacementClientAcceptedBefore: true, + oldClientRejectedAfter: true, + replacementClientAcceptedAfter: true, + fullPodReplacement: true, + allReplicasReadyThroughout: true, + }, + availability: { + databaseFailureWithdrewReadiness: true, + databaseFailurePreservedLiveness: true, + stalePodsDidNotRecoverInPlace: true, + freshPodsRecoveredAfterDatabase: true, + bothReplicasServedAfterRecovery: true, + }, + isolation: { + labelledClientAllowed: true, + unlabelledClientDenied: true, + wrongPortDenied: true, + kubernetesApiEgressDenied: true, + publicInternetEgressDenied: true, + cloudNativePgEgressAllowed: true, + managerSecretReadDenied: true, + managerMutationRbacDenied: true, + }, + durability: { + sourceRunStatus: 'failed', + retryRunCount: 1, + retryAttemptCount: 1, + retryEventCount: 2, + stoppedRunCount: 1, + stopEventCount: 1, + allowedAuditCount: 2, + deniedAuditCount: 1, + duplicateMutationCount: 0, + identityGeneration: 3, + weakAuthenticationAuditCount: 0, + survivedCloudNativePgFailover: true, + }, + gates: { + realThreeNodeKubernetes: true, + realCniPolicy: true, + threeInstanceCloudNativePg: true, + twoManagerPodsOnDistinctNodes: true, + tls13ProductClientAcrossBothPods: true, + strongUserRetryAndStop: true, + identityProjectionRotation: true, + certificateRevocationRollout: true, + databaseReadinessFence: true, + durableFactsSurvivedFailover: true, + leastPrivilege: true, + passed: true, + }, + limitations: [...LIMITATIONS], + }; +} + +function mutate(change) { + const report = structuredClone(validReport()); + change(report); + return validateRunManagementKubernetesLiveReport(report); +} + +test('accepts the exact content-free Run management Kubernetes report', () => { + assert.deepEqual(validateRunManagementKubernetesLiveReport(validReport()), { + schemaVersion: 1, + fixture: FIXTURE, + findings: [], + compatible: true, + }); +}); + +test('rejects topology, schema and deployment weakening', () => { + for (const [code, change] of [ + [ + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_PLATFORM', + (report) => { + report.platform.workerNodes = 1; + }, + ], + [ + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DATABASE', + (report) => { + report.database.controlCoreCapability = 55; + }, + ], + [ + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DEPLOYMENT', + (report) => { + report.deployment.nodeIdentitySha256[1] = + report.deployment.nodeIdentitySha256[0]; + }, + ], + ]) { + assert.ok(mutate(change).findings.some((entry) => entry.code === code)); + } +}); + +test('rejects widened client, incomplete rotations and false availability', () => { + for (const [code, change] of [ + [ + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_CLIENT', + (report) => { + report.client.rbacGranted = true; + }, + ], + [ + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_IDENTITY_ROTATION', + (report) => { + report.identityRotation.revokedOldAssertionRejected = false; + }, + ], + [ + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_CERTIFICATE_ROTATION', + (report) => { + report.certificateRotation.fullPodReplacement = false; + }, + ], + [ + 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_AVAILABILITY', + (report) => { + report.availability.stalePodsDidNotRecoverInPlace = false; + }, + ], + ]) { + assert.ok(mutate(change).findings.some((entry) => entry.code === code)); + } +}); + +test('rejects incomplete isolation, durable drift and secret-shaped content', () => { + assert.ok( + mutate((report) => { + report.isolation.publicInternetEgressDenied = false; + }).findings.some( + (entry) => entry.code === 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_ISOLATION', + ), + ); + assert.ok( + mutate((report) => { + report.durability.retryRunCount = 2; + }).findings.some( + (entry) => entry.code === 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DURABILITY', + ), + ); + assert.ok( + mutate((report) => { + report.client.token = + 'eyJ0123456789012345.eyJ0123456789012345.abc0123456789012345'; + }).findings.some( + (entry) => + entry.code === 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_SECRET_EXPOSURE', + ), + ); +}); + +module.exports = { validReport }; diff --git a/test/back/ql3RunManagementKubernetesLiveContract.test.cjs b/test/back/ql3RunManagementKubernetesLiveContract.test.cjs new file mode 100644 index 00000000..fba62dc8 --- /dev/null +++ b/test/back/ql3RunManagementKubernetesLiveContract.test.cjs @@ -0,0 +1,99 @@ +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); +const { test } = require('node:test'); + +const { + identity, + retryCommand, + stopCommand, +} = require('../../scripts/ql3-run-management-kubernetes-live-contract.cjs'); + +test('Run management live report path is mandatory before mutation begins', () => { + const script = path.resolve( + __dirname, + '../../scripts/ql3-run-management-kubernetes-live-contract.cjs', + ); + const result = spawnSync(process.execPath, [script], { + encoding: 'utf8', + env: { ...process.env, QL3_RUN_MANAGEMENT_KUBERNETES_LIVE: '1' }, + }); + assert.equal(result.status, 1); + assert.match(result.stderr, /--report=\/absolute\/private-report\.json/); + assert.doesNotMatch(result.stderr, /Docker\/Kubernetes/); +}); + +test('Run live identity is audience, type, purpose and assurance bound', () => { + const key = identity.reviewedKey('run-live-test-key'); + const document = identity.keyset(1, [key]); + assert.equal(document.audience, 'qinglong3-run-management'); + const strong = identity.assertion(key, 'strong-unit-test'); + const [header, payload, signature] = strong.split('.'); + assert.ok(signature.length > 32); + assert.deepEqual(JSON.parse(Buffer.from(header, 'base64url')), { + alg: 'EdDSA', + kid: 'run-live-test-key', + typ: 'ql3-run-management+jwt', + }); + const claims = JSON.parse(Buffer.from(payload, 'base64url')); + assert.equal(claims.aud, 'qinglong3-run-management'); + assert.equal(claims.ql3_purpose, 'run-management'); + assert.equal(claims.sub, 'run-operator'); + assert.deepEqual(claims.amr, ['pwd', 'otp']); + const weak = JSON.parse( + Buffer.from( + identity.weakAssertion(key, 'weak-unit-test').split('.')[1], + 'base64url', + ), + ); + assert.equal(weak.acr, 'urn:ql3:password'); + assert.deepEqual(weak.amr, ['pwd']); +}); + +test('Run live commands bind exact mutation, source fence and distinct audits', () => { + const retry = retryCommand( + 'project-a', + 'source-a', + 'request-a', + '123e4567-e89b-42d3-a456-426614174000', + 1, + ); + assert.equal(retry.operation, 'run.retry'); + assert.equal(retry.request.body.expectedRunStatus, 'failed'); + assert.equal(retry.request.body.expectedRunVersion, 3); + assert.notEqual( + retry.request.auditEventId, + retry.request.failureAuditEventId, + ); + const stop = stopCommand( + 'project-a', + 'run-a', + 'request-b', + '123e4567-e89b-42d3-a456-426614174001', + 2, + ); + assert.equal(stop.operation, 'run.stop'); + assert.equal(stop.request.body.schema, 'qinglong/run-cancellation@v1'); + assert.notEqual(stop.request.auditEventId, stop.request.failureAuditEventId); +}); + +test('Run live runner remains opt-in, layered and observation backed', () => { + const source = fs.readFileSync( + path.resolve( + __dirname, + '../../scripts/ql3-run-management-kubernetes-live-contract.cjs', + ), + 'utf8', + ); + assert.match(source, /QL3_RUN_MANAGEMENT_KUBERNETES_LIVE !== '1'/); + assert.match(source, /reviewedOperatorManifest\(operatorManifestFile\)/); + assert.match(source, /validateRunManagementKubernetesLiveReport/); + assert.match(source, /createManagementClientExecutor/); + assert.match(source, /durableRunManagementFacts/); + assert.match(source, /Run identity ledger rollback surge failure/); + assert.match(source, /CloudNativePG primary promotion/); + assert.match(source, /migrationCount: 57/); + assert.match(source, /controlCoreCapability: 56/); + assert.doesNotMatch(source, /kubectl.*logs/); +});