feat(ql3): add cluster run management live gate

.github/workflows/ql3-run-management-live.yml

@@ -0,0 +1,124 @@
+name: QingLong 3.0 Run Management Kubernetes live evidence
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ql3-run-management-live-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  run-management-live:
+    name: Three-node Run retry, stop, OIDC, mTLS and CloudNativePG
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v6
+      - uses: pnpm/action-setup@v6
+        with:
+          version: '8.3.1'
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '24.18.0'
+          cache: pnpm
+          cache-dependency-path: pnpm-lock.yaml
+
+      - name: Require enough ephemeral disk for three isolated K3s nodes
+        shell: bash
+        run: |
+          set -euo pipefail
+          available_kib="$(df -Pk / | awk 'NR == 2 { print $4 }')"
+          minimum_kib="$((25 * 1024 * 1024))"
+          if (( available_kib < minimum_kib )); then
+            echo "Run management live gate requires at least 25 GiB free; found ${available_kib} KiB" >&2
+            exit 1
+          fi
+          docker volume ls --quiet --filter dangling=true | sort > \
+            "${RUNNER_TEMP}/ql3-dangling-volumes.before"
+          docker system df
+
+      - name: Install workspace dependencies without lifecycle scripts
+        run: pnpm install --frozen-lockfile --ignore-scripts
+
+      - name: Install verified kubectl v1.32.8
+        shell: bash
+        run: |
+          set -euo pipefail
+          curl --fail --location --silent --show-error \
+            --output "${RUNNER_TEMP}/kubectl" \
+            https://dl.k8s.io/release/v1.32.8/bin/linux/amd64/kubectl
+          curl --fail --location --silent --show-error \
+            --output "${RUNNER_TEMP}/kubectl.sha256" \
+            https://dl.k8s.io/release/v1.32.8/bin/linux/amd64/kubectl.sha256
+          test "$(cat "${RUNNER_TEMP}/kubectl.sha256")" = \
+            "$(sha256sum "${RUNNER_TEMP}/kubectl" | cut -d ' ' -f 1)"
+          chmod 0755 "${RUNNER_TEMP}/kubectl"
+
+      - name: Fetch checksum-locked CloudNativePG operator manifest
+        shell: bash
+        run: |
+          set -euo pipefail
+          curl --fail --location --silent --show-error \
+            --output "${RUNNER_TEMP}/cloudnative-pg.yaml" \
+            https://github.com/cloudnative-pg/cloudnative-pg/releases/download/v1.30.0/cnpg-1.30.0.yaml
+          echo "f8bede43fe4ee0d478c2355b204a36876b2ae4faac60f2a9452280b293da3b88  ${RUNNER_TEMP}/cloudnative-pg.yaml" | sha256sum --check
+
+      - name: Recheck static Run management and evidence contracts
+        run: |
+          pnpm audit:cluster-deployment:ql3
+          node --test \
+            test/back/ql3RunManagementDeployment.test.cjs \
+            test/back/ql3RunManagementKubernetesLiveContract.test.cjs \
+            test/back/ql3RunManagementKubernetesLiveAudit.test.cjs
+          pnpm --filter @qinglong/cluster-admin build
+          pnpm --filter @qinglong/cluster-control build
+
+      - name: Preload the digest-bound K3s distribution
+        run: docker pull rancher/k3s:v1.34.3-k3s1
+
+      - name: Prove Run retry and stop over three nodes, mTLS, OIDC and CloudNativePG
+        env:
+          QL3_RUN_MANAGEMENT_KUBERNETES_LIVE: '1'
+          QL3_KUBECTL_BIN: ${{ runner.temp }}/kubectl
+          QL3_CNPG_OPERATOR_MANIFEST_FILE: ${{ runner.temp }}/cloudnative-pg.yaml
+          QL3_RUN_MANAGEMENT_REPORT: ${{ runner.temp }}/ql3-run-management/report.json
+        shell: bash
+        run: |
+          set -euo pipefail
+          umask 077
+          mkdir -m 0700 "$(dirname "${QL3_RUN_MANAGEMENT_REPORT}")"
+          node scripts/ql3-run-management-kubernetes-live-contract.cjs \
+            "--report=${QL3_RUN_MANAGEMENT_REPORT}"
+
+      - name: Re-audit private report and isolated cleanup
+        env:
+          QL3_RUN_MANAGEMENT_REPORT: ${{ runner.temp }}/ql3-run-management/report.json
+        shell: bash
+        run: |
+          set -euo pipefail
+          test "$(stat -c '%a' "${QL3_RUN_MANAGEMENT_REPORT}")" = '600'
+          pnpm audit:run-management-kubernetes-live:ql3 \
+            "--report=${QL3_RUN_MANAGEMENT_REPORT}"
+          sha256sum "${QL3_RUN_MANAGEMENT_REPORT}"
+          test -z "$(docker ps -aq --filter name=ql3-run-live-)"
+          test -z "$(docker network ls -q --filter name=ql3-run-live-)"
+          docker volume ls --quiet --filter dangling=true | sort > \
+            "${RUNNER_TEMP}/ql3-dangling-volumes.after"
+          diff --unified \
+            "${RUNNER_TEMP}/ql3-dangling-volumes.before" \
+            "${RUNNER_TEMP}/ql3-dangling-volumes.after"
+          docker system df
+
+      - name: Upload audited content-free Run management evidence
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: ql3-run-management-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/ql3-run-management/report.json
+          if-no-files-found: error
+          retention-days: 14
+          compression-level: 9
+          overwrite: false
+          include-hidden-files: false

docs/QINGLONG_3_0_ARCHITECTURE_RFC.md

@@ -11,6 +11,19 @@
 
 最新增量证据（2026-08-12）：
 
+- D-298/ADR-0386（实现完成；固定本机 live 证据待锁定输入可达后补录）
+  Cluster `run.retry | run.stop` 已建立人工触发的真实三节点 Kubernetes 组合门：1 control-plane + 2 worker K3s/Flannel、3 实例
+  CloudNativePG 1.30.0/PostgreSQL 18.4、2 个跨节点 Run manager Pod、TLS 1.3 mTLS、purpose-bound OIDC strong User、identity
+  generation overlap/revoke/rollback、client CRL rotation、CNPG primary promotion、数据库断连 readiness/liveness fence、CNI 与 RBAC
+  least privilege，并以最终 PostgreSQL facts 验证 retry/stop 首写和 exact replay 无重复。报告 schema 固定 migration 57/control-core
+  capability 56，只允许 `0600` content-free evidence，敏感材料与隐藏 limitation 均失败。实现没有新增 package、生产依赖、migration、表、
+  角色或默认 workload；live 代码按通用管理 Kubernetes helper、Run scenario、离线 audit 分层，现有 Approval 回归通过。当前静态/负向合同
+  20/20；本机首次 live 取锁定 CNPG manifest 时 GitHub 443 连接 75 秒无数据超时且无缓存，因此没有降低 checksum 或把静态结果冒充 live
+  证据。完整 18-package build/test 退出 0，backend 1,174 pass/2 conditional skip/0 fail，14 个 Profile artifact 全部 compatible；基础
+  Edge 为 2,467,343 bytes/295 files/53 modules，RSS delta 10,928,128 bytes，低于既有低配门限。package/dependency/local-image boundary
+  全绿，仍为 18 package、无 single-source/shallow package。PostgreSQL 18.4 arm64 HA 通过 123 gates、timeline `1→2`，报告 SHA-256
+  `6adb8c9de8929ff54b522e9a251e3081d9dd004c1a91f72f83c33288ddce63a9` 且 Docker 零残留。固定 K3s/CNPG live report 是 ADR 从
+  Proposed 转为 Accepted 的唯一剩余证据。
 - D-297/ADR-0385（已接受）
   Local Edge/Standalone 已补齐强认证 `run.stop` 产品入口，并与既有 `run.retry` 统一为同一个 caller-driven `ql3-run retry|stop`
   binary，不新增 package、migration、表、索引、进程、listener、timer、watcher、连接、cache 或 sidecar。stop 只接受 POSIX 私有命令文件

docs/adr/ADR-0386-cluster-run-management-kubernetes-live-evidence.md

@@ -0,0 +1,51 @@
+# ADR-0386：Cluster Run Management Kubernetes 多节点实证
+
+- 状态：Proposed（实现与静态回归完成；固定本机 live 证据待锁定输入可达后补录）
+- 日期：2026-08-12
+- 关联 RFC：QL-RFC-0001 D-298
+- 前置决策：ADR-0359、ADR-0364、ADR-0366、ADR-0382、ADR-0383、ADR-0384
+
+## 上下文
+
+Cluster 已有强认证 `run.retry | run.stop`、专用 `ql3_run_manager`、两副本 Kubernetes overlay 和 PostgreSQL HA Docker 证据，但这些证据没有在同一真实 CNI/Pod/CloudNativePG 环境中组合。静态 YAML、进程内测试和单机 PostgreSQL promotion 不能证明跨节点 Pod、mTLS/OIDC rotation、NetworkPolicy、readiness withdrawal 与数据库主切换同时成立。
+
+部署用户跨度很大：Edge 路由设备不能为 Cluster 门增加任何常驻成本；Cluster 节点又需要真实多副本证据。因此本门必须是 `workflow_dispatch` 的短生命周期发布门，不能进入默认 CI、默认 overlay 或 Local artifact。
+
+## 决策
+
+1. 新增 opt-in 三节点 K3s live gate：1 control-plane + 2 worker、内置 Flannel、3 实例 CloudNativePG 1.30.0/PostgreSQL 18.4、2 个 Run manager Pod 且位于不同节点。
+2. 直接构建当前 Cluster Admin/Control 镜像，应用既有 `operations/run-management/cloudnative-pg`；不新增 package、生产依赖、migration、表、角色、常驻进程或 sidecar。
+3. 产品 client 以 tokenless、无 RBAC、immutable Secret 输入的 caller-driven Job 精确连接每个 Pod。`run.retry` 必须得到 `accepted → existing`，随后 `run.stop` 必须得到 `accepted → already_requested`；最终数据库只能有 1 个 retry Run、1 个 Attempt、2 个 retry Event、1 个 cancellation Event、2 个 allowed audit，重复 mutation 为 0。
+4. 弱认证在业务层前返回 401 且不写项目 audit；强但无 RoleBinding 的 User 返回 403 并写恰好 1 个 denied audit。报告不保存 assertion、证书、密钥、DSN、kubeconfig 或业务内容。
+5. OIDC keyset 执行 generation 1→2 overlap→3 revoke，回滚 generation 必须启动失败且始终保留 2 Ready Pod；mTLS CRL 更新必须替换全部 Pod、旧 client 被拒绝、新 client 可用，rollout 不低于 2 Ready。
+6. 删除当前 CNPG primary 并确认 promotion 与恢复 3 实例；切断 `-rw` Service endpoint 时两个 Pod 都返回 503、readiness 为 503、liveness 为 200，恢复 endpoint 后旧 Pod不得原地恢复，必须由新 Pod 服务。
+7. CNI 必须观测 labelled client allowed、unlabelled/wrong-port denied、manager→CNPG allowed、manager→Kubernetes API/public internet denied；ServiceAccount 无 Secret read 和 Deployment patch 权限。
+8. live 代码分三层：通用 Kubernetes 管理 helper、Run 领域 scenario/SQL facts、严格离线 report audit。通用 executor 的 retry error code 改为显式配置且保留旧默认值，Approval live 回归必须通过。禁止复制身份 assertion、健康探针和平台原语；新增能力不得形成 workspace 微包。
+
+## 证据合同
+
+离线报告固定为 `qinglong/run-management-kubernetes-live-contract@v1`，要求：
+
+- K3s `v1.32+`、锁定 `rancher/k3s:v1.34.3-k3s1`、3 个独立 PodCIDR；
+- migration count 57、control-core capability 56、PostgreSQL version number 180004；
+- 两个唯一 Pod/Node digest、PDB minAvailable 1、maxUnavailable 0、每 Pod Pool max 2；
+- retry/stop exact replay、认证负向门、identity/certificate rotation、availability、isolation、durability 全为精确字段；
+- `0600` regular file、no overwrite、无敏感材料；三条非生产 limitation 不得隐藏。
+
+## Package 与低配影响
+
+本切片不新增 workspace package。当前 package boundary 仍要求每个 `src/` 根文件都是 manifest 证明的入口；实现位于领域目录，高密度目录受 hard cap。文件少不是拆包理由，部署/authority/稳定多消费者才是 package 边界。该 live gate 只在人工发布作业中创建 Docker/K3s/CNPG 资源，完成后全部删除；Edge/Standalone 的文件、module、RSS、timer、listener 与连接预算不变。
+
+## 验收状态
+
+- Run/Approval live 静态合同、离线审计与既有 deployment 回归：20/20；Run report 审计含 topology/schema/deployment/client/rotation/availability/isolation/durability/secret 反向 fixture。
+- runner、scenario、platform、identity、PKI 与 audit 均通过 Node syntax check；现有 Run deployment 合同通过。
+- 本机真实门尝试下载校验和锁定的 CNPG manifest 时，`github.com:443` 在 75 秒无数据后超时；本机无缓存。未降低 checksum、未改用未审计输入，也未把静态结果记录为 live 通过。
+- 完整 18-package build/test 退出 0；backend 1,174 pass/2 conditional skip/0 fail；14 个 Edge/Standalone Profile artifact 全部 compatible。基础 Edge 为 2,467,343 bytes/295 files/53 loaded modules，RSS delta 10,928,128 bytes，仍低于 4 MiB/512 files/16 MiB 门限。
+- package/dependency/local-image boundary 全部 compatible，18 个 workspace package 无 single-source/shallow package；本切片没有改变任一 Local artifact 的生产闭包。
+- PostgreSQL 18.4 arm64 HA 通过 123 gates、primary timeline `1→2`，报告 SHA-256 `6adb8c9de8929ff54b522e9a251e3081d9dd004c1a91f72f83c33288ddce63a9`；运行后 `ql3-ha-*` container、volume、network 均为空。
+- 固定 K3s/CloudNativePG live report 仍是 ADR 从 Proposed 转为 Accepted 的唯一未满足证据。
+
+## 后果
+
+Cluster 有了可重复执行的真实组合门，但 Docker-host K3s 仍不是生产 control-plane HA 或 STONITH 证明，local deterministic IdP 也不替代外部 IdP 集成。工作流只提供 release candidate evidence，不改变默认部署。后续可把同构的 Approval/Automation runner 继续迁移到共享平台层，但不得为去除文本重复而一次性重写已通过的独立 live 流程。

docs/adr/README.md

@@ -389,6 +389,7 @@
 | [ADR-0383](./ADR-0383-strong-cluster-run-management-plane.md)                                                 | 强认证的 Cluster Run Management Plane 与专用数据库角色                                                                           | Accepted                                |
 | [ADR-0384](./ADR-0384-strong-cluster-run-stop-management.md)                                                  | 强认证的 Cluster Run Stop Management 与列级数据库权限                                                                            | Accepted                                |
 | [ADR-0385](./ADR-0385-strong-local-run-stop-product-entry.md)                                                 | 强认证的 Local Run Stop 产品入口与原子审计                                                                                        | Accepted                                |
+| [ADR-0386](./ADR-0386-cluster-run-management-kubernetes-live-evidence.md)                                     | Cluster Run Management Kubernetes 多节点实证                                                                                       | Proposed（实现完成，固定 live 证据待补录） |
 
 ## 规则
 

package.json

@@ -35,6 +35,8 @@
     "audit:automation-management-kubernetes-live:ql3": "node scripts/ql3-automation-management-kubernetes-live-audit.cjs",
     "test:approval-management-kubernetes-live:ql3": "pnpm --filter @qinglong/runtime-core build && node scripts/ql3-approval-management-kubernetes-live-contract.cjs",
     "audit:approval-management-kubernetes-live:ql3": "node scripts/ql3-approval-management-kubernetes-live-audit.cjs",
+    "test:run-management-kubernetes-live:ql3": "pnpm --filter @qinglong/cluster-admin build && pnpm --filter @qinglong/cluster-control build && node scripts/ql3-run-management-kubernetes-live-contract.cjs",
+    "audit:run-management-kubernetes-live:ql3": "node scripts/ql3-run-management-kubernetes-live-audit.cjs",
     "test:provider-credential-test-kubernetes-live:ql3": "pnpm --filter @qinglong/cluster-admin build && node scripts/ql3-provider-credential-test-kubernetes-live-contract.cjs",
     "audit:provider-credential-test-kubernetes-live:ql3": "node scripts/ql3-provider-credential-test-kubernetes-live-audit.cjs",
     "test:prompt-output-key-retirement-kubernetes-live:ql3": "pnpm --filter @qinglong/cluster-admin build && node scripts/ql3-prompt-output-key-retirement-kubernetes-live-contract.cjs",

scripts/lib/ql3-management-kubernetes-live-platform.cjs

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const assert = require('node:assert/strict');
+
+function imageIdDigest(image) {
+  assert.match(image.Id, /^sha256:[a-f0-9]{64}$/);
+  return image.Id;
+}
+
+function localManifest(rendered, imageName, localImage) {
+  const occurrences = rendered.split(imageName).length - 1;
+  assert.ok(occurrences >= 1, 'reviewed image reference is missing');
+  return rendered
+    .replaceAll(imageName, localImage)
+    .replaceAll('imagePullPolicy: IfNotPresent', 'imagePullPolicy: Never');
+}
+
+function applySecret(fixture, name, type, stringData) {
+  fixture.apply({
+    apiVersion: 'v1',
+    kind: 'Secret',
+    metadata: {
+      name,
+      namespace: 'qinglong3-system',
+      labels: { 'cnpg.io/reload': 'true' },
+    },
+    immutable: false,
+    type,
+    stringData,
+  });
+}
+
+function psql(fixture, podName, sql) {
+  return fixture.kubectl(
+    [
+      '-n',
+      'qinglong3-system',
+      'exec',
+      podName,
+      '-c',
+      'postgres',
+      '--',
+      'psql',
+      '--username=postgres',
+      '--dbname=qinglong',
+      '--no-psqlrc',
+      '--tuples-only',
+      '--no-align',
+      '--set=ON_ERROR_STOP=1',
+      '--command',
+      sql,
+    ],
+    { capture: true, quiet: true },
+  ).stdout;
+}
+
+function currentPrimaryPod(fixture) {
+  const primaryName = fixture.kubectlJson([
+    '-n',
+    'qinglong3-system',
+    'get',
+    'cluster',
+    'ql3-postgres',
+  ]).status.currentPrimary;
+  assert.match(primaryName || '', /^ql3-postgres-[1-9][0-9]*$/);
+  const pods = fixture.kubectlJson([
+    '-n',
+    'qinglong3-system',
+    'get',
+    'pods',
+    '-l',
+    'cnpg.io/cluster=ql3-postgres',
+  ]).items;
+  const primary = pods.find((pod) => pod.metadata.name === primaryName);
+  assert.ok(primary, 'CloudNativePG primary Pod not found');
+  return primary;
+}
+
+module.exports = {
+  applySecret,
+  currentPrimaryPod,
+  imageIdDigest,
+  localManifest,
+  psql,
+};

scripts/lib/ql3-management-kubernetes-live.cjs

@@ -100,9 +100,7 @@ async function waitForTwoPreserved(options) {
         '-l',
         'app.kubernetes.io/name=' + options.deployment,
       ])
-      .items.filter(
+      .items.filter((pod) => pod.metadata.deletionTimestamp === undefined);
-        (pod) => pod.metadata.deletionTimestamp === undefined,
-      );
     const ready = pods.filter(podReady);
     minimumReady = Math.min(minimumReady, ready.length);
     const replacements = ready.filter(
@@ -117,10 +115,7 @@ async function waitForTwoPreserved(options) {
       : {
           ready: false,
           fact:
-            ready.length +
+            ready.length + ' ready, ' + replacements.length + ' replacements',
-            ' ready, ' +
-            replacements.length +
-            ' replacements',
         };
   });
   assert.ok(
@@ -131,6 +126,17 @@ async function waitForTwoPreserved(options) {
 }
 
 function createManagementClientExecutor(options) {
+  const retryableClientCodes = options.retryableClientCodes ?? [
+    'QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED',
+  ];
+  assert.ok(
+    Array.isArray(retryableClientCodes) &&
+      retryableClientCodes.length >= 1 &&
+      retryableClientCodes.every(
+        (code) => typeof code === 'string' && /^[A-Z0-9_]{1,128}$/.test(code),
+      ),
+    'management client retryable codes are invalid',
+  );
   options.fixture.apply({
     apiVersion: 'v1',
     kind: 'ServiceAccount',
@@ -243,9 +249,14 @@ function createManagementClientExecutor(options) {
                       '--command=/tmp/command.json ' +
                       '--assertion=/tmp/assertion.jwt 2>&1)"',
                     '  status=$?',
-                    '  if [ "$status" -eq 0 ] || { ! printf \'%s\' "$output" | ' +
+                    '  if [ "$status" -eq 0 ] || { ' +
-                      'grep -q QL3_PLUGIN_PACKAGE_MANAGEMENT_CLIENT_REQUEST_FAILED && ' +
+                      retryableClientCodes
-                      '! printf \'%s\' "$output" | grep -q \'"statusCode":503\'; } || ' +
+                        .map(
+                          (code) =>
+                            '! printf \'%s\' "$output" | grep -q ' + code,
+                        )
+                        .join(' && ') +
+                      ' && ! printf \'%s\' "$output" | grep -q \'"statusCode":503\'; } || ' +
                       '[ "$attempt" -ge 60 ]; then',
                     '    break',
                     '  fi',
@@ -373,7 +384,10 @@ function createManagementClientExecutor(options) {
       assert.equal(output.event, 'command_completed');
       assert.equal(output.result.operation, definition.command.operation);
       if (expected.resultStatus) {
-        assert.ok(expected.resultStatus.includes(output.result.status));
+        const status = expected.resultField
+          ? output.result[expected.resultField]?.status
+          : output.result.status;
+        assert.ok(expected.resultStatus.includes(status));
       }
     } else {
       assert.equal(
@@ -409,6 +423,36 @@ function createManagementClientExecutor(options) {
   };
 }
 
+function managementHealthStatus(options) {
+  const script = [
+    "const fs=require('node:fs');const https=require('node:https');",
+    "const request=https.request({host:'127.0.0.1',port:Number(process.argv[1]),path:process.argv[2],",
+    'servername:process.argv[3],ca:fs.readFileSync(process.argv[4]),',
+    "minVersion:'TLSv1.3',maxVersion:'TLSv1.3',rejectUnauthorized:true,agent:false},",
+    "(response)=>{response.resume();response.on('end',()=>process.stdout.write(String(response.statusCode)))});",
+    "request.on('error',(error)=>{process.stderr.write(error.message);process.exitCode=1});request.end();",
+  ].join('\n');
+  return Number(
+    options.fixture.kubectl(
+      [
+        '-n',
+        options.namespace,
+        'exec',
+        options.podName,
+        '--',
+        'node',
+        '-e',
+        script,
+        String(options.port),
+        options.route,
+        options.servername,
+        options.caFile,
+      ],
+      { capture: true, quiet: true },
+    ).stdout,
+  );
+}
+
 function podTcpProbe(options) {
   const script = [
     "const net=require('node:net');let finished=false;",
@@ -437,9 +481,7 @@ function podTcpProbe(options) {
 async function clientTcpProbe(options) {
   const labels = {
     'app.kubernetes.io/name': options.appName,
-    ...(options.labelled
+    ...(options.labelled ? { [options.networkPolicyLabel]: 'true' } : {}),
-      ? { [options.networkPolicyLabel]: 'true' }
-      : {}),
   };
   const script = [
     "const fs=require('node:fs');const net=require('node:net');let finished=false;let attempt=0;let socket;",
@@ -497,10 +539,7 @@ async function clientTcpProbe(options) {
       },
     },
   });
-  const observed = await waitFor(
+  const observed = await waitFor(options.name + ' completion', 180_000, () => {
-    options.name + ' completion',
-    180_000,
-    () => {
     const job = options.fixture.kubectlJson([
       '-n',
       options.namespace,
@@ -513,14 +552,12 @@ async function clientTcpProbe(options) {
         condition.type === 'Complete' && condition.status === 'True',
     );
     const failed = job.status.conditions?.some(
-        (condition) =>
+      (condition) => condition.type === 'Failed' && condition.status === 'True',
-          condition.type === 'Failed' && condition.status === 'True',
     );
     return complete || failed
       ? { ready: true, value: { complete, failed } }
       : { ready: false, fact: JSON.stringify(job.status ?? {}) };
-    },
+  });
-  );
   const probePod = (
     await waitFor(options.name + ' terminal pod', 30_000, () => {
       const pods = options.fixture.kubectlJson([
@@ -541,16 +578,8 @@ async function clientTcpProbe(options) {
   const terminated = probePod.status.containerStatuses[0].state.terminated;
   const observation =
     options.name + ': ' + (terminated.message ?? 'no-message');
-  assert.equal(
+  assert.equal(observed.value.complete, options.expectedConnected, observation);
-    observed.value.complete,
+  assert.equal(observed.value.failed, !options.expectedConnected, observation);
-    options.expectedConnected,
-    observation,
-  );
-  assert.equal(
-    observed.value.failed,
-    !options.expectedConnected,
-    observation,
-  );
   assert.equal(
     terminated.exitCode === 0,
     options.expectedConnected,
@@ -562,14 +591,7 @@ async function clientTcpProbe(options) {
     assert.match(terminated.message ?? '', /^denied:/);
   }
   options.fixture.kubectl(
-    [
+    ['-n', options.namespace, 'delete', 'job', options.name, '--wait=false'],
-      '-n',
-      options.namespace,
-      'delete',
-      'job',
-      options.name,
-      '--wait=false',
-    ],
     { capture: true, quiet: true },
   );
   return options.expectedConnected
@@ -580,6 +602,7 @@ async function clientTcpProbe(options) {
 module.exports = {
   clientTcpProbe,
   createManagementClientExecutor,
+  managementHealthStatus,
   patchManagementGeneration,
   podReady,
   podTcpProbe,

scripts/lib/ql3-management-live-identity.cjs

@@ -57,7 +57,8 @@ function createManagementIdentityCeremony(options) {
     });
   }
 
-  function assertion(key, suffix = crypto.randomUUID()) {
+  function assertionForSubject(key, subject, suffix = crypto.randomUUID()) {
+    assert.match(subject, /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/);
     const now = Math.floor(Date.now() / 1_000);
     const header = Buffer.from(
       JSON.stringify({
@@ -77,6 +78,43 @@ function createManagementIdentityCeremony(options) {
         iss: options.issuer,
         jti: options.jtiPrefix + '-' + suffix,
         ql3_purpose: options.purpose,
+        sub: subject,
+      }),
+    ).toString('base64url');
+    const signed = header + '.' + payload;
+    return (
+      signed +
+      '.' +
+      crypto
+        .sign(null, Buffer.from(signed, 'ascii'), key.privateKey)
+        .toString('base64url')
+    );
+  }
+
+  function assertion(key, suffix = crypto.randomUUID()) {
+    return assertionForSubject(key, options.subject, suffix);
+  }
+
+  function weakAssertion(key, suffix = crypto.randomUUID()) {
+    const now = Math.floor(Date.now() / 1_000);
+    const header = Buffer.from(
+      JSON.stringify({
+        alg: 'EdDSA',
+        kid: key.kid,
+        typ: options.tokenType,
+      }),
+    ).toString('base64url');
+    const payload = Buffer.from(
+      JSON.stringify({
+        acr: 'urn:ql3:password',
+        amr: ['pwd'],
+        aud: options.audience,
+        auth_time: now - 1,
+        exp: now + 290,
+        iat: now,
+        iss: options.issuer,
+        jti: options.jtiPrefix + '-weak-' + suffix,
+        ql3_purpose: options.purpose,
         sub: options.subject,
       }),
     ).toString('base64url');
@@ -90,7 +128,13 @@ function createManagementIdentityCeremony(options) {
     );
   }
 
-  return Object.freeze({ assertion, keyset, reviewedKey });
+  return Object.freeze({
+    assertion,
+    assertionForSubject,
+    keyset,
+    reviewedKey,
+    weakAssertion,
+  });
 }
 
 module.exports = { createManagementIdentityCeremony };

scripts/lib/ql3-run-management-kubernetes-live-scenario.cjs

@@ -0,0 +1,194 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const assert = require('node:assert/strict');
+const crypto = require('node:crypto');
+
+function eventId(ordinal) {
+  assert.ok(Number.isSafeInteger(ordinal) && ordinal >= 1 && ordinal < 1e12);
+  return '41000000-0000-4000-8000-' + String(ordinal).padStart(12, '0');
+}
+
+function retryCommand(projectId, sourceRunId, requestId, mutationId, ordinal) {
+  return Object.freeze({
+    schemaVersion: 1,
+    operation: 'run.retry',
+    request: Object.freeze({
+      projectId,
+      sourceRunId,
+      requestId,
+      auditEventId: eventId(ordinal),
+      failureAuditEventId: eventId(ordinal + 500_000),
+      body: Object.freeze({
+        schema: 'qinglong/run-manual-retry@v1',
+        mutationId,
+        expectedRunVersion: 3,
+        expectedRunStatus: 'failed',
+      }),
+    }),
+  });
+}
+
+function stopCommand(projectId, runId, requestId, mutationId, ordinal) {
+  return Object.freeze({
+    schemaVersion: 1,
+    operation: 'run.stop',
+    request: Object.freeze({
+      projectId,
+      runId,
+      requestId,
+      auditEventId: eventId(ordinal),
+      failureAuditEventId: eventId(ordinal + 500_000),
+      body: Object.freeze({
+        schema: 'qinglong/run-cancellation@v1',
+        mutationId,
+      }),
+    }),
+  });
+}
+
+function sqlString(value) {
+  assert.equal(typeof value, 'string');
+  return "'" + value.replaceAll("'", "''") + "'";
+}
+
+function seedRunManagement(fixture, podName, values, psql) {
+  const nowMs = Date.now();
+  const sourceDigest = 'a'.repeat(64);
+  const taskRevision = `qltd:v1:1:${sourceDigest}`;
+  const taskDigest = 'b'.repeat(64);
+  const planDigest = 'c'.repeat(64);
+  psql(
+    fixture,
+    podName,
+    [
+      'BEGIN;',
+      'INSERT INTO "ql3"."projects" (id, name, slug, status, version, created_at_ms, updated_at_ms)',
+      `VALUES (${sqlString(
+        values.projectId,
+      )}, 'Run Management Live', ${sqlString(
+        values.projectId,
+      )}, 'active', 1, ${nowMs}, ${nowMs});`,
+      'INSERT INTO "ql3"."project_role_bindings" (project_id, subject_type, subject_id, version, state, role, mutation_id, changed_by_type, changed_by_id, created_at_ms)',
+      `VALUES (${sqlString(values.projectId)}, 'user', ${sqlString(
+        values.operatorId,
+      )}, 1, 'active', 'operator', ${sqlString(
+        'binding-' + values.suffix,
+      )}, 'system', 'run-management-live', ${nowMs});`,
+      'INSERT INTO "ql3"."task_definitions" (project_id, task_id, current_revision, created_at_ms, updated_at_ms)',
+      `VALUES (${sqlString(values.projectId)}, ${sqlString(
+        values.taskId,
+      )}, 1, ${nowMs}, ${nowMs});`,
+      'INSERT INTO "ql3"."task_definition_revisions" (project_id, task_id, revision, mutation_id, name, kind, spec_json, labels_json, enabled, content_digest, created_at_ms)',
+      `VALUES (${sqlString(values.projectId)}, ${sqlString(
+        values.taskId,
+      )}, 1, ${sqlString(
+        crypto.randomUUID(),
+      )}::uuid, 'Run Management Live Task', 'command', '{"schema":"qinglong/command@v1","config":{"command":{"kind":"argv","file":"/bin/echo","args":["run-management-live"]}}}'::jsonb, '{}'::jsonb, true, ${sqlString(
+        taskDigest,
+      )}, ${nowMs});`,
+      'INSERT INTO "ql3"."task_execution_revisions" (project_id, task_id, source_revision, task_revision, source_content_digest, executor_type, plan_schema, plan_json, content_digest, created_at_ms)',
+      `VALUES (${sqlString(values.projectId)}, ${sqlString(
+        values.taskId,
+      )}, 1, ${sqlString(taskRevision)}, ${sqlString(
+        sourceDigest,
+      )}, 'remote_worker', 'qinglong/command-execution@v1', '{"file":"/bin/echo","args":["run-management-live"]}'::jsonb, ${sqlString(
+        planDigest,
+      )}, ${nowMs});`,
+      'INSERT INTO "ql3"."runs" (id, project_id, task_id, task_revision, task_name, task_snapshot_ref, trigger_type, execution_origin, execution_owner, status, version, event_sequence, priority, created_at_ms, queued_at_ms, finished_at_ms, error_code, error_summary)',
+      `VALUES (${sqlString(values.sourceRunId)}, ${sqlString(
+        values.projectId,
+      )}, ${sqlString(values.taskId)}, ${sqlString(
+        taskRevision,
+      )}, 'Run Management Live Task', ${sqlString(
+        taskRevision,
+      )}, 'manual', 'manual', 'runtime', 'failed', 3, 3, 0, ${nowMs}, ${nowMs}, ${nowMs}, 'LIVE_SOURCE_FAILURE', 'terminal source for Run management live');`,
+      'INSERT INTO "ql3"."run_attempts" (id, run_id, attempt, status, executor_type, callback_sequence, created_at_ms, finished_at_ms, error_code, error_summary)',
+      `VALUES (${sqlString(values.sourceAttemptId)}, ${sqlString(
+        values.sourceRunId,
+      )}, 1, 'failed', 'remote_worker', 0, ${nowMs}, ${nowMs}, 'LIVE_SOURCE_FAILURE', 'terminal source for Run management live');`,
+      'INSERT INTO "ql3"."run_events" (id, run_id, sequence, type, dedupe_key, actor_type, actor_id, attempt_id, payload, created_at_ms)',
+      `VALUES (${sqlString(crypto.randomUUID())}, ${sqlString(
+        values.sourceRunId,
+      )}, 1, 'run.created', ${sqlString(
+        'run-management-live-created-' + values.suffix,
+      )}, 'user', ${sqlString(values.operatorId)}, ${sqlString(
+        values.sourceAttemptId,
+      )}, '{"status":"created","version":1}'::jsonb, ${nowMs}),`,
+      `(${sqlString(crypto.randomUUID())}, ${sqlString(
+        values.sourceRunId,
+      )}, 2, 'run.queued', ${sqlString(
+        'run-management-live-queued-' + values.suffix,
+      )}, 'user', ${sqlString(values.operatorId)}, ${sqlString(
+        values.sourceAttemptId,
+      )}, '{"from_status":"created","to_status":"queued","version":2}'::jsonb, ${nowMs}),`,
+      `(${sqlString(crypto.randomUUID())}, ${sqlString(
+        values.sourceRunId,
+      )}, 3, 'run.failed', ${sqlString(
+        'run-management-live-failed-' + values.suffix,
+      )}, 'executor', 'run-management-live', ${sqlString(
+        values.sourceAttemptId,
+      )}, '{"from_status":"queued","to_status":"failed","version":3,"error_code":"LIVE_SOURCE_FAILURE"}'::jsonb, ${nowMs});`,
+      'COMMIT;',
+    ].join('\n'),
+  );
+  return Object.freeze({ taskRevision, taskDigest, planDigest });
+}
+
+function durableRunManagementFacts(fixture, podName, values, psql) {
+  return JSON.parse(
+    psql(
+      fixture,
+      podName,
+      [
+        'SELECT json_build_object(',
+        '  \'sourceRunStatus\', (SELECT status FROM "ql3"."runs" WHERE id = ' +
+          sqlString(values.sourceRunId) +
+          '),',
+        '  \'retryRunCount\', (SELECT count(*)::integer FROM "ql3"."runs" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          " AND trigger_type = 'run_manual_retry'),",
+        '  \'retryAttemptCount\', (SELECT count(*)::integer FROM "ql3"."run_attempts" AS attempt JOIN "ql3"."runs" AS run ON run.id = attempt.run_id WHERE run.project_id = ' +
+          sqlString(values.projectId) +
+          " AND run.trigger_type = 'run_manual_retry'),",
+        '  \'retryEventCount\', (SELECT count(*)::integer FROM "ql3"."run_events" AS event JOIN "ql3"."runs" AS run ON run.id = event.run_id WHERE run.project_id = ' +
+          sqlString(values.projectId) +
+          " AND run.trigger_type = 'run_manual_retry' AND event.type IN ('run.created', 'run.queued')),",
+        '  \'stoppedRunCount\', (SELECT count(*)::integer FROM "ql3"."runs" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          " AND trigger_type = 'run_manual_retry' AND cancel_requested_at_ms IS NOT NULL AND cancel_reason = 'user'),",
+        '  \'stopEventCount\', (SELECT count(*)::integer FROM "ql3"."run_events" AS event JOIN "ql3"."runs" AS run ON run.id = event.run_id WHERE run.project_id = ' +
+          sqlString(values.projectId) +
+          " AND event.type = 'run.cancel_requested'),",
+        '  \'allowedAuditCount\', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          " AND operation_id IN ('run.retry', 'run.stop') AND outcome = 'allowed'),",
+        '  \'deniedAuditCount\', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          " AND operation_id IN ('run.retry', 'run.stop') AND outcome = 'denied'),",
+        '  \'weakAuthenticationAuditCount\', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          " AND request_id = 'run-live-weak'),",
+        '  \'duplicateMutationCount\', greatest(0, (SELECT count(*)::integer FROM "ql3"."runs" WHERE project_id = ' +
+          sqlString(values.projectId) +
+          ' AND trigger_type = \'run_manual_retry\') - 1) + greatest(0, (SELECT count(*)::integer FROM "ql3"."run_events" AS event JOIN "ql3"."runs" AS run ON run.id = event.run_id WHERE run.project_id = ' +
+          sqlString(values.projectId) +
+          " AND event.type = 'run.cancel_requested') - 1),",
+        '  \'identityGeneration\', (SELECT generation::integer FROM "ql3"."plugin_package_identity_keyset_ledger" WHERE authority = \'run-management\'),',
+        '  \'migrationCount\', (SELECT count(*)::integer FROM "ql3"."schema_migrations"),',
+        '  \'controlCoreCapability\', (SELECT contract_version::integer FROM "ql3"."schema_capabilities" WHERE contract_name = \'control-core\'),',
+        "  'postgresVersionNumber', current_setting('server_version_num')::integer)",
+      ].join('\n'),
+    ),
+  );
+}
+
+module.exports = {
+  durableRunManagementFacts,
+  eventId,
+  retryCommand,
+  seedRunManagement,
+  sqlString,
+  stopCommand,
+};

scripts/ql3-approval-management-kubernetes-live-contract.cjs

@@ -16,6 +16,7 @@ const { createMutualTlsPki } = require('./lib/ql3-live-pki.cjs');
 const {
   clientTcpProbe,
   createManagementClientExecutor,
+  managementHealthStatus,
   patchManagementGeneration,
   podReady,
   podTcpProbe,
@@ -90,10 +91,7 @@ const identity = createManagementIdentityCeremony({
 });
 
 function sha256(value) {
-  return (
+  return 'sha256:' + crypto.createHash('sha256').update(value).digest('hex');
-    'sha256:' +
-    crypto.createHash('sha256').update(value).digest('hex')
-  );
 }
 
 function randomSecret() {
@@ -102,9 +100,7 @@ function randomSecret() {
 
 function eventId(ordinal) {
   assert.ok(Number.isSafeInteger(ordinal) && ordinal >= 1 && ordinal < 1e12);
-  return (
+  return '40000000-0000-4000-8000-' + String(ordinal).padStart(12, '0');
-    '40000000-0000-4000-8000-' + String(ordinal).padStart(12, '0')
-  );
 }
 
 function reviewedKey(kid) {
@@ -119,75 +115,12 @@ function assertion(key, suffix) {
   return identity.assertion(key, suffix);
 }
 
-function assertionForSubject(
+function assertionForSubject(key, subject, suffix = crypto.randomUUID()) {
-  key,
+  return identity.assertionForSubject(key, subject, 'subject-' + suffix);
-  subject,
-  suffix = crypto.randomUUID(),
-) {
-  assert.match(subject, /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/);
-  const now = Math.floor(Date.now() / 1_000);
-  const header = Buffer.from(
-    JSON.stringify({
-      alg: 'EdDSA',
-      kid: key.kid,
-      typ: 'ql3-approval-management+jwt',
-    }),
-  ).toString('base64url');
-  const payload = Buffer.from(
-    JSON.stringify({
-      acr: 'urn:ql3:mfa',
-      amr: ['pwd', 'otp'],
-      aud: AUDIENCE,
-      auth_time: now - 1,
-      exp: now + 290,
-      iat: now,
-      iss: ISSUER,
-      jti: 'ql3-approval-live-subject-' + suffix,
-      ql3_purpose: 'approval-management',
-      sub: subject,
-    }),
-  ).toString('base64url');
-  const signed = header + '.' + payload;
-  return (
-    signed +
-    '.' +
-    crypto
-      .sign(null, Buffer.from(signed, 'ascii'), key.privateKey)
-      .toString('base64url')
-  );
 }
 
 function weakAssertion(key, suffix = crypto.randomUUID()) {
-  const now = Math.floor(Date.now() / 1_000);
+  return identity.weakAssertion(key, suffix);
-  const header = Buffer.from(
-    JSON.stringify({
-      alg: 'EdDSA',
-      kid: key.kid,
-      typ: 'ql3-approval-management+jwt',
-    }),
-  ).toString('base64url');
-  const payload = Buffer.from(
-    JSON.stringify({
-      acr: 'urn:ql3:password',
-      amr: ['pwd'],
-      aud: AUDIENCE,
-      auth_time: now - 1,
-      exp: now + 290,
-      iat: now,
-      iss: ISSUER,
-      jti: 'ql3-approval-live-weak-' + suffix,
-      ql3_purpose: 'approval-management',
-      sub: 'approval-operator',
-    }),
-  ).toString('base64url');
-  const signed = header + '.' + payload;
-  return (
-    signed +
-    '.' +
-    crypto
-      .sign(null, Buffer.from(signed, 'ascii'), key.privateKey)
-      .toString('base64url')
-  );
 }
 
 function commandBase(projectId, approvalRequestId, requestId, ordinal) {
@@ -204,12 +137,7 @@ function inspectCommand(projectId, approvalRequestId, requestId, ordinal) {
   return Object.freeze({
     schemaVersion: 1,
     operation: 'approval.inspect',
-    request: commandBase(
+    request: commandBase(projectId, approvalRequestId, requestId, ordinal),
-      projectId,
-      approvalRequestId,
-      requestId,
-      ordinal,
-    ),
   });
 }
 
@@ -224,12 +152,7 @@ function decisionCommand(
     schemaVersion: 1,
     operation: 'approval.decide',
     request: Object.freeze({
-      ...commandBase(
+      ...commandBase(projectId, approvalRequestId, requestId, ordinal),
-        projectId,
-        approvalRequestId,
-        requestId,
-        ordinal,
-      ),
       expectedVersion: 1,
       expectedAction: ACTION,
       decisionId,
@@ -327,12 +250,7 @@ function loadApprovalContract() {
   return require(file);
 }
 
-function seedApproval(
+function seedApproval(fixture, primaryPod, projectId, approvalRequestId) {
-  fixture,
-  primaryPod,
-  projectId,
-  approvalRequestId,
-) {
   const { approvalRequestDigest, createApprovalRequest } =
     loadApprovalContract();
   const requestedAtMs = Date.now() - 1_000;
@@ -422,31 +340,15 @@ function patchGeneration(fixture, generation, annotations = {}) {
 }
 
 function healthStatus(fixture, pod, route) {
-  const script = [
+  return managementHealthStatus({
-    "const fs=require('node:fs');const https=require('node:https');",
+    fixture,
-    "const request=https.request({host:'127.0.0.1',port:8447,path:process.argv[1],",
+    namespace: NAMESPACE,
-    "servername:process.argv[2],ca:fs.readFileSync('/var/run/secrets/qinglong3/approval-management-tls/ca.crt'),",
+    podName: pod.metadata.name,
-    "minVersion:'TLSv1.3',maxVersion:'TLSv1.3',rejectUnauthorized:true,agent:false},",
+    port: 8447,
-    "(response)=>{response.resume();response.on('end',()=>process.stdout.write(String(response.statusCode)))});",
-    "request.on('error',(error)=>{process.stderr.write(error.message);process.exitCode=1});request.end();",
-  ].join('\n');
-  return Number(
-    fixture.kubectl(
-      [
-        '-n',
-        NAMESPACE,
-        'exec',
-        pod.metadata.name,
-        '--',
-        'node',
-        '-e',
-        script,
     route,
-        SERVERNAME,
+    servername: SERVERNAME,
-      ],
+    caFile: '/var/run/secrets/qinglong3/approval-management-tls/ca.crt',
-      { capture: true, quiet: true },
+  });
-    ).stdout,
-  );
 }
 
 function privateReportPath(argv) {
@@ -507,10 +409,7 @@ async function main(argv = process.argv.slice(2)) {
       );
       const preloadTag = imageTag(reviewedImage);
       run(fixture.docker, ['tag', reviewedImage, preloadTag]);
-      fixture.loadImage(
+      fixture.loadImage(preloadTag, path.basename(preloadTag) + '.tar');
-        preloadTag,
-        path.basename(preloadTag) + '.tar',
-      );
     }
 
     const sourceRevision = run('git', ['rev-parse', 'HEAD'], {
@@ -615,10 +514,7 @@ async function main(argv = process.argv.slice(2)) {
       '--timeout=20m',
     ]);
     const databasePods = (
-      await waitFor(
+      await waitFor('three ready CloudNativePG instances', 600_000, () => {
-        'three ready CloudNativePG instances',
-        600_000,
-        () => {
         const pods = fixture
           .kubectlJson([
             '-n',
@@ -635,8 +531,7 @@ async function main(argv = process.argv.slice(2)) {
               ready: false,
               fact: pods.length + '/3 ready database Pods',
             };
-        },
+      })
-      )
     ).value;
 
     const migrationManifest = localManifest(
@@ -685,12 +580,7 @@ async function main(argv = process.argv.slice(2)) {
     const approvalRequestId = 'approval-request-' + suffix;
     const decisionId = 'approval-decision-' + suffix;
     const primary = currentPrimaryPod(fixture);
-    seedApproval(
+    seedApproval(fixture, primary, projectId, approvalRequestId);
-      fixture,
-      primary,
-      projectId,
-      approvalRequestId,
-    );
 
     const pki = createMutualTlsPki({
       directory: fixture.temporary,
@@ -764,13 +654,8 @@ async function main(argv = process.argv.slice(2)) {
       1,
     );
     assert.equal(
-      fixture.kubectlJson([
+      fixture.kubectlJson(['-n', NAMESPACE, 'get', 'pdb', DEPLOYMENT]).spec
-        '-n',
+        .minAvailable,
-        NAMESPACE,
-        'get',
-        'pdb',
-        DEPLOYMENT,
-      ]).spec.minAvailable,
       1,
     );
     for (const pod of managerPods) {
@@ -876,9 +761,7 @@ async function main(argv = process.argv.slice(2)) {
       { statusCode: 403, responseCode: 'forbidden' },
     );
 
-    const generation1Uids = new Set(
+    const generation1Uids = new Set(managerPods.map((pod) => pod.metadata.uid));
-      managerPods.map((pod) => pod.metadata.uid),
-    );
     applyIdentity(keysets[1]);
     patchGeneration(fixture, 2);
     const generation2 = await waitForTwoPreserved({
@@ -925,9 +808,7 @@ async function main(argv = process.argv.slice(2)) {
     assert.equal(decided.output.result.approval.state, 'approved');
     assert.equal(decided.output.result.approval.version, 2);
 
-    const generation2Uids = new Set(
+    const generation2Uids = new Set(managerPods.map((pod) => pod.metadata.uid));
-      managerPods.map((pod) => pod.metadata.uid),
-    );
     applyIdentity(keysets[2]);
     patchGeneration(fixture, 3);
     const generation3 = await waitForTwoPreserved({
@@ -987,9 +868,7 @@ async function main(argv = process.argv.slice(2)) {
             '-l',
             'app.kubernetes.io/name=' + DEPLOYMENT,
           ])
-          .items.filter(
+          .items.filter((pod) => pod.metadata.deletionTimestamp === undefined);
-            (pod) => pod.metadata.deletionTimestamp === undefined,
-          );
         const ready = pods.filter(podReady);
         const candidate = pods.find(
           (pod) =>
@@ -1127,10 +1006,7 @@ async function main(argv = process.argv.slice(2)) {
             };
       },
     );
-    await waitFor(
+    await waitFor('CloudNativePG recovery to three instances', 900_000, () => {
-      'CloudNativePG recovery to three instances',
-      900_000,
-      () => {
       const status = fixture.kubectlJson([
         '-n',
         NAMESPACE,
@@ -1146,8 +1022,7 @@ async function main(argv = process.argv.slice(2)) {
               String(status.readyInstances ?? 0) +
               '/3 ready database instances',
           };
-      },
+    });
-    );
 
     const databaseService = fixture.kubectlJson([
       '-n',
@@ -1175,8 +1050,7 @@ async function main(argv = process.argv.slice(2)) {
       managerPods.map((pod, index) =>
         executeClient(
           {
-            name:
+            name: 'ql3-approval-database-unavailable-' + String(index + 1),
-              'ql3-approval-database-unavailable-' + String(index + 1),
             target: pod,
             command: decisionCommand(
               projectId,
@@ -1209,9 +1083,7 @@ async function main(argv = process.argv.slice(2)) {
         ? { ready: true, value: current }
         : {
             ready: false,
-            fact:
+            fact: String(current.status.readyReplicas ?? 0) + ' ready replicas',
-              String(current.status.readyReplicas ?? 0) +
-              ' ready replicas',
           };
     });
     assert.deepEqual(
@@ -1238,10 +1110,7 @@ async function main(argv = process.argv.slice(2)) {
         },
       ]),
     ]);
-    await waitFor(
+    await waitFor('restored CloudNativePG service endpoint', 120_000, () => {
-      'restored CloudNativePG service endpoint',
-      120_000,
-      () => {
       const endpoints = fixture.kubectlJson([
         '-n',
         NAMESPACE,
@@ -1258,15 +1127,12 @@ async function main(argv = process.argv.slice(2)) {
             ready: false,
             fact: String(count ?? 0) + ' service endpoints',
           };
-      },
+    });
-    );
     assert.deepEqual(
       managerPods.map((pod) => healthStatus(fixture, pod, '/readyz')),
       [503, 503],
     );
-    const staleUids = new Set(
+    const staleUids = new Set(managerPods.map((pod) => pod.metadata.uid));
-      managerPods.map((pod) => pod.metadata.uid),
-    );
     patchGeneration(fixture, '3-database-recovered');
     managerPods = await readyManagementPods({
       ...managerOptions(fixture),
@@ -1277,8 +1143,7 @@ async function main(argv = process.argv.slice(2)) {
       managerPods.map((pod, index) =>
         executeClient(
           {
-            name:
+            name: 'ql3-approval-database-recovered-' + String(index + 1),
-              'ql3-approval-database-recovered-' + String(index + 1),
             target: pod,
             command: decisionCommand(
               projectId,
@@ -1402,10 +1267,7 @@ async function main(argv = process.argv.slice(2)) {
           resource,
           '-n',
           NAMESPACE,
-          '--as=system:serviceaccount:' +
+          '--as=system:serviceaccount:' + NAMESPACE + ':' + DEPLOYMENT,
-            NAMESPACE +
-            ':' +
-            DEPLOYMENT,
         ],
         { capture: true, quiet: true, allowFailure: true },
       );
@@ -1576,12 +1438,8 @@ async function main(argv = process.argv.slice(2)) {
         port: 8447,
         replicas: deployment.spec.replicas,
         readyReplicas: managerPods.length,
-        podIdentitySha256: managerPods.map((pod) =>
+        podIdentitySha256: managerPods.map((pod) => sha256(pod.metadata.uid)),
-          sha256(pod.metadata.uid),
+        nodeIdentitySha256: managerPods.map((pod) => sha256(pod.spec.nodeName)),
-        ),
-        nodeIdentitySha256: managerPods.map((pod) =>
-          sha256(pod.spec.nodeName),
-        ),
         serviceAccount: DEPLOYMENT,
         automountServiceAccountToken: false,
         requiredPodAntiAffinity: true,
@@ -1620,8 +1478,7 @@ async function main(argv = process.argv.slice(2)) {
         activeNewAssertionAccepted: replayed.statusCode === 200,
         rollbackSurgeFailedClosed: Boolean(rollback.value),
         twoReadyReplicasPreserved:
-          generation2.minimumReady >= 2 &&
+          generation2.minimumReady >= 2 && generation3.minimumReady >= 2,
-          generation3.minimumReady >= 2,
         durableGenerationReachedThree: durable.identityGeneration === 3,
       },
       certificateRotation: {
@@ -1630,16 +1487,13 @@ async function main(argv = process.argv.slice(2)) {
         previousBundleSha256,
         currentBundleSha256,
         oldClientAcceptedBefore: initialRequests[0].statusCode === 200,
-        replacementClientAcceptedBefore:
+        replacementClientAcceptedBefore: initialRequests[1].statusCode === 200,
-          initialRequests[1].statusCode === 200,
         oldClientRejectedAfter: revokedCertificate.statusCode === 401,
-        replacementClientAcceptedAfter:
+        replacementClientAcceptedAfter: activeCertificate.statusCode === 200,
-          activeCertificate.statusCode === 200,
         fullPodReplacement: managerPods.every(
           (pod) => !preCertificateUids.has(pod.metadata.uid),
         ),
-        allReplicasReadyThroughout:
+        allReplicasReadyThroughout: certificateRollout.minimumReady >= 2,
-          certificateRollout.minimumReady >= 2,
       },
       availability: {
         databaseFailureWithdrewReadiness: true,
@@ -1660,8 +1514,7 @@ async function main(argv = process.argv.slice(2)) {
         publicInternetEgressDenied,
         cloudNativePgEgressAllowed,
         managerSecretReadDenied: canI('get', 'secrets') === 'no',
-        managerMutationRbacDenied:
+        managerMutationRbacDenied: canI('patch', 'deployments.apps') === 'no',
-          canI('patch', 'deployments.apps') === 'no',
       },
       durability: {
         approvalVersion: durable.approvalVersion,
@@ -1686,15 +1539,13 @@ async function main(argv = process.argv.slice(2)) {
         twoManagerPodsOnDistinctNodes:
           new Set(managerPods.map((pod) => pod.spec.nodeName)).size === 2,
         tls13ProductClientAcrossBothPods:
-          new Set(baselineSuccesses.map((entry) => entry.targetPod)).size >=
+          new Set(baselineSuccesses.map((entry) => entry.targetPod)).size >= 2,
-          2,
         strongUserDecision:
           weakUserRejected.statusCode === 401 &&
           outsiderDenied.statusCode === 403 &&
           decided.statusCode === 200,
         identityProjectionRotation: durable.identityGeneration === 3,
-        certificateRevocationRollout:
+        certificateRevocationRollout: revokedCertificate.statusCode === 401,
-          revokedCertificate.statusCode === 401,
         databaseReadinessFence: true,
         durableFactsSurvivedFailover: true,
         leastPrivilege: rolesLeastPrivilege,
@@ -1739,7 +1590,9 @@ if (require.main === module) {
   main().catch((error) => {
     process.stderr.write(
       'QL3 approval management Kubernetes live contract failed: ' +
-        (error instanceof Error ? error.stack || error.message : String(error)) +
+        (error instanceof Error
+          ? error.stack || error.message
+          : String(error)) +
         '\n',
     );
     process.exitCode = 1;

scripts/ql3-run-management-kubernetes-live-audit.cjs

@@ -0,0 +1,488 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+const FIXTURE = 'qinglong/run-management-kubernetes-live-contract@v1';
+const LIMITATIONS = Object.freeze([
+  'three privileged K3s Docker nodes are not production infrastructure or control-plane HA evidence',
+  'identity assertions use a deterministic local strong-User ceremony rather than an external IdP',
+  'CloudNativePG failover inside one Docker host is not infrastructure STONITH evidence',
+]);
+const BANNED_KEYS = new Set([
+  'assertion',
+  'authorization',
+  'bearer',
+  'certificate',
+  'clientkey',
+  'connectionstring',
+  'dsn',
+  'kubeconfig',
+  'password',
+  'privatekey',
+  'secret',
+  'tlskey',
+  'token',
+]);
+
+function exactKeys(value, expected) {
+  return (
+    value !== null &&
+    typeof value === 'object' &&
+    !Array.isArray(value) &&
+    JSON.stringify(Object.keys(value).sort()) ===
+      JSON.stringify([...expected].sort())
+  );
+}
+
+function isDigest(value) {
+  return typeof value === 'string' && /^sha256:[a-f0-9]{64}$/.test(value);
+}
+
+function isIsoTime(value) {
+  return (
+    typeof value === 'string' &&
+    /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/.test(value) &&
+    Number.isFinite(Date.parse(value))
+  );
+}
+
+function isToken(value, maximum = 128) {
+  return (
+    typeof value === 'string' &&
+    value.length >= 1 &&
+    value.length <= maximum &&
+    /^[A-Za-z0-9][A-Za-z0-9._:/@+-]*$/.test(value)
+  );
+}
+
+function containsSensitiveMaterial(value, key = '') {
+  if (BANNED_KEYS.has(key.toLowerCase())) return true;
+  if (typeof value === 'string') {
+    return (
+      /-----BEGIN (?:CERTIFICATE|(?:RSA |EC |OPENSSH )?PRIVATE KEY)-----/.test(
+        value,
+      ) ||
+      /postgres(?:ql)?:\/\/[^/\s]+:[^@\s]+@/i.test(value) ||
+      /\beyJ[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}\b/.test(
+        value,
+      )
+    );
+  }
+  if (Array.isArray(value)) {
+    return value.some((entry) => containsSensitiveMaterial(entry));
+  }
+  if (value && typeof value === 'object') {
+    return Object.entries(value).some(([childKey, child]) =>
+      containsSensitiveMaterial(child, childKey),
+    );
+  }
+  return false;
+}
+
+function validKubernetesVersion(value) {
+  const match =
+    typeof value === 'string'
+      ? /^v1\.([0-9]{2,3})\.([0-9]+)(?:[-+][0-9A-Za-z](?:[0-9A-Za-z.-]{0,62}[0-9A-Za-z])?)?$/.exec(
+          value,
+        )
+      : null;
+  return Boolean(match && Number(match[1]) >= 32);
+}
+
+function uniqueDigests(value, count) {
+  return (
+    Array.isArray(value) &&
+    value.length === count &&
+    value.every(isDigest) &&
+    new Set(value).size === count
+  );
+}
+
+function allTrue(value, expected) {
+  return (
+    exactKeys(value, expected) && expected.every((key) => value[key] === true)
+  );
+}
+
+function validateRunManagementKubernetesLiveReport(report) {
+  const findings = [];
+  const reject = (code, detail) =>
+    findings.push(Object.freeze({ code, detail }));
+  if (
+    !exactKeys(report, [
+      'schemaVersion',
+      'fixture',
+      'observedAt',
+      'platform',
+      'database',
+      'deployment',
+      'client',
+      'identityRotation',
+      'certificateRotation',
+      'availability',
+      'isolation',
+      'durability',
+      'gates',
+      'limitations',
+    ]) ||
+    report?.schemaVersion !== 1 ||
+    report?.fixture !== FIXTURE ||
+    !isIsoTime(report?.observedAt)
+  ) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_REPORT_SHAPE',
+      'the report must use the exact versioned Run management live envelope',
+    );
+  }
+  if (containsSensitiveMaterial(report)) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_SECRET_EXPOSURE',
+      'the report must not contain credentials, assertions, certificates, DSNs, kubeconfig or private keys',
+    );
+  }
+
+  const platform = report?.platform;
+  if (
+    !exactKeys(platform, [
+      'distribution',
+      'kubernetesVersion',
+      'architecture',
+      'kubernetesImageId',
+      'managementImageId',
+      'cniName',
+      'cniDistributionBinding',
+      'controlPlaneNodes',
+      'workerNodes',
+      'cniReadyNodes',
+    ]) ||
+    platform?.distribution !== 'k3s' ||
+    !validKubernetesVersion(platform?.kubernetesVersion) ||
+    !['amd64', 'arm64'].includes(platform?.architecture) ||
+    !isDigest(platform?.kubernetesImageId) ||
+    !isDigest(platform?.managementImageId) ||
+    platform?.cniName !== 'flannel' ||
+    platform?.cniDistributionBinding !== 'rancher/k3s:v1.34.3-k3s1' ||
+    platform?.controlPlaneNodes !== 1 ||
+    platform?.workerNodes !== 2 ||
+    platform?.cniReadyNodes !== 3
+  ) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_PLATFORM',
+      'the fixture must bind three real K3s nodes, embedded Flannel and exact images',
+    );
+  }
+
+  const database = report?.database;
+  if (
+    !exactKeys(database, [
+      'operator',
+      'operatorVersion',
+      'postgresVersionNumber',
+      'postgresImageId',
+      'instances',
+      'readyInstances',
+      'managerRole',
+      'migrationCount',
+      'controlCoreCapability',
+      'tlsVerified',
+      'primaryChangedDuringFailover',
+    ]) ||
+    database?.operator !== 'cloudnative-pg' ||
+    !isToken(database?.operatorVersion, 64) ||
+    database?.postgresVersionNumber !== 180004 ||
+    !isDigest(database?.postgresImageId) ||
+    database?.instances !== 3 ||
+    database?.readyInstances !== 3 ||
+    database?.managerRole !== 'ql3_run_manager' ||
+    database?.migrationCount !== 57 ||
+    database?.controlCoreCapability !== 56 ||
+    database?.tlsVerified !== true ||
+    database?.primaryChangedDuringFailover !== true
+  ) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DATABASE',
+      'three TLS CloudNativePG instances must run migration 57, capability 56 and the isolated Run manager role',
+    );
+  }
+
+  const deployment = report?.deployment;
+  if (
+    !exactKeys(deployment, [
+      'namespace',
+      'service',
+      'port',
+      'replicas',
+      'readyReplicas',
+      'podIdentitySha256',
+      'nodeIdentitySha256',
+      'serviceAccount',
+      'automountServiceAccountToken',
+      'requiredPodAntiAffinity',
+      'podDisruptionBudgetMinAvailable',
+      'maxUnavailable',
+      'maxConnectionsPerPod',
+    ]) ||
+    deployment?.namespace !== 'qinglong3-system' ||
+    deployment?.service !== 'ql3-run-management' ||
+    deployment?.port !== 8448 ||
+    deployment?.replicas !== 2 ||
+    deployment?.readyReplicas !== 2 ||
+    !uniqueDigests(deployment?.podIdentitySha256, 2) ||
+    !uniqueDigests(deployment?.nodeIdentitySha256, 2) ||
+    deployment?.serviceAccount !== 'ql3-run-management' ||
+    deployment?.automountServiceAccountToken !== false ||
+    deployment?.requiredPodAntiAffinity !== true ||
+    deployment?.podDisruptionBudgetMinAvailable !== 1 ||
+    deployment?.maxUnavailable !== 0 ||
+    deployment?.maxConnectionsPerPod !== 2
+  ) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DEPLOYMENT',
+      'two tokenless Run manager replicas must be ready on distinct nodes with the exact budget',
+    );
+  }
+
+  const client = report?.client;
+  if (
+    !exactKeys(client, [
+      'binary',
+      'operations',
+      'inputKind',
+      'inputImmutable',
+      'callerDrivenJob',
+      'backoffLimit',
+      'serviceAccountTokenMounted',
+      'rbacGranted',
+      'transportProtocol',
+      'mutualTls',
+      'servernameVerified',
+      'exactPodRequests',
+      'retryStatuses',
+      'stopStatuses',
+      'responseRedacted',
+    ]) ||
+    client?.binary !== 'ql3-run-client' ||
+    JSON.stringify(client?.operations) !==
+      JSON.stringify(['run.retry', 'run.stop']) ||
+    client?.inputKind !== 'Secret' ||
+    client?.inputImmutable !== true ||
+    client?.callerDrivenJob !== true ||
+    client?.backoffLimit !== 0 ||
+    client?.serviceAccountTokenMounted !== false ||
+    client?.rbacGranted !== false ||
+    client?.transportProtocol !== 'TLSv1.3' ||
+    client?.mutualTls !== true ||
+    client?.servernameVerified !== true ||
+    client?.exactPodRequests !== 6 ||
+    JSON.stringify(client?.retryStatuses) !==
+      JSON.stringify(['accepted', 'existing', 'existing']) ||
+    JSON.stringify(client?.stopStatuses) !==
+      JSON.stringify(['accepted', 'already_requested', 'already_requested']) ||
+    client?.responseRedacted !== true
+  ) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_CLIENT',
+      'immutable caller-driven clients must retry and stop with exact replay across both Pods over TLS 1.3 mTLS',
+    );
+  }
+
+  if (
+    !allTrue(report?.identityRotation, [
+      'overlapOldAssertionAccepted',
+      'overlapNewAssertionAccepted',
+      'revokedOldAssertionRejected',
+      'activeNewAssertionAccepted',
+      'rollbackSurgeFailedClosed',
+      'twoReadyReplicasPreserved',
+      'durableGenerationReachedThree',
+    ])
+  ) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_IDENTITY_ROTATION',
+      'identity overlap, revoke, rollback rejection and two-ready availability are mandatory',
+    );
+  }
+
+  const certificate = report?.certificateRotation;
+  if (
+    !exactKeys(certificate, [
+      'previousSerialSha256',
+      'currentSerialSha256',
+      'previousBundleSha256',
+      'currentBundleSha256',
+      'oldClientAcceptedBefore',
+      'replacementClientAcceptedBefore',
+      'oldClientRejectedAfter',
+      'replacementClientAcceptedAfter',
+      'fullPodReplacement',
+      'allReplicasReadyThroughout',
+    ]) ||
+    !isDigest(certificate?.previousSerialSha256) ||
+    !isDigest(certificate?.currentSerialSha256) ||
+    certificate?.previousSerialSha256 === certificate?.currentSerialSha256 ||
+    !isDigest(certificate?.previousBundleSha256) ||
+    !isDigest(certificate?.currentBundleSha256) ||
+    certificate?.previousBundleSha256 === certificate?.currentBundleSha256 ||
+    ![
+      'oldClientAcceptedBefore',
+      'replacementClientAcceptedBefore',
+      'oldClientRejectedAfter',
+      'replacementClientAcceptedAfter',
+      'fullPodReplacement',
+      'allReplicasReadyThroughout',
+    ].every((key) => certificate?.[key] === true)
+  ) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_CERTIFICATE_ROTATION',
+      'CRL rotation must replace all Pods without dropping below two ready replicas',
+    );
+  }
+
+  if (
+    !allTrue(report?.availability, [
+      'databaseFailureWithdrewReadiness',
+      'databaseFailurePreservedLiveness',
+      'stalePodsDidNotRecoverInPlace',
+      'freshPodsRecoveredAfterDatabase',
+      'bothReplicasServedAfterRecovery',
+    ])
+  ) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_AVAILABILITY',
+      'database loss must withdraw readiness, preserve liveness and require fresh manager Pods',
+    );
+  }
+  if (
+    !allTrue(report?.isolation, [
+      'labelledClientAllowed',
+      'unlabelledClientDenied',
+      'wrongPortDenied',
+      'kubernetesApiEgressDenied',
+      'publicInternetEgressDenied',
+      'cloudNativePgEgressAllowed',
+      'managerSecretReadDenied',
+      'managerMutationRbacDenied',
+    ])
+  ) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_ISOLATION',
+      'CNI and Kubernetes RBAC least-privilege observations are incomplete',
+    );
+  }
+
+  const durability = report?.durability;
+  if (
+    !exactKeys(durability, [
+      'sourceRunStatus',
+      'retryRunCount',
+      'retryAttemptCount',
+      'retryEventCount',
+      'stoppedRunCount',
+      'stopEventCount',
+      'allowedAuditCount',
+      'deniedAuditCount',
+      'duplicateMutationCount',
+      'identityGeneration',
+      'weakAuthenticationAuditCount',
+      'survivedCloudNativePgFailover',
+    ]) ||
+    durability?.sourceRunStatus !== 'failed' ||
+    durability?.retryRunCount !== 1 ||
+    durability?.retryAttemptCount !== 1 ||
+    durability?.retryEventCount !== 2 ||
+    durability?.stoppedRunCount !== 1 ||
+    durability?.stopEventCount !== 1 ||
+    durability?.allowedAuditCount !== 2 ||
+    durability?.deniedAuditCount !== 1 ||
+    durability?.duplicateMutationCount !== 0 ||
+    durability?.identityGeneration !== 3 ||
+    durability?.weakAuthenticationAuditCount !== 0 ||
+    durability?.survivedCloudNativePgFailover !== true
+  ) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DURABILITY',
+      'exact Run, Attempt, Event, cancellation and audit facts must survive failover without duplicates',
+    );
+  }
+
+  if (
+    !allTrue(report?.gates, [
+      'realThreeNodeKubernetes',
+      'realCniPolicy',
+      'threeInstanceCloudNativePg',
+      'twoManagerPodsOnDistinctNodes',
+      'tls13ProductClientAcrossBothPods',
+      'strongUserRetryAndStop',
+      'identityProjectionRotation',
+      'certificateRevocationRollout',
+      'databaseReadinessFence',
+      'durableFactsSurvivedFailover',
+      'leastPrivilege',
+      'passed',
+    ])
+  ) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_GATES',
+      'every independently observed release gate must pass',
+    );
+  }
+  if (JSON.stringify(report?.limitations) !== JSON.stringify(LIMITATIONS)) {
+    reject(
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_LIMITATIONS',
+      'the exact non-production limitations must remain visible',
+    );
+  }
+  return Object.freeze({
+    schemaVersion: 1,
+    fixture: FIXTURE,
+    findings: Object.freeze(findings),
+    compatible: findings.length === 0,
+  });
+}
+
+function main(argv = process.argv.slice(2)) {
+  const argument = argv[0] === '--' ? argv.slice(1) : argv;
+  if (
+    argument.length !== 1 ||
+    !argument[0].startsWith('--report=') ||
+    !path.isAbsolute(argument[0].slice('--report='.length))
+  ) {
+    throw new Error(
+      'usage: ql3-run-management-kubernetes-live-audit --report=/absolute/private-report.json',
+    );
+  }
+  const reportFile = argument[0].slice('--report='.length);
+  const stat = fs.lstatSync(reportFile);
+  if (!stat.isFile() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0) {
+    throw new Error(
+      'Run management Kubernetes live report must be a private regular file',
+    );
+  }
+  const audit = validateRunManagementKubernetesLiveReport(
+    JSON.parse(fs.readFileSync(reportFile, 'utf8')),
+  );
+  process.stdout.write(JSON.stringify(audit, null, 2) + '\n');
+  if (!audit.compatible) process.exitCode = 1;
+}
+
+if (require.main === module) {
+  try {
+    main();
+  } catch (error) {
+    process.stderr.write(
+      'QL3 Run management Kubernetes live audit failed: ' +
+        (error instanceof Error ? error.message : String(error)) +
+        '\n',
+    );
+    process.exitCode = 1;
+  }
+}
+
+module.exports = {
+  FIXTURE,
+  LIMITATIONS,
+  validateRunManagementKubernetesLiveReport,
+};

test/back/ql3RunManagementKubernetesLiveAudit.test.cjs

@@ -0,0 +1,244 @@
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+
+const {
+  FIXTURE,
+  LIMITATIONS,
+  validateRunManagementKubernetesLiveReport,
+} = require('../../scripts/ql3-run-management-kubernetes-live-audit.cjs');
+
+function digest(character) {
+  return 'sha256:' + character.repeat(64);
+}
+
+function validReport() {
+  return {
+    schemaVersion: 1,
+    fixture: FIXTURE,
+    observedAt: '2026-08-12T12:00:00.000Z',
+    platform: {
+      distribution: 'k3s',
+      kubernetesVersion: 'v1.34.3+k3s1',
+      architecture: 'arm64',
+      kubernetesImageId: digest('1'),
+      managementImageId: digest('2'),
+      cniName: 'flannel',
+      cniDistributionBinding: 'rancher/k3s:v1.34.3-k3s1',
+      controlPlaneNodes: 1,
+      workerNodes: 2,
+      cniReadyNodes: 3,
+    },
+    database: {
+      operator: 'cloudnative-pg',
+      operatorVersion: '1.30.0',
+      postgresVersionNumber: 180004,
+      postgresImageId: digest('3'),
+      instances: 3,
+      readyInstances: 3,
+      managerRole: 'ql3_run_manager',
+      migrationCount: 57,
+      controlCoreCapability: 56,
+      tlsVerified: true,
+      primaryChangedDuringFailover: true,
+    },
+    deployment: {
+      namespace: 'qinglong3-system',
+      service: 'ql3-run-management',
+      port: 8448,
+      replicas: 2,
+      readyReplicas: 2,
+      podIdentitySha256: [digest('4'), digest('5')],
+      nodeIdentitySha256: [digest('6'), digest('7')],
+      serviceAccount: 'ql3-run-management',
+      automountServiceAccountToken: false,
+      requiredPodAntiAffinity: true,
+      podDisruptionBudgetMinAvailable: 1,
+      maxUnavailable: 0,
+      maxConnectionsPerPod: 2,
+    },
+    client: {
+      binary: 'ql3-run-client',
+      operations: ['run.retry', 'run.stop'],
+      inputKind: 'Secret',
+      inputImmutable: true,
+      callerDrivenJob: true,
+      backoffLimit: 0,
+      serviceAccountTokenMounted: false,
+      rbacGranted: false,
+      transportProtocol: 'TLSv1.3',
+      mutualTls: true,
+      servernameVerified: true,
+      exactPodRequests: 6,
+      retryStatuses: ['accepted', 'existing', 'existing'],
+      stopStatuses: ['accepted', 'already_requested', 'already_requested'],
+      responseRedacted: true,
+    },
+    identityRotation: {
+      overlapOldAssertionAccepted: true,
+      overlapNewAssertionAccepted: true,
+      revokedOldAssertionRejected: true,
+      activeNewAssertionAccepted: true,
+      rollbackSurgeFailedClosed: true,
+      twoReadyReplicasPreserved: true,
+      durableGenerationReachedThree: true,
+    },
+    certificateRotation: {
+      previousSerialSha256: digest('8'),
+      currentSerialSha256: digest('9'),
+      previousBundleSha256: digest('a'),
+      currentBundleSha256: digest('b'),
+      oldClientAcceptedBefore: true,
+      replacementClientAcceptedBefore: true,
+      oldClientRejectedAfter: true,
+      replacementClientAcceptedAfter: true,
+      fullPodReplacement: true,
+      allReplicasReadyThroughout: true,
+    },
+    availability: {
+      databaseFailureWithdrewReadiness: true,
+      databaseFailurePreservedLiveness: true,
+      stalePodsDidNotRecoverInPlace: true,
+      freshPodsRecoveredAfterDatabase: true,
+      bothReplicasServedAfterRecovery: true,
+    },
+    isolation: {
+      labelledClientAllowed: true,
+      unlabelledClientDenied: true,
+      wrongPortDenied: true,
+      kubernetesApiEgressDenied: true,
+      publicInternetEgressDenied: true,
+      cloudNativePgEgressAllowed: true,
+      managerSecretReadDenied: true,
+      managerMutationRbacDenied: true,
+    },
+    durability: {
+      sourceRunStatus: 'failed',
+      retryRunCount: 1,
+      retryAttemptCount: 1,
+      retryEventCount: 2,
+      stoppedRunCount: 1,
+      stopEventCount: 1,
+      allowedAuditCount: 2,
+      deniedAuditCount: 1,
+      duplicateMutationCount: 0,
+      identityGeneration: 3,
+      weakAuthenticationAuditCount: 0,
+      survivedCloudNativePgFailover: true,
+    },
+    gates: {
+      realThreeNodeKubernetes: true,
+      realCniPolicy: true,
+      threeInstanceCloudNativePg: true,
+      twoManagerPodsOnDistinctNodes: true,
+      tls13ProductClientAcrossBothPods: true,
+      strongUserRetryAndStop: true,
+      identityProjectionRotation: true,
+      certificateRevocationRollout: true,
+      databaseReadinessFence: true,
+      durableFactsSurvivedFailover: true,
+      leastPrivilege: true,
+      passed: true,
+    },
+    limitations: [...LIMITATIONS],
+  };
+}
+
+function mutate(change) {
+  const report = structuredClone(validReport());
+  change(report);
+  return validateRunManagementKubernetesLiveReport(report);
+}
+
+test('accepts the exact content-free Run management Kubernetes report', () => {
+  assert.deepEqual(validateRunManagementKubernetesLiveReport(validReport()), {
+    schemaVersion: 1,
+    fixture: FIXTURE,
+    findings: [],
+    compatible: true,
+  });
+});
+
+test('rejects topology, schema and deployment weakening', () => {
+  for (const [code, change] of [
+    [
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_PLATFORM',
+      (report) => {
+        report.platform.workerNodes = 1;
+      },
+    ],
+    [
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DATABASE',
+      (report) => {
+        report.database.controlCoreCapability = 55;
+      },
+    ],
+    [
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DEPLOYMENT',
+      (report) => {
+        report.deployment.nodeIdentitySha256[1] =
+          report.deployment.nodeIdentitySha256[0];
+      },
+    ],
+  ]) {
+    assert.ok(mutate(change).findings.some((entry) => entry.code === code));
+  }
+});
+
+test('rejects widened client, incomplete rotations and false availability', () => {
+  for (const [code, change] of [
+    [
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_CLIENT',
+      (report) => {
+        report.client.rbacGranted = true;
+      },
+    ],
+    [
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_IDENTITY_ROTATION',
+      (report) => {
+        report.identityRotation.revokedOldAssertionRejected = false;
+      },
+    ],
+    [
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_CERTIFICATE_ROTATION',
+      (report) => {
+        report.certificateRotation.fullPodReplacement = false;
+      },
+    ],
+    [
+      'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_AVAILABILITY',
+      (report) => {
+        report.availability.stalePodsDidNotRecoverInPlace = false;
+      },
+    ],
+  ]) {
+    assert.ok(mutate(change).findings.some((entry) => entry.code === code));
+  }
+});
+
+test('rejects incomplete isolation, durable drift and secret-shaped content', () => {
+  assert.ok(
+    mutate((report) => {
+      report.isolation.publicInternetEgressDenied = false;
+    }).findings.some(
+      (entry) => entry.code === 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_ISOLATION',
+    ),
+  );
+  assert.ok(
+    mutate((report) => {
+      report.durability.retryRunCount = 2;
+    }).findings.some(
+      (entry) => entry.code === 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_DURABILITY',
+    ),
+  );
+  assert.ok(
+    mutate((report) => {
+      report.client.token =
+        'eyJ0123456789012345.eyJ0123456789012345.abc0123456789012345';
+    }).findings.some(
+      (entry) =>
+        entry.code === 'QL3_RUN_MANAGEMENT_KUBERNETES_LIVE_SECRET_EXPOSURE',
+    ),
+  );
+});
+
+module.exports = { validReport };

test/back/ql3RunManagementKubernetesLiveContract.test.cjs

@@ -0,0 +1,99 @@
+const assert = require('node:assert/strict');
+const { spawnSync } = require('node:child_process');
+const fs = require('node:fs');
+const path = require('node:path');
+const { test } = require('node:test');
+
+const {
+  identity,
+  retryCommand,
+  stopCommand,
+} = require('../../scripts/ql3-run-management-kubernetes-live-contract.cjs');
+
+test('Run management live report path is mandatory before mutation begins', () => {
+  const script = path.resolve(
+    __dirname,
+    '../../scripts/ql3-run-management-kubernetes-live-contract.cjs',
+  );
+  const result = spawnSync(process.execPath, [script], {
+    encoding: 'utf8',
+    env: { ...process.env, QL3_RUN_MANAGEMENT_KUBERNETES_LIVE: '1' },
+  });
+  assert.equal(result.status, 1);
+  assert.match(result.stderr, /--report=\/absolute\/private-report\.json/);
+  assert.doesNotMatch(result.stderr, /Docker\/Kubernetes/);
+});
+
+test('Run live identity is audience, type, purpose and assurance bound', () => {
+  const key = identity.reviewedKey('run-live-test-key');
+  const document = identity.keyset(1, [key]);
+  assert.equal(document.audience, 'qinglong3-run-management');
+  const strong = identity.assertion(key, 'strong-unit-test');
+  const [header, payload, signature] = strong.split('.');
+  assert.ok(signature.length > 32);
+  assert.deepEqual(JSON.parse(Buffer.from(header, 'base64url')), {
+    alg: 'EdDSA',
+    kid: 'run-live-test-key',
+    typ: 'ql3-run-management+jwt',
+  });
+  const claims = JSON.parse(Buffer.from(payload, 'base64url'));
+  assert.equal(claims.aud, 'qinglong3-run-management');
+  assert.equal(claims.ql3_purpose, 'run-management');
+  assert.equal(claims.sub, 'run-operator');
+  assert.deepEqual(claims.amr, ['pwd', 'otp']);
+  const weak = JSON.parse(
+    Buffer.from(
+      identity.weakAssertion(key, 'weak-unit-test').split('.')[1],
+      'base64url',
+    ),
+  );
+  assert.equal(weak.acr, 'urn:ql3:password');
+  assert.deepEqual(weak.amr, ['pwd']);
+});
+
+test('Run live commands bind exact mutation, source fence and distinct audits', () => {
+  const retry = retryCommand(
+    'project-a',
+    'source-a',
+    'request-a',
+    '123e4567-e89b-42d3-a456-426614174000',
+    1,
+  );
+  assert.equal(retry.operation, 'run.retry');
+  assert.equal(retry.request.body.expectedRunStatus, 'failed');
+  assert.equal(retry.request.body.expectedRunVersion, 3);
+  assert.notEqual(
+    retry.request.auditEventId,
+    retry.request.failureAuditEventId,
+  );
+  const stop = stopCommand(
+    'project-a',
+    'run-a',
+    'request-b',
+    '123e4567-e89b-42d3-a456-426614174001',
+    2,
+  );
+  assert.equal(stop.operation, 'run.stop');
+  assert.equal(stop.request.body.schema, 'qinglong/run-cancellation@v1');
+  assert.notEqual(stop.request.auditEventId, stop.request.failureAuditEventId);
+});
+
+test('Run live runner remains opt-in, layered and observation backed', () => {
+  const source = fs.readFileSync(
+    path.resolve(
+      __dirname,
+      '../../scripts/ql3-run-management-kubernetes-live-contract.cjs',
+    ),
+    'utf8',
+  );
+  assert.match(source, /QL3_RUN_MANAGEMENT_KUBERNETES_LIVE !== '1'/);
+  assert.match(source, /reviewedOperatorManifest\(operatorManifestFile\)/);
+  assert.match(source, /validateRunManagementKubernetesLiveReport/);
+  assert.match(source, /createManagementClientExecutor/);
+  assert.match(source, /durableRunManagementFacts/);
+  assert.match(source, /Run identity ledger rollback surge failure/);
+  assert.match(source, /CloudNativePG primary promotion/);
+  assert.match(source, /migrationCount: 57/);
+  assert.match(source, /controlCoreCapability: 56/);
+  assert.doesNotMatch(source, /kubectl.*logs/);
+});