From b731b434f7e11d7910e4a6102f09fbc4a61b3e3e Mon Sep 17 00:00:00 2001 From: whyour Date: Mon, 24 Aug 2026 16:19:51 +0800 Subject: [PATCH] feat(ql3): prove mounted secret provider rotation --- .github/workflows/ql3-ci.yml | 68 +++ docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 11 +- ...y-gated-mounted-cluster-secret-provider.md | 10 +- ...-config-reconciliation-and-task-binding.md | 6 +- ...ding-and-mounted-provider-live-rotation.md | 130 +++++ docs/adr/README.md | 1 + ...e-secret-binding-kubernetes-live-audit.cjs | 105 +++- ...ecret-binding-kubernetes-live-contract.cjs | 483 +++++++++++++++++- ...-secret-binding-mounted-provider-actor.cjs | 141 +++++ ...eSecretBindingKubernetesLiveAudit.test.cjs | 49 +- 10 files changed, 956 insertions(+), 48 deletions(-) create mode 100644 docs/adr/ADR-0494-postgresql-secret-binding-and-mounted-provider-live-rotation.md create mode 100644 scripts/ql3-plugin-package-secret-binding-mounted-provider-actor.cjs diff --git a/.github/workflows/ql3-ci.yml b/.github/workflows/ql3-ci.yml index f09e0d22..bd696298 100644 --- a/.github/workflows/ql3-ci.yml +++ b/.github/workflows/ql3-ci.yml @@ -1149,6 +1149,74 @@ jobs: QL3_KUBECTL_BIN: ${{ github.workspace }}/kubectl run: pnpm test:provider-credential-test-kubernetes-live:ql3 + cluster-secret-binding-mounted-provider-kubernetes-live: + name: Secret binding approval and mounted provider rotation live contract + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@v6 + - uses: pnpm/action-setup@v6 + with: + version: '8.3.1' + - uses: actions/setup-node@v6 + with: + node-version: '24.18.0' + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - name: Install verified kubectl v1.34.3 + run: | + curl --fail --location --silent --show-error \ + --output kubectl \ + https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl + curl --fail --location --silent --show-error \ + --output kubectl.sha256 \ + https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl.sha256 + test "$(cat kubectl.sha256)" = "$(sha256sum kubectl | cut -d ' ' -f 1)" + chmod +x kubectl + - name: Preload digest-bound K3s and PostgreSQL fixtures + run: | + docker pull rancher/k3s@sha256:71abd3a56f57884c62732e0e0d87606052cb5f8555b7db7e8e33c04570b8175c + docker tag \ + rancher/k3s@sha256:71abd3a56f57884c62732e0e0d87606052cb5f8555b7db7e8e33c04570b8175c \ + rancher/k3s:v1.34.3-k3s1 + docker pull postgres@sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296 + docker tag \ + postgres@sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296 \ + postgres:18.4-bookworm + - name: Install workspace dependencies without lifecycle scripts + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Verify the backward-compatible offline evidence contract + run: node --test test/back/ql3PluginPackageSecretBindingKubernetesLiveAudit.test.cjs + - name: Prove approval, two-replica rotation and missing projection fail-close + env: + QL3_PLUGIN_PACKAGE_SECRET_BINDING_KUBERNETES_LIVE: '1' + QL3_KUBECTL_BIN: ${{ github.workspace }}/kubectl + QL3_SECRET_BINDING_LIVE_REPORT: ${{ runner.temp }}/ql3-secret-binding/report.json + run: | + umask 077 + mkdir -m 0700 "$(dirname "${QL3_SECRET_BINDING_LIVE_REPORT}")" + pnpm test:plugin-package-secret-binding-kubernetes-live:ql3 \ + "--report=${QL3_SECRET_BINDING_LIVE_REPORT}" + - name: Re-audit content-free evidence and isolated cleanup + env: + QL3_SECRET_BINDING_LIVE_REPORT: ${{ runner.temp }}/ql3-secret-binding/report.json + run: | + test "$(stat -c '%a' "${QL3_SECRET_BINDING_LIVE_REPORT}")" = '600' + pnpm audit:plugin-package-secret-binding-kubernetes-live:ql3 \ + "--report=${QL3_SECRET_BINDING_LIVE_REPORT}" + sha256sum "${QL3_SECRET_BINDING_LIVE_REPORT}" + test -z "$(docker ps -aq --filter name=ql3-secret-binding-live-)" + - name: Upload audited content-free mounted provider evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ql3-secret-binding-provider-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/ql3-secret-binding/report.json + if-no-files-found: error + retention-days: 14 + compression-level: 9 + overwrite: false + include-hidden-files: false + cluster-plugin-package-kubernetes-live: name: Plugin Package Kubernetes CAS and RBAC live contract runs-on: ubuntu-24.04 diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 303fe114..e065eb9f 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -68,6 +68,14 @@ `manual_required/identity_custody_required`，本切片不声称旧 credential、session、token、Policy 或 Audit 已迁移。Secret/Config v52 fixture 现已用真实 `complete → replay → verify` 完成 v3 闭环，不再由测试伪造 receipt、直接推进 head 或手工回收 storage；真正未知表和 Legacy 身份回归 继续失败关闭。 + 第十切片以 D-399/ADR-0494 完成 Cluster `mounted-files` provider 的真实在线子门，而没有增加 Edge/Standalone 闭包。固定 K3s + `v1.34.3+k3s1` 三节点和 PostgreSQL 18.4 中，两个跨节点 management replica 通过正式 mTLS client 完成 plan、跨副本 replay、四眼审批与 inspect； + direct exact-key executor 以无 ServiceAccount token、不可读取 Secret API、单一只读 `0440` projection 完成 binding exactly once，数据库 material + match 为 0。两个 production provider observer 以 required anti-affinity 位于不同节点，在无 Secret API 权限、无 token、deny-all network、非 root/ + 只读 root 下先观察 generation 1，再于 Secret resourceVersion-fenced replace 后无重启观察 generation 2；删除 Secret 后空 projection 以 + `QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE` 失败关闭。observer 将 content-free JSON 写入 termination log，审计不依赖偶发 EOF 的 kubelet logs。 + `qinglong/plugin-package-secret-binding-kubernetes-live@v2` 私有 `0600` 报告 24/24 gates 为 true、离线 findings 为空，同时 verifier 继续接受 immutable + v1 shape。该门不声明 control-plane HA、PostgreSQL 物理 failover或直接 Vault/KMS/HSM；也没有实现 Cluster Legacy Env migration ledger。 全部 evidence 不含原 Env name/value、目标 ciphertext/key ID 或 row body。v52 Local SQLite 完整测试为 `247/247`，publisher 定向回归 `6/6`；fresh Edge readiness 为 contract v52、104 migrations、89 required tables、SQLite 3.53.3、`DELETE` journal。Local Admin 为 `96/96`，ADR-0493 后 Local Owner 有效结果为 `301 total / 294 pass / 7 conditional skip / 0 fail`；完整 backend 为 @@ -79,7 +87,8 @@ 因而不重跑且不重新占有 PostgreSQL HA 证明；相邻已通过的 remote CI/HA 只作为基线。 D-385～D-388 的 `config.sh`/Keyv/SSH data-directory lineage 与 SQLite `Envs` 保持分离；当前无稳定生产 schema 的历史 `Configs` 表继续 sealed+manual， - 不猜字段。后续切片必须完成真实 Edge 空间证据和 Cluster Secret provider live gate。D-397 apply + 不猜字段。ADR-0494 已关闭基础 Cluster mounted-files provider live 子门；后续切片仍必须完成固定低性能设备的真实 Edge 空间/写放大/断电证据， + 以及 Cluster Legacy Env migration 的专用 PostgreSQL SERIALIZABLE ledger、Task/Trigger revision mutation、直接外部 custody adapter 与 HA promotion 后 receipt replay。D-397 apply 只声明 sealed source retained 且 `physicalErasureGuaranteed=false`；明文销毁必须在 restart/ readiness、观察窗和 rollback retention 之后另行强认证。Cluster 必须使用 PostgreSQL SERIALIZABLE ledger、外部 KMS/Secret provider 与 HA evidence， 不复用 Local SQLite/POSIX authority，也不得把明文写入 PostgreSQL、ConfigMap、Pod env 或 Job command。 diff --git a/docs/adr/ADR-0233-authority-gated-mounted-cluster-secret-provider.md b/docs/adr/ADR-0233-authority-gated-mounted-cluster-secret-provider.md index 606bec62..e171b0a4 100644 --- a/docs/adr/ADR-0233-authority-gated-mounted-cluster-secret-provider.md +++ b/docs/adr/ADR-0233-authority-gated-mounted-cluster-secret-provider.md @@ -144,6 +144,11 @@ Unicode、大小写和 Kubernetes key 兼容问题。canonical ref hash 更小 `24526848` bytes、零 OOM/oom_kill； 6. workspace 仍为 20 个 QL3 package，没有新增 migration、表、生产依赖、timer、 watcher、listener、Pool、连接、sidecar 或 Kubernetes API 权限。 +7. ADR-0494 已在真实三节点 K3s 中以两个跨节点 observer 证明 Kubernetes Secret + atomic-writer rotation：两副本无重启观察新 generation，均无 Secret API 权限、 + ServiceAccount token 与网络权限，projection 为只读 `0440`；删除 projection 后以 + `QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE` 失败关闭。v2 私有报告 24/24 gates + 为 true，且报告/termination message 不含值或 SecretRef。 ## 尚未关闭 @@ -152,5 +157,6 @@ Unicode、大小写和 Kubernetes key 兼容问题。canonical ref hash 更小 2. 直接 Vault/KMS/HSM adapter 的可选供应链、认证、rate limit 和 outage contract； 3. Worker materialization 后的 tmpfs/文件清除、Executor-specific injection 与真实 Pod/节点回收证据； -4. Kubernetes Secret/CSI/Vault Agent live rotation、并发 delivery、raw-wire - response loss 和多副本故障证据。 +4. 直接 CSI/Vault Agent adapter 自身的 live rotation/credential outage、raw-wire + response loss、节点丢失与 Kubernetes control-plane 多副本故障证据；基础 + Kubernetes Secret 双副本 atomic rotation 已由 ADR-0494 关闭。 diff --git a/docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md b/docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md index f4a86fb3..cd581dc7 100644 --- a/docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md +++ b/docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md @@ -1,6 +1,6 @@ # ADR-0491：有界 Secret/Config Reconciliation 与任务环境绑定 -- 状态：Proposed（D-397 已实现 Legacy Env inspection、私有有界 row plan、durable plan publication、独立 signed decision、逐项 Automation adoption provenance、Local SQLite 原子 application publisher、Owner prepared/apply/rollback 编排与 ADR-0492 completion v3；真实 Edge 空间证据和 Cluster Secret provider live gate 尚未完成） +- 状态：Proposed（D-397 已实现 Legacy Env inspection、私有有界 row plan、durable plan publication、独立 signed decision、逐项 Automation adoption provenance、Local SQLite 原子 application publisher、Owner prepared/apply/rollback 编排、ADR-0492 completion v3，并由 ADR-0494 完成 Cluster mounted-files provider live 子门；真实 Edge 空间证据、Cluster migration ledger 与直接外部 custody gate 尚未完成） - 日期：2026-08-23 - 决策：D-397 - 关联：ADR-0073、ADR-0074、ADR-0092、ADR-0094、ADR-0480、ADR-0482、ADR-0483、ADR-0484、ADR-0485、ADR-0486、ADR-0487、ADR-0488、ADR-0490 @@ -145,4 +145,6 @@ D-397 当前八切片已经实现：absent、unsupported、Edge over-budget、2. 本切片当前验证：Local SQLite `247/247`，其中 Secret/Config application publisher 定向回归 `6/6`；fresh Edge readiness 为 contract v52、104 migrations、89 required tables、SQLite 3.53.3、`DELETE` journal。Local Admin 为 `96/96`；ADR-0493 后 Local Owner 有效结果为 `301 total / 294 pass / 7 conditional skip / 0 fail`。完整 backend 为 `1567 total / 1565 pass / 2 conditional skip / 0 fail`。package boundary、精确 Cluster dependency/legacy boundary、122-module Edge import、service-manager bridge import、本地镜像与 `14/14` Local artifact audit 全部 compatible；Local Admin 为 49 source / 48 nested / 1 root export，Local Owner 为 188/187/1，workspace 仍为 18 packages 且没有单文件或浅层 package。基础 Edge/Standalone 为 `2,635,529 / 2,635,607 bytes`、323 files、58 loaded modules，且没有 Cluster/PostgreSQL 闭包。本切片不改 PostgreSQL schema、连接、role、Pool、容器或 Kubernetes 拓扑，因此不重跑且不重新占有 PostgreSQL HA 证明。 -转为 Accepted 前仍必须完成：真实 Edge 空间预算证据与 Cluster Secret provider live gate；ADR-0492 已完成本机 completion schema 演进和 completed-head 后 rollback material 回收，ADR-0493 又让没有 Legacy 身份输入的 fresh v52 目标身份经 signed `retain_target` 正确形成 no-effect，并精确消除六张已知目标表的 `unknown` 误判。Legacy `Auths/Users` 或真正未知表仍保持 manual；本切片的 Local Owner 编排或 PostgreSQL HA 证据不得冒充外部密钥托管。 +ADR-0494 已完成 Cluster `mounted-files` provider live 子门：真实三节点 K3s 中两个 management replica、direct exact-key executor 和两个跨节点 provider observer 完成 PostgreSQL durable approval/binding、Kubernetes atomic projection rotation、无 Secret API 权限/ServiceAccount token、只读 `0440`、内容脱敏及删除后 fail-closed；v2 私有报告 24/24 gates 为 true，并保持 v1 verifier 兼容。该门不增加 Edge 闭包，也不等于直接 Vault/KMS/HSM custody。 + +转为 Accepted 前仍必须完成：固定低性能 Edge 设备的真实空间/写放大/断电恢复证据，以及 Cluster Legacy Env migration 的专用 PostgreSQL SERIALIZABLE ledger、Task/Trigger revision mutation、外部 custody adapter 和 HA promotion 后 receipt replay。ADR-0492 已完成本机 completion schema 演进和 completed-head 后 rollback material 回收，ADR-0493 又让没有 Legacy 身份输入的 fresh v52 目标身份经 signed `retain_target` 正确形成 no-effect，并精确消除六张已知目标表的 `unknown` 误判。Legacy `Auths/Users` 或真正未知表仍保持 manual；本切片的 Local Owner 编排、ADR-0494 的 mounted-files gate 或 PostgreSQL HA 证据都不得冒充完整 Cluster migration 与外部密钥托管。 diff --git a/docs/adr/ADR-0494-postgresql-secret-binding-and-mounted-provider-live-rotation.md b/docs/adr/ADR-0494-postgresql-secret-binding-and-mounted-provider-live-rotation.md new file mode 100644 index 00000000..94926798 --- /dev/null +++ b/docs/adr/ADR-0494-postgresql-secret-binding-and-mounted-provider-live-rotation.md @@ -0,0 +1,130 @@ +# ADR-0494：PostgreSQL Secret Binding 与 Mounted Provider 在线轮换门 + +- 状态：Accepted +- 日期：2026-08-24 +- 决策：D-399 +- 关联：ADR-0129、ADR-0141、ADR-0233、ADR-0491、ADR-0492、ADR-0493 + +## 背景 + +ADR-0233 已提供 production `mounted-files` Secret provider，但原有证明主要来自 +单元测试、静态部署审计和受限容器资源门。D-397 又要求 Cluster Secret migration +不能借用 Local SQLite/POSIX authority，必须证明 PostgreSQL durable approval、精确 +Secret 投影和多节点运行时在同一真实 Kubernetes 门中仍然失败关闭。 + +旧的 Secret binding live contract 还落后于当前 executor base：base 已支持创建 +action-scoped Job，而兼容门仍假设外层 executor 直接挂载值；同时成功 Pod 的 kubelet +logs 偶发 `EOF` 会让已经完成的证据在报告阶段丢失。门禁不能通过增加 Secret API +权限、ServiceAccount token、watcher、sidecar 或把值写入报告来规避这些问题。 + +## 决策 + +### 1. 复用现有生产边界，不增加新的运行时闭包 + +本门直接构建正式 `ql3-cluster-admin` 和 `ql3-cluster-control` 镜像，provider 使用 +ADR-0233 的 `createClusterMountedSecretProvider`。不新增 workspace package、生产依赖、 +PostgreSQL migration/table/role、daemon、timer、watcher、listener、sidecar 或 Secret +API client；Edge/Standalone import graph 不加载本门或 Cluster 依赖。 + +兼容 Secret binding executor 在 fixture 中显式进入 direct exact-key 模式：关闭 +action controller、禁用 ServiceAccount token，只挂载当前批准计划所需的一个 +projection key，并设置 `QL3_PLUGIN_PACKAGE_EXECUTOR_SECRET_ROOT`。这证明既有 +PostgreSQL approval/binding 语义和 direct provider 消费，不改写 production base 的 +action-scoped controller 决策；controller 的 digest-pinned Job/admission contract 继续由 +其独立测试与部署审计证明。 + +### 2. 三节点拓扑与 durable approval 必须同时成立 + +live fixture 启动一台 K3s server 和两台 agent。两个 management Pod 使用 +anti-affinity 分布到不同节点，通过正式 mTLS client 完成 plan、跨副本 replay、propose、 +四眼 decide 和 inspect。PostgreSQL 18.4 保存 approval、execution 和 binding；数据库中 +只能出现 SecretRef/plan/evidence digest，不能出现 Secret material。最小权限 manager +不能读取 binding 表，management Pod 不挂载业务 Secret。 + +外层 executor 只读挂载 exact-key `0440` projection，不能 get/list Kubernetes Secret， +且没有 ServiceAccount token。它必须发布一次 binding、消费一次 approval 并成功完成； +任一持久化 fence 或 projection 漂移都失败关闭。 + +### 3. 两个 provider observer 必须无重启观察原子轮换 + +另建与 approval 中相同 SecretRef/projection key 对应的可变 Kubernetes Secret，并启动 +两个 observer Job。required pod anti-affinity 要求它们位于不同节点；二者都运行正式 +Cluster provider、非 root、只读 root filesystem、drop ALL capability，并满足： + +- `automountServiceAccountToken=false`； +- ServiceAccount 对 Secret `get/list/patch` 均为 `no`； +- deny-all NetworkPolicy，无 ingress/egress； +- projection read-only、`defaultMode=0440`，只包含一个精确 hash key； +- 第一代 material 被观察后才以 resourceVersion-fenced `replace` 写入第二代； +- 两个 Pod 都在不重启的情况下观察到第二代； +- stdout、termination message 和最终报告不含两代值或 SecretRef。 + +随后删除 Secret，并以 `optional: true` 空 projection 启动一次性 observer；生产 provider +必须返回 `QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE`。这里的 optional 只允许 Pod 启动， +不允许 material resolve 降级成功。 + +### 4. 证据不依赖 kubelet 日志可用性 + +observer 只输出 content-free JSON，并同时写入 `/dev/termination-log`。审计优先读取 +PodStatus 中的终止消息，只有没有该消息时才回退 kubelet logs。因此已完成 Job 的证据 +不会因节点日志通道 `EOF` 丢失，也不需要扩大 Kubernetes API/RBAC。 + +私有报告使用 `qinglong/plugin-package-secret-binding-kubernetes-live@v2`，以 owner-only +`0600` 原子发布。v2 在原 v1 字段上增加 control image、provider 拓扑、轮换、RBAC、 +投影模式、脱敏和 missing fail-closed 证据；离线 verifier 继续接受 immutable v1 shape， +但绝不允许 v1 报告伪装成 v2 provider 证明。 + +## 被拒绝的替代方案 + +### 给 control Pod 增加 Secret API 权限 + +拒绝。轮换由 Kubernetes atomic writer 投影完成；get/list/watch 会扩大 credential、网络、 +缓存和审计面。 + +### 用 `disk-pressure` toleration 或降低 kubelet eviction 阈值通过门禁 + +拒绝。这会掩盖真实资源不足。本机运行先清理明确未使用且可重建的镜像/缓存，并在用户 +授权后只回收未被容器引用的匿名卷，再从健康磁盘启动全新集群。 + +### 只验证单副本或重启后读取新值 + +拒绝。单副本不能证明 topology separation；重启后读取只能证明重新挂载，不能证明 +atomic writer rotation 被现有 provider 请求观察。 + +### 把 Secret 值或 SecretRef 写入报告便于排障 + +拒绝。报告只保留 digest、计数、布尔值、错误码和节点名哈希。排障不得扩大 material +custody。 + +## 当前验证 + +2026-08-24 本机 Apple Silicon 完整 live gate 已通过： + +- K3s `v1.34.3+k3s1`，3 个 Ready 节点； +- PostgreSQL `server_version_num=180004`； +- 2 个 management replica 位于不同节点； +- direct exact-key executor Job 成功，binding exactly once，数据库 material match 为 0； +- 2 个 provider observer 位于不同节点，第一代与轮换代均为 `2/2`； +- resourceVersion 前进，Pod 未重启，Secret API 三个权限均为 false； +- read-only `0440`、无 token、deny-all network，输出脱敏； +- Secret 删除后以 `QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE` 失败关闭； +- v2 报告 24/24 gates 为 true，离线审计 findings 为空，文件权限 `0600`； +- v1 verifier 兼容与 v2 drift/sensitive rejection 定向测试通过。 + +CI 新增独立 `cluster-secret-binding-mounted-provider-kubernetes-live` Job，固定 K3s、 +PostgreSQL 和 kubectl 版本，运行完整 live gate、离线复核并上传低敏报告。 + +## 边界与后续门禁 + +本 ADR 关闭 ADR-0491 的 Cluster mounted-files provider live 子门，但不把 D-397 整体 +转为 Accepted，也不声明： + +1. Kubernetes control-plane HA；本 fixture 是单 server + 双 agent； +2. PostgreSQL 物理 failover；它由独立 125-gate HA contract 证明； +3. 直接 Vault/KMS/HSM adapter、CSI/Vault Agent 自身故障或 credential rotation； +4. Cluster Legacy Env migration 的专用 SERIALIZABLE ledger、Task/Trigger mutation 和 + promotion 后 receipt replay； +5. 固定低性能路由设备上的真实空间、写放大、断电与恢复证据。 + +因此 ADR-0491 仍保持 Proposed。下一步优先完成固定 Edge 硬件空间门，随后实现并证明 +Cluster migration ledger；直接外部 custody adapter 作为可选、独立供应链继续设计。 diff --git a/docs/adr/README.md b/docs/adr/README.md index dbf0fff9..cc65a977 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -497,6 +497,7 @@ | [ADR-0491](./ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md) | 有界 Secret/Config Reconciliation 与任务环境绑定 | Proposed（D-397 local apply/rollback/completion complete；Edge/Cluster gate pending） | | [ADR-0492](./ADR-0492-compatible-secret-config-completion-v3.md) | 兼容 Secret/Config 的 Reconciliation Completion v3 | Accepted | | [ADR-0493](./ADR-0493-target-native-reconciliation-domain-preservation.md) | 目标原生 Reconciliation 域分类与身份保留 | Accepted | +| [ADR-0494](./ADR-0494-postgresql-secret-binding-and-mounted-provider-live-rotation.md) | PostgreSQL Secret Binding 与 Mounted Provider 在线轮换门 | Accepted | ## 规则 diff --git a/scripts/ql3-plugin-package-secret-binding-kubernetes-live-audit.cjs b/scripts/ql3-plugin-package-secret-binding-kubernetes-live-audit.cjs index 1973d0ac..2ac7aee8 100644 --- a/scripts/ql3-plugin-package-secret-binding-kubernetes-live-audit.cjs +++ b/scripts/ql3-plugin-package-secret-binding-kubernetes-live-audit.cjs @@ -5,11 +5,13 @@ const fs = require('node:fs'); const path = require('node:path'); -const FIXTURE = 'qinglong/plugin-package-secret-binding-kubernetes-live@v1'; +const LEGACY_FIXTURE = + 'qinglong/plugin-package-secret-binding-kubernetes-live@v1'; +const FIXTURE = 'qinglong/plugin-package-secret-binding-kubernetes-live@v2'; const SHA256 = /^sha256:[a-f0-9]{64}$/; const FORBIDDEN_KEY = /(secretRef|secretValue|material|assertion|jwt|password|dsn|privateKey|certificate|kubeconfig|podUid|nodeUid|podName|nodeName)/i; -const REQUIRED_GATES = Object.freeze([ +const CORE_GATES = Object.freeze([ 'realThreeNodeKubernetes', 'twoManagementReplicasOnDistinctNodes', 'formalHttpsClientCommands', @@ -25,6 +27,21 @@ const REQUIRED_GATES = Object.freeze([ 'executorHasNoServiceAccountToken', 'executorProjectionReadOnly', 'databaseContainsNoSensitiveValue', +]); +const LEGACY_REQUIRED_GATES = Object.freeze([ + ...CORE_GATES, + 'passed', +]); +const REQUIRED_GATES = Object.freeze([ + ...CORE_GATES, + 'twoProviderReplicasOnDistinctNodes', + 'productionMountedProviderUsed', + 'atomicProjectionRotationObserved', + 'providerCannotReadSecretApi', + 'providerHasNoServiceAccountToken', + 'providerProjectionReadOnly', + 'providerOutputSensitiveFree', + 'missingProjectionFailsClosed', 'passed', ]); @@ -63,21 +80,16 @@ function scan(value, findings, location = 'report') { function validatePluginPackageSecretBindingKubernetesLiveReport(report) { const findings = []; + const legacy = + report?.schemaVersion === 1 && report?.fixture === LEGACY_FIXTURE; + const current = report?.schemaVersion === 2 && report?.fixture === FIXTURE; if ( + (!legacy && !current) || !exact(report, [ - 'schemaVersion', - 'fixture', - 'observedAtMs', - 'platform', - 'management', - 'review', - 'executor', - 'persistence', - 'gates', - 'limitations', + 'schemaVersion', 'fixture', 'observedAtMs', 'platform', 'management', + 'review', 'executor', 'persistence', ...(current ? ['provider'] : []), + 'gates', 'limitations', ]) || - report.schemaVersion !== 1 || - report.fixture !== FIXTURE || !Number.isSafeInteger(report.observedAtMs) || report.observedAtMs < 1 ) { @@ -85,17 +97,16 @@ function validatePluginPackageSecretBindingKubernetesLiveReport(report) { } if ( !exact(report.platform, [ - 'architecture', - 'kubernetesVersion', - 'nodeCount', - 'postgresVersionNumber', - 'adminImageId', + 'architecture', 'kubernetesVersion', 'nodeCount', + 'postgresVersionNumber', 'adminImageId', + ...(current ? ['controlImageId'] : []), ]) || !['amd64', 'arm64'].includes(report.platform?.architecture) || typeof report.platform?.kubernetesVersion !== 'string' || report.platform?.nodeCount !== 3 || report.platform?.postgresVersionNumber !== 180004 || - !SHA256.test(report.platform?.adminImageId ?? '') + !SHA256.test(report.platform?.adminImageId ?? '') || + (current && !SHA256.test(report.platform?.controlImageId ?? '')) ) { findings.push('platform evidence is invalid'); } @@ -200,14 +211,58 @@ function validatePluginPackageSecretBindingKubernetesLiveReport(report) { findings.push('durable binding evidence is invalid'); } if ( - !exact(report.gates, REQUIRED_GATES) || - REQUIRED_GATES.some((gate) => report.gates?.[gate] !== true) + current && + (!exact(report.provider, [ + 'provider', + 'replicas', + 'distinctNodeHashes', + 'serviceAccountTokenMounted', + 'canGetSecrets', + 'canListSecrets', + 'canPatchSecrets', + 'projectionReadOnly', + 'projectionMode', + 'firstGenerationObserved', + 'rotatedGenerationObserved', + 'resourceVersionAdvanced', + 'outputSensitiveFree', + 'missingProjectionRejected', + 'missingErrorCode', + ]) || + report.provider?.provider !== 'mounted-files' || + report.provider?.replicas !== 2 || + !Array.isArray(report.provider?.distinctNodeHashes) || + report.provider.distinctNodeHashes.length !== 2 || + new Set(report.provider.distinctNodeHashes).size !== 2 || + !report.provider.distinctNodeHashes.every((value) => + SHA256.test(value), + ) || + report.provider.serviceAccountTokenMounted !== false || + report.provider.canGetSecrets !== false || + report.provider.canListSecrets !== false || + report.provider.canPatchSecrets !== false || + report.provider.projectionReadOnly !== true || + report.provider.projectionMode !== '0440' || + report.provider.firstGenerationObserved !== 2 || + report.provider.rotatedGenerationObserved !== 2 || + report.provider.resourceVersionAdvanced !== true || + report.provider.outputSensitiveFree !== true || + report.provider.missingProjectionRejected !== true || + report.provider.missingErrorCode !== + 'QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE') + ) { + findings.push('mounted provider evidence is invalid'); + } + const requiredGates = current ? REQUIRED_GATES : LEGACY_REQUIRED_GATES; + if ( + !exact(report.gates, requiredGates) || + requiredGates.some((gate) => report.gates?.[gate] !== true) ) { findings.push('one or more required gates are false or missing'); } if ( !Array.isArray(report.limitations) || - report.limitations.length !== 2 || + report.limitations.length !== (current ? 3 : 2) || report.limitations.some( (value) => typeof value !== 'string' || value.length < 16 || value.length > 512, @@ -218,7 +273,7 @@ function validatePluginPackageSecretBindingKubernetesLiveReport(report) { scan(report, findings); return Object.freeze({ schemaVersion: 1, - fixture: FIXTURE, + fixture: current ? FIXTURE : LEGACY_FIXTURE, findings: Object.freeze(findings), compatible: findings.length === 0, }); @@ -254,6 +309,8 @@ if (require.main === module) { module.exports = { FIXTURE, + LEGACY_FIXTURE, + LEGACY_REQUIRED_GATES, REQUIRED_GATES, validatePluginPackageSecretBindingKubernetesLiveReport, }; diff --git a/scripts/ql3-plugin-package-secret-binding-kubernetes-live-contract.cjs b/scripts/ql3-plugin-package-secret-binding-kubernetes-live-contract.cjs index 596f344c..1c4a03ef 100644 --- a/scripts/ql3-plugin-package-secret-binding-kubernetes-live-contract.cjs +++ b/scripts/ql3-plugin-package-secret-binding-kubernetes-live-contract.cjs @@ -42,6 +42,11 @@ const REVIEWER_ID = 'secret-binding-reviewer'; const ACTION_REF = 'secret-binding:secret-binding-live:v1'; const APPROVAL_ID = 'secret-binding-live-approval'; const ADMIN_IMAGE_BASE = 'ql3-secret-binding-kubernetes-live'; +const CONTROL_IMAGE_BASE = 'ql3-secret-binding-mounted-provider-live'; +const PROVIDER_SECRET = 'ql3-cluster-worker-values-live'; +const PROVIDER_SERVICE_ACCOUNT = 'ql3-mounted-secret-provider-live'; +const PROVIDER_ACTOR_CONFIG = 'ql3-mounted-secret-provider-live-actor'; +const PROVIDER_ROOT = '/var/run/secrets/qinglong3/worker-values'; const ZERO_DIGEST = 'sha256:' + '0'.repeat(64); const ISSUER = 'https://identity.qinglong.test/'; const AUDIENCE = 'qinglong3-plugin-package-management'; @@ -428,11 +433,6 @@ async function waitJob(fixture, name, timeoutMs = 600_000) { } function jobLog(fixture, name) { - const logs = fixture.kubectl( - ['-n', NAMESPACE, 'logs', 'job/' + name, '--all-containers=true'], - { capture: true, quiet: true, allowFailure: true }, - ); - if (logs.status === 0) return logs.stdout; const pod = fixture.kubectlJson([ '-n', NAMESPACE, @@ -444,10 +444,13 @@ function jobLog(fixture, name) { const messages = pod?.status?.containerStatuses ?.map((status) => status.state?.terminated?.message) .filter(Boolean); - if (!messages?.length) { - throw new Error(name + ' output is unavailable: ' + logs.stderr); - } - return messages.join('\n'); + if (messages?.length) return messages.join('\n'); + const logs = fixture.kubectl( + ['-n', NAMESPACE, 'logs', 'job/' + name, '--all-containers=true'], + { capture: true, quiet: true, allowFailure: true }, + ); + if (logs.status === 0) return logs.stdout; + throw new Error(name + ' output is unavailable: ' + logs.stderr); } function lastJsonLine(output, predicate) { @@ -838,7 +841,8 @@ function canI(fixture, serviceAccount, verb) { return result.stdout === 'yes'; } -function renderExecutor(fixture, adminImage) { +function renderExecutor(fixture, adminImage, projectionKey) { + assert.match(projectionKey, /^[a-f0-9]{64}$/); const rendered = fixture.kubectl( [ 'kustomize', @@ -852,6 +856,7 @@ function renderExecutor(fixture, adminImage) { cronJob.spec.suspend = true; cronJob.spec.jobTemplate.spec.backoffLimit = 0; const template = cronJob.spec.jobTemplate.spec.template.spec; + template.automountServiceAccountToken = false; const container = template.containers[0]; container.image = adminImage; container.imagePullPolicy = 'Never'; @@ -885,6 +890,14 @@ function renderExecutor(fixture, adminImage) { name: 'QL3_POSTGRES_ALLOW_INSECURE', value: 'true', }); + setEnvironment(container, { + name: 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_ENABLED', + value: 'false', + }); + setEnvironment(container, { + name: 'QL3_PLUGIN_PACKAGE_EXECUTOR_SECRET_ROOT', + value: '/var/run/secrets/qinglong3/plugin-package-values', + }); for (const entry of roleEnvironment( 'PACKAGE_EXECUTOR', 'package-executor-password', @@ -901,9 +914,33 @@ function renderExecutor(fixture, adminImage) { container.volumeMounts = container.volumeMounts.filter( (entry) => entry.name !== 'postgres-package-executor-ca', ); + assert.equal( + container.volumeMounts.some( + (entry) => entry.name === 'plugin-package-values', + ), + false, + ); + container.volumeMounts.push({ + name: 'plugin-package-values', + mountPath: '/var/run/secrets/qinglong3/plugin-package-values', + readOnly: true, + }); template.volumes = template.volumes.filter( (entry) => entry.name !== 'postgres-package-executor-ca', ); + assert.equal( + template.volumes.some((entry) => entry.name === 'plugin-package-values'), + false, + ); + template.volumes.push({ + name: 'plugin-package-values', + secret: { + secretName: 'ql3-cluster-plugin-package-values', + optional: false, + defaultMode: 288, + items: [{ key: projectionKey, path: projectionKey }], + }, + }); const networkPolicy = resources.find( (value) => value.kind === 'NetworkPolicy', ); @@ -953,6 +990,334 @@ SELECT json_build_object( return JSON.parse(psql(fixture, DATABASE, sql).stdout); } +function replaceProviderSecret( + fixture, + projectionKey, + value, + resourceVersion = undefined, +) { + const manifest = { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: PROVIDER_SECRET, + namespace: NAMESPACE, + ...(resourceVersion === undefined ? {} : { resourceVersion }), + }, + type: 'Opaque', + data: { [projectionKey]: Buffer.from(value, 'utf8').toString('base64') }, + }; + return resourceVersion === undefined + ? fixture.create(manifest) + : fixture.kubectl(['replace', '-f', '-'], { + input: `${JSON.stringify(manifest)}\n`, + capture: true, + quiet: true, + }); +} + +function providerObserverJob(controlImage, name, projectionKey, missing) { + const pair = !missing; + const labels = { + 'app.kubernetes.io/name': 'ql3-mounted-secret-provider-live', + 'qinglong.io/provider-observer-pair': pair ? 'true' : 'false', + }; + return { + apiVersion: 'batch/v1', + kind: 'Job', + metadata: { name, namespace: NAMESPACE }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: 300, + ttlSecondsAfterFinished: 600, + template: { + metadata: { labels }, + spec: { + serviceAccountName: PROVIDER_SERVICE_ACCOUNT, + automountServiceAccountToken: false, + enableServiceLinks: false, + restartPolicy: 'Never', + securityContext: { + runAsNonRoot: true, + runAsUser: 10001, + runAsGroup: 10001, + fsGroup: 10001, + fsGroupChangePolicy: 'OnRootMismatch', + seccompProfile: { type: 'RuntimeDefault' }, + }, + ...(pair + ? { + affinity: { + podAntiAffinity: { + requiredDuringSchedulingIgnoredDuringExecution: [ + { + labelSelector: { + matchLabels: { + 'qinglong.io/provider-observer-pair': 'true', + }, + }, + topologyKey: 'kubernetes.io/hostname', + }, + ], + }, + }, + } + : {}), + containers: [ + { + name: 'observer', + image: controlImage, + imagePullPolicy: 'Never', + command: ['/bin/sh', '-c'], + args: [ + [ + 'set +e', + 'output="$(node /opt/ql3-live/actor.cjs 2>&1)"', + 'status=$?', + 'printf \'%s\\n\' "$output" > /dev/termination-log', + 'printf \'%s\\n\' "$output"', + 'exit "$status"', + ].join('\n'), + ], + terminationMessagePolicy: 'File', + env: [ + { name: 'NODE_PATH', value: '/opt/qinglong/node_modules' }, + ...(missing + ? [{ name: 'QL3_LIVE_EXPECT_MISSING', value: 'true' }] + : []), + ], + ...(pair + ? { + readinessProbe: { + exec: { + command: [ + '/bin/sh', + '-c', + 'test -f /tmp/ql3-mounted-secret-first-observed', + ], + }, + periodSeconds: 1, + failureThreshold: 180, + }, + } + : {}), + securityContext: { + allowPrivilegeEscalation: false, + readOnlyRootFilesystem: true, + capabilities: { drop: ['ALL'] }, + }, + resources: { + requests: { cpu: '5m', memory: '16Mi' }, + limits: { cpu: '100m', memory: '64Mi' }, + }, + volumeMounts: [ + { + name: 'actor', + mountPath: '/opt/ql3-live', + readOnly: true, + }, + { + name: 'provider-values', + mountPath: PROVIDER_ROOT, + readOnly: true, + }, + { name: 'tmp', mountPath: '/tmp' }, + ], + }, + ], + volumes: [ + { + name: 'actor', + configMap: { + name: PROVIDER_ACTOR_CONFIG, + defaultMode: 292, + }, + }, + { + name: 'provider-values', + secret: { + secretName: PROVIDER_SECRET, + optional: missing, + defaultMode: 288, + items: [{ key: projectionKey, path: projectionKey }], + }, + }, + { name: 'tmp', emptyDir: { medium: 'Memory', sizeLimit: '1Mi' } }, + ], + }, + }, + }, + }; +} + +async function proveMountedProviderRotation({ + fixture, + controlImage, + projectionKey, + secretRef, + firstValue, + secondValue, +}) { + fixture.create({ + apiVersion: 'v1', + kind: 'ServiceAccount', + metadata: { name: PROVIDER_SERVICE_ACCOUNT, namespace: NAMESPACE }, + automountServiceAccountToken: false, + }); + fixture.create({ + apiVersion: 'v1', + kind: 'ConfigMap', + metadata: { name: PROVIDER_ACTOR_CONFIG, namespace: NAMESPACE }, + immutable: true, + data: { + 'actor.cjs': fs.readFileSync( + path.join( + ROOT, + 'scripts/ql3-plugin-package-secret-binding-mounted-provider-actor.cjs', + ), + 'utf8', + ), + }, + }); + fixture.create({ + apiVersion: 'networking.k8s.io/v1', + kind: 'NetworkPolicy', + metadata: { + name: 'ql3-mounted-secret-provider-live-deny-all', + namespace: NAMESPACE, + }, + spec: { + podSelector: { + matchLabels: { + 'app.kubernetes.io/name': 'ql3-mounted-secret-provider-live', + }, + }, + policyTypes: ['Ingress', 'Egress'], + }, + }); + replaceProviderSecret(fixture, projectionKey, firstValue); + const firstResourceVersion = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'secret', + PROVIDER_SECRET, + ]).metadata.resourceVersion; + const names = ['ql3-mounted-provider-a', 'ql3-mounted-provider-b']; + for (const name of names) { + fixture.create( + providerObserverJob(controlImage, name, projectionKey, false), + ); + } + const ready = await waitFor('two mounted provider observers', 300_000, () => { + const pods = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'pods', + '-l', + 'qinglong.io/provider-observer-pair=true', + ]).items; + const readyPods = pods.filter(podReady); + return readyPods.length === 2 + ? { ready: true, value: readyPods } + : { ready: false, fact: `${readyPods.length}/2 observers Ready` }; + }); + const providerPods = ready.value; + assert.equal(new Set(providerPods.map((pod) => pod.spec.nodeName)).size, 2); + for (const pod of providerPods) { + assert.equal(pod.spec.automountServiceAccountToken, false); + const mount = pod.spec.containers[0].volumeMounts.find( + (entry) => entry.name === 'provider-values', + ); + assert.equal(mount?.readOnly, true); + assert.equal( + pod.spec.volumes.some((volume) => + volume.projected?.sources?.some( + (source) => source.serviceAccountToken !== undefined, + ), + ), + false, + ); + } + const current = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'secret', + PROVIDER_SECRET, + ]); + replaceProviderSecret( + fixture, + projectionKey, + secondValue, + current.metadata.resourceVersion, + ); + const secondResourceVersion = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'secret', + PROVIDER_SECRET, + ]).metadata.resourceVersion; + assert.notEqual(secondResourceVersion, firstResourceVersion); + await Promise.all(names.map((name) => waitJob(fixture, name, 300_000))); + const observations = names.map((name) => + lastJsonLine( + jobLog(fixture, name), + (value) => value.event === 'mounted_secret_rotation_observed', + ), + ); + assert.ok(observations.every((value) => value.generations === 2)); + const combinedOutput = names.map((name) => jobLog(fixture, name)).join('\n'); + assert.equal(combinedOutput.includes(firstValue), false); + assert.equal(combinedOutput.includes(secondValue), false); + assert.equal(combinedOutput.includes(secretRef), false); + + fixture.kubectl([ + '-n', + NAMESPACE, + 'delete', + 'secret', + PROVIDER_SECRET, + '--wait=true', + ]); + const missingName = 'ql3-mounted-provider-missing'; + fixture.create( + providerObserverJob(controlImage, missingName, projectionKey, true), + ); + await waitJob(fixture, missingName, 300_000); + const missing = lastJsonLine( + jobLog(fixture, missingName), + (value) => value.event === 'mounted_secret_missing_rejected', + ); + assert.equal(missing.errorCode, 'QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE'); + + return Object.freeze({ + provider: 'mounted-files', + replicas: providerPods.length, + distinctNodeHashes: providerPods + .map((pod) => sha256(pod.spec.nodeName)) + .sort(), + serviceAccountTokenMounted: false, + canGetSecrets: canI(fixture, PROVIDER_SERVICE_ACCOUNT, 'get'), + canListSecrets: canI(fixture, PROVIDER_SERVICE_ACCOUNT, 'list'), + canPatchSecrets: canI(fixture, PROVIDER_SERVICE_ACCOUNT, 'patch'), + projectionReadOnly: true, + projectionMode: '0440', + firstGenerationObserved: observations.length, + rotatedGenerationObserved: observations.length, + resourceVersionAdvanced: secondResourceVersion !== firstResourceVersion, + outputSensitiveFree: + !combinedOutput.includes(firstValue) && + !combinedOutput.includes(secondValue) && + !combinedOutput.includes(secretRef), + missingProjectionRejected: + missing.errorCode === 'QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE', + missingErrorCode: missing.errorCode, + }); +} + async function main(argv = process.argv.slice(2)) { const reportFile = privateReportPath(argv); if (process.env.QL3_PLUGIN_PACKAGE_SECRET_BINDING_KUBERNETES_LIVE !== '1') { @@ -967,7 +1332,9 @@ async function main(argv = process.argv.slice(2)) { const suffix = process.pid.toString(36) + '-' + crypto.randomBytes(3).toString('hex'); const adminImage = ADMIN_IMAGE_BASE + ':' + suffix; + const controlImage = CONTROL_IMAGE_BASE + ':' + suffix; let adminImageBuilt = false; + let controlImageBuilt = false; try { const nodes = await fixture.start(); assert.equal(nodes.length, 3); @@ -990,6 +1357,20 @@ async function main(argv = process.argv.slice(2)) { fixture.loadImage(adminImage, 'secret-binding-admin.tar'); const adminImageInfo = fixture.inspectImage(adminImage); assert.ok(['amd64', 'arm64'].includes(adminImageInfo.Architecture)); + run(fixture.docker, [ + 'build', + '--file', + 'deploy/containers/ql3-cluster-control/Dockerfile', + '--tag', + controlImage, + '--build-arg', + 'SOURCE_REVISION=' + sourceRevision, + '.', + ]); + controlImageBuilt = true; + fixture.loadImage(controlImage, 'secret-binding-control.tar'); + const controlImageInfo = fixture.inspectImage(controlImage); + assert.equal(controlImageInfo.Architecture, adminImageInfo.Architecture); fixture.apply({ apiVersion: 'v1', @@ -1000,7 +1381,20 @@ async function main(argv = process.argv.slice(2)) { for (const resource of postgresResources(superuserPassword)) { fixture.apply(resource); } - await waitFor('PostgreSQL readiness', 300_000, () => { + try { + await waitFor('PostgreSQL readiness', 300_000, () => { + const pod = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'pod', + POSTGRES, + ]); + return podReady(pod) + ? { ready: true, value: pod } + : { ready: false, fact: pod.status?.phase ?? 'unknown' }; + }); + } catch (error) { const pod = fixture.kubectlJson([ '-n', NAMESPACE, @@ -1008,10 +1402,30 @@ async function main(argv = process.argv.slice(2)) { 'pod', POSTGRES, ]); - return podReady(pod) - ? { ready: true, value: pod } - : { ready: false, fact: pod.status?.phase ?? 'unknown' }; - }); + const events = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'events', + '--field-selector', + 'involvedObject.name=' + POSTGRES, + ]); + throw new Error( + `${error instanceof Error ? error.message : String(error)}; ` + + `node=${pod.spec?.nodeName ?? 'unscheduled'}; ` + + `conditions=${JSON.stringify(pod.status?.conditions ?? [])}; ` + + `containerStatuses=${JSON.stringify( + pod.status?.containerStatuses ?? [], + )}; events=${JSON.stringify( + (events.items ?? []).map((event) => ({ + reason: event.reason, + message: event.message, + count: event.count, + })), + )}`, + { cause: error }, + ); + } const passwords = Object.fromEntries( ROLE_NAMES.map((role) => [role, randomSecret()]), ); @@ -1187,7 +1601,11 @@ async function main(argv = process.argv.slice(2)) { applySecret(fixture, 'ql3-cluster-plugin-package-values', { [prerequisite.projectionKey]: sensitiveValue, }); - const executorResources = renderExecutor(fixture, adminImage); + const executorResources = renderExecutor( + fixture, + adminImage, + prerequisite.projectionKey, + ); for (const resource of executorResources) fixture.apply(resource); fixture.kubectl([ '-n', @@ -1247,9 +1665,21 @@ async function main(argv = process.argv.slice(2)) { ); assert.notEqual(managerBindingRead.status, 0); + const mountedProvider = await proveMountedProviderRotation({ + fixture, + controlImage, + projectionKey: prerequisite.projectionKey, + secretRef: prerequisite.secretRef, + firstValue: sensitiveValue, + secondValue: 'ql3-live-rotated-' + randomSecret(), + }); + assert.equal(mountedProvider.canGetSecrets, false); + assert.equal(mountedProvider.canListSecrets, false); + assert.equal(mountedProvider.canPatchSecrets, false); + const report = { - schemaVersion: 1, - fixture: 'qinglong/plugin-package-secret-binding-kubernetes-live@v1', + schemaVersion: 2, + fixture: 'qinglong/plugin-package-secret-binding-kubernetes-live@v2', observedAtMs: Date.now(), platform: { architecture: adminImageInfo.Architecture, @@ -1258,6 +1688,7 @@ async function main(argv = process.argv.slice(2)) { nodeCount: nodes.length, postgresVersionNumber, adminImageId: imageId(adminImageInfo), + controlImageId: imageId(controlImageInfo), }, management: { replicas: managementPods.length, @@ -1308,6 +1739,7 @@ async function main(argv = process.argv.slice(2)) { !executorOutput.includes(prerequisite.secretRef), }, persistence, + provider: mountedProvider, gates: { realThreeNodeKubernetes: true, twoManagementReplicasOnDistinctNodes: true, @@ -1324,11 +1756,20 @@ async function main(argv = process.argv.slice(2)) { executorHasNoServiceAccountToken: true, executorProjectionReadOnly: true, databaseContainsNoSensitiveValue: true, + twoProviderReplicasOnDistinctNodes: true, + productionMountedProviderUsed: true, + atomicProjectionRotationObserved: true, + providerCannotReadSecretApi: true, + providerHasNoServiceAccountToken: true, + providerProjectionReadOnly: true, + providerOutputSensitiveFree: true, + missingProjectionFailsClosed: true, passed: true, }, limitations: [ 'single-server k3s control plane is not Kubernetes control-plane HA evidence', 'PostgreSQL physical failover is proven by the independent 125-gate HA contract', + 'the mounted-files gate proves Kubernetes projection, not a direct Vault KMS or HSM adapter', ], }; const audit = @@ -1353,6 +1794,12 @@ async function main(argv = process.argv.slice(2)) { quiet: true, }); } + if (controlImageBuilt) { + run(fixture.docker, ['image', 'rm', '-f', controlImage], { + capture: true, + quiet: true, + }); + } } } diff --git a/scripts/ql3-plugin-package-secret-binding-mounted-provider-actor.cjs b/scripts/ql3-plugin-package-secret-binding-mounted-provider-actor.cjs new file mode 100644 index 00000000..47508bcb --- /dev/null +++ b/scripts/ql3-plugin-package-secret-binding-mounted-provider-actor.cjs @@ -0,0 +1,141 @@ +#!/usr/bin/env node + +'use strict'; + +const assert = require('node:assert/strict'); +const { createHash } = require('node:crypto'); +const fs = require('node:fs'); + +const { + createSecretRef, +} = require('@qinglong/runtime-core/secret-reference'); +const { + ClusterMountedSecretProviderError, + createClusterMountedSecretProvider, +} = require('@qinglong/cluster-control/mounted-secret-provider'); + +const ROOT = '/var/run/secrets/qinglong3/worker-values'; +const FIRST_OBSERVATION = '/tmp/ql3-mounted-secret-first-observed'; +const SECRET_REF = createSecretRef({ + projectId: 'secret-binding-kubernetes-live', + name: 'runtime-token', + version: 1, +}); + +function authority() { + return Object.freeze({ + workerId: 'worker-secret-provider-live', + workerSessionId: '018f0000-0000-7000-8000-000000000001', + workerGeneration: 1, + runId: 'run-secret-provider-live', + attemptId: 'attempt-secret-provider-live', + projectId: 'secret-binding-kubernetes-live', + taskId: 'task-secret-provider-live', + taskRevision: 'revision-1', + executionDigest: 'a'.repeat(64), + offerId: 'offer-secret-provider-live', + leaseGeneration: 1, + leaseVersion: 1, + secretRefs: Object.freeze([SECRET_REF]), + }); +} + +function digest(value) { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +function delay(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +async function resolveDigest(provider) { + const resolution = await provider.resolve(authority()); + try { + assert.equal(resolution.values.length, 1); + assert.equal(resolution.values[0].secretRef, SECRET_REF); + assert.equal(typeof resolution.values[0].value, 'string'); + return digest(resolution.values[0].value); + } finally { + await resolution.dispose?.(); + } +} + +async function observeMissing(provider) { + try { + await provider.resolve(authority()); + } catch (error) { + assert.ok(error instanceof ClusterMountedSecretProviderError); + assert.equal(error.reason, 'material_unavailable'); + process.stdout.write( + `${JSON.stringify({ + schemaVersion: 1, + event: 'mounted_secret_missing_rejected', + errorCode: error.code, + })}\n`, + ); + return; + } + throw new Error('missing mounted Secret projection was accepted'); +} + +async function observeRotation(provider) { + const deadline = Date.now() + 180_000; + let firstDigest; + let observations = 0; + let unavailable = 0; + while (Date.now() < deadline) { + try { + const currentDigest = await resolveDigest(provider); + observations += 1; + if (!firstDigest) { + firstDigest = currentDigest; + fs.writeFileSync(FIRST_OBSERVATION, '', { flag: 'wx', mode: 0o600 }); + } else if (currentDigest !== firstDigest) { + process.stdout.write( + `${JSON.stringify({ + schemaVersion: 1, + event: 'mounted_secret_rotation_observed', + generations: 2, + observations, + unavailable, + })}\n`, + ); + return; + } + } catch (error) { + if (!(error instanceof ClusterMountedSecretProviderError)) throw error; + unavailable += 1; + } + await delay(250); + } + throw new Error('mounted Secret rotation was not observed before timeout'); +} + +async function main() { + const provider = await createClusterMountedSecretProvider({ + rootDirectory: ROOT, + }); + if (process.env.QL3_LIVE_EXPECT_MISSING === 'true') { + await observeMissing(provider); + return; + } + await observeRotation(provider); +} + +if (require.main === module) { + main().catch((error) => { + process.stderr.write( + `QL3 mounted Secret live actor failed: ${ + error instanceof Error ? error.message : String(error) + }\n`, + ); + process.exitCode = 1; + }); +} + +module.exports = { + FIRST_OBSERVATION, + SECRET_REF, + authority, + digest, +}; diff --git a/test/back/ql3PluginPackageSecretBindingKubernetesLiveAudit.test.cjs b/test/back/ql3PluginPackageSecretBindingKubernetesLiveAudit.test.cjs index 5a7e11fa..6ad8fb1e 100644 --- a/test/back/ql3PluginPackageSecretBindingKubernetesLiveAudit.test.cjs +++ b/test/back/ql3PluginPackageSecretBindingKubernetesLiveAudit.test.cjs @@ -5,6 +5,8 @@ const { test } = require('node:test'); const { FIXTURE, + LEGACY_FIXTURE, + LEGACY_REQUIRED_GATES, REQUIRED_GATES, validatePluginPackageSecretBindingKubernetesLiveReport, } = require('../../scripts/ql3-plugin-package-secret-binding-kubernetes-live-audit.cjs'); @@ -12,7 +14,7 @@ const { function report() { const digest = 'a'.repeat(64); return { - schemaVersion: 1, + schemaVersion: 2, fixture: FIXTURE, observedAtMs: 1, platform: { @@ -21,6 +23,7 @@ function report() { nodeCount: 3, postgresVersionNumber: 180004, adminImageId: `sha256:${digest}`, + controlImageId: `sha256:${'9'.repeat(64)}`, }, management: { replicas: 2, @@ -70,14 +73,48 @@ function report() { executionSucceeded: true, sensitiveMatchCount: 0, }, + provider: { + provider: 'mounted-files', + replicas: 2, + distinctNodeHashes: [ + `sha256:${'a'.repeat(64)}`, + `sha256:${'b'.repeat(64)}`, + ], + serviceAccountTokenMounted: false, + canGetSecrets: false, + canListSecrets: false, + canPatchSecrets: false, + projectionReadOnly: true, + projectionMode: '0440', + firstGenerationObserved: 2, + rotatedGenerationObserved: 2, + resourceVersionAdvanced: true, + outputSensitiveFree: true, + missingProjectionRejected: true, + missingErrorCode: 'QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE', + }, gates: Object.fromEntries(REQUIRED_GATES.map((gate) => [gate, true])), limitations: [ 'single-server k3s control plane is not Kubernetes control-plane HA evidence', 'PostgreSQL physical failover is proven by the independent 125-gate HA contract', + 'the mounted-files gate proves Kubernetes projection, not a direct Vault KMS or HSM adapter', ], }; } +function legacyReport() { + const value = report(); + value.schemaVersion = 1; + value.fixture = LEGACY_FIXTURE; + delete value.platform.controlImageId; + delete value.provider; + value.gates = Object.fromEntries( + LEGACY_REQUIRED_GATES.map((gate) => [gate, true]), + ); + value.limitations = value.limitations.slice(0, 2); + return value; +} + test('accepts one exact low-sensitive Secret binding Kubernetes report', () => { assert.deepEqual( validatePluginPackageSecretBindingKubernetesLiveReport(report()).findings, @@ -85,15 +122,25 @@ test('accepts one exact low-sensitive Secret binding Kubernetes report', () => { ); }); +test('continues to verify the immutable v1 report shape', () => { + assert.deepEqual( + validatePluginPackageSecretBindingKubernetesLiveReport(legacyReport()) + .findings, + [], + ); +}); + test('rejects false gates, topology drift and sensitive material', () => { const invalid = report(); invalid.gates.realExecutorJob = false; invalid.management.distinctNodeHashes[1] = invalid.management.distinctNodeHashes[0]; + invalid.provider.missingProjectionRejected = false; invalid.executor.secretRef = 'qlsecret:v1:forbidden'; const findings = validatePluginPackageSecretBindingKubernetesLiveReport(invalid).findings; assert.ok(findings.some((value) => value.includes('management'))); assert.ok(findings.some((value) => value.includes('gates'))); + assert.ok(findings.some((value) => value.includes('provider'))); assert.ok(findings.some((value) => value.includes('forbidden'))); });