mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 08:05:22 +08:00
feat(ql3): prove mounted secret provider rotation
This commit is contained in:
@@ -1149,6 +1149,74 @@ jobs:
|
||||
QL3_KUBECTL_BIN: ${{ github.workspace }}/kubectl
|
||||
run: pnpm test:provider-credential-test-kubernetes-live:ql3
|
||||
|
||||
cluster-secret-binding-mounted-provider-kubernetes-live:
|
||||
name: Secret binding approval and mounted provider rotation live contract
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: pnpm/action-setup@v6
|
||||
with:
|
||||
version: '8.3.1'
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '24.18.0'
|
||||
cache: pnpm
|
||||
cache-dependency-path: pnpm-lock.yaml
|
||||
- name: Install verified kubectl v1.34.3
|
||||
run: |
|
||||
curl --fail --location --silent --show-error \
|
||||
--output kubectl \
|
||||
https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl
|
||||
curl --fail --location --silent --show-error \
|
||||
--output kubectl.sha256 \
|
||||
https://dl.k8s.io/release/v1.34.3/bin/linux/amd64/kubectl.sha256
|
||||
test "$(cat kubectl.sha256)" = "$(sha256sum kubectl | cut -d ' ' -f 1)"
|
||||
chmod +x kubectl
|
||||
- name: Preload digest-bound K3s and PostgreSQL fixtures
|
||||
run: |
|
||||
docker pull rancher/k3s@sha256:71abd3a56f57884c62732e0e0d87606052cb5f8555b7db7e8e33c04570b8175c
|
||||
docker tag \
|
||||
rancher/k3s@sha256:71abd3a56f57884c62732e0e0d87606052cb5f8555b7db7e8e33c04570b8175c \
|
||||
rancher/k3s:v1.34.3-k3s1
|
||||
docker pull postgres@sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296
|
||||
docker tag \
|
||||
postgres@sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296 \
|
||||
postgres:18.4-bookworm
|
||||
- name: Install workspace dependencies without lifecycle scripts
|
||||
run: pnpm install --frozen-lockfile --ignore-scripts
|
||||
- name: Verify the backward-compatible offline evidence contract
|
||||
run: node --test test/back/ql3PluginPackageSecretBindingKubernetesLiveAudit.test.cjs
|
||||
- name: Prove approval, two-replica rotation and missing projection fail-close
|
||||
env:
|
||||
QL3_PLUGIN_PACKAGE_SECRET_BINDING_KUBERNETES_LIVE: '1'
|
||||
QL3_KUBECTL_BIN: ${{ github.workspace }}/kubectl
|
||||
QL3_SECRET_BINDING_LIVE_REPORT: ${{ runner.temp }}/ql3-secret-binding/report.json
|
||||
run: |
|
||||
umask 077
|
||||
mkdir -m 0700 "$(dirname "${QL3_SECRET_BINDING_LIVE_REPORT}")"
|
||||
pnpm test:plugin-package-secret-binding-kubernetes-live:ql3 \
|
||||
"--report=${QL3_SECRET_BINDING_LIVE_REPORT}"
|
||||
- name: Re-audit content-free evidence and isolated cleanup
|
||||
env:
|
||||
QL3_SECRET_BINDING_LIVE_REPORT: ${{ runner.temp }}/ql3-secret-binding/report.json
|
||||
run: |
|
||||
test "$(stat -c '%a' "${QL3_SECRET_BINDING_LIVE_REPORT}")" = '600'
|
||||
pnpm audit:plugin-package-secret-binding-kubernetes-live:ql3 \
|
||||
"--report=${QL3_SECRET_BINDING_LIVE_REPORT}"
|
||||
sha256sum "${QL3_SECRET_BINDING_LIVE_REPORT}"
|
||||
test -z "$(docker ps -aq --filter name=ql3-secret-binding-live-)"
|
||||
- name: Upload audited content-free mounted provider evidence
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ql3-secret-binding-provider-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/ql3-secret-binding/report.json
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
compression-level: 9
|
||||
overwrite: false
|
||||
include-hidden-files: false
|
||||
|
||||
cluster-plugin-package-kubernetes-live:
|
||||
name: Plugin Package Kubernetes CAS and RBAC live contract
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
@@ -68,6 +68,14 @@
|
||||
`manual_required/identity_custody_required`,本切片不声称旧 credential、session、token、Policy 或 Audit 已迁移。Secret/Config v52 fixture
|
||||
现已用真实 `complete → replay → verify` 完成 v3 闭环,不再由测试伪造 receipt、直接推进 head 或手工回收 storage;真正未知表和 Legacy 身份回归
|
||||
继续失败关闭。
|
||||
第十切片以 D-399/ADR-0494 完成 Cluster `mounted-files` provider 的真实在线子门,而没有增加 Edge/Standalone 闭包。固定 K3s
|
||||
`v1.34.3+k3s1` 三节点和 PostgreSQL 18.4 中,两个跨节点 management replica 通过正式 mTLS client 完成 plan、跨副本 replay、四眼审批与 inspect;
|
||||
direct exact-key executor 以无 ServiceAccount token、不可读取 Secret API、单一只读 `0440` projection 完成 binding exactly once,数据库 material
|
||||
match 为 0。两个 production provider observer 以 required anti-affinity 位于不同节点,在无 Secret API 权限、无 token、deny-all network、非 root/
|
||||
只读 root 下先观察 generation 1,再于 Secret resourceVersion-fenced replace 后无重启观察 generation 2;删除 Secret 后空 projection 以
|
||||
`QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE` 失败关闭。observer 将 content-free JSON 写入 termination log,审计不依赖偶发 EOF 的 kubelet logs。
|
||||
`qinglong/plugin-package-secret-binding-kubernetes-live@v2` 私有 `0600` 报告 24/24 gates 为 true、离线 findings 为空,同时 verifier 继续接受 immutable
|
||||
v1 shape。该门不声明 control-plane HA、PostgreSQL 物理 failover或直接 Vault/KMS/HSM;也没有实现 Cluster Legacy Env migration ledger。
|
||||
全部 evidence 不含原 Env name/value、目标 ciphertext/key ID 或 row body。v52 Local SQLite 完整测试为 `247/247`,publisher 定向回归 `6/6`;fresh Edge
|
||||
readiness 为 contract v52、104 migrations、89 required tables、SQLite 3.53.3、`DELETE` journal。Local Admin 为 `96/96`,ADR-0493 后 Local Owner 有效结果为
|
||||
`301 total / 294 pass / 7 conditional skip / 0 fail`;完整 backend 为
|
||||
@@ -79,7 +87,8 @@
|
||||
因而不重跑且不重新占有 PostgreSQL HA 证明;相邻已通过的 remote CI/HA 只作为基线。
|
||||
|
||||
D-385~D-388 的 `config.sh`/Keyv/SSH data-directory lineage 与 SQLite `Envs` 保持分离;当前无稳定生产 schema 的历史 `Configs` 表继续 sealed+manual,
|
||||
不猜字段。后续切片必须完成真实 Edge 空间证据和 Cluster Secret provider live gate。D-397 apply
|
||||
不猜字段。ADR-0494 已关闭基础 Cluster mounted-files provider live 子门;后续切片仍必须完成固定低性能设备的真实 Edge 空间/写放大/断电证据,
|
||||
以及 Cluster Legacy Env migration 的专用 PostgreSQL SERIALIZABLE ledger、Task/Trigger revision mutation、直接外部 custody adapter 与 HA promotion 后 receipt replay。D-397 apply
|
||||
只声明 sealed source retained 且 `physicalErasureGuaranteed=false`;明文销毁必须在 restart/
|
||||
readiness、观察窗和 rollback retention 之后另行强认证。Cluster 必须使用 PostgreSQL SERIALIZABLE ledger、外部 KMS/Secret provider 与 HA evidence,
|
||||
不复用 Local SQLite/POSIX authority,也不得把明文写入 PostgreSQL、ConfigMap、Pod env 或 Job command。
|
||||
|
||||
@@ -144,6 +144,11 @@ Unicode、大小写和 Kubernetes key 兼容问题。canonical ref hash 更小
|
||||
`24526848` bytes、零 OOM/oom_kill;
|
||||
6. workspace 仍为 20 个 QL3 package,没有新增 migration、表、生产依赖、timer、
|
||||
watcher、listener、Pool、连接、sidecar 或 Kubernetes API 权限。
|
||||
7. ADR-0494 已在真实三节点 K3s 中以两个跨节点 observer 证明 Kubernetes Secret
|
||||
atomic-writer rotation:两副本无重启观察新 generation,均无 Secret API 权限、
|
||||
ServiceAccount token 与网络权限,projection 为只读 `0440`;删除 projection 后以
|
||||
`QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE` 失败关闭。v2 私有报告 24/24 gates
|
||||
为 true,且报告/termination message 不含值或 SecretRef。
|
||||
|
||||
## 尚未关闭
|
||||
|
||||
@@ -152,5 +157,6 @@ Unicode、大小写和 Kubernetes key 兼容问题。canonical ref hash 更小
|
||||
2. 直接 Vault/KMS/HSM adapter 的可选供应链、认证、rate limit 和 outage contract;
|
||||
3. Worker materialization 后的 tmpfs/文件清除、Executor-specific injection 与真实
|
||||
Pod/节点回收证据;
|
||||
4. Kubernetes Secret/CSI/Vault Agent live rotation、并发 delivery、raw-wire
|
||||
response loss 和多副本故障证据。
|
||||
4. 直接 CSI/Vault Agent adapter 自身的 live rotation/credential outage、raw-wire
|
||||
response loss、节点丢失与 Kubernetes control-plane 多副本故障证据;基础
|
||||
Kubernetes Secret 双副本 atomic rotation 已由 ADR-0494 关闭。
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# ADR-0491:有界 Secret/Config Reconciliation 与任务环境绑定
|
||||
|
||||
- 状态:Proposed(D-397 已实现 Legacy Env inspection、私有有界 row plan、durable plan publication、独立 signed decision、逐项 Automation adoption provenance、Local SQLite 原子 application publisher、Owner prepared/apply/rollback 编排与 ADR-0492 completion v3;真实 Edge 空间证据和 Cluster Secret provider live gate 尚未完成)
|
||||
- 状态:Proposed(D-397 已实现 Legacy Env inspection、私有有界 row plan、durable plan publication、独立 signed decision、逐项 Automation adoption provenance、Local SQLite 原子 application publisher、Owner prepared/apply/rollback 编排、ADR-0492 completion v3,并由 ADR-0494 完成 Cluster mounted-files provider live 子门;真实 Edge 空间证据、Cluster migration ledger 与直接外部 custody gate 尚未完成)
|
||||
- 日期:2026-08-23
|
||||
- 决策:D-397
|
||||
- 关联:ADR-0073、ADR-0074、ADR-0092、ADR-0094、ADR-0480、ADR-0482、ADR-0483、ADR-0484、ADR-0485、ADR-0486、ADR-0487、ADR-0488、ADR-0490
|
||||
@@ -145,4 +145,6 @@ D-397 当前八切片已经实现:absent、unsupported、Edge over-budget、2.
|
||||
|
||||
本切片当前验证:Local SQLite `247/247`,其中 Secret/Config application publisher 定向回归 `6/6`;fresh Edge readiness 为 contract v52、104 migrations、89 required tables、SQLite 3.53.3、`DELETE` journal。Local Admin 为 `96/96`;ADR-0493 后 Local Owner 有效结果为 `301 total / 294 pass / 7 conditional skip / 0 fail`。完整 backend 为 `1567 total / 1565 pass / 2 conditional skip / 0 fail`。package boundary、精确 Cluster dependency/legacy boundary、122-module Edge import、service-manager bridge import、本地镜像与 `14/14` Local artifact audit 全部 compatible;Local Admin 为 49 source / 48 nested / 1 root export,Local Owner 为 188/187/1,workspace 仍为 18 packages 且没有单文件或浅层 package。基础 Edge/Standalone 为 `2,635,529 / 2,635,607 bytes`、323 files、58 loaded modules,且没有 Cluster/PostgreSQL 闭包。本切片不改 PostgreSQL schema、连接、role、Pool、容器或 Kubernetes 拓扑,因此不重跑且不重新占有 PostgreSQL HA 证明。
|
||||
|
||||
转为 Accepted 前仍必须完成:真实 Edge 空间预算证据与 Cluster Secret provider live gate;ADR-0492 已完成本机 completion schema 演进和 completed-head 后 rollback material 回收,ADR-0493 又让没有 Legacy 身份输入的 fresh v52 目标身份经 signed `retain_target` 正确形成 no-effect,并精确消除六张已知目标表的 `unknown` 误判。Legacy `Auths/Users` 或真正未知表仍保持 manual;本切片的 Local Owner 编排或 PostgreSQL HA 证据不得冒充外部密钥托管。
|
||||
ADR-0494 已完成 Cluster `mounted-files` provider live 子门:真实三节点 K3s 中两个 management replica、direct exact-key executor 和两个跨节点 provider observer 完成 PostgreSQL durable approval/binding、Kubernetes atomic projection rotation、无 Secret API 权限/ServiceAccount token、只读 `0440`、内容脱敏及删除后 fail-closed;v2 私有报告 24/24 gates 为 true,并保持 v1 verifier 兼容。该门不增加 Edge 闭包,也不等于直接 Vault/KMS/HSM custody。
|
||||
|
||||
转为 Accepted 前仍必须完成:固定低性能 Edge 设备的真实空间/写放大/断电恢复证据,以及 Cluster Legacy Env migration 的专用 PostgreSQL SERIALIZABLE ledger、Task/Trigger revision mutation、外部 custody adapter 和 HA promotion 后 receipt replay。ADR-0492 已完成本机 completion schema 演进和 completed-head 后 rollback material 回收,ADR-0493 又让没有 Legacy 身份输入的 fresh v52 目标身份经 signed `retain_target` 正确形成 no-effect,并精确消除六张已知目标表的 `unknown` 误判。Legacy `Auths/Users` 或真正未知表仍保持 manual;本切片的 Local Owner 编排、ADR-0494 的 mounted-files gate 或 PostgreSQL HA 证据都不得冒充完整 Cluster migration 与外部密钥托管。
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
# ADR-0494:PostgreSQL Secret Binding 与 Mounted Provider 在线轮换门
|
||||
|
||||
- 状态:Accepted
|
||||
- 日期:2026-08-24
|
||||
- 决策:D-399
|
||||
- 关联:ADR-0129、ADR-0141、ADR-0233、ADR-0491、ADR-0492、ADR-0493
|
||||
|
||||
## 背景
|
||||
|
||||
ADR-0233 已提供 production `mounted-files` Secret provider,但原有证明主要来自
|
||||
单元测试、静态部署审计和受限容器资源门。D-397 又要求 Cluster Secret migration
|
||||
不能借用 Local SQLite/POSIX authority,必须证明 PostgreSQL durable approval、精确
|
||||
Secret 投影和多节点运行时在同一真实 Kubernetes 门中仍然失败关闭。
|
||||
|
||||
旧的 Secret binding live contract 还落后于当前 executor base:base 已支持创建
|
||||
action-scoped Job,而兼容门仍假设外层 executor 直接挂载值;同时成功 Pod 的 kubelet
|
||||
logs 偶发 `EOF` 会让已经完成的证据在报告阶段丢失。门禁不能通过增加 Secret API
|
||||
权限、ServiceAccount token、watcher、sidecar 或把值写入报告来规避这些问题。
|
||||
|
||||
## 决策
|
||||
|
||||
### 1. 复用现有生产边界,不增加新的运行时闭包
|
||||
|
||||
本门直接构建正式 `ql3-cluster-admin` 和 `ql3-cluster-control` 镜像,provider 使用
|
||||
ADR-0233 的 `createClusterMountedSecretProvider`。不新增 workspace package、生产依赖、
|
||||
PostgreSQL migration/table/role、daemon、timer、watcher、listener、sidecar 或 Secret
|
||||
API client;Edge/Standalone import graph 不加载本门或 Cluster 依赖。
|
||||
|
||||
兼容 Secret binding executor 在 fixture 中显式进入 direct exact-key 模式:关闭
|
||||
action controller、禁用 ServiceAccount token,只挂载当前批准计划所需的一个
|
||||
projection key,并设置 `QL3_PLUGIN_PACKAGE_EXECUTOR_SECRET_ROOT`。这证明既有
|
||||
PostgreSQL approval/binding 语义和 direct provider 消费,不改写 production base 的
|
||||
action-scoped controller 决策;controller 的 digest-pinned Job/admission contract 继续由
|
||||
其独立测试与部署审计证明。
|
||||
|
||||
### 2. 三节点拓扑与 durable approval 必须同时成立
|
||||
|
||||
live fixture 启动一台 K3s server 和两台 agent。两个 management Pod 使用
|
||||
anti-affinity 分布到不同节点,通过正式 mTLS client 完成 plan、跨副本 replay、propose、
|
||||
四眼 decide 和 inspect。PostgreSQL 18.4 保存 approval、execution 和 binding;数据库中
|
||||
只能出现 SecretRef/plan/evidence digest,不能出现 Secret material。最小权限 manager
|
||||
不能读取 binding 表,management Pod 不挂载业务 Secret。
|
||||
|
||||
外层 executor 只读挂载 exact-key `0440` projection,不能 get/list Kubernetes Secret,
|
||||
且没有 ServiceAccount token。它必须发布一次 binding、消费一次 approval 并成功完成;
|
||||
任一持久化 fence 或 projection 漂移都失败关闭。
|
||||
|
||||
### 3. 两个 provider observer 必须无重启观察原子轮换
|
||||
|
||||
另建与 approval 中相同 SecretRef/projection key 对应的可变 Kubernetes Secret,并启动
|
||||
两个 observer Job。required pod anti-affinity 要求它们位于不同节点;二者都运行正式
|
||||
Cluster provider、非 root、只读 root filesystem、drop ALL capability,并满足:
|
||||
|
||||
- `automountServiceAccountToken=false`;
|
||||
- ServiceAccount 对 Secret `get/list/patch` 均为 `no`;
|
||||
- deny-all NetworkPolicy,无 ingress/egress;
|
||||
- projection read-only、`defaultMode=0440`,只包含一个精确 hash key;
|
||||
- 第一代 material 被观察后才以 resourceVersion-fenced `replace` 写入第二代;
|
||||
- 两个 Pod 都在不重启的情况下观察到第二代;
|
||||
- stdout、termination message 和最终报告不含两代值或 SecretRef。
|
||||
|
||||
随后删除 Secret,并以 `optional: true` 空 projection 启动一次性 observer;生产 provider
|
||||
必须返回 `QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE`。这里的 optional 只允许 Pod 启动,
|
||||
不允许 material resolve 降级成功。
|
||||
|
||||
### 4. 证据不依赖 kubelet 日志可用性
|
||||
|
||||
observer 只输出 content-free JSON,并同时写入 `/dev/termination-log`。审计优先读取
|
||||
PodStatus 中的终止消息,只有没有该消息时才回退 kubelet logs。因此已完成 Job 的证据
|
||||
不会因节点日志通道 `EOF` 丢失,也不需要扩大 Kubernetes API/RBAC。
|
||||
|
||||
私有报告使用 `qinglong/plugin-package-secret-binding-kubernetes-live@v2`,以 owner-only
|
||||
`0600` 原子发布。v2 在原 v1 字段上增加 control image、provider 拓扑、轮换、RBAC、
|
||||
投影模式、脱敏和 missing fail-closed 证据;离线 verifier 继续接受 immutable v1 shape,
|
||||
但绝不允许 v1 报告伪装成 v2 provider 证明。
|
||||
|
||||
## 被拒绝的替代方案
|
||||
|
||||
### 给 control Pod 增加 Secret API 权限
|
||||
|
||||
拒绝。轮换由 Kubernetes atomic writer 投影完成;get/list/watch 会扩大 credential、网络、
|
||||
缓存和审计面。
|
||||
|
||||
### 用 `disk-pressure` toleration 或降低 kubelet eviction 阈值通过门禁
|
||||
|
||||
拒绝。这会掩盖真实资源不足。本机运行先清理明确未使用且可重建的镜像/缓存,并在用户
|
||||
授权后只回收未被容器引用的匿名卷,再从健康磁盘启动全新集群。
|
||||
|
||||
### 只验证单副本或重启后读取新值
|
||||
|
||||
拒绝。单副本不能证明 topology separation;重启后读取只能证明重新挂载,不能证明
|
||||
atomic writer rotation 被现有 provider 请求观察。
|
||||
|
||||
### 把 Secret 值或 SecretRef 写入报告便于排障
|
||||
|
||||
拒绝。报告只保留 digest、计数、布尔值、错误码和节点名哈希。排障不得扩大 material
|
||||
custody。
|
||||
|
||||
## 当前验证
|
||||
|
||||
2026-08-24 本机 Apple Silicon 完整 live gate 已通过:
|
||||
|
||||
- K3s `v1.34.3+k3s1`,3 个 Ready 节点;
|
||||
- PostgreSQL `server_version_num=180004`;
|
||||
- 2 个 management replica 位于不同节点;
|
||||
- direct exact-key executor Job 成功,binding exactly once,数据库 material match 为 0;
|
||||
- 2 个 provider observer 位于不同节点,第一代与轮换代均为 `2/2`;
|
||||
- resourceVersion 前进,Pod 未重启,Secret API 三个权限均为 false;
|
||||
- read-only `0440`、无 token、deny-all network,输出脱敏;
|
||||
- Secret 删除后以 `QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE` 失败关闭;
|
||||
- v2 报告 24/24 gates 为 true,离线审计 findings 为空,文件权限 `0600`;
|
||||
- v1 verifier 兼容与 v2 drift/sensitive rejection 定向测试通过。
|
||||
|
||||
CI 新增独立 `cluster-secret-binding-mounted-provider-kubernetes-live` Job,固定 K3s、
|
||||
PostgreSQL 和 kubectl 版本,运行完整 live gate、离线复核并上传低敏报告。
|
||||
|
||||
## 边界与后续门禁
|
||||
|
||||
本 ADR 关闭 ADR-0491 的 Cluster mounted-files provider live 子门,但不把 D-397 整体
|
||||
转为 Accepted,也不声明:
|
||||
|
||||
1. Kubernetes control-plane HA;本 fixture 是单 server + 双 agent;
|
||||
2. PostgreSQL 物理 failover;它由独立 125-gate HA contract 证明;
|
||||
3. 直接 Vault/KMS/HSM adapter、CSI/Vault Agent 自身故障或 credential rotation;
|
||||
4. Cluster Legacy Env migration 的专用 SERIALIZABLE ledger、Task/Trigger mutation 和
|
||||
promotion 后 receipt replay;
|
||||
5. 固定低性能路由设备上的真实空间、写放大、断电与恢复证据。
|
||||
|
||||
因此 ADR-0491 仍保持 Proposed。下一步优先完成固定 Edge 硬件空间门,随后实现并证明
|
||||
Cluster migration ledger;直接外部 custody adapter 作为可选、独立供应链继续设计。
|
||||
@@ -497,6 +497,7 @@
|
||||
| [ADR-0491](./ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md) | 有界 Secret/Config Reconciliation 与任务环境绑定 | Proposed(D-397 local apply/rollback/completion complete;Edge/Cluster gate pending) |
|
||||
| [ADR-0492](./ADR-0492-compatible-secret-config-completion-v3.md) | 兼容 Secret/Config 的 Reconciliation Completion v3 | Accepted |
|
||||
| [ADR-0493](./ADR-0493-target-native-reconciliation-domain-preservation.md) | 目标原生 Reconciliation 域分类与身份保留 | Accepted |
|
||||
| [ADR-0494](./ADR-0494-postgresql-secret-binding-and-mounted-provider-live-rotation.md) | PostgreSQL Secret Binding 与 Mounted Provider 在线轮换门 | Accepted |
|
||||
|
||||
## 规则
|
||||
|
||||
|
||||
@@ -5,11 +5,13 @@
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const FIXTURE = 'qinglong/plugin-package-secret-binding-kubernetes-live@v1';
|
||||
const LEGACY_FIXTURE =
|
||||
'qinglong/plugin-package-secret-binding-kubernetes-live@v1';
|
||||
const FIXTURE = 'qinglong/plugin-package-secret-binding-kubernetes-live@v2';
|
||||
const SHA256 = /^sha256:[a-f0-9]{64}$/;
|
||||
const FORBIDDEN_KEY =
|
||||
/(secretRef|secretValue|material|assertion|jwt|password|dsn|privateKey|certificate|kubeconfig|podUid|nodeUid|podName|nodeName)/i;
|
||||
const REQUIRED_GATES = Object.freeze([
|
||||
const CORE_GATES = Object.freeze([
|
||||
'realThreeNodeKubernetes',
|
||||
'twoManagementReplicasOnDistinctNodes',
|
||||
'formalHttpsClientCommands',
|
||||
@@ -25,6 +27,21 @@ const REQUIRED_GATES = Object.freeze([
|
||||
'executorHasNoServiceAccountToken',
|
||||
'executorProjectionReadOnly',
|
||||
'databaseContainsNoSensitiveValue',
|
||||
]);
|
||||
const LEGACY_REQUIRED_GATES = Object.freeze([
|
||||
...CORE_GATES,
|
||||
'passed',
|
||||
]);
|
||||
const REQUIRED_GATES = Object.freeze([
|
||||
...CORE_GATES,
|
||||
'twoProviderReplicasOnDistinctNodes',
|
||||
'productionMountedProviderUsed',
|
||||
'atomicProjectionRotationObserved',
|
||||
'providerCannotReadSecretApi',
|
||||
'providerHasNoServiceAccountToken',
|
||||
'providerProjectionReadOnly',
|
||||
'providerOutputSensitiveFree',
|
||||
'missingProjectionFailsClosed',
|
||||
'passed',
|
||||
]);
|
||||
|
||||
@@ -63,21 +80,16 @@ function scan(value, findings, location = 'report') {
|
||||
|
||||
function validatePluginPackageSecretBindingKubernetesLiveReport(report) {
|
||||
const findings = [];
|
||||
const legacy =
|
||||
report?.schemaVersion === 1 && report?.fixture === LEGACY_FIXTURE;
|
||||
const current = report?.schemaVersion === 2 && report?.fixture === FIXTURE;
|
||||
if (
|
||||
(!legacy && !current) ||
|
||||
!exact(report, [
|
||||
'schemaVersion',
|
||||
'fixture',
|
||||
'observedAtMs',
|
||||
'platform',
|
||||
'management',
|
||||
'review',
|
||||
'executor',
|
||||
'persistence',
|
||||
'gates',
|
||||
'limitations',
|
||||
'schemaVersion', 'fixture', 'observedAtMs', 'platform', 'management',
|
||||
'review', 'executor', 'persistence', ...(current ? ['provider'] : []),
|
||||
'gates', 'limitations',
|
||||
]) ||
|
||||
report.schemaVersion !== 1 ||
|
||||
report.fixture !== FIXTURE ||
|
||||
!Number.isSafeInteger(report.observedAtMs) ||
|
||||
report.observedAtMs < 1
|
||||
) {
|
||||
@@ -85,17 +97,16 @@ function validatePluginPackageSecretBindingKubernetesLiveReport(report) {
|
||||
}
|
||||
if (
|
||||
!exact(report.platform, [
|
||||
'architecture',
|
||||
'kubernetesVersion',
|
||||
'nodeCount',
|
||||
'postgresVersionNumber',
|
||||
'adminImageId',
|
||||
'architecture', 'kubernetesVersion', 'nodeCount',
|
||||
'postgresVersionNumber', 'adminImageId',
|
||||
...(current ? ['controlImageId'] : []),
|
||||
]) ||
|
||||
!['amd64', 'arm64'].includes(report.platform?.architecture) ||
|
||||
typeof report.platform?.kubernetesVersion !== 'string' ||
|
||||
report.platform?.nodeCount !== 3 ||
|
||||
report.platform?.postgresVersionNumber !== 180004 ||
|
||||
!SHA256.test(report.platform?.adminImageId ?? '')
|
||||
!SHA256.test(report.platform?.adminImageId ?? '') ||
|
||||
(current && !SHA256.test(report.platform?.controlImageId ?? ''))
|
||||
) {
|
||||
findings.push('platform evidence is invalid');
|
||||
}
|
||||
@@ -200,14 +211,58 @@ function validatePluginPackageSecretBindingKubernetesLiveReport(report) {
|
||||
findings.push('durable binding evidence is invalid');
|
||||
}
|
||||
if (
|
||||
!exact(report.gates, REQUIRED_GATES) ||
|
||||
REQUIRED_GATES.some((gate) => report.gates?.[gate] !== true)
|
||||
current &&
|
||||
(!exact(report.provider, [
|
||||
'provider',
|
||||
'replicas',
|
||||
'distinctNodeHashes',
|
||||
'serviceAccountTokenMounted',
|
||||
'canGetSecrets',
|
||||
'canListSecrets',
|
||||
'canPatchSecrets',
|
||||
'projectionReadOnly',
|
||||
'projectionMode',
|
||||
'firstGenerationObserved',
|
||||
'rotatedGenerationObserved',
|
||||
'resourceVersionAdvanced',
|
||||
'outputSensitiveFree',
|
||||
'missingProjectionRejected',
|
||||
'missingErrorCode',
|
||||
]) ||
|
||||
report.provider?.provider !== 'mounted-files' ||
|
||||
report.provider?.replicas !== 2 ||
|
||||
!Array.isArray(report.provider?.distinctNodeHashes) ||
|
||||
report.provider.distinctNodeHashes.length !== 2 ||
|
||||
new Set(report.provider.distinctNodeHashes).size !== 2 ||
|
||||
!report.provider.distinctNodeHashes.every((value) =>
|
||||
SHA256.test(value),
|
||||
) ||
|
||||
report.provider.serviceAccountTokenMounted !== false ||
|
||||
report.provider.canGetSecrets !== false ||
|
||||
report.provider.canListSecrets !== false ||
|
||||
report.provider.canPatchSecrets !== false ||
|
||||
report.provider.projectionReadOnly !== true ||
|
||||
report.provider.projectionMode !== '0440' ||
|
||||
report.provider.firstGenerationObserved !== 2 ||
|
||||
report.provider.rotatedGenerationObserved !== 2 ||
|
||||
report.provider.resourceVersionAdvanced !== true ||
|
||||
report.provider.outputSensitiveFree !== true ||
|
||||
report.provider.missingProjectionRejected !== true ||
|
||||
report.provider.missingErrorCode !==
|
||||
'QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE')
|
||||
) {
|
||||
findings.push('mounted provider evidence is invalid');
|
||||
}
|
||||
const requiredGates = current ? REQUIRED_GATES : LEGACY_REQUIRED_GATES;
|
||||
if (
|
||||
!exact(report.gates, requiredGates) ||
|
||||
requiredGates.some((gate) => report.gates?.[gate] !== true)
|
||||
) {
|
||||
findings.push('one or more required gates are false or missing');
|
||||
}
|
||||
if (
|
||||
!Array.isArray(report.limitations) ||
|
||||
report.limitations.length !== 2 ||
|
||||
report.limitations.length !== (current ? 3 : 2) ||
|
||||
report.limitations.some(
|
||||
(value) =>
|
||||
typeof value !== 'string' || value.length < 16 || value.length > 512,
|
||||
@@ -218,7 +273,7 @@ function validatePluginPackageSecretBindingKubernetesLiveReport(report) {
|
||||
scan(report, findings);
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
fixture: FIXTURE,
|
||||
fixture: current ? FIXTURE : LEGACY_FIXTURE,
|
||||
findings: Object.freeze(findings),
|
||||
compatible: findings.length === 0,
|
||||
});
|
||||
@@ -254,6 +309,8 @@ if (require.main === module) {
|
||||
|
||||
module.exports = {
|
||||
FIXTURE,
|
||||
LEGACY_FIXTURE,
|
||||
LEGACY_REQUIRED_GATES,
|
||||
REQUIRED_GATES,
|
||||
validatePluginPackageSecretBindingKubernetesLiveReport,
|
||||
};
|
||||
|
||||
@@ -42,6 +42,11 @@ const REVIEWER_ID = 'secret-binding-reviewer';
|
||||
const ACTION_REF = 'secret-binding:secret-binding-live:v1';
|
||||
const APPROVAL_ID = 'secret-binding-live-approval';
|
||||
const ADMIN_IMAGE_BASE = 'ql3-secret-binding-kubernetes-live';
|
||||
const CONTROL_IMAGE_BASE = 'ql3-secret-binding-mounted-provider-live';
|
||||
const PROVIDER_SECRET = 'ql3-cluster-worker-values-live';
|
||||
const PROVIDER_SERVICE_ACCOUNT = 'ql3-mounted-secret-provider-live';
|
||||
const PROVIDER_ACTOR_CONFIG = 'ql3-mounted-secret-provider-live-actor';
|
||||
const PROVIDER_ROOT = '/var/run/secrets/qinglong3/worker-values';
|
||||
const ZERO_DIGEST = 'sha256:' + '0'.repeat(64);
|
||||
const ISSUER = 'https://identity.qinglong.test/';
|
||||
const AUDIENCE = 'qinglong3-plugin-package-management';
|
||||
@@ -428,11 +433,6 @@ async function waitJob(fixture, name, timeoutMs = 600_000) {
|
||||
}
|
||||
|
||||
function jobLog(fixture, name) {
|
||||
const logs = fixture.kubectl(
|
||||
['-n', NAMESPACE, 'logs', 'job/' + name, '--all-containers=true'],
|
||||
{ capture: true, quiet: true, allowFailure: true },
|
||||
);
|
||||
if (logs.status === 0) return logs.stdout;
|
||||
const pod = fixture.kubectlJson([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
@@ -444,10 +444,13 @@ function jobLog(fixture, name) {
|
||||
const messages = pod?.status?.containerStatuses
|
||||
?.map((status) => status.state?.terminated?.message)
|
||||
.filter(Boolean);
|
||||
if (!messages?.length) {
|
||||
throw new Error(name + ' output is unavailable: ' + logs.stderr);
|
||||
}
|
||||
return messages.join('\n');
|
||||
if (messages?.length) return messages.join('\n');
|
||||
const logs = fixture.kubectl(
|
||||
['-n', NAMESPACE, 'logs', 'job/' + name, '--all-containers=true'],
|
||||
{ capture: true, quiet: true, allowFailure: true },
|
||||
);
|
||||
if (logs.status === 0) return logs.stdout;
|
||||
throw new Error(name + ' output is unavailable: ' + logs.stderr);
|
||||
}
|
||||
|
||||
function lastJsonLine(output, predicate) {
|
||||
@@ -838,7 +841,8 @@ function canI(fixture, serviceAccount, verb) {
|
||||
return result.stdout === 'yes';
|
||||
}
|
||||
|
||||
function renderExecutor(fixture, adminImage) {
|
||||
function renderExecutor(fixture, adminImage, projectionKey) {
|
||||
assert.match(projectionKey, /^[a-f0-9]{64}$/);
|
||||
const rendered = fixture.kubectl(
|
||||
[
|
||||
'kustomize',
|
||||
@@ -852,6 +856,7 @@ function renderExecutor(fixture, adminImage) {
|
||||
cronJob.spec.suspend = true;
|
||||
cronJob.spec.jobTemplate.spec.backoffLimit = 0;
|
||||
const template = cronJob.spec.jobTemplate.spec.template.spec;
|
||||
template.automountServiceAccountToken = false;
|
||||
const container = template.containers[0];
|
||||
container.image = adminImage;
|
||||
container.imagePullPolicy = 'Never';
|
||||
@@ -885,6 +890,14 @@ function renderExecutor(fixture, adminImage) {
|
||||
name: 'QL3_POSTGRES_ALLOW_INSECURE',
|
||||
value: 'true',
|
||||
});
|
||||
setEnvironment(container, {
|
||||
name: 'QL3_PLUGIN_PACKAGE_SECRET_ACTION_CONTROLLER_ENABLED',
|
||||
value: 'false',
|
||||
});
|
||||
setEnvironment(container, {
|
||||
name: 'QL3_PLUGIN_PACKAGE_EXECUTOR_SECRET_ROOT',
|
||||
value: '/var/run/secrets/qinglong3/plugin-package-values',
|
||||
});
|
||||
for (const entry of roleEnvironment(
|
||||
'PACKAGE_EXECUTOR',
|
||||
'package-executor-password',
|
||||
@@ -901,9 +914,33 @@ function renderExecutor(fixture, adminImage) {
|
||||
container.volumeMounts = container.volumeMounts.filter(
|
||||
(entry) => entry.name !== 'postgres-package-executor-ca',
|
||||
);
|
||||
assert.equal(
|
||||
container.volumeMounts.some(
|
||||
(entry) => entry.name === 'plugin-package-values',
|
||||
),
|
||||
false,
|
||||
);
|
||||
container.volumeMounts.push({
|
||||
name: 'plugin-package-values',
|
||||
mountPath: '/var/run/secrets/qinglong3/plugin-package-values',
|
||||
readOnly: true,
|
||||
});
|
||||
template.volumes = template.volumes.filter(
|
||||
(entry) => entry.name !== 'postgres-package-executor-ca',
|
||||
);
|
||||
assert.equal(
|
||||
template.volumes.some((entry) => entry.name === 'plugin-package-values'),
|
||||
false,
|
||||
);
|
||||
template.volumes.push({
|
||||
name: 'plugin-package-values',
|
||||
secret: {
|
||||
secretName: 'ql3-cluster-plugin-package-values',
|
||||
optional: false,
|
||||
defaultMode: 288,
|
||||
items: [{ key: projectionKey, path: projectionKey }],
|
||||
},
|
||||
});
|
||||
const networkPolicy = resources.find(
|
||||
(value) => value.kind === 'NetworkPolicy',
|
||||
);
|
||||
@@ -953,6 +990,334 @@ SELECT json_build_object(
|
||||
return JSON.parse(psql(fixture, DATABASE, sql).stdout);
|
||||
}
|
||||
|
||||
function replaceProviderSecret(
|
||||
fixture,
|
||||
projectionKey,
|
||||
value,
|
||||
resourceVersion = undefined,
|
||||
) {
|
||||
const manifest = {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
metadata: {
|
||||
name: PROVIDER_SECRET,
|
||||
namespace: NAMESPACE,
|
||||
...(resourceVersion === undefined ? {} : { resourceVersion }),
|
||||
},
|
||||
type: 'Opaque',
|
||||
data: { [projectionKey]: Buffer.from(value, 'utf8').toString('base64') },
|
||||
};
|
||||
return resourceVersion === undefined
|
||||
? fixture.create(manifest)
|
||||
: fixture.kubectl(['replace', '-f', '-'], {
|
||||
input: `${JSON.stringify(manifest)}\n`,
|
||||
capture: true,
|
||||
quiet: true,
|
||||
});
|
||||
}
|
||||
|
||||
function providerObserverJob(controlImage, name, projectionKey, missing) {
|
||||
const pair = !missing;
|
||||
const labels = {
|
||||
'app.kubernetes.io/name': 'ql3-mounted-secret-provider-live',
|
||||
'qinglong.io/provider-observer-pair': pair ? 'true' : 'false',
|
||||
};
|
||||
return {
|
||||
apiVersion: 'batch/v1',
|
||||
kind: 'Job',
|
||||
metadata: { name, namespace: NAMESPACE },
|
||||
spec: {
|
||||
backoffLimit: 0,
|
||||
activeDeadlineSeconds: 300,
|
||||
ttlSecondsAfterFinished: 600,
|
||||
template: {
|
||||
metadata: { labels },
|
||||
spec: {
|
||||
serviceAccountName: PROVIDER_SERVICE_ACCOUNT,
|
||||
automountServiceAccountToken: false,
|
||||
enableServiceLinks: false,
|
||||
restartPolicy: 'Never',
|
||||
securityContext: {
|
||||
runAsNonRoot: true,
|
||||
runAsUser: 10001,
|
||||
runAsGroup: 10001,
|
||||
fsGroup: 10001,
|
||||
fsGroupChangePolicy: 'OnRootMismatch',
|
||||
seccompProfile: { type: 'RuntimeDefault' },
|
||||
},
|
||||
...(pair
|
||||
? {
|
||||
affinity: {
|
||||
podAntiAffinity: {
|
||||
requiredDuringSchedulingIgnoredDuringExecution: [
|
||||
{
|
||||
labelSelector: {
|
||||
matchLabels: {
|
||||
'qinglong.io/provider-observer-pair': 'true',
|
||||
},
|
||||
},
|
||||
topologyKey: 'kubernetes.io/hostname',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
containers: [
|
||||
{
|
||||
name: 'observer',
|
||||
image: controlImage,
|
||||
imagePullPolicy: 'Never',
|
||||
command: ['/bin/sh', '-c'],
|
||||
args: [
|
||||
[
|
||||
'set +e',
|
||||
'output="$(node /opt/ql3-live/actor.cjs 2>&1)"',
|
||||
'status=$?',
|
||||
'printf \'%s\\n\' "$output" > /dev/termination-log',
|
||||
'printf \'%s\\n\' "$output"',
|
||||
'exit "$status"',
|
||||
].join('\n'),
|
||||
],
|
||||
terminationMessagePolicy: 'File',
|
||||
env: [
|
||||
{ name: 'NODE_PATH', value: '/opt/qinglong/node_modules' },
|
||||
...(missing
|
||||
? [{ name: 'QL3_LIVE_EXPECT_MISSING', value: 'true' }]
|
||||
: []),
|
||||
],
|
||||
...(pair
|
||||
? {
|
||||
readinessProbe: {
|
||||
exec: {
|
||||
command: [
|
||||
'/bin/sh',
|
||||
'-c',
|
||||
'test -f /tmp/ql3-mounted-secret-first-observed',
|
||||
],
|
||||
},
|
||||
periodSeconds: 1,
|
||||
failureThreshold: 180,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
securityContext: {
|
||||
allowPrivilegeEscalation: false,
|
||||
readOnlyRootFilesystem: true,
|
||||
capabilities: { drop: ['ALL'] },
|
||||
},
|
||||
resources: {
|
||||
requests: { cpu: '5m', memory: '16Mi' },
|
||||
limits: { cpu: '100m', memory: '64Mi' },
|
||||
},
|
||||
volumeMounts: [
|
||||
{
|
||||
name: 'actor',
|
||||
mountPath: '/opt/ql3-live',
|
||||
readOnly: true,
|
||||
},
|
||||
{
|
||||
name: 'provider-values',
|
||||
mountPath: PROVIDER_ROOT,
|
||||
readOnly: true,
|
||||
},
|
||||
{ name: 'tmp', mountPath: '/tmp' },
|
||||
],
|
||||
},
|
||||
],
|
||||
volumes: [
|
||||
{
|
||||
name: 'actor',
|
||||
configMap: {
|
||||
name: PROVIDER_ACTOR_CONFIG,
|
||||
defaultMode: 292,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'provider-values',
|
||||
secret: {
|
||||
secretName: PROVIDER_SECRET,
|
||||
optional: missing,
|
||||
defaultMode: 288,
|
||||
items: [{ key: projectionKey, path: projectionKey }],
|
||||
},
|
||||
},
|
||||
{ name: 'tmp', emptyDir: { medium: 'Memory', sizeLimit: '1Mi' } },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function proveMountedProviderRotation({
|
||||
fixture,
|
||||
controlImage,
|
||||
projectionKey,
|
||||
secretRef,
|
||||
firstValue,
|
||||
secondValue,
|
||||
}) {
|
||||
fixture.create({
|
||||
apiVersion: 'v1',
|
||||
kind: 'ServiceAccount',
|
||||
metadata: { name: PROVIDER_SERVICE_ACCOUNT, namespace: NAMESPACE },
|
||||
automountServiceAccountToken: false,
|
||||
});
|
||||
fixture.create({
|
||||
apiVersion: 'v1',
|
||||
kind: 'ConfigMap',
|
||||
metadata: { name: PROVIDER_ACTOR_CONFIG, namespace: NAMESPACE },
|
||||
immutable: true,
|
||||
data: {
|
||||
'actor.cjs': fs.readFileSync(
|
||||
path.join(
|
||||
ROOT,
|
||||
'scripts/ql3-plugin-package-secret-binding-mounted-provider-actor.cjs',
|
||||
),
|
||||
'utf8',
|
||||
),
|
||||
},
|
||||
});
|
||||
fixture.create({
|
||||
apiVersion: 'networking.k8s.io/v1',
|
||||
kind: 'NetworkPolicy',
|
||||
metadata: {
|
||||
name: 'ql3-mounted-secret-provider-live-deny-all',
|
||||
namespace: NAMESPACE,
|
||||
},
|
||||
spec: {
|
||||
podSelector: {
|
||||
matchLabels: {
|
||||
'app.kubernetes.io/name': 'ql3-mounted-secret-provider-live',
|
||||
},
|
||||
},
|
||||
policyTypes: ['Ingress', 'Egress'],
|
||||
},
|
||||
});
|
||||
replaceProviderSecret(fixture, projectionKey, firstValue);
|
||||
const firstResourceVersion = fixture.kubectlJson([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
'get',
|
||||
'secret',
|
||||
PROVIDER_SECRET,
|
||||
]).metadata.resourceVersion;
|
||||
const names = ['ql3-mounted-provider-a', 'ql3-mounted-provider-b'];
|
||||
for (const name of names) {
|
||||
fixture.create(
|
||||
providerObserverJob(controlImage, name, projectionKey, false),
|
||||
);
|
||||
}
|
||||
const ready = await waitFor('two mounted provider observers', 300_000, () => {
|
||||
const pods = fixture.kubectlJson([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
'get',
|
||||
'pods',
|
||||
'-l',
|
||||
'qinglong.io/provider-observer-pair=true',
|
||||
]).items;
|
||||
const readyPods = pods.filter(podReady);
|
||||
return readyPods.length === 2
|
||||
? { ready: true, value: readyPods }
|
||||
: { ready: false, fact: `${readyPods.length}/2 observers Ready` };
|
||||
});
|
||||
const providerPods = ready.value;
|
||||
assert.equal(new Set(providerPods.map((pod) => pod.spec.nodeName)).size, 2);
|
||||
for (const pod of providerPods) {
|
||||
assert.equal(pod.spec.automountServiceAccountToken, false);
|
||||
const mount = pod.spec.containers[0].volumeMounts.find(
|
||||
(entry) => entry.name === 'provider-values',
|
||||
);
|
||||
assert.equal(mount?.readOnly, true);
|
||||
assert.equal(
|
||||
pod.spec.volumes.some((volume) =>
|
||||
volume.projected?.sources?.some(
|
||||
(source) => source.serviceAccountToken !== undefined,
|
||||
),
|
||||
),
|
||||
false,
|
||||
);
|
||||
}
|
||||
const current = fixture.kubectlJson([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
'get',
|
||||
'secret',
|
||||
PROVIDER_SECRET,
|
||||
]);
|
||||
replaceProviderSecret(
|
||||
fixture,
|
||||
projectionKey,
|
||||
secondValue,
|
||||
current.metadata.resourceVersion,
|
||||
);
|
||||
const secondResourceVersion = fixture.kubectlJson([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
'get',
|
||||
'secret',
|
||||
PROVIDER_SECRET,
|
||||
]).metadata.resourceVersion;
|
||||
assert.notEqual(secondResourceVersion, firstResourceVersion);
|
||||
await Promise.all(names.map((name) => waitJob(fixture, name, 300_000)));
|
||||
const observations = names.map((name) =>
|
||||
lastJsonLine(
|
||||
jobLog(fixture, name),
|
||||
(value) => value.event === 'mounted_secret_rotation_observed',
|
||||
),
|
||||
);
|
||||
assert.ok(observations.every((value) => value.generations === 2));
|
||||
const combinedOutput = names.map((name) => jobLog(fixture, name)).join('\n');
|
||||
assert.equal(combinedOutput.includes(firstValue), false);
|
||||
assert.equal(combinedOutput.includes(secondValue), false);
|
||||
assert.equal(combinedOutput.includes(secretRef), false);
|
||||
|
||||
fixture.kubectl([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
'delete',
|
||||
'secret',
|
||||
PROVIDER_SECRET,
|
||||
'--wait=true',
|
||||
]);
|
||||
const missingName = 'ql3-mounted-provider-missing';
|
||||
fixture.create(
|
||||
providerObserverJob(controlImage, missingName, projectionKey, true),
|
||||
);
|
||||
await waitJob(fixture, missingName, 300_000);
|
||||
const missing = lastJsonLine(
|
||||
jobLog(fixture, missingName),
|
||||
(value) => value.event === 'mounted_secret_missing_rejected',
|
||||
);
|
||||
assert.equal(missing.errorCode, 'QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE');
|
||||
|
||||
return Object.freeze({
|
||||
provider: 'mounted-files',
|
||||
replicas: providerPods.length,
|
||||
distinctNodeHashes: providerPods
|
||||
.map((pod) => sha256(pod.spec.nodeName))
|
||||
.sort(),
|
||||
serviceAccountTokenMounted: false,
|
||||
canGetSecrets: canI(fixture, PROVIDER_SERVICE_ACCOUNT, 'get'),
|
||||
canListSecrets: canI(fixture, PROVIDER_SERVICE_ACCOUNT, 'list'),
|
||||
canPatchSecrets: canI(fixture, PROVIDER_SERVICE_ACCOUNT, 'patch'),
|
||||
projectionReadOnly: true,
|
||||
projectionMode: '0440',
|
||||
firstGenerationObserved: observations.length,
|
||||
rotatedGenerationObserved: observations.length,
|
||||
resourceVersionAdvanced: secondResourceVersion !== firstResourceVersion,
|
||||
outputSensitiveFree:
|
||||
!combinedOutput.includes(firstValue) &&
|
||||
!combinedOutput.includes(secondValue) &&
|
||||
!combinedOutput.includes(secretRef),
|
||||
missingProjectionRejected:
|
||||
missing.errorCode === 'QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE',
|
||||
missingErrorCode: missing.errorCode,
|
||||
});
|
||||
}
|
||||
|
||||
async function main(argv = process.argv.slice(2)) {
|
||||
const reportFile = privateReportPath(argv);
|
||||
if (process.env.QL3_PLUGIN_PACKAGE_SECRET_BINDING_KUBERNETES_LIVE !== '1') {
|
||||
@@ -967,7 +1332,9 @@ async function main(argv = process.argv.slice(2)) {
|
||||
const suffix =
|
||||
process.pid.toString(36) + '-' + crypto.randomBytes(3).toString('hex');
|
||||
const adminImage = ADMIN_IMAGE_BASE + ':' + suffix;
|
||||
const controlImage = CONTROL_IMAGE_BASE + ':' + suffix;
|
||||
let adminImageBuilt = false;
|
||||
let controlImageBuilt = false;
|
||||
try {
|
||||
const nodes = await fixture.start();
|
||||
assert.equal(nodes.length, 3);
|
||||
@@ -990,6 +1357,20 @@ async function main(argv = process.argv.slice(2)) {
|
||||
fixture.loadImage(adminImage, 'secret-binding-admin.tar');
|
||||
const adminImageInfo = fixture.inspectImage(adminImage);
|
||||
assert.ok(['amd64', 'arm64'].includes(adminImageInfo.Architecture));
|
||||
run(fixture.docker, [
|
||||
'build',
|
||||
'--file',
|
||||
'deploy/containers/ql3-cluster-control/Dockerfile',
|
||||
'--tag',
|
||||
controlImage,
|
||||
'--build-arg',
|
||||
'SOURCE_REVISION=' + sourceRevision,
|
||||
'.',
|
||||
]);
|
||||
controlImageBuilt = true;
|
||||
fixture.loadImage(controlImage, 'secret-binding-control.tar');
|
||||
const controlImageInfo = fixture.inspectImage(controlImage);
|
||||
assert.equal(controlImageInfo.Architecture, adminImageInfo.Architecture);
|
||||
|
||||
fixture.apply({
|
||||
apiVersion: 'v1',
|
||||
@@ -1000,7 +1381,20 @@ async function main(argv = process.argv.slice(2)) {
|
||||
for (const resource of postgresResources(superuserPassword)) {
|
||||
fixture.apply(resource);
|
||||
}
|
||||
await waitFor('PostgreSQL readiness', 300_000, () => {
|
||||
try {
|
||||
await waitFor('PostgreSQL readiness', 300_000, () => {
|
||||
const pod = fixture.kubectlJson([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
'get',
|
||||
'pod',
|
||||
POSTGRES,
|
||||
]);
|
||||
return podReady(pod)
|
||||
? { ready: true, value: pod }
|
||||
: { ready: false, fact: pod.status?.phase ?? 'unknown' };
|
||||
});
|
||||
} catch (error) {
|
||||
const pod = fixture.kubectlJson([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
@@ -1008,10 +1402,30 @@ async function main(argv = process.argv.slice(2)) {
|
||||
'pod',
|
||||
POSTGRES,
|
||||
]);
|
||||
return podReady(pod)
|
||||
? { ready: true, value: pod }
|
||||
: { ready: false, fact: pod.status?.phase ?? 'unknown' };
|
||||
});
|
||||
const events = fixture.kubectlJson([
|
||||
'-n',
|
||||
NAMESPACE,
|
||||
'get',
|
||||
'events',
|
||||
'--field-selector',
|
||||
'involvedObject.name=' + POSTGRES,
|
||||
]);
|
||||
throw new Error(
|
||||
`${error instanceof Error ? error.message : String(error)}; ` +
|
||||
`node=${pod.spec?.nodeName ?? 'unscheduled'}; ` +
|
||||
`conditions=${JSON.stringify(pod.status?.conditions ?? [])}; ` +
|
||||
`containerStatuses=${JSON.stringify(
|
||||
pod.status?.containerStatuses ?? [],
|
||||
)}; events=${JSON.stringify(
|
||||
(events.items ?? []).map((event) => ({
|
||||
reason: event.reason,
|
||||
message: event.message,
|
||||
count: event.count,
|
||||
})),
|
||||
)}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
const passwords = Object.fromEntries(
|
||||
ROLE_NAMES.map((role) => [role, randomSecret()]),
|
||||
);
|
||||
@@ -1187,7 +1601,11 @@ async function main(argv = process.argv.slice(2)) {
|
||||
applySecret(fixture, 'ql3-cluster-plugin-package-values', {
|
||||
[prerequisite.projectionKey]: sensitiveValue,
|
||||
});
|
||||
const executorResources = renderExecutor(fixture, adminImage);
|
||||
const executorResources = renderExecutor(
|
||||
fixture,
|
||||
adminImage,
|
||||
prerequisite.projectionKey,
|
||||
);
|
||||
for (const resource of executorResources) fixture.apply(resource);
|
||||
fixture.kubectl([
|
||||
'-n',
|
||||
@@ -1247,9 +1665,21 @@ async function main(argv = process.argv.slice(2)) {
|
||||
);
|
||||
assert.notEqual(managerBindingRead.status, 0);
|
||||
|
||||
const mountedProvider = await proveMountedProviderRotation({
|
||||
fixture,
|
||||
controlImage,
|
||||
projectionKey: prerequisite.projectionKey,
|
||||
secretRef: prerequisite.secretRef,
|
||||
firstValue: sensitiveValue,
|
||||
secondValue: 'ql3-live-rotated-' + randomSecret(),
|
||||
});
|
||||
assert.equal(mountedProvider.canGetSecrets, false);
|
||||
assert.equal(mountedProvider.canListSecrets, false);
|
||||
assert.equal(mountedProvider.canPatchSecrets, false);
|
||||
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
fixture: 'qinglong/plugin-package-secret-binding-kubernetes-live@v1',
|
||||
schemaVersion: 2,
|
||||
fixture: 'qinglong/plugin-package-secret-binding-kubernetes-live@v2',
|
||||
observedAtMs: Date.now(),
|
||||
platform: {
|
||||
architecture: adminImageInfo.Architecture,
|
||||
@@ -1258,6 +1688,7 @@ async function main(argv = process.argv.slice(2)) {
|
||||
nodeCount: nodes.length,
|
||||
postgresVersionNumber,
|
||||
adminImageId: imageId(adminImageInfo),
|
||||
controlImageId: imageId(controlImageInfo),
|
||||
},
|
||||
management: {
|
||||
replicas: managementPods.length,
|
||||
@@ -1308,6 +1739,7 @@ async function main(argv = process.argv.slice(2)) {
|
||||
!executorOutput.includes(prerequisite.secretRef),
|
||||
},
|
||||
persistence,
|
||||
provider: mountedProvider,
|
||||
gates: {
|
||||
realThreeNodeKubernetes: true,
|
||||
twoManagementReplicasOnDistinctNodes: true,
|
||||
@@ -1324,11 +1756,20 @@ async function main(argv = process.argv.slice(2)) {
|
||||
executorHasNoServiceAccountToken: true,
|
||||
executorProjectionReadOnly: true,
|
||||
databaseContainsNoSensitiveValue: true,
|
||||
twoProviderReplicasOnDistinctNodes: true,
|
||||
productionMountedProviderUsed: true,
|
||||
atomicProjectionRotationObserved: true,
|
||||
providerCannotReadSecretApi: true,
|
||||
providerHasNoServiceAccountToken: true,
|
||||
providerProjectionReadOnly: true,
|
||||
providerOutputSensitiveFree: true,
|
||||
missingProjectionFailsClosed: true,
|
||||
passed: true,
|
||||
},
|
||||
limitations: [
|
||||
'single-server k3s control plane is not Kubernetes control-plane HA evidence',
|
||||
'PostgreSQL physical failover is proven by the independent 125-gate HA contract',
|
||||
'the mounted-files gate proves Kubernetes projection, not a direct Vault KMS or HSM adapter',
|
||||
],
|
||||
};
|
||||
const audit =
|
||||
@@ -1353,6 +1794,12 @@ async function main(argv = process.argv.slice(2)) {
|
||||
quiet: true,
|
||||
});
|
||||
}
|
||||
if (controlImageBuilt) {
|
||||
run(fixture.docker, ['image', 'rm', '-f', controlImage], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { createHash } = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
|
||||
const {
|
||||
createSecretRef,
|
||||
} = require('@qinglong/runtime-core/secret-reference');
|
||||
const {
|
||||
ClusterMountedSecretProviderError,
|
||||
createClusterMountedSecretProvider,
|
||||
} = require('@qinglong/cluster-control/mounted-secret-provider');
|
||||
|
||||
const ROOT = '/var/run/secrets/qinglong3/worker-values';
|
||||
const FIRST_OBSERVATION = '/tmp/ql3-mounted-secret-first-observed';
|
||||
const SECRET_REF = createSecretRef({
|
||||
projectId: 'secret-binding-kubernetes-live',
|
||||
name: 'runtime-token',
|
||||
version: 1,
|
||||
});
|
||||
|
||||
function authority() {
|
||||
return Object.freeze({
|
||||
workerId: 'worker-secret-provider-live',
|
||||
workerSessionId: '018f0000-0000-7000-8000-000000000001',
|
||||
workerGeneration: 1,
|
||||
runId: 'run-secret-provider-live',
|
||||
attemptId: 'attempt-secret-provider-live',
|
||||
projectId: 'secret-binding-kubernetes-live',
|
||||
taskId: 'task-secret-provider-live',
|
||||
taskRevision: 'revision-1',
|
||||
executionDigest: 'a'.repeat(64),
|
||||
offerId: 'offer-secret-provider-live',
|
||||
leaseGeneration: 1,
|
||||
leaseVersion: 1,
|
||||
secretRefs: Object.freeze([SECRET_REF]),
|
||||
});
|
||||
}
|
||||
|
||||
function digest(value) {
|
||||
return createHash('sha256').update(value, 'utf8').digest('hex');
|
||||
}
|
||||
|
||||
function delay(milliseconds) {
|
||||
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
}
|
||||
|
||||
async function resolveDigest(provider) {
|
||||
const resolution = await provider.resolve(authority());
|
||||
try {
|
||||
assert.equal(resolution.values.length, 1);
|
||||
assert.equal(resolution.values[0].secretRef, SECRET_REF);
|
||||
assert.equal(typeof resolution.values[0].value, 'string');
|
||||
return digest(resolution.values[0].value);
|
||||
} finally {
|
||||
await resolution.dispose?.();
|
||||
}
|
||||
}
|
||||
|
||||
async function observeMissing(provider) {
|
||||
try {
|
||||
await provider.resolve(authority());
|
||||
} catch (error) {
|
||||
assert.ok(error instanceof ClusterMountedSecretProviderError);
|
||||
assert.equal(error.reason, 'material_unavailable');
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
event: 'mounted_secret_missing_rejected',
|
||||
errorCode: error.code,
|
||||
})}\n`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
throw new Error('missing mounted Secret projection was accepted');
|
||||
}
|
||||
|
||||
async function observeRotation(provider) {
|
||||
const deadline = Date.now() + 180_000;
|
||||
let firstDigest;
|
||||
let observations = 0;
|
||||
let unavailable = 0;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const currentDigest = await resolveDigest(provider);
|
||||
observations += 1;
|
||||
if (!firstDigest) {
|
||||
firstDigest = currentDigest;
|
||||
fs.writeFileSync(FIRST_OBSERVATION, '', { flag: 'wx', mode: 0o600 });
|
||||
} else if (currentDigest !== firstDigest) {
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
event: 'mounted_secret_rotation_observed',
|
||||
generations: 2,
|
||||
observations,
|
||||
unavailable,
|
||||
})}\n`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!(error instanceof ClusterMountedSecretProviderError)) throw error;
|
||||
unavailable += 1;
|
||||
}
|
||||
await delay(250);
|
||||
}
|
||||
throw new Error('mounted Secret rotation was not observed before timeout');
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const provider = await createClusterMountedSecretProvider({
|
||||
rootDirectory: ROOT,
|
||||
});
|
||||
if (process.env.QL3_LIVE_EXPECT_MISSING === 'true') {
|
||||
await observeMissing(provider);
|
||||
return;
|
||||
}
|
||||
await observeRotation(provider);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(
|
||||
`QL3 mounted Secret live actor failed: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
FIRST_OBSERVATION,
|
||||
SECRET_REF,
|
||||
authority,
|
||||
digest,
|
||||
};
|
||||
@@ -5,6 +5,8 @@ const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
FIXTURE,
|
||||
LEGACY_FIXTURE,
|
||||
LEGACY_REQUIRED_GATES,
|
||||
REQUIRED_GATES,
|
||||
validatePluginPackageSecretBindingKubernetesLiveReport,
|
||||
} = require('../../scripts/ql3-plugin-package-secret-binding-kubernetes-live-audit.cjs');
|
||||
@@ -12,7 +14,7 @@ const {
|
||||
function report() {
|
||||
const digest = 'a'.repeat(64);
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
schemaVersion: 2,
|
||||
fixture: FIXTURE,
|
||||
observedAtMs: 1,
|
||||
platform: {
|
||||
@@ -21,6 +23,7 @@ function report() {
|
||||
nodeCount: 3,
|
||||
postgresVersionNumber: 180004,
|
||||
adminImageId: `sha256:${digest}`,
|
||||
controlImageId: `sha256:${'9'.repeat(64)}`,
|
||||
},
|
||||
management: {
|
||||
replicas: 2,
|
||||
@@ -70,14 +73,48 @@ function report() {
|
||||
executionSucceeded: true,
|
||||
sensitiveMatchCount: 0,
|
||||
},
|
||||
provider: {
|
||||
provider: 'mounted-files',
|
||||
replicas: 2,
|
||||
distinctNodeHashes: [
|
||||
`sha256:${'a'.repeat(64)}`,
|
||||
`sha256:${'b'.repeat(64)}`,
|
||||
],
|
||||
serviceAccountTokenMounted: false,
|
||||
canGetSecrets: false,
|
||||
canListSecrets: false,
|
||||
canPatchSecrets: false,
|
||||
projectionReadOnly: true,
|
||||
projectionMode: '0440',
|
||||
firstGenerationObserved: 2,
|
||||
rotatedGenerationObserved: 2,
|
||||
resourceVersionAdvanced: true,
|
||||
outputSensitiveFree: true,
|
||||
missingProjectionRejected: true,
|
||||
missingErrorCode: 'QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE',
|
||||
},
|
||||
gates: Object.fromEntries(REQUIRED_GATES.map((gate) => [gate, true])),
|
||||
limitations: [
|
||||
'single-server k3s control plane is not Kubernetes control-plane HA evidence',
|
||||
'PostgreSQL physical failover is proven by the independent 125-gate HA contract',
|
||||
'the mounted-files gate proves Kubernetes projection, not a direct Vault KMS or HSM adapter',
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function legacyReport() {
|
||||
const value = report();
|
||||
value.schemaVersion = 1;
|
||||
value.fixture = LEGACY_FIXTURE;
|
||||
delete value.platform.controlImageId;
|
||||
delete value.provider;
|
||||
value.gates = Object.fromEntries(
|
||||
LEGACY_REQUIRED_GATES.map((gate) => [gate, true]),
|
||||
);
|
||||
value.limitations = value.limitations.slice(0, 2);
|
||||
return value;
|
||||
}
|
||||
|
||||
test('accepts one exact low-sensitive Secret binding Kubernetes report', () => {
|
||||
assert.deepEqual(
|
||||
validatePluginPackageSecretBindingKubernetesLiveReport(report()).findings,
|
||||
@@ -85,15 +122,25 @@ test('accepts one exact low-sensitive Secret binding Kubernetes report', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('continues to verify the immutable v1 report shape', () => {
|
||||
assert.deepEqual(
|
||||
validatePluginPackageSecretBindingKubernetesLiveReport(legacyReport())
|
||||
.findings,
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects false gates, topology drift and sensitive material', () => {
|
||||
const invalid = report();
|
||||
invalid.gates.realExecutorJob = false;
|
||||
invalid.management.distinctNodeHashes[1] =
|
||||
invalid.management.distinctNodeHashes[0];
|
||||
invalid.provider.missingProjectionRejected = false;
|
||||
invalid.executor.secretRef = 'qlsecret:v1:forbidden';
|
||||
const findings =
|
||||
validatePluginPackageSecretBindingKubernetesLiveReport(invalid).findings;
|
||||
assert.ok(findings.some((value) => value.includes('management')));
|
||||
assert.ok(findings.some((value) => value.includes('gates')));
|
||||
assert.ok(findings.some((value) => value.includes('provider')));
|
||||
assert.ok(findings.some((value) => value.includes('forbidden')));
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user