From ac0992081efa09d9cc7aa98e3a9afd241443df51 Mon Sep 17 00:00:00 2001 From: whyour Date: Wed, 26 Aug 2026 03:31:13 +0800 Subject: [PATCH] feat(ql3): validate cluster credential key rotation --- deploy/kubernetes/ql3-cluster/README.md | 21 +- .../ql3-cluster/base/deployment.yaml | 9 +- .../security-administration/base/job.yaml | 6 +- .../input-secret.example.yaml | 3 +- .../cloudnative-pg/credentials.example.yaml | 3 +- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 6 +- ...-kubernetes-security-administration-job.md | 4 +- ...d-cluster-api-credential-pepper-keyring.md | 8 +- docs/operations/ql3-alpha-candidate.md | 12 + .../ql3-cluster-security-administration.md | 4 +- ...usterAdministrationKubernetesInputStage.ts | 2 +- ...dministrationKubernetesInputStage.test.cjs | 31 +- ...ql3-cloudnativepg-barman-live-contract.cjs | 34 +- .../ql3-cloudnativepg-deployment-audit.cjs | 7 +- scripts/ql3-cloudnativepg-live-contract.cjs | 6 +- scripts/ql3-cluster-deployment-audit.cjs | 14 +- ...gin-package-recovery-e2e-live-contract.cjs | 38 +- ...curity-administration-kubernetes-audit.cjs | 11 +- ...y-administration-kubernetes-live-audit.cjs | 87 +- ...dministration-kubernetes-live-contract.cjs | 820 ++++++++++++++++-- ...orker-kubernetes-rollout-live-contract.cjs | 56 +- ...AdministrationKubernetesLiveAudit.test.cjs | 55 +- ...inistrationKubernetesLiveContract.test.cjs | 49 +- 23 files changed, 1119 insertions(+), 167 deletions(-) diff --git a/deploy/kubernetes/ql3-cluster/README.md b/deploy/kubernetes/ql3-cluster/README.md index 9acfb2e1..c9ea1ff8 100644 --- a/deploy/kubernetes/ql3-cluster/README.md +++ b/deploy/kubernetes/ql3-cluster/README.md @@ -958,22 +958,25 @@ stringData: -----BEGIN CERTIFICATE----- REPLACE_WITH_OPERATOR_CA_BUNDLE -----END CERTIFICATE----- - api-credential-pepper: REPLACE_WITH_CANONICAL_32_BYTE_BASE64URL + api-credential-pepper-keyring.json: | + {"schemaVersion":1,"activePepperKeyId":"legacy-v1","keys":[{"pepperKeyId":"legacy-v1","pepper":"REPLACE_WITH_CANONICAL_32_BYTE_BASE64URL"}]} ``` The URL must not contain `ssl*` query parameters. TLS is configured separately and is fixed to `verify-full` in the committed deployment. The servername is mandatory, must be an explicit DNS name rather than an IP literal, and must match the endpoint certificate SAN. Only -`postgres-ca.crt` is projected from this Secret into the runtime trust mount; -the URL, servername and pepper remain environment-only values. The CA loader -requires an absolute path to a regular file that is not group/world writable, -1–256 KiB, and contains 1–16 unique PEM X.509 CA certificates with no trailing -data. +`postgres-ca.crt` and `api-credential-pepper-keyring.json` are projected from +this Secret into the runtime private mount; only the URL and servername remain +environment values. The keyring is a canonical, bounded 1–2 generation file; +the singleton above is the bootstrap form. The CA loader requires an absolute +path to a regular file that is not group/world writable, 1–256 KiB, and +contains 1–16 unique PEM X.509 CA certificates with no trailing data. -The trust bundle is loaded once for each new application activation. Rotate the -Secret and perform a controlled Deployment rollout; an active Pool never -silently changes trust roots in place. Use the +The trust bundle and API credential keyring are loaded once for each new +application activation. Rotate the Secret and perform a controlled Deployment +rollout; an active process never silently changes either authority in place. +Use the [old → overlap → new runbook](operations/postgres-ca-rotation.md) and its `audit:postgres-ca-overlap:ql3` preflight rather than replacing a trust root in one step. diff --git a/deploy/kubernetes/ql3-cluster/base/deployment.yaml b/deploy/kubernetes/ql3-cluster/base/deployment.yaml index 0f3f6e07..98d798cb 100644 --- a/deploy/kubernetes/ql3-cluster/base/deployment.yaml +++ b/deploy/kubernetes/ql3-cluster/base/deployment.yaml @@ -113,11 +113,8 @@ spec: secretKeyRef: name: ql3-cluster-control-runtime key: postgres-tls-servername - - name: QL3_API_CREDENTIAL_PEPPER - valueFrom: - secretKeyRef: - name: ql3-cluster-control-runtime - key: api-credential-pepper + - name: QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE + value: /var/run/secrets/qinglong3/postgres-runtime/api-credential-pepper-keyring.json - name: QL3_POSTGRES_WORKER_INGRESS_URL valueFrom: secretKeyRef: @@ -250,6 +247,8 @@ spec: items: - key: postgres-ca.crt path: ca.crt + - key: api-credential-pepper-keyring.json + path: api-credential-pepper-keyring.json - name: postgres-worker-ingress-ca secret: secretName: ql3-cluster-worker-ingress diff --git a/deploy/kubernetes/ql3-cluster/operations/security-administration/base/job.yaml b/deploy/kubernetes/ql3-cluster/operations/security-administration/base/job.yaml index 291e431e..e51e86fd 100644 --- a/deploy/kubernetes/ql3-cluster/operations/security-administration/base/job.yaml +++ b/deploy/kubernetes/ql3-cluster/operations/security-administration/base/job.yaml @@ -72,7 +72,7 @@ spec: - --command=/var/run/qinglong3/security-administration-private/input/command.json - --assertion=/var/run/qinglong3/security-administration-private/input/assertion.jwt - --keyset=/var/run/qinglong3/security-administration-private/input/keyset.json - - --pepper=/var/run/qinglong3/security-administration-private/input/pepper + - --pepper-keyring=/var/run/qinglong3/security-administration-private/input/pepper-keyring.json securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -120,8 +120,8 @@ spec: path: assertion.jwt - key: keyset.json path: keyset.json - - key: pepper - path: pepper + - key: pepper-keyring.json + path: pepper-keyring.json - name: private-input emptyDir: medium: Memory diff --git a/deploy/kubernetes/ql3-cluster/operations/security-administration/input-secret.example.yaml b/deploy/kubernetes/ql3-cluster/operations/security-administration/input-secret.example.yaml index 969b42cb..753b74ad 100644 --- a/deploy/kubernetes/ql3-cluster/operations/security-administration/input-secret.example.yaml +++ b/deploy/kubernetes/ql3-cluster/operations/security-administration/input-secret.example.yaml @@ -11,4 +11,5 @@ stringData: {"schemaVersion":1,"operation":"audit.list","request":{"limit":25,"filter":{"outcome":"allowed"}}} assertion.jwt: REPLACE_WITH_SHORT_LIVED_MULTI_FACTOR_ASSERTION keyset.json: REPLACE_WITH_PINNED_SECURITY_ADMINISTRATION_KEYSET - pepper: REPLACE_WITH_CANONICAL_32_BYTE_BASE64URL_API_CREDENTIAL_PEPPER + pepper-keyring.json: | + {"schemaVersion":1,"activePepperKeyId":"REPLACE_WITH_ACTIVE_KEY_ID","keys":[{"pepperKeyId":"REPLACE_WITH_ACTIVE_KEY_ID","pepper":"REPLACE_WITH_CANONICAL_32_BYTE_BASE64URL_API_CREDENTIAL_PEPPER"}]} diff --git a/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/credentials.example.yaml b/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/credentials.example.yaml index 40f17e98..2c75d201 100644 --- a/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/credentials.example.yaml +++ b/deploy/kubernetes/ql3-cluster/operators/cloudnative-pg/credentials.example.yaml @@ -175,7 +175,8 @@ metadata: namespace: qinglong3-system type: Opaque stringData: - api-credential-pepper: REPLACE_WITH_CANONICAL_32_BYTE_BASE64URL + api-credential-pepper-keyring.json: | + {"schemaVersion":1,"activePepperKeyId":"REPLACE_WITH_ACTIVE_KEY_ID","keys":[{"pepperKeyId":"REPLACE_WITH_ACTIVE_KEY_ID","pepper":"REPLACE_WITH_CANONICAL_32_BYTE_BASE64URL"}]} --- apiVersion: v1 kind: Secret diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 5eedb6f6..bd0a3165 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,9 +11,9 @@ 最新增量证据（2026-08-26）： -- Alpha 阶段产物门（已实现、首次远端产物待本提交 CI）：`QingLong 3.0 CI` 的显式 `produce_alpha_artifacts` 门只在手动里程碑运行中归档已经通过原生测试的镜像，而不把普通 push/PR 的中间构建冒充发布。Local amd64/arm64 Docker archive 必须先通过 OS vulnerability policy、SBOM/实际库存复核、128 MiB entrypoint、Edge/Standalone fresh SQLite lifecycle 和真实 Local API cancellation；Cluster control/control-ai/admin/worker 的双架构 archive 必须通过对应 vulnerability、non-root、SBOM inventory 门，Admin 额外通过产品 facade smoke。每个 30 天 artifact 绑定完整 source SHA、3.0 版本、架构、image ID、archive SHA-256、CycloneDX SBOM、已通过 gate 和试运行/回退说明。Local 可称为用户隔离试运行候选；Cluster 仅是 integration candidate，因为离线 tag 尚未满足 public GHCR digest、catalog、deployment-lock、生产 HA/DR/CSI/IdP 门。只有实际远端 run 成功并记录 artifact 名/digest 后，才把本阶段标记为“已有可下载产物”；当前仍不能宣称公开 3.0 release。 +- Alpha 阶段产物门（已有本地可加载候选，远端归档待显式授权）：`QingLong 3.0 CI` 的显式 `produce_alpha_artifacts` 门只在手动里程碑运行中归档已经通过原生测试的镜像，而不把普通 push/PR 的中间构建冒充发布。source `b45a5e04b7f49ffdadd5117b6b5253c6f1c05430` 已产出 Local arm64 Docker archive（SHA-256 `58bbc250833c9e86321718aea70ac0a637699b84c18531fa7a82b35e90b7fa83`）、CycloneDX SBOM、manifest、verification evidence 与 checksum；重新 `docker load` 后的 image ID/architecture/user、read-only/no-network smoke、128 MiB Edge/Standalone lifecycle、SQLite integrity 与 HIGH/CRITICAL=0 已复验。对应远端 CI 40/40 与原生 arm64 Local image job 已通过，但因尚未授权向 GitHub 上传源码派生镜像/SBOM，当前是 owner-private 的本地阶段产物，不是公开可下载 release。Local 可称为用户隔离试运行候选；Cluster 仍只是 integration candidate，因为离线 tag 尚未满足 public GHCR digest、catalog、deployment-lock、生产 HA/DR/CSI/IdP 门。 -- D-407/ADR-0502（代码门已通过，Kubernetes 双代 live ceremony 待验收）：Cluster API credential pepper 从“数据库保存 key ID、运行时却只有一个固定 material”收敛为最多 old/new 两代的显式 keyring。Security Administration 只用 active key 签发并持久化 exact ID；Cluster Control 按 credential record 精确选一把 key，未知 ID/material 一律 unavailable，绝不 fallback 或遍历，因此认证热路径仍为一次摘要。旧 raw pepper 只通过 `legacy-v1` singleton bridge 保持兼容。新增 `pepper.references` 以数据库时间返回最多 64 个当前 latest active/unexpired credential ID 和 `hasMore`，只作为退休前检查，不执行删除。keyring 文件有 2 KiB、canonical/no-symlink/private/stable-read 边界，无 watcher/timer/新连接池；Edge/Standalone package、依赖与常驻资源零变化。`cluster-admin` 完整回归为 `458 total / 455 pass / 3 conditional skip / 0 fail`，`cluster-control` 为 `281 total / 279 pass / 2 conditional skip / 0 fail`，D-407 聚焦构建与 33 项测试通过；`3.0.0-alpha.1` 身份下 18-package clean build/test 退出 0，backend 为 `1592 total / 1590 pass / 2 conditional skip / 0 fail`。当前 D-406 Kubernetes stager 仍只支持单 pepper，material GC、持久 active catalog、索引/大规模查询计划、远程 UI/API、双人复核和真实 K3s/CNPG overlap→activate→contract 仍是后续门禁。 +- D-407/ADR-0502（代码与静态门已通过，Kubernetes 双代 live ceremony 待验收）：Cluster API credential pepper 从“数据库保存 key ID、运行时却只有一个固定 material”收敛为最多 old/new 两代的显式 keyring。Security Administration 只用 active key 签发并持久化 exact ID；Cluster Control 按 credential record 精确选一把 key，未知 ID/material 一律 unavailable，绝不 fallback 或遍历，因此认证热路径仍为一次摘要。旧 raw pepper 只通过 `legacy-v1` singleton bridge 保持通用 CLI/进程兼容；Kubernetes Job 和常驻 Cluster Control manifest 已统一为 keyring-only，不再维护第二套单值 Secret 注入模式。新增 `pepper.references` 以数据库时间返回最多 64 个当前 latest active/unexpired credential ID 和 `hasMore`，只作为退休前检查，不执行删除。keyring 文件有 2 KiB、canonical/no-symlink/private/stable-read 边界，无 watcher/timer/新连接池；Edge/Standalone package、依赖与常驻资源零变化。三节点 live 合同现已编码 overlap-old-active、activate-new、旧代引用 1→0、contract-new，并在每个阶段滚动真实双副本 Cluster Control：有效但无 Project role 的 old/new credential 必须到达生产 HTTP 认证面后返回 403，收缩后的旧 credential 必须返回 401。content-free schema 固定 2→1 代收缩、数据库跨两个 credential 的 1 个旧代/3 个新代版本事实、四次授权拒绝与一次认证拒绝；本轮 Kubernetes/report/stager 聚焦测试 23/23，Security Administration、Cluster Deployment、CloudNativePG 三个静态审计及 54 项相关共享 live fixture 测试通过。18-package clean build/test 已完成；Worker Runtime 的 3 个 loopback 用例在沙箱内因 `listen EPERM` 失败后，于允许 loopback 的同一工作区重跑为 `135/135`。完整 `cluster-admin` 为 `459 total / 456 pass / 3 conditional skip / 0 fail`，`cluster-control` 为 `281 total / 279 pass / 2 conditional skip / 0 fail`，最新 backend 为 `1594 total / 1592 pass / 2 conditional skip / 0 fail`。material GC、持久 active catalog、索引/大规模查询计划、远程 UI/API、双人复核和真实 K3s/CNPG live 验收仍是后续门禁。 - D-406/ADR-0501（已验收）：Cluster Security Administration 现在有可选的一次性 Kubernetes Job，而不是要求每个部署者自行拼装。通用 PostgreSQL、CloudNativePG、credential delivery 与组合入口均显式 opt-in，不进入共享 operations；因此 Edge/Standalone 以及默认 Cluster 的 package、依赖、启动路径和常驻资源仍为零增量。Job 固定 non-root/read-only/drop-all、无 ServiceAccount token/RBAC、`backoffLimit=0`、300 秒 deadline、600 秒 TTL、每容器 25m/48 MiB request 和 250m/128 MiB limit；主命令仍只打开一个 admin PostgreSQL connection。stager 保留 kubelet Secret symlink 兼容性，同时以 realpath confinement、`O_NOFOLLOW`、读前/读后 inode 复验和独立大小上限把四个输入复制成 memory-backed `0700/0600` 私有边界，失败时主容器不可启动；issue/rotate 才选择 PVC delivery component，token 只进入唯一 `0600` no-replace 文件。真实 arm64 live gate 已在 1 control-plane + 2 worker 的 K3s `v1.34.3+k3s1`、Flannel、CloudNativePG 1.30.0、3 个 PostgreSQL 18.4 实例、migration 71/control-core capability 70 上完成 register、audit query、issue、response-loss exact replay、rotate、revoke；证明 kubelet `0440` immutable Secret 投影可被私有化、`0444` 扩权会在主容器启动前失败、RWO PVC 跨 Job 保留 `0700/0600` no-replace 交付、Kubernetes API/公网 egress 与 Secret/Job RBAC 均被拒绝。全部管理 Job 继续以 UID/GID 10001 运行；仅 live local-path fixture 用一个无网络、无 API token、drop-all 的 root Job 把实现特有的 `02777 root:10001` PVC 根收紧为 `02770`，不能据此声称生产 CSI 加密或 custody 已验收。content-free `0600` 报告独立审计为 `compatible=true/findings=[]`，SHA-256 为 `e5c24af77034e1a2efee062107176e218c11a9f9f0d6c8c7308fdc280b0a82cf`；18-package clean build/test 退出 0，当前 `cluster-admin` 为 `456 total / 453 pass / 3 conditional skip / 0 fail`，backend 为 `1590 total / 1588 pass / 2 conditional skip / 0 fail`。D-406 关闭单主机 K3s/CNPG/PVC 的产品命令与权限边界门，但不冒充生产 Kubernetes control-plane HA、跨主机 STONITH/DR、加密 CSI、外部 IdP、双人复核/break-glass、pepper rotation、audit retention/export/alert、并发 dispatch 或远程 UI/API。 @@ -10082,7 +10082,7 @@ PR-8 的本机最新增量由 ADR-0075/0076/0077/0078/0079/0080/0081/0082/0083/0 ADR-0087 Owner package 更新同样适用于上段 PR-8 累计描述：现行产品 CLI 只能经 console facade 到达其内部 bootstrap/credential-recovery，三个历史 ceremony package 名都只表示旧切片；`ql3-owner-gc` 由 maintenance 直接提供，不再拥有独立 importer。 -PR-8 的 cluster ADR-0049 未完成项由 ADR-0050/0051/0500/0501 继续收敛：Identity register/enable/disable、credential issue/rotate/revoke、mutation ledger、强 actor、同事务 audit、有界 audit query，以及常驻 `/api/v3` 的认证前 overload shield 已孵化完成；D-405 新增默认无 listener、一次只执行一个命令并关闭单连接 admin authority 的 `ql3-security-admin` 产品 CLI，credential token 只向私有 no-replace 文件交付。D-406 又补齐默认不安装、无 Kubernetes API authority 的一次性 Job、私密输入 stager、CloudNativePG 与 PVC delivery 契约，并已由真实三节点 K3s、三实例 PostgreSQL 18.4、response-loss replay、PVC custody、网络/RBAC 拒绝与失败清理 live ceremony 验收。该单 Docker host fixture 仍不证明生产 control-plane HA、跨主机 STONITH/DR 或加密 CSI；远程 API/UI、管理入口独立 rate limit、双人复核或 break-glass、pepper rotation、audit retention/export/alert 也仍未完成。不得把短生命周期 CLI、一次性 Job、application service 或 process-local HTTP shield 的存在解释为 cluster-control 已获得管理 authority 或全局 quota。 +PR-8 的 cluster ADR-0049 未完成项由 ADR-0050/0051/0500/0501/0502 继续收敛：Identity register/enable/disable、credential issue/rotate/revoke、mutation ledger、强 actor、同事务 audit、有界 audit query，以及常驻 `/api/v3` 的认证前 overload shield 已孵化完成；D-405 新增默认无 listener、一次只执行一个命令并关闭单连接 admin authority 的 `ql3-security-admin` 产品 CLI，credential token 只向私有 no-replace 文件交付。D-406 又补齐默认不安装、无 Kubernetes API authority 的一次性 Job、私密输入 stager、CloudNativePG 与 PVC delivery 契约，并已由真实三节点 K3s、三实例 PostgreSQL 18.4、response-loss replay、PVC custody、网络/RBAC 拒绝与失败清理 live ceremony 验收。D-407 进一步把常驻 Cluster Control 与 Admin Job 的 Kubernetes 配置收敛为最多 old/new 两代的 keyring-only 文件投影，保留一次精确摘要认证和显式滚动重启；overlap→activate→contract live 合同已经编码，但受审三节点门仍待执行。该单 Docker host fixture 仍不证明生产 control-plane HA、跨主机 STONITH/DR 或加密 CSI；远程 API/UI、管理入口独立 rate limit、双人复核或 break-glass、自动 rotation/material GC、audit retention/export/alert 也仍未完成。不得把短生命周期 CLI、一次性 Job、application service 或 process-local HTTP shield 的存在解释为 cluster-control 已获得管理 authority 或全局 quota。 未进入当前孵化切片的代码在通过对应 Gate 前必须保持不可达：不得仅因 schema、service 或 Primary 编排器已存在，就让旧 Controller、Scheduler、gRPC callback 或 Shell 脚本直接写入新状态表或调用新 Executor。已接入的 Shadow 观察只能通过默认关闭的 Feature Flag 和 origin owner 决策到达，不得调用 Executor、再次 spawn 或改变 Legacy 返回结果。manual `runSingle` 只增加 owner selection seam；默认没有 router。HTTP bootstrap 每次启动只读取一次 manifest，缺失、禁用、拒绝或非 primary 时保持 Legacy 且不加载重组件；显式 accepted manual primary 会在恢复门禁通过后安装唯一 owner，选中后禁止回退双跑。ADR-0445 已让 `ScheduleService.runTask` 的 subscription/system/script 在显式 origin flag 下只观察同一个 Legacy ChildProcess；system crond 的 `scheduled_system`、once/boot/grpc 与这些来源的 Primary owner 切换仍须独立门禁。 diff --git a/docs/adr/ADR-0501-opt-in-kubernetes-security-administration-job.md b/docs/adr/ADR-0501-opt-in-kubernetes-security-administration-job.md index f531d1f8..9f4a711c 100644 --- a/docs/adr/ADR-0501-opt-in-kubernetes-security-administration-job.md +++ b/docs/adr/ADR-0501-opt-in-kubernetes-security-administration-job.md @@ -23,7 +23,7 @@ ServiceAccount 和 Pod 都关闭自动 token 挂载，不创建 Role、RoleBindi Secret volume 必须兼容 kubelet 的版本目录与 symlink 投影，默认 `0440` 以便固定的非 root group 读取；但 ADR-0500 的命令拒绝 symlink 和 group/world 权限。因此在既有 `@qinglong/cluster-admin/security-administration` 内增加专用 init stager，而不新建 package。 -stager 只接受固定的 `command.json`、`assertion.jwt`、`keyset.json` 和 `pepper`，分别有 64 KiB、16 KiB、256 KiB 与 256 B 上限。它解析 kubelet symlink 后仍要求 realpath 留在投影 authority 内，以 `O_NOFOLLOW` 打开最终文件，复验类型、权限、大小和读前/读后 inode 状态，再清零源 Buffer。目标目录必须不存在，由 stager 创建为 `0700`；文件以 `0600`、`fsync` 和 hard-link no-replace 发布到 1 MiB memory-backed `emptyDir`。任何输入失败都清理已发布目标，主容器不会启动。 +stager 只接受固定的 `command.json`、`assertion.jwt`、`keyset.json` 和 `pepper-keyring.json`，分别有 64 KiB、16 KiB、256 KiB 与 2 KiB 上限。Kubernetes 路径以 D-407 keyring 为唯一 canonical 输入；旧单 pepper 只保留在通用 CLI 兼容桥，不形成第二套 Kubernetes 配置模式。stager 解析 kubelet symlink 后仍要求 realpath 留在投影 authority 内，以 `O_NOFOLLOW` 打开最终文件，复验类型、权限、大小和读前/读后 inode 状态，再清零源 Buffer。目标目录必须不存在，由 stager 创建为 `0700`；文件以 `0600`、`fsync` 和 hard-link no-replace 发布到 1 MiB memory-backed `emptyDir`。任何输入失败都清理已发布目标，主容器不会启动。 ### 3. 数据库和进程权限保持最小化 @@ -57,7 +57,7 @@ Identity 变更、revoke 和 audit query 使用无 delivery 的 base。只有 `c ## 验证 -- stager 聚焦测试覆盖真实 kubelet symlink 布局、`0700/0600` 收紧、持久 delivery 目录复验、realpath 逃逸、world-readable material、目标不可覆盖和 CLI 无敏感回显。 +- stager 聚焦测试覆盖真实 kubelet symlink 布局、2 KiB keyring 边界、`0700/0600` 收紧、持久 delivery 目录复验、realpath 逃逸、world-readable material、目标不可覆盖和 CLI 无敏感回显。 - 部署审计冻结无 API token/RBAC、caller-driven/零重试/deadline/TTL、non-root/read-only/drop-all、资源上限、固定 CLI、内存私有输入、独立 admin credential、CloudNativePG egress、PVC delivery 和默认聚合不可达；失败注入覆盖权限扩大、非持久 delivery 与误入共享 aggregate。 - `kubectl kustomize` 已分别渲染 base、CloudNativePG、credential-delivery 和 CloudNativePG + delivery 四个入口。 - 18-package clean build/test 退出 0；当前 `cluster-admin` 为 456 total / 453 pass / 3 conditional skip / 0 fail，backend 为 1590 total / 1588 pass / 2 conditional skip / 0 fail。 diff --git a/docs/adr/ADR-0502-bounded-cluster-api-credential-pepper-keyring.md b/docs/adr/ADR-0502-bounded-cluster-api-credential-pepper-keyring.md index 7c8b918e..08819cf3 100644 --- a/docs/adr/ADR-0502-bounded-cluster-api-credential-pepper-keyring.md +++ b/docs/adr/ADR-0502-bounded-cluster-api-credential-pepper-keyring.md @@ -17,7 +17,7 @@ QingLong 3.0 同时服务低配路由设备与集群节点。pepper rotation 只 `runtime-core` 提供 schema v1 keyring：一个 `activePepperKeyId` 和 1–2 个唯一 `{pepperKeyId,pepper}`。每个 ID 与 32-byte canonical base64url material 都使用既有 credential contract 校验，active ID 必须存在于 keys。原单 pepper 配置只通过显式 `legacy-v1` singleton bridge 保持兼容，不允许自动发现、环境合并或第三代历史 key。 -Cluster Control 可从 `QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE` 读取不超过 2 KiB 的 canonical 私有 JSON；它与旧 `QL3_API_CREDENTIAL_PEPPER` 必须二选一。文件必须是 canonical absolute regular file、不可为 symlink、不可向 group/world 开放，并在稳定 inode/mtime/size 下读取。运行时不安装 watcher；切换 keyring 后由部署系统执行受控滚动重启。 +Cluster Control 可从 `QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE` 读取不超过 2 KiB 的 canonical 私有 JSON；它与旧 `QL3_API_CREDENTIAL_PEPPER` 必须二选一。文件必须是 canonical absolute regular file、不可为 symlink、不可向 group/world 写入，并在稳定 inode/mtime/size 下读取。Kubernetes 以只读 Secret volume 和 Pod 专属 fsGroup 提供该文件。运行时不安装 watcher；切换 keyring 后由部署系统执行受控滚动重启。 ### 2. 认证严格按 durable key ID 选择 @@ -70,10 +70,12 @@ CLI 在 `--pepper` 与 `--pepper-keyring` 中必须精确选择一个。keyring - Cluster Control 覆盖 old/new overlap、stored key exact selection、unknown stored key fail-closed、旧环境变量 bridge、私有 keyring 文件与双来源拒绝； - Security Administration 覆盖新 credential 绑定 active key、keyring CLI、引用查询 exact command/result 与旧单 pepper bridge； - PostgreSQL repository 覆盖数据库时间、latest active/unexpired 过滤、limit+1、空引用与畸形行 fail-closed； -- 当前 `cluster-admin` 完整回归为 458 total / 455 pass / 3 conditional skip / 0 fail，`cluster-control` 为 281 total / 279 pass / 2 conditional skip / 0 fail；D-407 聚焦构建与 33 项测试通过，`3.0.0-alpha.1` 身份下 18-package clean build/test 退出 0，backend 为 1592 total / 1590 pass / 2 conditional skip / 0 fail。 +- Kubernetes Security Administration 只投影 `pepper-keyring.json`，stager 以 2 KiB 边界私有化到 memory-backed `0700/0600` 目录；常驻 Cluster Control 同样只挂载 keyring 文件，不再从 Secret 注入旧单值环境变量； +- overlap-old-active、activate-new、旧代引用 1→0 与 contract-new 的三节点 K3s/CloudNativePG live 合同已经编码；它在三次 rollout 中保持两个反亲和 Cluster Control 副本，并以真实 `/api/v3` 请求证明 old/new 在 overlap 期间认证成功但因无 Project role 返回 403、old 在 contract 后返回 401、new 仍返回 403。content-free schema 只记录状态码、引用计数、1 个旧代/3 个新代 durable version 与审计计数；远程 live 门通过前不得宣称该 ceremony 已验收； +- 当前 `cluster-admin` 完整回归为 459 total / 456 pass / 3 conditional skip / 0 fail，`cluster-control` 为 281 total / 279 pass / 2 conditional skip / 0 fail；本轮 Kubernetes/report/stager 聚焦测试 23 项、相关共享 live fixture 静态测试 54 项以及三个部署审计通过，`3.0.0-alpha.1` 身份下 18-package clean build/test 完成（Worker Runtime 3 个 loopback 用例在沙箱外复验为 135/135），backend 为 1594 total / 1592 pass / 2 conditional skip / 0 fail。 ## 影响与剩余门禁 D-407 关闭了“数据库记录 key ID、运行时却只能使用一个固定 pepper”的结构性缺口。Edge/Standalone package、依赖和常驻路径零变化；Cluster 每个认证请求仍只解析 record 并计算一个摘要，管理引用检查只在显式短命令中打开一个 admin connection。 -D-406 Kubernetes stager/Job 仍只接受单 `pepper` 文件，不能用它执行双代 ceremony；在增加 keyring Secret 投影、失败注入和真实 K3s/CNPG overlap→activate→contract 验收前，Kubernetes 部署继续停留在 singleton bridge。远程管理 API/UI、双人复核/break-glass、material GC、audit retention/export/alert 和大规模引用查询计划也仍未完成。 +D-406 Kubernetes stager/Job 与常驻 Cluster Control manifest 已收敛为 keyring-only；旧单 pepper 只存在于通用 CLI/进程配置兼容桥。真实 K3s/CNPG overlap→activate→contract live 合同已实现但仍需在受审远程门中验收，因此当前不能把实现完成解释为 live ceremony 已关闭。远程管理 API/UI、双人复核/break-glass、material GC、audit retention/export/alert 和大规模引用查询计划也仍未完成。 diff --git a/docs/operations/ql3-alpha-candidate.md b/docs/operations/ql3-alpha-candidate.md index 15ac9f9b..a345e7f9 100644 --- a/docs/operations/ql3-alpha-candidate.md +++ b/docs/operations/ql3-alpha-candidate.md @@ -12,6 +12,18 @@ 只有第一等级可以称为本阶段“用户可试运行产物”。Cluster archive 是工程集成产物，因为离线 per-architecture tag 不满足正式 Kubernetes deployment-lock 的 GHCR immutable digest 与 catalog provenance。 +## 当前阶段实物（2026-08-26） + +当前已经存在一份 owner-private、可重新加载的 Local arm64 候选，而不是只有源码或 Dockerfile： + +- source revision：`b45a5e04b7f49ffdadd5117b6b5253c6f1c05430`； +- image：`qinglong3-local-application:ci-arm64`，image ID `sha256:59e39cd0c71e5a5c2bc99c599d5aa240c59f215008f8d12fde4243c984274426`； +- archive SHA-256：`58bbc250833c9e86321718aea70ac0a637699b84c18531fa7a82b35e90b7fa83`； +- 同目录包含 `manifest.json`、`verification-evidence.json`、CycloneDX 1.5 SBOM、`README.md` 与 `SHA256SUMS`，全套 checksum 和 `docker load` 后身份/smoke 已复验； +- HIGH/CRITICAL OS vulnerability 为 0；128 MiB、0.5 CPU、read-only、no-network、drop-all 下的 Edge/Standalone fresh lifecycle 与 SQLite integrity 已通过；原生 Linux arm64 CI 另行覆盖 macOS bind-mount 无法等价证明的 Local API cancellation。 + +该实物保存在工作区忽略目录，不进入 Git，也尚未上传 GitHub。远端 40/40 CI 与原生 arm64 image job 已通过；公开下载仍需维护者明确授权上传。它足以作为单架构内部试运行材料，但在 amd64 同级 archive 和远端 artifact identity 未齐全前，不得把它升级为完整双架构阶段里程碑或公开 release。 + ## 生成 在 GitHub Actions 手动运行 `QingLong 3.0 CI`，选择目标 `next` 提交并设置 `produce_alpha_artifacts=true`。普通 push/PR 不上传大镜像，避免每次开发提交都制造伪里程碑和额外存储成本。 diff --git a/docs/operations/ql3-cluster-security-administration.md b/docs/operations/ql3-cluster-security-administration.md index bbb70e99..dbae3b33 100644 --- a/docs/operations/ql3-cluster-security-administration.md +++ b/docs/operations/ql3-cluster-security-administration.md @@ -170,7 +170,7 @@ Cluster Control 使用 `QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE`；它与旧 `QL3 - `credential-delivery`：在 base 上增加调用方提供的 RWO PVC； - `cloudnative-pg-credential-delivery`：CloudNativePG 与 PVC 交付的组合。 -把 `input-secret.example.yaml` 复制到仓库外的私有目录，替换四个占位值，并保持 `immutable: true`。示例不属于任何 Kustomization。非签发操作不要选择 delivery overlay；`credential.issue` / `credential.rotate` 必须先按 `delivery-pvc.example.yaml` 创建受加密、受访问控制的 PVC，并把 manifest 中的 `replace-with-unique-delivery.json` 改为本次唯一文件名。 +把 `input-secret.example.yaml` 复制到仓库外的私有目录，替换四个占位值，并保持 `immutable: true`。第四项必须是最多 old/new 两代、2 KiB 内的 canonical `pepper-keyring.json`；Kubernetes 不接受旧单 pepper 文件。示例不属于任何 Kustomization。非签发操作不要选择 delivery overlay；`credential.issue` / `credential.rotate` 必须先按 `delivery-pvc.example.yaml` 创建受加密、受访问控制的 PVC，并把 manifest 中的 `replace-with-unique-delivery.json` 改为本次唯一文件名。 以 CloudNativePG 的无 delivery audit query 为例： @@ -189,4 +189,4 @@ kubectl logs job/ql3-security-administration -n qinglong3-system \ ## 当前边界 -本入口没有远程 API/UI、双人复核或 break-glass、自动 pepper rotation/material GC、audit retention/export/alert。D-407 已提供 old/new 双代 keyring、active issuance、exact-key authentication 和退休前引用检查，但 active 切换仍由显式配置更新加滚动重启完成。现有 Kubernetes Job stager 只接受单 pepper，尚未完成 keyring Secret 投影和真实 overlap→activate→contract live gate，因此不能用 D-406 模板宣称 Kubernetes pepper rotation 已完成。可选 Job 的静态契约与单主机 K3s + PostgreSQL/PVC ceremony 已验收，但仍不默认安装，也不证明生产基础设施 HA/DR 或存储加密；admin database credential 始终不得进入常驻 Cluster Control。命令决策见 [ADR-0500](../adr/ADR-0500-short-lived-cluster-security-administration-command.md)，部署决策见 [ADR-0501](../adr/ADR-0501-opt-in-kubernetes-security-administration-job.md)，双代 keyring 见 [ADR-0502](../adr/ADR-0502-bounded-cluster-api-credential-pepper-keyring.md)。 +本入口没有远程管理 API/UI、双人复核或 break-glass、自动 pepper rotation/material GC、audit retention/export/alert。D-407 已提供 old/new 双代 keyring、active issuance、exact-key authentication 和退休前引用检查；active 切换仍由显式 Secret 更新加受控滚动重启完成。Kubernetes Job stager 与常驻 Cluster Control manifest 已收敛为 keyring-only；overlap→activate→contract live 合同会在三次 rollout 中保持两个反亲和 Control 副本，并用真实 `/api/v3` 请求区分“认证成功但未授权”的 403 与旧 credential 收缩后被拒绝的 401。该合同已编码并通过聚焦/静态门，但远程三节点 K3s/CNPG live 验收仍待执行，因此不能宣称 Kubernetes pepper rotation 已完成；它的内部 HTTP probe 也不替代外部 ingress TLS 验收。可选 Job 仍不默认安装，也不证明生产基础设施 HA/DR 或存储加密；admin database credential 始终不得进入常驻 Cluster Control。命令决策见 [ADR-0500](../adr/ADR-0500-short-lived-cluster-security-administration-command.md)，部署决策见 [ADR-0501](../adr/ADR-0501-opt-in-kubernetes-security-administration-job.md)，双代 keyring 见 [ADR-0502](../adr/ADR-0502-bounded-cluster-api-credential-pepper-keyring.md)。 diff --git a/packages/ql3-cluster-admin/src/security-administration/clusterAdministrationKubernetesInputStage.ts b/packages/ql3-cluster-admin/src/security-administration/clusterAdministrationKubernetesInputStage.ts index 918f7617..2ffd6fec 100644 --- a/packages/ql3-cluster-admin/src/security-administration/clusterAdministrationKubernetesInputStage.ts +++ b/packages/ql3-cluster-admin/src/security-administration/clusterAdministrationKubernetesInputStage.ts @@ -27,7 +27,7 @@ const INPUTS = Object.freeze([ Object.freeze({ name: 'command.json', maximumBytes: 64 * 1024 }), Object.freeze({ name: 'assertion.jwt', maximumBytes: 16 * 1024 }), Object.freeze({ name: 'keyset.json', maximumBytes: 256 * 1024 }), - Object.freeze({ name: 'pepper', maximumBytes: 256 }), + Object.freeze({ name: 'pepper-keyring.json', maximumBytes: 2 * 1024 }), ]); export interface ClusterAdministrationKubernetesInputStagePaths { diff --git a/packages/ql3-cluster-admin/test/clusterAdministrationKubernetesInputStage.test.cjs b/packages/ql3-cluster-admin/test/clusterAdministrationKubernetesInputStage.test.cjs index c96a2a6f..794425e4 100644 --- a/packages/ql3-cluster-admin/test/clusterAdministrationKubernetesInputStage.test.cjs +++ b/packages/ql3-cluster-admin/test/clusterAdministrationKubernetesInputStage.test.cjs @@ -39,7 +39,11 @@ function projectedInput() { 'command.json': '{"schemaVersion":1,"operation":"audit.list"}\n', 'assertion.jwt': 'signed.assertion.value', 'keyset.json': '{"keys":[]}', - pepper: 'A'.repeat(43), + 'pepper-keyring.json': JSON.stringify({ + schemaVersion: 1, + activePepperKeyId: 'legacy-v1', + keys: [{ pepperKeyId: 'legacy-v1', pepper: 'A'.repeat(43) }], + }), }; for (const [name, value] of Object.entries(inputs)) { const versionFile = join(versionDirectory, name); @@ -88,6 +92,31 @@ test('copies a Kubernetes projected Secret into a private immutable input bounda assert.equal(JSON.stringify(result).includes('A'.repeat(43)), false); }); +test('rejects a projected pepper keyring larger than the runtime boundary', () => { + const fixture = projectedInput(); + const keyringFile = resolve( + fixture.sourceDirectory, + '..data', + 'pepper-keyring.json', + ); + chmodSync(keyringFile, 0o640); + writeFileSync( + keyringFile, + 'A'.repeat(2 * 1024 + 1), + ); + chmodSync(keyringFile, 0o440); + + assert.throws( + () => + stageClusterAdministrationKubernetesInputs({ + sourceDirectory: fixture.sourceDirectory, + targetDirectory: fixture.targetDirectory, + }), + /file authority is invalid/, + ); + assert.throws(() => lstatSync(fixture.targetDirectory)); +}); + test('accepts only the exact root-owned sticky Kubernetes mount authority', () => { assert.equal( isClusterAdministrationProjectedSourceDirectoryAuthority({ diff --git a/scripts/ql3-cloudnativepg-barman-live-contract.cjs b/scripts/ql3-cloudnativepg-barman-live-contract.cjs index bc80433a..110b523e 100644 --- a/scripts/ql3-cloudnativepg-barman-live-contract.cjs +++ b/scripts/ql3-cloudnativepg-barman-live-contract.cjs @@ -676,7 +676,14 @@ function postgresRestoreApplicationProbeResources({ kind: 'List', items: Object.freeze([ kubernetesSecret('ql3-dr', secretName, { - 'api-credential-pepper': apiCredentialPepper, + 'api-credential-pepper-keyring.json': `${JSON.stringify({ + schemaVersion: 1, + activePepperKeyId: 'legacy-v1', + keys: [{ + pepperKeyId: 'legacy-v1', + pepper: apiCredentialPepper, + }], + })}\n`, }), { apiVersion: 'apps/v1', @@ -753,11 +760,10 @@ function postgresRestoreApplicationProbeResources({ value: `qinglong3-dr-${suffix}`, }, { name: 'QL3_POSTGRES_MAX_CONNECTIONS', value: '2' }, - secretValue( - 'QL3_API_CREDENTIAL_PEPPER', - secretName, - 'api-credential-pepper', - ), + { + name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE', + value: '/var/run/secrets/qinglong3/api-credential/keyring.json', + }, ], ports: [{ name: 'http', containerPort: 5800 }], startupProbe: { @@ -794,6 +800,11 @@ function postgresRestoreApplicationProbeResources({ mountPath: '/var/run/secrets/qinglong3/postgres', readOnly: true, }, + { + name: 'api-credential-keyring', + mountPath: '/var/run/secrets/qinglong3/api-credential', + readOnly: true, + }, ], }, ], @@ -810,6 +821,17 @@ function postgresRestoreApplicationProbeResources({ items: [{ key: 'ca.crt', path: 'ca.crt' }], }, }, + { + name: 'api-credential-keyring', + secret: { + secretName, + defaultMode: 288, + items: [{ + key: 'api-credential-pepper-keyring.json', + path: 'keyring.json', + }], + }, + }, ], }, }, diff --git a/scripts/ql3-cloudnativepg-deployment-audit.cjs b/scripts/ql3-cloudnativepg-deployment-audit.cjs index 535daa59..4f232f63 100644 --- a/scripts/ql3-cloudnativepg-deployment-audit.cjs +++ b/scripts/ql3-cloudnativepg-deployment-audit.cjs @@ -354,13 +354,13 @@ function assertSecretBoundary(readFile, root, findings) { if (name === 'ql3-cluster-control-runtime') { if ( secret?.type !== 'Opaque' || - secret?.stringData?.['api-credential-pepper'] !== - 'REPLACE_WITH_CANONICAL_32_BYTE_BASE64URL' + secret?.stringData?.['api-credential-pepper-keyring.json'] !== + '{"schemaVersion":1,"activePepperKeyId":"REPLACE_WITH_ACTIVE_KEY_ID","keys":[{"pepperKeyId":"REPLACE_WITH_ACTIVE_KEY_ID","pepper":"REPLACE_WITH_CANONICAL_32_BYTE_BASE64URL"}]}\n' ) { findings.push( finding( 'QL3_CNPG_RUNTIME_SECRET_EXAMPLE', - 'runtime Secret example may contain only the placeholder credential pepper', + 'runtime Secret example must contain only the bounded placeholder credential pepper keyring', ), ); } @@ -503,6 +503,7 @@ function assertMigrationBinding(readFile, root, findings) { env.has('QL3_POSTGRES_MIGRATION_URL') || env.has('QL3_POSTGRES_RUNTIME_URL') || env.has('QL3_API_CREDENTIAL_PEPPER') || + env.has('QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE') || env.get('QL3_POSTGRES_MIGRATION_HOST')?.value !== PRIMARY_DNS || env.get('QL3_POSTGRES_MIGRATION_PORT')?.value !== '5432' || env.get('QL3_POSTGRES_MIGRATION_DATABASE')?.value !== 'qinglong' || diff --git a/scripts/ql3-cloudnativepg-live-contract.cjs b/scripts/ql3-cloudnativepg-live-contract.cjs index b0872961..ed922e0e 100644 --- a/scripts/ql3-cloudnativepg-live-contract.cjs +++ b/scripts/ql3-cloudnativepg-live-contract.cjs @@ -760,7 +760,11 @@ async function main() { ); } applySecret('ql3-cluster-control-runtime', 'Opaque', { - 'api-credential-pepper': randomSecret(), + 'api-credential-pepper-keyring.json': `${JSON.stringify({ + schemaVersion: 1, + activePepperKeyId: 'legacy-v1', + keys: [{ pepperKeyId: 'legacy-v1', pepper: randomSecret() }], + })}\n`, }); const workerIngressTls = createWorkerIngressTls(tempDirectory); applySecret('ql3-cluster-worker-ingress', 'Opaque', { diff --git a/scripts/ql3-cluster-deployment-audit.cjs b/scripts/ql3-cluster-deployment-audit.cjs index 812f5378..25e1effc 100644 --- a/scripts/ql3-cluster-deployment-audit.cjs +++ b/scripts/ql3-cluster-deployment-audit.cjs @@ -796,6 +796,10 @@ function assertKubernetes(readFile, root, findings) { 'QL3_POSTGRES_TLS_CA_FILE', '/var/run/secrets/qinglong3/postgres-runtime/ca.crt', ], + [ + 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE', + '/var/run/secrets/qinglong3/postgres-runtime/api-credential-pepper-keyring.json', + ], ['QL3_WORKER_INGRESS_ENABLED', 'true'], ['QL3_WORKER_INGRESS_HOST', '0.0.0.0'], ['QL3_WORKER_INGRESS_PORT', '5801'], @@ -854,7 +858,14 @@ function assertKubernetes(readFile, root, findings) { runtimeCaVolume?.secret?.secretName !== 'ql3-cluster-control-runtime' || runtimeCaVolume?.secret?.defaultMode !== 0o444 || JSON.stringify(runtimeCaVolume?.secret?.items) !== - JSON.stringify([{ key: 'postgres-ca.crt', path: 'ca.crt' }]) + JSON.stringify([ + { key: 'postgres-ca.crt', path: 'ca.crt' }, + { + key: 'api-credential-pepper-keyring.json', + path: 'api-credential-pepper-keyring.json', + }, + ]) || + env.has('QL3_API_CREDENTIAL_PEPPER') ) { findings.push( finding( @@ -866,7 +877,6 @@ function assertKubernetes(readFile, root, findings) { for (const [name, key] of [ ['QL3_POSTGRES_RUNTIME_URL', 'postgres-runtime-url'], ['QL3_POSTGRES_TLS_SERVERNAME', 'postgres-tls-servername'], - ['QL3_API_CREDENTIAL_PEPPER', 'api-credential-pepper'], ]) { const secret = env.get(name)?.valueFrom?.secretKeyRef; if (secret?.name !== 'ql3-cluster-control-runtime' || secret?.key !== key) { diff --git a/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs b/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs index 887d2ecb..775be6d9 100644 --- a/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs +++ b/scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs @@ -1483,7 +1483,14 @@ function applyRuntimeAfterRecovery(recoveryJob, migrationJobValue, secrets) { type: 'Opaque', stringData: { 'postgres-runtime-url': `postgresql://ql3_runtime:${secrets.runtime}@${POSTGRES_NAME}:5432/qinglong`, - 'api-credential-pepper': randomBytes(32).toString('base64url'), + 'api-credential-pepper-keyring.json': `${JSON.stringify({ + schemaVersion: 1, + activePepperKeyId: 'legacy-v1', + keys: [{ + pepperKeyId: 'legacy-v1', + pepper: randomBytes(32).toString('base64url'), + }], + })}\n`, }, }, 'create runtime-only credential after recovery success', @@ -1535,21 +1542,34 @@ function applyRuntimeAfterRecovery(recoveryJob, migrationJobValue, secrets) { }, }, { - name: 'QL3_API_CREDENTIAL_PEPPER', - valueFrom: { - secretKeyRef: { - name: 'ql3-cluster-control-runtime', - key: 'api-credential-pepper', - }, - }, + name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE', + value: '/var/run/secrets/qinglong3/api-credential/keyring.json', + }, + ]; + container.volumeMounts = [ + { name: 'tmp', mountPath: '/tmp' }, + { + name: 'api-credential-keyring', + mountPath: '/var/run/secrets/qinglong3/api-credential', + readOnly: true, }, ]; - container.volumeMounts = [{ name: 'tmp', mountPath: '/tmp' }]; resource.spec.template.spec.volumes = [ { name: 'tmp', emptyDir: { medium: 'Memory', sizeLimit: '16Mi' }, }, + { + name: 'api-credential-keyring', + secret: { + secretName: 'ql3-cluster-control-runtime', + defaultMode: 288, + items: [{ + key: 'api-credential-pepper-keyring.json', + path: 'keyring.json', + }], + }, + }, ]; } for (const resource of resources) { diff --git a/scripts/ql3-security-administration-kubernetes-audit.cjs b/scripts/ql3-security-administration-kubernetes-audit.cjs index 83d05212..36f0781e 100644 --- a/scripts/ql3-security-administration-kubernetes-audit.cjs +++ b/scripts/ql3-security-administration-kubernetes-audit.cjs @@ -150,7 +150,7 @@ function auditSecurityAdministrationKubernetes(options = {}) { `--command=${PRIVATE_ROOT}/input/command.json`, `--assertion=${PRIVATE_ROOT}/input/assertion.jwt`, `--keyset=${PRIVATE_ROOT}/input/keyset.json`, - `--pepper=${PRIVATE_ROOT}/input/pepper`, + `--pepper-keyring=${PRIVATE_ROOT}/input/pepper-keyring.json`, ]; if ( JSON.stringify(stager?.command) !== @@ -181,7 +181,7 @@ function auditSecurityAdministrationKubernetes(options = {}) { 'command.json', 'assertion.jwt', 'keyset.json', - 'pepper', + 'pepper-keyring.json', ]) || privateInput?.emptyDir?.medium !== 'Memory' || privateInput?.emptyDir?.sizeLimit !== '1Mi' || @@ -241,7 +241,12 @@ function auditSecurityAdministrationKubernetes(options = {}) { inputExample?.metadata?.name !== 'ql3-security-administration-input' || JSON.stringify(Object.keys(inputExample?.stringData ?? {}).sort()) !== JSON.stringify( - ['command.json', 'assertion.jwt', 'keyset.json', 'pepper'].sort(), + [ + 'command.json', + 'assertion.jwt', + 'keyset.json', + 'pepper-keyring.json', + ].sort(), ) || JSON.stringify(aggregate).includes('security-administration') ) { diff --git a/scripts/ql3-security-administration-kubernetes-live-audit.cjs b/scripts/ql3-security-administration-kubernetes-live-audit.cjs index 9f0f3ea7..f86bcc7e 100644 --- a/scripts/ql3-security-administration-kubernetes-live-audit.cjs +++ b/scripts/ql3-security-administration-kubernetes-live-audit.cjs @@ -13,6 +13,7 @@ const LIMITATIONS = Object.freeze([ 'CloudNativePG inside one Docker host is not infrastructure STONITH or disaster-recovery evidence', 'the local-path ReadWriteOnce volume is not encrypted production CSI custody evidence', 'a dedicated root storage-fixture Job constrains the local-path volume root before every non-root administration Job', + 'the credential probe reaches the production HTTP surface inside the isolated cluster and is not external ingress TLS evidence', ]); const BANNED_KEYS = new Set([ 'assertion', @@ -143,6 +144,7 @@ function validateSecurityAdministrationKubernetesLiveReport(report) { 'architecture', 'kubernetesImageId', 'administrationImageId', + 'controlImageId', 'cniName', 'cniDistributionBinding', 'controlPlaneNodes', @@ -154,6 +156,7 @@ function validateSecurityAdministrationKubernetesLiveReport(report) { !['amd64', 'arm64'].includes(platform?.architecture) || !isSha256(platform?.kubernetesImageId) || !isSha256(platform?.administrationImageId) || + !isSha256(platform?.controlImageId) || platform?.cniName !== 'flannel' || platform?.cniDistributionBinding !== 'rancher/k3s:v1.34.3-k3s1' || platform?.controlPlaneNodes !== 1 || @@ -213,6 +216,10 @@ function validateSecurityAdministrationKubernetesLiveReport(report) { 'operations', 'completedJobs', 'failedJobs', + 'authenticationProbeJobs', + 'controlReplicas', + 'controlRollouts', + 'controlReplicaAntiAffinity', 'callerDriven', 'backoffLimit', 'activeDeadlineSeconds', @@ -221,19 +228,39 @@ function validateSecurityAdministrationKubernetesLiveReport(report) { 'serviceAccountTokenMounted', 'rbacGranted', 'responseLossReplayObserved', + 'overlapGenerationCount', + 'contractedGenerationCount', + 'activeGenerationChanged', + 'oldReferencesBeforeActivation', + 'oldReferencesAfterActivation', + 'oldReferencesAfterConvergence', + 'oldAuthenticationBeforeActivation', + 'oldAuthenticationDuringOverlap', + 'newAuthenticationDuringOverlap', + 'oldAuthenticationRejectedAfterConvergence', + 'newAuthenticationAfterContraction', + 'contractedToActiveGeneration', 'sensitiveMaterialReported', ]) || JSON.stringify(ceremony?.operations) !== JSON.stringify([ 'identity.register', 'audit.list', - 'credential.issue', - 'credential.issue.replay', - 'credential.rotate', - 'credential.revoke', + 'credential.issue.old', + 'credential.issue.old.replay', + 'credential.key-references.before-activate', + 'credential.issue.new', + 'credential.rotate.new', + 'credential.key-references.after-activate', + 'credential.revoke.old', + 'credential.key-references.after-converge', ]) || - ceremony?.completedJobs !== 6 || + ceremony?.completedJobs !== 10 || ceremony?.failedJobs !== 1 || + ceremony?.authenticationProbeJobs !== 5 || + ceremony?.controlReplicas !== 2 || + ceremony?.controlRollouts !== 3 || + ceremony?.controlReplicaAntiAffinity !== true || ceremony?.callerDriven !== true || ceremony?.backoffLimit !== 0 || ceremony?.activeDeadlineSeconds !== 300 || @@ -242,12 +269,24 @@ function validateSecurityAdministrationKubernetesLiveReport(report) { ceremony?.serviceAccountTokenMounted !== false || ceremony?.rbacGranted !== false || ceremony?.responseLossReplayObserved !== true || + ceremony?.overlapGenerationCount !== 2 || + ceremony?.contractedGenerationCount !== 1 || + ceremony?.activeGenerationChanged !== true || + ceremony?.oldReferencesBeforeActivation !== 1 || + ceremony?.oldReferencesAfterActivation !== 1 || + ceremony?.oldReferencesAfterConvergence !== 0 || + ceremony?.oldAuthenticationBeforeActivation !== true || + ceremony?.oldAuthenticationDuringOverlap !== true || + ceremony?.newAuthenticationDuringOverlap !== true || + ceremony?.oldAuthenticationRejectedAfterConvergence !== true || + ceremony?.newAuthenticationAfterContraction !== true || + ceremony?.contractedToActiveGeneration !== true || ceremony?.sensitiveMaterialReported !== false ) { findings.push( finding( 'QL3_SECURITY_ADMINISTRATION_KUBERNETES_LIVE_CEREMONY', - 'six serial caller-created commands plus one failed input stage must use the exact tokenless Job contract', + 'ten serial caller-created commands, five content-free authentication probes and one failed input stage must prove the exact two-replica overlap, activation, convergence and contraction contract', ), ); } @@ -299,7 +338,7 @@ function validateSecurityAdministrationKubernetesLiveReport(report) { delivery?.fixtureProvisionerRanAsRoot !== true || delivery?.privateDirectoryMode !== '0700' || delivery?.fileMode !== '0600' || - delivery?.fileCount !== 2 || + delivery?.fileCount !== 3 || !isSha256(delivery?.issueDigest) || !isSha256(delivery?.rotationDigest) || delivery?.issueDigest === delivery?.rotationDigest || @@ -342,28 +381,42 @@ function validateSecurityAdministrationKubernetesLiveReport(report) { !exactKeys(durability, [ 'identityVersion', 'identityStatus', - 'credentialVersion', - 'credentialState', + 'oldCredentialVersion', + 'oldCredentialState', + 'newCredentialVersion', + 'newCredentialState', 'identityMutationCount', 'credentialMutationCount', 'issueMutationCount', 'credentialVersionCount', + 'oldGenerationVersionCount', + 'newGenerationVersionCount', + 'latestGenerationsAreNew', 'allowedAuditCount', + 'authenticationDeniedAuditCount', + 'authenticationRejectedAuditCount', ]) || durability?.identityVersion !== 1 || durability?.identityStatus !== 'active' || - durability?.credentialVersion !== 3 || - durability?.credentialState !== 'revoked' || + durability?.oldCredentialVersion !== 2 || + durability?.oldCredentialState !== 'revoked' || + durability?.newCredentialVersion !== 2 || + durability?.newCredentialState !== 'active' || durability?.identityMutationCount !== 1 || - durability?.credentialMutationCount !== 3 || + durability?.credentialMutationCount !== 4 || durability?.issueMutationCount !== 1 || - durability?.credentialVersionCount !== 3 || - durability?.allowedAuditCount !== 4 + durability?.credentialVersionCount !== 4 || + durability?.oldGenerationVersionCount !== 1 || + durability?.newGenerationVersionCount !== 3 || + durability?.latestGenerationsAreNew !== true || + durability?.allowedAuditCount !== 5 || + durability?.authenticationDeniedAuditCount !== 4 || + durability?.authenticationRejectedAuditCount !== 1 ) { findings.push( finding( 'QL3_SECURITY_ADMINISTRATION_KUBERNETES_LIVE_DURABILITY', - 'the database must retain one identity mutation and exactly three credential generations without replay duplication', + 'the database must retain one old-key version followed by three new-key versions across two credentials, exact replay and the expected authentication audits', ), ); } @@ -375,6 +428,9 @@ function validateSecurityAdministrationKubernetesLiveReport(report) { 'evidenceJobsDeleted', 'storageProvisionJobDeleted', 'deliveryVolumeClaimDeleted', + 'controlDeploymentDeleted', + 'controlServiceDeleted', + 'controlRuntimeSecretDeleted', ]) ) { findings.push( @@ -392,6 +448,7 @@ function validateSecurityAdministrationKubernetesLiveReport(report) { 'realKubeletSecretProjection', 'realAdministrationProductCommands', 'realPersistentCredentialCustody', + 'realClusterControlAuthenticationRotation', 'responseLossReplay', 'failedInputStageClosed', 'leastPrivilege', diff --git a/scripts/ql3-security-administration-kubernetes-live-contract.cjs b/scripts/ql3-security-administration-kubernetes-live-contract.cjs index 68140d36..6396a9f3 100644 --- a/scripts/ql3-security-administration-kubernetes-live-contract.cjs +++ b/scripts/ql3-security-administration-kubernetes-live-contract.cjs @@ -42,6 +42,8 @@ const NAMESPACE = 'qinglong3-system'; const NAME = 'ql3-security-administration'; const POSTGRES_CLUSTER = 'ql3-postgres'; const DELIVERY_CLAIM = 'ql3-security-administration-delivery'; +const CONTROL_NAME = 'ql3-security-live-control'; +const CONTROL_RUNTIME_SECRET = 'ql3-security-live-control-runtime'; const ADMIN_IMAGE_BASE = 'ql3-security-administration-live'; const CONTROL_IMAGE_BASE = 'ql3-security-administration-migration-live'; const ISSUER = 'https://identity.qinglong.test/'; @@ -85,6 +87,23 @@ function randomSecret() { return crypto.randomBytes(32).toString('base64url'); } +function apiCredentialPepperKeyring(activePepperKeyId, keys) { + assert.ok(keys.length === 1 || keys.length === 2); + assert.ok(keys.some((key) => key.pepperKeyId === activePepperKeyId)); + return Object.freeze({ + schemaVersion: 1, + activePepperKeyId, + keys: Object.freeze( + keys.map((key) => + Object.freeze({ + pepperKeyId: key.pepperKeyId, + pepper: key.pepper, + }), + ), + ), + }); +} + function sqlString(value) { assert.equal(typeof value, 'string'); return `'${value.replaceAll("'", "''")}'`; @@ -114,6 +133,14 @@ function auditListCommand() { }); } +function pepperReferencesCommand(pepperKeyId) { + return Object.freeze({ + schemaVersion: 1, + operation: 'pepper.references', + request: Object.freeze({ pepperKeyId, limit: 64 }), + }); +} + function credentialIssueCommand(values) { return Object.freeze({ schemaVersion: 1, @@ -153,7 +180,7 @@ function credentialRevokeCommand(values) { request: Object.freeze({ mutationId: values.revokeMutationId, requestId: values.revokeRequestId, - expectedCurrentVersion: 2, + expectedCurrentVersion: values.expectedCurrentVersion ?? 2, credentialId: values.credentialId, subject: values.subject, }), @@ -221,7 +248,7 @@ function findNamed(values, name) { } function inputAuthorityEvidenceSource() { - return `'use strict';const fs=require('node:fs');const path=require('node:path');const names=['command.json','assertion.jwt','keyset.json','pepper'];const source='/var/run/secrets/qinglong3/security-administration-projected';const parent='/var/run/qinglong3/security-administration-private';const target=parent+'/input';const deliveryRoot='/var/lib/qinglong3/security-administration-delivery';const deliveryPrivate=deliveryRoot+'/private';const facts=(value)=>{const status=fs.lstatSync(value);return{mode:(status.mode&0o7777).toString(8),uid:status.uid,gid:status.gid,directory:status.isDirectory(),file:status.isFile(),symlink:status.isSymbolicLink()}};const sourceReal=fs.realpathSync(source);const files=names.map((name)=>{const candidate=source+'/'+name;const resolved=fs.realpathSync(candidate);const relative=path.relative(sourceReal,resolved);return{name,link:facts(candidate),resolved:facts(resolved),confined:relative!==''&&relative!=='..'&&!relative.startsWith('../')&&!path.isAbsolute(relative)}});fs.writeFileSync('/dev/termination-log',JSON.stringify({schema:'qinglong/security-administration-input-authority@v1',source:facts(source),parent:facts(parent),targetExists:fs.existsSync(target),deliveryRoot:fs.existsSync(deliveryRoot)?facts(deliveryRoot):null,deliveryPrivateExists:fs.existsSync(deliveryPrivate),files}));`; + return `'use strict';const fs=require('node:fs');const path=require('node:path');const names=['command.json','assertion.jwt','keyset.json','pepper-keyring.json'];const source='/var/run/secrets/qinglong3/security-administration-projected';const parent='/var/run/qinglong3/security-administration-private';const target=parent+'/input';const deliveryRoot='/var/lib/qinglong3/security-administration-delivery';const deliveryPrivate=deliveryRoot+'/private';const facts=(value)=>{const status=fs.lstatSync(value);return{mode:(status.mode&0o7777).toString(8),uid:status.uid,gid:status.gid,directory:status.isDirectory(),file:status.isFile(),symlink:status.isSymbolicLink()}};const sourceReal=fs.realpathSync(source);const files=names.map((name)=>{const candidate=source+'/'+name;const resolved=fs.realpathSync(candidate);const relative=path.relative(sourceReal,resolved);return{name,link:facts(candidate),resolved:facts(resolved),confined:relative!==''&&relative!=='..'&&!relative.startsWith('../')&&!path.isAbsolute(relative)}});fs.writeFileSync('/dev/termination-log',JSON.stringify({schema:'qinglong/security-administration-input-authority@v1',source:facts(source),parent:facts(parent),targetExists:fs.existsSync(target),deliveryRoot:fs.existsSync(deliveryRoot)?facts(deliveryRoot):null,deliveryPrivateExists:fs.existsSync(deliveryPrivate),files}));`; } function deliveryVolumeProvisionSource() { @@ -508,7 +535,7 @@ async function runAdministrationJob({ command, assertion, keyset, - pepper, + pepperKeyring, deliveryFile, expectedComplete, projectedMode, @@ -533,7 +560,7 @@ async function runAdministrationJob({ 'command.json': `${JSON.stringify(command)}\n`, 'assertion.jwt': `${assertion}\n`, 'keyset.json': `${JSON.stringify(keyset)}\n`, - pepper: `${pepper}\n`, + 'pepper-keyring.json': `${JSON.stringify(pepperKeyring)}\n`, }, }); createdSecrets.add(inputSecretName); @@ -704,6 +731,346 @@ async function runCustodyEvidence({ } } +function clusterControlResources(controlImage) { + const labels = Object.freeze({ + 'app.kubernetes.io/name': CONTROL_NAME, + 'app.kubernetes.io/component': 'control-plane', + 'app.kubernetes.io/part-of': 'qinglong3', + }); + return Object.freeze([ + { + apiVersion: 'v1', + kind: 'Service', + metadata: { name: CONTROL_NAME, namespace: NAMESPACE, labels }, + spec: { + selector: labels, + ports: [{ name: 'http', port: 5800, targetPort: 'http' }], + }, + }, + { + apiVersion: 'apps/v1', + kind: 'Deployment', + metadata: { name: CONTROL_NAME, namespace: NAMESPACE, labels }, + spec: { + replicas: 2, + minReadySeconds: 2, + progressDeadlineSeconds: 300, + strategy: { + type: 'RollingUpdate', + rollingUpdate: { maxUnavailable: 0, maxSurge: 1 }, + }, + selector: { matchLabels: labels }, + template: { + metadata: { labels }, + spec: { + serviceAccountName: 'ql3-cluster-control', + automountServiceAccountToken: false, + enableServiceLinks: false, + terminationGracePeriodSeconds: 20, + securityContext: { + runAsNonRoot: true, + runAsUser: 10001, + runAsGroup: 10001, + fsGroup: 10001, + fsGroupChangePolicy: 'OnRootMismatch', + seccompProfile: { type: 'RuntimeDefault' }, + }, + affinity: { + podAntiAffinity: { + requiredDuringSchedulingIgnoredDuringExecution: [{ + topologyKey: 'kubernetes.io/hostname', + labelSelector: { matchLabels: labels }, + }], + }, + }, + containers: [{ + name: 'cluster-control', + image: controlImage, + imagePullPolicy: 'Never', + securityContext: { + allowPrivilegeEscalation: false, + readOnlyRootFilesystem: true, + capabilities: { drop: ['ALL'] }, + }, + env: [ + { name: 'QL_DEPLOYMENT_PROFILE', value: 'cluster-control' }, + { name: 'QL3_CLUSTER_CONTROL_ENABLED', value: 'true' }, + { name: 'QL3_CLUSTER_HTTP_HOST', value: '0.0.0.0' }, + { name: 'QL3_CLUSTER_HTTP_PORT', value: '5800' }, + { name: 'QL3_CLUSTER_HTTP_DRAIN_TIMEOUT_MS', value: '10000' }, + { name: 'QL3_WORKER_INGRESS_ENABLED', value: 'false' }, + { name: 'QL3_POSTGRES_TLS_MODE', value: 'verify-full' }, + { + name: 'QL3_POSTGRES_TLS_CA_FILE', + value: '/var/run/secrets/qinglong3/postgres/ca.crt', + }, + { + name: 'QL3_POSTGRES_TLS_SERVERNAME', + value: 'ql3-postgres-rw.qinglong3-system.svc', + }, + { name: 'QL3_POSTGRES_MAX_CONNECTIONS', value: '2' }, + { + name: 'QL3_POSTGRES_APPLICATION_NAME', + value: 'qinglong3-security-live-control', + }, + { + name: 'QL3_CLUSTER_REPLICA_ID', + valueFrom: { + fieldRef: { + apiVersion: 'v1', + fieldPath: 'metadata.name', + }, + }, + }, + { + name: 'QL3_POSTGRES_RUNTIME_URL', + valueFrom: { + secretKeyRef: { + name: CONTROL_RUNTIME_SECRET, + key: 'postgres-runtime-url', + }, + }, + }, + { + name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE', + value: '/var/run/secrets/qinglong3/api-credential/keyring.json', + }, + ], + ports: [{ name: 'http', containerPort: 5800 }], + startupProbe: { + httpGet: { path: '/livez', port: 'http' }, + periodSeconds: 2, + timeoutSeconds: 1, + failureThreshold: 60, + }, + readinessProbe: { + httpGet: { path: '/readyz', port: 'http' }, + periodSeconds: 2, + timeoutSeconds: 1, + failureThreshold: 10, + }, + resources: { + requests: { cpu: '25m', memory: '96Mi' }, + limits: { cpu: '500m', memory: '256Mi' }, + }, + volumeMounts: [ + { name: 'tmp', mountPath: '/tmp' }, + { + name: 'postgres-ca', + mountPath: '/var/run/secrets/qinglong3/postgres', + readOnly: true, + }, + { + name: 'api-credential-keyring', + mountPath: '/var/run/secrets/qinglong3/api-credential', + readOnly: true, + }, + ], + }], + volumes: [ + { + name: 'tmp', + emptyDir: { medium: 'Memory', sizeLimit: '16Mi' }, + }, + { + name: 'postgres-ca', + secret: { + secretName: 'ql3-postgres-ca', + defaultMode: 292, + items: [{ key: 'ca.crt', path: 'ca.crt' }], + }, + }, + { + name: 'api-credential-keyring', + secret: { + secretName: CONTROL_RUNTIME_SECRET, + defaultMode: 292, + items: [{ + key: 'api-credential-pepper-keyring.json', + path: 'keyring.json', + }], + }, + }, + ], + }, + }, + }, + }, + ]); +} + +function applyControlRuntimeSecret(fixture, runtimeDatabaseUrl, keyring) { + applySecret(fixture, CONTROL_RUNTIME_SECRET, 'Opaque', { + 'postgres-runtime-url': runtimeDatabaseUrl, + 'api-credential-pepper-keyring.json': `${JSON.stringify(keyring)}\n`, + }); +} + +async function waitForControlRollout(fixture, restart) { + if (restart) { + fixture.kubectl([ + '-n', + NAMESPACE, + 'rollout', + 'restart', + `deployment/${CONTROL_NAME}`, + ]); + } + fixture.kubectl([ + '-n', + NAMESPACE, + 'rollout', + 'status', + `deployment/${CONTROL_NAME}`, + '--timeout=5m', + ]); + return ( + await waitFor('two ready Cluster Control replicas', 120_000, () => { + const deployment = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'deployment', + CONTROL_NAME, + ]); + const pods = fixture.kubectlJson([ + '-n', + NAMESPACE, + 'get', + 'pods', + '-l', + `app.kubernetes.io/name=${CONTROL_NAME}`, + ]).items; + const ready = pods.filter(podReady); + return deployment.status.availableReplicas === 2 && ready.length === 2 + ? { + ready: true, + value: Object.freeze({ + replicas: ready.length, + nodes: Object.freeze( + ready.map((pod) => pod.spec.nodeName).sort(), + ), + }), + } + : { + ready: false, + fact: JSON.stringify({ + availableReplicas: deployment.status.availableReplicas ?? 0, + readyPods: ready.length, + }), + }; + }) + ).value; +} + +function credentialAuthenticationProbeSource() { + return String.raw` +const fs=require('node:fs');const http=require('node:http'); +const finish=(value,status)=>{fs.writeFileSync('/dev/termination-log',JSON.stringify(value),{encoding:'utf8',mode:0o600});process.exitCode=status}; +const sleep=(milliseconds)=>new Promise((resolve)=>setTimeout(resolve,milliseconds)); +const request=(token,requestId)=>new Promise((resolve)=>{let settled=false;const done=(status)=>{if(settled)return;settled=true;resolve(status)};const call=http.get({host:'ql3-security-live-control',port:5800,path:'/api/v3/projects/prj_default/runs?limit=1',headers:{authorization:'Bearer '+token,'x-request-id':requestId}},(response)=>{const status=response.statusCode??null;response.resume();response.once('end',()=>done(status));response.once('error',()=>done(null))});call.setTimeout(5000,()=>{call.destroy();done(null)});call.once('error',()=>done(null))}); +(async()=>{let bytes;try{const file='/delivery/private/'+process.argv[1];const expected=Number(process.argv[2]);const requestId=process.argv[3];const status=fs.lstatSync(file);if(!status.isFile()||status.isSymbolicLink()||(status.mode&0o777)!==0o600||status.uid!==process.geteuid()||status.size<1||status.size>32768)throw new Error('FILE');bytes=fs.readFileSync(file);const delivery=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(bytes));if(typeof delivery.token!=='string'||!/^ql3c_[A-Za-z0-9_-]+_[A-Za-z0-9_-]{43}$/.test(delivery.token)||!Number.isSafeInteger(delivery.notBeforeAtMs)||!Number.isSafeInteger(delivery.expiresAtMs))throw new Error('SCHEMA');const wait=Math.max(0,delivery.notBeforeAtMs-Date.now()+1000);if(wait>360000||Date.now()+wait>=delivery.expiresAtMs)throw new Error('LIFETIME');if(wait>0)await sleep(wait);const observed=await request(delivery.token,requestId);if(observed!==expected)throw Object.assign(new Error('STATUS'),{observed});finish({schemaVersion:1,passed:true,expectedStatus:expected,observedStatus:observed},0)}catch(error){finish({schemaVersion:1,passed:false,code:error instanceof Error&&/^[A-Z]+$/.test(error.message)?error.message:'UNAVAILABLE',observedStatus:Number.isInteger(error?.observed)?error.observed:null},1)}finally{bytes?.fill(0)}})();`; +} + +async function runCredentialAuthenticationProbe({ + fixture, + adminImage, + name, + deliveryFile, + expectedStatus, + requestId, + createdEvidenceJobs, +}) { + fixture.create({ + apiVersion: 'batch/v1', + kind: 'Job', + metadata: { name, namespace: NAMESPACE }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: 420, + ttlSecondsAfterFinished: 600, + template: { + metadata: { + labels: { + 'app.kubernetes.io/name': NAME, + 'app.kubernetes.io/component': 'credential-authentication-evidence', + 'ql3.live/evidence': 'credential-authentication', + }, + }, + spec: { + serviceAccountName: NAME, + automountServiceAccountToken: false, + enableServiceLinks: false, + restartPolicy: 'Never', + securityContext: { + runAsNonRoot: true, + runAsUser: 10001, + runAsGroup: 10001, + fsGroup: 10001, + fsGroupChangePolicy: 'OnRootMismatch', + seccompProfile: { type: 'RuntimeDefault' }, + }, + containers: [{ + name: 'authentication-evidence', + image: adminImage, + imagePullPolicy: 'Never', + command: [ + 'node', + '-e', + credentialAuthenticationProbeSource(), + deliveryFile, + String(expectedStatus), + requestId, + ], + terminationMessagePolicy: 'File', + securityContext: { + allowPrivilegeEscalation: false, + readOnlyRootFilesystem: true, + capabilities: { drop: ['ALL'] }, + }, + resources: { + requests: { cpu: '5m', memory: '16Mi' }, + limits: { cpu: '100m', memory: '64Mi' }, + }, + volumeMounts: [{ + name: 'delivery', + mountPath: '/delivery', + readOnly: true, + }], + }], + volumes: [{ + name: 'delivery', + persistentVolumeClaim: { claimName: DELIVERY_CLAIM }, + }], + }, + }, + }, + }); + createdEvidenceJobs.add(name); + try { + const snapshot = await terminalJobSnapshot(fixture, name, 480_000); + const state = snapshot.pod.status.containerStatuses?.[0]?.state?.terminated; + const statusEvidence = state?.message || JSON.stringify({ + exitCode: state?.exitCode ?? null, + reason: state?.reason ?? null, + }); + assert.equal(snapshot.complete, true, statusEvidence); + assert.equal(snapshot.failed, false, statusEvidence); + assert.equal(state?.exitCode, 0, state?.message); + const evidence = JSON.parse(state.message); + assert.deepEqual(evidence, { + schemaVersion: 1, + passed: true, + expectedStatus, + observedStatus: expectedStatus, + }); + return evidence; + } finally { + deleteResource(fixture, `job/${name}`); + createdEvidenceJobs.delete(name); + } +} + async function provisionDeliveryVolume({ fixture, adminImage, @@ -900,6 +1267,7 @@ async function main(argv = process.argv.slice(2)) { fixture.loadImage(image, archive); } const adminImageInfo = fixture.inspectImage(adminImage); + const controlImageInfo = fixture.inspectImage(controlImage); const postgresImageInfo = fixture.inspectImage(POSTGRES_IMAGE); const k3sImageInfo = fixture.inspectImage(fixture.k3sImage); @@ -1098,21 +1466,70 @@ async function main(argv = process.argv.slice(2)) { const values = Object.freeze({ subject: Object.freeze({ type: 'api_app', id: `d406-live-${suffix}` }), - credentialId: `d406-live-${suffix}`, identityMutationId: crypto.randomUUID(), + registerRequestId: `d406-register-${suffix}`, + }); + const oldCredential = Object.freeze({ + subject: values.subject, + credentialId: `d407-old-${suffix}`, + issueMutationId: crypto.randomUUID(), + revokeMutationId: crypto.randomUUID(), + issueRequestId: `d407-old-issue-${suffix}`, + revokeRequestId: `d407-old-revoke-${suffix}`, + }); + const newCredential = Object.freeze({ + subject: values.subject, + credentialId: `d407-new-${suffix}`, issueMutationId: crypto.randomUUID(), rotateMutationId: crypto.randomUUID(), - revokeMutationId: crypto.randomUUID(), - registerRequestId: `d406-register-${suffix}`, - issueRequestId: `d406-issue-${suffix}`, - rotateRequestId: `d406-rotate-${suffix}`, - revokeRequestId: `d406-revoke-${suffix}`, + issueRequestId: `d407-new-issue-${suffix}`, + rotateRequestId: `d407-new-rotate-${suffix}`, + }); + const authenticationRequestIds = Object.freeze({ + oldBeforeActivation: `d407-auth-old-before-${suffix}`, + oldDuringOverlap: `d407-auth-old-overlap-${suffix}`, + newDuringOverlap: `d407-auth-new-overlap-${suffix}`, + oldAfterConvergence: `d407-auth-old-contracted-${suffix}`, + newAfterContraction: `d407-auth-new-contracted-${suffix}`, }); const key = identity.reviewedKey('security-administration-live-key-1'); const keyset = identity.keyset(1, [key]); - const pepper = randomSecret(); + const oldPepperKeyId = 'd407-old'; + const newPepperKeyId = 'd407-new'; + const oldPepper = randomSecret(); + const newPepper = randomSecret(); + const overlapOldActive = apiCredentialPepperKeyring(oldPepperKeyId, [ + { pepperKeyId: oldPepperKeyId, pepper: oldPepper }, + { pepperKeyId: newPepperKeyId, pepper: newPepper }, + ]); + const overlapNewActive = apiCredentialPepperKeyring(newPepperKeyId, [ + { pepperKeyId: oldPepperKeyId, pepper: oldPepper }, + { pepperKeyId: newPepperKeyId, pepper: newPepper }, + ]); + const contractedNew = apiCredentialPepperKeyring(newPepperKeyId, [ + { pepperKeyId: newPepperKeyId, pepper: newPepper }, + ]); const register = identityRegisterCommand(values); - const revoke = credentialRevokeCommand(values); + const revokeOld = credentialRevokeCommand({ + ...oldCredential, + expectedCurrentVersion: 1, + }); + const runtimeDatabaseUrl = + `postgresql://ql3_runtime:${passwords.ql3_runtime}` + + '@ql3-postgres-rw.qinglong3-system.svc:5432/qinglong'; + applyControlRuntimeSecret( + fixture, + runtimeDatabaseUrl, + overlapOldActive, + ); + for (const resource of clusterControlResources(controlImage)) { + fixture.apply(resource); + } + const controlBeforeActivation = await waitForControlRollout( + fixture, + false, + ); + assert.equal(new Set(controlBeforeActivation.nodes).size, 2); await runAdministrationJob({ fixture, @@ -1121,7 +1538,7 @@ async function main(argv = process.argv.slice(2)) { command: register, assertion: identity.assertion(key, 'invalid-input'), keyset, - pepper, + pepperKeyring: overlapOldActive, expectedComplete: false, projectedMode: 0o444, createdJobs, @@ -1134,7 +1551,7 @@ async function main(argv = process.argv.slice(2)) { command: register, assertion: identity.assertion(key, 'register'), keyset, - pepper, + pepperKeyring: overlapOldActive, expectedComplete: true, createdJobs, createdSecrets, @@ -1146,27 +1563,30 @@ async function main(argv = process.argv.slice(2)) { command: auditListCommand(), assertion: identity.assertion(key, 'audit'), keyset, - pepper, + pepperKeyring: overlapOldActive, expectedComplete: true, createdJobs, createdSecrets, }); - const issueNotBeforeAtMs = Date.now() + 5 * 60 * 1000; - const issue = credentialIssueCommand({ - ...values, - notBeforeAtMs: issueNotBeforeAtMs, - expiresAtMs: issueNotBeforeAtMs + 60 * 60 * 1000, + const oldIssueNotBeforeAtMs = Date.now() + 2 * 60 * 1000; + const issueOld = credentialIssueCommand({ + ...oldCredential, + notBeforeAtMs: oldIssueNotBeforeAtMs, + expiresAtMs: oldIssueNotBeforeAtMs + 60 * 60 * 1000, }); - const issueAssertion = identity.assertion(key, 'issue-response-loss'); + const oldIssueAssertion = identity.assertion( + key, + 'old-issue-response-loss', + ); await runAdministrationJob({ fixture, template: deliveryTemplate, - name: 'ql3-security-live-issue', - command: issue, - assertion: issueAssertion, + name: 'ql3-security-live-old-issue', + command: issueOld, + assertion: oldIssueAssertion, keyset, - pepper, - deliveryFile: 'issue.json', + pepperKeyring: overlapOldActive, + deliveryFile: 'old-issue.json', expectedComplete: true, createdJobs, createdSecrets, @@ -1182,19 +1602,19 @@ async function main(argv = process.argv.slice(2)) { fixture, adminImage, name: 'ql3-security-live-evidence-issued', - expected: [{ name: 'issue.json', operation: 'credential.issue' }], + expected: [{ name: 'old-issue.json', operation: 'credential.issue' }], kubernetesServiceIp, createdEvidenceJobs, }); await runAdministrationJob({ fixture, template: deliveryTemplate, - name: 'ql3-security-live-issue-replay', - command: issue, - assertion: issueAssertion, + name: 'ql3-security-live-old-issue-replay', + command: issueOld, + assertion: oldIssueAssertion, keyset, - pepper, - deliveryFile: 'issue.json', + pepperKeyring: overlapOldActive, + deliveryFile: 'old-issue.json', expectedComplete: true, createdJobs, createdSecrets, @@ -1203,26 +1623,142 @@ async function main(argv = process.argv.slice(2)) { fixture, adminImage, name: 'ql3-security-live-evidence-replay', - expected: [{ name: 'issue.json', operation: 'credential.issue' }], + expected: [{ name: 'old-issue.json', operation: 'credential.issue' }], kubernetesServiceIp, createdEvidenceJobs, }); assert.deepEqual(replayEvidence.files, issuedEvidence.files); - const rotateNotBeforeAtMs = Date.now() + 5 * 60 * 1000; - const rotate = credentialRotateCommand({ - ...values, - notBeforeAtMs: rotateNotBeforeAtMs, - expiresAtMs: rotateNotBeforeAtMs + 60 * 60 * 1000, + const oldAuthenticationBeforeActivation = + await runCredentialAuthenticationProbe({ + fixture, + adminImage, + name: 'ql3-security-live-auth-old-before-activate', + deliveryFile: 'old-issue.json', + expectedStatus: 403, + requestId: authenticationRequestIds.oldBeforeActivation, + createdEvidenceJobs, + }); + await runAdministrationJob({ + fixture, + template: baseTemplate, + name: 'ql3-security-live-old-references-before-activate', + command: pepperReferencesCommand(oldPepperKeyId), + assertion: identity.assertion(key, 'old-references-before-activate'), + keyset, + pepperKeyring: overlapOldActive, + expectedComplete: true, + createdJobs, + createdSecrets, + }); + const referencesBeforeActivate = Number( + psql( + fixture, + currentPrimaryPod(fixture).metadata.name, + `SELECT count(*)::integer FROM "ql3"."api_credentials" current_record WHERE credential_id = ${sqlString( + oldCredential.credentialId, + )} AND pepper_key_id = ${sqlString( + oldPepperKeyId, + )} AND state = 'active' AND expires_at_ms > (extract(epoch FROM statement_timestamp()) * 1000)::bigint AND NOT EXISTS (SELECT 1 FROM "ql3"."api_credentials" newer WHERE newer.credential_id = current_record.credential_id AND newer.version > current_record.version)`, + ), + ); + assert.equal(referencesBeforeActivate, 1); + applyControlRuntimeSecret( + fixture, + runtimeDatabaseUrl, + overlapNewActive, + ); + const controlDuringOverlap = await waitForControlRollout(fixture, true); + assert.equal(new Set(controlDuringOverlap.nodes).size, 2); + const oldAuthenticationDuringOverlap = + await runCredentialAuthenticationProbe({ + fixture, + adminImage, + name: 'ql3-security-live-auth-old-overlap', + deliveryFile: 'old-issue.json', + expectedStatus: 403, + requestId: authenticationRequestIds.oldDuringOverlap, + createdEvidenceJobs, + }); + const newIssueNotBeforeAtMs = Date.now() + 2 * 60 * 1000; + const issueNew = credentialIssueCommand({ + ...newCredential, + notBeforeAtMs: newIssueNotBeforeAtMs, + expiresAtMs: newIssueNotBeforeAtMs + 60 * 60 * 1000, }); await runAdministrationJob({ fixture, template: deliveryTemplate, - name: 'ql3-security-live-rotate', - command: rotate, - assertion: identity.assertion(key, 'rotate'), + name: 'ql3-security-live-new-issue', + command: issueNew, + assertion: identity.assertion(key, 'new-issue'), keyset, - pepper, - deliveryFile: 'rotate.json', + pepperKeyring: overlapNewActive, + deliveryFile: 'new-issue.json', + expectedComplete: true, + createdJobs, + createdSecrets, + }); + const newRotateNotBeforeAtMs = Date.now() + 2 * 60 * 1000; + const rotateNew = credentialRotateCommand({ + ...newCredential, + notBeforeAtMs: newRotateNotBeforeAtMs, + expiresAtMs: newRotateNotBeforeAtMs + 60 * 60 * 1000, + }); + await runAdministrationJob({ + fixture, + template: deliveryTemplate, + name: 'ql3-security-live-new-rotate', + command: rotateNew, + assertion: identity.assertion(key, 'new-rotate'), + keyset, + pepperKeyring: overlapNewActive, + deliveryFile: 'new-rotate.json', + expectedComplete: true, + createdJobs, + createdSecrets, + }); + const newAuthenticationDuringOverlap = + await runCredentialAuthenticationProbe({ + fixture, + adminImage, + name: 'ql3-security-live-auth-new-overlap', + deliveryFile: 'new-rotate.json', + expectedStatus: 403, + requestId: authenticationRequestIds.newDuringOverlap, + createdEvidenceJobs, + }); + await runAdministrationJob({ + fixture, + template: baseTemplate, + name: 'ql3-security-live-old-references-after-activate', + command: pepperReferencesCommand(oldPepperKeyId), + assertion: identity.assertion(key, 'old-references-after-activate'), + keyset, + pepperKeyring: overlapNewActive, + expectedComplete: true, + createdJobs, + createdSecrets, + }); + const referencesAfterActivate = Number( + psql( + fixture, + currentPrimaryPod(fixture).metadata.name, + `SELECT count(*)::integer FROM "ql3"."api_credentials" current_record WHERE credential_id = ${sqlString( + oldCredential.credentialId, + )} AND pepper_key_id = ${sqlString( + oldPepperKeyId, + )} AND version = (SELECT max(version) FROM "ql3"."api_credentials" WHERE credential_id = current_record.credential_id) AND state = 'active' AND expires_at_ms > (extract(epoch FROM statement_timestamp()) * 1000)::bigint`, + ), + ); + assert.equal(referencesAfterActivate, 1); + await runAdministrationJob({ + fixture, + template: baseTemplate, + name: 'ql3-security-live-old-revoke', + command: revokeOld, + assertion: identity.assertion(key, 'old-revoke'), + keyset, + pepperKeyring: overlapNewActive, expectedComplete: true, createdJobs, createdSecrets, @@ -1230,32 +1766,68 @@ async function main(argv = process.argv.slice(2)) { await runAdministrationJob({ fixture, template: baseTemplate, - name: 'ql3-security-live-revoke', - command: revoke, - assertion: identity.assertion(key, 'revoke'), + name: 'ql3-security-live-old-references-after-converge', + command: pepperReferencesCommand(oldPepperKeyId), + assertion: identity.assertion(key, 'old-references-after-converge'), keyset, - pepper, + pepperKeyring: overlapNewActive, expectedComplete: true, createdJobs, createdSecrets, }); + const referencesAfterConvergence = Number( + psql( + fixture, + currentPrimaryPod(fixture).metadata.name, + `SELECT count(*)::integer FROM "ql3"."api_credentials" current_record WHERE credential_id = ${sqlString( + oldCredential.credentialId, + )} AND pepper_key_id = ${sqlString( + oldPepperKeyId, + )} AND version = (SELECT max(version) FROM "ql3"."api_credentials" WHERE credential_id = current_record.credential_id) AND state = 'active' AND expires_at_ms > (extract(epoch FROM statement_timestamp()) * 1000)::bigint`, + ), + ); + assert.equal(referencesAfterConvergence, 0); + applyControlRuntimeSecret(fixture, runtimeDatabaseUrl, contractedNew); + const controlAfterContraction = await waitForControlRollout(fixture, true); + assert.equal(new Set(controlAfterContraction.nodes).size, 2); + const oldAuthenticationAfterConvergence = + await runCredentialAuthenticationProbe({ + fixture, + adminImage, + name: 'ql3-security-live-auth-old-contracted', + deliveryFile: 'old-issue.json', + expectedStatus: 401, + requestId: authenticationRequestIds.oldAfterConvergence, + createdEvidenceJobs, + }); + const newAuthenticationAfterContraction = + await runCredentialAuthenticationProbe({ + fixture, + adminImage, + name: 'ql3-security-live-auth-new-contracted', + deliveryFile: 'new-rotate.json', + expectedStatus: 403, + requestId: authenticationRequestIds.newAfterContraction, + createdEvidenceJobs, + }); const finalEvidence = await runCustodyEvidence({ fixture, adminImage, name: 'ql3-security-live-evidence-final', expected: [ - { name: 'issue.json', operation: 'credential.issue' }, - { name: 'rotate.json', operation: 'credential.rotate' }, + { name: 'old-issue.json', operation: 'credential.issue' }, + { name: 'new-issue.json', operation: 'credential.issue' }, + { name: 'new-rotate.json', operation: 'credential.rotate' }, ], kubernetesServiceIp, createdEvidenceJobs, }); - assert.equal(finalEvidence.files.length, 2); + assert.equal(finalEvidence.files.length, 3); const issueFile = finalEvidence.files.find( - (file) => file.name === 'issue.json', + (file) => file.name === 'old-issue.json', ); const rotationFile = finalEvidence.files.find( - (file) => file.name === 'rotate.json', + (file) => file.name === 'new-rotate.json', ); assert.ok(issueFile && rotationFile); assert.notEqual(issueFile.digest, rotationFile.digest); @@ -1274,45 +1846,82 @@ async function main(argv = process.argv.slice(2)) { ` 'identityStatus', (SELECT status FROM "ql3"."identity_subjects" WHERE subject_type = ${sqlString( values.subject.type, )} AND subject_id = ${sqlString(values.subject.id)}),`, - ` 'credentialVersion', (SELECT max(version)::integer FROM "ql3"."api_credentials" WHERE credential_id = ${sqlString( - values.credentialId, + ` 'oldCredentialVersion', (SELECT max(version)::integer FROM "ql3"."api_credentials" WHERE credential_id = ${sqlString( + oldCredential.credentialId, )}),`, - ` 'credentialState', (SELECT state FROM "ql3"."api_credentials" WHERE credential_id = ${sqlString( - values.credentialId, + ` 'oldCredentialState', (SELECT state FROM "ql3"."api_credentials" WHERE credential_id = ${sqlString( + oldCredential.credentialId, + )} ORDER BY version DESC LIMIT 1),`, + ` 'newCredentialVersion', (SELECT max(version)::integer FROM "ql3"."api_credentials" WHERE credential_id = ${sqlString( + newCredential.credentialId, + )}),`, + ` 'newCredentialState', (SELECT state FROM "ql3"."api_credentials" WHERE credential_id = ${sqlString( + newCredential.credentialId, )} ORDER BY version DESC LIMIT 1),`, ` 'identityMutationCount', (SELECT count(*)::integer FROM "ql3"."identity_subject_mutations" WHERE subject_type = ${sqlString( values.subject.type, )} AND subject_id = ${sqlString(values.subject.id)}),`, - ` 'credentialMutationCount', (SELECT count(*)::integer FROM "ql3"."api_credential_mutations" WHERE credential_id = ${sqlString( - values.credentialId, - )}),`, + ` 'credentialMutationCount', (SELECT count(*)::integer FROM "ql3"."api_credential_mutations" WHERE credential_id IN (${[ + oldCredential.credentialId, + newCredential.credentialId, + ].map(sqlString).join(',')})),`, ` 'issueMutationCount', (SELECT count(*)::integer FROM "ql3"."api_credential_mutations" WHERE mutation_id = ${sqlString( - values.issueMutationId, + oldCredential.issueMutationId, )}::uuid),`, - ` 'credentialVersionCount', (SELECT count(*)::integer FROM "ql3"."api_credentials" WHERE credential_id = ${sqlString( - values.credentialId, - )}),`, + ` 'credentialVersionCount', (SELECT count(*)::integer FROM "ql3"."api_credentials" WHERE credential_id IN (${[ + oldCredential.credentialId, + newCredential.credentialId, + ].map(sqlString).join(',')})),`, + ` 'oldGenerationVersionCount', (SELECT count(*)::integer FROM "ql3"."api_credentials" WHERE credential_id IN (${[ + oldCredential.credentialId, + newCredential.credentialId, + ].map(sqlString).join(',')}) AND pepper_key_id = ${sqlString(oldPepperKeyId)}),`, + ` 'newGenerationVersionCount', (SELECT count(*)::integer FROM "ql3"."api_credentials" WHERE credential_id IN (${[ + oldCredential.credentialId, + newCredential.credentialId, + ].map(sqlString).join(',')}) AND pepper_key_id = ${sqlString(newPepperKeyId)}),`, + ` 'latestGenerationsAreNew', (SELECT count(*) = 2 FROM "ql3"."api_credentials" current_record WHERE credential_id IN (${[ + oldCredential.credentialId, + newCredential.credentialId, + ].map(sqlString).join(',')}) AND version = (SELECT max(version) FROM "ql3"."api_credentials" WHERE credential_id = current_record.credential_id) AND pepper_key_id = ${sqlString(newPepperKeyId)}),`, ` 'allowedAuditCount', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE request_id IN (${[ values.registerRequestId, - values.issueRequestId, - values.rotateRequestId, - values.revokeRequestId, + oldCredential.issueRequestId, + newCredential.issueRequestId, + newCredential.rotateRequestId, + oldCredential.revokeRequestId, ] .map(sqlString) - .join(',')}) AND outcome = 'allowed'))`, + .join(',')}) AND outcome = 'allowed'),`, + ` 'authenticationDeniedAuditCount', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE request_id IN (${[ + authenticationRequestIds.oldBeforeActivation, + authenticationRequestIds.oldDuringOverlap, + authenticationRequestIds.newDuringOverlap, + authenticationRequestIds.newAfterContraction, + ].map(sqlString).join(',')}) AND outcome = 'denied'),`, + ` 'authenticationRejectedAuditCount', (SELECT count(*)::integer FROM "ql3"."security_audit_events" WHERE request_id = ${sqlString( + authenticationRequestIds.oldAfterConvergence, + )} AND outcome = 'authentication_rejected'))`, ].join('\n'), ), ); assert.deepEqual(durable, { identityVersion: 1, identityStatus: 'active', - credentialVersion: 3, - credentialState: 'revoked', + oldCredentialVersion: 2, + oldCredentialState: 'revoked', + newCredentialVersion: 2, + newCredentialState: 'active', identityMutationCount: 1, - credentialMutationCount: 3, + credentialMutationCount: 4, issueMutationCount: 1, - credentialVersionCount: 3, - allowedAuditCount: 4, + credentialVersionCount: 4, + oldGenerationVersionCount: 1, + newGenerationVersionCount: 3, + latestGenerationsAreNew: true, + allowedAuditCount: 5, + authenticationDeniedAuditCount: 4, + authenticationRejectedAuditCount: 1, }); const adminRole = JSON.parse( psql( @@ -1365,6 +1974,9 @@ async function main(argv = process.argv.slice(2)) { assert.equal(createdJobs.size, 0); assert.equal(createdSecrets.size, 0); assert.equal(createdEvidenceJobs.size, 0); + deleteResource(fixture, `deployment/${CONTROL_NAME}`); + deleteResource(fixture, `service/${CONTROL_NAME}`); + deleteResource(fixture, `secret/${CONTROL_RUNTIME_SECRET}`); deleteResource(fixture, `persistentvolumeclaim/${DELIVERY_CLAIM}`); await waitFor('delivery PVC deletion', 60_000, () => resourceAbsent( @@ -1399,6 +2011,18 @@ async function main(argv = process.argv.slice(2)) { fixture, `persistentvolumeclaim/${DELIVERY_CLAIM}`, ), + controlDeploymentDeleted: resourceAbsent( + fixture, + `deployment/${CONTROL_NAME}`, + ), + controlServiceDeleted: resourceAbsent( + fixture, + `service/${CONTROL_NAME}`, + ), + controlRuntimeSecretDeleted: resourceAbsent( + fixture, + `secret/${CONTROL_RUNTIME_SECRET}`, + ), }; assert.equal(Object.values(cleanup).every(Boolean), true); @@ -1412,6 +2036,7 @@ async function main(argv = process.argv.slice(2)) { architecture, kubernetesImageId: imageIdDigest(k3sImageInfo), administrationImageId: imageIdDigest(adminImageInfo), + controlImageId: imageIdDigest(controlImageInfo), cniName: 'flannel', cniDistributionBinding: fixture.k3sImage, controlPlaneNodes: 1, @@ -1437,13 +2062,24 @@ async function main(argv = process.argv.slice(2)) { operations: [ 'identity.register', 'audit.list', - 'credential.issue', - 'credential.issue.replay', - 'credential.rotate', - 'credential.revoke', + 'credential.issue.old', + 'credential.issue.old.replay', + 'credential.key-references.before-activate', + 'credential.issue.new', + 'credential.rotate.new', + 'credential.key-references.after-activate', + 'credential.revoke.old', + 'credential.key-references.after-converge', ], - completedJobs: 6, + completedJobs: 10, failedJobs: 1, + authenticationProbeJobs: 5, + controlReplicas: controlAfterContraction.replicas, + controlRollouts: 3, + controlReplicaAntiAffinity: + new Set(controlBeforeActivation.nodes).size === 2 && + new Set(controlDuringOverlap.nodes).size === 2 && + new Set(controlAfterContraction.nodes).size === 2, callerDriven: true, backoffLimit: baseTemplate.spec.backoffLimit, activeDeadlineSeconds: baseTemplate.spec.activeDeadlineSeconds, @@ -1452,6 +2088,29 @@ async function main(argv = process.argv.slice(2)) { serviceAccountTokenMounted: false, rbacGranted: false, responseLossReplayObserved: true, + overlapGenerationCount: overlapOldActive.keys.length, + contractedGenerationCount: contractedNew.keys.length, + activeGenerationChanged: + overlapOldActive.activePepperKeyId !== + overlapNewActive.activePepperKeyId, + oldReferencesBeforeActivation: referencesBeforeActivate, + oldReferencesAfterActivation: referencesAfterActivate, + oldReferencesAfterConvergence: referencesAfterConvergence, + oldAuthenticationBeforeActivation: + oldAuthenticationBeforeActivation.observedStatus === 403, + oldAuthenticationDuringOverlap: + oldAuthenticationDuringOverlap.observedStatus === 403, + newAuthenticationDuringOverlap: + newAuthenticationDuringOverlap.observedStatus === 403, + oldAuthenticationRejectedAfterConvergence: + oldAuthenticationAfterConvergence.observedStatus === 401, + newAuthenticationAfterContraction: + newAuthenticationAfterContraction.observedStatus === 403, + contractedToActiveGeneration: + contractedNew.activePepperKeyId === + overlapNewActive.activePepperKeyId && + contractedNew.keys[0]?.pepperKeyId === + contractedNew.activePepperKeyId, sensitiveMaterialReported: false, }, inputBoundary: { @@ -1500,7 +2159,10 @@ async function main(argv = process.argv.slice(2)) { Number(finalCluster.status.readyInstances) === 3, realKubeletSecretProjection: true, realAdministrationProductCommands: true, - realPersistentCredentialCustody: finalEvidence.files.length === 2, + realPersistentCredentialCustody: finalEvidence.files.length === 3, + realClusterControlAuthenticationRotation: + durable.authenticationDeniedAuditCount === 4 && + durable.authenticationRejectedAuditCount === 1, responseLossReplay: durable.issueMutationCount === 1, failedInputStageClosed: true, leastPrivilege: true, @@ -1555,6 +2217,8 @@ if (require.main === module) { module.exports = { auditListCommand, + clusterControlResources, + credentialAuthenticationProbeSource, credentialIssueCommand, credentialRevokeCommand, credentialRotateCommand, diff --git a/scripts/ql3-worker-kubernetes-rollout-live-contract.cjs b/scripts/ql3-worker-kubernetes-rollout-live-contract.cjs index 57ffd86a..e3d2e3b2 100644 --- a/scripts/ql3-worker-kubernetes-rollout-live-contract.cjs +++ b/scripts/ql3-worker-kubernetes-rollout-live-contract.cjs @@ -1617,6 +1617,25 @@ async function main(argv = process.argv.slice(2)) { 'tls.crt': pkiMaterial.oldClientCertificate, }, }); + apply({ + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: 'ql3-worker-control-live-runtime', + namespace: NAMESPACE, + }, + type: 'Opaque', + stringData: { + 'api-credential-pepper-keyring.json': `${JSON.stringify({ + schemaVersion: 1, + activePepperKeyId: 'legacy-v1', + keys: [{ + pepperKeyId: 'legacy-v1', + pepper: Buffer.alloc(32, 29).toString('base64url'), + }], + })}\n`, + }, + }); const workerArchitecture = remoteWorkerArchitectureForNodeRuntime( process.arch, process.config.variables.arm_version, ); @@ -1700,8 +1719,8 @@ async function main(argv = process.argv.slice(2)) { { name: 'QL3_POSTGRES_ALLOW_INSECURE', value: 'true' }, { name: 'QL3_POSTGRES_MAX_CONNECTIONS', value: '2' }, { - name: 'QL3_API_CREDENTIAL_PEPPER', - value: Buffer.alloc(32, 29).toString('base64url'), + name: 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE', + value: '/var/run/secrets/qinglong3/api-credential/keyring.json', }, { name: 'QL3_WORKER_INGRESS_ENABLED', value: 'true' }, { name: 'QL3_WORKER_INGRESS_HOST', value: '0.0.0.0' }, @@ -1738,12 +1757,35 @@ async function main(argv = process.argv.slice(2)) { timeoutSeconds: 1, failureThreshold: 20, }, - volumeMounts: [{ name: 'tls', mountPath: '/tls', readOnly: true }], - }], - volumes: [{ - name: 'tls', - secret: { secretName: 'ql3-worker-ingress-tls-live', defaultMode: 288 }, + volumeMounts: [ + { name: 'tls', mountPath: '/tls', readOnly: true }, + { + name: 'api-credential-keyring', + mountPath: '/var/run/secrets/qinglong3/api-credential', + readOnly: true, + }, + ], }], + volumes: [ + { + name: 'tls', + secret: { + secretName: 'ql3-worker-ingress-tls-live', + defaultMode: 288, + }, + }, + { + name: 'api-credential-keyring', + secret: { + secretName: 'ql3-worker-control-live-runtime', + defaultMode: 288, + items: [{ + key: 'api-credential-pepper-keyring.json', + path: 'keyring.json', + }], + }, + }, + ], }, }, }, diff --git a/test/back/ql3SecurityAdministrationKubernetesLiveAudit.test.cjs b/test/back/ql3SecurityAdministrationKubernetesLiveAudit.test.cjs index 3670f5b2..c14cf49f 100644 --- a/test/back/ql3SecurityAdministrationKubernetesLiveAudit.test.cjs +++ b/test/back/ql3SecurityAdministrationKubernetesLiveAudit.test.cjs @@ -20,6 +20,7 @@ function report() { architecture: 'amd64', kubernetesImageId: digest('1'), administrationImageId: digest('2'), + controlImageId: digest('4'), cniName: 'flannel', cniDistributionBinding: 'rancher/k3s:v1.34.3-k3s1', controlPlaneNodes: 1, @@ -45,13 +46,21 @@ function report() { operations: [ 'identity.register', 'audit.list', - 'credential.issue', - 'credential.issue.replay', - 'credential.rotate', - 'credential.revoke', + 'credential.issue.old', + 'credential.issue.old.replay', + 'credential.key-references.before-activate', + 'credential.issue.new', + 'credential.rotate.new', + 'credential.key-references.after-activate', + 'credential.revoke.old', + 'credential.key-references.after-converge', ], - completedJobs: 6, + completedJobs: 10, failedJobs: 1, + authenticationProbeJobs: 5, + controlReplicas: 2, + controlRollouts: 3, + controlReplicaAntiAffinity: true, callerDriven: true, backoffLimit: 0, activeDeadlineSeconds: 300, @@ -60,6 +69,18 @@ function report() { serviceAccountTokenMounted: false, rbacGranted: false, responseLossReplayObserved: true, + overlapGenerationCount: 2, + contractedGenerationCount: 1, + activeGenerationChanged: true, + oldReferencesBeforeActivation: 1, + oldReferencesAfterActivation: 1, + oldReferencesAfterConvergence: 0, + oldAuthenticationBeforeActivation: true, + oldAuthenticationDuringOverlap: true, + newAuthenticationDuringOverlap: true, + oldAuthenticationRejectedAfterConvergence: true, + newAuthenticationAfterContraction: true, + contractedToActiveGeneration: true, sensitiveMaterialReported: false, }, inputBoundary: { @@ -80,7 +101,7 @@ function report() { fixtureProvisionerRanAsRoot: true, privateDirectoryMode: '0700', fileMode: '0600', - fileCount: 2, + fileCount: 3, issueDigest: digest('4'), rotationDigest: digest('5'), distinctRotationMaterial: true, @@ -100,13 +121,20 @@ function report() { durability: { identityVersion: 1, identityStatus: 'active', - credentialVersion: 3, - credentialState: 'revoked', + oldCredentialVersion: 2, + oldCredentialState: 'revoked', + newCredentialVersion: 2, + newCredentialState: 'active', identityMutationCount: 1, - credentialMutationCount: 3, + credentialMutationCount: 4, issueMutationCount: 1, - credentialVersionCount: 3, - allowedAuditCount: 4, + credentialVersionCount: 4, + oldGenerationVersionCount: 1, + newGenerationVersionCount: 3, + latestGenerationsAreNew: true, + allowedAuditCount: 5, + authenticationDeniedAuditCount: 4, + authenticationRejectedAuditCount: 1, }, cleanup: { jobsDeleted: true, @@ -114,6 +142,9 @@ function report() { evidenceJobsDeleted: true, storageProvisionJobDeleted: true, deliveryVolumeClaimDeleted: true, + controlDeploymentDeleted: true, + controlServiceDeleted: true, + controlRuntimeSecretDeleted: true, }, gates: { realThreeNodeKubernetes: true, @@ -121,6 +152,7 @@ function report() { realKubeletSecretProjection: true, realAdministrationProductCommands: true, realPersistentCredentialCustody: true, + realClusterControlAuthenticationRotation: true, responseLossReplay: true, failedInputStageClosed: true, leastPrivilege: true, @@ -142,6 +174,7 @@ test('rejects widened authority, false custody and replay duplication', () => { candidate.ceremony.serviceAccountTokenMounted = true; candidate.deliveryCustody.noReplaceReplayPreserved = false; candidate.durability.issueMutationCount = 2; + candidate.ceremony.oldReferencesAfterConvergence = 1; candidate.gates.passed = false; const codes = validateSecurityAdministrationKubernetesLiveReport( candidate, diff --git a/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs b/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs index 208f0945..6d6040af 100644 --- a/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs +++ b/test/back/ql3SecurityAdministrationKubernetesLiveContract.test.cjs @@ -6,6 +6,8 @@ const { test } = require('node:test'); const { auditListCommand, + clusterControlResources, + credentialAuthenticationProbeSource, credentialIssueCommand, credentialRevokeCommand, credentialRotateCommand, @@ -188,6 +190,43 @@ test('waits for per-Pod network policy before mounting private material', () => assert.doesNotMatch(source, /readFile|process\.env|console\.log/); }); +test('runs the credential ceremony against two real anti-affine control replicas', () => { + const [service, deployment] = clusterControlResources( + 'qinglong3-cluster-control:test', + ); + assert.equal(service.kind, 'Service'); + assert.equal(service.spec.ports[0].port, 5800); + assert.equal(deployment.kind, 'Deployment'); + assert.equal(deployment.spec.replicas, 2); + assert.equal(deployment.spec.strategy.rollingUpdate.maxUnavailable, 0); + assert.equal( + deployment.spec.template.spec.affinity.podAntiAffinity + .requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey, + 'kubernetes.io/hostname', + ); + const environment = deployment.spec.template.spec.containers[0].env; + assert.ok( + environment.some( + (entry) => + entry.name === 'QL3_API_CREDENTIAL_PEPPER_KEYRING_FILE' && + entry.value.endsWith('/keyring.json'), + ), + ); + assert.equal( + environment.some((entry) => entry.name === 'QL3_API_CREDENTIAL_PEPPER'), + false, + ); +}); + +test('keeps the real authentication probe content-free', () => { + const source = credentialAuthenticationProbeSource(); + assert.match(source, /ql3-security-live-control/); + assert.match(source, /\/api\/v3\/projects\/prj_default\/runs\?limit=1/); + assert.match(source, /observedStatus:observed/); + assert.match(source, /bytes\?\.fill\(0\)/); + assert.doesNotMatch(source, /console\.log|process\.env/); +}); + test('live runner remains opt-in, reviewed, cleanup-bound and log-free', () => { const source = fs.readFileSync( path.resolve( @@ -206,7 +245,15 @@ test('live runner remains opt-in, reviewed, cleanup-bound and log-free', () => { assert.match(source, /net\.bridge\.bridge-nf-call-iptables=1/); assert.match(source, /wait-network-policy/); assert.match(source, /projectedMode: 0o444/); - assert.match(source, /credential\.issue\.replay/); + assert.match(source, /credential\.issue\.old\.replay/); + assert.match(source, /ql3-security-live-auth-old-before-activate/); + assert.match(source, /ql3-security-live-auth-old-overlap/); + assert.match(source, /ql3-security-live-auth-new-overlap/); + assert.match(source, /ql3-security-live-auth-old-contracted/); + assert.match(source, /ql3-security-live-auth-new-contracted/); + assert.match(source, /expectedStatus: 401/); + assert.match(source, /expectedStatus: 403/); + assert.match(source, /controlRollouts: 3/); assert.match(source, /FallbackToLogsOnError/); assert.match(source, /failureMessage: 'rejected'/); assert.match(