feat(ql3): add direct Vault KV worker secret custody

This commit is contained in:
whyour
2026-08-25 00:16:59 +08:00
parent a64d08bda1
commit 686873f8ec
25 changed files with 2830 additions and 37 deletions
+86
View File
@@ -1219,6 +1219,92 @@ jobs:
overwrite: false
include-hidden-files: false
cluster-vault-kv-worker-secret-live:
name: Vault KV Worker Secret direct custody (${{ matrix.arch }})
runs-on: ${{ matrix.runner }}
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-24.04
arch: x64
kubernetesArch: amd64
- runner: ubuntu-24.04-arm
arch: arm64
kubernetesArch: arm64
steps:
- uses: actions/checkout@v6
- uses: pnpm/action-setup@v6
with:
version: '8.3.1'
- uses: actions/setup-node@v6
with:
node-version: '24.18.0'
cache: pnpm
cache-dependency-path: pnpm-lock.yaml
- name: Verify native runner architecture
run: node -e "if (process.arch !== '${{ matrix.arch }}') throw new Error('unexpected architecture ' + process.arch)"
- name: Install workspace dependencies without lifecycle scripts
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Install verified kubectl v1.34.3
shell: bash
run: |
set -euo pipefail
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl" \
"https://dl.k8s.io/release/v1.34.3/bin/linux/${{ matrix.kubernetesArch }}/kubectl"
curl --fail --location --silent --show-error \
--output "${RUNNER_TEMP}/kubectl.sha256" \
"https://dl.k8s.io/release/v1.34.3/bin/linux/${{ matrix.kubernetesArch }}/kubectl.sha256"
test "$(cat "${RUNNER_TEMP}/kubectl.sha256")" = \
"$(sha256sum "${RUNNER_TEMP}/kubectl" | cut -d ' ' -f 1)"
chmod 0755 "${RUNNER_TEMP}/kubectl"
- name: Build the direct custody adapter
run: |
pnpm --filter @qinglong/runtime-core build
pnpm --filter @qinglong/cluster-control build
- name: Verify the fail-closed adapter and evidence contracts
run: |
node --test \
packages/ql3-cluster-control/test/vaultKvSecretProvider.test.cjs \
packages/ql3-cluster-control/test/vaultKvWorkerIngressConfig.test.cjs \
test/back/ql3VaultKvWorkerSecretLiveAudit.test.cjs \
test/back/ql3VaultKvWorkerSecretDeploymentAudit.test.cjs
pnpm audit:vault-kv-worker-secret-deployment:ql3
"${RUNNER_TEMP}/kubectl" kustomize \
deploy/kubernetes/ql3-cluster/vault-kv-worker-secret \
>/dev/null
- name: Preload the digest-pinned native Vault image
run: docker pull docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569
- name: Prove TLS, least privilege, value and token rotation, seal and recovery
env:
QL3_RUN_VAULT_KV_WORKER_SECRET_LIVE: 'true'
QL3_VAULT_KV_WORKER_SECRET_REPORT: ${{ runner.temp }}/ql3-vault-kv-worker-secret/report.json
run: |
umask 077
mkdir -m 0700 "$(dirname "${QL3_VAULT_KV_WORKER_SECRET_REPORT}")"
node scripts/ql3-vault-kv-worker-secret-live-contract.cjs >/dev/null
- name: Re-audit content-free direct custody evidence and cleanup
env:
QL3_VAULT_KV_WORKER_SECRET_REPORT: ${{ runner.temp }}/ql3-vault-kv-worker-secret/report.json
run: |
test "$(stat -c '%a' "${QL3_VAULT_KV_WORKER_SECRET_REPORT}")" = '600'
pnpm audit:vault-kv-worker-secret-live:ql3 \
"--report=${QL3_VAULT_KV_WORKER_SECRET_REPORT}"
sha256sum "${QL3_VAULT_KV_WORKER_SECRET_REPORT}"
test -z "$(docker ps -aq --filter name=ql3-vault-kv-worker-)"
- name: Upload audited content-free Vault KV custody evidence
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ql3-vault-kv-worker-secret-${{ matrix.arch }}-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/ql3-vault-kv-worker-secret/report.json
if-no-files-found: error
retention-days: 14
compression-level: 9
overwrite: false
include-hidden-files: false
cluster-plugin-package-kubernetes-live:
name: Plugin Package Kubernetes CAS and RBAC live contract
runs-on: ubuntu-24.04
+16
View File
@@ -1032,6 +1032,22 @@ Kubernetes API permission. Version-pinned SecretRefs use a different hash and
remain immutable by policy; retain their files until referenced Run/Artifact
retention has completed.
For deployments that require direct external custody, use the optional
`vault-kv-worker-secret` overlay instead of projecting value files:
```bash
kubectl kustomize \
deploy/kubernetes/ql3-cluster/vault-kv-worker-secret >/dev/null
```
That overlay mounts only a private Vault CA and a short-lived, orphan,
non-renewable read token. The adapter revalidates the token and fetches exact
digest-derived KV v2 paths over TLS 1.3 for every authorized delivery; it has no
cache, watcher, renewal loop or fallback to `mounted-files`. Follow the
overlay's README for the exact policy, payload and token-rotation contract.
External Vault HA, seal custody, audit devices, backup and disaster recovery
remain deployment-specific gates.
Create a distinct migration Secret:
```yaml
@@ -0,0 +1,50 @@
# Vault KV v2 Worker Secret custody
This overlay replaces the base `mounted-files` value projection with direct,
TLS 1.3 Vault KV v2 resolution. The control Pods mount only the private Vault
CA and one short-lived read token; Secret values and opaque Legacy Env bundles
remain in Vault and are fetched only after the durable Worker delivery authority
has been validated.
The exact Vault policy is read-only:
```hcl
path "worker-secrets/data/values/production/*" {
capabilities = ["read"]
}
path "auth/token/lookup-self" {
capabilities = ["read"]
}
```
Issue an orphan, non-renewable service token with only that policy and a TTL no
greater than 15 minutes. Rotate the projected `token` atomically before expiry;
the adapter rereads and revalidates it for every authorized resolution. A Vault
Agent or external Secret operator may own this projection, but must not project
the actual Worker Secret values.
Store each value at
`worker-secrets/data/values/production/<sha256(canonical SecretRef)>` with this
exact KV payload:
```json
{
"schemaVersion": 1,
"secretRefDigest": "<same 64-hex path key>",
"encoding": "base64",
"value": "<canonical base64 bytes>"
}
```
Create the two private Kubernetes Secret objects from
`credentials.example.yaml` through the deployment authority, then render:
```sh
kubectl kustomize deploy/kubernetes/ql3-cluster/vault-kv-worker-secret
```
The overlay intentionally does not create a NetworkPolicy because the base
control-plane deployment has no universal egress policy and external Vault
topologies differ. If the namespace is default-deny, explicitly allow DNS,
PostgreSQL, the artifact store and TCP 8200 only to the reviewed Vault identity.
@@ -0,0 +1,20 @@
# Create these objects through the deployment's Secret authority; do not commit
# real CA or token material. The Vault token must be an orphan, non-renewable
# service token with only ql3-worker-secret-read and at most a 15-minute TTL.
apiVersion: v1
kind: Secret
metadata:
name: ql3-cluster-worker-vault-trust
namespace: qinglong3-system
type: Opaque
stringData:
ca.pem: REPLACE_WITH_PRIVATE_VAULT_CA_PEM
---
apiVersion: v1
kind: Secret
metadata:
name: ql3-cluster-worker-vault-auth
namespace: qinglong3-system
type: Opaque
stringData:
token: REPLACE_WITH_SHORT_LIVED_ORPHAN_VAULT_TOKEN
@@ -0,0 +1,58 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ql3-cluster-control
spec:
template:
spec:
containers:
- name: cluster-control
env:
- name: QL3_WORKER_SECRET_PROVIDER
value: vault-kv-v2
- name: QL3_WORKER_SECRET_ROOT_DIRECTORY
$patch: delete
- name: QL3_WORKER_SECRET_VAULT_ENDPOINT
value: https://vault.vault.svc.cluster.local:8200
- name: QL3_WORKER_SECRET_VAULT_CA_FILE
value: /var/run/secrets/qinglong3/worker-vault-trust/ca.pem
- name: QL3_WORKER_SECRET_VAULT_TOKEN_FILE
value: /var/run/secrets/qinglong3/worker-vault-auth/token
- name: QL3_WORKER_SECRET_VAULT_KV_MOUNT
value: worker-secrets
- name: QL3_WORKER_SECRET_VAULT_PATH_PREFIX
value: values/production
- name: QL3_WORKER_SECRET_VAULT_EXPECTED_POLICY
value: ql3-worker-secret-read
- name: QL3_WORKER_SECRET_VAULT_MAX_TOKEN_TTL_SECONDS
value: '900'
- name: QL3_WORKER_SECRET_VAULT_REQUEST_TIMEOUT_MS
value: '5000'
- name: QL3_WORKER_SECRET_VAULT_MAX_CONCURRENCY
value: '4'
volumeMounts:
- name: worker-secret-values
$patch: delete
- name: worker-vault-trust
mountPath: /var/run/secrets/qinglong3/worker-vault-trust
readOnly: true
- name: worker-vault-auth
mountPath: /var/run/secrets/qinglong3/worker-vault-auth
readOnly: true
volumes:
- name: worker-secret-values
$patch: delete
- name: worker-vault-trust
secret:
secretName: ql3-cluster-worker-vault-trust
defaultMode: 292
items:
- key: ca.pem
path: ca.pem
- name: worker-vault-auth
secret:
secretName: ql3-cluster-worker-vault-auth
defaultMode: 288
items:
- key: token
path: token
@@ -0,0 +1,8 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../base
patches:
- path: deployment-patch.yaml
+7 -6
View File
@@ -11,6 +11,8 @@
最新增量证据(2026-08-24):
- D-404/ADR-0499(已验收):Cluster Worker 现在有可选的直接外部 Secret custody adapter,而不再只能依赖 Kubernetes Secret value projection。`vault-kv-v2` 位于既有 `@qinglong/cluster-control` Remote Execution 子域,只有显式选择 provider 的 Cluster 进程才动态加载;基础部署继续使用 `mounted-files`Edge/Standalone 不新增 package、依赖、daemon、timer、watcher、连接池或常驻内存。adapter 只在 durable Run/Attempt/Lease/Worker Session/execution digest/SecretRef authority 通过后,用 `SHA-256(canonical SecretRef)` 路径读取 KV v2;只接受显式私有 CA 的 TLS 1.3、每次重新打开的短期 orphan/non-renewable service token 和唯一精确 policy,不跟随 redirect、不使用系统 CA 回退、不缓存值或 token。Kubernetes overlay 删除 value Secret projection,只挂载 CA 与 token;普通 Secret、opaque environment bundle 和总响应仍受原 16/96/256 KiB 边界约束,空 Secret 保持合法,异常 envelope、metadata、digest、token、TLS 或 Vault availability 均失败关闭且不回退。真实 arm64 Vault 1.21.4 gate 已完成 3-share/2-threshold init、两个普通 Secret 与一个 bundle、value/token 原子轮换、旧 accessor revoke、缺失 material、不可信 CA、seal/unseal 与同持久存储容器替换;`0600` content-free 报告 SHA-256 为 `df225509cb763009b610cb0aea2207e0b07b5e05a44cf8cf0dff1633c1624d52`audit 为 `compatible=true/findings=[]`。Cluster Control 为 `279 total / 277 pass / 2 conditional skip / 0 fail`backend 为 `1574 total / 1572 pass / 2 conditional skip / 0 fail`18-package clean build/test 退出 0package、Cluster dependency、122-module Edge import、部署和 14 档 Local artifact 审计全部 compatible,基础 Edge/Standalone 仍为 `2,669,390 / 2,669,468 bytes`、325 files、58 modules。共享 CI 新增原生 x64/arm64 live matrix。该 fixture 关闭 QingLong 直接 custody adapter/data-boundary 门,不冒充生产 Vault HA、KMS/HSM seal、审计设备或灾备证明;ADR-0491 现在只剩固定低性能物理 Edge 的真实空间、RSS/I/O、写放大、ENOSPC 与断电恢复门。
- D-397/ADR-0491(进行中):Secret/Config reconciliation 已先冻结行为保持边界,不能把“密文已保存”冒充“任务已迁移”。2.x `Envs`
的 active 行必须按 `isPinned DESC、position DESC、createdAt ASC、id ASC` 重放旧顺序,同名值用 `&` 形成唯一 effective Secret;该
Secret 后续必须在同一事务中绑定到经 Automation adoption ledger 证明的全部 Legacy Task 新修订,并同步追加指向新 Task revision 的 Trigger/dispatch
@@ -87,8 +89,7 @@
因而不重跑且不重新占有 PostgreSQL HA 证明;相邻已通过的 remote CI/HA 只作为基线。
D-385D-388 的 `config.sh`/Keyv/SSH data-directory lineage 与 SQLite `Envs` 保持分离;当前无稳定生产 schema 的历史 `Configs` 表继续 sealed+manual
不猜字段。ADR-0494 已关闭基础 Cluster mounted-files provider live 子门;后续切片仍必须完成固定低性能设备的真实 Edge 空间/写放大/断电证据,
以及 Cluster Legacy Env migration 的专用 PostgreSQL SERIALIZABLE ledger、Task/Trigger revision mutation 与直接外部 custody adapter。HA promotion 后 receipt replay 已由 D-403/ADR-0498 关闭。D-397 apply
不猜字段。ADR-0494 已关闭基础 Cluster mounted-files provider live 子门ADR-0495ADR-0498 已关闭 Cluster plan/application/HA replayADR-0499 已关闭直接 Vault KV 外部 custody adapter;后续仍必须完成固定低性能设备的真实 Edge 空间、RSS/I/O、写放大、ENOSPC 与断电证据。D-397 apply
只声明 sealed source retained 且 `physicalErasureGuaranteed=false`;明文销毁必须在 restart/
readiness、观察窗和 rollback retention 之后另行强认证。Cluster 必须使用 PostgreSQL SERIALIZABLE ledger、外部 KMS/Secret provider 与 HA evidence
不复用 Local SQLite/POSIX authority,也不得把明文写入 PostgreSQL、ConfigMap、Pod env 或 Job command。
@@ -174,8 +175,8 @@
`a7c8a05e08c748d677475a09ce2998b741ed9326e27678613e93d431bc3769aa`
真实用例覆盖 Trigger 固定 Task r1、Task current r2、原子生成 Task r3/Trigger r2 并重定向 pin
同时证明 bundle ref-only execution、schedule reset、无流消费 replay 与数据库角色隔离。D-402
关闭 mutation/receipt 边界;相邻 D-403 已继续关闭 promotion 后 receipt replaydirect external
custody 固定低性能 Edge 物理证据是 ADR-0491 转 Accepted 前门禁。
关闭 mutation/receipt 边界;相邻 D-403 已继续关闭 promotion 后 receipt replayD-404/ADR-0499 已关闭 direct external
custody adapter固定低性能 Edge 物理证据是 ADR-0491 转 Accepted 前唯一剩余门禁。
- D-403/ADR-0498(已验收):ADR-0497 application receipt 已进入真实 PostgreSQL 18 physical HA
领域门,而不是继续引用通用表存活作为间接证据。主库与 standby 达到
@@ -199,8 +200,8 @@
`1569 total / 1567 pass / 2 conditional skip / 0 fail`,四项 package/依赖/import 边界均 compatible。
基础 Edge/Standalone 仍只包含 Local SQLite、runtime-core 与 SemVer,均为 325 files/58 modules
大小 `2,669,390 / 2,669,468 bytes`,距 4 MiB 上限保留 `1,524,914 / 1,524,836 bytes`;它只证明
常驻闭包未扩大,不冒充真实设备证据。D-403 关闭 promotion replayADR-0491 现在只剩 direct
external custody 与固定低性能 Edge 空间/写放大/断电恢复两项硬门。
常驻闭包未扩大,不冒充真实设备证据。D-403 关闭 promotion replayD-404/ADR-0499 关闭 direct
external custody adapterADR-0491 现在只剩固定低性能 Edge 空间、RSS/I/O、写放大、ENOSPC 与断电恢复硬门。
- D-396/ADR-0490(已验收):Run History 不再只有永久 `manual_external`,但也没有被错误实现为 Legacy 日志到 3.0 Run ledger 的回灌。
新的 Local adapter 以 ADR-0482 sealed capture bundle 作为 append-only 保全资产:Legacy history 必须逐事实选择 `retain_both`Target history
@@ -1,9 +1,9 @@
# ADR-0491:有界 Secret/Config Reconciliation 与任务环境绑定
- 状态:ProposedD-397 已实现 Legacy Env inspection、私有有界 row plan、durable plan publication、独立 signed decision、逐项 Automation adoption provenance、Local SQLite 原子 application publisher、Owner prepared/apply/rollback 编排、ADR-0492 completion v3ADR-0494 完成 Cluster mounted-files provider live 子门,ADR-0495 完成 content-free Cluster plan ledgerADR-0496 完成 opaque environment bundle 数据面,ADR-0497 完成 Cluster Task/Trigger 原子 mutation 与 receiptADR-0498 完成 promotion 后 exact replay;真实 Edge 空间证据与直接外部 custody gate 尚未完成)
- 状态:ProposedD-397 已实现 Legacy Env inspection、私有有界 row plan、durable plan publication、独立 signed decision、逐项 Automation adoption provenance、Local SQLite 原子 application publisher、Owner prepared/apply/rollback 编排、ADR-0492 completion v3ADR-0494 完成 Cluster mounted-files provider live 子门,ADR-0495 完成 content-free Cluster plan ledgerADR-0496 完成 opaque environment bundle 数据面,ADR-0497 完成 Cluster Task/Trigger 原子 mutation 与 receiptADR-0498 完成 promotion 后 exact replayADR-0499 完成直接 Vault KV 外部 custody adapter;仅固定物理 Edge 证据尚未完成)
- 日期:2026-08-23
- 决策:D-397
- 关联:ADR-0073、ADR-0074、ADR-0092、ADR-0094、ADR-0480、ADR-0482、ADR-0483、ADR-0484、ADR-0485、ADR-0486、ADR-0487、ADR-0488、ADR-0490、ADR-0495、ADR-0496、ADR-0497、ADR-0498
- 关联:ADR-0073、ADR-0074、ADR-0092、ADR-0094、ADR-0480、ADR-0482、ADR-0483、ADR-0484、ADR-0485、ADR-0486、ADR-0487、ADR-0488、ADR-0490、ADR-0495、ADR-0496、ADR-0497、ADR-0498、ADR-0499
## 背景
@@ -147,4 +147,4 @@ D-397 当前八切片已经实现:absent、unsupported、Edge over-budget、2.
ADR-0494 已完成 Cluster `mounted-files` provider live 子门:真实三节点 K3s 中两个 management replica、direct exact-key executor 和两个跨节点 provider observer 完成 PostgreSQL durable approval/binding、Kubernetes atomic projection rotation、无 Secret API 权限/ServiceAccount token、只读 `0440`、内容脱敏及删除后 fail-closedv2 私有报告 24/24 gates 为 true,并保持 v1 verifier 兼容。该门不增加 Edge 闭包,也不等于直接 Vault/KMS/HSM custody。
转为 Accepted 前仍必须完成固定低性能 Edge 设备的真实空间/写放大/断电恢复证据,以及直接外部 custody adapter。ADR-0495 已完成专用 PostgreSQL plan ledgerADR-0496 已完成只保存 pinned bundle ref、通过 fenced remote delivery 取回 typed carrier 并在 Worker 内存展开的数据面;ADR-0497 又在一个 Project-serialized SERIALIZABLE transaction 中完成逐项 Task/Trigger current-head revalidation、revision/execution mutation、schedule reset 和 content-free append-only receipt,并支持合法历史 Task pin。ADR-0498 已证明该首次提交在 `remote_apply` 下到达 standby,并在 timeline `1→2``pg_rewind` 重建同步副本后由 Automation Manager 精确重放为 `existing`,不打开 mutation stream 或增加 durable row。它仍不写入 Secret material,也不等于 direct Vault/KMS/HSM custody。ADR-0492 已完成本机 completion schema 演进和 completed-head 后 rollback material 回收,ADR-0493 又让没有 Legacy 身份输入的 fresh v52 目标身份经 signed `retain_target` 正确形成 no-effect,并精确消除六张已知目标表的 `unknown` 误判。Legacy `Auths/Users` 或真正未知表仍保持 manualLocal Owner 编排、mounted-files gate、plan/application ledger 或 HA replay 证据都不得冒充完整外部密钥托管
转为 Accepted 前仍必须完成固定低性能 Edge 设备的真实空间、RSS/I/O、写放大、ENOSPC 与断电恢复证据。ADR-0495 已完成专用 PostgreSQL plan ledgerADR-0496 已完成只保存 pinned bundle ref、通过 fenced remote delivery 取回 typed carrier 并在 Worker 内存展开的数据面;ADR-0497 又在一个 Project-serialized SERIALIZABLE transaction 中完成逐项 Task/Trigger current-head revalidation、revision/execution mutation、schedule reset 和 content-free append-only receipt,并支持合法历史 Task pin。ADR-0498 已证明该首次提交在 `remote_apply` 下到达 standby,并在 timeline `1→2``pg_rewind` 重建同步副本后由 Automation Manager 精确重放为 `existing`,不打开 mutation stream 或增加 durable row。ADR-0499 进一步增加可选 `vault-kv-v2` provider:它只在 durable delivery authority 通过后,以 digest-derived path、显式 CA、TLS 1.3 和每次重新验证的短期最小权限 token 直接读取外部 VaultKubernetes overlay 不再投影真实值,本地 arm64 live gate 已覆盖 value/token rotation、revoke、seal/unseal、容器替换持久恢复与 content-free evidence。该单机 Vault fixture 只关闭 QingLong adapter/data-boundary 门,不冒充生产 Vault HA、KMS/HSM seal、审计设备或灾备证明。ADR-0492 已完成本机 completion schema 演进和 completed-head 后 rollback material 回收,ADR-0493 又让没有 Legacy 身份输入的 fresh v52 目标身份经 signed `retain_target` 正确形成 no-effect,并精确消除六张已知目标表的 `unknown` 误判。Legacy `Auths/Users` 或真正未知表仍保持 manual。
@@ -0,0 +1,104 @@
# ADR-0499:直接 Vault KV Worker Secret 外部托管
- 状态:Accepted
- 日期:2026-08-24
- 决策:D-404
- 关联:ADR-0058、ADR-0114、ADR-0233、ADR-0491、ADR-0494、ADR-0496、ADR-0497、ADR-0498
## 背景
ADR-0494 已证明 `mounted-files` provider 可以在 Kubernetes atomic projection 轮换后无重启读取新值,并且 Worker ingress 不需要 Secret API 权限。但该模式仍由 Kubernetes Secret 保存真实值,不能满足要求控制面只持有短期访问能力、密钥材料始终由外部托管系统保管的部署。
QingLong 3.0 需要一个可选的直接外部 custody adapter,同时保持小型 Edge/Standalone 默认闭包不变,也不能把 Vault token、Secret 值、Legacy Env 名称或 provider 路径写入 PostgreSQL、Pod environment、公开 evidence 或日志。
## 决策
### 1. Provider 边界
在既有 `@qinglong/cluster-control` Remote Execution 子域中增加 `vault-kv-v2` provider,不新建 workspace package。基础 Kubernetes 部署继续默认使用 `mounted-files`;只有显式选择 `QL3_WORKER_SECRET_PROVIDER=vault-kv-v2` 的 Cluster overlay 才加载该实现。Edge/Standalone 不导入 Cluster Control,因此其制品、常驻内存和依赖树不增加 Vault 客户端。
provider 只在 durable Worker delivery authority 已经通过 Run、Attempt、Lease、Session、execution digest 和精确 SecretRef 集合校验后执行。每个路径由 `SHA-256(canonical SecretRef)` 推导,不把 SecretRef 或环境变量名放入 URL。普通 Secret 与 opaque environment bundle 继续分别服从 16 KiB、96 KiB 和总交付上限。
### 2. 传输与身份
实现只接受无凭据、无 path/query/fragment 的 `https://` authority,使用显式投影 CA,并要求协商 TLS 1.3;不跟随 redirect,不使用系统 CA 回退。控制 Pod 只投影 CA 和短期 token,不投影真实 Secret 值,也不挂载 ServiceAccount token。
token 必须同时满足:
- orphan、non-renewable、service token
- 只含一个精确配置的读取 policy;
- 剩余 TTL 大于 0 且不超过配置上限,部署默认上限为 900 秒;
- 每次 `verify``resolve` 都重新打开投影文件并调用 `auth/token/lookup-self`
实现没有 cache、watcher、timer、renewal loop 或后台连接。token 轮换由外部部署 authority 原子替换投影文件;旧 token 的撤销与新 token 的签发不由 QingLong 接管。Vault 不可达、封存、CA 不可信、token 过宽/过期或材料缺失时交付失败关闭,不回退到 `mounted-files`
### 3. KV v2 数据合同
每个值固定存放在:
```text
<mount>/data/<prefix>/<sha256(canonical SecretRef)>
```
KV payload 必须是 exact shape
```json
{
"schemaVersion": 1,
"secretRefDigest": "<same 64-hex digest>",
"encoding": "base64",
"value": "<canonical base64>"
}
```
KV metadata 必须表明 version 为正整数、未删除且未销毁。空 Secret 是合法的 canonical base64;超限、非 canonical 编码、digest 不一致、未知字段或异常 metadata 均失败关闭。成功 material 在调用方 `dispose()` 时清零;报告和错误只暴露有界分类,不包含 endpoint path、token、SecretRef 或 value。
### 4. Kubernetes 部署
`deploy/kubernetes/ql3-cluster/vault-kv-worker-secret` overlay 删除基础部署的 value Secret env/mount/volume,只增加私有 CA 与短期 token 投影。示例文件只含占位符,仓库不保存可用 credential。
Vault policy 固定为:
```hcl
path "worker-secrets/data/values/production/*" {
capabilities = ["read"]
}
path "auth/token/lookup-self" {
capabilities = ["read"]
}
```
外部 Vault 的 HA、unseal、KMS/HSM seal、审计设备、备份和灾备属于部署方独立门禁。本 ADR 证明 QingLong 的直接 custody 数据边界,不把单机 Vault file storage fixture 冒充生产 Vault HA。
## 被拒绝的替代方案
### 在 PostgreSQL 或 ConfigMap 保存密文
拒绝。即使值已加密,也会扩大数据库/配置平面的 custody、备份和读取权限,并把轮换与数据库生命周期绑定。
### 给控制 Pod Kubernetes Secret API 权限
拒绝。它扩大为 namespace 级读取能力,无法把访问限制到 durable delivery authority 中的精确引用。
### 引入 Vault SDK、Agent sidecar 或常驻续租器作为必选依赖
拒绝。直接 HTTPS adapter 已能满足有界请求;额外 SDK 或 sidecar 会扩大依赖、镜像、常驻内存和凭据生命周期。部署方可以负责 token projection,但不得通过该机制把真实值重新投影进 Pod。
## 验证
真实 arm64 live contract 使用 digest-pinned Vault 1.21.4、私有 TLS CA、3-share/2-threshold 初始化和短期 orphan token,完成两个普通 Secret 与一个 opaque environment bundle 的读取,并证明:
- value 轮换和 token 原子轮换均无需重启;
- 旧 token accessor 撤销后不再可用;
- 缺失 material、不可信 CA 和 sealed Vault 均失败关闭;
- unseal 后恢复,Vault 容器在同一持久存储上替换后值仍可读取;
- 私有 `0600` 报告不含 token、SecretRef、value、证书私钥或 Vault path。
最终本地报告 SHA-256 为 `df225509cb763009b610cb0aea2207e0b07b5e05a44cf8cf0dff1633c1624d52`,离线 audit 为 `compatible=true``findings=[]`。共享 CI 在原生 x64/arm64 runner 上分别构建 provider、审计 overlay、拉取同一 digest-pinned Vault image 并重跑完整 live contract;远程运行结果作为提交后的独立证据。
本地完整验证中,Cluster Control 为 `279 total / 277 pass / 2 conditional skip / 0 fail`backend 为 `1574 total / 1572 pass / 2 conditional skip / 0 fail`18-package clean build/test 退出 0。package boundary 保持 18 packages、`singleSourcePackages=[]``shallowSourcePackages=[]`Cluster dependency、122-module Edge import、service-manager bridge、Cluster deployment、Local image、Vault overlay 和 14 档 Local artifact 均 compatible。基础 Edge/Standalone 制品仍为 `2,669,390 / 2,669,468 bytes`、325 files、58 loaded modules,不包含 Cluster Control 或 Vault adapter。
## 影响与剩余门禁
D-404 关闭 ADR-0491 的直接外部 custody adapter 门。ADR-0491 转 Accepted 前只剩固定低性能物理 Edge 的真实空间、RSS/I/O、写放大、ENOSPC 与断电恢复证据;开发机、Docker、CI runner 和 Cluster Vault live gate 均不能替代该设备证据。
+3 -1
View File
@@ -494,13 +494,15 @@
| [ADR-0488](./ADR-0488-cross-domain-reconciliation-completion-fence.md) | 跨领域 Reconciliation 完成围栏与目标重启授权 | Accepted |
| [ADR-0489](./ADR-0489-service-manager-completion-restart-lineage.md) | Service Manager 完成围栏重启谱系 | Accepted |
| [ADR-0490](./ADR-0490-run-history-terminal-preservation.md) | Run History 终态保全与跨领域完成证明 | Accepted |
| [ADR-0491](./ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md) | 有界 Secret/Config Reconciliation 与任务环境绑定 | ProposedD-397 local apply/rollback/completion completeEdge/Cluster gate pending |
| [ADR-0491](./ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md) | 有界 Secret/Config Reconciliation 与任务环境绑定 | Proposed直接外部 custody 已完成;仅固定物理 Edge gate pending |
| [ADR-0492](./ADR-0492-compatible-secret-config-completion-v3.md) | 兼容 Secret/Config 的 Reconciliation Completion v3 | Accepted |
| [ADR-0493](./ADR-0493-target-native-reconciliation-domain-preservation.md) | 目标原生 Reconciliation 域分类与身份保留 | Accepted |
| [ADR-0494](./ADR-0494-postgresql-secret-binding-and-mounted-provider-live-rotation.md) | PostgreSQL Secret Binding 与 Mounted Provider 在线轮换门 | Accepted |
| [ADR-0495](./ADR-0495-content-free-cluster-legacy-env-migration-plan-ledger.md) | 无敏感内容的 Cluster Legacy Env 迁移计划账本 | Accepted |
| [ADR-0496](./ADR-0496-opaque-cluster-environment-bundle-delivery.md) | Cluster 不透明环境 Bundle 的有界交付与 Worker 内存展开 | Accepted |
| [ADR-0497](./ADR-0497-atomic-cluster-legacy-env-migration-application.md) | Cluster Legacy Env 的原子 Task/Trigger 迁移与只追加回执 | Accepted |
| [ADR-0498](./ADR-0498-cluster-legacy-env-application-ha-replay.md) | Cluster Legacy Env Application 的 HA Promotion 后精确重放 | Accepted |
| [ADR-0499](./ADR-0499-direct-vault-kv-worker-secret-custody.md) | 直接 Vault KV Worker Secret 外部托管 | Accepted |
## 规则
+3
View File
@@ -87,6 +87,9 @@
"audit:prompt-output-external-recovery-deployment:ql3": "node scripts/ql3-prompt-output-external-recovery-deployment-audit.cjs",
"custody:vault-transit:ql3": "node scripts/ql3-vault-transit-prompt-output-custody.cjs",
"test:vault-transit-custody-live:ql3": "pnpm --filter @qinglong/ai build && pnpm --filter @qinglong/cluster-admin build && node scripts/ql3-vault-transit-prompt-output-custody-live-contract.cjs",
"test:vault-kv-worker-secret-live:ql3": "pnpm --filter @qinglong/runtime-core build && pnpm --filter @qinglong/cluster-control build && node scripts/ql3-vault-kv-worker-secret-live-contract.cjs",
"audit:vault-kv-worker-secret-live:ql3": "node scripts/ql3-vault-kv-worker-secret-live-audit.cjs",
"audit:vault-kv-worker-secret-deployment:ql3": "node scripts/ql3-vault-kv-worker-secret-deployment-audit.cjs",
"test:postgres-backup-prompt-output-recovery-live:ql3": "pnpm --filter @qinglong/ai build && pnpm --filter @qinglong/cluster-admin build && node scripts/ql3-postgres-prompt-output-recovery-live-contract.cjs",
"audit:cluster-deployment:ql3": "node scripts/ql3-cluster-deployment-audit.cjs",
"audit:cluster-copilot-console:ql3": "node scripts/ql3-cluster-copilot-console-audit.cjs",
@@ -150,6 +150,11 @@
"require": "./dist/remote-execution/mountedSecretProvider.js",
"default": "./dist/remote-execution/mountedSecretProvider.js"
},
"./vault-kv-secret-provider": {
"types": "./dist/remote-execution/vaultKvSecretProvider.d.ts",
"require": "./dist/remote-execution/vaultKvSecretProvider.js",
"default": "./dist/remote-execution/vaultKvSecretProvider.js"
},
"./trusted-tool-result-keyring": {
"types": "./dist/trusted-tool/key-management/toolResultProjectedKeyring.d.ts",
"require": "./dist/trusted-tool/key-management/toolResultProjectedKeyring.js",
@@ -128,8 +128,11 @@ async function createWorkerArtifactBinding(
async function createWorkerSecretProvider(
config: NonNullable<EnabledClusterWorkerIngressConfig['secret']>,
): Promise<Readonly<RemoteWorkerSecretValueProvider>> {
if (config.provider !== 'mounted-files') {
throw new TypeError('Cluster Worker Secret provider is unsupported');
if (config.provider === 'vault-kv-v2') {
const provider = await import(
'../remote-execution/vaultKvSecretProvider.js'
);
return provider.createClusterVaultKvSecretProvider(config);
}
const provider = await import('../remote-execution/mountedSecretProvider.js');
return provider.createClusterMountedSecretProvider({
@@ -0,0 +1,637 @@
// Remote Execution owns direct external Secret custody resolution.
import { Buffer } from 'node:buffer';
import https from 'node:https';
import { isIP } from 'node:net';
import { basename, dirname, isAbsolute, normalize } from 'node:path';
import type { TLSSocket } from 'node:tls';
import {
MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES,
MAX_REMOTE_SECRET_VALUE_BYTES,
normalizeRemoteWorkerSecretDeliveryAuthority,
type RemoteWorkerSecretDeliveryAuthority,
type RemoteWorkerSecretResolution,
type RemoteWorkerSecretValueProvider,
} from '@qinglong/runtime-core/remote-secret-delivery';
import { secretProjectionFileName } from '@qinglong/runtime-core/secret-projection';
import { PrivateProjectedFileReader } from '../security/privateProjectedFile';
const MAX_CA_BYTES = 1024 * 1024;
const MAX_TOKEN_BYTES = 4096;
const MAX_RESPONSE_BYTES =
MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES + 16 * 1024;
const TOKEN_PATTERN = /^[\x21-\x7e]{1,4096}$/;
const NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/;
const PREFIX_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_/-]{0,511}$/;
const FILE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,252}$/;
const BASE64_PATTERN =
/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;
export interface ClusterVaultKvSecretProviderOptions {
readonly endpoint: string;
readonly caFile: string;
readonly tokenFile: string;
readonly kvMount: string;
readonly pathPrefix: string;
readonly expectedPolicy: string;
readonly maximumTokenTtlSeconds: number;
readonly requestTimeoutMs: number;
readonly maximumConcurrency: number;
readonly namespace?: string;
}
interface NormalizedClusterVaultKvSecretProviderOptions
extends ClusterVaultKvSecretProviderOptions {
readonly endpoint: string;
readonly caRootDirectory: string;
readonly caFileName: string;
readonly tokenRootDirectory: string;
readonly tokenFileName: string;
}
export interface ClusterVaultKvRequest {
readonly endpoint: string;
readonly path: string;
readonly requestTimeoutMs: number;
readonly namespace?: string;
}
export type ClusterVaultKvRequester = (
request: Readonly<ClusterVaultKvRequest>,
material: Readonly<{ readonly ca: Buffer; readonly token: string }>,
) => Promise<unknown>;
export interface ClusterVaultKvSecretProviderDependencies {
readonly request?: ClusterVaultKvRequester;
}
export class ClusterVaultKvSecretProviderError extends Error {
readonly code = 'QL3_CLUSTER_VAULT_KV_SECRET_UNAVAILABLE';
constructor(
readonly reason:
| 'invalid_configuration'
| 'trust_unavailable'
| 'authentication_unavailable'
| 'material_unavailable',
options?: ErrorOptions,
) {
super(`Cluster Vault KV Secret provider failed: ${reason}`, options);
this.name = 'ClusterVaultKvSecretProviderError';
}
}
function invalidConfiguration(cause?: unknown): never {
throw new ClusterVaultKvSecretProviderError('invalid_configuration', {
cause: cause instanceof Error ? cause : undefined,
});
}
function boundedInteger(
value: unknown,
minimum: number,
maximum: number,
): number {
if (
typeof value !== 'number' ||
!Number.isSafeInteger(value) ||
value < minimum ||
value > maximum
) {
return invalidConfiguration();
}
return value;
}
function safeName(value: unknown): string {
if (typeof value !== 'string' || !NAME_PATTERN.test(value)) {
return invalidConfiguration();
}
return value;
}
function safePrefix(value: unknown): string {
if (
typeof value !== 'string' ||
!PREFIX_PATTERN.test(value) ||
value.includes('//') ||
value.split('/').some((part) => part === '.' || part === '..')
) {
return invalidConfiguration();
}
return value;
}
function absoluteProjectedFile(value: unknown): Readonly<{
rootDirectory: string;
fileName: string;
}> {
if (
typeof value !== 'string' ||
!isAbsolute(value) ||
normalize(value) !== value ||
value.includes('\0') ||
Buffer.byteLength(value, 'utf8') > 4096
) {
return invalidConfiguration();
}
const fileName = basename(value);
const rootDirectory = dirname(value);
if (
!FILE_NAME_PATTERN.test(fileName) ||
rootDirectory === value ||
rootDirectory === dirname(rootDirectory)
) {
return invalidConfiguration();
}
return Object.freeze({ rootDirectory, fileName });
}
export function normalizeClusterVaultKvSecretProviderOptions(
value: ClusterVaultKvSecretProviderOptions,
): Readonly<NormalizedClusterVaultKvSecretProviderOptions> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
return invalidConfiguration();
}
let endpoint: URL;
try {
endpoint = new URL(value.endpoint);
} catch (cause) {
return invalidConfiguration(cause);
}
if (
endpoint.protocol !== 'https:' ||
endpoint.username ||
endpoint.password ||
endpoint.pathname !== '/' ||
endpoint.search ||
endpoint.hash
) {
return invalidConfiguration();
}
const ca = absoluteProjectedFile(value.caFile);
const token = absoluteProjectedFile(value.tokenFile);
const namespace =
value.namespace === undefined ? undefined : safePrefix(value.namespace);
return Object.freeze({
endpoint: endpoint.toString(),
caFile: value.caFile,
caRootDirectory: ca.rootDirectory,
caFileName: ca.fileName,
tokenFile: value.tokenFile,
tokenRootDirectory: token.rootDirectory,
tokenFileName: token.fileName,
kvMount: safeName(value.kvMount),
pathPrefix: safePrefix(value.pathPrefix),
expectedPolicy: safeName(value.expectedPolicy),
maximumTokenTtlSeconds: boundedInteger(
value.maximumTokenTtlSeconds,
30,
3600,
),
requestTimeoutMs: boundedInteger(value.requestTimeoutMs, 100, 30_000),
maximumConcurrency: boundedInteger(value.maximumConcurrency, 1, 8),
...(namespace === undefined ? {} : { namespace }),
});
}
function exactKeys(value: unknown, expected: readonly string[]): boolean {
return (
value !== null &&
typeof value === 'object' &&
!Array.isArray(value) &&
JSON.stringify(Object.keys(value).sort()) ===
JSON.stringify([...expected].sort())
);
}
function canonicalBase64(value: unknown): Buffer {
if (
typeof value !== 'string' ||
value.length >
Math.ceil(MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES / 3) * 4 + 4 ||
!BASE64_PATTERN.test(value)
) {
throw new ClusterVaultKvSecretProviderError('material_unavailable');
}
const bytes = Buffer.from(value, 'base64');
if (bytes.toString('base64') !== value) {
bytes.fill(0);
throw new ClusterVaultKvSecretProviderError('material_unavailable');
}
return bytes;
}
function parseVaultKvValue(
value: unknown,
expectedSecretRefDigest: string,
maximumBytes: number,
): Buffer {
const envelope = value as {
readonly data?: {
readonly data?: unknown;
readonly metadata?: {
readonly deletion_time?: unknown;
readonly destroyed?: unknown;
readonly version?: unknown;
};
};
};
const material = envelope?.data?.data as {
readonly schemaVersion?: unknown;
readonly secretRefDigest?: unknown;
readonly encoding?: unknown;
readonly value?: unknown;
};
const metadata = envelope?.data?.metadata;
if (
!exactKeys(material, [
'encoding',
'schemaVersion',
'secretRefDigest',
'value',
]) ||
material.schemaVersion !== 1 ||
material.secretRefDigest !== expectedSecretRefDigest ||
material.encoding !== 'base64' ||
metadata?.destroyed !== false ||
metadata?.deletion_time !== '' ||
!Number.isSafeInteger(metadata?.version) ||
Number(metadata?.version) < 1
) {
throw new ClusterVaultKvSecretProviderError('material_unavailable');
}
const bytes = canonicalBase64(material.value);
if (bytes.byteLength > maximumBytes) {
bytes.fill(0);
throw new ClusterVaultKvSecretProviderError('material_unavailable');
}
return bytes;
}
function parseToken(bytes: Buffer): string {
const raw = bytes.toString('utf8');
const token = raw.endsWith('\n') ? raw.slice(0, -1) : raw;
if (!TOKEN_PATTERN.test(token) || (raw !== token && raw !== `${token}\n`)) {
throw new ClusterVaultKvSecretProviderError('authentication_unavailable');
}
return token;
}
function secretValue(bytes: Buffer): string {
try {
const value = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
if (value.includes('\0')) throw new Error('Secret contains NUL');
return value;
} catch (cause) {
throw new ClusterVaultKvSecretProviderError('material_unavailable', {
cause: cause instanceof Error ? cause : undefined,
});
}
}
function validateTokenLookup(
value: unknown,
expectedPolicy: string,
maximumTokenTtlSeconds: number,
): void {
const data = (value as { readonly data?: Record<string, unknown> })?.data;
if (
!data ||
!Array.isArray(data.policies) ||
JSON.stringify(data.policies) !== JSON.stringify([expectedPolicy]) ||
data.orphan !== true ||
data.renewable !== false ||
data.type !== 'service' ||
!Number.isSafeInteger(data.ttl) ||
Number(data.ttl) < 1 ||
Number(data.ttl) > maximumTokenTtlSeconds
) {
throw new ClusterVaultKvSecretProviderError('authentication_unavailable');
}
}
async function requestVaultJson(
request: Readonly<ClusterVaultKvRequest>,
material: Readonly<{ readonly ca: Buffer; readonly token: string }>,
): Promise<unknown> {
const target = new URL(request.path, request.endpoint);
return new Promise((resolve, reject) => {
const requestHandle = https.request(
target,
{
method: 'GET',
ca: material.ca,
rejectUnauthorized: true,
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
...(isIP(target.hostname) === 0 ? { servername: target.hostname } : {}),
headers: {
accept: 'application/json',
'x-vault-token': material.token,
...(request.namespace === undefined
? {}
: { 'x-vault-namespace': request.namespace }),
},
},
(response) => {
const socket = response.socket as TLSSocket;
const peerAuthorized = socket.authorized;
const tlsProtocol = socket.getProtocol();
const chunks: Buffer[] = [];
let length = 0;
const dispose = () =>
chunks.splice(0).forEach((chunk) => chunk.fill(0));
response.on('data', (chunk: Buffer) => {
length += chunk.byteLength;
if (length > MAX_RESPONSE_BYTES) {
response.destroy(
new ClusterVaultKvSecretProviderError('material_unavailable'),
);
return;
}
chunks.push(Buffer.from(chunk));
});
response.once('error', (cause) => {
dispose();
reject(cause);
});
response.on('end', () => {
const bytes = Buffer.concat(chunks);
dispose();
try {
if (
response.statusCode !== 200 ||
peerAuthorized !== true ||
tlsProtocol !== 'TLSv1.3'
) {
throw new ClusterVaultKvSecretProviderError(
'material_unavailable',
);
}
resolve(JSON.parse(bytes.toString('utf8')));
} catch (cause) {
reject(cause);
} finally {
bytes.fill(0);
}
});
},
);
requestHandle.setTimeout(request.requestTimeoutMs, () => {
requestHandle.destroy(
new ClusterVaultKvSecretProviderError('material_unavailable'),
);
});
requestHandle.once('error', reject);
requestHandle.end();
});
}
/**
* A no-cache, no-watcher Vault KV v2 adapter. Every authorized delivery reads
* the projected short-lived token again, revalidates its least-privilege
* policy, and resolves only digest-derived paths over pinned TLS 1.3.
*/
export class ClusterVaultKvSecretProvider
implements RemoteWorkerSecretValueProvider
{
readonly #options: Readonly<NormalizedClusterVaultKvSecretProviderOptions>;
readonly #caReader: PrivateProjectedFileReader;
readonly #tokenReader: PrivateProjectedFileReader;
readonly #request: ClusterVaultKvRequester;
constructor(
options: ClusterVaultKvSecretProviderOptions,
dependencies: ClusterVaultKvSecretProviderDependencies = {},
) {
this.#options = normalizeClusterVaultKvSecretProviderOptions(options);
if (
!dependencies ||
typeof dependencies !== 'object' ||
Array.isArray(dependencies) ||
(dependencies.request !== undefined &&
typeof dependencies.request !== 'function')
) {
invalidConfiguration();
}
this.#request = dependencies.request ?? requestVaultJson;
try {
this.#caReader = new PrivateProjectedFileReader({
rootDirectory: this.#options.caRootDirectory,
minimumBytes: 32,
maximumBytes: MAX_CA_BYTES,
access: 'read_only_keyring',
});
this.#tokenReader = new PrivateProjectedFileReader({
rootDirectory: this.#options.tokenRootDirectory,
minimumBytes: 1,
maximumBytes: MAX_TOKEN_BYTES + 1,
access: 'private_material',
});
} catch (cause) {
invalidConfiguration(cause);
}
}
async #withCredentials<T>(
action: (ca: Buffer, token: string) => Promise<T>,
): Promise<T> {
let ca: Buffer | undefined;
let tokenBytes: Buffer | undefined;
let token: string | undefined;
try {
[ca, tokenBytes] = await Promise.all([
this.#caReader.read(this.#options.caFileName),
this.#tokenReader.read(this.#options.tokenFileName),
]);
token = parseToken(tokenBytes);
return await action(ca, token);
} catch (cause) {
if (cause instanceof ClusterVaultKvSecretProviderError) throw cause;
throw new ClusterVaultKvSecretProviderError('trust_unavailable', {
cause: cause instanceof Error ? cause : undefined,
});
} finally {
ca?.fill(0);
tokenBytes?.fill(0);
token = undefined;
}
}
async #requestJson(
path: string,
ca: Buffer,
token: string,
): Promise<unknown> {
try {
return await this.#request(
Object.freeze({
endpoint: this.#options.endpoint,
path,
requestTimeoutMs: this.#options.requestTimeoutMs,
...(this.#options.namespace === undefined
? {}
: { namespace: this.#options.namespace }),
}),
Object.freeze({ ca, token }),
);
} catch (cause) {
if (cause instanceof ClusterVaultKvSecretProviderError) throw cause;
throw new ClusterVaultKvSecretProviderError('material_unavailable', {
cause: cause instanceof Error ? cause : undefined,
});
}
}
async #authenticate(ca: Buffer, token: string): Promise<void> {
try {
const response = await this.#requestJson(
'/v1/auth/token/lookup-self',
ca,
token,
);
validateTokenLookup(
response,
this.#options.expectedPolicy,
this.#options.maximumTokenTtlSeconds,
);
} catch (cause) {
if (
cause instanceof ClusterVaultKvSecretProviderError &&
cause.reason === 'authentication_unavailable'
) {
throw cause;
}
throw new ClusterVaultKvSecretProviderError(
'authentication_unavailable',
{ cause: cause instanceof Error ? cause : undefined },
);
}
}
async verify(): Promise<void> {
await this.#withCredentials(async (ca, token) => {
await this.#authenticate(ca, token);
});
}
async resolve(
authority: Readonly<RemoteWorkerSecretDeliveryAuthority>,
): Promise<Readonly<RemoteWorkerSecretResolution>> {
let normalized: Readonly<RemoteWorkerSecretDeliveryAuthority>;
try {
normalized = normalizeRemoteWorkerSecretDeliveryAuthority(authority);
} catch (cause) {
throw new ClusterVaultKvSecretProviderError('material_unavailable', {
cause: cause instanceof Error ? cause : undefined,
});
}
return this.#withCredentials(async (ca, token) => {
await this.#authenticate(ca, token);
const inputs = [
...normalized.secretRefs.map((secretRef) =>
Object.freeze({
kind: 'secret' as const,
secretRef,
maximumBytes: MAX_REMOTE_SECRET_VALUE_BYTES,
}),
),
...normalized.environmentBundleRefs.map((secretRef) =>
Object.freeze({
kind: 'environment-bundle' as const,
secretRef,
maximumBytes: MAX_REMOTE_ENVIRONMENT_BUNDLE_VALUE_BYTES,
}),
),
];
const buffers: Buffer[] = [];
try {
const resolved: Array<
Readonly<{
kind: 'secret' | 'environment-bundle';
secretRef: string;
value: string;
byteLength: number;
}>
> = [];
for (
let offset = 0;
offset < inputs.length;
offset += this.#options.maximumConcurrency
) {
const batch = inputs.slice(
offset,
offset + this.#options.maximumConcurrency,
);
const values = await Promise.all(
batch.map(async (input) => {
const digest = secretProjectionFileName(input.secretRef);
const response = await this.#requestJson(
`/v1/${encodeURIComponent(
this.#options.kvMount,
)}/data/${this.#options.pathPrefix
.split('/')
.map(encodeURIComponent)
.join('/')}/${digest}`,
ca,
token,
);
const bytes = parseVaultKvValue(
response,
digest,
input.maximumBytes,
);
buffers.push(bytes);
return Object.freeze({
kind: input.kind,
secretRef: input.secretRef,
value: secretValue(bytes),
byteLength: bytes.byteLength,
});
}),
);
resolved.push(...values);
}
const values = resolved
.filter(({ kind }) => kind === 'secret')
.map(({ secretRef, value }) => Object.freeze({ secretRef, value }));
const totalBytes = resolved
.filter(({ kind }) => kind === 'secret')
.reduce((sum, item) => sum + item.byteLength, 0);
if (totalBytes > MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES) {
throw new ClusterVaultKvSecretProviderError('material_unavailable');
}
const environmentBundles = resolved
.filter(({ kind }) => kind === 'environment-bundle')
.map(({ secretRef, value }) => Object.freeze({ secretRef, value }));
let disposed = false;
return Object.freeze({
values: Object.freeze(values),
environmentBundles: Object.freeze(environmentBundles),
dispose() {
if (disposed) return;
disposed = true;
buffers.splice(0).forEach((bytes) => bytes.fill(0));
},
});
} catch (cause) {
buffers.splice(0).forEach((bytes) => bytes.fill(0));
if (cause instanceof ClusterVaultKvSecretProviderError) throw cause;
throw new ClusterVaultKvSecretProviderError('material_unavailable', {
cause: cause instanceof Error ? cause : undefined,
});
}
});
}
}
export async function createClusterVaultKvSecretProvider(
options: ClusterVaultKvSecretProviderOptions,
): Promise<Readonly<ClusterVaultKvSecretProvider>> {
const provider = new ClusterVaultKvSecretProvider(options);
await provider.verify();
return provider;
}
@@ -52,6 +52,24 @@ export interface ClusterWorkerMountedSecretConfig {
readonly rootDirectory: string;
}
export interface ClusterWorkerVaultKvSecretConfig {
readonly provider: 'vault-kv-v2';
readonly endpoint: string;
readonly caFile: string;
readonly tokenFile: string;
readonly kvMount: string;
readonly pathPrefix: string;
readonly expectedPolicy: string;
readonly maximumTokenTtlSeconds: number;
readonly requestTimeoutMs: number;
readonly maximumConcurrency: number;
readonly namespace?: string;
}
export type ClusterWorkerSecretConfig =
| ClusterWorkerMountedSecretConfig
| ClusterWorkerVaultKvSecretConfig;
export interface EnabledClusterWorkerIngressConfig {
readonly enabled: true;
readonly profile: 'cluster-control';
@@ -70,7 +88,7 @@ export interface EnabledClusterWorkerIngressConfig {
readonly workerCredentialPepper: string;
}>;
readonly artifact: Readonly<ClusterWorkerArtifactS3Config>;
readonly secret?: Readonly<ClusterWorkerMountedSecretConfig>;
readonly secret?: Readonly<ClusterWorkerSecretConfig>;
}
export type ClusterWorkerIngressConfig =
@@ -327,13 +345,11 @@ function workerArtifactS3(
);
if (
prefix !== undefined &&
(
!/^[A-Za-z0-9][A-Za-z0-9/_=-]{0,254}$/.test(prefix) ||
(!/^[A-Za-z0-9][A-Za-z0-9/_=-]{0,254}$/.test(prefix) ||
prefix.startsWith('/') ||
prefix.endsWith('/') ||
prefix.includes('//') ||
prefix.split('/').some((segment) => segment === '.' || segment === '..')
)
prefix.split('/').some((segment) => segment === '.' || segment === '..'))
) {
throw new ClusterWorkerIngressConfigError(
'QL3_WORKER_ARTIFACT_S3_PREFIX is invalid',
@@ -388,11 +404,7 @@ function workerArtifactS3(
endpoint = parsed.origin;
}
const encryptionMode =
boundedValue(
environment,
'QL3_WORKER_ARTIFACT_S3_ENCRYPTION',
3,
) ?? 's3';
boundedValue(environment, 'QL3_WORKER_ARTIFACT_S3_ENCRYPTION', 3) ?? 's3';
if (encryptionMode !== 's3' && encryptionMode !== 'kms') {
throw new ClusterWorkerIngressConfigError(
'QL3_WORKER_ARTIFACT_S3_ENCRYPTION must be s3 or kms',
@@ -415,9 +427,7 @@ function workerArtifactS3(
bucket,
region,
...(prefix === undefined ? {} : { prefix }),
...(expectedBucketOwner === undefined
? {}
: { expectedBucketOwner }),
...(expectedBucketOwner === undefined ? {} : { expectedBucketOwner }),
...(endpoint === undefined ? {} : { endpoint }),
forcePathStyle: booleanValue(
environment,
@@ -433,18 +443,96 @@ function workerArtifactS3(
function workerSecret(
environment: ClusterWorkerIngressEnvironment,
): Readonly<ClusterWorkerMountedSecretConfig> | undefined {
const provider = boundedValue(
environment,
'QL3_WORKER_SECRET_PROVIDER',
32,
);
): Readonly<ClusterWorkerSecretConfig> | undefined {
const provider = boundedValue(environment, 'QL3_WORKER_SECRET_PROVIDER', 32);
if (provider === undefined || provider === 'disabled') return undefined;
if (provider !== 'mounted-files') {
if (provider !== 'mounted-files' && provider !== 'vault-kv-v2') {
throw new ClusterWorkerIngressConfigError(
'QL3_WORKER_SECRET_PROVIDER must be disabled or mounted-files',
'QL3_WORKER_SECRET_PROVIDER must be disabled, mounted-files or vault-kv-v2',
);
}
if (provider === 'vault-kv-v2') {
const endpoint = boundedValue(
environment,
'QL3_WORKER_SECRET_VAULT_ENDPOINT',
2048,
true,
)!;
let parsedEndpoint: URL;
try {
parsedEndpoint = new URL(endpoint);
} catch {
throw new ClusterWorkerIngressConfigError(
'QL3_WORKER_SECRET_VAULT_ENDPOINT must be an HTTPS authority',
);
}
if (
parsedEndpoint.protocol !== 'https:' ||
parsedEndpoint.username ||
parsedEndpoint.password ||
parsedEndpoint.pathname !== '/' ||
parsedEndpoint.search ||
parsedEndpoint.hash
) {
throw new ClusterWorkerIngressConfigError(
'QL3_WORKER_SECRET_VAULT_ENDPOINT must be an HTTPS authority',
);
}
const namespace = boundedValue(
environment,
'QL3_WORKER_SECRET_VAULT_NAMESPACE',
512,
);
return Object.freeze({
provider,
endpoint: parsedEndpoint.toString(),
caFile: absoluteFile(environment, 'QL3_WORKER_SECRET_VAULT_CA_FILE'),
tokenFile: absoluteFile(
environment,
'QL3_WORKER_SECRET_VAULT_TOKEN_FILE',
),
kvMount: boundedValue(
environment,
'QL3_WORKER_SECRET_VAULT_KV_MOUNT',
128,
true,
)!,
pathPrefix: boundedValue(
environment,
'QL3_WORKER_SECRET_VAULT_PATH_PREFIX',
512,
true,
)!,
expectedPolicy: boundedValue(
environment,
'QL3_WORKER_SECRET_VAULT_EXPECTED_POLICY',
128,
true,
)!,
maximumTokenTtlSeconds: integerValue(
environment,
'QL3_WORKER_SECRET_VAULT_MAX_TOKEN_TTL_SECONDS',
900,
30,
3600,
),
requestTimeoutMs: integerValue(
environment,
'QL3_WORKER_SECRET_VAULT_REQUEST_TIMEOUT_MS',
5000,
100,
30_000,
),
maximumConcurrency: integerValue(
environment,
'QL3_WORKER_SECRET_VAULT_MAX_CONCURRENCY',
4,
1,
8,
),
...(namespace === undefined ? {} : { namespace }),
});
}
return Object.freeze({
provider,
rootDirectory: absoluteFile(
@@ -0,0 +1,297 @@
'use strict';
const assert = require('node:assert/strict');
const { chmod, mkdtemp, rename, rm, writeFile } = require('node:fs/promises');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const { createSecretRef } = require('@qinglong/runtime-core/secret-reference');
const {
ClusterVaultKvSecretProvider,
ClusterVaultKvSecretProviderError,
normalizeClusterVaultKvSecretProviderOptions,
} = require('@qinglong/cluster-control/vault-kv-secret-provider');
const {
secretProjectionFileName,
} = require('@qinglong/runtime-core/secret-projection');
const SECRET_REF = createSecretRef({
projectId: 'project-1',
name: 'api-token',
});
const SECOND_SECRET_REF = createSecretRef({
projectId: 'project-1',
name: 'certificate',
version: 3,
});
const ENVIRONMENT_BUNDLE_REF = createSecretRef({
projectId: 'project-1',
name: 'legacy-env-bundle',
version: 4,
});
function authority(secretRefs = [SECRET_REF], environmentBundleRefs = []) {
return {
workerId: 'worker-1',
workerSessionId: '018f0000-0000-7000-8000-000000000001',
workerGeneration: 1,
runId: 'run-1',
attemptId: 'attempt-1',
projectId: 'project-1',
taskId: 'task-1',
taskRevision: 'revision-1',
executionDigest: 'a'.repeat(64),
offerId: 'offer-1',
leaseGeneration: 1,
leaseVersion: 1,
secretRefs,
environmentBundleRefs,
};
}
function lookup(policy = 'ql3-worker-secret-read', overrides = {}) {
return {
data: {
policies: [policy],
orphan: true,
renewable: false,
type: 'service',
ttl: 600,
...overrides,
},
};
}
function envelope(secretRef, value, overrides = {}) {
return {
data: {
data: {
schemaVersion: 1,
secretRefDigest: secretProjectionFileName(secretRef),
encoding: 'base64',
value: Buffer.from(value).toString('base64'),
...overrides,
},
metadata: {
deletion_time: '',
destroyed: false,
version: 1,
},
},
};
}
async function projectedFile(file, value, mode) {
await writeFile(file, value);
await chmod(file, mode);
}
async function fixture(t) {
const directory = await mkdtemp(path.join(os.tmpdir(), 'ql3-vault-kv-'));
t.after(() => rm(directory, { recursive: true, force: true }));
await chmod(directory, 0o700);
const caFile = path.join(directory, 'ca.pem');
const tokenFile = path.join(directory, 'token');
await projectedFile(caFile, 'x'.repeat(64), 0o440);
await projectedFile(tokenFile, 'token-generation-one\n', 0o440);
return {
directory,
caFile,
tokenFile,
options: {
endpoint: 'https://vault.internal:8200',
caFile,
tokenFile,
kvMount: 'worker-secrets',
pathPrefix: 'values/production',
expectedPolicy: 'ql3-worker-secret-read',
maximumTokenTtlSeconds: 900,
requestTimeoutMs: 5000,
maximumConcurrency: 2,
},
};
}
test('normalizes only a pinned HTTPS Vault KV v2 authority', async (t) => {
const { options } = await fixture(t);
assert.deepEqual(
normalizeClusterVaultKvSecretProviderOptions({
...options,
namespace: 'organization/team-a',
}),
{
...options,
endpoint: 'https://vault.internal:8200/',
caRootDirectory: path.dirname(options.caFile),
caFileName: path.basename(options.caFile),
tokenRootDirectory: path.dirname(options.tokenFile),
tokenFileName: path.basename(options.tokenFile),
namespace: 'organization/team-a',
},
);
for (const invalid of [
{ ...options, endpoint: 'http://vault.internal:8200' },
{ ...options, endpoint: 'https://user@vault.internal:8200' },
{ ...options, endpoint: 'https://vault.internal:8200/v1' },
{ ...options, caFile: 'relative.pem' },
{ ...options, kvMount: '../secret' },
{ ...options, pathPrefix: 'values//production' },
{ ...options, maximumTokenTtlSeconds: 3601 },
{ ...options, maximumConcurrency: 9 },
]) {
assert.throws(
() => normalizeClusterVaultKvSecretProviderOptions(invalid),
ClusterVaultKvSecretProviderError,
);
}
});
test('revalidates the short-lived token and reads digest paths without a cache', async (t) => {
const { options, tokenFile, directory } = await fixture(t);
const requests = [];
let generation = 1;
let active = 0;
let maximumActive = 0;
const values = new Map([
[secretProjectionFileName(SECRET_REF), () => `token-${generation}`],
[secretProjectionFileName(SECOND_SECRET_REF), () => 'certificate-value'],
[
secretProjectionFileName(ENVIRONMENT_BUNDLE_REF),
() =>
JSON.stringify({
schema: 'qinglong/environment-bundle@v1',
entries: [{ name: 'LEGACY_VALUE', value: `bundle-${generation}` }],
}),
],
]);
const provider = new ClusterVaultKvSecretProvider(options, {
async request(request, material) {
requests.push({ ...request, token: material.token });
if (request.path === '/v1/auth/token/lookup-self') return lookup();
active += 1;
maximumActive = Math.max(maximumActive, active);
await new Promise((resolve) => setImmediate(resolve));
active -= 1;
const digest = request.path.split('/').at(-1);
const selected = values.get(digest);
assert.ok(selected, 'only digest-derived reviewed paths are requested');
const secretRef =
digest === secretProjectionFileName(SECRET_REF)
? SECRET_REF
: digest === secretProjectionFileName(SECOND_SECRET_REF)
? SECOND_SECRET_REF
: ENVIRONMENT_BUNDLE_REF;
return envelope(secretRef, selected());
},
});
await provider.verify();
const first = await provider.resolve(
authority([SECRET_REF, SECOND_SECRET_REF], [ENVIRONMENT_BUNDLE_REF]),
);
assert.deepEqual(first.values, [
{ secretRef: SECRET_REF, value: 'token-1' },
{ secretRef: SECOND_SECRET_REF, value: 'certificate-value' },
]);
assert.equal(first.environmentBundles[0].secretRef, ENVIRONMENT_BUNDLE_REF);
assert.match(first.environmentBundles[0].value, /bundle-1/);
await first.dispose();
await first.dispose();
assert.equal(maximumActive, 2);
generation = 2;
const replacement = path.join(directory, 'token.replacement');
await projectedFile(replacement, 'token-generation-two\n', 0o440);
await rename(replacement, tokenFile);
const second = await provider.resolve(authority());
assert.deepEqual(second.values, [
{ secretRef: SECRET_REF, value: 'token-2' },
]);
await second.dispose();
assert.equal(
requests.filter(({ path: requestPath }) =>
requestPath.endsWith(secretProjectionFileName(SECRET_REF)),
).length,
2,
);
assert.equal(requests.at(-2).token, 'token-generation-two');
assert.equal(
requests.some(({ path: requestPath }) =>
requestPath.includes(encodeURIComponent(SECRET_REF)),
),
false,
);
});
test('preserves an authorized empty Secret value', async (t) => {
const { options } = await fixture(t);
const provider = new ClusterVaultKvSecretProvider(options, {
request(request) {
return Promise.resolve(
request.path === '/v1/auth/token/lookup-self'
? lookup()
: envelope(SECRET_REF, ''),
);
},
});
const resolution = await provider.resolve(authority());
assert.deepEqual(resolution.values, [{ secretRef: SECRET_REF, value: '' }]);
await resolution.dispose();
});
test('fails closed for broad tokens, unsafe projections and untrusted material', async (t) => {
const { options, tokenFile } = await fixture(t);
const broad = new ClusterVaultKvSecretProvider(options, {
request(request) {
return Promise.resolve(
request.path === '/v1/auth/token/lookup-self'
? lookup('ql3-worker-secret-read', {
policies: ['default', 'ql3-worker-secret-read'],
})
: envelope(SECRET_REF, 'not-reached'),
);
},
});
await assert.rejects(
broad.verify(),
(error) =>
error instanceof ClusterVaultKvSecretProviderError &&
error.reason === 'authentication_unavailable',
);
const mismatched = new ClusterVaultKvSecretProvider(options, {
request(request) {
return Promise.resolve(
request.path === '/v1/auth/token/lookup-self'
? lookup()
: envelope(SECRET_REF, 'private-value', {
secretRefDigest: 'f'.repeat(64),
}),
);
},
});
await assert.rejects(
mismatched.resolve(authority()),
(error) =>
error instanceof ClusterVaultKvSecretProviderError &&
error.reason === 'material_unavailable' &&
error.message.includes('private-value') === false,
);
await chmod(tokenFile, 0o666);
const unsafe = new ClusterVaultKvSecretProvider(options, {
request() {
throw new Error('unsafe token must fail before transport');
},
});
await assert.rejects(
unsafe.verify(),
(error) =>
error instanceof ClusterVaultKvSecretProviderError &&
error.reason === 'trust_unavailable',
);
await assert.rejects(
mismatched.resolve({ ...authority(), projectId: 'other-project' }),
ClusterVaultKvSecretProviderError,
);
});
@@ -0,0 +1,86 @@
'use strict';
const assert = require('node:assert/strict');
const path = require('node:path');
const { test } = require('node:test');
const {
ClusterWorkerIngressConfigError,
loadClusterWorkerIngressConfig,
} = require('@qinglong/cluster-control/worker-ingress-config');
const FIXTURES = path.join(__dirname, 'fixtures', 'mtls');
const BASE_ENV = Object.freeze({
QL3_WORKER_INGRESS_ENABLED: 'true',
QL_DEPLOYMENT_PROFILE: 'cluster-control',
QL3_POSTGRES_WORKER_INGRESS_URL:
'postgresql://ql3_worker_ingress:secret@database.internal:5432/qinglong',
QL3_WORKER_INGRESS_POSTGRES_TLS_SERVERNAME: 'database.internal',
QL3_WORKER_CREDENTIAL_PEPPER: 'A'.repeat(43),
QL3_WORKER_ARTIFACT_S3_BUCKET: 'qinglong-worker-artifacts',
QL3_WORKER_ARTIFACT_S3_REGION: 'us-east-1',
QL3_WORKER_INGRESS_TLS_PRIVATE_KEY_FILE: path.join(
FIXTURES,
'server-key.pem',
),
QL3_WORKER_INGRESS_TLS_CERTIFICATE_FILE: path.join(
FIXTURES,
'server-cert.pem',
),
QL3_WORKER_INGRESS_TLS_CLIENT_CA_FILE: path.join(FIXTURES, 'ca-cert.pem'),
});
test('loads the explicit Vault KV v2 Worker Secret provider', () => {
const config = loadClusterWorkerIngressConfig({
...BASE_ENV,
QL3_WORKER_SECRET_PROVIDER: 'vault-kv-v2',
QL3_WORKER_SECRET_VAULT_ENDPOINT: 'https://vault.internal:8200',
QL3_WORKER_SECRET_VAULT_CA_FILE: '/run/vault/trust/ca.pem',
QL3_WORKER_SECRET_VAULT_TOKEN_FILE: '/run/vault/auth/token',
QL3_WORKER_SECRET_VAULT_KV_MOUNT: 'worker-secrets',
QL3_WORKER_SECRET_VAULT_PATH_PREFIX: 'values/production',
QL3_WORKER_SECRET_VAULT_EXPECTED_POLICY: 'ql3-worker-secret-read',
QL3_WORKER_SECRET_VAULT_MAX_TOKEN_TTL_SECONDS: '600',
QL3_WORKER_SECRET_VAULT_REQUEST_TIMEOUT_MS: '4000',
QL3_WORKER_SECRET_VAULT_MAX_CONCURRENCY: '3',
QL3_WORKER_SECRET_VAULT_NAMESPACE: 'organization/team-a',
});
assert.deepEqual(config.secret, {
provider: 'vault-kv-v2',
endpoint: 'https://vault.internal:8200/',
caFile: '/run/vault/trust/ca.pem',
tokenFile: '/run/vault/auth/token',
kvMount: 'worker-secrets',
pathPrefix: 'values/production',
expectedPolicy: 'ql3-worker-secret-read',
maximumTokenTtlSeconds: 600,
requestTimeoutMs: 4000,
maximumConcurrency: 3,
namespace: 'organization/team-a',
});
});
test('rejects incomplete or non-HTTPS Vault Worker Secret configuration', () => {
const base = {
...BASE_ENV,
QL3_WORKER_SECRET_PROVIDER: 'vault-kv-v2',
QL3_WORKER_SECRET_VAULT_ENDPOINT: 'https://vault.internal:8200',
QL3_WORKER_SECRET_VAULT_CA_FILE: '/run/vault/trust/ca.pem',
QL3_WORKER_SECRET_VAULT_TOKEN_FILE: '/run/vault/auth/token',
QL3_WORKER_SECRET_VAULT_KV_MOUNT: 'worker-secrets',
QL3_WORKER_SECRET_VAULT_PATH_PREFIX: 'values/production',
QL3_WORKER_SECRET_VAULT_EXPECTED_POLICY: 'ql3-worker-secret-read',
};
for (const environment of [
{ ...base, QL3_WORKER_SECRET_VAULT_ENDPOINT: 'http://vault.internal' },
{ ...base, QL3_WORKER_SECRET_VAULT_CA_FILE: 'relative.pem' },
{ ...base, QL3_WORKER_SECRET_VAULT_TOKEN_FILE: undefined },
{ ...base, QL3_WORKER_SECRET_VAULT_MAX_TOKEN_TTL_SECONDS: '3601' },
{ ...base, QL3_WORKER_SECRET_VAULT_MAX_CONCURRENCY: '9' },
]) {
assert.throws(
() => loadClusterWorkerIngressConfig(environment),
ClusterWorkerIngressConfigError,
);
}
});
@@ -0,0 +1,188 @@
#!/usr/bin/env node
'use strict';
const fs = require('node:fs');
const path = require('node:path');
const yaml = require('js-yaml');
function finding(code, detail) {
return Object.freeze({ code, detail });
}
function load(readFile, root, relativePath) {
return yaml.load(readFile(path.join(root, relativePath), 'utf8'));
}
function loadDocuments(source) {
const documents = [];
yaml.loadAll(source, (document) => {
if (document) documents.push(document);
});
return documents;
}
function named(values, name) {
return (values ?? []).find((value) => value?.name === name);
}
function auditVaultKvWorkerSecretDeployment(options = {}) {
const root = path.resolve(options.root ?? path.join(__dirname, '..'));
const readFile = options.readFile ?? fs.readFileSync;
const findings = [];
try {
const directory = 'deploy/kubernetes/ql3-cluster/vault-kv-worker-secret';
const kustomization = load(
readFile,
root,
`${directory}/kustomization.yaml`,
);
const patch = load(readFile, root, `${directory}/deployment-patch.yaml`);
const credentials = loadDocuments(
readFile(path.join(root, directory, 'credentials.example.yaml'), 'utf8'),
);
const readme = readFile(path.join(root, directory, 'README.md'), 'utf8');
if (
kustomization?.apiVersion !== 'kustomize.config.k8s.io/v1beta1' ||
kustomization?.kind !== 'Kustomization' ||
JSON.stringify(kustomization?.resources) !==
JSON.stringify(['../base']) ||
JSON.stringify(kustomization?.patches) !==
JSON.stringify([{ path: 'deployment-patch.yaml' }])
) {
findings.push(
finding(
'QL3_VAULT_KV_WORKER_SECRET_KUSTOMIZATION_INVALID',
'the Vault overlay must patch only the reviewed Cluster base',
),
);
}
const pod = patch?.spec?.template?.spec;
const container = named(pod?.containers, 'cluster-control');
const env = new Map(
(container?.env ?? []).map((entry) => [entry.name, entry]),
);
const expectedEnvironment = new Map([
['QL3_WORKER_SECRET_PROVIDER', 'vault-kv-v2'],
[
'QL3_WORKER_SECRET_VAULT_ENDPOINT',
'https://vault.vault.svc.cluster.local:8200',
],
[
'QL3_WORKER_SECRET_VAULT_CA_FILE',
'/var/run/secrets/qinglong3/worker-vault-trust/ca.pem',
],
[
'QL3_WORKER_SECRET_VAULT_TOKEN_FILE',
'/var/run/secrets/qinglong3/worker-vault-auth/token',
],
['QL3_WORKER_SECRET_VAULT_KV_MOUNT', 'worker-secrets'],
['QL3_WORKER_SECRET_VAULT_PATH_PREFIX', 'values/production'],
['QL3_WORKER_SECRET_VAULT_EXPECTED_POLICY', 'ql3-worker-secret-read'],
['QL3_WORKER_SECRET_VAULT_MAX_TOKEN_TTL_SECONDS', '900'],
['QL3_WORKER_SECRET_VAULT_REQUEST_TIMEOUT_MS', '5000'],
['QL3_WORKER_SECRET_VAULT_MAX_CONCURRENCY', '4'],
]);
if (
env.size !== expectedEnvironment.size + 1 ||
env.get('QL3_WORKER_SECRET_ROOT_DIRECTORY')?.$patch !== 'delete' ||
[...expectedEnvironment].some(
([name, value]) => env.get(name)?.value !== value,
)
) {
findings.push(
finding(
'QL3_VAULT_KV_WORKER_SECRET_ENVIRONMENT_INVALID',
'the overlay must select exact Vault KV v2 authority and bounds while deleting the mounted-value root',
),
);
}
const mounts = container?.volumeMounts ?? [];
const valuesMount = named(mounts, 'worker-secret-values');
const trustMount = named(mounts, 'worker-vault-trust');
const authMount = named(mounts, 'worker-vault-auth');
const volumes = pod?.volumes ?? [];
const valuesVolume = named(volumes, 'worker-secret-values');
const trustVolume = named(volumes, 'worker-vault-trust');
const authVolume = named(volumes, 'worker-vault-auth');
if (
mounts.length !== 3 ||
valuesMount?.$patch !== 'delete' ||
trustMount?.mountPath !==
'/var/run/secrets/qinglong3/worker-vault-trust' ||
trustMount?.readOnly !== true ||
authMount?.mountPath !== '/var/run/secrets/qinglong3/worker-vault-auth' ||
authMount?.readOnly !== true ||
volumes.length !== 3 ||
valuesVolume?.$patch !== 'delete' ||
trustVolume?.secret?.secretName !== 'ql3-cluster-worker-vault-trust' ||
trustVolume?.secret?.defaultMode !== 0o444 ||
JSON.stringify(trustVolume?.secret?.items) !==
JSON.stringify([{ key: 'ca.pem', path: 'ca.pem' }]) ||
authVolume?.secret?.secretName !== 'ql3-cluster-worker-vault-auth' ||
authVolume?.secret?.defaultMode !== 0o440 ||
JSON.stringify(authVolume?.secret?.items) !==
JSON.stringify([{ key: 'token', path: 'token' }])
) {
findings.push(
finding(
'QL3_VAULT_KV_WORKER_SECRET_PROJECTION_INVALID',
'only read-only Vault trust and short-lived auth projections are allowed; the value projection must be deleted',
),
);
}
if (
credentials.length !== 2 ||
credentials[0]?.kind !== 'Secret' ||
credentials[0]?.metadata?.name !== 'ql3-cluster-worker-vault-trust' ||
credentials[0]?.stringData?.['ca.pem'] !==
'REPLACE_WITH_PRIVATE_VAULT_CA_PEM' ||
credentials[1]?.kind !== 'Secret' ||
credentials[1]?.metadata?.name !== 'ql3-cluster-worker-vault-auth' ||
credentials[1]?.stringData?.token !==
'REPLACE_WITH_SHORT_LIVED_ORPHAN_VAULT_TOKEN'
) {
findings.push(
finding(
'QL3_VAULT_KV_WORKER_SECRET_EXAMPLE_INVALID',
'credential examples must contain only explicit non-production placeholders',
),
);
}
if (
!readme.includes('auth/token/lookup-self') ||
!readme.includes('capabilities = ["read"]') ||
!/must not project\s+the actual Worker Secret values/.test(readme) ||
!readme.includes('default-deny')
) {
findings.push(
finding(
'QL3_VAULT_KV_WORKER_SECRET_OPERATIONS_INVALID',
'operations guidance must preserve exact policy, direct custody and explicit egress boundaries',
),
);
}
} catch (error) {
findings.push(
finding(
'QL3_VAULT_KV_WORKER_SECRET_DEPLOYMENT_AUDIT_UNAVAILABLE',
error instanceof Error ? error.message : String(error),
),
);
}
return Object.freeze({
schemaVersion: 1,
provider: 'vault-kv-v2',
mountedValueProjection: false,
findings: Object.freeze(findings),
compatible: findings.length === 0,
});
}
if (require.main === module) {
const report = auditVaultKvWorkerSecretDeployment();
process.stdout.write(`${JSON.stringify(report)}\n`);
if (!report.compatible) process.exitCode = 1;
}
module.exports = { auditVaultKvWorkerSecretDeployment };
@@ -0,0 +1,192 @@
#!/usr/bin/env node
'use strict';
const fs = require('node:fs');
const path = require('node:path');
const FIXTURE = 'qinglong/vault-kv-worker-secret-direct-custody-live@v1';
const IMAGE =
'docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569';
const SHA256 = /^sha256:[a-f0-9]{64}$/;
const VERSION = /^\d+\.\d+\.\d+(?:[+-][A-Za-z0-9.-]+)?$/;
const FORBIDDEN_KEY =
/(secretRef|secretValue|clientToken|rootToken|accessor|endpoint|pathPrefix|tokenFile|caFile|materialValue|materialBytes|privateKey)/i;
const REQUIRED_GATES = Object.freeze([
'digestPinnedVaultImage',
'nativeVaultArchitecture',
'tls13WithExplicitPrivateCa',
'untrustedCaRejected',
'initializedWithThreeOfTwoSealAuthority',
'kvV2ExternalCustody',
'oneExactReadOnlyPolicy',
'shortLivedOrphanNonRenewableToken',
'tokenRevalidatedPerResolution',
'digestDerivedPathsOnly',
'normalSecretBoundPreserved',
'opaqueEnvironmentBundleBoundPreserved',
'valueRotationObservedWithoutControlRestart',
'tokenRotationObservedWithoutControlRestart',
'revokedTokenRemoved',
'missingMaterialFailsClosed',
'sealedVaultFailsClosed',
'thresholdUnsealRestoresResolution',
'persistentValuesSurviveContainerReplacement',
'reportIsContentFree',
'passed',
]);
function exact(value, keys) {
return Boolean(
value &&
typeof value === 'object' &&
!Array.isArray(value) &&
JSON.stringify(Object.keys(value).sort()) ===
JSON.stringify([...keys].sort()),
);
}
function scan(value, findings, location = 'report') {
if (
typeof value === 'string' &&
/(hvs\.[A-Za-z0-9_-]{16,}|-----BEGIN (?:RSA |EC )?PRIVATE KEY-----|qlsecret:v1:|vault-private-|bundle-private-)/.test(
value,
)
) {
findings.push(`${location} contains sensitive material`);
return;
}
if (Array.isArray(value)) {
value.forEach((entry, index) =>
scan(entry, findings, `${location}[${index}]`),
);
return;
}
if (!value || typeof value !== 'object') return;
for (const [key, entry] of Object.entries(value)) {
if (FORBIDDEN_KEY.test(key)) {
findings.push(`${location}.${key} is forbidden`);
}
scan(entry, findings, `${location}.${key}`);
}
}
function validateVaultKvWorkerSecretLiveReport(report) {
const findings = [];
if (
!exact(report, [
'schemaVersion',
'fixture',
'platform',
'custody',
'gates',
'limitations',
]) ||
report?.schemaVersion !== 1 ||
report?.fixture !== FIXTURE
) {
findings.push('report envelope is invalid');
}
if (
!exact(report?.platform, [
'architecture',
'vaultImage',
'vaultImageId',
'vaultVersion',
'transport',
'storage',
]) ||
!['amd64', 'arm64'].includes(report?.platform?.architecture) ||
report?.platform?.vaultImage !== IMAGE ||
!SHA256.test(report?.platform?.vaultImageId ?? '') ||
!VERSION.test(report?.platform?.vaultVersion ?? '') ||
report?.platform?.transport !== 'TLSv1.3 with an explicit private CA' ||
report?.platform?.storage !== 'persistent file barrier fixture'
) {
findings.push('platform evidence is invalid');
}
if (
!exact(report?.custody, [
'provider',
'kvVersion',
'policyCount',
'maximumTokenTtlSeconds',
'tokenLeaseSeconds',
'secretCount',
'environmentBundleCount',
'observedVersions',
'containerReplacements',
]) ||
report?.custody?.provider !== 'vault-kv-v2' ||
report?.custody?.kvVersion !== 2 ||
report?.custody?.policyCount !== 1 ||
report?.custody?.maximumTokenTtlSeconds !== 900 ||
report?.custody?.tokenLeaseSeconds !== 600 ||
report?.custody?.secretCount !== 2 ||
report?.custody?.environmentBundleCount !== 1 ||
JSON.stringify(report?.custody?.observedVersions) !==
JSON.stringify([1, 2]) ||
report?.custody?.containerReplacements !== 1
) {
findings.push('custody evidence is invalid');
}
if (
!exact(report?.gates, REQUIRED_GATES) ||
REQUIRED_GATES.some((gate) => report?.gates?.[gate] !== true)
) {
findings.push('one or more required gates are false or missing');
}
if (
!Array.isArray(report?.limitations) ||
report.limitations.length !== 3 ||
report.limitations.some(
(value) =>
typeof value !== 'string' || value.length < 32 || value.length > 512,
)
) {
findings.push('limitations are invalid');
}
scan(report, findings);
return Object.freeze({
schemaVersion: 1,
fixture: FIXTURE,
findings: Object.freeze(findings),
compatible: findings.length === 0,
});
}
function main(argv) {
if (argv.length !== 1 || !argv[0].startsWith('--report=/')) {
throw new Error(
'usage: ql3-vault-kv-worker-secret-live-audit --report=/absolute/report.json',
);
}
const reportPath = argv[0].slice('--report='.length);
if (path.resolve(reportPath) !== reportPath) {
throw new Error('report path is invalid');
}
const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
const result = validateVaultKvWorkerSecretLiveReport(report);
process.stdout.write(`${JSON.stringify(result)}\n`);
if (!result.compatible) process.exitCode = 1;
}
if (require.main === module) {
try {
main(process.argv.slice(2));
} catch (error) {
process.stderr.write(
`QL3 Vault KV Worker Secret live audit failed: ${
error instanceof Error ? error.message : String(error)
}\n`,
);
process.exitCode = 1;
}
}
module.exports = {
FIXTURE,
IMAGE,
REQUIRED_GATES,
validateVaultKvWorkerSecretLiveReport,
};
@@ -0,0 +1,814 @@
#!/usr/bin/env node
'use strict';
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const { randomBytes } = require('node:crypto');
const {
chmodSync,
mkdirSync,
mkdtempSync,
readFileSync,
renameSync,
rmSync,
writeFileSync,
} = require('node:fs');
const https = require('node:https');
const { isIP } = require('node:net');
const os = require('node:os');
const path = require('node:path');
const {
createSecretRef,
} = require('../packages/ql3-runtime-core/dist/secret/secretReference.js');
const {
secretProjectionFileName,
} = require('../packages/ql3-runtime-core/dist/secret/secretProjection.js');
const {
ClusterVaultKvSecretProviderError,
createClusterVaultKvSecretProvider,
} = require('../packages/ql3-cluster-control/dist/remote-execution/vaultKvSecretProvider.js');
const IMAGE =
'docker.io/hashicorp/vault@sha256:4e33b126a59c0c333b76fb4e894722462659a6bec7c48c9ee8cea56fccfd2569';
const FIXTURE = 'qinglong/vault-kv-worker-secret-direct-custody-live@v1';
const MAX_RESPONSE_BYTES = 256 * 1024;
const VAULT_TIMEOUT_MS = 10_000;
const POLICY = 'ql3-worker-secret-read';
const MOUNT = 'worker-secrets';
const PREFIX = 'values/production';
function docker(args, options = {}) {
const result = spawnSync('docker', args, {
encoding: 'utf8',
timeout: options.timeoutMs ?? 120_000,
maxBuffer: 4 * 1024 * 1024,
});
if (result.error) throw result.error;
if (result.status !== 0 && !options.allowFailure) {
throw new Error(`Docker command failed: ${args[0] ?? 'unknown'}`);
}
return {
status: result.status,
stdout: result.stdout.trim(),
stderr: result.stderr.trim(),
};
}
function privateFile(directory, name, value, mode = 0o440) {
const target = path.join(directory, name);
writeFileSync(target, value, { mode: 0o600, flag: 'wx' });
chmodSync(target, mode);
return target;
}
function openssl(args) {
const result = spawnSync('openssl', args, {
encoding: 'utf8',
timeout: 30_000,
maxBuffer: 1024 * 1024,
});
if (result.error) throw result.error;
if (result.status !== 0) throw new Error('OpenSSL command failed');
}
function generateTlsAuthority(directory) {
const tlsDirectory = path.join(directory, 'tls');
mkdirSync(tlsDirectory, { mode: 0o700 });
const caKeyFile = path.join(tlsDirectory, 'ca-key.pem');
const caFile = path.join(tlsDirectory, 'ca.pem');
const serverKeyFile = path.join(tlsDirectory, 'server-key.pem');
const serverRequestFile = path.join(tlsDirectory, 'server.csr');
const serverCertificateFile = path.join(tlsDirectory, 'server.pem');
const untrustedCaKeyFile = path.join(tlsDirectory, 'untrusted-ca-key.pem');
const untrustedCaFile = path.join(tlsDirectory, 'untrusted-ca.pem');
const extensionsFile = privateFile(
tlsDirectory,
'server-extensions.cnf',
[
'basicConstraints=critical,CA:FALSE',
'keyUsage=critical,digitalSignature,keyEncipherment',
'extendedKeyUsage=serverAuth',
'subjectAltName=IP:127.0.0.1,DNS:localhost',
'',
].join('\n'),
0o400,
);
openssl([
'req',
'-x509',
'-newkey',
'rsa:2048',
'-sha256',
'-nodes',
'-days',
'2',
'-subj',
'/CN=QingLong 3 Vault KV Live Root',
'-addext',
'basicConstraints=critical,CA:TRUE,pathlen:0',
'-addext',
'keyUsage=critical,keyCertSign,cRLSign',
'-keyout',
caKeyFile,
'-out',
caFile,
]);
openssl([
'req',
'-new',
'-newkey',
'rsa:2048',
'-sha256',
'-nodes',
'-subj',
'/CN=127.0.0.1',
'-keyout',
serverKeyFile,
'-out',
serverRequestFile,
]);
openssl([
'x509',
'-req',
'-sha256',
'-days',
'2',
'-in',
serverRequestFile,
'-CA',
caFile,
'-CAkey',
caKeyFile,
'-CAcreateserial',
'-extfile',
extensionsFile,
'-out',
serverCertificateFile,
]);
openssl([
'req',
'-x509',
'-newkey',
'rsa:2048',
'-sha256',
'-nodes',
'-days',
'2',
'-subj',
'/CN=QingLong 3 Untrusted Vault KV Root',
'-addext',
'basicConstraints=critical,CA:TRUE,pathlen:0',
'-addext',
'keyUsage=critical,keyCertSign,cRLSign',
'-keyout',
untrustedCaKeyFile,
'-out',
untrustedCaFile,
]);
chmodSync(caKeyFile, 0o400);
chmodSync(caFile, 0o440);
chmodSync(serverKeyFile, 0o400);
chmodSync(serverRequestFile, 0o400);
chmodSync(serverCertificateFile, 0o440);
chmodSync(untrustedCaKeyFile, 0o400);
chmodSync(untrustedCaFile, 0o440);
return Object.freeze({
tlsDirectory,
caFile,
serverCertificateFile,
serverKeyFile,
untrustedCaFile,
});
}
function vaultJson(endpoint, ca, token, method, requestPath, body) {
const bytes = body === undefined ? null : Buffer.from(JSON.stringify(body));
const target = new URL(requestPath, endpoint);
return new Promise((resolve, reject) => {
const request = https.request(
target,
{
method,
ca,
rejectUnauthorized: true,
minVersion: 'TLSv1.3',
maxVersion: 'TLSv1.3',
...(isIP(target.hostname) === 0 ? { servername: target.hostname } : {}),
headers: {
accept: 'application/json',
...(bytes === null
? {}
: {
'content-type': 'application/json',
'content-length': String(bytes.byteLength),
}),
...(token === null ? {} : { 'x-vault-token': token }),
},
},
(response) => {
const tlsProtocol = response.socket.getProtocol();
const peerAuthorized = response.socket.authorized;
const chunks = [];
let length = 0;
response.on('data', (chunk) => {
length += chunk.byteLength;
if (length > MAX_RESPONSE_BYTES) {
response.destroy(new Error('Vault response exceeded live limit'));
return;
}
chunks.push(Buffer.from(chunk));
});
response.once('error', reject);
response.on('end', () => {
const responseBytes = Buffer.concat(chunks);
chunks.forEach((chunk) => chunk.fill(0));
try {
resolve({
statusCode: response.statusCode,
value: responseBytes.byteLength
? JSON.parse(responseBytes.toString('utf8'))
: null,
tlsProtocol,
peerAuthorized,
});
} catch (cause) {
reject(cause);
} finally {
responseBytes.fill(0);
}
});
},
);
request.setTimeout(VAULT_TIMEOUT_MS, () =>
request.destroy(new Error('Vault live request timed out')),
);
request.once('error', reject);
request.once('close', () => bytes?.fill(0));
request.end(bytes ?? undefined);
});
}
async function waitForVault(endpoint, ca, expected) {
let lastError;
for (let attempt = 0; attempt < 120; attempt += 1) {
try {
const health = await vaultJson(
endpoint,
ca,
null,
'GET',
'/v1/sys/health',
);
if (
health.statusCode === expected.statusCode &&
health.value?.initialized === expected.initialized &&
health.value?.sealed === expected.sealed
) {
assert.equal(health.tlsProtocol, 'TLSv1.3');
assert.equal(health.peerAuthorized, true);
return health;
}
} catch (cause) {
lastError = cause;
}
await new Promise((resolve) => setTimeout(resolve, 125));
}
const detail =
lastError instanceof Error
? `${
typeof lastError.code === 'string' ? lastError.code : lastError.name
}:${lastError.message}`
: 'no-health-response';
throw new Error(
`Vault server did not reach the expected state (${detail.slice(0, 256)})`,
{ cause: lastError },
);
}
function startVaultContainer(container, publish, directory, tls) {
const uid = typeof process.getuid === 'function' ? process.getuid() : 100;
const gid = typeof process.getgid === 'function' ? process.getgid() : 1000;
docker([
'run',
'--detach',
'--rm',
'--name',
container,
'--publish',
publish,
'--user',
`${uid}:${gid}`,
'--cap-drop',
'ALL',
'--cap-add',
'IPC_LOCK',
'--security-opt',
'no-new-privileges:true',
'--read-only',
'--tmpfs',
'/tmp:rw,noexec,nosuid,size=16m',
'--volume',
`${path.join(directory, 'vault-server.hcl')}:/vault/config/server.hcl:ro`,
'--volume',
`${tls.tlsDirectory}:/vault/tls:ro`,
'--volume',
`${path.join(directory, 'vault-data')}:/vault/file:rw`,
'--entrypoint',
'/bin/vault',
IMAGE,
'server',
'-config=/vault/config/server.hcl',
]);
return docker(['inspect', container, '--format', '{{.Id}}']).stdout;
}
async function unsealVault(endpoint, ca, unsealKeys) {
let result;
for (const key of unsealKeys.slice(0, 2)) {
result = await vaultJson(endpoint, ca, null, 'POST', '/v1/sys/unseal', {
key: key.toString('base64'),
});
assert.equal(result.statusCode, 200);
}
assert.equal(result.value?.sealed, false);
assert.equal(result.value?.t, 2);
assert.equal(result.value?.n, 3);
}
function authority(secretRefs, environmentBundleRefs = []) {
return {
workerId: 'worker-vault-live',
workerSessionId: '018f0000-0000-7000-8000-000000000001',
workerGeneration: 1,
runId: 'run-vault-live',
attemptId: 'attempt-vault-live',
projectId: 'project-vault-live',
taskId: 'task-vault-live',
taskRevision: 'revision-vault-live',
executionDigest: 'a'.repeat(64),
offerId: 'offer-vault-live',
leaseGeneration: 1,
leaseVersion: 1,
secretRefs,
environmentBundleRefs,
};
}
function kvEnvelope(secretRef, value) {
return {
schemaVersion: 1,
secretRefDigest: secretProjectionFileName(secretRef),
encoding: 'base64',
value: Buffer.from(value).toString('base64'),
};
}
async function putSecret(endpoint, ca, rootToken, secretRef, value) {
const digest = secretProjectionFileName(secretRef);
const response = await vaultJson(
endpoint,
ca,
rootToken,
'POST',
`/v1/${MOUNT}/data/${PREFIX}/${digest}`,
{ data: kvEnvelope(secretRef, value) },
);
assert.equal(response.statusCode, 200);
assert.equal(Number.isSafeInteger(response.value?.data?.version), true);
return response.value.data.version;
}
async function createLeastPrivilegeToken(endpoint, ca, rootToken) {
const response = await vaultJson(
endpoint,
ca,
rootToken,
'POST',
'/v1/auth/token/create-orphan',
{
policies: [POLICY],
no_default_policy: true,
renewable: false,
ttl: '10m',
display_name: 'ql3-worker-secret-live',
},
);
assert.equal(response.statusCode, 200);
assert.equal(response.value?.auth?.policies?.length, 1);
assert.equal(response.value.auth.policies[0], POLICY);
assert.equal(response.value.auth.orphan, true);
assert.equal(response.value.auth.renewable, false);
assert.equal(response.value.auth.lease_duration, 600);
assert.equal(typeof response.value.auth.client_token, 'string');
assert.equal(typeof response.value.auth.accessor, 'string');
return {
token: response.value.auth.client_token,
accessor: response.value.auth.accessor,
};
}
async function main() {
if (process.env.QL3_RUN_VAULT_KV_WORKER_SECRET_LIVE !== 'true') {
throw new Error('QL3_RUN_VAULT_KV_WORKER_SECRET_LIVE=true is required');
}
docker(['version', '--format', '{{.Server.Version}}']);
docker(['image', 'inspect', IMAGE]);
const suffix = `${process.pid}-${randomBytes(3).toString('hex')}`;
const container = `ql3-vault-kv-worker-${suffix}`;
const directory = mkdtempSync(path.join(os.tmpdir(), 'ql3-vault-kv-live-'));
chmodSync(directory, 0o700);
const unsealKeys = [];
let ca;
let rootToken;
let firstToken;
let secondToken;
let started = false;
try {
const tls = generateTlsAuthority(directory);
ca = readFileSync(tls.caFile);
const untrustedCa = readFileSync(tls.untrustedCaFile);
mkdirSync(path.join(directory, 'vault-data'), { mode: 0o700 });
privateFile(
directory,
'vault-server.hcl',
[
'ui = false',
'disable_mlock = false',
'api_addr = "https://127.0.0.1:8200"',
'cluster_addr = "https://127.0.0.1:8201"',
'storage "file" {',
' path = "/vault/file"',
'}',
'listener "tcp" {',
' address = "0.0.0.0:8200"',
' cluster_address = "0.0.0.0:8201"',
' tls_cert_file = "/vault/tls/server.pem"',
' tls_key_file = "/vault/tls/server-key.pem"',
' tls_min_version = "tls13"',
' tls_max_version = "tls13"',
'}',
'',
].join('\n'),
);
const firstContainerId = startVaultContainer(
container,
'127.0.0.1::8200',
directory,
tls,
);
started = true;
const portOutput = docker(['port', container, '8200/tcp']).stdout;
const portMatch = /^127\.0\.0\.1:([1-9][0-9]*)$/.exec(portOutput);
assert.ok(portMatch, 'Vault host port must be loopback-only');
const hostPort = Number(portMatch[1]);
const endpoint = `https://127.0.0.1:${hostPort}`;
const uninitialized = await waitForVault(endpoint, ca, {
statusCode: 501,
initialized: false,
sealed: true,
});
const initialization = await vaultJson(
endpoint,
ca,
null,
'POST',
'/v1/sys/init',
{ secret_shares: 3, secret_threshold: 2 },
);
assert.equal(initialization.statusCode, 200);
for (const encoded of initialization.value.keys_base64) {
const key = Buffer.from(encoded, 'base64');
assert.equal(key.toString('base64'), encoded);
unsealKeys.push(key);
}
rootToken = initialization.value.root_token;
initialization.value.keys = [];
initialization.value.keys_base64 = [];
initialization.value.root_token = '';
await unsealVault(endpoint, ca, unsealKeys);
const initialHealth = await waitForVault(endpoint, ca, {
statusCode: 200,
initialized: true,
sealed: false,
});
const mount = await vaultJson(
endpoint,
ca,
rootToken,
'POST',
`/v1/sys/mounts/${MOUNT}`,
{ type: 'kv', options: { version: '2' } },
);
assert.equal(mount.statusCode, 204);
const policy = await vaultJson(
endpoint,
ca,
rootToken,
'PUT',
`/v1/sys/policies/acl/${POLICY}`,
{
policy: [
`path "${MOUNT}/data/${PREFIX}/*" { capabilities = ["read"] }`,
'path "auth/token/lookup-self" { capabilities = ["read"] }',
].join('\n'),
},
);
assert.equal(policy.statusCode, 204);
const secretRef = createSecretRef({
projectId: 'project-vault-live',
name: 'legacy-token',
version: 2,
});
const secondSecretRef = createSecretRef({
projectId: 'project-vault-live',
name: 'legacy-certificate',
version: 1,
});
const bundleRef = createSecretRef({
projectId: 'project-vault-live',
name: 'legacy-environment-bundle',
version: 3,
});
const missingRef = createSecretRef({
projectId: 'project-vault-live',
name: 'not-provisioned',
});
const firstValue = `vault-private-generation-one-${suffix}`;
const secondValue = `vault-private-generation-two-${suffix}`;
const certificateValue = `vault-private-certificate-${suffix}`;
const bundleValue = JSON.stringify({
schema: 'qinglong/environment-bundle@v1',
entries: [{ name: 'LEGACY_ENV', value: `bundle-private-${suffix}` }],
});
const firstVersion = await putSecret(
endpoint,
ca,
rootToken,
secretRef,
firstValue,
);
await putSecret(endpoint, ca, rootToken, secondSecretRef, certificateValue);
await putSecret(endpoint, ca, rootToken, bundleRef, bundleValue);
firstToken = await createLeastPrivilegeToken(endpoint, ca, rootToken);
const tokenFile = privateFile(
directory,
'worker-token',
`${firstToken.token}\n`,
0o440,
);
const options = {
endpoint,
caFile: tls.caFile,
tokenFile,
kvMount: MOUNT,
pathPrefix: PREFIX,
expectedPolicy: POLICY,
maximumTokenTtlSeconds: 900,
requestTimeoutMs: 5000,
maximumConcurrency: 2,
};
const initialLookup = await vaultJson(
endpoint,
ca,
firstToken.token,
'GET',
'/v1/auth/token/lookup-self',
);
assert.equal(initialLookup.statusCode, 200);
assert.deepEqual(initialLookup.value?.data?.policies, [POLICY]);
assert.equal(initialLookup.value?.data?.orphan, true);
assert.equal(initialLookup.value?.data?.renewable, false);
assert.equal(initialLookup.value?.data?.type, 'service');
assert.equal(
Number.isSafeInteger(initialLookup.value?.data?.ttl) &&
initialLookup.value.data.ttl > 0 &&
initialLookup.value.data.ttl <= 900,
true,
);
const provider = await createClusterVaultKvSecretProvider(options);
const first = await provider.resolve(
authority([secretRef, secondSecretRef], [bundleRef]),
);
assert.deepEqual(first.values, [
{ secretRef, value: firstValue },
{ secretRef: secondSecretRef, value: certificateValue },
]);
assert.deepEqual(first.environmentBundles, [
{ secretRef: bundleRef, value: bundleValue },
]);
await first.dispose();
const rotatedVersion = await putSecret(
endpoint,
ca,
rootToken,
secretRef,
secondValue,
);
assert.equal(rotatedVersion, firstVersion + 1);
const rotated = await provider.resolve(authority([secretRef]));
assert.deepEqual(rotated.values, [{ secretRef, value: secondValue }]);
await rotated.dispose();
await assert.rejects(
provider.resolve(authority([missingRef])),
(error) =>
error instanceof ClusterVaultKvSecretProviderError &&
error.reason === 'material_unavailable',
);
const untrustedProvider = createClusterVaultKvSecretProvider({
...options,
caFile: tls.untrustedCaFile,
});
await assert.rejects(untrustedProvider, ClusterVaultKvSecretProviderError);
secondToken = await createLeastPrivilegeToken(endpoint, ca, rootToken);
const tokenReplacement = privateFile(
directory,
'worker-token.next',
`${secondToken.token}\n`,
0o440,
);
renameSync(tokenReplacement, tokenFile);
const revoke = await vaultJson(
endpoint,
ca,
rootToken,
'POST',
'/v1/auth/token/revoke-accessor',
{ accessor: firstToken.accessor },
);
assert.equal(revoke.statusCode, 204);
const afterTokenRotation = await provider.resolve(authority([secretRef]));
assert.equal(afterTokenRotation.values[0].value, secondValue);
await afterTokenRotation.dispose();
const seal = await vaultJson(
endpoint,
ca,
rootToken,
'PUT',
'/v1/sys/seal',
);
assert.equal(seal.statusCode, 204);
await waitForVault(endpoint, ca, {
statusCode: 503,
initialized: true,
sealed: true,
});
await assert.rejects(
provider.resolve(authority([secretRef])),
ClusterVaultKvSecretProviderError,
);
await unsealVault(endpoint, ca, unsealKeys);
const postUnseal = await provider.resolve(authority([secretRef]));
assert.equal(postUnseal.values[0].value, secondValue);
await postUnseal.dispose();
docker(['rm', '--force', container]);
started = false;
const secondContainerId = startVaultContainer(
container,
`127.0.0.1:${hostPort}:8200`,
directory,
tls,
);
started = true;
assert.notEqual(secondContainerId, firstContainerId);
const sealedAfterReplacement = await waitForVault(endpoint, ca, {
statusCode: 503,
initialized: true,
sealed: true,
});
await unsealVault(endpoint, ca, unsealKeys);
const postReplacementHealth = await waitForVault(endpoint, ca, {
statusCode: 200,
initialized: true,
sealed: false,
});
const postReplacement = await provider.resolve(authority([secretRef]));
assert.equal(postReplacement.values[0].value, secondValue);
await postReplacement.dispose();
const imageId = docker([
'image',
'inspect',
IMAGE,
'--format',
'{{.Id}}',
]).stdout;
const architecture = docker([
'image',
'inspect',
IMAGE,
'--format',
'{{.Architecture}}',
]).stdout;
const report = {
schemaVersion: 1,
fixture: FIXTURE,
platform: {
architecture,
vaultImage: IMAGE,
vaultImageId: imageId,
vaultVersion: postReplacementHealth.value.version,
transport: 'TLSv1.3 with an explicit private CA',
storage: 'persistent file barrier fixture',
},
custody: {
provider: 'vault-kv-v2',
kvVersion: 2,
policyCount: 1,
maximumTokenTtlSeconds: 900,
tokenLeaseSeconds: 600,
secretCount: 2,
environmentBundleCount: 1,
observedVersions: [firstVersion, rotatedVersion],
containerReplacements: 1,
},
gates: {
digestPinnedVaultImage: true,
nativeVaultArchitecture: ['amd64', 'arm64'].includes(architecture),
tls13WithExplicitPrivateCa:
uninitialized.tlsProtocol === 'TLSv1.3' &&
initialHealth.tlsProtocol === 'TLSv1.3' &&
postReplacementHealth.tlsProtocol === 'TLSv1.3',
untrustedCaRejected: true,
initializedWithThreeOfTwoSealAuthority: true,
kvV2ExternalCustody: true,
oneExactReadOnlyPolicy: true,
shortLivedOrphanNonRenewableToken: true,
tokenRevalidatedPerResolution: true,
digestDerivedPathsOnly: true,
normalSecretBoundPreserved: true,
opaqueEnvironmentBundleBoundPreserved: true,
valueRotationObservedWithoutControlRestart: true,
tokenRotationObservedWithoutControlRestart: true,
revokedTokenRemoved: true,
missingMaterialFailsClosed: true,
sealedVaultFailsClosed: true,
thresholdUnsealRestoresResolution: true,
persistentValuesSurviveContainerReplacement:
sealedAfterReplacement.value.sealed === true,
reportIsContentFree: true,
passed: true,
},
limitations: [
'single-host file storage is not Vault integrated-storage HA or an HSM seal quorum',
'the short-lived private CA and service tokens are live fixture authorities rather than enterprise PKI or workload identity',
'the live gate proves direct external custody resolution and rotation, not fixed physical Edge storage behavior',
],
};
const serialized = JSON.stringify(report);
for (const forbidden of [
rootToken,
firstToken.token,
secondToken.token,
firstValue,
secondValue,
certificateValue,
bundleValue,
directory,
endpoint,
]) {
assert.equal(serialized.includes(forbidden), false);
}
const outputPath = process.env.QL3_VAULT_KV_WORKER_SECRET_REPORT;
if (outputPath !== undefined) {
if (!path.isAbsolute(outputPath))
throw new Error('report path is invalid');
writeFileSync(outputPath, `${serialized}\n`, {
flag: 'wx',
mode: 0o600,
});
}
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
untrustedCa.fill(0);
} finally {
ca?.fill(0);
unsealKeys.forEach((key) => key.fill(0));
rootToken = undefined;
firstToken = undefined;
secondToken = undefined;
if (started) docker(['rm', '--force', container], { allowFailure: true });
rmSync(directory, { recursive: true, force: true });
}
}
if (require.main === module) {
main().catch((error) => {
process.stderr.write(
`ql3 Vault KV Worker Secret live contract failed: ${
error instanceof Error ? error.stack ?? error.message : String(error)
}\n`,
);
process.exitCode = 1;
});
}
module.exports = { FIXTURE, IMAGE };
+1 -1
View File
@@ -919,7 +919,7 @@ test('CLI rejects symlinks, open arguments, output aliasing and policy ambiguity
test('source-surface audit freezes every reviewed cluster and worker authority', () => {
assert.deepEqual(auditDeploymentImageSurfaces(root), {
schemaVersion: 1,
deploymentYamlFiles: 224,
deploymentYamlFiles: 227,
imageOccurrences: {
control: 2,
'control-ai': 1,
+2 -2
View File
@@ -385,10 +385,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', (
rootSourceFileRoles: clusterControl.rootSourceFileRoles,
},
{
sourceFiles: 65,
sourceFiles: 66,
rootSourceFiles: 2,
rootSourceLines: 195,
nestedSourceFiles: 63,
nestedSourceFiles: 64,
rootSourceFileRoles: {
'aiCli.ts': 'binary_entry',
'cli.ts': 'binary_entry',
@@ -0,0 +1,47 @@
'use strict';
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const { test } = require('node:test');
const {
auditVaultKvWorkerSecretDeployment,
} = require('../../scripts/ql3-vault-kv-worker-secret-deployment-audit.cjs');
const ROOT = path.resolve(__dirname, '../..');
test('accepts the exact direct Vault KV deployment overlay', () => {
assert.deepEqual(auditVaultKvWorkerSecretDeployment({ root: ROOT }), {
schemaVersion: 1,
provider: 'vault-kv-v2',
mountedValueProjection: false,
findings: [],
compatible: true,
});
});
test('rejects a restored mounted-value projection or broad token TTL', () => {
const readFile = (file, encoding) => {
const source = fs.readFileSync(file, encoding);
if (!file.endsWith('deployment-patch.yaml')) return source;
return source
.replace(
"QL3_WORKER_SECRET_VAULT_MAX_TOKEN_TTL_SECONDS\n value: '900'",
"QL3_WORKER_SECRET_VAULT_MAX_TOKEN_TTL_SECONDS\n value: '3600'",
)
.replace(
'- name: worker-secret-values\n $patch: delete',
'- name: worker-secret-values\n mountPath: /run/values',
);
};
const result = auditVaultKvWorkerSecretDeployment({ root: ROOT, readFile });
assert.equal(result.compatible, false);
assert.deepEqual(
result.findings.map(({ code }) => code),
[
'QL3_VAULT_KV_WORKER_SECRET_ENVIRONMENT_INVALID',
'QL3_VAULT_KV_WORKER_SECRET_PROJECTION_INVALID',
],
);
});
@@ -0,0 +1,88 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
FIXTURE,
IMAGE,
REQUIRED_GATES,
validateVaultKvWorkerSecretLiveReport,
} = require('../../scripts/ql3-vault-kv-worker-secret-live-audit.cjs');
function fixture() {
return {
schemaVersion: 1,
fixture: FIXTURE,
platform: {
architecture: 'arm64',
vaultImage: IMAGE,
vaultImageId: `sha256:${'a'.repeat(64)}`,
vaultVersion: '1.20.2',
transport: 'TLSv1.3 with an explicit private CA',
storage: 'persistent file barrier fixture',
},
custody: {
provider: 'vault-kv-v2',
kvVersion: 2,
policyCount: 1,
maximumTokenTtlSeconds: 900,
tokenLeaseSeconds: 600,
secretCount: 2,
environmentBundleCount: 1,
observedVersions: [1, 2],
containerReplacements: 1,
},
gates: Object.fromEntries(REQUIRED_GATES.map((gate) => [gate, true])),
limitations: [
'single-host file storage is not Vault integrated-storage HA or an HSM seal quorum',
'the short-lived private CA and service tokens are fixture authorities rather than enterprise PKI',
'the live gate proves direct custody resolution rather than physical Edge storage behavior',
],
};
}
test('accepts the exact content-free Vault KV direct custody report', () => {
assert.deepEqual(validateVaultKvWorkerSecretLiveReport(fixture()), {
schemaVersion: 1,
fixture: FIXTURE,
findings: [],
compatible: true,
});
});
test('rejects false, missing or widened Vault KV gates', () => {
const falseGate = fixture();
falseGate.gates.sealedVaultFailsClosed = false;
assert.equal(
validateVaultKvWorkerSecretLiveReport(falseGate).compatible,
false,
);
const widened = fixture();
widened.gates.unreviewed = true;
assert.equal(
validateVaultKvWorkerSecretLiveReport(widened).compatible,
false,
);
const missing = fixture();
delete missing.gates.tokenRevalidatedPerResolution;
assert.equal(
validateVaultKvWorkerSecretLiveReport(missing).compatible,
false,
);
});
test('rejects sensitive material or widened report shape', () => {
const sensitive = fixture();
sensitive.limitations[0] =
'vault-private-generation-one-must-never-enter-the-report';
assert.match(
validateVaultKvWorkerSecretLiveReport(sensitive).findings.join('; '),
/sensitive material/,
);
const widened = fixture();
widened.endpoint = 'https://vault.private:8200';
const findings = validateVaultKvWorkerSecretLiveReport(widened).findings;
assert.ok(findings.includes('report envelope is invalid'));
assert.match(findings.join('; '), /endpoint is forbidden/);
});
+1 -1
View File
@@ -102,7 +102,7 @@ test('audits one source-derived QingLong 3 release identity', () => {
legacyRootExcluded: true,
workspacePackageCount: 18,
containerRootCount: 4,
deploymentFileCount: 242,
deploymentFileCount: 246,
deploymentImageReferences: 32,
deploymentVersionOccurrences: 36,
compatible: true,