mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
test(ql3): audit plugin recovery evidence
This commit is contained in:
whyour
@@ -1061,7 +1061,21 @@ jobs:
|
||||
- name: Prove migration, durable OCI recovery and rollout ordering
|
||||
env:
|
||||
QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE: '1'
|
||||
QL3_SOURCE_REVISION: ${{ github.sha }}
|
||||
QL3_KIND_BIN: ${{ github.workspace }}/kind-linux-amd64
|
||||
QL3_KUBECTL_BIN: ${{ github.workspace }}/kubectl
|
||||
QL3_KIND_CLUSTER: ql3-plugin-recovery-e2e-ci
|
||||
run: pnpm test:plugin-package-recovery-e2e:ql3
|
||||
run: |
|
||||
install -d -m 0700 "${RUNNER_TEMP}/ql3-plugin-package-recovery-e2e"
|
||||
pnpm test:plugin-package-recovery-e2e:ql3 -- \
|
||||
--report="${RUNNER_TEMP}/ql3-plugin-package-recovery-e2e/report.json"
|
||||
pnpm audit:plugin-package-recovery-e2e:ql3 -- \
|
||||
--report="${RUNNER_TEMP}/ql3-plugin-package-recovery-e2e/report.json"
|
||||
- name: Upload Plugin Package recovery E2E evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ql3-plugin-package-recovery-e2e-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/ql3-plugin-package-recovery-e2e/report.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
- 2026-08-14 终态恢复更新：Secret Action controller 不再把所有终态 Job 仅计为瞬时 `recoveryRequired`。Job 到达 Complete/Failed 后已停止执行，controller 会用 started execution 的原 lease fence 复验不可变业务结果：首次 binding 必须与 approval plan、`startedAtMs` 推导出的 binding 完全一致；transition 必须与 plan、authority evidence、commit time 推导出的 receipt 完全一致。精确 durable result 存在时补写 `succeeded`，即使 Job 已被 TTL 清理也能收敛；Failed 且无 durable mutation 时写 `failed`；Complete 但无 receipt 时以 `indeterminate` 写 `blocked`。Job 在 start barrier 前终态或审批过期且尚未创建时，controller 复用既有 claim→release fence 写 `blocked`，不让坏 Job 永久占据 reconciler 页首。任何 stored result 漂移继续抛出 conflict，`executing + Job 缺失 + receipt 缺失` 继续要求人工处理，绝不自动重建可能已产生副作用的动作。该切片不修改共享 execution schema、PostgreSQL migration 或角色权限，不新增 package、连接与常驻进程；Cluster controller 复用现有 package-executor Pool，Edge 零变化。controller/process 定向 21/21，cluster-admin 全包 348 项为 345 pass/3 条件 skip/0 fail；完整 18-package 串行 build/test 退出 0；backend 1196 项为 1194 pass/2 条件 skip/0 fail；package boundary、cluster dependency、edge import、cluster deployment 均无 finding，部署/包边界聚焦测试 61/61。PostgreSQL `18.4` arm64 physical HA 125 项、timeline `1→2` 通过，报告 SHA-256 为 `bec512767fbbd7774baa9366698f60c25c8b017ed66f459b154d143fe86293bc`，临时 Docker 资源已清理。
|
||||
- 2026-08-14 人工恢复更新（ADR-0397，已接受）：上述唯一保留的 `executing + Job/receipt 均缺失` 不确定窗口现在具有显式 Cluster 产品处置路径。既有 Approval management mTLS/OIDC endpoint 新增 `approval.recover.inspect|resolve`，只接受五分钟内 `multi_factor|hardware` User、独立 `approval.recover` 权限、二次认证、exact execution version/digest 和外部 evidence SHA-256。只允许 Secret binding/transition action；`confirm_failed` 写 failed，`abandon_unknown` 写 blocked，永远禁止人工 succeeded、Job 重建或 execution 重置。PostgreSQL `pg-0065`/capability v64 新增不可变 resolution ledger 与单个 SECURITY DEFINER resolver，在同一事务内锁 Policy/execution fence、写 allowed audit、推进终态并写 receipt；Approval manager 只有 dispatch/execution/resolution SELECT 与函数 EXECUTE，没有 execution UPDATE。通用 execution repository 与 Worker Credential 调用链保持不变。真实 PostgreSQL 18.4 已从空库完成 65 migration，证明原子提交、exact replay 不重复审计和 direct UPDATE `42501`；实现不新增 package、依赖、Pod、Pool、daemon、timer、watcher 或 Edge/Standalone 负担。18-package clean build/test 退出 0，backend 1,194 pass/2 skip/0 fail，package/dependency/edge/deployment 审计零 finding；新 migration 与 repository 内聚到 `approved-action` 领域，migration ledger 直属源码保持审定上限 65。PostgreSQL 18.4 arm64 physical HA 125 项 gate、timeline `1→2` 通过，报告 SHA-256 为 `6d4921cba74475d15722a13c6a8034793c0ee25681bc7dcaf91024927c5752fe`，临时 Docker 资源已清理。
|
||||
- 2026-08-14 升级失败收口（ADR-0398，进行中）：已存在旧 active 的 `upgrade|reinstall|rollback` 不再先切 active pointer 再物化 Package 资源。共享 activation prerequisite sequence 固定为 Secret binding/transition receipt 就绪后，从 staged install 与 immutable lock 构建候选 generation，按既有字节上限完成 Manifest、Task、Workflow、Prompt、Tool 语义物化，并以 generation digest 预发布 immutable revision；只有成功后才进入 active pointer CAS。确定性候选错误把本次 install 写为 `failed(activation_fact_conflict)`，`activeLockDigest` 保持 `previousActiveLockDigest` 且 publisher 不被调用；瞬时文件/OCI/数据库故障保持 staged 重试。generation 1 没有旧版本可保留，Secret-aware 首次安装继续使用 ADR-0395 的 post-activation B1 binding ceremony，不被候选门错误拒绝。激活后的 publication recovery 只复用 revision 做 generation-fenced reconciliation。实现复用既有 18 个 package、双方言 repository、Local 单 SQLite authority 与 Cluster caller-driven recovery Job/单 Pool，不新增 migration、表、依赖、daemon、timer、watcher、listener、连接池或常驻 cache。定向 Runtime Core 21/21、Runtime Core 全量 548/548、Local Application 47 pass/4 条件 skip、Cluster Admin 347 pass/3 条件 skip；18-package clean build/test 退出 0，backend 1196 项为 1194 pass/2 条件 skip/0 fail，package boundary、cluster dependency、cluster deployment 与 edge import 审计均通过。PostgreSQL `18.4` arm64 physical HA 125 项、timeline `1→2` 通过，报告 SHA-256 `75d7a52be75c22b2aacf32f2d7e2c432a467ebaab4d639668ff3a4b98767a17e`。真实 Kubernetes 失败升级未移动 active pointer/head 的现场门与固定物理低配设备证据仍待闭合。
|
||||
- 2026-08-14 失败升级现场门更新：既有 PostgreSQL/OCI/Kubernetes recovery E2E 已升级为 report v2。门先激活 signed OCI generation 1，再排入包含合法 Task 与循环 Workflow 的 generation 2；第一次 recovery 必须因 transition receipt 缺失而失败并保持 staged，提交 content-free receipt 后，第二次 recovery 必须确定性写入 `failed(activation_fact_conflict)`，且 generation 2 materialized revision 数量为 0。门在升级前后逐字比较 active ConfigMap UID、`resourceVersion` 与完整 `active.json`，从而排除“先切 pointer、再补偿”的假安全；OCI v1 六路径各取一次、v2 六路径各取两次，全部要求 HTTPS、exact Basic authentication 与零 redirect。runtime rollout 只绑定最终成功 recovery Job，ConfigMap-only RBAC 与 runtime 数据库隔离保持不变。源码契约 7/7、18-package clean build/test、backend 1194 pass/2 skip/0 fail、package/dependency/deployment/edge 审计均通过；PostgreSQL 18.4 arm64 physical HA 125 gate、timeline `1→2` 通过，报告 SHA-256 `8560469694c67776e5e4c70977f8bde8d4f5635f8e7d1c293ef449dc6da59f72`。本机 admin/control 镜像构建成功，但固定 Kind 1.32.8 节点镜像不在缓存且受限网络无法取得，门在创建节点前中止并确认无遗留集群/容器；因此远端 Kubernetes 成功记录与固定物理低配设备证据仍阻断 ADR Accepted。
|
||||
- 2026-08-14 失败升级现场门更新：既有 PostgreSQL/OCI/Kubernetes recovery E2E 已升级为 report v2。门先激活 signed OCI generation 1，再排入包含合法 Task 与循环 Workflow 的 generation 2；第一次 recovery 必须因 transition receipt 缺失而失败并保持 staged，提交 content-free receipt 后，第二次 recovery 必须确定性写入 `failed(activation_fact_conflict)`，且 generation 2 materialized revision 数量为 0。门在升级前后逐字比较 active ConfigMap UID、`resourceVersion` 与完整 `active.json`，从而排除“先切 pointer、再补偿”的假安全；OCI v1 六路径各取一次、v2 六路径各取两次，全部要求 HTTPS、exact Basic authentication 与零 redirect。runtime rollout 只绑定最终成功 recovery Job，ConfigMap-only RBAC 与 runtime 数据库隔离保持不变。证据链现强制使用 canonical absolute path 原子写入 `0600` no-replace 私有报告，绑定 40-hex source revision 与 admin/control OCI revision label；持久报告只保存 active JSON digest，不保存原始 pointer、credential、DSN、kubeconfig、证书或 Secret material。独立离线审计以 `O_NOFOLLOW` 和 inode/mode/size 复验文件，并 exact-shape 校验 provenance、ordering、数据库/OCI/RBAC/runtime 事实、11 个 gate 和 limitation；CI 审计成功后上传固定 14 天的 evidence artifact。该链路只增加验收代码，不增加产品 package、依赖或低配/集群运行时常驻开销。本轮 producer/离线审计契约 14/14、18-package clean build/test 退出 0、backend 1201 pass/2 skip/0 fail，package/dependency/deployment/edge 审计均通过；PostgreSQL 18.4 arm64 physical HA 125 gate、timeline `1→2` 通过，报告 SHA-256 `8560469694c67776e5e4c70977f8bde8d4f5635f8e7d1c293ef449dc6da59f72`。本机 admin/control 镜像构建成功，但固定 Kind 1.32.8 节点镜像不在缓存且受限网络无法取得，门在创建节点前中止并确认无遗留集群/容器；因此远端 Kubernetes 成功记录与固定物理低配设备证据仍阻断 ADR Accepted。
|
||||
- D-302/ADR-0390（已接受）
|
||||
Cluster operator context 增加无网络、无 mutation 的内建 `ql3-cluster-admin context validate` 预检。它先复用 owner-private context
|
||||
reader，再让每个 entry 经过与真实请求相同的 production HTTPS/Kubernetes configuration preparation，验证精确 route、hostname、CA、
|
||||
|
||||
@@ -70,12 +70,19 @@ ConfigMap 和数据库 head 之间会出现第二次分布式提交窗口，并
|
||||
`resourceVersion` 与完整 `active.json`，因此不能用“错误切换后再补偿回来”冒充旧版本未移动；OCI v1 六个路径各读取一次，v2 六个路径
|
||||
各读取两次，全部要求 HTTPS、exact Basic authentication、200 且无 redirect。最终 runtime rollout 仍只绑定最后一个成功 recovery Job，
|
||||
recovery ServiceAccount 继续只有 ConfigMap `get|create|update`，runtime 角色仍不能读取安装 authority。
|
||||
- 18-package clean build/test 在允许 loopback TLS 的环境退出 0；backend 1196 项为
|
||||
1194 pass/2 条件 skip/0 fail。新增/更新的 recovery E2E 源码契约 7/7，Runtime Core 定向 21/21。package boundary 保持 18 个 package 且
|
||||
- v2 现场门现在强制接收 canonical absolute `--report`，以 `0600` 临时文件、`fsync` 与 no-replace hard link 原子发布
|
||||
owner-private 报告；目标已存在、父目录为 symlink、缺少显式 opt-in 或缺少 40-hex `QL3_SOURCE_REVISION` 时，均在访问
|
||||
Docker/Kind 前失败关闭。admin/control 镜像的 OCI revision label 必须与报告源码 revision 相同；报告只保存 active JSON 的
|
||||
SHA-256，不保存原始 pointer、Registry credential、数据库 DSN、kubeconfig、证书或 Secret material。独立离线审计对 envelope、
|
||||
镜像 provenance、六段单调 ordering、数据库精确计数、OCI 18 次认证请求、ConfigMap-only RBAC、双节点 runtime 绑定、全部
|
||||
11 个 gate 与三项 limitation 做 exact-shape 校验，并以 `O_NOFOLLOW`/inode/mode/size 复验私有报告。CI 在独立 job 内先审计，
|
||||
再上传固定 14 天的 source-bound evidence artifact；这一证据链只属于验收，不新增产品 package、依赖或运行时常驻开销。
|
||||
- 18-package clean build/test 在允许 loopback TLS 的环境退出 0；backend 1203 项为
|
||||
1201 pass/2 条件 skip/0 fail。新增/更新的 recovery E2E producer/离线审计契约 14/14，Runtime Core 定向 21/21。package boundary 保持 18 个 package 且
|
||||
`singleSourcePackages=[]`、`shallowSourcePackages=[]`；cluster dependency、cluster deployment
|
||||
与 edge import 审计均无 finding。
|
||||
- PostgreSQL `18.4` arm64 physical HA 通过 125 项门，timeline `1→2`，报告 SHA-256
|
||||
`8560469694c67776e5e4c70977f8bde8d4f5635f8e7d1c293ef449dc6da59f72`，临时 Docker
|
||||
资源已清理。本机已成功构建现场门所需 admin/control 镜像，但固定 `kindest/node:v1.32.8` 不在本地缓存，受限网络拉取数分钟无进度；
|
||||
门在创建任何 Kind 节点前被中止，并确认没有遗留集群或容器。因此 v2 门的代码与离线契约已完成，但仍不能计为真实 Kubernetes
|
||||
门在创建任何 Kind 节点前被中止，并确认没有遗留集群或容器。因此 v2 门、私有报告与离线审计代码已完成，但仍不能计为真实 Kubernetes
|
||||
现场通过；远端 CI 成功记录与固定物理低配设备证据仍待完成，本 ADR 保持 Proposed。
|
||||
|
||||
+1
-1
@@ -401,7 +401,7 @@
|
||||
| [ADR-0395](./ADR-0395-owner-confirmed-plugin-package-secret-binding.md) | Owner 确认的 Plugin Package Secret 首次绑定 | Proposed |
|
||||
| [ADR-0396](./ADR-0396-generation-transition-plugin-package-secret-binding.md) | 按 Package Generation 切换 Plugin Package Secret Binding | Proposed |
|
||||
| [ADR-0397](./ADR-0397-explicit-cluster-secret-action-manual-recovery.md) | Cluster Secret Action 显式人工恢复 | Accepted（实现、单节点 PostgreSQL、完整 workspace/后端/边界与 physical HA 门完成） |
|
||||
| [ADR-0398](./ADR-0398-pre-activation-plugin-package-candidate-qualification.md) | Plugin Package 激活前候选资格校验与自动保留旧版本 | Proposed（v2 现场门已编排、全量/HA 门完成；远端 Kubernetes 成功记录与物理低配门待闭合） |
|
||||
| [ADR-0398](./ADR-0398-pre-activation-plugin-package-candidate-qualification.md) | Plugin Package 激活前候选资格校验与自动保留旧版本 | Proposed（v2 私有报告/离线审计已编排、全量/HA 门完成；远端 Kubernetes 成功记录与物理低配门待闭合） |
|
||||
|
||||
## 规则
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@
|
||||
"test:plugin-package-secret-binding-kubernetes-live:ql3": "pnpm --filter @qinglong/cluster-admin build && node scripts/ql3-plugin-package-secret-binding-kubernetes-live-contract.cjs",
|
||||
"audit:plugin-package-secret-binding-kubernetes-live:ql3": "node scripts/ql3-plugin-package-secret-binding-kubernetes-live-audit.cjs",
|
||||
"test:plugin-package-recovery-e2e:ql3": "pnpm --filter @qinglong/cluster-admin check && pnpm --filter @qinglong/cluster-control check && node scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs",
|
||||
"audit:plugin-package-recovery-e2e:ql3": "node scripts/ql3-plugin-package-recovery-e2e-live-audit.cjs",
|
||||
"benchmark:edge": "node scripts/ql3-edge-benchmark.cjs",
|
||||
"benchmark:cluster-control": "node scripts/ql3-cluster-control-benchmark.cjs",
|
||||
"benchmark:worker": "node scripts/ql3-worker-resource-benchmark.cjs",
|
||||
|
||||
@@ -0,0 +1,508 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
'use strict';
|
||||
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const FIXTURE = 'qinglong/plugin-package-recovery-e2e-live-contract@v2';
|
||||
const LIMITATIONS = Object.freeze([
|
||||
'isolated PostgreSQL uses explicit TLS disable; production manifests remain verify-full',
|
||||
'the authenticated HTTPS OCI Distribution fixture implements the immutable GET/referrers surface used by the resolver, not a production registry storage implementation',
|
||||
'the disposable Kind control plane is single-replica; this gate proves workload ordering, not Kubernetes control-plane HA',
|
||||
]);
|
||||
const GATE_KEYS = Object.freeze([
|
||||
'healthyInitialActivation',
|
||||
'missingTransitionFailedClosed',
|
||||
'invalidUpgradeRejectedBeforeActivation',
|
||||
'activePointerUidUnchanged',
|
||||
'activePointerResourceVersionUnchanged',
|
||||
'activePointerJsonUnchanged',
|
||||
'candidateRevisionAbsent',
|
||||
'exactAuthenticatedOciRequests',
|
||||
'recoveryRbacLeastPrivilege',
|
||||
'runtimeRolledOutAfterRecovery',
|
||||
'passed',
|
||||
]);
|
||||
const BANNED_KEYS = new Set([
|
||||
'activejson',
|
||||
'assertion',
|
||||
'authorization',
|
||||
'bearer',
|
||||
'certificate',
|
||||
'connectionstring',
|
||||
'credential',
|
||||
'dsn',
|
||||
'kubeconfig',
|
||||
'password',
|
||||
'privatekey',
|
||||
'secret',
|
||||
'tlskey',
|
||||
'token',
|
||||
]);
|
||||
const SHA256 = /^[a-f0-9]{64}$/;
|
||||
const SHA256_ID = /^sha256:[a-f0-9]{64}$/;
|
||||
const UUID =
|
||||
/^[a-f0-9]{8}-[a-f0-9]{4}-[1-8][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/i;
|
||||
|
||||
function finding(code, detail) {
|
||||
return Object.freeze({ code, detail });
|
||||
}
|
||||
|
||||
function exactKeys(value, expected) {
|
||||
return Boolean(
|
||||
value &&
|
||||
typeof value === 'object' &&
|
||||
!Array.isArray(value) &&
|
||||
JSON.stringify(Object.keys(value).sort()) ===
|
||||
JSON.stringify([...expected].sort()),
|
||||
);
|
||||
}
|
||||
|
||||
function imageDigest(value) {
|
||||
if (typeof value !== 'string') return undefined;
|
||||
return value.match(/sha256:[a-f0-9]{64}$/)?.[0];
|
||||
}
|
||||
|
||||
function safeToken(value, maximum = 256) {
|
||||
return (
|
||||
typeof value === 'string' &&
|
||||
value.length > 0 &&
|
||||
value.length <= maximum &&
|
||||
/^[A-Za-z0-9][A-Za-z0-9._:/@+-]*$/.test(value)
|
||||
);
|
||||
}
|
||||
|
||||
function sensitiveMaterial(value, key = '') {
|
||||
if (BANNED_KEYS.has(key.toLowerCase())) return true;
|
||||
if (typeof value === 'string') {
|
||||
return (
|
||||
/-----BEGIN (?:CERTIFICATE|(?:RSA |EC |OPENSSH )?PRIVATE KEY)-----/.test(
|
||||
value,
|
||||
) ||
|
||||
/postgres(?:ql)?:\/\/[^/\s]+:[^@\s]+@/i.test(value) ||
|
||||
/\bBasic\s+[A-Za-z0-9+/=_-]{8,}/i.test(value) ||
|
||||
/\beyJ[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{8,}\./.test(value) ||
|
||||
/\bqlsecret(?::|\/\/)/i.test(value)
|
||||
);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.some((entry) => sensitiveMaterial(entry));
|
||||
}
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.entries(value).some(([entryKey, entry]) =>
|
||||
sensitiveMaterial(entry, entryKey),
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function validIso(value) {
|
||||
return (
|
||||
typeof value === 'string' &&
|
||||
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/.test(value) &&
|
||||
Number.isFinite(Date.parse(value))
|
||||
);
|
||||
}
|
||||
|
||||
function validRuntime(report) {
|
||||
const runtime = report.runtime;
|
||||
const ordering = report.ordering;
|
||||
return (
|
||||
exactKeys(runtime, [
|
||||
'replicas',
|
||||
'creationTimestamp',
|
||||
'recoveryJobUid',
|
||||
'recoveryCompletedAt',
|
||||
'nodes',
|
||||
'imageIds',
|
||||
]) &&
|
||||
runtime.replicas === 2 &&
|
||||
runtime.creationTimestamp === ordering?.runtimeCreatedAt &&
|
||||
runtime.recoveryJobUid === ordering?.rejectionRecoveryJobUid &&
|
||||
runtime.recoveryCompletedAt === ordering?.rejectionRecoveryCompletedAt &&
|
||||
Array.isArray(runtime.nodes) &&
|
||||
runtime.nodes.length === 2 &&
|
||||
new Set(runtime.nodes).size === 2 &&
|
||||
runtime.nodes.every((value) => safeToken(value)) &&
|
||||
Array.isArray(runtime.imageIds) &&
|
||||
runtime.imageIds.length === 1 &&
|
||||
imageDigest(runtime.imageIds[0]) === report.images?.controlBuildId
|
||||
);
|
||||
}
|
||||
|
||||
function validatePluginPackageRecoveryE2ELiveReport(report) {
|
||||
const findings = [];
|
||||
if (
|
||||
!exactKeys(report, [
|
||||
'schema',
|
||||
'observedAt',
|
||||
'sourceRevision',
|
||||
'passed',
|
||||
'cluster',
|
||||
'architecture',
|
||||
'elapsedMs',
|
||||
'images',
|
||||
'ordering',
|
||||
'failedUpgrade',
|
||||
'database',
|
||||
'oci',
|
||||
'kubernetes',
|
||||
'runtime',
|
||||
'gates',
|
||||
'limitations',
|
||||
]) ||
|
||||
report?.schema !== FIXTURE ||
|
||||
report?.passed !== true ||
|
||||
!validIso(report?.observedAt) ||
|
||||
!/^[a-f0-9]{40}$/.test(report?.sourceRevision ?? '') ||
|
||||
!/^ql3-plugin-recovery-e2e(?:-[a-z0-9](?:[-a-z0-9]{0,24}[a-z0-9])?)?$/.test(
|
||||
report?.cluster ?? '',
|
||||
) ||
|
||||
!['amd64', 'arm64'].includes(report?.architecture) ||
|
||||
!Number.isSafeInteger(report?.elapsedMs) ||
|
||||
report.elapsedMs < 1 ||
|
||||
report.elapsedMs > 60 * 60 * 1000
|
||||
) {
|
||||
findings.push(
|
||||
finding('QL3_PLUGIN_RECOVERY_E2E_ENVELOPE', 'report envelope is invalid'),
|
||||
);
|
||||
}
|
||||
|
||||
const images = report?.images;
|
||||
if (
|
||||
!exactKeys(images, [
|
||||
'adminBuildId',
|
||||
'adminSourceRevision',
|
||||
'controlBuildId',
|
||||
'controlSourceRevision',
|
||||
'postgresRepositoryDigest',
|
||||
'migrationImageId',
|
||||
'initialRecoveryImageId',
|
||||
'stageRecoveryImageId',
|
||||
'rejectionRecoveryImageId',
|
||||
'postgresImageId',
|
||||
]) ||
|
||||
!SHA256_ID.test(images?.adminBuildId ?? '') ||
|
||||
!SHA256_ID.test(images?.controlBuildId ?? '') ||
|
||||
images?.adminSourceRevision !== report?.sourceRevision ||
|
||||
images?.controlSourceRevision !== report?.sourceRevision ||
|
||||
images?.postgresRepositoryDigest !==
|
||||
'postgres@sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296' ||
|
||||
!imageDigest(images?.migrationImageId) ||
|
||||
imageDigest(images?.initialRecoveryImageId) !==
|
||||
imageDigest(images?.migrationImageId) ||
|
||||
imageDigest(images?.stageRecoveryImageId) !==
|
||||
imageDigest(images?.migrationImageId) ||
|
||||
imageDigest(images?.rejectionRecoveryImageId) !==
|
||||
imageDigest(images?.migrationImageId) ||
|
||||
!imageDigest(images?.postgresImageId)
|
||||
) {
|
||||
findings.push(
|
||||
finding('QL3_PLUGIN_RECOVERY_E2E_IMAGES', 'image provenance is invalid'),
|
||||
);
|
||||
}
|
||||
|
||||
const ordering = report?.ordering;
|
||||
const timeKeys = [
|
||||
'migrationCompletedAt',
|
||||
'initialRecoveryCompletedAt',
|
||||
'upgradeStageFailedAt',
|
||||
'transitionCompletedAt',
|
||||
'rejectionRecoveryCompletedAt',
|
||||
'runtimeCreatedAt',
|
||||
];
|
||||
const uidKeys = [
|
||||
'migrationJobUid',
|
||||
'initialRecoveryJobUid',
|
||||
'upgradeStageJobUid',
|
||||
'transitionJobUid',
|
||||
'rejectionRecoveryJobUid',
|
||||
'runtimeBoundRecoveryJobUid',
|
||||
];
|
||||
const times = timeKeys.map((key) => Date.parse(ordering?.[key]));
|
||||
if (
|
||||
!exactKeys(ordering, [...uidKeys, ...timeKeys]) ||
|
||||
uidKeys.some((key) => !UUID.test(ordering?.[key] ?? '')) ||
|
||||
ordering?.runtimeBoundRecoveryJobUid !==
|
||||
ordering?.rejectionRecoveryJobUid ||
|
||||
timeKeys.some((key) => !validIso(ordering?.[key])) ||
|
||||
times.some((value, index) => index > 0 && value < times[index - 1])
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_ORDERING',
|
||||
'Job and rollout ordering is invalid',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const failed = report?.failedUpgrade;
|
||||
if (
|
||||
!exactKeys(failed, [
|
||||
'stageFailure',
|
||||
'transitionReceiptDigest',
|
||||
'rejectionReason',
|
||||
'candidateRevisionCount',
|
||||
'activePointerUnchanged',
|
||||
]) ||
|
||||
!exactKeys(failed?.stageFailure, ['jobUid', 'reason', 'durableState']) ||
|
||||
failed?.stageFailure?.jobUid !== ordering?.upgradeStageJobUid ||
|
||||
failed?.stageFailure?.reason !==
|
||||
'ClusterPluginPackageRecoveryRequiredError' ||
|
||||
failed?.stageFailure?.durableState !== 'staged' ||
|
||||
!SHA256.test(failed?.transitionReceiptDigest ?? '') ||
|
||||
failed?.rejectionReason !== 'activation_fact_conflict' ||
|
||||
failed?.candidateRevisionCount !== 0 ||
|
||||
failed?.activePointerUnchanged !== true
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_FAILED_UPGRADE',
|
||||
'failed upgrade evidence is invalid',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const database = report?.database;
|
||||
const databaseKeys = [
|
||||
'migrationCount',
|
||||
'capabilityVersion',
|
||||
'initialState',
|
||||
'initialActiveLockDigest',
|
||||
'upgradeState',
|
||||
'upgradePreviousActiveLockDigest',
|
||||
'upgradeActiveLockDigest',
|
||||
'upgradeFailureReason',
|
||||
'initialMutationCount',
|
||||
'upgradeMutationCount',
|
||||
'headInstallationId',
|
||||
'transitionReceiptCount',
|
||||
'initialRevisionCount',
|
||||
'upgradeRevisionCount',
|
||||
'recoverableCount',
|
||||
];
|
||||
if (
|
||||
!exactKeys(database, databaseKeys) ||
|
||||
database?.migrationCount !== 65 ||
|
||||
database?.capabilityVersion !== 64 ||
|
||||
database?.initialState !== 'active' ||
|
||||
database?.upgradeState !== 'failed' ||
|
||||
!SHA256.test(database?.initialActiveLockDigest ?? '') ||
|
||||
database?.upgradePreviousActiveLockDigest !==
|
||||
database?.initialActiveLockDigest ||
|
||||
database?.upgradeActiveLockDigest !== database?.initialActiveLockDigest ||
|
||||
database?.upgradeFailureReason !== 'activation_fact_conflict' ||
|
||||
database?.initialMutationCount !== 4 ||
|
||||
database?.upgradeMutationCount !== 3 ||
|
||||
database?.headInstallationId !== 'install-plugin-recovery-e2e-upgrade' ||
|
||||
database?.transitionReceiptCount !== 1 ||
|
||||
database?.initialRevisionCount !== 1 ||
|
||||
database?.upgradeRevisionCount !== 0 ||
|
||||
database?.recoverableCount !== 0
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_DATABASE',
|
||||
'durable database evidence is invalid',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const oci = report?.oci;
|
||||
if (
|
||||
!exactKeys(oci, [
|
||||
'https',
|
||||
'authentication',
|
||||
'authenticatedRequestCount',
|
||||
'requestCount',
|
||||
'uniquePaths',
|
||||
'initialRequestCount',
|
||||
'upgradeRequestCount',
|
||||
'redirects',
|
||||
]) ||
|
||||
oci?.https !== true ||
|
||||
oci?.authentication !== 'exact-registry-basic' ||
|
||||
oci?.authenticatedRequestCount !== 18 ||
|
||||
oci?.requestCount !== 18 ||
|
||||
oci?.uniquePaths !== 12 ||
|
||||
oci?.initialRequestCount !== 6 ||
|
||||
oci?.upgradeRequestCount !== 12 ||
|
||||
oci?.redirects !== 0
|
||||
) {
|
||||
findings.push(
|
||||
finding('QL3_PLUGIN_RECOVERY_E2E_OCI', 'OCI request evidence is invalid'),
|
||||
);
|
||||
}
|
||||
|
||||
const kubernetes = report?.kubernetes;
|
||||
const pointer = kubernetes?.activePointer;
|
||||
const rbac = kubernetes?.rbac;
|
||||
if (
|
||||
!exactKeys(kubernetes, ['activePointer', 'rbac']) ||
|
||||
!exactKeys(pointer, [
|
||||
'name',
|
||||
'uid',
|
||||
'resourceVersion',
|
||||
'activeJsonDigest',
|
||||
'intentDigest',
|
||||
'activationRef',
|
||||
]) ||
|
||||
!safeToken(pointer?.name) ||
|
||||
!UUID.test(pointer?.uid ?? '') ||
|
||||
!/^[1-9][0-9]*$/.test(pointer?.resourceVersion ?? '') ||
|
||||
!SHA256.test(pointer?.activeJsonDigest ?? '') ||
|
||||
!SHA256.test(pointer?.intentDigest ?? '') ||
|
||||
!safeToken(pointer?.activationRef) ||
|
||||
!exactKeys(rbac, [
|
||||
'getConfigMaps',
|
||||
'createConfigMaps',
|
||||
'updateConfigMaps',
|
||||
'listConfigMaps',
|
||||
'deleteConfigMaps',
|
||||
'getSecrets',
|
||||
]) ||
|
||||
rbac?.getConfigMaps !== true ||
|
||||
rbac?.createConfigMaps !== true ||
|
||||
rbac?.updateConfigMaps !== true ||
|
||||
rbac?.listConfigMaps !== false ||
|
||||
rbac?.deleteConfigMaps !== false ||
|
||||
rbac?.getSecrets !== false
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_KUBERNETES',
|
||||
'active pointer or RBAC evidence is invalid',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
if (!validRuntime(report ?? {})) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_RUNTIME',
|
||||
'runtime rollout evidence is invalid',
|
||||
),
|
||||
);
|
||||
}
|
||||
if (
|
||||
!exactKeys(report?.gates, GATE_KEYS) ||
|
||||
GATE_KEYS.some((key) => report?.gates?.[key] !== true)
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_GATES',
|
||||
'every gate must be explicitly true',
|
||||
),
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Array.isArray(report?.limitations) ||
|
||||
JSON.stringify([...report.limitations].sort()) !==
|
||||
JSON.stringify([...LIMITATIONS].sort())
|
||||
) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_LIMITATIONS',
|
||||
'limitations must remain exact',
|
||||
),
|
||||
);
|
||||
}
|
||||
if (sensitiveMaterial(report)) {
|
||||
findings.push(
|
||||
finding(
|
||||
'QL3_PLUGIN_RECOVERY_E2E_SENSITIVE',
|
||||
'report contains a forbidden key or sensitive material',
|
||||
),
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
fixture: FIXTURE,
|
||||
findings: Object.freeze(findings),
|
||||
compatible: findings.length === 0,
|
||||
});
|
||||
}
|
||||
|
||||
function readPrivateReport(filePath) {
|
||||
if (!path.isAbsolute(filePath) || path.resolve(filePath) !== filePath) {
|
||||
throw new Error('report path must be absolute and canonical');
|
||||
}
|
||||
const before = fs.lstatSync(filePath);
|
||||
if (
|
||||
!before.isFile() ||
|
||||
before.isSymbolicLink() ||
|
||||
before.size < 2 ||
|
||||
before.size > 1024 * 1024 ||
|
||||
(before.mode & 0o777) !== 0o600
|
||||
) {
|
||||
throw new Error(
|
||||
'report must be an owner-private regular file between 2 bytes and 1 MiB',
|
||||
);
|
||||
}
|
||||
const descriptor = fs.openSync(
|
||||
filePath,
|
||||
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
try {
|
||||
const opened = fs.fstatSync(descriptor);
|
||||
if (
|
||||
opened.dev !== before.dev ||
|
||||
opened.ino !== before.ino ||
|
||||
opened.size !== before.size ||
|
||||
(opened.mode & 0o777) !== 0o600
|
||||
) {
|
||||
throw new Error('report identity or mode changed while opening');
|
||||
}
|
||||
const text = fs.readFileSync(descriptor, 'utf8');
|
||||
const after = fs.fstatSync(descriptor);
|
||||
const currentPath = fs.lstatSync(filePath);
|
||||
if (
|
||||
after.size !== opened.size ||
|
||||
after.mtimeMs !== opened.mtimeMs ||
|
||||
currentPath.dev !== opened.dev ||
|
||||
currentPath.ino !== opened.ino ||
|
||||
currentPath.size !== opened.size ||
|
||||
(currentPath.mode & 0o777) !== 0o600
|
||||
) {
|
||||
throw new Error('report changed while reading');
|
||||
}
|
||||
return JSON.parse(text);
|
||||
} finally {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
function main(argv) {
|
||||
if (argv.length !== 1 || !argv[0].startsWith('--report=')) {
|
||||
throw new Error(
|
||||
'usage: ql3-plugin-package-recovery-e2e-live-audit --report=/absolute/report.json',
|
||||
);
|
||||
}
|
||||
const result = validatePluginPackageRecoveryE2ELiveReport(
|
||||
readPrivateReport(argv[0].slice('--report='.length)),
|
||||
);
|
||||
process.stdout.write(`${JSON.stringify(result)}\n`);
|
||||
if (!result.compatible) process.exitCode = 1;
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
try {
|
||||
main(process.argv.slice(2));
|
||||
} catch (error) {
|
||||
process.stderr.write(
|
||||
`QL3 Plugin Package recovery E2E live audit failed: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}\n`,
|
||||
);
|
||||
process.exitCode = 2;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
FIXTURE,
|
||||
GATE_KEYS,
|
||||
LIMITATIONS,
|
||||
readPrivateReport,
|
||||
validatePluginPackageRecoveryE2ELiveReport,
|
||||
};
|
||||
@@ -6,7 +6,7 @@ const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { randomBytes } = require('node:crypto');
|
||||
const { createHash, randomBytes } = require('node:crypto');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const yaml = require('js-yaml');
|
||||
|
||||
@@ -181,7 +181,62 @@ function imageExists(image) {
|
||||
);
|
||||
}
|
||||
|
||||
function buildImages() {
|
||||
function sourceRevision() {
|
||||
const value = process.env.QL3_SOURCE_REVISION ?? '';
|
||||
if (!/^[a-f0-9]{40}$/.test(value)) {
|
||||
fail(
|
||||
'QL3_SOURCE_REVISION must be the exact lowercase 40-hex source revision',
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function privateReportPath(argv) {
|
||||
if (
|
||||
argv.length !== 1 ||
|
||||
!argv[0].startsWith('--report=') ||
|
||||
!path.isAbsolute(argv[0].slice('--report='.length))
|
||||
) {
|
||||
fail(
|
||||
'usage: ql3-plugin-package-recovery-e2e-live-contract ' +
|
||||
'--report=/absolute/private-report.json',
|
||||
);
|
||||
}
|
||||
const reportFile = argv[0].slice('--report='.length);
|
||||
if (path.resolve(reportFile) !== reportFile) {
|
||||
fail('Plugin Package recovery E2E report path must be canonical');
|
||||
}
|
||||
if (fs.existsSync(reportFile)) {
|
||||
fail('refusing to overwrite the Plugin Package recovery E2E report');
|
||||
}
|
||||
const parent = fs.lstatSync(path.dirname(reportFile));
|
||||
if (!parent.isDirectory() || parent.isSymbolicLink()) {
|
||||
fail('Plugin Package recovery E2E report parent must be a real directory');
|
||||
}
|
||||
return reportFile;
|
||||
}
|
||||
|
||||
function writePrivateReport(reportFile, report) {
|
||||
const temporaryReport = path.join(
|
||||
path.dirname(reportFile),
|
||||
`.${path.basename(reportFile)}.${process.pid}.` +
|
||||
`${randomBytes(6).toString('hex')}.tmp`,
|
||||
);
|
||||
let descriptor;
|
||||
try {
|
||||
descriptor = fs.openSync(temporaryReport, 'wx', 0o600);
|
||||
fs.writeFileSync(descriptor, `${JSON.stringify(report, null, 2)}\n`);
|
||||
fs.fsyncSync(descriptor);
|
||||
fs.closeSync(descriptor);
|
||||
descriptor = undefined;
|
||||
fs.linkSync(temporaryReport, reportFile);
|
||||
} finally {
|
||||
if (descriptor !== undefined) fs.closeSync(descriptor);
|
||||
fs.rmSync(temporaryReport, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function buildImages(revision) {
|
||||
if (process.env.QL3_SKIP_IMAGE_BUILD === '1') {
|
||||
assert.equal(imageExists(ADMIN_IMAGE), true, `${ADMIN_IMAGE} is absent`);
|
||||
assert.equal(
|
||||
@@ -200,7 +255,7 @@ function buildImages() {
|
||||
'--tag',
|
||||
ADMIN_IMAGE,
|
||||
'--build-arg',
|
||||
'SOURCE_REVISION=ql3-plugin-recovery-e2e-live',
|
||||
`SOURCE_REVISION=${revision}`,
|
||||
'.',
|
||||
],
|
||||
{ label: 'build current QingLong 3.0 cluster-admin image' },
|
||||
@@ -214,7 +269,7 @@ function buildImages() {
|
||||
'--tag',
|
||||
CONTROL_IMAGE,
|
||||
'--build-arg',
|
||||
'SOURCE_REVISION=ql3-plugin-recovery-e2e-live',
|
||||
`SOURCE_REVISION=${revision}`,
|
||||
'.',
|
||||
],
|
||||
{ label: 'build current QingLong 3.0 cluster-control image' },
|
||||
@@ -248,6 +303,15 @@ function imageId(image) {
|
||||
)[0].Id;
|
||||
}
|
||||
|
||||
function imageSourceRevision(image) {
|
||||
return JSON.parse(
|
||||
run(DOCKER, ['image', 'inspect', image], {
|
||||
capture: true,
|
||||
quiet: true,
|
||||
}).stdout,
|
||||
)[0].Config?.Labels?.['org.opencontainers.image.revision'];
|
||||
}
|
||||
|
||||
function createRegistryCertificate(root) {
|
||||
const caKey = path.join(root, 'registry-ca.key');
|
||||
const caCert = path.join(root, 'registry-ca.crt');
|
||||
@@ -1264,6 +1328,19 @@ function activePointerEvidence(fixture) {
|
||||
});
|
||||
}
|
||||
|
||||
function reportActivePointer(pointer) {
|
||||
return Object.freeze({
|
||||
name: pointer.name,
|
||||
uid: pointer.uid,
|
||||
resourceVersion: pointer.resourceVersion,
|
||||
activeJsonDigest: createHash('sha256')
|
||||
.update(pointer.activeJson, 'utf8')
|
||||
.digest('hex'),
|
||||
intentDigest: pointer.intentDigest,
|
||||
activationRef: pointer.activationRef,
|
||||
});
|
||||
}
|
||||
|
||||
function registryEvidence(fixture) {
|
||||
const output = kubectl(
|
||||
['-n', NAMESPACE, 'logs', REGISTRY_NAME, '-c', 'registry'],
|
||||
@@ -1542,12 +1619,14 @@ function diagnostics() {
|
||||
if (snapshot.stdout) process.stderr.write(`${snapshot.stdout}\n`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
async function main(argv = process.argv.slice(2)) {
|
||||
const reportFile = privateReportPath(argv);
|
||||
if (process.env.QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE !== '1') {
|
||||
fail(
|
||||
'Refusing to create a live cluster without QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE=1',
|
||||
);
|
||||
}
|
||||
const revision = sourceRevision();
|
||||
const clusterName = exactClusterName();
|
||||
const existing = kind(['get', 'clusters'], {
|
||||
capture: true,
|
||||
@@ -1605,7 +1684,13 @@ async function main() {
|
||||
let created = false;
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
buildImages();
|
||||
buildImages(revision);
|
||||
const adminSourceRevision = imageSourceRevision(ADMIN_IMAGE);
|
||||
const controlSourceRevision = imageSourceRevision(CONTROL_IMAGE);
|
||||
assert.equal(adminSourceRevision, revision);
|
||||
assert.equal(controlSourceRevision, revision);
|
||||
const adminBuildId = imageId(ADMIN_IMAGE);
|
||||
const controlBuildId = imageId(CONTROL_IMAGE);
|
||||
ensurePostgresImage();
|
||||
fs.writeFileSync(
|
||||
kindConfig,
|
||||
@@ -1829,13 +1914,17 @@ async function main() {
|
||||
]);
|
||||
const report = Object.freeze({
|
||||
schema: REPORT_SCHEMA,
|
||||
observedAt: new Date().toISOString(),
|
||||
sourceRevision: revision,
|
||||
passed: true,
|
||||
cluster: clusterName,
|
||||
architecture: fixtureArchitecture,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
images: Object.freeze({
|
||||
adminBuildId: imageId(ADMIN_IMAGE),
|
||||
controlBuildId: imageId(CONTROL_IMAGE),
|
||||
adminBuildId,
|
||||
adminSourceRevision,
|
||||
controlBuildId,
|
||||
controlSourceRevision,
|
||||
postgresRepositoryDigest: POSTGRES_REPOSITORY_DIGEST,
|
||||
migrationImageId,
|
||||
initialRecoveryImageId,
|
||||
@@ -1871,17 +1960,38 @@ async function main() {
|
||||
database,
|
||||
oci,
|
||||
kubernetes: Object.freeze({
|
||||
activePointer: pointerAfterRejection,
|
||||
activePointer: reportActivePointer(pointerAfterRejection),
|
||||
rbac,
|
||||
}),
|
||||
runtime,
|
||||
gates: Object.freeze({
|
||||
healthyInitialActivation: true,
|
||||
missingTransitionFailedClosed: true,
|
||||
invalidUpgradeRejectedBeforeActivation: true,
|
||||
activePointerUidUnchanged: true,
|
||||
activePointerResourceVersionUnchanged: true,
|
||||
activePointerJsonUnchanged: true,
|
||||
candidateRevisionAbsent: true,
|
||||
exactAuthenticatedOciRequests: true,
|
||||
recoveryRbacLeastPrivilege: true,
|
||||
runtimeRolledOutAfterRecovery: true,
|
||||
passed: true,
|
||||
}),
|
||||
limitations: Object.freeze([
|
||||
'isolated PostgreSQL uses explicit TLS disable; production manifests remain verify-full',
|
||||
'the authenticated HTTPS OCI Distribution fixture implements the immutable GET/referrers surface used by the resolver, not a production registry storage implementation',
|
||||
'the disposable Kind control plane is single-replica; this gate proves workload ordering, not Kubernetes control-plane HA',
|
||||
]),
|
||||
});
|
||||
process.stdout.write(`${JSON.stringify(report)}\n`);
|
||||
writePrivateReport(reportFile, report);
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
fixture: REPORT_SCHEMA,
|
||||
reportWritten: true,
|
||||
passed: true,
|
||||
})}\n`,
|
||||
);
|
||||
} catch (error) {
|
||||
diagnostics();
|
||||
throw error;
|
||||
@@ -1902,13 +2012,21 @@ async function main() {
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
process.stderr.write(
|
||||
`${JSON.stringify({
|
||||
schema: 'qinglong/plugin-package-recovery-e2e-live-failure@v1',
|
||||
name: error?.name ?? 'Error',
|
||||
message: error?.message ?? 'unknown failure',
|
||||
})}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(
|
||||
`${JSON.stringify({
|
||||
schema: 'qinglong/plugin-package-recovery-e2e-live-failure@v1',
|
||||
name: error?.name ?? 'Error',
|
||||
message: error?.message ?? 'unknown failure',
|
||||
})}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
REPORT_SCHEMA,
|
||||
privateReportPath,
|
||||
writePrivateReport,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const root = path.resolve(__dirname, '../..');
|
||||
const producerPath = path.join(
|
||||
root,
|
||||
'scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs',
|
||||
);
|
||||
const {
|
||||
FIXTURE,
|
||||
GATE_KEYS,
|
||||
readPrivateReport,
|
||||
validatePluginPackageRecoveryE2ELiveReport,
|
||||
} = require('../../scripts/ql3-plugin-package-recovery-e2e-live-audit.cjs');
|
||||
const {
|
||||
writePrivateReport,
|
||||
} = require('../../scripts/ql3-plugin-package-recovery-e2e-live-contract.cjs');
|
||||
|
||||
function validReport() {
|
||||
const admin = `sha256:${'a'.repeat(64)}`;
|
||||
const control = `sha256:${'b'.repeat(64)}`;
|
||||
const sourceRevision = 'c'.repeat(40);
|
||||
const lock = 'd'.repeat(64);
|
||||
return {
|
||||
schema: FIXTURE,
|
||||
observedAt: '2026-08-14T08:00:07.000Z',
|
||||
sourceRevision,
|
||||
passed: true,
|
||||
cluster: 'ql3-plugin-recovery-e2e-ci',
|
||||
architecture: 'arm64',
|
||||
elapsedMs: 420000,
|
||||
images: {
|
||||
adminBuildId: admin,
|
||||
adminSourceRevision: sourceRevision,
|
||||
controlBuildId: control,
|
||||
controlSourceRevision: sourceRevision,
|
||||
postgresRepositoryDigest:
|
||||
'postgres@sha256:1961f96e6029a02c3812d7cb329a3b03a3ac2bb067058dec17b0f5596aca9296',
|
||||
migrationImageId: `docker://${admin}`,
|
||||
initialRecoveryImageId: `docker://${admin}`,
|
||||
stageRecoveryImageId: `docker://${admin}`,
|
||||
rejectionRecoveryImageId: `docker://${admin}`,
|
||||
postgresImageId: `containerd://sha256:${'e'.repeat(64)}`,
|
||||
},
|
||||
ordering: {
|
||||
migrationJobUid: '00000000-0000-4000-8000-000000000001',
|
||||
migrationCompletedAt: '2026-08-14T08:00:01.000Z',
|
||||
initialRecoveryJobUid: '00000000-0000-4000-8000-000000000002',
|
||||
initialRecoveryCompletedAt: '2026-08-14T08:00:02.000Z',
|
||||
upgradeStageJobUid: '00000000-0000-4000-8000-000000000003',
|
||||
upgradeStageFailedAt: '2026-08-14T08:00:03.000Z',
|
||||
transitionJobUid: '00000000-0000-4000-8000-000000000004',
|
||||
transitionCompletedAt: '2026-08-14T08:00:04.000Z',
|
||||
rejectionRecoveryJobUid: '00000000-0000-4000-8000-000000000005',
|
||||
rejectionRecoveryCompletedAt: '2026-08-14T08:00:05.000Z',
|
||||
runtimeCreatedAt: '2026-08-14T08:00:06.000Z',
|
||||
runtimeBoundRecoveryJobUid: '00000000-0000-4000-8000-000000000005',
|
||||
},
|
||||
failedUpgrade: {
|
||||
stageFailure: {
|
||||
jobUid: '00000000-0000-4000-8000-000000000003',
|
||||
reason: 'ClusterPluginPackageRecoveryRequiredError',
|
||||
durableState: 'staged',
|
||||
},
|
||||
transitionReceiptDigest: 'f'.repeat(64),
|
||||
rejectionReason: 'activation_fact_conflict',
|
||||
candidateRevisionCount: 0,
|
||||
activePointerUnchanged: true,
|
||||
},
|
||||
database: {
|
||||
migrationCount: 65,
|
||||
capabilityVersion: 64,
|
||||
initialState: 'active',
|
||||
initialActiveLockDigest: lock,
|
||||
upgradeState: 'failed',
|
||||
upgradePreviousActiveLockDigest: lock,
|
||||
upgradeActiveLockDigest: lock,
|
||||
upgradeFailureReason: 'activation_fact_conflict',
|
||||
initialMutationCount: 4,
|
||||
upgradeMutationCount: 3,
|
||||
headInstallationId: 'install-plugin-recovery-e2e-upgrade',
|
||||
transitionReceiptCount: 1,
|
||||
initialRevisionCount: 1,
|
||||
upgradeRevisionCount: 0,
|
||||
recoverableCount: 0,
|
||||
},
|
||||
oci: {
|
||||
https: true,
|
||||
authentication: 'exact-registry-basic',
|
||||
authenticatedRequestCount: 18,
|
||||
requestCount: 18,
|
||||
uniquePaths: 12,
|
||||
initialRequestCount: 6,
|
||||
upgradeRequestCount: 12,
|
||||
redirects: 0,
|
||||
},
|
||||
kubernetes: {
|
||||
activePointer: {
|
||||
name: 'ql3-plugin-package-active-e2e-monitor',
|
||||
uid: '00000000-0000-4000-8000-000000000006',
|
||||
resourceVersion: '42',
|
||||
activeJsonDigest: '1'.repeat(64),
|
||||
intentDigest: '2'.repeat(64),
|
||||
activationRef: 'activation-e2e-monitor-1',
|
||||
},
|
||||
rbac: {
|
||||
getConfigMaps: true,
|
||||
createConfigMaps: true,
|
||||
updateConfigMaps: true,
|
||||
listConfigMaps: false,
|
||||
deleteConfigMaps: false,
|
||||
getSecrets: false,
|
||||
},
|
||||
},
|
||||
runtime: {
|
||||
replicas: 2,
|
||||
creationTimestamp: '2026-08-14T08:00:06.000Z',
|
||||
recoveryJobUid: '00000000-0000-4000-8000-000000000005',
|
||||
recoveryCompletedAt: '2026-08-14T08:00:05.000Z',
|
||||
nodes: ['worker-a', 'worker-b'],
|
||||
imageIds: [`docker://${control}`],
|
||||
},
|
||||
gates: Object.fromEntries(GATE_KEYS.map((key) => [key, true])),
|
||||
limitations: [
|
||||
'isolated PostgreSQL uses explicit TLS disable; production manifests remain verify-full',
|
||||
'the authenticated HTTPS OCI Distribution fixture implements the immutable GET/referrers surface used by the resolver, not a production registry storage implementation',
|
||||
'the disposable Kind control plane is single-replica; this gate proves workload ordering, not Kubernetes control-plane HA',
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
test('offline audit accepts one exact low-sensitive recovery report', () => {
|
||||
const result = validatePluginPackageRecoveryE2ELiveReport(validReport());
|
||||
assert.equal(result.compatible, true);
|
||||
assert.deepEqual(result.findings, []);
|
||||
});
|
||||
|
||||
test('offline audit rejects broken upgrade, ordering and image relationships', () => {
|
||||
const report = validReport();
|
||||
report.database.upgradeRevisionCount = 1;
|
||||
report.ordering.transitionCompletedAt = '2026-08-14T07:59:59.000Z';
|
||||
report.runtime.imageIds = [`docker://sha256:${'9'.repeat(64)}`];
|
||||
report.gates.activePointerJsonUnchanged = false;
|
||||
const codes = validatePluginPackageRecoveryE2ELiveReport(report).findings.map(
|
||||
(value) => value.code,
|
||||
);
|
||||
assert.ok(codes.includes('QL3_PLUGIN_RECOVERY_E2E_DATABASE'));
|
||||
assert.ok(codes.includes('QL3_PLUGIN_RECOVERY_E2E_ORDERING'));
|
||||
assert.ok(codes.includes('QL3_PLUGIN_RECOVERY_E2E_RUNTIME'));
|
||||
assert.ok(codes.includes('QL3_PLUGIN_RECOVERY_E2E_GATES'));
|
||||
});
|
||||
|
||||
test('offline audit rejects forbidden keys and credential-shaped material', () => {
|
||||
const report = validReport();
|
||||
report.password = 'not-allowed';
|
||||
report.limitations[0] =
|
||||
'postgresql://operator:credential@database.example/qinglong';
|
||||
const result = validatePluginPackageRecoveryE2ELiveReport(report);
|
||||
assert.equal(result.compatible, false);
|
||||
assert.ok(
|
||||
result.findings.some(
|
||||
(value) => value.code === 'QL3_PLUGIN_RECOVERY_E2E_SENSITIVE',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('producer writes an atomic owner-private no-replace report', (context) => {
|
||||
const temporary = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'ql3-recovery-report-'),
|
||||
);
|
||||
context.after(() => fs.rmSync(temporary, { recursive: true, force: true }));
|
||||
const reportFile = path.join(temporary, 'report.json');
|
||||
writePrivateReport(reportFile, validReport());
|
||||
assert.equal(fs.statSync(reportFile).mode & 0o777, 0o600);
|
||||
assert.equal(readPrivateReport(reportFile).schema, FIXTURE);
|
||||
assert.deepEqual(
|
||||
fs.readdirSync(temporary).filter((value) => value.endsWith('.tmp')),
|
||||
[],
|
||||
);
|
||||
assert.throws(
|
||||
() => writePrivateReport(reportFile, validReport()),
|
||||
/EEXIST|exist/i,
|
||||
);
|
||||
});
|
||||
|
||||
test('private report reader rejects relative, symlinked and broad-mode reports', (context) => {
|
||||
const temporary = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'ql3-recovery-audit-'),
|
||||
);
|
||||
context.after(() => fs.rmSync(temporary, { recursive: true, force: true }));
|
||||
const reportFile = path.join(temporary, 'report.json');
|
||||
fs.writeFileSync(reportFile, '{}\n', { mode: 0o600 });
|
||||
assert.throws(() => readPrivateReport('report.json'), /absolute/);
|
||||
const link = path.join(temporary, 'link.json');
|
||||
fs.symlinkSync(reportFile, link);
|
||||
assert.throws(() => readPrivateReport(link), /owner-private/);
|
||||
fs.chmodSync(reportFile, 0o640);
|
||||
assert.throws(() => readPrivateReport(reportFile), /owner-private/);
|
||||
});
|
||||
|
||||
test('producer refuses missing authority before invoking Docker or Kind', (context) => {
|
||||
const temporary = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'ql3-recovery-producer-'),
|
||||
);
|
||||
context.after(() => fs.rmSync(temporary, { recursive: true, force: true }));
|
||||
const report = path.join(temporary, 'report.json');
|
||||
const noReport = spawnSync(process.execPath, [producerPath], {
|
||||
cwd: root,
|
||||
encoding: 'utf8',
|
||||
});
|
||||
assert.notEqual(noReport.status, 0);
|
||||
assert.match(noReport.stderr, /--report=\/absolute/);
|
||||
const noOptIn = spawnSync(
|
||||
process.execPath,
|
||||
[producerPath, `--report=${report}`],
|
||||
{
|
||||
cwd: root,
|
||||
env: { ...process.env, QL3_DOCKER_BIN: '/not/invoked/docker' },
|
||||
encoding: 'utf8',
|
||||
},
|
||||
);
|
||||
assert.notEqual(noOptIn.status, 0);
|
||||
assert.match(noOptIn.stderr, /QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE=1/);
|
||||
const noRevision = spawnSync(
|
||||
process.execPath,
|
||||
[producerPath, `--report=${report}`],
|
||||
{
|
||||
cwd: root,
|
||||
env: {
|
||||
...process.env,
|
||||
QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE: '1',
|
||||
QL3_KIND_BIN: '/not/invoked/kind',
|
||||
},
|
||||
encoding: 'utf8',
|
||||
},
|
||||
);
|
||||
assert.notEqual(noRevision.status, 0);
|
||||
assert.match(noRevision.stderr, /QL3_SOURCE_REVISION/);
|
||||
});
|
||||
@@ -122,9 +122,33 @@ test('package script and independent CI job execute the full gate', () => {
|
||||
const job = workflow.jobs['cluster-plugin-package-recovery-e2e'];
|
||||
assert.ok(job);
|
||||
assert.equal(job['timeout-minutes'], 35);
|
||||
assert.equal(
|
||||
packageJson.scripts['audit:plugin-package-recovery-e2e:ql3'],
|
||||
'node scripts/ql3-plugin-package-recovery-e2e-live-audit.cjs',
|
||||
);
|
||||
assert.ok(
|
||||
job.steps.some(
|
||||
(step) => step.run === 'pnpm test:plugin-package-recovery-e2e:ql3',
|
||||
(step) =>
|
||||
String(step.run).includes(
|
||||
'pnpm test:plugin-package-recovery-e2e:ql3',
|
||||
) &&
|
||||
String(step.run).includes('--report=') &&
|
||||
String(step.run).includes('pnpm audit:plugin-package-recovery-e2e:ql3'),
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
job.steps.some(
|
||||
(step) =>
|
||||
step.env?.QL3_SOURCE_REVISION === '${{ github.sha }}' &&
|
||||
step.env?.QL3_PLUGIN_PACKAGE_RECOVERY_E2E_LIVE === '1',
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
job.steps.some(
|
||||
(step) =>
|
||||
step.if === 'always()' &&
|
||||
String(step.uses).startsWith('actions/upload-artifact@') &&
|
||||
step.with?.['retention-days'] === 14,
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
@@ -136,6 +160,16 @@ test('package script and independent CI job execute the full gate', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('gate persists only a source-bound low-sensitive private report', () => {
|
||||
assert.match(live, /privateReportPath\(argv\)/);
|
||||
assert.match(live, /writePrivateReport\(reportFile, report\)/);
|
||||
assert.match(live, /QL3_SOURCE_REVISION/);
|
||||
assert.match(live, /org\.opencontainers\.image\.revision/);
|
||||
assert.match(live, /activeJsonDigest/);
|
||||
assert.doesNotMatch(live, /activePointer: pointerAfterRejection/);
|
||||
assert.match(live, /reportWritten: true/);
|
||||
});
|
||||
|
||||
test('private Registry evidence uses one exact Secret file and authenticated requests', () => {
|
||||
assert.match(fixture, /request\.headers\.authorization !== authorization/);
|
||||
assert.match(fixture, /www-authenticate/);
|
||||
|
||||
Reference in New Issue
Block a user