From 51145a802856d7b375edc7488d8b56ad127d1586 Mon Sep 17 00:00:00 2001 From: whyour Date: Fri, 14 Aug 2026 14:53:44 +0800 Subject: [PATCH] test(ql3): qualify failed upgrades on edge devices --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 3 +- ...-envelopes-and-physical-device-evidence.md | 13 + ...-plugin-package-candidate-qualification.md | 23 +- package.json | 1 + scripts/ql3-linux-resource-gate.cjs | 92 +- scripts/ql3-physical-edge-evidence.cjs | 89 +- ...plugin-package-recovery-edge-benchmark.cjs | 863 ++++++++++++++++++ test/back/ql3LinuxResourceGate.test.cjs | 47 +- test/back/ql3PhysicalEdgeEvidence.test.cjs | 47 +- ...luginPackageRecoveryEdgeBenchmark.test.cjs | 49 + 10 files changed, 1156 insertions(+), 71 deletions(-) create mode 100644 scripts/ql3-plugin-package-recovery-edge-benchmark.cjs create mode 100644 test/back/ql3PluginPackageRecoveryEdgeBenchmark.test.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 2efc1fa5..e177361b 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -36,6 +36,7 @@ - 2026-08-14 人工恢复更新（ADR-0397，已接受）：上述唯一保留的 `executing + Job/receipt 均缺失` 不确定窗口现在具有显式 Cluster 产品处置路径。既有 Approval management mTLS/OIDC endpoint 新增 `approval.recover.inspect|resolve`，只接受五分钟内 `multi_factor|hardware` User、独立 `approval.recover` 权限、二次认证、exact execution version/digest 和外部 evidence SHA-256。只允许 Secret binding/transition action；`confirm_failed` 写 failed，`abandon_unknown` 写 blocked，永远禁止人工 succeeded、Job 重建或 execution 重置。PostgreSQL `pg-0065`/capability v64 新增不可变 resolution ledger 与单个 SECURITY DEFINER resolver，在同一事务内锁 Policy/execution fence、写 allowed audit、推进终态并写 receipt；Approval manager 只有 dispatch/execution/resolution SELECT 与函数 EXECUTE，没有 execution UPDATE。通用 execution repository 与 Worker Credential 调用链保持不变。真实 PostgreSQL 18.4 已从空库完成 65 migration，证明原子提交、exact replay 不重复审计和 direct UPDATE `42501`；实现不新增 package、依赖、Pod、Pool、daemon、timer、watcher 或 Edge/Standalone 负担。18-package clean build/test 退出 0，backend 1,194 pass/2 skip/0 fail，package/dependency/edge/deployment 审计零 finding；新 migration 与 repository 内聚到 `approved-action` 领域，migration ledger 直属源码保持审定上限 65。PostgreSQL 18.4 arm64 physical HA 125 项 gate、timeline `1→2` 通过，报告 SHA-256 为 `6d4921cba74475d15722a13c6a8034793c0ee25681bc7dcaf91024927c5752fe`，临时 Docker 资源已清理。 - 2026-08-14 升级失败收口（ADR-0398，进行中）：已存在旧 active 的 `upgrade|reinstall|rollback` 不再先切 active pointer 再物化 Package 资源。共享 activation prerequisite sequence 固定为 Secret binding/transition receipt 就绪后，从 staged install 与 immutable lock 构建候选 generation，按既有字节上限完成 Manifest、Task、Workflow、Prompt、Tool 语义物化，并以 generation digest 预发布 immutable revision；只有成功后才进入 active pointer CAS。确定性候选错误把本次 install 写为 `failed(activation_fact_conflict)`，`activeLockDigest` 保持 `previousActiveLockDigest` 且 publisher 不被调用；瞬时文件/OCI/数据库故障保持 staged 重试。generation 1 没有旧版本可保留，Secret-aware 首次安装继续使用 ADR-0395 的 post-activation B1 binding ceremony，不被候选门错误拒绝。激活后的 publication recovery 只复用 revision 做 generation-fenced reconciliation。实现复用既有 18 个 package、双方言 repository、Local 单 SQLite authority 与 Cluster caller-driven recovery Job/单 Pool，不新增 migration、表、依赖、daemon、timer、watcher、listener、连接池或常驻 cache。定向 Runtime Core 21/21、Runtime Core 全量 548/548、Local Application 47 pass/4 条件 skip、Cluster Admin 347 pass/3 条件 skip；18-package clean build/test 退出 0，backend 1196 项为 1194 pass/2 条件 skip/0 fail，package boundary、cluster dependency、cluster deployment 与 edge import 审计均通过。PostgreSQL `18.4` arm64 physical HA 125 项、timeline `1→2` 通过，报告 SHA-256 `75d7a52be75c22b2aacf32f2d7e2c432a467ebaab4d639668ff3a4b98767a17e`。真实 Kubernetes 失败升级未移动 active pointer/head 的现场门与固定物理低配设备证据仍待闭合。 - 2026-08-14 失败升级现场门更新：既有 PostgreSQL/OCI/Kubernetes recovery E2E 已升级为 report v2。门先激活 signed OCI generation 1，再排入包含合法 Task 与循环 Workflow 的 generation 2；第一次 recovery 必须因 transition receipt 缺失而失败并保持 staged，提交 content-free receipt 后，第二次 recovery 必须确定性写入 `failed(activation_fact_conflict)`，且 generation 2 materialized revision 数量为 0。门在升级前后逐字比较 active ConfigMap UID、`resourceVersion` 与完整 `active.json`，从而排除“先切 pointer、再补偿”的假安全；OCI v1 六路径各取一次、v2 六路径各取两次，全部要求 HTTPS、exact Basic authentication 与零 redirect。runtime rollout 只绑定最终成功 recovery Job，ConfigMap-only RBAC 与 runtime 数据库隔离保持不变。证据链现强制使用 canonical absolute path 原子写入 `0600` no-replace 私有报告，绑定 40-hex source revision 与 admin/control OCI revision label；持久报告只保存 active JSON digest，不保存原始 pointer、credential、DSN、kubeconfig、证书或 Secret material。独立离线审计以 `O_NOFOLLOW` 和 inode/mode/size 复验文件，并 exact-shape 校验 provenance、ordering、数据库/OCI/RBAC/runtime 事实、11 个 gate 和 limitation；CI 审计成功后上传固定 14 天的 evidence artifact。该链路只增加验收代码，不增加产品 package、依赖或低配/集群运行时常驻开销。本轮 producer/离线审计契约 14/14、18-package clean build/test 退出 0、backend 1201 pass/2 skip/0 fail，package/dependency/deployment/edge 审计均通过；PostgreSQL 18.4 arm64 physical HA 125 gate、timeline `1→2` 通过，报告 SHA-256 `8560469694c67776e5e4c70977f8bde8d4f5635f8e7d1c293ef449dc6da59f72`。本机 admin/control 镜像构建成功，但固定 Kind 1.32.8 节点镜像不在缓存且受限网络无法取得，门在创建节点前中止并确认无遗留集群/容器；因此远端 Kubernetes 成功记录与固定物理低配设备证据仍阻断 ADR Accepted。 + - 2026-08-14 固定低配失败升级 workload 更新：新增 `plugin_package_failed_upgrade_edge_candidate`，在 fresh production migration SQLite 上用正式 install/materialized repositories、正式 recovery coordinator 和正式资源物化 prerequisite，先激活 generation 1，再恢复含循环 Workflow 的 generation 2。exact gate 要求候选从 staged 进入 `failed(activation_fact_conflict)`、旧 active/previous digest 均保留、stage/publisher/inspect 零调用、候选 revision 零行、byte source 四路径各一次、`integrity_check=ok`，并限制耗时、RSS delta 与数据库 logical/allocated growth。它已进入 128 MiB router stress、256 MiB Edge release 和统一 physical recorder；CI cgroup 报告永久保持 `supported=false`，只有固定型号、无虚拟化设备在声明数据盘生成的统一私有总报告才是物理候选，且仍不证明断电。Node `v24.18.0` arm64 开发机本轮观测 14.390 ms、RSS delta 3,014,656 bytes、SQLite 增长 8,192 bytes，只作为可运行性样本。阶段完整验证为 18-package clean build/test 退出 0、backend 1,203 pass/2 skip/0 fail、相关 backend/发布契约 34/34，package boundary 仍为 18 个且 `singleSourcePackages=[]`、`shallowSourcePackages=[]`，cluster dependency/deployment、Edge import 和 service bridge import 审计零 finding。固定设备统一报告与远端 Kubernetes 成功记录仍是 ADR-0398 转 Accepted 的阻断项。 - D-302/ADR-0390（已接受） Cluster operator context 增加无网络、无 mutation 的内建 `ql3-cluster-admin context validate` 预检。它先复用 owner-private context reader，再让每个 entry 经过与真实请求相同的 production HTTPS/Kubernetes configuration preparation，验证精确 route、hostname、CA、 @@ -5911,7 +5912,7 @@ ADR-0088 将 Linux CI 拆为三个不能互相替代的档位： 三档都固定 Node `24.18.0` 和 Debian slim，只在原生 Linux x64/arm64 runner 上执行。`scripts/ql3-linux-resource-gate.cjs` 必须从容器内部证明 cgroup v2 的 memory/swap/cpu/PID 精确限额、工作前后零 `max`/OOM 事件、非 root、只读根与 workspace、有界可写 `/tmp`、seccomp 和 `NoNewPrivs`；Docker CLI 参数、`os.totalmem()`、QEMU 或一次成功退出均不能替代这些事实。Workflow/Prompt crash report 永久保持 `physicalPowerLossProven=false`；固定设备受控断电前，它只能证明进程崩溃恢复。Docker tmpfs 上的 logical/allocated growth 也不能推导闪存 FTL 写放大。 -当前 Edge 产品候选仍是 1 CPU、256 MiB RAM、1 GiB 可用持久空间，推荐 512 MiB，但正式支持阈值必须来自固定物理设备。`scripts/ql3-physical-edge-evidence.cjs` 已提供 fail-closed candidate recorder：绑定 exact-shape 设备 manifest、实际架构/内存/kernel/libc/文件系统/容量、Edge 与真实数据盘 Node SQLite 基准，拒绝容器/VM 指示、symlink/路径漂移和覆盖已有输出，以 `0600` + fsync 发布 SHA-256 绑定报告，并永久输出 `supported=false`。D-87 又增加了同设备同 boot 的进程 idle sampler 与专用文件系统 fault probe；D-88 增加了通过正式 TaskDefinition Repository 在 fresh v14 SQLite 上写入并完整扫描 100/1000/10000 三档的规模记录器，同时记录 RSS、耗时与数据库 logical/allocated bytes。基础记录器只能导入私有、摘要有效且 scope 精确的同设备同 boot 报告。规模报告不证明 scheduler 吞吐，也不替代 2.x adopted migration 的时间与磁盘峰值。物理支持仍需补齐冷启动/首次 ready、整机指标、adopted migration、application recovery、断电和 release signature；规模协议虽已实现，仍待固定实机采集。Cluster 必须另行记录副本数、PostgreSQL/连接池、claim/ACK/completion 吞吐、重复率、队列深度和 failover；不得从 512 MiB 空载门禁推导生产规格。 +当前 Edge 产品候选仍是 1 CPU、256 MiB RAM、1 GiB 可用持久空间，推荐 512 MiB，但正式支持阈值必须来自固定物理设备。`scripts/ql3-physical-edge-evidence.cjs` 已提供 fail-closed candidate recorder：绑定 exact-shape 设备 manifest、实际架构/内存/kernel/libc/文件系统/容量、Edge 与真实数据盘 Node SQLite 基准，拒绝容器/VM 指示、symlink/路径漂移和覆盖已有输出，以 `0600` + fsync 发布 SHA-256 绑定报告，并永久输出 `supported=false`。D-87 又增加了同设备同 boot 的进程 idle sampler 与专用文件系统 fault probe；D-88 增加了通过正式 TaskDefinition Repository 在 fresh v14 SQLite 上写入并完整扫描 100/1000/10000 三档的规模记录器，同时记录 RSS、耗时与数据库 logical/allocated bytes。D-306B2/ADR-0398 进一步把真实 Plugin Package 失败升级加入基础 workload：在同一 scratch 数据盘的 fresh production migration SQLite 上先激活 generation 1，再以正式 recovery coordinator、候选物化 prerequisite 和 SQLite repositories 恢复含循环 Workflow 的 generation 2；只有 v2 进入 `failed(activation_fact_conflict)`、旧 active digest 双重保留、publisher 零调用、候选 materialized revision 零行、`integrity_check=ok` 且耗时/RSS/logical 与 allocated 增长均未越界才通过。该 workload 也进入 128/256 MiB cgroup 门，但容器结果仍只属于 stress；统一 physical recorder 在无虚拟化固定设备上采集的总报告才属于物理候选。基础记录器只能导入私有、摘要有效且 scope 精确的同设备同 boot 报告。规模报告不证明 scheduler 吞吐，也不替代 2.x adopted migration 的时间与磁盘峰值；失败升级报告也永久声明 `physical_power_loss_not_proven`。物理支持仍需补齐冷启动/首次 ready、整机指标、adopted migration、application recovery、断电和 release signature；规模与失败升级协议虽已实现，仍待固定实机采集。Cluster 必须另行记录副本数、PostgreSQL/连接池、claim/ACK/completion 吞吐、重复率、队列深度和 failover；不得从 512 MiB 空载门禁推导生产规格。 冷启动/首次 ready 只能对最终 application artifact 与冻结的 readiness contract 计时，不能用 Executor benchmark 或模块 import 代替。TaskDefinition schema、独立 execution revision digest、版本化 Trigger schema、Repository 与读取路径现已由 local SQLite capability v16 冻结，规模记录器只能调用正式端口；它生成的仍是物理 candidate，固定设备报告采集前不形成支持结论。fresh schema migration 计时只属于该规模报告的局部度量，2.x adopted database 仍须单独记录 migration 时间、替表额外磁盘峰值与采样精度。对应证据未完成前，candidate report 保留其余 required evidence。 diff --git a/docs/adr/ADR-0088-linux-resource-envelopes-and-physical-device-evidence.md b/docs/adr/ADR-0088-linux-resource-envelopes-and-physical-device-evidence.md index 385837d0..61e8228f 100644 --- a/docs/adr/ADR-0088-linux-resource-envelopes-and-physical-device-evidence.md +++ b/docs/adr/ADR-0088-linux-resource-envelopes-and-physical-device-evidence.md @@ -152,6 +152,19 @@ pnpm evidence:physical-edge -- \ 记录器拒绝 macOS、容器/VM 指示、架构/内存/文件系统漂移、symlink、根目录数据路径、已有输出和非 canonical 路径；SQLite 基准的临时库实际位于声明的数据文件系统。输出以 `0600`、no-replace、fsync 发布，并绑定 SHA-256。`storageMedium`、设备型号和 SoC 仍是 operator-declared provenance，SHA-256 也不是签名，因此报告始终为 `supported=false`。 +基础记录器现在还在同一真实数据文件系统的私有 scratch 中运行 +`plugin_package_failed_upgrade_edge_candidate`：在 fresh production migration SQLite +上先激活 generation 1，再持久化一个包含循环 Workflow 的 generation 2 staged +升级，并通过正式 recovery coordinator、正式候选资源物化 prerequisite 和正式 SQLite +repositories 恢复。报告只有在 generation 2 精确进入 +`failed(activation_fact_conflict)`、旧 `activeLockDigest` 与 +`previousActiveLockDigest` 均保留、activation publisher 零调用、候选 materialized +revision 零行、SQLite `integrity_check=ok`，且耗时、RSS delta、logical/allocated +数据库增长均处于固定上限内时才通过。相同 workload 同时进入 128 MiB router stress +与 256 MiB Edge release cgroup 门，前者仍只是 CI stress；只有由本记录器在无虚拟化 +指示的固定设备和声明数据盘上生成时，才属于物理候选证据。两条路径都明确保留 +`physical_power_loss_not_proven`，不能推出断电安全或正式最低配置。 + ### 补充 idle 证据 idle sampler 只观察一个已经启动的 QingLong Node 进程，采样窗口为 30 至 3600 秒，间隔为 1 至 60 秒且必须整除窗口。manifest 必须绑定 device、PID 与期望 executable；采样期间 PID、boot ID、进程 start ticks、UID、executable 或命令摘要任一漂移都失败关闭： diff --git a/docs/adr/ADR-0398-pre-activation-plugin-package-candidate-qualification.md b/docs/adr/ADR-0398-pre-activation-plugin-package-candidate-qualification.md index b1deaf48..ae282b3a 100644 --- a/docs/adr/ADR-0398-pre-activation-plugin-package-candidate-qualification.md +++ b/docs/adr/ADR-0398-pre-activation-plugin-package-candidate-qualification.md @@ -48,7 +48,9 @@ ConfigMap 和数据库 head 之间会出现第二次分布式提交窗口，并 - Local 与 Cluster 组合测试证明恢复顺序一致，既有 active publication/reconciliation 不回归。 - 完整 18-package、backend、package/dependency/deployment/edge/import 审计通过。 - 真实 PostgreSQL/Kubernetes 门证明失败升级没有移动 active ConfigMap/head；physical HA 门通过。 -- 固定物理低配设备证据仍由 ADR-0396 单独阻断，不能用开发机观测替代。 +- 固定物理低配设备必须运行真实 SQLite 失败升级 workload，证明旧 active 保留、候选 + revision 零写入、publisher 零调用、数据库完整性及耗时/RSS/数据盘增长上限；容器 + cgroup 结果只能作为前置 stress，不能用开发机或虚拟化观测替代。 ## 影响与替代方案 @@ -77,8 +79,8 @@ ConfigMap 和数据库 head 之间会出现第二次分布式提交窗口，并 镜像 provenance、六段单调 ordering、数据库精确计数、OCI 18 次认证请求、ConfigMap-only RBAC、双节点 runtime 绑定、全部 11 个 gate 与三项 limitation 做 exact-shape 校验，并以 `O_NOFOLLOW`/inode/mode/size 复验私有报告。CI 在独立 job 内先审计， 再上传固定 14 天的 source-bound evidence artifact；这一证据链只属于验收，不新增产品 package、依赖或运行时常驻开销。 -- 18-package clean build/test 在允许 loopback TLS 的环境退出 0；backend 1203 项为 - 1201 pass/2 条件 skip/0 fail。新增/更新的 recovery E2E producer/离线审计契约 14/14，Runtime Core 定向 21/21。package boundary 保持 18 个 package 且 +- 18-package clean build/test 在允许 loopback TLS 的环境退出 0；backend 1205 项为 + 1203 pass/2 条件 skip/0 fail。新增/更新的 recovery E2E producer/离线审计契约 14/14，Runtime Core 定向 21/21。package boundary 保持 18 个 package 且 `singleSourcePackages=[]`、`shallowSourcePackages=[]`；cluster dependency、cluster deployment 与 edge import 审计均无 finding。 - PostgreSQL `18.4` arm64 physical HA 通过 125 项门，timeline `1→2`，报告 SHA-256 @@ -86,3 +88,18 @@ ConfigMap 和数据库 head 之间会出现第二次分布式提交窗口，并 资源已清理。本机已成功构建现场门所需 admin/control 镜像，但固定 `kindest/node:v1.32.8` 不在本地缓存，受限网络拉取数分钟无进度； 门在创建任何 Kind 节点前被中止，并确认没有遗留集群或容器。因此 v2 门、私有报告与离线审计代码已完成，但仍不能计为真实 Kubernetes 现场通过；远端 CI 成功记录与固定物理低配设备证据仍待完成，本 ADR 保持 Proposed。 +- 固定低配资格门的产品工作负载已落地：`evidence:plugin-package-recovery-edge` + 使用 fresh production migration SQLite、正式 install/materialized repositories、正式 + recovery coordinator 与正式语义物化门，实测无效 generation 2 从 staged 进入 failed， + 旧 active generation 保留、publisher 0 次、候选 materialized revision 0 行、数据库 + `integrity_check=ok`。该工作负载已接入 128/256 MiB Linux 资源门和统一 physical Edge + recorder；统一聚合器会 exact-shape 拒绝缺失、改名或失败报告，并把 + `plugin_package_failed_upgrade_retains_active_generation` 列为独立 collected/remaining + evidence。当前本机 arm64 执行仅是开发验证，尚未产生固定型号、无虚拟化设备上的 + owner-private 总报告，也未证明受控断电，因此不能把本 ADR 转为 Accepted。 + 本轮真实 workload 在 Node `v24.18.0` arm64 开发机观测为 14.390 ms、RSS delta + 3,014,656 bytes、SQLite logical/allocated growth 各 8,192 bytes，均低于 + 10,000 ms、96 MiB、4 MiB 的候选上限；这些数值只证明脚本与门禁可运行，不是 + 固定设备支持结论。对应 backend 新增路径、物理聚合和 Linux release 契约 34/34； + package boundary、cluster dependency、cluster deployment、Edge import 与 service + bridge import 审计均通过。 diff --git a/package.json b/package.json index 1fe81a92..77eb9aab 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "benchmark:worker": "node scripts/ql3-worker-resource-benchmark.cjs", "benchmark:ai-profile": "node scripts/ql3-ai-profile-benchmark.cjs", "evidence:physical-edge": "node scripts/ql3-physical-edge-evidence.cjs", + "evidence:plugin-package-recovery-edge": "node scripts/ql3-plugin-package-recovery-edge-benchmark.cjs", "evidence:physical-edge-idle": "node scripts/ql3-physical-edge-idle-sampler.cjs", "evidence:physical-edge-fault": "node scripts/ql3-physical-edge-fault-probe.cjs", "evidence:physical-edge-task-scale": "node scripts/ql3-physical-edge-task-scale.cjs", diff --git a/scripts/ql3-linux-resource-gate.cjs b/scripts/ql3-linux-resource-gate.cjs index fb2d765d..864082f3 100644 --- a/scripts/ql3-linux-resource-gate.cjs +++ b/scripts/ql3-linux-resource-gate.cjs @@ -295,41 +295,41 @@ function createWorkloadPlans(root, tierName) { ...(tierName === 'edge-release-ci' ? ['edge', 'standalone'].map((profile) => Object.freeze({ - name: `local-ai-prompt-durable-output-${profile}`, - format: 'node_test', - nodeArgs: Object.freeze([ - '--test', - '--test-name-pattern=executes one active Package Prompt', - path.join( - root, - 'packages/ql3-local-application/test/activation.test.cjs', - ), - ]), - env: Object.freeze({ - QL3_PROMPT_RESOURCE_PROFILE: profile, - QL3_PROMPT_RESOURCE_OUTPUT_BYTES: String( - PROMPT_RESOURCE_OUTPUT_BYTES, - ), - }), - maxProcessRssBytes: tier.promptMaxProcessRssMb * MIB, - contract: Object.freeze({ - kind: 'durable_prompt_output_resource', - profile, - journalMode: profile === 'edge' ? 'delete' : 'wal', - durableOutputBytes: PROMPT_RESOURCE_OUTPUT_BYTES, - providerCalls: 2, - keyLoads: 1, - keyResolutions: 1, - exactReplay: true, - contentFree: true, - maxLogicalWriteAmplificationPermille: 3_000, - maxAllocatedWriteAmplificationPermille: 3_500, - maxWalWriteAmplificationPermille: - profile === 'standalone' ? 3_000 : 0, - requireWalGrowth: profile === 'standalone', - runAttempts: 0, - physicalPowerLossProven: false, - }), + name: `local-ai-prompt-durable-output-${profile}`, + format: 'node_test', + nodeArgs: Object.freeze([ + '--test', + '--test-name-pattern=executes one active Package Prompt', + path.join( + root, + 'packages/ql3-local-application/test/activation.test.cjs', + ), + ]), + env: Object.freeze({ + QL3_PROMPT_RESOURCE_PROFILE: profile, + QL3_PROMPT_RESOURCE_OUTPUT_BYTES: String( + PROMPT_RESOURCE_OUTPUT_BYTES, + ), + }), + maxProcessRssBytes: tier.promptMaxProcessRssMb * MIB, + contract: Object.freeze({ + kind: 'durable_prompt_output_resource', + profile, + journalMode: profile === 'edge' ? 'delete' : 'wal', + durableOutputBytes: PROMPT_RESOURCE_OUTPUT_BYTES, + providerCalls: 2, + keyLoads: 1, + keyResolutions: 1, + exactReplay: true, + contentFree: true, + maxLogicalWriteAmplificationPermille: 3_000, + maxAllocatedWriteAmplificationPermille: 3_500, + maxWalWriteAmplificationPermille: + profile === 'standalone' ? 3_000 : 0, + requireWalGrowth: profile === 'standalone', + runAttempts: 0, + physicalPowerLossProven: false, + }), }), ) : []), @@ -395,10 +395,7 @@ function createWorkloadPlans(root, tierName) { profiles: Object.freeze(['edge', 'standalone']), crashPointsPerProfile: 7, scenarios: 14, - boundaries: Object.freeze([ - 'model_start', - 'model_completion', - ]), + boundaries: Object.freeze(['model_start', 'model_completion']), mechanism: 'process_sigkill_then_database_reopen', physicalPowerLossProven: false, }), @@ -426,6 +423,20 @@ function createWorkloadPlans(root, tierName) { }), ] : []), + Object.freeze({ + name: 'plugin-package-failed-upgrade', + format: 'json', + script: path.join( + root, + 'scripts/ql3-plugin-package-recovery-edge-benchmark.cjs', + ), + args: Object.freeze([ + '--json', + '--max-duration-ms=10000', + `--max-rss-delta-mb=${tier.edgeMaxRssDeltaMb}`, + `--max-database-growth-bytes=${4 * MIB}`, + ]), + }), ]); } return Object.freeze([ @@ -489,8 +500,7 @@ function parseNodeTestReport(output, plan) { evidence.exactReplay !== contract.exactReplay || evidence.contentFree !== contract.contentFree || evidence.durableFacts?.attempts !== contract.runAttempts || - evidence.physicalPowerLossProven !== - contract.physicalPowerLossProven || + evidence.physicalPowerLossProven !== contract.physicalPowerLossProven || !Number.isSafeInteger( evidence.databaseLogicalWriteAmplificationPermille, ) || diff --git a/scripts/ql3-physical-edge-evidence.cjs b/scripts/ql3-physical-edge-evidence.cjs index 549d4e8a..2f314186 100644 --- a/scripts/ql3-physical-edge-evidence.cjs +++ b/scripts/ql3-physical-edge-evidence.cjs @@ -5,6 +5,9 @@ const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); const { spawnSync } = require('node:child_process'); +const { + validateReport: validatePluginPackageRecoveryEdgeReport, +} = require('./ql3-plugin-package-recovery-edge-benchmark.cjs'); const MIB = 1024 * 1024; const GIB = 1024 * MIB; @@ -710,12 +713,72 @@ function runEvidenceWorkloads(root, dataPath) { ], { TMPDIR: scratchPath }, ), + runJsonWorkload( + root, + 'plugin-package-failed-upgrade', + 'scripts/ql3-plugin-package-recovery-edge-benchmark.cjs', + [ + '--json', + '--max-duration-ms=10000', + '--max-rss-delta-mb=96', + `--max-database-growth-bytes=${4 * MIB}`, + ], + { TMPDIR: scratchPath }, + ), ]); } finally { fs.rmSync(scratchPath, { recursive: true, force: false }); } } +function validateEvidenceWorkloads(workloads) { + if (!Array.isArray(workloads) || workloads.length !== 3) { + return Object.freeze(['physical evidence workloads are incomplete']); + } + const expectedNames = [ + 'edge-executor', + 'node-sqlite-on-device-storage', + 'plugin-package-failed-upgrade', + ]; + if ( + workloads.some( + (workload, index) => + !hasExactKeys(workload, ['name', 'report']) || + workload.name !== expectedNames[index], + ) + ) { + return Object.freeze(['physical evidence workload identity is invalid']); + } + const violations = []; + const edge = workloads[0].report; + if ( + edge?.schemaVersion !== 1 || + edge?.profile !== 'edge' || + edge?.gates?.passed !== true || + !Array.isArray(edge?.gates?.violations) || + edge.gates.violations.length !== 0 + ) { + violations.push('edge executor workload did not pass exactly'); + } + const sqlite = workloads[1].report; + if ( + sqlite?.journalMode !== 'delete' || + sqlite?.synchronous !== 'full' || + sqlite?.integrityCheck !== 'ok' || + !Number.isSafeInteger(sqlite?.databaseBytes) || + sqlite.databaseBytes < 1 || + !isNonNegativeNumber(sqlite?.rssDeltaMb) + ) { + violations.push('on-device SQLite workload is invalid'); + } + violations.push( + ...validatePluginPackageRecoveryEdgeReport(workloads[2].report).map( + (violation) => `Plugin Package recovery workload: ${violation}`, + ), + ); + return Object.freeze(violations); +} + function canonicalDigest(value) { return crypto .createHash('sha256') @@ -1402,7 +1465,10 @@ function buildEvidenceReport({ supplementalEvidence = [], generatedAt, }) { - const violations = validateObservedPlatform(manifest, observed); + const violations = [ + ...validateObservedPlatform(manifest, observed), + ...validateEvidenceWorkloads(workloads), + ]; const idleEvidence = supplementalEvidence.find( ({ evidenceClass }) => evidenceClass === 'physical_edge_idle_candidate', ); @@ -1491,6 +1557,19 @@ function buildEvidenceReport({ if (directServiceStopEvidence) { collectedEvidence.push('init_managed_graceful_application_stop'); } + const pluginPackageRecoveryEvidence = Array.isArray(workloads) + ? workloads.find(({ name }) => name === 'plugin-package-failed-upgrade') + : undefined; + const pluginPackageRecoveryQualified = + pluginPackageRecoveryEvidence !== undefined && + validatePluginPackageRecoveryEdgeReport( + pluginPackageRecoveryEvidence.report, + ).length === 0; + if (pluginPackageRecoveryQualified) { + collectedEvidence.push( + 'plugin_package_failed_upgrade_retains_active_generation', + ); + } const remainingRequiredEvidence = [ ...(directServiceStartEvidence ? ['firmware_and_bootloader_power_on_to_linux_kernel_clock'] @@ -1513,6 +1592,13 @@ function buildEvidenceReport({ 'power_loss_restart', 'release_archive_signature', ]; + if (!pluginPackageRecoveryQualified) { + remainingRequiredEvidence.splice( + 1, + 0, + 'plugin_package_failed_upgrade_retains_active_generation', + ); + } if (!idleEvidence) { remainingRequiredEvidence.splice( 1, @@ -1713,6 +1799,7 @@ module.exports = { validateServiceStartEvidenceReport, validateDirectServiceStartEvidenceReport, validateDirectServiceStopEvidenceReport, + validateEvidenceWorkloads, validateTaskScaleWorkload, validateObservedPlatform, writeNoReplace, diff --git a/scripts/ql3-plugin-package-recovery-edge-benchmark.cjs b/scripts/ql3-plugin-package-recovery-edge-benchmark.cjs new file mode 100644 index 00000000..2a907698 --- /dev/null +++ b/scripts/ql3-plugin-package-recovery-edge-benchmark.cjs @@ -0,0 +1,863 @@ +#!/usr/bin/env node + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { performance } = require('node:perf_hooks'); + +const MIB = 1024 * 1024; +const DEFAULT_MAX_DURATION_MS = 10_000; +const DEFAULT_MAX_RSS_DELTA_MB = 96; +const DEFAULT_MAX_DATABASE_GROWTH_BYTES = 4 * MIB; + +const REPORT_KEYS = Object.freeze([ + 'calls', + 'database', + 'durable', + 'evidenceClass', + 'gates', + 'identity', + 'limitations', + 'measurement', + 'recovery', + 'schemaVersion', + 'supported', + 'thresholds', +]); +const CALL_KEYS = Object.freeze([ + 'byteSourceClose', + 'byteSourceOpen', + 'byteSourceRead', + 'publisherInspect', + 'publisherPublish', + 'stage', +]); +const DATABASE_KEYS = Object.freeze([ + 'allocatedBytes', + 'allocatedGrowthBytes', + 'files', + 'integrityCheck', + 'journalMode', + 'logicalBytes', + 'logicalGrowthBytes', + 'materializedCandidateRevisions', + 'synchronous', +]); +const DATABASE_FILE_KEYS = Object.freeze([ + 'allocatedBytes', + 'logicalBytes', + 'suffix', +]); +const DURABLE_KEYS = Object.freeze([ + 'activeLockDigestPreserved', + 'failedFrom', + 'failureReason', + 'headInstallationIsCandidate', + 'previousActiveLockDigestPreserved', + 'state', +]); +const GATE_KEYS = Object.freeze(['passed', 'violations']); +const IDENTITY_KEYS = Object.freeze(['architecture', 'node', 'platform']); +const MEASUREMENT_KEYS = Object.freeze([ + 'durationMs', + 'rssAfterBytes', + 'rssBeforeBytes', + 'rssDeltaBytes', +]); +const RECOVERY_KEYS = Object.freeze([ + 'deferred', + 'manualRequired', + 'pages', + 'remaining', + 'retry', + 'safeToAdmit', + 'scanned', + 'settled', + 'superseded', +]); +const THRESHOLD_KEYS = Object.freeze([ + 'maxDatabaseGrowthBytes', + 'maxDurationMs', + 'maxRssDeltaBytes', +]); +const EXPECTED_LIMITATIONS = Object.freeze([ + 'container_or_vm_resource_limits_are_not_physical_support_evidence', + 'physical_power_loss_not_proven', +]); + +class QingLong3PluginPackageRecoveryEdgeBenchmarkError extends Error { + constructor(message) { + super( + `QingLong 3.0 Plugin Package recovery Edge benchmark failed: ${message}`, + ); + this.name = 'QingLong3PluginPackageRecoveryEdgeBenchmarkError'; + } +} + +function hasExactKeys(value, expected) { + return ( + value !== null && + typeof value === 'object' && + !Array.isArray(value) && + JSON.stringify(Object.keys(value).sort()) === JSON.stringify(expected) + ); +} + +function positiveNumber(value, label) { + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new QingLong3PluginPackageRecoveryEdgeBenchmarkError( + `${label} must be a positive number`, + ); + } + return parsed; +} + +function positiveSafeInteger(value, label) { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 1) { + throw new QingLong3PluginPackageRecoveryEdgeBenchmarkError( + `${label} must be a positive safe integer`, + ); + } + return parsed; +} + +function parseArguments(argv) { + const options = { + json: false, + maxDatabaseGrowthBytes: DEFAULT_MAX_DATABASE_GROWTH_BYTES, + maxDurationMs: DEFAULT_MAX_DURATION_MS, + maxRssDeltaBytes: DEFAULT_MAX_RSS_DELTA_MB * MIB, + }; + for (const argument of argv) { + if (argument === '--') continue; + if (argument === '--json') { + options.json = true; + continue; + } + const [name, raw] = argument.split('=', 2); + if (raw === undefined) { + throw new QingLong3PluginPackageRecoveryEdgeBenchmarkError( + `unsupported argument ${argument}`, + ); + } + if (name === '--max-duration-ms') { + options.maxDurationMs = positiveNumber(raw, name); + } else if (name === '--max-rss-delta-mb') { + options.maxRssDeltaBytes = positiveNumber(raw, name) * MIB; + } else if (name === '--max-database-growth-bytes') { + options.maxDatabaseGrowthBytes = positiveSafeInteger(raw, name); + } else { + throw new QingLong3PluginPackageRecoveryEdgeBenchmarkError( + `unsupported argument ${name}`, + ); + } + } + return Object.freeze(options); +} + +function round(value) { + return Math.round(value * 1_000) / 1_000; +} + +function sha256(value) { + return crypto.createHash('sha256').update(value).digest('hex'); +} + +function storageSnapshot(databasePath) { + const files = [ + ['database', databasePath], + ['-journal', `${databasePath}-journal`], + ['-wal', `${databasePath}-wal`], + ['-shm', `${databasePath}-shm`], + ] + .filter(([, filePath]) => fs.existsSync(filePath)) + .map(([suffix, filePath]) => { + const stat = fs.statSync(filePath); + return Object.freeze({ + suffix, + logicalBytes: stat.size, + allocatedBytes: stat.blocks * 512, + }); + }); + return Object.freeze({ + files: Object.freeze(files), + logicalBytes: files.reduce((sum, file) => sum + file.logicalBytes, 0), + allocatedBytes: files.reduce((sum, file) => sum + file.allocatedBytes, 0), + }); +} + +function packageFixture(runtime, generation, previous, invalidWorkflow) { + const packageArchitecture = + process.arch === 'x64' + ? 'amd64' + : process.arch === 'arm' + ? 'arm/v7' + : process.arch; + const task = (id) => ({ + schema: 'qinglong/plugin-package-task-resource@v1', + id, + name: `Recovery ${id}`, + labels: { 'plugin.qinglong.io/source': 'edge-recovery-benchmark' }, + enabled: true, + kind: 'command', + spec: { + schema: 'qinglong/command@v1', + config: { + command: { + kind: 'argv', + file: '/usr/bin/printf', + args: [id], + }, + }, + }, + }); + const workflow = { + schema: 'qinglong/plugin-package-workflow-resource@v1', + id: 'recovery', + name: 'Recovery qualification', + enabled: true, + steps: invalidWorkflow + ? [ + { id: 'collect', task: 'collect', needs: ['report'] }, + { id: 'report', task: 'report', needs: ['collect'] }, + ] + : [ + { id: 'collect', task: 'collect', needs: [] }, + { id: 'report', task: 'report', needs: ['collect'] }, + ], + }; + const values = { + 'tasks/collect.json': task('collect'), + 'tasks/report.json': task('report'), + 'workflows/recovery.json': workflow, + }; + const bytes = Object.fromEntries( + Object.entries(values).map(([resourcePath, value]) => [ + resourcePath, + Buffer.from(JSON.stringify(value)), + ]), + ); + const descriptors = Object.entries(bytes) + .map(([resourcePath, material]) => ({ + path: resourcePath, + bytes: material.byteLength, + digest: sha256(material), + })) + .sort((left, right) => left.path.localeCompare(right.path)); + const manifest = { + apiVersion: runtime.PLUGIN_PACKAGE_API_VERSION, + kind: runtime.PLUGIN_PACKAGE_KIND, + metadata: { + name: 'edge-recovery-benchmark', + displayName: 'Edge recovery benchmark', + version: `${generation}.0.0`, + description: 'Bounded failed-upgrade qualification workload', + license: 'Apache-2.0', + }, + spec: { + compatibility: { + qinglong: '>=3.0.0-0 <4.0.0', + architectures: [packageArchitecture], + deploymentProfiles: ['edge'], + }, + runtimes: [], + resources: { + memory: { recommended: '16Mi' }, + disk: { install: '4Mi', working: '8Mi' }, + }, + permissions: { + network: { allowedHosts: [] }, + secrets: [], + tools: ['system.command'], + }, + contents: { + tasks: ['tasks/collect.json', 'tasks/report.json'], + workflows: ['workflows/recovery.json'], + prompts: [], + tools: [], + }, + }, + }; + const environment = { + qinglongVersion: '3.0.0-alpha.0', + architecture: packageArchitecture, + deploymentProfile: 'edge', + runtimes: [], + availableMemoryBytes: 128 * MIB, + availableDiskBytes: 256 * MIB, + }; + const plan = runtime.planPluginPackageInstall( + manifest, + environment, + previous?.manifest, + ); + const artifactDigest = sha256(`edge-recovery-artifact-${generation}`); + const action = { + lockId: `edge-recovery-lock-${generation}`, + projectId: 'default', + manifest, + plan, + environment, + ...(previous ? { previousManifest: previous.manifest } : {}), + source: { + kind: 'offline', + locator: `offline:sha256:${artifactDigest}`, + artifactDigest, + artifactBytes: Object.values(bytes).reduce( + (sum, material) => sum + material.byteLength, + 0, + ), + contentDigest: runtime.pluginPackageContentTreeDigest(descriptors), + }, + architecture: packageArchitecture, + deploymentProfile: 'edge', + targetGeneration: generation, + ...(previous ? { previousLockDigest: previous.lock.lockDigest } : {}), + }; + const lock = runtime.createPluginPackageLock({ + ...action, + approval: { + requestId: `edge-recovery-approval-${generation}`, + requestVersion: 1, + dispatchId: `edge-recovery-dispatch-${generation}`, + actionDigest: runtime.pluginPackageInstallActionDigest(action), + previewDigest: runtime.pluginPackageInstallPlanDigest(plan), + approvedBy: { type: 'user', id: 'edge-owner' }, + approvedAtMs: 100 + generation, + expiresAtMs: 100_000, + fence: { projectVersion: 1, bindingVersion: 1 }, + }, + createdAtMs: 200 + generation, + }); + return Object.freeze({ + bytes: Object.freeze(bytes), + lock, + manifest, + manifestBytes: Buffer.from( + runtime.serializePluginPackageManifest(manifest), + ), + }); +} + +function stage(runtime, value, record, occurredAtMs) { + return runtime.transitionPluginPackageInstall(value.lock, record, { + type: 'stage_completed', + mutationId: `edge-recovery-stage-${value.lock.targetGeneration}`, + occurredAtMs, + stageRef: `edge-recovery:${value.lock.lockDigest}`, + artifactDigest: value.lock.source.artifactDigest, + manifestDigest: value.lock.manifestDigest, + contentDigest: value.lock.source.contentDigest, + evidenceDigest: sha256( + `edge-recovery-stage-${value.lock.targetGeneration}`, + ), + }); +} + +async function createActiveGeneration(runtime, repository, value) { + const queued = runtime.createPluginPackageInstall(value.lock, { + installationId: 'edge-recovery-install-1', + mutationId: 'edge-recovery-create-1', + occurredAtMs: 301, + }); + await repository.create( + runtime.pluginPackageInstallCreate(value.lock, queued, null), + ); + const staged = stage(runtime, value, queued, 302); + await repository.commit(runtime.pluginPackageInstallCommit(queued, staged)); + const activating = runtime.transitionPluginPackageInstall( + value.lock, + staged, + { + type: 'activation_started', + mutationId: 'edge-recovery-activate-1', + occurredAtMs: 303, + }, + ); + await repository.commit( + runtime.pluginPackageInstallCommit(staged, activating), + ); + const active = runtime.transitionPluginPackageInstall( + value.lock, + activating, + { + type: 'activation_committed', + mutationId: 'edge-recovery-commit-1', + occurredAtMs: 304, + activationRef: `edge-active:${value.lock.lockDigest}`, + intentDigest: runtime.pluginPackageActivationIntentDigest( + value.lock, + activating, + ), + generation: 1, + contentDigest: value.lock.source.contentDigest, + }, + ); + await repository.commit( + runtime.pluginPackageInstallCommit(activating, active), + ); + return active; +} + +async function createStagedUpgrade(runtime, repository, value, active) { + const queued = runtime.createPluginPackageInstall(value.lock, { + installationId: 'edge-recovery-install-2', + mutationId: 'edge-recovery-create-2', + occurredAtMs: 401, + }); + await repository.create( + runtime.pluginPackageInstallCreate(value.lock, queued, active), + ); + const staged = stage(runtime, value, queued, 402); + await repository.commit(runtime.pluginPackageInstallCommit(queued, staged)); + return staged; +} + +function runtimeDependencies() { + const pluginPackage = require('../packages/ql3-runtime-core/dist/plugin-package/pluginPackage'); + const install = require('../packages/ql3-runtime-core/dist/plugin-package/installation/pluginPackageInstall'); + const bundle = require('../packages/ql3-runtime-core/dist/plugin-package/pluginPackageBundle'); + const recovery = require('../packages/ql3-runtime-core/dist/plugin-package/installation/pluginPackageRecovery'); + const materialization = require('../packages/ql3-runtime-core/dist/plugin-package/pluginPackageResourceMaterialization'); + const generation = require('../packages/ql3-runtime-core/dist/plugin-package/pluginPackageResourceGeneration'); + const semantics = require('../packages/ql3-runtime-core/dist/task-definition/taskSpecSemantic'); + return { + ...pluginPackage, + ...install, + ...bundle, + ...recovery, + ...materialization, + ...generation, + ...semantics, + }; +} + +async function runBenchmark(options) { + if (Number(process.versions.node.split('.')[0]) < 24) { + throw new QingLong3PluginPackageRecoveryEdgeBenchmarkError( + 'Node.js 24 or newer is required', + ); + } + const { DatabaseSync } = require('node:sqlite'); + const runtime = runtimeDependencies(); + const { + LocalSqlitePluginPackageInstallRepository, + } = require('../packages/ql3-local-sqlite/dist/plugin-package/pluginPackageInstallRepository'); + const { + LocalSqlitePluginPackageMaterializedRevisionRepository, + } = require('../packages/ql3-local-sqlite/dist/plugin-package/pluginPackageMaterializedRevisionRepository'); + const { + migrateLocalSqliteDatabase, + } = require('../packages/ql3-local-sqlite/dist/migration/migration'); + const temporaryRoot = fs.mkdtempSync( + path.join(os.tmpdir(), 'ql3-package-recovery-edge-'), + ); + fs.chmodSync(temporaryRoot, 0o700); + const databasePath = path.join(temporaryRoot, 'recovery.sqlite'); + let database; + try { + database = new DatabaseSync(databasePath, { + allowExtension: false, + allowUnknownNamedParameters: false, + defensive: true, + enableDoubleQuotedStringLiterals: false, + enableForeignKeyConstraints: true, + timeout: 1_000, + }); + await migrateLocalSqliteDatabase(database); + database.exec('PRAGMA journal_mode = DELETE; PRAGMA synchronous = FULL;'); + const baseline = storageSnapshot(databasePath); + const repository = new LocalSqlitePluginPackageInstallRepository(database); + const registry = runtime.createBuiltInTaskSpecSemanticRegistry(); + const materializedRepository = + new LocalSqlitePluginPackageMaterializedRevisionRepository( + database, + registry, + ); + const first = packageFixture(runtime, 1, null, false); + const second = packageFixture(runtime, 2, first, true); + const calls = { + byteSourceClose: 0, + byteSourceOpen: 0, + byteSourceRead: 0, + publisherInspect: 0, + publisherPublish: 0, + stage: 0, + }; + const rssBeforeBytes = process.memoryUsage().rss; + const startedAt = performance.now(); + const active = await createActiveGeneration(runtime, repository, first); + const staged = await createStagedUpgrade( + runtime, + repository, + second, + active, + ); + const prerequisite = + new runtime.PluginPackageResourceActivationPrerequisite({ + byteSource: { + async open(generation) { + if ( + generation.lockDigest !== second.lock.lockDigest || + generation.installationId !== staged.installationId + ) { + throw new Error('byte source received another generation'); + } + calls.byteSourceOpen += 1; + return { + async read(resourcePath, maximumBytes) { + calls.byteSourceRead += 1; + const material = + resourcePath === 'package.json' + ? second.manifestBytes + : second.bytes[resourcePath]; + if (!material || material.byteLength > maximumBytes) { + throw new Error('byte source path or bound is invalid'); + } + return new Uint8Array(material); + }, + async close() { + calls.byteSourceClose += 1; + }, + }; + }, + }, + materializedRepository, + taskSpecSemanticRegistry: registry, + }); + const coordinator = new runtime.PluginPackageRecoveryCoordinator({ + repository, + stageProvider: { + async stage() { + calls.stage += 1; + throw new Error('staged upgrade must not be staged again'); + }, + }, + publisher: { + async publish() { + calls.publisherPublish += 1; + throw new Error('invalid candidate must not be published'); + }, + async inspect() { + calls.publisherInspect += 1; + throw new Error('invalid candidate must not be inspected'); + }, + }, + activationPrerequisite: prerequisite, + now: () => 500, + }); + const recovery = await coordinator.recover({ pageSize: 1, maxPages: 2 }); + const durationMs = performance.now() - startedAt; + const rssAfterBytes = process.memoryUsage().rss; + const durable = await repository.find(staged.projectId, staged.packageName); + const candidateGeneration = + runtime.createPluginPackageResourceGenerationFromReferences({ + installationId: staged.installationId, + projectId: staged.projectId, + packageName: staged.packageName, + lockDigest: second.lock.lockDigest, + generation: second.lock.targetGeneration, + previousActiveLockDigest: first.lock.lockDigest, + contentDigest: second.lock.source.contentDigest, + resources: second.lock.resources, + }); + const materializedCandidateRevisions = database + .prepare( + `SELECT count(*) AS count + FROM "QingLong3PluginPackageMaterializedRevisions" + WHERE generation_digest = ?`, + ) + .get(candidateGeneration.generationDigest).count; + const integrityCheck = database + .prepare('PRAGMA integrity_check') + .get().integrity_check; + const finalStorage = storageSnapshot(databasePath); + const logicalGrowthBytes = Math.max( + 0, + finalStorage.logicalBytes - baseline.logicalBytes, + ); + const allocatedGrowthBytes = Math.max( + 0, + finalStorage.allocatedBytes - baseline.allocatedBytes, + ); + const rssDeltaBytes = Math.max(0, rssAfterBytes - rssBeforeBytes); + const violations = []; + if ( + !durable || + durable.installationId !== staged.installationId || + durable.state !== 'failed' || + durable.failure?.reason !== 'activation_fact_conflict' || + durable.failure?.failedFrom !== 'staged' + ) { + violations.push('candidate did not fail closed from staged'); + } + if ( + durable?.previousActiveLockDigest !== first.lock.lockDigest || + durable?.activeLockDigest !== first.lock.lockDigest + ) { + violations.push('previous active lock was not preserved'); + } + if ( + calls.stage !== 0 || + calls.publisherPublish !== 0 || + calls.publisherInspect !== 0 + ) { + violations.push( + 'invalid candidate reached stage or activation publisher', + ); + } + if ( + calls.byteSourceOpen !== 1 || + calls.byteSourceRead !== 4 || + calls.byteSourceClose !== 1 + ) { + violations.push('candidate bytes were not read exactly once'); + } + if (materializedCandidateRevisions !== 0) { + violations.push('invalid candidate materialized revision was published'); + } + if (integrityCheck !== 'ok') { + violations.push(`SQLite integrity_check returned ${integrityCheck}`); + } + if (durationMs > options.maxDurationMs) { + violations.push('duration exceeded the configured bound'); + } + if (rssDeltaBytes > options.maxRssDeltaBytes) { + violations.push('RSS delta exceeded the configured bound'); + } + if ( + Math.max(logicalGrowthBytes, allocatedGrowthBytes) > + options.maxDatabaseGrowthBytes + ) { + violations.push('database growth exceeded the configured bound'); + } + const report = Object.freeze({ + schemaVersion: 1, + evidenceClass: 'plugin_package_failed_upgrade_edge_candidate', + supported: false, + identity: Object.freeze({ + node: process.version, + architecture: process.arch, + platform: process.platform, + }), + measurement: Object.freeze({ + durationMs: round(durationMs), + rssBeforeBytes, + rssAfterBytes, + rssDeltaBytes, + }), + thresholds: Object.freeze({ + maxDurationMs: options.maxDurationMs, + maxRssDeltaBytes: options.maxRssDeltaBytes, + maxDatabaseGrowthBytes: options.maxDatabaseGrowthBytes, + }), + database: Object.freeze({ + journalMode: 'delete', + synchronous: 'full', + integrityCheck, + logicalBytes: finalStorage.logicalBytes, + allocatedBytes: finalStorage.allocatedBytes, + logicalGrowthBytes, + allocatedGrowthBytes, + files: finalStorage.files, + materializedCandidateRevisions, + }), + durable: Object.freeze({ + state: durable?.state ?? null, + failureReason: durable?.failure?.reason ?? null, + failedFrom: durable?.failure?.failedFrom ?? null, + headInstallationIsCandidate: + durable?.installationId === staged.installationId, + previousActiveLockDigestPreserved: + durable?.previousActiveLockDigest === first.lock.lockDigest, + activeLockDigestPreserved: + durable?.activeLockDigest === first.lock.lockDigest, + }), + recovery: Object.freeze(recovery), + calls: Object.freeze(calls), + limitations: EXPECTED_LIMITATIONS, + gates: Object.freeze({ + passed: violations.length === 0, + violations: Object.freeze(violations), + }), + }); + const reportViolations = validateReport(report); + if (reportViolations.length > 0) { + throw new QingLong3PluginPackageRecoveryEdgeBenchmarkError( + `report contract rejected: ${reportViolations.join('; ')}`, + ); + } + if (!report.gates.passed) { + throw new QingLong3PluginPackageRecoveryEdgeBenchmarkError( + report.gates.violations.join('; '), + ); + } + return report; + } finally { + database?.close(); + fs.rmSync(temporaryRoot, { recursive: true, force: true }); + } +} + +function validateReport(report) { + const violations = []; + if (!hasExactKeys(report, REPORT_KEYS)) { + return Object.freeze(['report shape is invalid']); + } + if ( + report.schemaVersion !== 1 || + report.evidenceClass !== 'plugin_package_failed_upgrade_edge_candidate' || + report.supported !== false + ) { + violations.push('report identity is invalid'); + } + if ( + !hasExactKeys(report.identity, IDENTITY_KEYS) || + typeof report.identity.node !== 'string' || + !['arm64', 'x64', 'arm'].includes(report.identity.architecture) || + typeof report.identity.platform !== 'string' + ) { + violations.push('runtime identity is invalid'); + } + if ( + !hasExactKeys(report.measurement, MEASUREMENT_KEYS) || + !Number.isFinite(report.measurement.durationMs) || + report.measurement.durationMs < 0 || + !Number.isSafeInteger(report.measurement.rssBeforeBytes) || + report.measurement.rssBeforeBytes < 1 || + !Number.isSafeInteger(report.measurement.rssAfterBytes) || + report.measurement.rssAfterBytes < 1 || + !Number.isSafeInteger(report.measurement.rssDeltaBytes) || + report.measurement.rssDeltaBytes < 0 + ) { + violations.push('measurement is invalid'); + } + if ( + !hasExactKeys(report.thresholds, THRESHOLD_KEYS) || + !Number.isFinite(report.thresholds.maxDurationMs) || + report.thresholds.maxDurationMs <= 0 || + !Number.isSafeInteger(report.thresholds.maxRssDeltaBytes) || + report.thresholds.maxRssDeltaBytes < 1 || + !Number.isSafeInteger(report.thresholds.maxDatabaseGrowthBytes) || + report.thresholds.maxDatabaseGrowthBytes < 1 + ) { + violations.push('thresholds are invalid'); + } + if ( + !hasExactKeys(report.database, DATABASE_KEYS) || + report.database.journalMode !== 'delete' || + report.database.synchronous !== 'full' || + report.database.integrityCheck !== 'ok' || + report.database.materializedCandidateRevisions !== 0 || + ![ + 'logicalBytes', + 'allocatedBytes', + 'logicalGrowthBytes', + 'allocatedGrowthBytes', + ].every( + (key) => + Number.isSafeInteger(report.database[key]) && report.database[key] >= 0, + ) || + !Array.isArray(report.database.files) || + report.database.files.length < 1 || + report.database.files.length > 4 || + report.database.files.some( + (file) => + !hasExactKeys(file, DATABASE_FILE_KEYS) || + !['database', '-journal', '-wal', '-shm'].includes(file.suffix) || + !Number.isSafeInteger(file.logicalBytes) || + file.logicalBytes < 0 || + !Number.isSafeInteger(file.allocatedBytes) || + file.allocatedBytes < 0, + ) + ) { + violations.push('database evidence is invalid'); + } + if ( + !hasExactKeys(report.durable, DURABLE_KEYS) || + report.durable.state !== 'failed' || + report.durable.failureReason !== 'activation_fact_conflict' || + report.durable.failedFrom !== 'staged' || + report.durable.headInstallationIsCandidate !== true || + report.durable.previousActiveLockDigestPreserved !== true || + report.durable.activeLockDigestPreserved !== true + ) { + violations.push('durable recovery facts are invalid'); + } + if ( + !hasExactKeys(report.recovery, RECOVERY_KEYS) || + report.recovery.pages !== 1 || + report.recovery.scanned !== 1 || + report.recovery.settled !== 1 || + report.recovery.retry !== 0 || + report.recovery.manualRequired !== 0 || + report.recovery.superseded !== 0 || + report.recovery.deferred !== 0 || + report.recovery.remaining !== false || + report.recovery.safeToAdmit !== true + ) { + violations.push('recovery outcome is invalid'); + } + if ( + !hasExactKeys(report.calls, CALL_KEYS) || + report.calls.stage !== 0 || + report.calls.publisherPublish !== 0 || + report.calls.publisherInspect !== 0 || + report.calls.byteSourceOpen !== 1 || + report.calls.byteSourceRead !== 4 || + report.calls.byteSourceClose !== 1 + ) { + violations.push('authority call evidence is invalid'); + } + if ( + JSON.stringify(report.limitations) !== JSON.stringify(EXPECTED_LIMITATIONS) + ) { + violations.push('limitations are invalid'); + } + if ( + !hasExactKeys(report.gates, GATE_KEYS) || + report.gates.passed !== true || + !Array.isArray(report.gates.violations) || + report.gates.violations.length !== 0 + ) { + violations.push('gates did not pass exactly'); + } + if ( + report.measurement?.durationMs > report.thresholds?.maxDurationMs || + report.measurement?.rssDeltaBytes > report.thresholds?.maxRssDeltaBytes || + Math.max( + report.database?.logicalGrowthBytes ?? Number.POSITIVE_INFINITY, + report.database?.allocatedGrowthBytes ?? Number.POSITIVE_INFINITY, + ) > report.thresholds?.maxDatabaseGrowthBytes + ) { + violations.push('measurement exceeded its declared threshold'); + } + return Object.freeze(violations); +} + +async function main() { + const options = parseArguments(process.argv.slice(2)); + const report = await runBenchmark(options); + process.stdout.write( + `${JSON.stringify(report, null, options.json ? 0 : 2)}\n`, + ); +} + +module.exports = { + QingLong3PluginPackageRecoveryEdgeBenchmarkError, + parseArguments, + runBenchmark, + validateReport, +}; + +if (require.main === module) { + main().catch((error) => { + process.stderr.write( + `${error instanceof Error ? error.message : String(error)}\n`, + ); + process.exitCode = 1; + }); +} diff --git a/test/back/ql3LinuxResourceGate.test.cjs b/test/back/ql3LinuxResourceGate.test.cjs index b90fd04d..b1592087 100644 --- a/test/back/ql3LinuxResourceGate.test.cjs +++ b/test/back/ql3LinuxResourceGate.test.cjs @@ -117,6 +117,7 @@ test('builds tier-specific workload plans without shell commands', () => { 'local-workflow-sqlite-lock', 'local-workflow-admission-crash-recovery', 'local-workflow-control-crash-recovery', + 'plugin-package-failed-upgrade', ], ); assert.match(edge[0].script, /ql3-edge-benchmark\.cjs$/); @@ -135,10 +136,7 @@ test('builds tier-specific workload plans without shell commands', () => { ), true, ); - const edgeRelease = createWorkloadPlans( - '/workspace', - 'edge-release-ci', - ); + const edgeRelease = createWorkloadPlans('/workspace', 'edge-release-ci'); assert.deepEqual( edgeRelease.map(({ name }) => name), [ @@ -152,6 +150,7 @@ test('builds tier-specific workload plans without shell commands', () => { 'local-workflow-control-crash-recovery', 'local-ai-prompt-model-invocation-crash-recovery', 'local-ai-prompt-outer-transaction-crash-recovery', + 'plugin-package-failed-upgrade', ], ); assert.equal( @@ -185,17 +184,11 @@ test('builds tier-specific workload plans without shell commands', () => { assert.equal(edgeRelease[3].contract.exactReplay, true); assert.equal(edgeRelease[3].contract.contentFree, true); assert.equal(edgeRelease[3].contract.durableOutputBytes, 512 * 1024); - assert.equal( - edgeRelease[3].contract.maxWalWriteAmplificationPermille, - 0, - ); + assert.equal(edgeRelease[3].contract.maxWalWriteAmplificationPermille, 0); assert.equal(edgeRelease[4].contract.profile, 'standalone'); assert.equal(edgeRelease[4].contract.journalMode, 'wal'); assert.equal(edgeRelease[4].contract.requireWalGrowth, true); - assert.match( - edge[3].script, - /ql3-local-workflow-resource-benchmark\.cjs$/, - ); + assert.match(edge[3].script, /ql3-local-workflow-resource-benchmark\.cjs$/); assert.ok(edge[3].args.includes('--lock-samples=16')); assert.ok(edge[3].args.includes('--max-lock-p95-ms=500')); assert.equal(edge[4].contract.scenarios, 16); @@ -221,6 +214,12 @@ test('builds tier-specific workload plans without shell commands', () => { true, ); assert.equal(edgeRelease[9].contract.physicalPowerLossProven, false); + assert.match( + edge.at(-1).script, + /ql3-plugin-package-recovery-edge-benchmark\.cjs$/, + ); + assert.ok(edge.at(-1).args.includes('--max-rss-delta-mb=64')); + assert.ok(edgeRelease.at(-1).args.includes('--max-rss-delta-mb=96')); const cluster = createWorkloadPlans('/workspace', 'cluster-control-ci'); assert.deepEqual( cluster.map(({ name }) => name), @@ -248,7 +247,9 @@ test('fails closed when durable Prompt resource evidence drifts', () => { const [edgePlan, standalonePlan] = createWorkloadPlans( '/workspace', 'edge-release-ci', - ).filter(({ contract }) => contract?.kind === 'durable_prompt_output_resource'); + ).filter( + ({ contract }) => contract?.kind === 'durable_prompt_output_resource', + ); const evidence = { profile: 'edge', journalMode: 'delete', @@ -268,7 +269,9 @@ test('fails closed when durable Prompt resource evidence drifts', () => { physicalPowerLossProven: false, }; const output = (value) => - `tests 1\npass 1\nfail 0\nskipped 0\nQL3_RESOURCE_EVIDENCE=${JSON.stringify(value)}\n`; + `tests 1\npass 1\nfail 0\nskipped 0\nQL3_RESOURCE_EVIDENCE=${JSON.stringify( + value, + )}\n`; assert.equal( parseNodeTestReport(output(evidence), edgePlan).evidence.profile, 'edge', @@ -301,12 +304,8 @@ test('fails closed when durable Prompt resource evidence drifts', () => { }); test('fails closed when authenticated Workflow cancellation evidence drifts', () => { - const plan = createWorkloadPlans( - '/workspace', - 'edge-release-ci', - ).find( - ({ contract }) => - contract?.kind === 'local_workflow_product_lifecycle', + const plan = createWorkloadPlans('/workspace', 'edge-release-ci').find( + ({ contract }) => contract?.kind === 'local_workflow_product_lifecycle', ); assert.ok(plan); const completionEvidence = { @@ -332,10 +331,12 @@ test('fails closed when authenticated Workflow cancellation evidence drifts', () physicalPowerLossProven: false, }; const output = (value) => - `tests 2\npass 2\nfail 0\nskipped 0\nQL3_RESOURCE_EVIDENCE=${JSON.stringify(completionEvidence)}\nQL3_RESOURCE_EVIDENCE=${JSON.stringify(value)}\n`; + `tests 2\npass 2\nfail 0\nskipped 0\nQL3_RESOURCE_EVIDENCE=${JSON.stringify( + completionEvidence, + )}\nQL3_RESOURCE_EVIDENCE=${JSON.stringify(value)}\n`; assert.equal( - parseNodeTestReport(output(cancellationEvidence), plan) - .evidenceRecords[1].processExited, + parseNodeTestReport(output(cancellationEvidence), plan).evidenceRecords[1] + .processExited, true, ); assert.throws( diff --git a/test/back/ql3PhysicalEdgeEvidence.test.cjs b/test/back/ql3PhysicalEdgeEvidence.test.cjs index cee082e9..66a9274e 100644 --- a/test/back/ql3PhysicalEdgeEvidence.test.cjs +++ b/test/back/ql3PhysicalEdgeEvidence.test.cjs @@ -24,6 +24,9 @@ const { validateObservedPlatform, writeNoReplace, } = require('../../scripts/ql3-physical-edge-evidence.cjs'); +const { + runBenchmark: runPluginPackageRecoveryEdgeBenchmark, +} = require('../../scripts/ql3-plugin-package-recovery-edge-benchmark.cjs'); const { buildReport: buildAdoptionScaleReport, normalizeManifest: normalizeAdoptionScaleManifest, @@ -168,15 +171,49 @@ test('fails closed on architecture, memory, filesystem and virtualization drift' ); }); -test('produces digest-bound candidate evidence but never supported status', () => { +test('produces digest-bound candidate evidence but never supported status', async () => { + const pluginPackageRecovery = await runPluginPackageRecoveryEdgeBenchmark({ + maxDatabaseGrowthBytes: 4 * 1024 * 1024, + maxDurationMs: 10_000, + maxRssDeltaBytes: 96 * 1024 * 1024, + }); + const workloads = [ + { + name: 'edge-executor', + report: { + schemaVersion: 1, + profile: 'edge', + gates: { passed: true, violations: [] }, + }, + }, + { + name: 'node-sqlite-on-device-storage', + report: { + journalMode: 'delete', + synchronous: 'full', + integrityCheck: 'ok', + databaseBytes: 16_384, + rssDeltaMb: 1, + }, + }, + { + name: 'plugin-package-failed-upgrade', + report: pluginPackageRecovery, + }, + ]; const report = buildEvidenceReport({ manifest: manifest(), observed: observed(), - workloads: [{ name: 'edge-executor', report: { gates: { passed: true } } }], + workloads, generatedAt: '2026-07-22T00:00:00.000Z', }); assert.equal(report.supported, false); assert.equal(report.qualification.physicalCandidate, true); + assert.ok( + report.qualification.collectedEvidence.includes( + 'plugin_package_failed_upgrade_retains_active_generation', + ), + ); assert.equal(report.sha256.length, 64); const changed = buildEvidenceReport({ manifest: manifest({ deviceId: 'router-a2' }), @@ -184,6 +221,12 @@ test('produces digest-bound candidate evidence but never supported status', () = workloads: [], generatedAt: '2026-07-22T00:00:00.000Z', }); + assert.equal(changed.qualification.physicalCandidate, false); + assert.ok( + changed.qualification.remainingRequiredEvidence.includes( + 'plugin_package_failed_upgrade_retains_active_generation', + ), + ); assert.notEqual(changed.sha256, report.sha256); }); diff --git a/test/back/ql3PluginPackageRecoveryEdgeBenchmark.test.cjs b/test/back/ql3PluginPackageRecoveryEdgeBenchmark.test.cjs new file mode 100644 index 00000000..2505006e --- /dev/null +++ b/test/back/ql3PluginPackageRecoveryEdgeBenchmark.test.cjs @@ -0,0 +1,49 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + parseArguments, + runBenchmark, + validateReport, +} = require('../../scripts/ql3-plugin-package-recovery-edge-benchmark.cjs'); + +test('parses bounded failed-upgrade benchmark thresholds', () => { + assert.deepEqual( + parseArguments([ + '--json', + '--max-duration-ms=5000', + '--max-rss-delta-mb=64', + '--max-database-growth-bytes=2097152', + ]), + { + json: true, + maxDatabaseGrowthBytes: 2 * 1024 * 1024, + maxDurationMs: 5_000, + maxRssDeltaBytes: 64 * 1024 * 1024, + }, + ); + assert.throws( + () => parseArguments(['--max-rss-delta-mb=0']), + /positive number/, + ); + assert.throws(() => parseArguments(['--unknown=1']), /unsupported argument/); +}); + +test('uses real SQLite recovery to reject an invalid upgrade before publication', async () => { + const report = await runBenchmark({ + maxDatabaseGrowthBytes: 4 * 1024 * 1024, + maxDurationMs: 10_000, + maxRssDeltaBytes: 96 * 1024 * 1024, + }); + + assert.deepEqual(validateReport(report), []); + assert.equal(report.durable.state, 'failed'); + assert.equal(report.durable.activeLockDigestPreserved, true); + assert.equal(report.database.materializedCandidateRevisions, 0); + assert.equal(report.calls.publisherPublish, 0); + assert.equal(report.gates.passed, true); + + assert.deepEqual(validateReport({ ...report, supported: true }), [ + 'report identity is invalid', + ]); +});