From a64d08bda131f997a8478131be1336083e922238 Mon Sep 17 00:00:00 2001 From: whyour Date: Mon, 24 Aug 2026 22:33:15 +0800 Subject: [PATCH] test(ql3): prove legacy env replay across promotion --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 31 +- ...-config-reconciliation-and-task-binding.md | 6 +- ...luster-legacy-env-application-ha-replay.md | 72 ++++ scripts/ql3-postgres-ha-contract.cjs | 78 +++++ scripts/ql3-postgres-ha-evidence-audit.cjs | 118 +++++++ ...gres-ha-legacy-env-application-fixture.cjs | 331 ++++++++++++++++++ test/back/ql3PostgresHaEvidenceAudit.test.cjs | 66 ++++ 7 files changed, 696 insertions(+), 6 deletions(-) create mode 100644 docs/adr/ADR-0498-cluster-legacy-env-application-ha-replay.md create mode 100644 scripts/ql3-postgres-ha-legacy-env-application-fixture.cjs diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 5a240f85..3daa251b 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -88,7 +88,7 @@ D-385～D-388 的 `config.sh`/Keyv/SSH data-directory lineage 与 SQLite `Envs` 保持分离；当前无稳定生产 schema 的历史 `Configs` 表继续 sealed+manual， 不猜字段。ADR-0494 已关闭基础 Cluster mounted-files provider live 子门；后续切片仍必须完成固定低性能设备的真实 Edge 空间/写放大/断电证据， - 以及 Cluster Legacy Env migration 的专用 PostgreSQL SERIALIZABLE ledger、Task/Trigger revision mutation、直接外部 custody adapter 与 HA promotion 后 receipt replay。D-397 apply + 以及 Cluster Legacy Env migration 的专用 PostgreSQL SERIALIZABLE ledger、Task/Trigger revision mutation 与直接外部 custody adapter。HA promotion 后 receipt replay 已由 D-403/ADR-0498 关闭。D-397 apply 只声明 sealed source retained 且 `physicalErasureGuaranteed=false`；明文销毁必须在 restart/ readiness、观察窗和 rollback retention 之后另行强认证。Cluster 必须使用 PostgreSQL SERIALIZABLE ledger、外部 KMS/Secret provider 与 HA evidence， 不复用 Local SQLite/POSIX authority，也不得把明文写入 PostgreSQL、ConfigMap、Pod env 或 Job command。 @@ -174,8 +174,33 @@ `a7c8a05e08c748d677475a09ce2998b741ed9326e27678613e93d431bc3769aa`； 真实用例覆盖 Trigger 固定 Task r1、Task current r2、原子生成 Task r3/Trigger r2 并重定向 pin， 同时证明 bundle ref-only execution、schedule reset、无流消费 replay 与数据库角色隔离。D-402 - 关闭 mutation/receipt 边界；direct external custody、promotion 后 receipt replay 和固定低性能 Edge - 物理证据仍是 ADR-0491 转 Accepted 前的门禁。 + 关闭 mutation/receipt 边界；相邻 D-403 已继续关闭 promotion 后 receipt replay，direct external + custody 与固定低性能 Edge 物理证据仍是 ADR-0491 转 Accepted 前的门禁。 + +- D-403/ADR-0498（已验收）：ADR-0497 application receipt 已进入真实 PostgreSQL 18 physical HA + 领域门，而不是继续引用通用表存活作为间接证据。主库与 standby 达到 + `synchronous_commit=remote_apply`/`sync_state=sync` 后，Automation Manager 提交真实 plan 与 + application；standby 在 recovery 中读取的 receipt、逐项 ledger、Task r3/execution、Trigger r2 与 + schedule fence 共 17 项 content-free facts 与主库完全一致。随后门禁制造 replication partition、 + fence 旧主库、把 standby 晋升到 timeline 2，并以 `pg_rewind --write-recovery-conf` 将旧主库重接为 + 同步 standby。 + + 写入能力恢复后，新的 Automation Manager Pool 重放同一 intent，必须返回 `existing`；Task/Trigger + mutation stream factory 打开次数均为 0，application/Task/Trigger ledger 新增行均为 0，晋升后 17 项 + facts 仍与主库/standby 逐字段相同。独立 fixture 保持 disabled cron Trigger，避免污染 HA scheduler + 场景的候选集，但仍真实验证 schedule revision、state/claim fence 与 reset。HA private report 只包含 + count/revision/version/digest；evidence audit 强制 exact key set、三份 facts 同一性、两个 timeline state、 + replay side-effect 为零，并把 fixture SecretRef marker 加入 forbidden material。 + + PostgreSQL 18.6 arm64 HA 为 147/147 gates、timeline `1→2`，独立 audit 为 + `compatible=true/findings=[]`，最终报告 SHA-256 为 + `8bb61bc126ba96e7d4e20b1bfad4db960768c03473744d9d24e11b1b5b1a9286`。本切片不改 production + source、schema/migration/ACL、package、依赖、镜像或 Edge import；完整 backend 为 + `1569 total / 1567 pass / 2 conditional skip / 0 fail`，四项 package/依赖/import 边界均 compatible。 + 基础 Edge/Standalone 仍只包含 Local SQLite、runtime-core 与 SemVer，均为 325 files/58 modules， + 大小 `2,669,390 / 2,669,468 bytes`，距 4 MiB 上限保留 `1,524,914 / 1,524,836 bytes`；它只证明 + 常驻闭包未扩大，不冒充真实设备证据。D-403 关闭 promotion replay；ADR-0491 现在只剩 direct + external custody 与固定低性能 Edge 空间/写放大/断电恢复两项硬门。 - D-396/ADR-0490（已验收）：Run History 不再只有永久 `manual_external`，但也没有被错误实现为 Legacy 日志到 3.0 Run ledger 的回灌。 新的 Local adapter 以 ADR-0482 sealed capture bundle 作为 append-only 保全资产：Legacy history 必须逐事实选择 `retain_both`，Target history diff --git a/docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md b/docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md index 70713060..94653149 100644 --- a/docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md +++ b/docs/adr/ADR-0491-bounded-secret-config-reconciliation-and-task-binding.md @@ -1,9 +1,9 @@ # ADR-0491：有界 Secret/Config Reconciliation 与任务环境绑定 -- 状态：Proposed（D-397 已实现 Legacy Env inspection、私有有界 row plan、durable plan publication、独立 signed decision、逐项 Automation adoption provenance、Local SQLite 原子 application publisher、Owner prepared/apply/rollback 编排、ADR-0492 completion v3；ADR-0494 完成 Cluster mounted-files provider live 子门，ADR-0495 完成 content-free Cluster plan ledger，ADR-0496 完成 opaque environment bundle 数据面，ADR-0497 完成 Cluster Task/Trigger 原子 mutation 与 receipt；真实 Edge 空间证据、promotion 后 receipt replay 与直接外部 custody gate 尚未完成） +- 状态：Proposed（D-397 已实现 Legacy Env inspection、私有有界 row plan、durable plan publication、独立 signed decision、逐项 Automation adoption provenance、Local SQLite 原子 application publisher、Owner prepared/apply/rollback 编排、ADR-0492 completion v3；ADR-0494 完成 Cluster mounted-files provider live 子门，ADR-0495 完成 content-free Cluster plan ledger，ADR-0496 完成 opaque environment bundle 数据面，ADR-0497 完成 Cluster Task/Trigger 原子 mutation 与 receipt，ADR-0498 完成 promotion 后 exact replay；真实 Edge 空间证据与直接外部 custody gate 尚未完成） - 日期：2026-08-23 - 决策：D-397 -- 关联：ADR-0073、ADR-0074、ADR-0092、ADR-0094、ADR-0480、ADR-0482、ADR-0483、ADR-0484、ADR-0485、ADR-0486、ADR-0487、ADR-0488、ADR-0490 +- 关联：ADR-0073、ADR-0074、ADR-0092、ADR-0094、ADR-0480、ADR-0482、ADR-0483、ADR-0484、ADR-0485、ADR-0486、ADR-0487、ADR-0488、ADR-0490、ADR-0495、ADR-0496、ADR-0497、ADR-0498 ## 背景 @@ -147,4 +147,4 @@ D-397 当前八切片已经实现：absent、unsupported、Edge over-budget、2. ADR-0494 已完成 Cluster `mounted-files` provider live 子门：真实三节点 K3s 中两个 management replica、direct exact-key executor 和两个跨节点 provider observer 完成 PostgreSQL durable approval/binding、Kubernetes atomic projection rotation、无 Secret API 权限/ServiceAccount token、只读 `0440`、内容脱敏及删除后 fail-closed；v2 私有报告 24/24 gates 为 true，并保持 v1 verifier 兼容。该门不增加 Edge 闭包，也不等于直接 Vault/KMS/HSM custody。 -转为 Accepted 前仍必须完成：固定低性能 Edge 设备的真实空间/写放大/断电恢复证据、直接外部 custody adapter，以及 HA promotion 后对既有 application receipt 的 exact replay。ADR-0495 已完成专用 PostgreSQL plan ledger，ADR-0496 已完成只保存 pinned bundle ref、通过 fenced remote delivery 取回 typed carrier 并在 Worker 内存展开的数据面；ADR-0497 又在一个 Project-serialized SERIALIZABLE transaction 中完成逐项 Task/Trigger current-head revalidation、revision/execution mutation、schedule reset 和 content-free append-only receipt，并支持合法历史 Task pin。它仍不写入 Secret material、不等于 direct Vault/KMS/HSM custody，也尚未在 promotion 后重放同一 application receipt。ADR-0492 已完成本机 completion schema 演进和 completed-head 后 rollback material 回收，ADR-0493 又让没有 Legacy 身份输入的 fresh v52 目标身份经 signed `retain_target` 正确形成 no-effect，并精确消除六张已知目标表的 `unknown` 误判。Legacy `Auths/Users` 或真正未知表仍保持 manual；Local Owner 编排、mounted-files gate、plan/application ledger 或通用 PostgreSQL HA 证据都不得冒充完整外部密钥托管。 +转为 Accepted 前仍必须完成：固定低性能 Edge 设备的真实空间/写放大/断电恢复证据，以及直接外部 custody adapter。ADR-0495 已完成专用 PostgreSQL plan ledger，ADR-0496 已完成只保存 pinned bundle ref、通过 fenced remote delivery 取回 typed carrier 并在 Worker 内存展开的数据面；ADR-0497 又在一个 Project-serialized SERIALIZABLE transaction 中完成逐项 Task/Trigger current-head revalidation、revision/execution mutation、schedule reset 和 content-free append-only receipt，并支持合法历史 Task pin。ADR-0498 已证明该首次提交在 `remote_apply` 下到达 standby，并在 timeline `1→2`、`pg_rewind` 重建同步副本后由 Automation Manager 精确重放为 `existing`，不打开 mutation stream 或增加 durable row。它仍不写入 Secret material，也不等于 direct Vault/KMS/HSM custody。ADR-0492 已完成本机 completion schema 演进和 completed-head 后 rollback material 回收，ADR-0493 又让没有 Legacy 身份输入的 fresh v52 目标身份经 signed `retain_target` 正确形成 no-effect，并精确消除六张已知目标表的 `unknown` 误判。Legacy `Auths/Users` 或真正未知表仍保持 manual；Local Owner 编排、mounted-files gate、plan/application ledger 或 HA replay 证据都不得冒充完整外部密钥托管。 diff --git a/docs/adr/ADR-0498-cluster-legacy-env-application-ha-replay.md b/docs/adr/ADR-0498-cluster-legacy-env-application-ha-replay.md new file mode 100644 index 00000000..d3621e69 --- /dev/null +++ b/docs/adr/ADR-0498-cluster-legacy-env-application-ha-replay.md @@ -0,0 +1,72 @@ +# ADR-0498：Cluster Legacy Env application 的 HA 晋升后精确重放 + +- 状态：Accepted +- 日期：2026-08-24 +- 决策：D-403 +- 关联：ADR-0092、ADR-0094、ADR-0491、ADR-0495、ADR-0496、ADR-0497 + +## 背景 + +ADR-0497 已让 Automation Manager 在一个 Project-serialized `SERIALIZABLE` transaction 中提交 +Legacy Env plan、Task/Trigger 新 revision、execution revision、schedule reset 与只追加 application +receipt。此前 PostgreSQL HA 门只能证明通用 migration、role/readiness 及其他领域状态跨 timeline +晋升存活，尚未证明这一新 application 的首次提交已同步复制，也未证明晋升后的同一 intent 会读取既有 +receipt，而不是重新打开大规模 mutation stream 或追加第二组 durable state。 + +通用 HA 成功不能替代领域重放。若 promotion 后只检查表存在，Task/Trigger head、execution digest、 +schedule fence 或逐项 receipt 仍可能漂移；若重放重新消费最多 100,000/500,000 项的输入流，则恢复成本 +不再有界，也可能把“检查既有提交”退化成第二次迁移。 + +## 决策 + +扩展既有 PostgreSQL 18 physical HA contract，不新增 workspace package、生产依赖、daemon、controller、 +timer、watcher 或部署资源。领域逻辑放在独立 +`scripts/ql3-postgres-ha-legacy-env-application-fixture.cjs`，14,000 行以上的 HA orchestration 只保留 +fixture import、主库调用、晋升后调用、report 与 gate 装配。 + +门禁固定执行以下顺序： + +1. 主库与 standby 已进入 `synchronous_commit=remote_apply`、`sync_state=sync` 后，以 migration authority + 建立 active Project、Task r1、固定 Task r1 的 disabled cron Trigger r1，再把 Task 推进到 r2； +2. 以 `ql3_automation_manager` 发布 content-free plan，并通过 ADR-0497 repository 原子生成 Task r3、 + Trigger r2、execution revision、schedule reset 与 application receipt； +3. 在 standby 仍处于 recovery 时读取 17 项 content-free facts，要求与主库完全相同，然后才允许制造 + replication partition、fence 旧主库和 promotion； +4. promotion 后先以 `pg_rewind --write-recovery-conf` 把旧主库作为同步 standby 重接，恢复 + `remote_apply` 写入能力，再使用新的 Automation Manager Pool 重放完全相同的 application intent； +5. replay 必须返回 `existing`，Task/Trigger mutation stream factory 均不得被调用，receipt、逐项 ledger、 + Task/Trigger revision、execution digest 与 schedule fence 必须逐字段不变，新增 durable row 必须为 0。 + +Trigger fixture 保持 disabled，避免给 HA contract 中独立 scheduler failover 场景增加第二条合法可领取的 +schedule；它仍真实创建并迁移 cron schedule，因此会验证 revision、state/claim fence 与 reset。 + +## 证据边界 + +私有 HA report 只包含计数、revision、state/claim version 与 SHA-256 digest，不包含 Project/Task/Trigger +ID、SecretRef、Env name/value、row body、ciphertext、key ID、数据库 DSN 或 credential。evidence audit +现在强制要求三份事实对象只有固定 17 个 key、逐字段相同，并要求： + +- `replicatedBeforePromotion=true`； +- `exactReplayAfterPromotion=true` 且 `replayStatus=existing`； +- `mutationStreamsOpenedAfterPromotion=0`； +- `durableRowsAddedByReplay=0`； +- 两个新增 timeline state 位于同步复制之后、partition/promotion 的正确顺序； +- 缺证据、字段扩张、digest 非法、facts 漂移、SecretRef marker 泄漏或 gate 为 false 都失败关闭。 + +本 ADR 不声明 direct Vault/KMS/HSM custody，也不证明固定低性能 Edge 设备的空间、写放大或断电恢复； +这两项仍是 ADR-0491 转 Accepted 前的独立门禁。mounted-files live rotation 也不能替代直接外部 custody。 + +## 验证 + +PostgreSQL 18.6 arm64 physical HA 通过 `147/147` gates，timeline `1→2`。主库提交、standby WAL +投影与晋升后 replay 的 17 项 facts 完全一致；replay 返回 `existing`，mutation stream 打开次数与新增 +durable row 都为 0。独立 evidence audit 返回 `compatible=true`、`findings=[]`。最终私有报告 +SHA-256 为 `8bb61bc126ba96e7d4e20b1bfad4db960768c03473744d9d24e11b1b5b1a9286`。 + +本切片没有修改 production source、PostgreSQL schema/migration/ACL、package topology、依赖树、镜像或 +Edge/Standalone import graph；完整 backend 为 `1569 total / 1567 pass / 2 conditional skip / 0 fail`， +package/Cluster dependency/122-module Edge/service bridge 边界均 compatible。基础 Edge/Standalone +仍只包含 Local SQLite、runtime-core 与 SemVer，均为 325 files/58 modules，大小为 +`2,669,390 / 2,669,468 bytes`，距 4 MiB 上限保留 `1,524,914 / 1,524,836 bytes`。这些结果只证明 +常驻闭包未被 D-403 扩大，不冒充固定低性能设备的真实断电/写放大证据。D-403 把 ADR-0497 已有领域 +语义提升为 promotion 后可验证的发布证据。 diff --git a/scripts/ql3-postgres-ha-contract.cjs b/scripts/ql3-postgres-ha-contract.cjs index 0907b855..7f2f9927 100644 --- a/scripts/ql3-postgres-ha-contract.cjs +++ b/scripts/ql3-postgres-ha-contract.cjs @@ -379,6 +379,11 @@ const { persistCancellationDispatchHaFixture, verifyPromotedCancellationDispatchHaFixture, } = require('./ql3-postgres-ha-cancellation-dispatch-fixture.cjs'); +const { + clusterLegacyEnvMigrationApplicationFacts, + persistClusterLegacyEnvMigrationApplicationBeforePromotion, + verifyPromotedClusterLegacyEnvMigrationApplication, +} = require('./ql3-postgres-ha-legacy-env-application-fixture.cjs'); const { activateInstall, pluginPackageTaskReconciliationFixture, @@ -11405,6 +11410,7 @@ async function main(argv = process.argv.slice(2)) { let runAttemptLogRetention; let manualRunRetry; let cancellationDispatch; + let clusterLegacyEnvMigrationApplication; const startedAt = performance.now(); const timeline = []; let report; @@ -11920,6 +11926,43 @@ async function main(argv = process.argv.slice(2)) { state: 'synchronous_remote_apply_ready', atMs: Number((performance.now() - startedAt).toFixed(3)), }); + const legacyEnvAutomationDatabase = await databaseOpener( + 'automation-manager', + databaseUrl( + AUTOMATION_MANAGER_USER, + AUTOMATION_MANAGER_PASSWORD, + primaryPort, + ), + 'ql3-ha-legacy-env-application-primary', + )(); + try { + clusterLegacyEnvMigrationApplication = + await persistClusterLegacyEnvMigrationApplicationBeforePromotion({ + migrationPool: primaryDatabase.pool, + automationPool: legacyEnvAutomationDatabase.pool, + }); + } finally { + await legacyEnvAutomationDatabase.close(); + } + const replicatedLegacyEnvApplication = await waitFor(async () => { + const facts = await clusterLegacyEnvMigrationApplicationFacts( + standbyDatabase.pool, + clusterLegacyEnvMigrationApplication.fixture, + ); + return JSON.stringify(facts) === + JSON.stringify( + clusterLegacyEnvMigrationApplication.report.primaryBeforePromotion, + ) + ? facts + : null; + }, 'Cluster Legacy Env migration application WAL replay'); + clusterLegacyEnvMigrationApplication.report.standbyBeforePromotion = + replicatedLegacyEnvApplication; + clusterLegacyEnvMigrationApplication.report.replicatedBeforePromotion = true; + timeline.push({ + state: 'cluster_legacy_env_application_replicated', + atMs: Number((performance.now() - startedAt).toFixed(3)), + }); await waitFor(async () => { const replicated = await copilotFailureDiagnosisAdmissionFacts( standbyDatabase.pool, @@ -13465,6 +13508,11 @@ async function main(argv = process.argv.slice(2)) { database: recoveredAutomationManagerDatabase, report: automationManagementInspection, }); + await verifyPromotedClusterLegacyEnvMigrationApplication({ + automationPool: recoveredAutomationManagerDatabase.pool, + promotedPool: promotedDatabase.pool, + evidence: clusterLegacyEnvMigrationApplication, + }); } finally { await recoveredAutomationManagerDatabase.close(); } @@ -13472,6 +13520,10 @@ async function main(argv = process.argv.slice(2)) { state: 'automation_management_inspection_survived_promotion', atMs: Number((performance.now() - startedAt).toFixed(3)), }); + timeline.push({ + state: 'cluster_legacy_env_application_replayed_after_promotion', + atMs: Number((performance.now() - startedAt).toFixed(3)), + }); await verifyPluginPackageQuarantineAfterPromotion({ promotedPort: standbyPort, promotedDatabase, @@ -13954,6 +14006,8 @@ async function main(argv = process.argv.slice(2)) { approvalIdentityKeysetLedger, runManagementIdentityKeysetLedger, automationManagementInspection, + clusterLegacyEnvMigrationApplication: + clusterLegacyEnvMigrationApplication.report, pluginPackageLifecycle: pluginPackageLifecycle.report, pluginPackageQuarantine: pluginPackageQuarantine.report, pluginPackagePromptExecution: pluginPackageQuarantine.promptExecution, @@ -14312,6 +14366,30 @@ async function main(argv = process.argv.slice(2)) { .successfulReadAuditCount === 4 && automationManagementInspection.afterPromotion.taskRevision === 1 && automationManagementInspection.afterPromotion.triggerRevision === 1, + clusterLegacyEnvMigrationApplicationExactlyReplaysAfterPromotion: + clusterLegacyEnvMigrationApplication.report + .replicatedBeforePromotion === true && + clusterLegacyEnvMigrationApplication.report + .exactReplayAfterPromotion === true && + clusterLegacyEnvMigrationApplication.report.replayStatus === + 'existing' && + clusterLegacyEnvMigrationApplication.report + .mutationStreamsOpenedAfterPromotion === 0 && + clusterLegacyEnvMigrationApplication.report + .durableRowsAddedByReplay === 0 && + JSON.stringify( + clusterLegacyEnvMigrationApplication.report.primaryBeforePromotion, + ) === + JSON.stringify( + clusterLegacyEnvMigrationApplication.report + .standbyBeforePromotion, + ) && + JSON.stringify( + clusterLegacyEnvMigrationApplication.report.primaryBeforePromotion, + ) === + JSON.stringify( + clusterLegacyEnvMigrationApplication.report.promotedAfterReplay, + ), pluginPackageLifecycleCommitResponseLossConvergesExactlyOnce: true, pluginPackageLifecycleRunAndToolFencesTransitionAtomically: true, pluginPackageAutomationPublicationTransitionsAtomically: diff --git a/scripts/ql3-postgres-ha-evidence-audit.cjs b/scripts/ql3-postgres-ha-evidence-audit.cjs index 29d85f1c..af28a18a 100644 --- a/scripts/ql3-postgres-ha-evidence-audit.cjs +++ b/scripts/ql3-postgres-ha-evidence-audit.cjs @@ -15,11 +15,47 @@ const REQUIRED_TIMELINE_STATES = Object.freeze([ 'primary_ready', 'standby_streaming', 'synchronous_remote_apply_ready', + 'cluster_legacy_env_application_replicated', 'replication_partition_and_promotion_guard_verified', 'old_primary_fenced_and_admission_withdrawn', + 'cluster_legacy_env_application_replayed_after_promotion', 'standby_promoted_old_primary_rejoined_endpoint_switched', 'two_fresh_control_replicas_ready', ]); +const LEGACY_ENV_APPLICATION_FACT_KEYS = Object.freeze( + [ + 'executionContentDigest', + 'planRows', + 'receiptDigest', + 'receiptRows', + 'scheduleClaimVersion', + 'scheduleRevision', + 'scheduleStateVersion', + 'taskContentDigest', + 'taskCount', + 'taskItemRows', + 'taskRevision', + 'triggerContentDigest', + 'triggerCount', + 'triggerItemRows', + 'triggerRevision', + 'triggerTaskContentDigest', + 'triggerTaskRevision', + ].sort(), +); +const LEGACY_ENV_APPLICATION_KEYS = Object.freeze( + [ + 'contentFree', + 'durableRowsAddedByReplay', + 'exactReplayAfterPromotion', + 'mutationStreamsOpenedAfterPromotion', + 'primaryBeforePromotion', + 'promotedAfterReplay', + 'replayStatus', + 'replicatedBeforePromotion', + 'standbyBeforePromotion', + ].sort(), +); const FORBIDDEN_MATERIAL = Object.freeze([ 'postgresql://', 'ql3_migration_test', @@ -36,6 +72,8 @@ const FORBIDDEN_MATERIAL = Object.freeze([ 'ql3_worker_credential_executor_test', 'ql3_worker_ingress_test', 'ql3w_', + 'qlsecret:v1:', + 'ha-legacy-env-bundle-private', ]); function isObject(value) { @@ -250,6 +288,86 @@ function auditPostgresHaEvidence(report) { 'UNCOMMITTED_WRITE_SURVIVED', ); + const legacyEnvApplication = report.clusterLegacyEnvMigrationApplication; + add( + isObject(legacyEnvApplication), + 'CLUSTER_LEGACY_ENV_APPLICATION_EVIDENCE_MISSING', + ); + if (isObject(legacyEnvApplication)) { + add( + JSON.stringify(Object.keys(legacyEnvApplication).sort()) === + JSON.stringify(LEGACY_ENV_APPLICATION_KEYS), + 'CLUSTER_LEGACY_ENV_APPLICATION_EVIDENCE_WIDENED', + ); + add( + legacyEnvApplication.replicatedBeforePromotion === true, + 'CLUSTER_LEGACY_ENV_APPLICATION_NOT_REPLICATED', + ); + add( + legacyEnvApplication.exactReplayAfterPromotion === true && + legacyEnvApplication.replayStatus === 'existing', + 'CLUSTER_LEGACY_ENV_APPLICATION_REPLAY_INVALID', + ); + add( + legacyEnvApplication.mutationStreamsOpenedAfterPromotion === 0 && + legacyEnvApplication.durableRowsAddedByReplay === 0, + 'CLUSTER_LEGACY_ENV_APPLICATION_REPLAY_SIDE_EFFECT', + ); + add( + legacyEnvApplication.contentFree === true, + 'CLUSTER_LEGACY_ENV_APPLICATION_EVIDENCE_NOT_CONTENT_FREE', + ); + const factSets = [ + legacyEnvApplication.primaryBeforePromotion, + legacyEnvApplication.standbyBeforePromotion, + legacyEnvApplication.promotedAfterReplay, + ]; + for (const facts of factSets) { + add(isObject(facts), 'CLUSTER_LEGACY_ENV_APPLICATION_FACTS_INVALID'); + if (!isObject(facts)) continue; + add( + JSON.stringify(Object.keys(facts).sort()) === + JSON.stringify(LEGACY_ENV_APPLICATION_FACT_KEYS), + 'CLUSTER_LEGACY_ENV_APPLICATION_FACTS_WIDENED', + ); + add( + facts.planRows === 1 && + facts.receiptRows === 1 && + facts.taskItemRows === 1 && + facts.triggerItemRows === 1 && + facts.taskCount === 1 && + facts.triggerCount === 1 && + facts.taskRevision === 3 && + facts.triggerRevision === 2 && + facts.triggerTaskRevision === 3 && + facts.scheduleRevision === 2 && + facts.scheduleStateVersion === 1 && + facts.scheduleClaimVersion === 1, + 'CLUSTER_LEGACY_ENV_APPLICATION_FACTS_DRIFTED', + ); + for (const key of [ + 'receiptDigest', + 'taskContentDigest', + 'executionContentDigest', + 'triggerContentDigest', + 'triggerTaskContentDigest', + ]) { + add( + typeof facts[key] === 'string' && SHA256_PATTERN.test(facts[key]), + 'CLUSTER_LEGACY_ENV_APPLICATION_DIGEST_INVALID', + ); + } + } + add( + factSets.every( + (facts) => + isObject(facts) && + JSON.stringify(facts) === JSON.stringify(factSets[0]), + ), + 'CLUSTER_LEGACY_ENV_APPLICATION_FACTS_NOT_EXACT', + ); + } + const gates = report.gates; add(isObject(gates), 'GATES_MISSING'); if (isObject(gates)) { diff --git a/scripts/ql3-postgres-ha-legacy-env-application-fixture.cjs b/scripts/ql3-postgres-ha-legacy-env-application-fixture.cjs new file mode 100644 index 00000000..8e20fe32 --- /dev/null +++ b/scripts/ql3-postgres-ha-legacy-env-application-fixture.cjs @@ -0,0 +1,331 @@ +const assert = require('node:assert/strict'); + +const { + PostgresTaskDefinitionRepository, + PostgresTriggerRepository, +} = require('../packages/ql3-cluster-postgres/dist/entrypoints/admin.js'); +const { + PostgresClusterLegacyEnvMigrationPlanRepository, +} = require('../packages/ql3-cluster-postgres/dist/reconciliation/clusterLegacyEnvMigrationPlanRepository.js'); +const { + PostgresClusterLegacyEnvMigrationApplicationRepository, +} = require('../packages/ql3-cluster-postgres/dist/reconciliation/clusterLegacyEnvMigrationApplicationRepository.js'); +const { + createClusterLegacyEnvMigrationTaskMutationSetDigester, + createClusterLegacyEnvMigrationTriggerMutationSetDigester, +} = require('../packages/ql3-runtime-core/dist/migration/clusterLegacyEnvMigrationApplication.js'); +const { + createSecretRef, +} = require('../packages/ql3-runtime-core/dist/secret/secretReference.js'); + +const FIXTURE = Object.freeze({ + projectId: 'ha-legacy-env-application', + taskId: 'ha legacy env task', + triggerId: 'ha legacy env trigger', + planId: 'ha-legacy-env-plan', + planMutationId: '819f7900-0000-4000-8000-000000000001', + applicationId: 'ha-legacy-env-application-receipt', + applicationMutationId: '819f7900-0000-4000-8000-000000000002', + taskCreateMutationId: '819f7900-0000-4000-8000-000000000003', + triggerCreateMutationId: '819f7900-0000-4000-8000-000000000004', + taskAdvanceMutationId: '819f7900-0000-4000-8000-000000000005', + taskApplicationMutationId: '819f7900-0000-4000-8000-000000000006', + triggerApplicationMutationId: '819f7900-0000-4000-8000-000000000007', + privateBundleName: 'ha-legacy-env-bundle-private', +}); + +async function clusterLegacyEnvMigrationApplicationFacts(pool, fixture) { + const result = await pool.query( + `SELECT + receipt.receipt_digest AS "receiptDigest", + receipt.task_count AS "taskCount", + receipt.trigger_count AS "triggerCount", + task_item.revision AS "taskRevision", + task_item.content_digest AS "taskContentDigest", + task_item.execution_content_digest AS "executionContentDigest", + trigger_item.revision AS "triggerRevision", + trigger_item.content_digest AS "triggerContentDigest", + trigger_item.task_revision AS "triggerTaskRevision", + trigger_item.task_content_digest AS "triggerTaskContentDigest", + schedule.trigger_revision AS "scheduleRevision", + schedule.state_version AS "scheduleStateVersion", + schedule.claim_version AS "scheduleClaimVersion", + (SELECT count(*)::integer + FROM "ql3"."cluster_legacy_env_migration_plans" AS plan + WHERE plan.plan_id = receipt.plan_id) AS "planRows", + (SELECT count(*)::integer + FROM "ql3"."cluster_legacy_env_migration_application_receipts" AS stored + WHERE stored.application_id = receipt.application_id) AS "receiptRows", + (SELECT count(*)::integer + FROM "ql3"."cluster_legacy_env_migration_application_tasks" AS item + WHERE item.application_id = receipt.application_id) AS "taskItemRows", + (SELECT count(*)::integer + FROM "ql3"."cluster_legacy_env_migration_application_triggers" AS item + WHERE item.application_id = receipt.application_id) AS "triggerItemRows" + FROM "ql3"."cluster_legacy_env_migration_application_receipts" AS receipt + JOIN "ql3"."cluster_legacy_env_migration_application_tasks" AS task_item + ON task_item.application_id = receipt.application_id + AND task_item.ordinal = 0 + JOIN "ql3"."cluster_legacy_env_migration_application_triggers" AS trigger_item + ON trigger_item.application_id = receipt.application_id + AND trigger_item.ordinal = 0 + JOIN "ql3"."trigger_schedules" AS schedule + ON schedule.project_id = trigger_item.project_id + AND schedule.trigger_id = trigger_item.trigger_id + WHERE receipt.application_id = $1`, + [fixture.applicationId], + ); + assert.equal(result.rowCount, 1); + return Object.freeze({ ...result.rows[0] }); +} + +function assertExpectedFacts(facts) { + assert.equal(facts.planRows, 1); + assert.equal(facts.receiptRows, 1); + assert.equal(facts.taskItemRows, 1); + assert.equal(facts.triggerItemRows, 1); + assert.equal(facts.taskCount, 1); + assert.equal(facts.triggerCount, 1); + assert.equal(facts.taskRevision, 3); + assert.equal(facts.triggerRevision, 2); + assert.equal(facts.triggerTaskRevision, 3); + assert.equal(facts.scheduleRevision, 2); + assert.equal(facts.scheduleStateVersion, 1); + assert.equal(facts.scheduleClaimVersion, 1); + for (const digest of [ + facts.receiptDigest, + facts.taskContentDigest, + facts.executionContentDigest, + facts.triggerContentDigest, + facts.triggerTaskContentDigest, + ]) { + assert.match(digest, /^[0-9a-f]{64}$/); + } +} + +async function persistClusterLegacyEnvMigrationApplicationBeforePromotion( + options, +) { + const { migrationPool, automationPool } = options; + const observed = await migrationPool.query( + `SELECT floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint + AS "observedAtMs"`, + ); + const occurredAtMs = Number(observed.rows[0].observedAtMs) - 1_000; + await migrationPool.query( + `INSERT INTO "ql3"."projects" ( + id, name, slug, status, version, created_at_ms, updated_at_ms + ) VALUES ($1, 'HA Legacy Env application', $1, 'active', 1, $2, $2)`, + [FIXTURE.projectId, occurredAtMs], + ); + + const tasks = new PostgresTaskDefinitionRepository(migrationPool); + const triggers = new PostgresTriggerRepository(migrationPool); + const initialTask = ( + await tasks.appendTaskDefinitionRevision({ + projectId: FIXTURE.projectId, + taskId: FIXTURE.taskId, + expectedRevision: null, + mutationId: FIXTURE.taskCreateMutationId, + name: 'HA Legacy Env command', + description: 'promotion replay fixture', + kind: 'command', + spec: { + schema: 'qinglong/command@v1', + config: { + command: { kind: 'argv', file: '/bin/echo', args: ['ha'] }, + timeoutMs: 30_000, + }, + }, + labels: { 'qinglong.io/source': 'ha-contract' }, + enabled: true, + occurredAtMs, + }) + ).definition; + const initialTrigger = ( + await triggers.appendTriggerRevision({ + projectId: FIXTURE.projectId, + triggerId: FIXTURE.triggerId, + expectedRevision: null, + mutationId: FIXTURE.triggerCreateMutationId, + taskId: FIXTURE.taskId, + taskRevision: initialTask.revision, + taskContentDigest: initialTask.contentDigest, + spec: { + schema: 'qinglong/cron@v1', + config: { + expression: '* * * * *', + timezone: 'UTC', + misfirePolicy: 'skip', + }, + }, + enabled: false, + occurredAtMs: occurredAtMs + 1, + }) + ).trigger; + const currentTask = ( + await tasks.appendTaskDefinitionRevision({ + projectId: FIXTURE.projectId, + taskId: FIXTURE.taskId, + expectedRevision: initialTask.revision, + mutationId: FIXTURE.taskAdvanceMutationId, + name: initialTask.name, + description: initialTask.description, + kind: initialTask.kind, + spec: initialTask.spec, + labels: initialTask.labels, + enabled: initialTask.enabled, + occurredAtMs: occurredAtMs + 2, + }) + ).definition; + + const taskMutations = Object.freeze([ + Object.freeze({ + ordinal: 0, + taskId: FIXTURE.taskId, + previousRevision: currentTask.revision, + previousContentDigest: currentTask.contentDigest, + mutationId: FIXTURE.taskApplicationMutationId, + }), + ]); + const triggerMutations = Object.freeze([ + Object.freeze({ + ordinal: 0, + triggerId: FIXTURE.triggerId, + taskId: FIXTURE.taskId, + previousRevision: initialTrigger.revision, + previousContentDigest: initialTrigger.contentDigest, + previousTaskRevision: initialTask.revision, + previousTaskContentDigest: initialTask.contentDigest, + mutationId: FIXTURE.triggerApplicationMutationId, + }), + ]); + const taskDigester = createClusterLegacyEnvMigrationTaskMutationSetDigester(); + taskDigester.update(taskMutations[0]); + const taskSet = taskDigester.finish(); + const triggerDigester = + createClusterLegacyEnvMigrationTriggerMutationSetDigester(); + triggerDigester.update(triggerMutations[0]); + const triggerSet = triggerDigester.finish(); + const secretRef = createSecretRef({ + projectId: FIXTURE.projectId, + name: FIXTURE.privateBundleName, + version: 1, + }); + const plan = ( + await new PostgresClusterLegacyEnvMigrationPlanRepository( + automationPool, + ).publish({ + planId: FIXTURE.planId, + mutationId: FIXTURE.planMutationId, + projectId: FIXTURE.projectId, + source: { + reconciliationBundleDigest: '1'.repeat(64), + decisionDigest: '2'.repeat(64), + candidateSetDigest: '3'.repeat(64), + sourceRowCount: 1, + activeRowCount: 1, + disabledRowCount: 0, + effectiveBindingCount: 1, + }, + target: { + secretRef, + taskRevisionSetDigest: taskSet.revisionSetDigest, + triggerRevisionSetDigest: triggerSet.revisionSetDigest, + taskCount: taskSet.count, + triggerCount: triggerSet.count, + totalEffectiveBytes: 128, + }, + }) + ).plan; + const intent = Object.freeze({ + applicationId: FIXTURE.applicationId, + mutationId: FIXTURE.applicationMutationId, + projectId: FIXTURE.projectId, + planId: FIXTURE.planId, + planDigest: plan.planDigest, + taskMutationSetDigest: taskSet.mutationSetDigest, + triggerMutationSetDigest: triggerSet.mutationSetDigest, + }); + let taskStreamOpenCount = 0; + let triggerStreamOpenCount = 0; + const applications = + new PostgresClusterLegacyEnvMigrationApplicationRepository(automationPool); + const applied = await applications.apply(intent, { + taskMutations() { + taskStreamOpenCount += 1; + return taskMutations; + }, + triggerMutations() { + triggerStreamOpenCount += 1; + return triggerMutations; + }, + }); + assert.equal(applied.status, 'applied'); + assert.equal(taskStreamOpenCount, 1); + assert.equal(triggerStreamOpenCount, 1); + const primaryBeforePromotion = + await clusterLegacyEnvMigrationApplicationFacts(migrationPool, FIXTURE); + assertExpectedFacts(primaryBeforePromotion); + return { + fixture: Object.freeze({ + ...FIXTURE, + intent, + receipt: applied.receipt, + }), + report: { + primaryBeforePromotion, + standbyBeforePromotion: null, + promotedAfterReplay: null, + replicatedBeforePromotion: false, + exactReplayAfterPromotion: false, + replayStatus: null, + mutationStreamsOpenedAfterPromotion: null, + durableRowsAddedByReplay: null, + contentFree: true, + }, + }; +} + +async function verifyPromotedClusterLegacyEnvMigrationApplication(options) { + const { automationPool, promotedPool, evidence } = options; + let mutationStreamsOpened = 0; + const applications = + new PostgresClusterLegacyEnvMigrationApplicationRepository(automationPool); + const beforeReplay = await clusterLegacyEnvMigrationApplicationFacts( + promotedPool, + evidence.fixture, + ); + assert.deepEqual(beforeReplay, evidence.report.primaryBeforePromotion); + const replay = await applications.apply(evidence.fixture.intent, { + taskMutations() { + mutationStreamsOpened += 1; + throw new Error('exact replay must not reopen Task mutations'); + }, + triggerMutations() { + mutationStreamsOpened += 1; + throw new Error('exact replay must not reopen Trigger mutations'); + }, + }); + assert.equal(replay.status, 'existing'); + assert.deepEqual(replay.receipt, evidence.fixture.receipt); + const promotedAfterReplay = await clusterLegacyEnvMigrationApplicationFacts( + promotedPool, + evidence.fixture, + ); + assertExpectedFacts(promotedAfterReplay); + assert.deepEqual(promotedAfterReplay, beforeReplay); + evidence.report.promotedAfterReplay = promotedAfterReplay; + evidence.report.exactReplayAfterPromotion = true; + evidence.report.replayStatus = replay.status; + evidence.report.mutationStreamsOpenedAfterPromotion = mutationStreamsOpened; + evidence.report.durableRowsAddedByReplay = + promotedAfterReplay.receiptRows - beforeReplay.receiptRows; + return evidence.report; +} + +module.exports = { + clusterLegacyEnvMigrationApplicationFacts, + persistClusterLegacyEnvMigrationApplicationBeforePromotion, + verifyPromotedClusterLegacyEnvMigrationApplication, +}; diff --git a/test/back/ql3PostgresHaEvidenceAudit.test.cjs b/test/back/ql3PostgresHaEvidenceAudit.test.cjs index d26782eb..a7441eaf 100644 --- a/test/back/ql3PostgresHaEvidenceAudit.test.cjs +++ b/test/back/ql3PostgresHaEvidenceAudit.test.cjs @@ -6,6 +6,28 @@ const { auditPostgresHaEvidence, } = require('../../scripts/ql3-postgres-ha-evidence-audit.cjs'); +function legacyEnvApplicationFacts() { + return { + receiptDigest: '1'.repeat(64), + taskCount: 1, + triggerCount: 1, + taskRevision: 3, + taskContentDigest: '2'.repeat(64), + executionContentDigest: '3'.repeat(64), + triggerRevision: 2, + triggerContentDigest: '4'.repeat(64), + triggerTaskRevision: 3, + triggerTaskContentDigest: '5'.repeat(64), + scheduleRevision: 2, + scheduleStateVersion: 1, + scheduleClaimVersion: 1, + planRows: 1, + receiptRows: 1, + taskItemRows: 1, + triggerItemRows: 1, + }; +} + function fixture(architecture = 'arm64') { const gates = { passed: true }; for (let index = 0; index < 100; index += 1) gates[`gate${index}`] = true; @@ -66,12 +88,25 @@ function fixture(architecture = 'arm64') { synchronousState: 'sync', rejoinedAsWritablePrimary: false, }, + clusterLegacyEnvMigrationApplication: { + primaryBeforePromotion: legacyEnvApplicationFacts(), + standbyBeforePromotion: legacyEnvApplicationFacts(), + promotedAfterReplay: legacyEnvApplicationFacts(), + replicatedBeforePromotion: true, + exactReplayAfterPromotion: true, + replayStatus: 'existing', + mutationStreamsOpenedAfterPromotion: 0, + durableRowsAddedByReplay: 0, + contentFree: true, + }, timeline: [ 'primary_ready', 'standby_streaming', 'synchronous_remote_apply_ready', + 'cluster_legacy_env_application_replicated', 'replication_partition_and_promotion_guard_verified', 'old_primary_fenced_and_admission_withdrawn', + 'cluster_legacy_env_application_replayed_after_promotion', 'standby_promoted_old_primary_rejoined_endpoint_switched', 'two_fresh_control_replicas_ready', ].map((state, atMs) => ({ state, atMs })), @@ -92,11 +127,18 @@ test('rejects false gates, promotion drift and hidden private material', () => { const report = fixture(); report.gates.oldPrimaryFencedBeforePromotion = false; report.replication.promotedPrimaryTimeline = 1; + report.clusterLegacyEnvMigrationApplication.promotedAfterReplay.receiptRows = 2; report.privateValue = 'postgresql://private-credential'; const result = auditPostgresHaEvidence(report); assert.equal(result.compatible, false); assert.ok(result.findings.includes('GATE_FAILED')); assert.ok(result.findings.includes('PROMOTION_TIMELINE_INVALID')); + assert.ok( + result.findings.includes('CLUSTER_LEGACY_ENV_APPLICATION_FACTS_DRIFTED'), + ); + assert.ok( + result.findings.includes('CLUSTER_LEGACY_ENV_APPLICATION_FACTS_NOT_EXACT'), + ); assert.ok(result.findings.includes('PRIVATE_MATERIAL_PRESENT')); }); @@ -109,3 +151,27 @@ test('rejects reordered timeline and limitations drift', () => { assert.ok(result.findings.includes('LIMITATIONS_DRIFTED')); assert.ok(result.findings.some((finding) => finding.startsWith('TIMELINE_'))); }); + +test('rejects missing, widened or private Legacy Env replay evidence', () => { + const missing = fixture(); + delete missing.clusterLegacyEnvMigrationApplication; + assert.ok( + auditPostgresHaEvidence(missing).findings.includes( + 'CLUSTER_LEGACY_ENV_APPLICATION_EVIDENCE_MISSING', + ), + ); + + const widened = fixture(); + widened.clusterLegacyEnvMigrationApplication.privateRef = + 'qlsecret:v1:private'; + widened.clusterLegacyEnvMigrationApplication.primaryBeforePromotion.secretRef = + 'ha-legacy-env-bundle-private'; + const result = auditPostgresHaEvidence(widened); + assert.ok( + result.findings.includes('CLUSTER_LEGACY_ENV_APPLICATION_EVIDENCE_WIDENED'), + ); + assert.ok( + result.findings.includes('CLUSTER_LEGACY_ENV_APPLICATION_FACTS_WIDENED'), + ); + assert.ok(result.findings.includes('PRIVATE_MATERIAL_PRESENT')); +});