From 4d71dfb9375e2e5d93724634e7b1097a7511770b Mon Sep 17 00:00:00 2001 From: whyour Date: Sat, 22 Aug 2026 22:36:50 +0800 Subject: [PATCH] feat(local): fence reconciliation completion --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 17 + ...-reconciliation-apply-storage-lifecycle.md | 2 +- ...-domain-reconciliation-completion-fence.md | 87 ++ .../src/deployment/cutover/instanceLineage.ts | 11 +- .../src/deployment/localDeployment.ts | 27 + .../src/deployment/localDeploymentCli.ts | 10 +- .../application/automation/applyStorage.ts | 49 ++ .../reconciliation/completion/contract.ts | 355 +++++++++ .../reconciliation/completion/coordinator.ts | 754 ++++++++++++++++++ .../reconciliation/completion/evidence.ts | 195 +++++ .../reconciliationCapturePrepare.test.cjs | 336 ++++++++ scripts/ql3-cluster-dependency-audit.cjs | 5 + test/back/ql3ClusterDependencyAudit.test.cjs | 18 + test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 14 files changed, 1865 insertions(+), 5 deletions(-) create mode 100644 docs/adr/ADR-0488-cross-domain-reconciliation-completion-fence.md create mode 100644 packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/contract.ts create mode 100644 packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/coordinator.ts create mode 100644 packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/evidence.ts diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index a9ccf33c..e661d789 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -92,6 +92,23 @@ `shallowSourcePackages=[]`；Local Owner 为 `169 source / 168 nested / 1 root binary entry`。D-393 仍未代表完整 reconciliation：Automation 之外的 Secret、Plugin、Identity、history 等领域 adapter 尚未 apply，target service 也未获得 restart authority；下一切片是跨领域 completion fence，完成后才可进入 target restart/readiness。 +- D-394/ADR-0488（已接受）：跨领域 completion 不再由某个 adapter 状态或调用方自报列表推断。既有 Local Owner 在同一包内新增 + `deployment/reconciliation/completion/{contract,evidence,coordinator}` 与 + `reconciliation-complete|reconciliation-complete-verify`，重新打开 D-392 sealed application terminal 并按固定顺序推导全部八项证据。V1 只允许八域全 + `no_effect`，或仅 Automation 为 `adapter_required` 且 decision/apply/current target/head 完整一致、其余七域全 `no_effect`；任何 manual、其他 adapter、 + rolled-back 或 drift 都失败关闭。真实完整迁移库当前仍会在 Secret/Config、Run History、Identity/Policy/Audit、Unknown 等领域产生受保护的 + `manual_external`，因此 Automation applied 不会被冒充为全局完成，rollback backup 和 target-stop 状态均保持不变。 + + mutation 固定为 receipt no-replace publish → `0400/0500` seal → instance head CAS 到 `reconciliation_completed` → 可选 Automation backup collection； + 只有 completed head durable 后才删除数据库等量 backup，四个 crash/response-loss 窗口均可 exact replay。target runner 与 Service Manager 继续共享 + `assertLocalCutoverTargetHead`：application-planned、automation-applied、manual 和 rolled-back 都不能重启，只有 completed 可进入下一 generation 的 + `target_active`。Service Manager v1 intent 仍只有 prior service journal digest，没有独立 completion-head digest，因此 systemd/OpenRC 路径在后续 + compare-and-swap 中继续失败关闭；不得仅放宽共享 assertion 来伪装支持，后续必须以 v2 intent 同时绑定两条 lineage。verify 只读且不修复。实现未新增 + package、dependency、SQL、daemon、timer、watcher、listener、Pool 或 Cluster workload；workspace + 仍为 18 packages，Local Owner 为 `172 source / 171 nested / 1 root binary entry`。focused completion `2/2`、完整 Local Owner + `268 total / 261 pass / 7 conditional skip / 0 fail`、18-package clean build/逐包测试、dependency/package boundary `70/70` 和 122-module Edge + import audit 已通过；其余六个领域的 terminal adapter、Service Manager v2 binding、完整 target readiness/restart 演练和 Cluster 自有 completion + authority 仍是后续门禁。 - D-392/ADR-0485（已接受）：D-391 的 signed review 不能直接获得通用 DML authority；表级 `adopt_legacy/retain_both` 也不能证明 Automation 行级 command/trigger 兼容，更不能覆盖 Secret custody、append-only history、Plugin/AI 外部资产与 Identity/Policy 语义。 因此既有 Local Owner 新增 `reconciliation.application.prepare|commit|verify`，以 diff --git a/docs/adr/ADR-0487-reconciliation-apply-storage-lifecycle.md b/docs/adr/ADR-0487-reconciliation-apply-storage-lifecycle.md index 07d6b50e..d7be6a33 100644 --- a/docs/adr/ADR-0487-reconciliation-apply-storage-lifecycle.md +++ b/docs/adr/ADR-0487-reconciliation-apply-storage-lifecycle.md @@ -96,4 +96,4 @@ Applied backup 在尚未 rollback 时不得自动删除。跨 Automation、Secre - 完整 Local Owner 在真实 loopback 环境 `266 total / 259 pass / 7 conditional skip / 0 fail`；18-package clean build/逐包测试退出 0；真实 stopped-target Docker reconciliation `2/2`。 - cluster dependency 与 package boundary 组合门 `70/70`，workspace 保持 18 packages、`singleSourcePackages=[]`、`shallowSourcePackages=[]`；Local Owner 为 `169 source / 168 nested / 1 root binary entry`。 - 不新增 workspace package、production dependency、daemon、timer、watcher、listener、SQL migration、PostgreSQL role/ACL、Pool 或 cluster workload。 -- D-393 仍未完成：下一切片必须建立跨领域 completion fence；Secret、Plugin、Identity、history 等 adapter 及 target restart/readiness authority 仍关闭。 +- 后续 ADR-0488 已建立跨领域 completion fence：八域全 `no_effect` 可进入 `reconciliation_completed` 并获得 target generation 2 重启 authority；Automation apply 只有在其余七域同样 `no_effect` 时才能完成并回收 backup。当前完整迁移库仍有 Secret、Identity、history 等 manual 领域时会失败关闭并继续保留 rollback authority。 diff --git a/docs/adr/ADR-0488-cross-domain-reconciliation-completion-fence.md b/docs/adr/ADR-0488-cross-domain-reconciliation-completion-fence.md new file mode 100644 index 00000000..0964d1a4 --- /dev/null +++ b/docs/adr/ADR-0488-cross-domain-reconciliation-completion-fence.md @@ -0,0 +1,87 @@ +# ADR-0488：跨领域 Reconciliation 完成围栏与目标重启授权 + +- 状态：Accepted +- 日期：2026-08-22 +- 决策：D-394 +- 关联：ADR-0482、ADR-0485、ADR-0486、ADR-0487 + +## 背景 + +D-392 把 reconciliation 拆为固定八个领域，D-393 已实现首个 Automation plan、review、apply、verify 与 rollback。然而 instance lineage 在 Automation apply 后只停在 `reconciliation_automation_applied`，target start authority 又只接受早期 `legacy_stopped|target_active|manual_required`。系统因此同时存在两个缺口： + +- 没有一个证据能证明八个领域都已终结，不能安全地把单领域 apply 当作整体完成； +- 即使八个领域都明确选择 no-effect，也没有合法状态可进入 target generation 2。 + +直接允许 `reconciliation_automation_applied` 重启是不安全的。真实迁移后的目标 SQLite 仍可能在 Secret/Config、Run History、Identity/Policy/Audit 或 Unknown 领域包含 `manual_external`；Automation 写入成功不等于这些领域已经被处理。另一方面，永久拒绝 all-no-effect 场景会让完成了人工 review、明确保留 Target/排除 Legacy 的部署无法继续。 + +## 决策 + +### 1. 完成是独立的跨领域状态，而不是某个 adapter 的别名 + +实例谱系新增唯一终态 `reconciliation_completed`。它只能从以下 source head 前进： + +- `reconciliation_application_planned`：八个领域全部为 `no_effect`； +- `reconciliation_automation_applied`：只有 Automation 为 `adapter_required`、其余七个领域全部为 `no_effect`，且 Automation decision、apply receipt、当前 target snapshot 与 head 完整一致。 + +任何 `manual_external|adapter_and_manual`、非 Automation 的 `adapter_required`、Automation rolled-back 或证据漂移都失败关闭。当前完整迁移数据库通常仍包含多个受保护的 manual 领域，因此 Automation apply 不会被误判为 restart-ready；backup 继续保留，等待后续领域 adapter 或显式 rollback。 + +### 2. 调用方不能自报八领域完成 + +`local.deployment.reconciliation.complete` 只接收 completion/application/head digest，以及可选的 Automation apply digest/identity。coordinator 重新打开已封存的 application terminal，并严格按 D-392 固定顺序从 plan 推导八项 domain evidence： + +- `no_effect` 绑定 application domain `summaryDigest`； +- Automation adapter 绑定真实 `applyDigest`。 + +receipt 固定包含八项、`adapterCount=0|1`、application plan、source head、实例 identity、generation、时间与自身 digest。调用方不能提交任意 domain 数组、路径、row body、Secret、reviewer 或 credential 作为完成声明。 + +### 3. Receipt-first、head-second、重资产最后回收 + +mutation 顺序固定为： + +1. 重新验证 application、可选 Automation apply、当前 target snapshot 与 source head； +2. no-replace 发布 completion receipt，并封存为 `0400`；目录封为 `0500`； +3. CAS 前进到 `reconciliation_completed`； +4. 仅在 completed head 已 durable 后回收 Automation rollback backup，并把空 `backup/`、`rollback-work/` 封为 `0500`。 + +receipt publish、seal、head advance 或 backup collection 任一响应丢失都可由相同 exact command 重放。顺序禁止在 head 仍是 applied 时删除 backup；若在 head 完成后、回收前崩溃，重放只继续收紧存储，不恢复 rollback authority。 + +`local.deployment.reconciliation.complete.verify` 全程只读：验证 sealed receipt、八领域推导、application/Automation binding、当前 target snapshot、completed head 与已经收敛的存储形态，不修复漂移。 + +### 4. 直接 Target 重启只接受完成围栏 + +Docker target authority 继续拒绝 `reconciliation_application_planned`、`reconciliation_automation_applied` 和所有 manual/rolled-back 中间态；只有 `reconciliation_completed` 能重新进入 `target_active`，且调用方必须使用下一 generation。 + +Service Manager 虽然复用同一个 head assertion，但既有 v1 intent 只绑定 prior service journal `previousRecordDigest`，没有独立的 expected completion-head digest；其后续 compare-and-swap 因而继续拒绝 completed head。D-394 不通过忽略该比较来伪造兼容性。systemd/OpenRC 的 restart-ready 必须由后续 v2 intent 同时绑定 prior service record 与 exact completion head 后才能开放。 + +### 5. 部署规模与代码边界 + +实现位于既有 `@qinglong/local-owner-cli` 的 `deployment/reconciliation/completion/` 子域，按 contract/evidence/coordinator 三个职责文件组织；没有新拆 workspace package，也没有把实现平铺到 `src/` 根。 + +- Edge/Standalone：无 daemon、timer、watcher、listener、Pool 或常驻缓存；receipt 上限 64 KiB、completion catalog 上限 64，SQLite snapshot/hash 复用固定内存实现；completed 后同步释放数据库等量 rollback backup。 +- Cluster：本 ADR 只定义 Local SQLite authority。PostgreSQL/多副本必须使用独立的事务、snapshot 与 HA evidence，不能复用本机文件路径或把 Local receipt 当成 Cluster completion。 + +## 被拒绝的替代方案 + +### Automation applied 直接允许重启 + +拒绝。它会跳过其余七个领域，尤其会把 Secret custody、append-only history 和 Identity/Policy manual decision 静默吞掉。 + +### 调用方提交 `completedDomains[]` + +拒绝。自由数组无法证明来源、顺序、摘要和 current head，容易把 stale 或伪造 claim 变成 restart authority。 + +### 先删除 backup 再写 completed head + +拒绝。head CAS 失败会留下 applied 状态却失去 rollback source，形成不可恢复的部分提交。 + +### 为完成围栏新增独立 package/service/GC timer + +拒绝。该职责没有独立部署和扩缩容边界；额外 package、进程或周期扫描只会增加路由设备的安装元数据、RSS、唤醒和竞态。 + +## 验收证据 + +- completion focused `2/2`：覆盖 all-no-effect 的 receipt/seal/head 三个 response-loss 窗口、只读 verify、CLI content-free、直接 target generation 2 状态转换；同时覆盖 Automation 已 apply 但其他领域仍 manual 时拒绝完成、拒绝 target restart，并保持 `0400/0500` backup authority。 +- 完整 Local Owner `268 total / 261 pass / 7 conditional skip / 0 fail`；18-package clean build 与逐包测试 exit 0。 +- dependency/package boundary 组合门 `70/70`，Edge import audit 为 122 modules、0 forbidden；workspace 保持 18 packages、无 single-source/shallow package；Local Owner 为 `172 source / 171 nested / 1 root binary entry`。 +- 不新增 production dependency、SQL migration、daemon、timer、watcher、listener、Pool、PostgreSQL role/ACL 或 cluster workload。 +- Secret/Config、Run History、Plugin Package、AI/Tool、Identity/Policy/Audit、Unknown 的 terminal adapter，以及 Service Manager v2 completion-head binding/真实 restart 演练仍是后续工作；D-394 不把尚未实现的领域或部署方式宣称为完成。 diff --git a/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts b/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts index df8871bf..ccdd5706 100644 --- a/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts +++ b/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts @@ -37,6 +37,7 @@ export type LocalCutoverInstanceHeadState = | 'reconciliation_automation_apply_prepared' | 'reconciliation_automation_applied' | 'reconciliation_automation_rolled_back' + | 'reconciliation_completed' | 'rollback_prepared' | 'legacy_restart_requested' | 'legacy_running' @@ -182,6 +183,7 @@ function parseHead(value: unknown): Readonly { head.state !== 'reconciliation_automation_apply_prepared' && head.state !== 'reconciliation_automation_applied' && head.state !== 'reconciliation_automation_rolled_back' && + head.state !== 'reconciliation_completed' && head.state !== 'rollback_prepared' && head.state !== 'legacy_restart_requested' && head.state !== 'legacy_running' && @@ -362,6 +364,7 @@ export function advanceLocalCutoverInstanceHead( | 'reconciliation_automation_apply_prepared' | 'reconciliation_automation_applied' | 'reconciliation_automation_rolled_back' + | 'reconciliation_completed' | 'rollback_prepared' | 'legacy_restart_requested' | 'legacy_running' @@ -414,6 +417,7 @@ export function advanceLocalCutoverInstanceHead( current.state === 'reconciliation_automation_apply_prepared' || current.state === 'reconciliation_automation_applied' || current.state === 'reconciliation_automation_rolled_back' || + current.state === 'reconciliation_completed' || current.state === 'legacy_restart_requested' || current.state === 'legacy_running' || current.state === 'legacy_ready') @@ -427,7 +431,8 @@ export function advanceLocalCutoverInstanceHead( (state === 'legacy_stopped' && current.state === 'legacy_stop_requested') || (state === 'target_active' && (current.state === 'legacy_stopped' || - current.state === 'target_active')) || + current.state === 'target_active' || + current.state === 'reconciliation_completed')) || (state === 'target_stopped' && current.state === 'target_active') || (state === 'reconciliation_capture_prepared' && current.state === 'target_stopped') || @@ -457,6 +462,9 @@ export function advanceLocalCutoverInstanceHead( current.state === 'reconciliation_automation_apply_prepared') || (state === 'reconciliation_automation_rolled_back' && current.state === 'reconciliation_automation_applied') || + (state === 'reconciliation_completed' && + (current.state === 'reconciliation_application_planned' || + current.state === 'reconciliation_automation_applied')) || (state === 'rollback_prepared' && current.state === 'target_stopped') || (state === 'legacy_restart_requested' && current.state === 'rollback_prepared') || @@ -497,6 +505,7 @@ export function assertLocalCutoverTargetHead( head.activationDigest !== identity.request.expectedActivationDigest || (head.state !== 'legacy_stopped' && head.state !== 'target_active' && + head.state !== 'reconciliation_completed' && head.state !== 'manual_required') ) { configurationError( diff --git a/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts b/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts index 0e33ac37..57921220 100644 --- a/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts +++ b/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts @@ -154,6 +154,12 @@ import { verifyLocalReconciliationAutomationApply, verifyLocalReconciliationAutomationApplyCommandFile, } from './reconciliation/application/automation/applyCoordinator'; +import { + completeLocalReconciliation, + completeLocalReconciliationCommandFile, + verifyLocalReconciliationCompletion, + verifyLocalReconciliationCompletionCommandFile, +} from './reconciliation/completion/coordinator'; export { commitLocalReconciliationPlan, @@ -193,8 +199,29 @@ export { verifyLocalReconciliationAutomationApplyCommandFile, rollbackLocalReconciliationAutomationApply, rollbackLocalReconciliationAutomationApplyCommandFile, + completeLocalReconciliation, + completeLocalReconciliationCommandFile, + verifyLocalReconciliationCompletion, + verifyLocalReconciliationCompletionCommandFile, }; +export { + normalizeLocalReconciliationCompleteCommand, + normalizeLocalReconciliationCompletionVerifyCommand, + type LocalReconciliationCompleteCommand, + type LocalReconciliationCompletionAutomationBinding, + type LocalReconciliationCompletionAutomationOptions, + type LocalReconciliationCompletionOptions, + type LocalReconciliationCompletionResult, + type LocalReconciliationCompletionVerifyCommand, +} from './reconciliation/completion/contract'; +export { type LocalReconciliationCompletionDependencies } from './reconciliation/completion/coordinator'; +export { + normalizeLocalReconciliationCompletionReceipt, + type LocalReconciliationCompletionDomainEvidence, + type LocalReconciliationCompletionReceipt, +} from './reconciliation/completion/evidence'; + export { normalizeLocalReconciliationAutomationApplyCommand, normalizeLocalReconciliationAutomationApplyRollbackCommand, diff --git a/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts b/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts index c43c841c..b7da246c 100644 --- a/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts +++ b/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts @@ -33,6 +33,8 @@ import { applyLocalReconciliationAutomationCommandFile, verifyLocalReconciliationAutomationApplyCommandFile, rollbackLocalReconciliationAutomationApplyCommandFile, + completeLocalReconciliationCommandFile, + verifyLocalReconciliationCompletionCommandFile, writeLocalReconciliationReviewDiagnosticsCommandFile, prepareLocalDeploymentCommandFile, proveLocalDeploymentLegacyReadinessCommandFile, @@ -48,7 +50,7 @@ import { } from './localDeployment'; const USAGE = - 'Usage: ql3-local-deploy --command-file /absolute/private-command.json'; + 'Usage: ql3-local-deploy --command-file /absolute/private-command.json'; async function main(argv: readonly string[]): Promise { if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) { @@ -98,6 +100,8 @@ async function main(argv: readonly string[]): Promise { argv[0] !== 'reconciliation-automation-apply' && argv[0] !== 'reconciliation-automation-apply-verify' && argv[0] !== 'reconciliation-automation-apply-rollback' && + argv[0] !== 'reconciliation-complete' && + argv[0] !== 'reconciliation-complete-verify' && argv[0] !== 'compose-revision' && argv[0] !== 'compose-preflight' && argv[0] !== 'compose-apply' && @@ -210,6 +214,10 @@ async function main(argv: readonly string[]): Promise { ? verifyLocalReconciliationAutomationApplyCommandFile(argv[2]!) : argv[0] === 'reconciliation-automation-apply-rollback' ? rollbackLocalReconciliationAutomationApplyCommandFile(argv[2]!) + : argv[0] === 'reconciliation-complete' + ? completeLocalReconciliationCommandFile(argv[2]!) + : argv[0] === 'reconciliation-complete-verify' + ? verifyLocalReconciliationCompletionCommandFile(argv[2]!) : argv[0] === 'compose-revision' ? switchLocalDeploymentComposeRevisionCommandFile(argv[2]!) : argv[0] === 'compose-preflight' diff --git a/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyStorage.ts b/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyStorage.ts index 874c6e4e..2d556304 100644 --- a/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyStorage.ts +++ b/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyStorage.ts @@ -503,6 +503,55 @@ export function validateLocalReconciliationAutomationAppliedStorage( validateBackup(selected, intent, uid, [0o400]); } +/** + * Collects the rollback copy only after the cross-domain completion head is + * durable. The operation accepts its own partially collected layout so a + * response loss or process crash can be replayed without restoring authority. + */ +export function collectLocalReconciliationAutomationCompletedStorage( + selected: Readonly, + intent: Readonly, + uid: number, +): void { + directoryMode(selected.root, uid, [0o500], 'root'); + directoryMode(selected.backupRoot, uid, [0o700, 0o500], 'backup root'); + directoryMode( + selected.rollbackRoot, + uid, + [0o700, 0o500], + 'rollback work root', + ); + validateLocalReconciliationAutomationApplyCatalog(selected); + emptyDirectory(selected.rollbackRoot, 'rollback work root'); + stableJson(selected.intent, uid, [0o400], 'intent', [1n]); + stableJson(selected.receipt, uid, [0o400], 'receipt', [1n]); + if (fs.existsSync(selected.backup)) { + validateBackup(selected, intent, uid, [0o400]); + if ((fs.statSync(selected.backupRoot).mode & 0o777) !== 0o700) { + fs.chmodSync(selected.backupRoot, 0o700); + syncDirectory(selected.root); + } + unlinkIfPresent(selected.backup); + } + sealDirectory(selected.backupRoot, uid, 'backup root'); + sealDirectory(selected.rollbackRoot, uid, 'rollback work root'); + validateLocalReconciliationAutomationCompletedStorage(selected, uid); +} + +export function validateLocalReconciliationAutomationCompletedStorage( + selected: Readonly, + uid: number, +): void { + directoryMode(selected.root, uid, [0o500], 'root'); + directoryMode(selected.backupRoot, uid, [0o500], 'backup root'); + directoryMode(selected.rollbackRoot, uid, [0o500], 'rollback work root'); + validateLocalReconciliationAutomationApplyCatalog(selected); + emptyDirectory(selected.backupRoot, 'backup root'); + emptyDirectory(selected.rollbackRoot, 'rollback work root'); + stableJson(selected.intent, uid, [0o400], 'intent', [1n]); + stableJson(selected.receipt, uid, [0o400], 'receipt', [1n]); +} + export function prepareLocalReconciliationAutomationRollbackSource( selected: Readonly, intent: Readonly, diff --git a/packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/contract.ts b/packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/contract.ts new file mode 100644 index 00000000..4fd38c03 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/contract.ts @@ -0,0 +1,355 @@ +import path from 'node:path'; + +import { currentIdentity } from '../../foundation/contract'; +import { LocalDeploymentConfigurationError } from '../../foundation/error'; + +const DIGEST = /^[0-9a-f]{64}$/; +const UUID_V4 = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const UUID_V7 = + /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const SAFE_PATH = /^\/[A-Za-z0-9._/@-]+$/; + +export interface LocalReconciliationCompletionAutomationOptions { + readonly automationRoot: string; + readonly automationDecisionRoot: string; + readonly automationApplyRoot: string; + readonly targetDatabasePath: string; +} + +export interface LocalReconciliationCompletionOptions { + readonly deploymentRoot: string; + readonly applicationRoot: string; + readonly completionRoot: string; + readonly automation: Readonly | null; + readonly allowRootService: boolean; +} + +export interface LocalReconciliationCompletionAutomationBinding { + readonly automationId: string; + readonly decisionId: string; + readonly expectedApplyDigest: string; +} + +export interface LocalReconciliationCompleteCommand { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.reconciliation.complete'; + readonly options: Readonly; + readonly request: Readonly<{ + completionId: string; + applicationId: string; + expectedApplicationPlanDigest: string; + expectedHeadDigest: string; + automation: Readonly | null; + completedAtMs: number; + }>; +} + +export interface LocalReconciliationCompletionVerifyCommand { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.reconciliation.complete.verify'; + readonly options: Readonly; + readonly request: Readonly<{ + completionId: string; + applicationId: string; + expectedCompletionDigest: string; + automation: Readonly | null; + }>; +} + +export interface LocalReconciliationCompletionResult { + readonly schemaVersion: 1; + readonly operation: + | LocalReconciliationCompleteCommand['operation'] + | LocalReconciliationCompletionVerifyCommand['operation']; + readonly status: 'completed' | 'existing' | 'verified'; + readonly state: 'reconciliation_completed'; + readonly completionId: string; + readonly applicationId: string; + readonly completionDigest: string; + readonly domainCount: 8; + readonly adapterCount: 0 | 1; + readonly instanceHeadDigest: string; +} + +function fail(message: string): never { + throw new LocalDeploymentConfigurationError( + `reconciliation completion ${message}`, + ); +} + +function record(value: unknown, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + fail(`${label} must be an object`); + } + return value as Record; +} + +function exact( + value: Record, + keys: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + fail(`${label} shape is invalid`); + } +} + +function safePath(value: unknown, label: string): string { + if ( + typeof value !== 'string' || + !path.isAbsolute(value) || + path.parse(value).root === value || + path.normalize(value) !== value || + value.includes('\0') || + value.includes('//') || + !SAFE_PATH.test(value) || + Buffer.byteLength(value, 'utf8') > 4_096 + ) { + fail(`${label} must be a safe non-root absolute path`); + } + return value; +} + +function overlaps(left: string, right: string): boolean { + const relative = path.relative(left, right); + return ( + relative === '' || + (!relative.startsWith('..') && !path.isAbsolute(relative)) + ); +} + +function identifier(value: unknown, pattern: RegExp, label: string): string { + if (typeof value !== 'string' || !pattern.test(value)) { + fail(`${label} is invalid`); + } + return value; +} + +function digest(value: unknown, label: string): string { + return identifier(value, DIGEST, label); +} + +function normalizeAutomationOptions( + value: unknown, +): Readonly | null { + if (value === null) return null; + const selected = record(value, 'automation options'); + exact( + selected, + [ + 'automationApplyRoot', + 'automationDecisionRoot', + 'automationRoot', + 'targetDatabasePath', + ], + 'automation options', + ); + return Object.freeze({ + automationRoot: safePath(selected.automationRoot, 'automationRoot'), + automationDecisionRoot: safePath( + selected.automationDecisionRoot, + 'automationDecisionRoot', + ), + automationApplyRoot: safePath( + selected.automationApplyRoot, + 'automationApplyRoot', + ), + targetDatabasePath: safePath( + selected.targetDatabasePath, + 'targetDatabasePath', + ), + }); +} + +function normalizeOptions( + value: unknown, +): Readonly { + const selected = record(value, 'options'); + exact( + selected, + [ + 'allowRootService', + 'applicationRoot', + 'automation', + 'completionRoot', + 'deploymentRoot', + ], + 'options', + ); + if ( + typeof selected.allowRootService !== 'boolean' || + (currentIdentity().uid === 0) !== selected.allowRootService + ) { + fail('command identity is invalid'); + } + const automation = normalizeAutomationOptions(selected.automation); + const normalized = Object.freeze({ + deploymentRoot: safePath(selected.deploymentRoot, 'deploymentRoot'), + applicationRoot: safePath(selected.applicationRoot, 'applicationRoot'), + completionRoot: safePath(selected.completionRoot, 'completionRoot'), + automation, + allowRootService: selected.allowRootService, + }) as Readonly; + const roots = [ + normalized.deploymentRoot, + normalized.applicationRoot, + normalized.completionRoot, + ...(automation === null + ? [] + : [ + automation.automationRoot, + automation.automationDecisionRoot, + automation.automationApplyRoot, + ]), + ]; + for (let left = 0; left < roots.length; left += 1) { + for (let right = left + 1; right < roots.length; right += 1) { + if ( + overlaps(roots[left]!, roots[right]!) || + overlaps(roots[right]!, roots[left]!) + ) { + fail('authority roots overlap'); + } + } + } + if ( + automation !== null && + roots.some( + (root) => + overlaps(root, automation.targetDatabasePath) || + overlaps(automation.targetDatabasePath, root), + ) + ) { + fail('targetDatabasePath overlaps an authority root'); + } + return normalized; +} + +function normalizeAutomationBinding( + value: unknown, +): Readonly | null { + if (value === null) return null; + const selected = record(value, 'automation binding'); + exact( + selected, + ['automationId', 'decisionId', 'expectedApplyDigest'], + 'automation binding', + ); + return Object.freeze({ + automationId: identifier(selected.automationId, UUID_V4, 'automationId'), + decisionId: identifier(selected.decisionId, UUID_V7, 'decisionId'), + expectedApplyDigest: digest( + selected.expectedApplyDigest, + 'expectedApplyDigest', + ), + }); +} + +function command(value: unknown, operation: string) { + const selected = record(value, 'command'); + exact( + selected, + ['operation', 'options', 'request', 'schemaVersion'], + 'command', + ); + if (selected.schemaVersion !== 1 || selected.operation !== operation) { + fail('command version or operation is invalid'); + } + return Object.freeze({ + options: normalizeOptions(selected.options), + request: record(selected.request, 'request'), + }); +} + +export function normalizeLocalReconciliationCompleteCommand( + value: unknown, +): Readonly { + const selected = command(value, 'local.deployment.reconciliation.complete'); + exact( + selected.request, + [ + 'applicationId', + 'automation', + 'completedAtMs', + 'completionId', + 'expectedApplicationPlanDigest', + 'expectedHeadDigest', + ], + 'request', + ); + if ( + !Number.isSafeInteger(selected.request.completedAtMs) || + (selected.request.completedAtMs as number) < 0 + ) { + fail('completedAtMs is invalid'); + } + return Object.freeze({ + schemaVersion: 1, + operation: 'local.deployment.reconciliation.complete', + options: selected.options, + request: Object.freeze({ + completionId: identifier( + selected.request.completionId, + UUID_V4, + 'completionId', + ), + applicationId: identifier( + selected.request.applicationId, + UUID_V4, + 'applicationId', + ), + expectedApplicationPlanDigest: digest( + selected.request.expectedApplicationPlanDigest, + 'expectedApplicationPlanDigest', + ), + expectedHeadDigest: digest( + selected.request.expectedHeadDigest, + 'expectedHeadDigest', + ), + automation: normalizeAutomationBinding(selected.request.automation), + completedAtMs: selected.request.completedAtMs as number, + }), + }); +} + +export function normalizeLocalReconciliationCompletionVerifyCommand( + value: unknown, +): Readonly { + const selected = command( + value, + 'local.deployment.reconciliation.complete.verify', + ); + exact( + selected.request, + ['applicationId', 'automation', 'completionId', 'expectedCompletionDigest'], + 'request', + ); + return Object.freeze({ + schemaVersion: 1, + operation: 'local.deployment.reconciliation.complete.verify', + options: selected.options, + request: Object.freeze({ + completionId: identifier( + selected.request.completionId, + UUID_V4, + 'completionId', + ), + applicationId: identifier( + selected.request.applicationId, + UUID_V4, + 'applicationId', + ), + expectedCompletionDigest: digest( + selected.request.expectedCompletionDigest, + 'expectedCompletionDigest', + ), + automation: normalizeAutomationBinding(selected.request.automation), + }), + }); +} diff --git a/packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/coordinator.ts b/packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/coordinator.ts new file mode 100644 index 00000000..756770f9 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/coordinator.ts @@ -0,0 +1,754 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { readPrivateLocalCommandFile } from '@qinglong/local-command-file'; +import { inspectLocalSqliteSnapshot } from '@qinglong/local-sqlite/rollout-safety'; + +import { currentIdentity } from '../../foundation/contract'; +import { LocalDeploymentConfigurationError } from '../../foundation/error'; +import { + preflightPublishedFile, + publishExactFile, + validatePrivateDirectory, +} from '../../foundation/files'; +import { + advanceLocalCutoverInstanceHead, + readLocalCutoverInstanceHead, + type LocalCutoverInstanceHead, +} from '../../cutover/instanceLineage'; +import { readLocalReconciliationApplicationTerminal } from '../application/coordinator'; +import type { LocalReconciliationApplicationTerminal } from '../application/coordinator'; +import { + collectLocalReconciliationAutomationCompletedStorage, + localReconciliationAutomationApplyPaths, + readLocalReconciliationAutomationApplyIntent, + readLocalReconciliationAutomationApplyReceipt, + validateLocalReconciliationAutomationAppliedStorage, + validateLocalReconciliationAutomationApplyCatalog, + validateLocalReconciliationAutomationApplyLayout, + validateLocalReconciliationAutomationCompletedStorage, +} from '../application/automation/applyStorage'; +import type { + LocalReconciliationAutomationApplyIntent, + LocalReconciliationAutomationApplyReceipt, +} from '../application/automation/applyEvidence'; +import { readLocalReconciliationAutomationDecisionTerminal } from '../application/automation/decisionCoordinator'; +import type { LocalReconciliationCompletionDomainEvidence } from './evidence'; +import { + buildLocalReconciliationCompletionReceipt, + localReconciliationCompletionReceiptContents, + normalizeLocalReconciliationCompletionReceipt, + type LocalReconciliationCompletionReceipt, +} from './evidence'; +import { + normalizeLocalReconciliationCompleteCommand, + normalizeLocalReconciliationCompletionVerifyCommand, + type LocalReconciliationCompleteCommand, + type LocalReconciliationCompletionOptions, + type LocalReconciliationCompletionResult, + type LocalReconciliationCompletionVerifyCommand, +} from './contract'; + +const MAX_COMPLETIONS = 64; +const MAX_RECEIPT_BYTES = 64 * 1024; + +interface CompletionPaths { + readonly root: string; + readonly receipt: string; +} + +interface AutomationProof { + readonly intent: Readonly; + readonly receipt: Readonly; + readonly paths: ReturnType; +} + +export interface LocalReconciliationCompletionDependencies { + readonly inspectSnapshot?: typeof inspectLocalSqliteSnapshot; + readonly afterReceiptPublished?: () => void; + readonly afterTerminalSealed?: () => void; + readonly afterHeadAdvanced?: () => void; + readonly afterBackupCollected?: () => void; +} + +function fail(message: string, cause?: unknown): never { + throw new LocalDeploymentConfigurationError( + `reconciliation completion ${message}`, + { cause }, + ); +} + +function completionPaths( + completionRoot: string, + completionId: string, +): Readonly { + const root = path.join(completionRoot, completionId); + return Object.freeze({ root, receipt: path.join(root, 'receipt.json') }); +} + +function validateDirectory( + directory: string, + uid: number, + modes: readonly number[], + label: string, +): number { + let stat: fs.Stats; + try { + stat = fs.lstatSync(directory); + } catch (error) { + return fail(`${label} is unavailable`, error); + } + const mode = stat.mode & 0o777; + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + stat.uid !== uid || + !modes.includes(mode) || + fs.realpathSync(directory) !== directory + ) { + fail(`${label} identity is invalid`); + } + return mode; +} + +function validateCatalog(selected: Readonly, sealed: boolean) { + const allowed = new Set([ + 'receipt.json', + ...(!sealed ? ['.receipt.json.ql3-deploy-stage'] : []), + ]); + for (const entry of fs.readdirSync(selected.root, { withFileTypes: true })) { + if (!allowed.has(entry.name) || entry.isSymbolicLink()) { + fail('receipt catalog contains unknown material'); + } + } +} + +function ensureCompletionDirectory( + completionRoot: string, + completionId: string, + uid: number, +): Readonly { + const selected = completionPaths(completionRoot, completionId); + const entries = fs.readdirSync(completionRoot, { withFileTypes: true }); + if (entries.some((entry) => !entry.isDirectory() || entry.isSymbolicLink())) { + fail('completion catalog contains drift'); + } + if (entries.length >= MAX_COMPLETIONS && !fs.existsSync(selected.root)) { + fail('completion retention limit is reached'); + } + try { + fs.mkdirSync(selected.root, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') { + fail('receipt directory cannot be created', error); + } + } + validateDirectory(selected.root, uid, [0o700, 0o500], 'receipt directory'); + validateCatalog( + selected, + (fs.statSync(selected.root).mode & 0o777) === 0o500, + ); + return selected; +} + +function stableReceipt( + selected: Readonly, + uid: number, + modes: readonly number[], +): Readonly { + let descriptor: number | undefined; + let bytes: Buffer | undefined; + try { + const before = fs.lstatSync(selected.receipt, { bigint: true }); + if ( + !before.isFile() || + before.isSymbolicLink() || + Number(before.uid) !== uid || + !modes.includes(Number(before.mode) & 0o777) || + before.nlink !== 1n || + before.size < 2n || + before.size > BigInt(MAX_RECEIPT_BYTES) + ) { + fail('receipt identity is invalid'); + } + descriptor = fs.openSync( + selected.receipt, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + const opened = fs.fstatSync(descriptor, { bigint: true }); + if ( + !opened.isFile() || + opened.dev !== before.dev || + opened.ino !== before.ino || + opened.size !== before.size || + opened.mtimeNs !== before.mtimeNs || + opened.ctimeNs !== before.ctimeNs || + opened.mode !== before.mode || + opened.uid !== before.uid || + opened.nlink !== before.nlink + ) { + fail('receipt changed while opening'); + } + bytes = Buffer.alloc(Number(opened.size)); + let offset = 0; + while (offset < bytes.length) { + const count = fs.readSync( + descriptor, + bytes, + offset, + bytes.length - offset, + offset, + ); + if (count < 1) fail('receipt read stalled'); + offset += count; + } + const after = fs.fstatSync(descriptor, { bigint: true }); + const pathAfter = fs.lstatSync(selected.receipt, { bigint: true }); + if ( + after.dev !== opened.dev || + after.ino !== opened.ino || + after.size !== opened.size || + after.mtimeNs !== opened.mtimeNs || + after.ctimeNs !== opened.ctimeNs || + after.mode !== opened.mode || + after.uid !== opened.uid || + after.nlink !== opened.nlink || + pathAfter.dev !== before.dev || + pathAfter.ino !== before.ino || + pathAfter.size !== before.size || + pathAfter.mtimeNs !== before.mtimeNs || + pathAfter.ctimeNs !== before.ctimeNs || + pathAfter.mode !== before.mode || + pathAfter.uid !== before.uid || + pathAfter.nlink !== before.nlink + ) { + fail('receipt drifted while reading'); + } + return normalizeLocalReconciliationCompletionReceipt( + JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)), + ); + } catch (error) { + if (error instanceof LocalDeploymentConfigurationError) throw error; + return fail('receipt cannot be read', error); + } finally { + bytes?.fill(0); + if (descriptor !== undefined) fs.closeSync(descriptor); + } +} + +function sealCompletion( + selected: Readonly, + uid: number, +): void { + validateDirectory(selected.root, uid, [0o700, 0o500], 'receipt directory'); + stableReceipt(selected, uid, [0o600, 0o400]); + let descriptor = fs.openSync( + selected.receipt, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + try { + const opened = fs.fstatSync(descriptor); + if ( + !opened.isFile() || + opened.uid !== uid || + opened.nlink !== 1 || + ![0o600, 0o400].includes(opened.mode & 0o777) + ) { + fail('receipt cannot be sealed'); + } + if ((opened.mode & 0o777) !== 0o400) { + fs.fchmodSync(descriptor, 0o400); + fs.fsyncSync(descriptor); + } + } finally { + fs.closeSync(descriptor); + } + const rootMode = validateDirectory( + selected.root, + uid, + [0o700, 0o500], + 'receipt directory', + ); + descriptor = fs.openSync(selected.root, fs.constants.O_RDONLY); + try { + if (rootMode !== 0o500) fs.fchmodSync(descriptor, 0o500); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + validateCatalog(selected, true); + stableReceipt(selected, uid, [0o400]); +} + +function validateApplication( + terminal: Readonly, + applicationId: string, + applicationPlanDigest: string, + deploymentRoot: string, + applicationRoot: string, +): void { + if ( + terminal.intent.command.request.applicationId !== applicationId || + terminal.plan.applicationId !== applicationId || + terminal.plan.applicationPlanDigest !== applicationPlanDigest || + terminal.intent.command.options.deploymentRoot !== deploymentRoot || + terminal.intent.command.options.applicationRoot !== applicationRoot + ) { + fail('application authority is detached'); + } +} + +async function automationProof( + command: Readonly, + terminal: Readonly, + uid: number, + dependencies: LocalReconciliationCompletionDependencies, +): Promise | null> { + const automationDomain = terminal.plan.domains.find( + (domain) => domain.domain === 'automation', + ); + if (!automationDomain) fail('automation domain is absent'); + if (automationDomain.action === 'no_effect') { + if ( + command.options.automation !== null || + command.request.automation !== null + ) { + fail('no-effect completion must not carry automation authority'); + } + return null; + } + if ( + automationDomain.action !== 'adapter_required' || + command.options.automation === null || + command.request.automation === null + ) { + fail('automation domain is not terminally provable'); + } + const options = command.options.automation; + const binding = command.request.automation; + for (const [directory, label] of [ + [options.automationRoot, 'automationRoot'], + [options.automationDecisionRoot, 'automationDecisionRoot'], + [options.automationApplyRoot, 'automationApplyRoot'], + ] as const) { + validatePrivateDirectory(directory, uid, label); + } + const decision = await readLocalReconciliationAutomationDecisionTerminal( + { + deploymentRoot: command.options.deploymentRoot, + applicationRoot: command.options.applicationRoot, + automationRoot: options.automationRoot, + automationDecisionRoot: options.automationDecisionRoot, + allowRootService: command.options.allowRootService, + }, + binding.automationId, + uid, + ); + if ( + decision.receipt.decisionId !== binding.decisionId || + decision.context.application.plan.applicationPlanDigest !== + terminal.plan.applicationPlanDigest + ) { + fail('automation decision is detached from application authority'); + } + const selected = localReconciliationAutomationApplyPaths( + options.automationApplyRoot, + binding.automationId, + ); + validateLocalReconciliationAutomationApplyLayout(selected, uid); + validateLocalReconciliationAutomationApplyCatalog(selected); + const intent = readLocalReconciliationAutomationApplyIntent(selected, uid); + const receipt = readLocalReconciliationAutomationApplyReceipt(selected, uid); + if ( + intent.command.options.deploymentRoot !== command.options.deploymentRoot || + intent.command.options.applicationRoot !== + command.options.applicationRoot || + intent.command.options.automationRoot !== options.automationRoot || + intent.command.options.automationDecisionRoot !== + options.automationDecisionRoot || + intent.command.options.automationApplyRoot !== + options.automationApplyRoot || + intent.command.options.targetDatabasePath !== options.targetDatabasePath || + intent.command.request.automationId !== binding.automationId || + intent.command.request.decisionId !== binding.decisionId || + receipt.automationId !== binding.automationId || + receipt.decisionId !== binding.decisionId || + receipt.applyDigest !== binding.expectedApplyDigest || + receipt.preparationDigest !== intent.preparationDigest + ) { + fail('automation apply evidence is detached'); + } + const current = await ( + dependencies.inspectSnapshot ?? inspectLocalSqliteSnapshot + )({ + databasePath: options.targetDatabasePath, + profile: intent.profile, + }); + if (current.sha256 !== receipt.targetAfter.sha256) { + fail('automation target drifted after apply'); + } + return Object.freeze({ intent, receipt, paths: selected }); +} + +function domainEvidence( + terminal: Readonly, + automation: Readonly | null, +): readonly Readonly[] { + return Object.freeze( + terminal.plan.domains.map((domain) => { + if (domain.action === 'no_effect') { + return Object.freeze({ + domain: domain.domain, + action: 'no_effect' as const, + evidenceKind: 'application_summary' as const, + evidenceDigest: domain.summaryDigest, + }); + } + if ( + domain.domain === 'automation' && + domain.action === 'adapter_required' && + automation !== null + ) { + return Object.freeze({ + domain: domain.domain, + action: 'adapter_required' as const, + evidenceKind: 'automation_apply' as const, + evidenceDigest: automation.receipt.applyDigest, + }); + } + return fail(`${domain.domain} is not terminally reconciled`); + }), + ); +} + +function validateReceiptBinding( + receipt: Readonly, + terminal: Readonly, + expectedDomains: readonly Readonly[], + completionId: string, + applicationId: string, +): void { + if ( + receipt.completionId !== completionId || + receipt.applicationId !== applicationId || + receipt.profile !== terminal.intent.profile || + receipt.instanceId !== terminal.intent.instanceId || + receipt.cutoverId !== terminal.intent.cutoverId || + receipt.generation !== terminal.intent.generation || + receipt.activationDigest !== terminal.intent.activationDigest || + receipt.applicationPlanDigest !== terminal.plan.applicationPlanDigest || + JSON.stringify(receipt.domains) !== JSON.stringify(expectedDomains) + ) { + fail('receipt is detached from terminal domain evidence'); + } +} + +function advanceCompletedHead( + terminal: Readonly, + receipt: Readonly, + uid: number, +): Readonly { + return advanceLocalCutoverInstanceHead( + { + options: { + deploymentRoot: terminal.intent.command.options.deploymentRoot, + }, + request: { + cutoverId: terminal.intent.cutoverId, + profile: terminal.intent.profile, + instanceId: terminal.intent.instanceId, + expectedActivationDigest: terminal.intent.activationDigest, + requestedAtMs: receipt.completedAtMs, + }, + }, + uid, + 'reconciliation_completed', + terminal.intent.generation, + receipt.completionDigest, + ); +} + +function result( + operation: LocalReconciliationCompletionResult['operation'], + status: LocalReconciliationCompletionResult['status'], + receipt: Readonly, + head: Readonly, +): Readonly { + return Object.freeze({ + schemaVersion: 1, + operation, + status, + state: 'reconciliation_completed', + completionId: receipt.completionId, + applicationId: receipt.applicationId, + completionDigest: receipt.completionDigest, + domainCount: 8, + adapterCount: receipt.adapterCount, + instanceHeadDigest: head.headDigest, + }); +} + +function assertSourceHead( + head: Readonly, + expectedHeadDigest: string, + automation: Readonly | null, +): void { + const expectedState = + automation === null + ? 'reconciliation_application_planned' + : 'reconciliation_automation_applied'; + const expectedSource = + automation === null ? undefined : automation.receipt.applyDigest; + if ( + head.headDigest !== expectedHeadDigest || + head.state !== expectedState || + (expectedSource !== undefined && head.sourceRecordDigest !== expectedSource) + ) { + fail('completion lost source head compare-and-swap'); + } +} + +export async function completeLocalReconciliation( + value: unknown, + dependencies: LocalReconciliationCompletionDependencies = {}, +): Promise> { + const command = normalizeLocalReconciliationCompleteCommand(value); + const uid = currentIdentity().uid; + for (const [directory, label] of [ + [command.options.deploymentRoot, 'deploymentRoot'], + [command.options.applicationRoot, 'applicationRoot'], + [command.options.completionRoot, 'completionRoot'], + ] as const) { + validatePrivateDirectory(directory, uid, label); + } + const terminal = await readLocalReconciliationApplicationTerminal( + command.options.applicationRoot, + command.request.applicationId, + uid, + ); + validateApplication( + terminal, + command.request.applicationId, + command.request.expectedApplicationPlanDigest, + command.options.deploymentRoot, + command.options.applicationRoot, + ); + const automation = await automationProof( + command, + terminal, + uid, + dependencies, + ); + const domains = domainEvidence(terminal, automation); + const selected = ensureCompletionDirectory( + command.options.completionRoot, + command.request.completionId, + uid, + ); + let receipt: Readonly; + let status: 'completed' | 'existing' = 'completed'; + let head = readLocalCutoverInstanceHead( + command.options.deploymentRoot, + terminal.intent.instanceId, + uid, + ); + if (fs.existsSync(selected.receipt)) { + status = 'existing'; + receipt = stableReceipt(selected, uid, [0o600, 0o400]); + validateReceiptBinding( + receipt, + terminal, + domains, + command.request.completionId, + command.request.applicationId, + ); + if ( + receipt.sourceHeadDigest !== command.request.expectedHeadDigest || + receipt.completedAtMs !== command.request.completedAtMs + ) { + fail('completion command is not an exact replay'); + } + } else { + assertSourceHead(head, command.request.expectedHeadDigest, automation); + const adapterCount = domains.filter( + (domain) => domain.action === 'adapter_required', + ).length as 0 | 1; + const latestEvidenceAtMs = Math.max( + terminal.plan.committedAtMs, + automation?.receipt.appliedAtMs ?? 0, + ); + if (command.request.completedAtMs < latestEvidenceAtMs) { + fail('completion timestamp precedes terminal evidence'); + } + receipt = buildLocalReconciliationCompletionReceipt({ + completionId: command.request.completionId, + applicationId: command.request.applicationId, + profile: terminal.intent.profile, + instanceId: terminal.intent.instanceId, + cutoverId: terminal.intent.cutoverId, + generation: terminal.intent.generation, + activationDigest: terminal.intent.activationDigest, + applicationPlanDigest: terminal.plan.applicationPlanDigest, + sourceHeadDigest: head.headDigest, + domains, + adapterCount, + completedAtMs: command.request.completedAtMs, + }); + const serialized = localReconciliationCompletionReceiptContents(receipt); + preflightPublishedFile( + selected.receipt, + serialized, + 0o600, + uid, + 'reconciliation completion receipt', + ); + publishExactFile( + selected.receipt, + serialized, + 0o600, + uid, + 'reconciliation completion receipt', + ); + dependencies.afterReceiptPublished?.(); + } + sealCompletion(selected, uid); + dependencies.afterTerminalSealed?.(); + head = readLocalCutoverInstanceHead( + command.options.deploymentRoot, + terminal.intent.instanceId, + uid, + ); + if (head.state !== 'reconciliation_completed') { + assertSourceHead(head, receipt.sourceHeadDigest, automation); + if (automation !== null) { + validateLocalReconciliationAutomationAppliedStorage( + automation.paths, + automation.intent, + uid, + ); + } + head = advanceCompletedHead(terminal, receipt, uid); + } else if (head.sourceRecordDigest !== receipt.completionDigest) { + fail('completed head is detached from receipt'); + } + dependencies.afterHeadAdvanced?.(); + if (automation !== null) { + collectLocalReconciliationAutomationCompletedStorage( + automation.paths, + automation.intent, + uid, + ); + dependencies.afterBackupCollected?.(); + } + return result(command.operation, status, receipt, head); +} + +export async function verifyLocalReconciliationCompletion( + value: unknown, + dependencies: LocalReconciliationCompletionDependencies = {}, +): Promise> { + const command = normalizeLocalReconciliationCompletionVerifyCommand(value); + const uid = currentIdentity().uid; + for (const [directory, label] of [ + [command.options.deploymentRoot, 'deploymentRoot'], + [command.options.applicationRoot, 'applicationRoot'], + [command.options.completionRoot, 'completionRoot'], + ] as const) { + validatePrivateDirectory(directory, uid, label); + } + const selected = completionPaths( + command.options.completionRoot, + command.request.completionId, + ); + validateDirectory(selected.root, uid, [0o500], 'receipt directory'); + validateCatalog(selected, true); + const receipt = stableReceipt(selected, uid, [0o400]); + if ( + receipt.applicationId !== command.request.applicationId || + receipt.completionDigest !== command.request.expectedCompletionDigest + ) { + fail('verify command is detached from receipt'); + } + const terminal = await readLocalReconciliationApplicationTerminal( + command.options.applicationRoot, + command.request.applicationId, + uid, + ); + validateApplication( + terminal, + command.request.applicationId, + receipt.applicationPlanDigest, + command.options.deploymentRoot, + command.options.applicationRoot, + ); + const syntheticCompleteCommand = Object.freeze({ + schemaVersion: 1 as const, + operation: 'local.deployment.reconciliation.complete' as const, + options: command.options, + request: Object.freeze({ + completionId: receipt.completionId, + applicationId: receipt.applicationId, + expectedApplicationPlanDigest: receipt.applicationPlanDigest, + expectedHeadDigest: receipt.sourceHeadDigest, + automation: command.request.automation, + completedAtMs: receipt.completedAtMs, + }), + }); + const automation = await automationProof( + syntheticCompleteCommand, + terminal, + uid, + dependencies, + ); + const domains = domainEvidence(terminal, automation); + validateReceiptBinding( + receipt, + terminal, + domains, + command.request.completionId, + command.request.applicationId, + ); + const head = readLocalCutoverInstanceHead( + command.options.deploymentRoot, + terminal.intent.instanceId, + uid, + ); + if ( + head.state !== 'reconciliation_completed' || + head.sourceRecordDigest !== receipt.completionDigest + ) { + fail('completion receipt is detached from instance head'); + } + if (automation !== null) { + validateLocalReconciliationAutomationCompletedStorage( + automation.paths, + uid, + ); + } + return result(command.operation, 'verified', receipt, head); +} + +export function completeLocalReconciliationCommandFile( + filePath: string, + dependencies: LocalReconciliationCompletionDependencies = {}, +) { + return completeLocalReconciliation( + readPrivateLocalCommandFile(filePath), + dependencies, + ); +} + +export function verifyLocalReconciliationCompletionCommandFile( + filePath: string, + dependencies: LocalReconciliationCompletionDependencies = {}, +) { + return verifyLocalReconciliationCompletion( + readPrivateLocalCommandFile(filePath), + dependencies, + ); +} + +export type { + LocalReconciliationCompletionOptions, + LocalReconciliationCompletionVerifyCommand, +}; diff --git a/packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/evidence.ts b/packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/evidence.ts new file mode 100644 index 00000000..6b971e91 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/evidence.ts @@ -0,0 +1,195 @@ +import { LocalDeploymentConfigurationError } from '../../foundation/error'; +import { cutoverDigest } from '../../cutover/targetEvidence'; +import { + LOCAL_RECONCILIATION_PLAN_DOMAINS, + type LocalReconciliationPlanDomain, +} from '../planning/contract'; + +const RECEIPT_SCHEMA = 'qinglong3-local-reconciliation-completion-receipt'; +const DIGEST = /^[0-9a-f]{64}$/; +const UUID_V4 = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +export interface LocalReconciliationCompletionDomainEvidence { + readonly domain: LocalReconciliationPlanDomain; + readonly action: 'no_effect' | 'adapter_required'; + readonly evidenceKind: 'application_summary' | 'automation_apply'; + readonly evidenceDigest: string; +} + +export interface LocalReconciliationCompletionReceipt { + readonly schema: typeof RECEIPT_SCHEMA; + readonly schemaVersion: 1; + readonly state: 'reconciliation_completed'; + readonly completionId: string; + readonly applicationId: string; + readonly profile: 'edge' | 'standalone'; + readonly instanceId: string; + readonly cutoverId: string; + readonly generation: number; + readonly activationDigest: string; + readonly applicationPlanDigest: string; + readonly sourceHeadDigest: string; + readonly domains: readonly Readonly[]; + readonly adapterCount: 0 | 1; + readonly completedAtMs: number; + readonly completionDigest: string; +} + +function fail(message: string): never { + throw new LocalDeploymentConfigurationError( + `reconciliation completion evidence ${message}`, + ); +} + +function exact( + value: unknown, + keys: readonly string[], + label: string, +): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + fail(`${label} must be an object`); + } + const selected = value as Record; + const actual = Object.keys(selected).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + fail(`${label} shape is invalid`); + } + return selected; +} + +function domainEvidence( + value: unknown, + expectedDomain: LocalReconciliationPlanDomain, +): Readonly { + const selected = exact( + value, + ['action', 'domain', 'evidenceDigest', 'evidenceKind'], + 'domain evidence', + ); + const noEffect = + selected.action === 'no_effect' && + selected.evidenceKind === 'application_summary'; + const automation = + expectedDomain === 'automation' && + selected.action === 'adapter_required' && + selected.evidenceKind === 'automation_apply'; + if ( + selected.domain !== expectedDomain || + (!noEffect && !automation) || + typeof selected.evidenceDigest !== 'string' || + !DIGEST.test(selected.evidenceDigest) + ) { + fail('domain evidence is invalid'); + } + return Object.freeze({ + domain: expectedDomain, + action: selected.action, + evidenceKind: selected.evidenceKind, + evidenceDigest: selected.evidenceDigest, + }) as Readonly; +} + +export function buildLocalReconciliationCompletionReceipt( + input: Omit< + LocalReconciliationCompletionReceipt, + 'schema' | 'schemaVersion' | 'state' | 'completionDigest' + >, +): Readonly { + const payload = Object.freeze({ + schema: RECEIPT_SCHEMA, + schemaVersion: 1 as const, + state: 'reconciliation_completed' as const, + ...input, + }); + return Object.freeze({ + ...payload, + completionDigest: cutoverDigest(payload), + }); +} + +export function normalizeLocalReconciliationCompletionReceipt( + value: unknown, +): Readonly { + const selected = exact( + value, + [ + 'activationDigest', + 'adapterCount', + 'applicationId', + 'applicationPlanDigest', + 'completedAtMs', + 'completionDigest', + 'completionId', + 'cutoverId', + 'domains', + 'generation', + 'instanceId', + 'profile', + 'schema', + 'schemaVersion', + 'sourceHeadDigest', + 'state', + ], + 'receipt', + ); + if (!Array.isArray(selected.domains) || selected.domains.length !== 8) { + fail('receipt domain catalog is invalid'); + } + const rawDomains = selected.domains as unknown[]; + const domains = Object.freeze( + LOCAL_RECONCILIATION_PLAN_DOMAINS.map((domain, index) => + domainEvidence(rawDomains[index], domain), + ), + ); + const adapterCount = domains.filter( + (domain) => domain.action === 'adapter_required', + ).length; + const { completionDigest, ...raw } = selected; + const normalized = Object.freeze({ ...raw, domains }); + if ( + selected.schema !== RECEIPT_SCHEMA || + selected.schemaVersion !== 1 || + selected.state !== 'reconciliation_completed' || + typeof selected.completionId !== 'string' || + !UUID_V4.test(selected.completionId) || + typeof selected.applicationId !== 'string' || + !UUID_V4.test(selected.applicationId) || + (selected.profile !== 'edge' && selected.profile !== 'standalone') || + typeof selected.instanceId !== 'string' || + selected.instanceId.length < 1 || + typeof selected.cutoverId !== 'string' || + selected.cutoverId.length < 1 || + !Number.isSafeInteger(selected.generation) || + (selected.generation as number) < 1 || + ![ + selected.activationDigest, + selected.applicationPlanDigest, + selected.sourceHeadDigest, + completionDigest, + ].every( + (candidate) => typeof candidate === 'string' && DIGEST.test(candidate), + ) || + (selected.adapterCount !== 0 && selected.adapterCount !== 1) || + selected.adapterCount !== adapterCount || + !Number.isSafeInteger(selected.completedAtMs) || + (selected.completedAtMs as number) < 0 || + cutoverDigest(normalized) !== completionDigest + ) { + fail('receipt binding is invalid'); + } + return Object.freeze({ + ...normalized, + completionDigest, + }) as unknown as Readonly; +} + +export function localReconciliationCompletionReceiptContents( + receipt: Readonly, +): string { + return `${JSON.stringify(receipt, null, 2)}\n`; +} diff --git a/packages/ql3-local-owner-cli/test/reconciliationCapturePrepare.test.cjs b/packages/ql3-local-owner-cli/test/reconciliationCapturePrepare.test.cjs index aa215dcb..53f0adaf 100644 --- a/packages/ql3-local-owner-cli/test/reconciliationCapturePrepare.test.cjs +++ b/packages/ql3-local-owner-cli/test/reconciliationCapturePrepare.test.cjs @@ -11,6 +11,7 @@ const { commitLocalReconciliationCapture, commitLocalReconciliationApplication, commitLocalReconciliationAutomationDecision, + completeLocalReconciliation, applyLocalReconciliationAutomation, commitLocalReconciliationPlan, commitLocalReconciliationReview, @@ -27,6 +28,7 @@ const { verifyLocalReconciliationAutomationDecision, verifyLocalReconciliationAutomationApply, verifyLocalReconciliationAutomationPlan, + verifyLocalReconciliationCompletion, verifyLocalReconciliationPlan, verifyLocalReconciliationReview, writeLocalReconciliationReviewDiagnostics, @@ -42,6 +44,7 @@ const { } = require('@qinglong/local-sqlite/data-directory-application-commit'); const { advanceLocalCutoverInstanceHead, + assertLocalCutoverTargetHead, claimLocalCutoverInstance, readLocalCutoverInstanceHead, } = require('../dist/deployment/cutover/instanceLineage.js'); @@ -1356,6 +1359,108 @@ function automationDecisionCommitFixture( }; } +async function appliedAutomationFixture(t, options = {}) { + const state = await plannedAutomationFixture(t, { + suffix: options.suffix ?? 'completion-applied', + planId: options.planId ?? '00000000-0000-4000-8000-000000000481', + reviewId: options.reviewId ?? '00000000-0000-4000-8000-000000000482', + applicationId: + options.applicationId ?? '00000000-0000-4000-8000-000000000483', + automationId: + options.automationId ?? '00000000-0000-4000-8000-000000000484', + readyTarget: true, + }); + assert.equal(state.application.outcome, 'adapter_and_manual_required'); + const decisionId = + options.decisionId ?? '019b0000-0000-7000-8000-000000000481'; + const review = automationDecisionReviewFile( + state, + decisionId, + 'adopt', + 'reviewed_lossless', + options.suffix ?? 'completion-applied', + ); + const prepareCommand = automationDecisionPrepareCommand(state, decisionId); + const prepared = await prepareLocalReconciliationAutomationDecision( + prepareCommand, + ); + const commit = automationDecisionCommitFixture( + state, + { result: prepared, commandOptions: prepareCommand.options }, + review.filePath, + ); + const decision = await commitLocalReconciliationAutomationDecision( + commit.command, + commit.dependencies, + ); + const automationApplyRoot = path.join( + path.dirname(state.captureRoot), + `automation-apply-${options.suffix ?? 'completion-applied'}`, + ); + fs.mkdirSync(automationApplyRoot, { mode: 0o700 }); + const applyOptions = { + ...prepareCommand.options, + automationApplyRoot, + targetDatabasePath: state.targetDatabasePath, + ownerPepperKeyringDirectory: + state.command.options.ownerPepperKeyringDirectory, + credentialFilePath: state.command.options.credentialFilePath, + }; + const appliedAtMs = commit.command.request.committedAtMs + 1; + const applyCommand = { + schemaVersion: 1, + operation: 'local.deployment.reconciliation.automation.apply', + options: applyOptions, + request: { + decisionId, + automationId: state.automationCommand.request.automationId, + expectedDecisionDigest: decision.decisionDigest, + expectedHeadDigest: decision.instanceHeadDigest, + mutationId: options.mutationId ?? '00000000-0000-4000-8000-000000000485', + requestId: `automation-apply-${options.suffix ?? 'completion-applied'}`, + appliedAtMs, + }, + }; + const applyDependencies = { + async openAuthenticationDatabase() { + return { async close() {} }; + }, + async authenticate(_database, authenticateOptions) { + const authenticatedAtMs = authenticateOptions.now(); + return { + principal: { + subject: { type: 'user', id: 'review-owner' }, + authenticationId: 'local_reconciliation_automation_apply:test', + authenticatedAtMs, + expiresAtMs: authenticatedAtMs + 60 * 60 * 1_000, + assurance: 'local_console', + }, + databaseFence: { + credentialId: 'review-owner', + credentialVersion: 1, + pepperKeyId: 'review-owner-v1', + pepperVersion: 1, + }, + async confirm() {}, + }; + }, + }; + const applied = await applyLocalReconciliationAutomation( + applyCommand, + applyDependencies, + ); + return { + ...state, + decisionId, + decision, + automationApplyRoot, + applyOptions, + applyCommand, + applyDependencies, + applied, + }; +} + function dockerReadSealedSqlite(assetsDirectory, mode) { const source = mode === 'main_only_immutable' @@ -2908,6 +3013,237 @@ test('application coordinator plans eight content-free domains and verifies with assert.equal(cli.stdout.includes('Crontabs'), false); }); +test('completion fence seals all eight no-effect domains before target restart', async (t) => { + const state = await reviewedApplicationFixture(t, { + planId: '00000000-0000-4000-8000-000000000421', + reviewId: '00000000-0000-4000-8000-000000000422', + applicationId: '00000000-0000-4000-8000-000000000423', + reviewSuffix: 'completion-no-effect', + createDefaultSidecars: false, + initializeDatabases: automationDatabaseInitializer(), + mutateTarget(paths) { + return mutateAutomationTarget(paths); + }, + }); + const prepared = await prepareLocalReconciliationApplication( + state.prepareApplicationCommand, + ); + const application = await commitLocalReconciliationApplication( + applicationCommitCommand(state, prepared), + ); + assert.equal(application.outcome, 'no_effect_ready'); + const completionRoot = path.join( + path.dirname(state.captureRoot), + 'completion-no-effect', + ); + fs.mkdirSync(completionRoot, { mode: 0o700 }); + const command = { + schemaVersion: 1, + operation: 'local.deployment.reconciliation.complete', + options: { + deploymentRoot: state.deploymentRoot, + applicationRoot: state.applicationRoot, + completionRoot, + automation: null, + allowRootService: rootAcknowledgement(), + }, + request: { + completionId: '00000000-0000-4000-8000-000000000424', + applicationId: state.prepareApplicationCommand.request.applicationId, + expectedApplicationPlanDigest: application.applicationPlanDigest, + expectedHeadDigest: application.instanceHeadDigest, + automation: null, + completedAtMs: state.prepareApplicationCommand.request.preparedAtMs + 4, + }, + }; + for (const boundary of [ + 'afterReceiptPublished', + 'afterTerminalSealed', + 'afterHeadAdvanced', + ]) { + await assert.rejects( + completeLocalReconciliation(command, { + [boundary]() { + throw new Error(`completion ${boundary} response loss`); + }, + }), + new RegExp(`completion ${boundary} response loss`), + ); + } + const completed = await completeLocalReconciliation(command); + assert.equal(completed.status, 'existing'); + assert.equal(completed.state, 'reconciliation_completed'); + assert.equal(completed.domainCount, 8); + assert.equal(completed.adapterCount, 0); + const completionDirectory = path.join( + completionRoot, + command.request.completionId, + ); + assert.deepEqual(fs.readdirSync(completionDirectory), ['receipt.json']); + assert.equal(fs.statSync(completionDirectory).mode & 0o777, 0o500); + assert.equal( + fs.statSync(path.join(completionDirectory, 'receipt.json')).mode & 0o777, + 0o400, + ); + const receipt = JSON.parse( + fs.readFileSync(path.join(completionDirectory, 'receipt.json'), 'utf8'), + ); + assert.equal(receipt.domains.length, 8); + assert.equal( + receipt.domains.every( + (domain) => + domain.action === 'no_effect' && + domain.evidenceKind === 'application_summary', + ), + true, + ); + const verifyCommand = { + schemaVersion: 1, + operation: 'local.deployment.reconciliation.complete.verify', + options: command.options, + request: { + completionId: command.request.completionId, + applicationId: command.request.applicationId, + expectedCompletionDigest: completed.completionDigest, + automation: null, + }, + }; + const verified = await verifyLocalReconciliationCompletion(verifyCommand); + assert.equal(verified.status, 'verified'); + const commandPath = path.join(state.deploymentRoot, 'completion-verify.json'); + fs.writeFileSync(commandPath, `${JSON.stringify(verifyCommand)}\n`, { + mode: 0o600, + }); + const cli = spawnSync( + process.execPath, + [ + path.join(__dirname, '../dist/deployment/localDeploymentCli.js'), + 'reconciliation-complete-verify', + '--command-file', + commandPath, + ], + { encoding: 'utf8' }, + ); + assert.equal(cli.status, 0, cli.stderr); + assert.equal(JSON.parse(cli.stdout).status, 'verified'); + assert.equal(cli.stdout.includes(completionRoot), false); + const identity = { + options: { deploymentRoot: state.deploymentRoot }, + request: { + cutoverId: state.captureCommand.request.cutoverId, + profile: state.captureCommand.request.profile, + instanceId: state.captureCommand.request.instanceId, + expectedActivationDigest: + state.captureCommand.request.expectedActivationDigest, + requestedAtMs: command.request.completedAtMs + 1, + }, + }; + const restartHead = assertLocalCutoverTargetHead(identity, state.uid); + assert.equal(restartHead.state, 'reconciliation_completed'); + const activeHead = advanceLocalCutoverInstanceHead( + identity, + state.uid, + 'target_active', + 2, + 'e'.repeat(64), + ); + assert.equal(activeHead.state, 'target_active'); + assert.equal(activeHead.generation, 2); +}); + +test('completion fence retains automation rollback backup while other domains remain manual', async (t) => { + const state = await appliedAutomationFixture(t, { + suffix: 'completion-fence', + planId: '00000000-0000-4000-8000-000000000491', + reviewId: '00000000-0000-4000-8000-000000000492', + applicationId: '00000000-0000-4000-8000-000000000493', + automationId: '00000000-0000-4000-8000-000000000494', + decisionId: '019b0000-0000-7000-8000-000000000491', + mutationId: '00000000-0000-4000-8000-000000000495', + }); + const completionRoot = path.join( + path.dirname(state.captureRoot), + 'completion-automation-applied', + ); + fs.mkdirSync(completionRoot, { mode: 0o700 }); + const automation = { + automationId: state.automationCommand.request.automationId, + decisionId: state.decisionId, + expectedApplyDigest: state.applied.applyDigest, + }; + const completionOptions = { + deploymentRoot: state.deploymentRoot, + applicationRoot: state.applicationRoot, + completionRoot, + automation: { + automationRoot: state.automationRoot, + automationDecisionRoot: state.automationDecisionRoot, + automationApplyRoot: state.automationApplyRoot, + targetDatabasePath: state.targetDatabasePath, + }, + allowRootService: rootAcknowledgement(), + }; + const command = { + schemaVersion: 1, + operation: 'local.deployment.reconciliation.complete', + options: completionOptions, + request: { + completionId: '00000000-0000-4000-8000-000000000496', + applicationId: state.application.applicationId, + expectedApplicationPlanDigest: state.application.applicationPlanDigest, + expectedHeadDigest: state.applied.instanceHeadDigest, + automation, + completedAtMs: state.applyCommand.request.appliedAtMs + 1, + }, + }; + const applyRoot = path.join( + state.automationApplyRoot, + state.automationCommand.request.automationId, + ); + const backupRoot = path.join(applyRoot, 'backup'); + const backupPath = path.join(backupRoot, 'before.sqlite'); + const rollbackRoot = path.join(applyRoot, 'rollback-work'); + assert.equal(fs.existsSync(backupPath), true); + await assert.rejects( + completeLocalReconciliation(command), + /secret_and_config is not terminally reconciled/, + ); + assert.equal(fs.existsSync(backupPath), true); + assert.deepEqual(fs.readdirSync(backupRoot), ['before.sqlite']); + assert.deepEqual(fs.readdirSync(rollbackRoot), []); + assert.equal(fs.statSync(backupRoot).mode & 0o777, 0o500); + assert.equal(fs.statSync(rollbackRoot).mode & 0o777, 0o700); + assert.equal( + fs.existsSync(path.join(completionRoot, command.request.completionId)), + false, + ); + const head = readLocalCutoverInstanceHead( + state.deploymentRoot, + state.captureCommand.request.instanceId, + state.uid, + ); + assert.equal(head.state, 'reconciliation_automation_applied'); + assert.equal(head.sourceRecordDigest, state.applied.applyDigest); + assert.throws( + () => + assertLocalCutoverTargetHead( + { + options: { deploymentRoot: state.deploymentRoot }, + request: { + cutoverId: state.captureCommand.request.cutoverId, + profile: state.captureCommand.request.profile, + instanceId: state.captureCommand.request.instanceId, + expectedActivationDigest: + state.captureCommand.request.expectedActivationDigest, + requestedAtMs: command.request.completedAtMs + 1, + }, + }, + state.uid, + ), + /not bound to the instance lineage head/, + ); +}); + test('automation adapter builds a sealed row plan with bounded conflict evidence', async (t) => { const state = await reviewedApplicationFixture(t, { planId: '00000000-0000-4000-8000-000000000431', diff --git a/scripts/ql3-cluster-dependency-audit.cjs b/scripts/ql3-cluster-dependency-audit.cjs index 9aa7829d..1033ea47 100644 --- a/scripts/ql3-cluster-dependency-audit.cjs +++ b/scripts/ql3-cluster-dependency-audit.cjs @@ -2042,6 +2042,11 @@ function auditSourceImports(root, packagePath, findings) { 'src/deployment/reconciliation/application/automation/applyEvidence.ts' && specifier === '@qinglong/local-sqlite/rollout-safety' ) && + !( + path.relative(packageDirectory, filePath) === + 'src/deployment/reconciliation/completion/coordinator.ts' && + specifier === '@qinglong/local-sqlite/rollout-safety' + ) && !( path.relative(packageDirectory, filePath) === 'src/deployment/reconciliation/application/automation/planReader.ts' && diff --git a/test/back/ql3ClusterDependencyAudit.test.cjs b/test/back/ql3ClusterDependencyAudit.test.cjs index 34097a67..feb019f3 100644 --- a/test/back/ql3ClusterDependencyAudit.test.cjs +++ b/test/back/ql3ClusterDependencyAudit.test.cjs @@ -2158,7 +2158,12 @@ test('confines reconciliation automation apply authority to exact coordinators', root, 'packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation', ); + const completionDirectory = path.join( + root, + 'packages/ql3-local-owner-cli/src/deployment/reconciliation/completion', + ); fs.mkdirSync(automationDirectory, { recursive: true }); + fs.mkdirSync(completionDirectory, { recursive: true }); fs.writeFileSync( path.join(automationDirectory, 'decisionCoordinator.ts'), [ @@ -2182,6 +2187,14 @@ test('confines reconciliation automation apply authority to exact coordinators', path.join(automationDirectory, 'applyEvidence.ts'), "import type { Evidence } from '@qinglong/local-sqlite/rollout-safety';", ); + fs.writeFileSync( + path.join(completionDirectory, 'coordinator.ts'), + "import { inspect } from '@qinglong/local-sqlite/rollout-safety';", + ); + fs.writeFileSync( + path.join(completionDirectory, 'neighbor.ts'), + "import { backup } from '@qinglong/local-sqlite/rollout-safety';", + ); fs.writeFileSync( path.join(automationDirectory, 'neighbor.ts'), [ @@ -2224,6 +2237,11 @@ test('confines reconciliation automation apply authority to exact coordinators', file: 'packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/neighbor.ts', specifier: '@qinglong/runtime-core/security', }, + { + code: 'FORBIDDEN_LOCAL_ADOPTION_CLI_AUTHORITY_IMPORT', + file: 'packages/ql3-local-owner-cli/src/deployment/reconciliation/completion/neighbor.ts', + specifier: '@qinglong/local-sqlite/rollout-safety', + }, ], ); }); diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index 45b01579..6083f3ef 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -207,10 +207,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: localOwnerCli.rootSourceFileRoles, }, { - sourceFiles: 169, + sourceFiles: 172, rootSourceFiles: 1, rootSourceLines: 50, - nestedSourceFiles: 168, + nestedSourceFiles: 171, rootSourceFileRoles: { 'cli.ts': 'binary_entry' }, }, );