From 546b02a0bc0e010c98f76e45ed351e2b0316da8e Mon Sep 17 00:00:00 2001 From: whyour Date: Sat, 22 Aug 2026 20:41:54 +0800 Subject: [PATCH] feat(local): recover reconciliation automation apply --- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 24 + .../src/deployment/cutover/instanceLineage.ts | 18 + .../src/deployment/localDeployment.ts | 36 +- .../src/deployment/localDeploymentCli.ts | 14 +- .../application/automation/applyContract.ts | 380 +++++++ .../automation/applyCoordinator.ts | 927 ++++++++++++++++++ .../application/automation/applyEvidence.ts | 351 +++++++ .../automation/decisionCoordinator.ts | 115 ++- .../reconciliationCapturePrepare.test.cjs | 357 ++++++- test/back/ql3PackageBoundaryAudit.test.cjs | 4 +- 10 files changed, 2181 insertions(+), 45 deletions(-) create mode 100644 packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyContract.ts create mode 100644 packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyCoordinator.ts create mode 100644 packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyEvidence.ts diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 1a4cd568..3b3b0657 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -55,6 +55,30 @@ signed decision，完成 target 写前可恢复 backup 证明、Project Policy/credential 复验、幂等原子 transaction/replay、post-apply evidence 与显式 rollback。当前状态仍不代表 applied、restart-ready 或 reconciliation-complete。进入 apply 写语义时必须重新选择 SQLite/Docker 数据门禁，不能 复用本切片只读证明。 + + 第三切片已消费上述 signed decision 并建立首个真实写入闭环，而没有新增第二套 Task/Trigger DML。Local Admin 的 plan-bound verifier 现在只把 + 已验证的 decision scope 交给既有 `LocalSqliteLegacyAdoptionPublisher`；后者继续以 `BEGIN IMMEDIATE`、Project/RoleBinding version fence、 + mutation/decision exact replay、Task/Trigger collision 检查、security audit 与 adoption ledger 作为唯一写入 authority。Local Owner 新增 + `reconciliation-automation-apply|apply-verify|apply-rollback`，实例 lineage 严格前进为 + `reconciliation_automation_reviewed → reconciliation_automation_apply_prepared → reconciliation_automation_applied → + reconciliation_automation_rolled_back`。apply 在 DML 前重新证明原 target 仍处于同一 stopped authority，直接复用 `local-sqlite/rollout-safety` + 的 online SQLite backup、readiness/snapshot audit 和 no-replace stage/link publication；backup、intent 与 prepared head 完成后，再重新打开 sealed + Legacy bundle、逐行 plan 和 `0400/0500` signed authorization，并把最多 5 分钟的当前强认证 User 与原 reviewer 精确匹配。事务提交前还会复验 + prepared head、sealed bundle、authorization inode、当前 credential 与 Project Policy fence。database commit、receipt、head 任一 response-loss + 窗口通过相同 mutation/decision ledger 收敛，不能重复插入 Task/Trigger。 + + apply receipt 绑定 publication digest、adopted Task/Trigger/skipped counters、写后完整 SQLite snapshot 和写前 backup；verify 同时检查 signed + decision、instance head 与当前 target snapshot，stdout 不返回 reviewer、路径、row digest 或 command。rollback 不是裸文件覆盖：它先以当前 reviewer + 强认证和当前 Project Policy 重放已存在 publication，再要求 target 精确等于 apply snapshot，最后调用 `preserveDatabaseIdentity=true` 的 restore， + 保持 activation 所绑定的 SQLite inode。restore、rollback receipt 和 rollback head 的 response-loss 同样可恢复；可恢复 backup authority 因仍需支持 + apply 后显式 rollback，保持 current-UID `0700/0600`，不伪装成已经归档的 `0500/0400` terminal。聚焦 reconciliation 套件仍为 + `44 total / 42 pass / 2 conditional Docker skip / 0 fail`，但成功场景现在额外覆盖首次 apply、mutation replay、不同 reviewer 拒绝、六个 + backup/apply/rollback response-loss 窗口、content-free CLI verify、identity-preserving rollback 与 rollback replay。完整 Local Owner 在受限沙箱为 + `266 total / 256 pass / 7 conditional skip / 3 loopback-listen EPERM`，两个对应文件在沙箱外 `15/15` 通过；package/dependency boundary 为 + `69/69`。workspace 仍为 18 packages、`singleSourcePackages=[]`、`shallowSourcePackages=[]`，Local Owner 为 + `168 source / 167 nested / 1 root binary entry`，没有新增 package、production dependency、daemon、listener、timer 或 `src/` 根平铺。 + D-393 仍未代表完整 reconciliation：Automation 之外的 Secret、Plugin、Identity、history 等领域 adapter 尚未 apply，target service 也未获得 restart + authority；后续必须先完成 apply-root 的 retention/seal 策略和跨领域 completion fence，再进入 target restart/readiness。 - D-392/ADR-0485（已接受）：D-391 的 signed review 不能直接获得通用 DML authority；表级 `adopt_legacy/retain_both` 也不能证明 Automation 行级 command/trigger 兼容，更不能覆盖 Secret custody、append-only history、Plugin/AI 外部资产与 Identity/Policy 语义。 因此既有 Local Owner 新增 `reconciliation.application.prepare|commit|verify`，以 diff --git a/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts b/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts index 118025f9..df8871bf 100644 --- a/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts +++ b/packages/ql3-local-owner-cli/src/deployment/cutover/instanceLineage.ts @@ -34,6 +34,9 @@ export type LocalCutoverInstanceHeadState = | 'reconciliation_automation_planned' | 'reconciliation_automation_decision_prepared' | 'reconciliation_automation_reviewed' + | 'reconciliation_automation_apply_prepared' + | 'reconciliation_automation_applied' + | 'reconciliation_automation_rolled_back' | 'rollback_prepared' | 'legacy_restart_requested' | 'legacy_running' @@ -176,6 +179,9 @@ function parseHead(value: unknown): Readonly { head.state !== 'reconciliation_automation_planned' && head.state !== 'reconciliation_automation_decision_prepared' && head.state !== 'reconciliation_automation_reviewed' && + head.state !== 'reconciliation_automation_apply_prepared' && + head.state !== 'reconciliation_automation_applied' && + head.state !== 'reconciliation_automation_rolled_back' && head.state !== 'rollback_prepared' && head.state !== 'legacy_restart_requested' && head.state !== 'legacy_running' && @@ -353,6 +359,9 @@ export function advanceLocalCutoverInstanceHead( | 'reconciliation_automation_planned' | 'reconciliation_automation_decision_prepared' | 'reconciliation_automation_reviewed' + | 'reconciliation_automation_apply_prepared' + | 'reconciliation_automation_applied' + | 'reconciliation_automation_rolled_back' | 'rollback_prepared' | 'legacy_restart_requested' | 'legacy_running' @@ -402,6 +411,9 @@ export function advanceLocalCutoverInstanceHead( current.state === 'reconciliation_automation_planned' || current.state === 'reconciliation_automation_decision_prepared' || current.state === 'reconciliation_automation_reviewed' || + current.state === 'reconciliation_automation_apply_prepared' || + current.state === 'reconciliation_automation_applied' || + current.state === 'reconciliation_automation_rolled_back' || current.state === 'legacy_restart_requested' || current.state === 'legacy_running' || current.state === 'legacy_ready') @@ -439,6 +451,12 @@ export function advanceLocalCutoverInstanceHead( current.state === 'reconciliation_automation_planned') || (state === 'reconciliation_automation_reviewed' && current.state === 'reconciliation_automation_decision_prepared') || + (state === 'reconciliation_automation_apply_prepared' && + current.state === 'reconciliation_automation_reviewed') || + (state === 'reconciliation_automation_applied' && + current.state === 'reconciliation_automation_apply_prepared') || + (state === 'reconciliation_automation_rolled_back' && + current.state === 'reconciliation_automation_applied') || (state === 'rollback_prepared' && current.state === 'target_stopped') || (state === 'legacy_restart_requested' && current.state === 'rollback_prepared') || diff --git a/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts b/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts index 9a9785da..0e33ac37 100644 --- a/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts +++ b/packages/ql3-local-owner-cli/src/deployment/localDeployment.ts @@ -142,9 +142,18 @@ import { commitLocalReconciliationAutomationDecisionCommandFile, prepareLocalReconciliationAutomationDecision, prepareLocalReconciliationAutomationDecisionCommandFile, + readLocalReconciliationAutomationDecisionTerminal, verifyLocalReconciliationAutomationDecision, verifyLocalReconciliationAutomationDecisionCommandFile, } from './reconciliation/application/automation/decisionCoordinator'; +import { + applyLocalReconciliationAutomation, + applyLocalReconciliationAutomationCommandFile, + rollbackLocalReconciliationAutomationApply, + rollbackLocalReconciliationAutomationApplyCommandFile, + verifyLocalReconciliationAutomationApply, + verifyLocalReconciliationAutomationApplyCommandFile, +} from './reconciliation/application/automation/applyCoordinator'; export { commitLocalReconciliationPlan, @@ -173,12 +182,31 @@ export { verifyLocalReconciliationAutomationPlanCommandFile, prepareLocalReconciliationAutomationDecision, prepareLocalReconciliationAutomationDecisionCommandFile, + readLocalReconciliationAutomationDecisionTerminal, commitLocalReconciliationAutomationDecision, commitLocalReconciliationAutomationDecisionCommandFile, verifyLocalReconciliationAutomationDecision, verifyLocalReconciliationAutomationDecisionCommandFile, + applyLocalReconciliationAutomation, + applyLocalReconciliationAutomationCommandFile, + verifyLocalReconciliationAutomationApply, + verifyLocalReconciliationAutomationApplyCommandFile, + rollbackLocalReconciliationAutomationApply, + rollbackLocalReconciliationAutomationApplyCommandFile, }; +export { + normalizeLocalReconciliationAutomationApplyCommand, + normalizeLocalReconciliationAutomationApplyRollbackCommand, + normalizeLocalReconciliationAutomationApplyVerifyCommand, + type LocalReconciliationAutomationApplyCommand, + type LocalReconciliationAutomationApplyOptions, + type LocalReconciliationAutomationApplyResult, + type LocalReconciliationAutomationApplyRollbackCommand, + type LocalReconciliationAutomationApplyVerifyCommand, +} from './reconciliation/application/automation/applyContract'; +export { type LocalReconciliationAutomationApplyDependencies } from './reconciliation/application/automation/applyCoordinator'; + export { normalizeLocalReconciliationApplicationCommitCommand, normalizeLocalReconciliationApplicationPrepareCommand, @@ -214,9 +242,7 @@ export { type LocalReconciliationAutomationPlanResult, type LocalReconciliationAutomationVerifyCommand, } from './reconciliation/application/automation/contract'; -export { - type LocalReconciliationAutomationPlanDependencies, -} from './reconciliation/application/automation/coordinator'; +export { type LocalReconciliationAutomationPlanDependencies } from './reconciliation/application/automation/coordinator'; export { normalizeLocalReconciliationAutomationDecisionCommitCommand, normalizeLocalReconciliationAutomationDecisionPrepareCommand, @@ -229,9 +255,7 @@ export { type LocalReconciliationAutomationDecisionTerminalResult, type LocalReconciliationAutomationDecisionVerifyCommand, } from './reconciliation/application/automation/decisionContract'; -export { - type LocalReconciliationAutomationDecisionDependencies, -} from './reconciliation/application/automation/decisionCoordinator'; +export { type LocalReconciliationAutomationDecisionDependencies } from './reconciliation/application/automation/decisionCoordinator'; export { MAX_EDGE_LOCAL_RECONCILIATION_AUTOMATION_PLAN_BYTES, MAX_STANDALONE_LOCAL_RECONCILIATION_AUTOMATION_PLAN_BYTES, diff --git a/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts b/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts index 1700b8d9..c43c841c 100644 --- a/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts +++ b/packages/ql3-local-owner-cli/src/deployment/localDeploymentCli.ts @@ -30,6 +30,9 @@ import { prepareLocalReconciliationAutomationDecisionCommandFile, commitLocalReconciliationAutomationDecisionCommandFile, verifyLocalReconciliationAutomationDecisionCommandFile, + applyLocalReconciliationAutomationCommandFile, + verifyLocalReconciliationAutomationApplyCommandFile, + rollbackLocalReconciliationAutomationApplyCommandFile, writeLocalReconciliationReviewDiagnosticsCommandFile, prepareLocalDeploymentCommandFile, proveLocalDeploymentLegacyReadinessCommandFile, @@ -45,7 +48,7 @@ import { } from './localDeployment'; const USAGE = - 'Usage: ql3-local-deploy --command-file /absolute/private-command.json'; + 'Usage: ql3-local-deploy --command-file /absolute/private-command.json'; async function main(argv: readonly string[]): Promise { if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) { @@ -92,6 +95,9 @@ async function main(argv: readonly string[]): Promise { argv[0] !== 'reconciliation-automation-decision-prepare' && argv[0] !== 'reconciliation-automation-decision-commit' && argv[0] !== 'reconciliation-automation-decision-verify' && + argv[0] !== 'reconciliation-automation-apply' && + argv[0] !== 'reconciliation-automation-apply-verify' && + argv[0] !== 'reconciliation-automation-apply-rollback' && argv[0] !== 'compose-revision' && argv[0] !== 'compose-preflight' && argv[0] !== 'compose-apply' && @@ -198,6 +204,12 @@ async function main(argv: readonly string[]): Promise { ? commitLocalReconciliationAutomationDecisionCommandFile(argv[2]!) : argv[0] === 'reconciliation-automation-decision-verify' ? verifyLocalReconciliationAutomationDecisionCommandFile(argv[2]!) + : argv[0] === 'reconciliation-automation-apply' + ? applyLocalReconciliationAutomationCommandFile(argv[2]!) + : argv[0] === 'reconciliation-automation-apply-verify' + ? verifyLocalReconciliationAutomationApplyCommandFile(argv[2]!) + : argv[0] === 'reconciliation-automation-apply-rollback' + ? rollbackLocalReconciliationAutomationApplyCommandFile(argv[2]!) : argv[0] === 'compose-revision' ? switchLocalDeploymentComposeRevisionCommandFile(argv[2]!) : argv[0] === 'compose-preflight' diff --git a/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyContract.ts b/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyContract.ts new file mode 100644 index 00000000..4ffe7274 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyContract.ts @@ -0,0 +1,380 @@ +import path from 'node:path'; + +import { currentIdentity } from '../../../foundation/contract'; +import { LocalDeploymentConfigurationError } from '../../../foundation/error'; + +const DIGEST = /^[0-9a-f]{64}$/; +const UUID_V4 = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const UUID_V7 = + /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const SAFE_PATH = /^\/[A-Za-z0-9._/@-]+$/; +const SAFE_ID = /^[A-Za-z0-9._:@-]{1,256}$/; + +export interface LocalReconciliationAutomationApplyOptions { + readonly deploymentRoot: string; + readonly applicationRoot: string; + readonly automationRoot: string; + readonly automationDecisionRoot: string; + readonly automationApplyRoot: string; + readonly targetDatabasePath: string; + readonly ownerPepperKeyringDirectory: string; + readonly credentialFilePath: string; + readonly allowRootService: boolean; + readonly busyTimeoutMs?: number; +} + +export interface LocalReconciliationAutomationApplyCommand { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.reconciliation.automation.apply'; + readonly options: Readonly; + readonly request: Readonly<{ + decisionId: string; + automationId: string; + expectedDecisionDigest: string; + expectedHeadDigest: string; + mutationId: string; + requestId: string; + appliedAtMs: number; + }>; +} + +export interface LocalReconciliationAutomationApplyVerifyCommand { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.reconciliation.automation.apply.verify'; + readonly options: Readonly; + readonly request: Readonly<{ + decisionId: string; + automationId: string; + expectedApplyDigest: string; + }>; +} + +export interface LocalReconciliationAutomationApplyRollbackCommand { + readonly schemaVersion: 1; + readonly operation: 'local.deployment.reconciliation.automation.apply.rollback'; + readonly options: Readonly; + readonly request: Readonly<{ + decisionId: string; + automationId: string; + expectedApplyDigest: string; + expectedHeadDigest: string; + rolledBackAtMs: number; + }>; +} + +export interface LocalReconciliationAutomationApplyResult { + readonly schemaVersion: 1; + readonly operation: + | LocalReconciliationAutomationApplyCommand['operation'] + | LocalReconciliationAutomationApplyVerifyCommand['operation'] + | LocalReconciliationAutomationApplyRollbackCommand['operation']; + readonly status: 'applied' | 'existing' | 'verified' | 'rolled_back'; + readonly state: + | 'reconciliation_automation_applied' + | 'reconciliation_automation_rolled_back'; + readonly decisionId: string; + readonly automationId: string; + readonly applyDigest: string; + readonly publicationDigest: string; + readonly adoptedTaskCount: number; + readonly adoptedTriggerCount: number; + readonly backupSha256: string; + readonly instanceHeadDigest: string; +} + +function fail(message: string): never { + throw new LocalDeploymentConfigurationError( + `reconciliation automation apply ${message}`, + ); +} + +function record(value: unknown, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + fail(`${label} must be an object`); + } + return value as Record; +} + +function exact( + value: Record, + keys: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, i) => key !== expected[i]) + ) { + fail(`${label} shape is invalid`); + } +} + +function safePath(value: unknown, label: string): string { + if ( + typeof value !== 'string' || + !path.isAbsolute(value) || + path.parse(value).root === value || + path.normalize(value) !== value || + value.includes('\0') || + value.includes('//') || + !SAFE_PATH.test(value) || + Buffer.byteLength(value, 'utf8') > 4096 + ) + fail(`${label} must be a safe non-root absolute path`); + return value; +} + +function overlaps(left: string, right: string): boolean { + const relative = path.relative(left, right); + return ( + relative === '' || + (!relative.startsWith('..') && !path.isAbsolute(relative)) + ); +} + +function options( + value: unknown, +): Readonly { + const selected = record(value, 'options'); + const hasBusyTimeout = Object.hasOwn(selected, 'busyTimeoutMs'); + exact( + selected, + [ + 'allowRootService', + 'applicationRoot', + 'automationApplyRoot', + 'automationDecisionRoot', + 'automationRoot', + 'credentialFilePath', + 'deploymentRoot', + 'ownerPepperKeyringDirectory', + 'targetDatabasePath', + ...(hasBusyTimeout ? ['busyTimeoutMs'] : []), + ], + 'options', + ); + const identity = currentIdentity(); + if ( + typeof selected.allowRootService !== 'boolean' || + (identity.uid === 0) !== selected.allowRootService + ) + fail('command identity is invalid'); + const normalized = { + deploymentRoot: safePath(selected.deploymentRoot, 'deploymentRoot'), + applicationRoot: safePath(selected.applicationRoot, 'applicationRoot'), + automationRoot: safePath(selected.automationRoot, 'automationRoot'), + automationDecisionRoot: safePath( + selected.automationDecisionRoot, + 'automationDecisionRoot', + ), + automationApplyRoot: safePath( + selected.automationApplyRoot, + 'automationApplyRoot', + ), + targetDatabasePath: safePath( + selected.targetDatabasePath, + 'targetDatabasePath', + ), + ownerPepperKeyringDirectory: safePath( + selected.ownerPepperKeyringDirectory, + 'ownerPepperKeyringDirectory', + ), + credentialFilePath: safePath( + selected.credentialFilePath, + 'credentialFilePath', + ), + }; + const roots = [ + normalized.deploymentRoot, + normalized.applicationRoot, + normalized.automationRoot, + normalized.automationDecisionRoot, + normalized.automationApplyRoot, + ]; + for (let i = 0; i < roots.length; i += 1) { + for (let j = i + 1; j < roots.length; j += 1) { + if (overlaps(roots[i]!, roots[j]!) || overlaps(roots[j]!, roots[i]!)) { + fail('authority roots overlap'); + } + } + } + for (const candidate of [ + normalized.ownerPepperKeyringDirectory, + normalized.credentialFilePath, + ]) { + const relative = path.relative(normalized.deploymentRoot, candidate); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) { + fail('authentication material must be below deploymentRoot'); + } + } + if ( + roots.some( + (root) => + overlaps(root, normalized.targetDatabasePath) || + overlaps(normalized.targetDatabasePath, root), + ) + ) { + fail('targetDatabasePath overlaps an authority root'); + } + if ( + selected.busyTimeoutMs !== undefined && + (!Number.isSafeInteger(selected.busyTimeoutMs) || + (selected.busyTimeoutMs as number) < 1 || + (selected.busyTimeoutMs as number) > 60_000) + ) { + fail('busyTimeoutMs is invalid'); + } + return Object.freeze({ + ...normalized, + allowRootService: selected.allowRootService as boolean, + ...(selected.busyTimeoutMs === undefined + ? {} + : { busyTimeoutMs: selected.busyTimeoutMs as number }), + }); +} + +function command(value: unknown, operation: string) { + const selected = record(value, 'command'); + exact( + selected, + ['operation', 'options', 'request', 'schemaVersion'], + 'command', + ); + if (selected.schemaVersion !== 1 || selected.operation !== operation) + fail('command version or operation is invalid'); + return { + options: options(selected.options), + request: record(selected.request, 'request'), + } as const; +} + +function id(value: unknown, pattern: RegExp, label: string): string { + if (typeof value !== 'string' || !pattern.test(value)) + fail(`${label} is invalid`); + return value; +} + +function digest(value: unknown, label: string): string { + return id(value, DIGEST, label); +} + +export function normalizeLocalReconciliationAutomationApplyCommand( + value: unknown, +): Readonly { + const selected = command( + value, + 'local.deployment.reconciliation.automation.apply', + ); + exact( + selected.request, + [ + 'appliedAtMs', + 'automationId', + 'decisionId', + 'expectedDecisionDigest', + 'expectedHeadDigest', + 'mutationId', + 'requestId', + ], + 'request', + ); + if ( + !Number.isSafeInteger(selected.request.appliedAtMs) || + (selected.request.appliedAtMs as number) < 0 || + typeof selected.request.requestId !== 'string' || + !SAFE_ID.test(selected.request.requestId) + ) + fail('apply request timestamp or requestId is invalid'); + return Object.freeze({ + schemaVersion: 1, + operation: 'local.deployment.reconciliation.automation.apply', + options: selected.options, + request: Object.freeze({ + decisionId: id(selected.request.decisionId, UUID_V7, 'decisionId'), + automationId: id(selected.request.automationId, UUID_V4, 'automationId'), + expectedDecisionDigest: digest( + selected.request.expectedDecisionDigest, + 'expectedDecisionDigest', + ), + expectedHeadDigest: digest( + selected.request.expectedHeadDigest, + 'expectedHeadDigest', + ), + mutationId: id(selected.request.mutationId, UUID_V4, 'mutationId'), + requestId: selected.request.requestId, + appliedAtMs: selected.request.appliedAtMs as number, + }), + }); +} + +export function normalizeLocalReconciliationAutomationApplyVerifyCommand( + value: unknown, +): Readonly { + const selected = command( + value, + 'local.deployment.reconciliation.automation.apply.verify', + ); + exact( + selected.request, + ['automationId', 'decisionId', 'expectedApplyDigest'], + 'request', + ); + return Object.freeze({ + schemaVersion: 1, + operation: 'local.deployment.reconciliation.automation.apply.verify', + options: selected.options, + request: Object.freeze({ + decisionId: id(selected.request.decisionId, UUID_V7, 'decisionId'), + automationId: id(selected.request.automationId, UUID_V4, 'automationId'), + expectedApplyDigest: digest( + selected.request.expectedApplyDigest, + 'expectedApplyDigest', + ), + }), + }); +} + +export function normalizeLocalReconciliationAutomationApplyRollbackCommand( + value: unknown, +): Readonly { + const selected = command( + value, + 'local.deployment.reconciliation.automation.apply.rollback', + ); + exact( + selected.request, + [ + 'automationId', + 'decisionId', + 'expectedApplyDigest', + 'expectedHeadDigest', + 'rolledBackAtMs', + ], + 'request', + ); + if ( + !Number.isSafeInteger(selected.request.rolledBackAtMs) || + (selected.request.rolledBackAtMs as number) < 0 + ) + fail('rolledBackAtMs is invalid'); + return Object.freeze({ + schemaVersion: 1, + operation: 'local.deployment.reconciliation.automation.apply.rollback', + options: selected.options, + request: Object.freeze({ + decisionId: id(selected.request.decisionId, UUID_V7, 'decisionId'), + automationId: id(selected.request.automationId, UUID_V4, 'automationId'), + expectedApplyDigest: digest( + selected.request.expectedApplyDigest, + 'expectedApplyDigest', + ), + expectedHeadDigest: digest( + selected.request.expectedHeadDigest, + 'expectedHeadDigest', + ), + rolledBackAtMs: selected.request.rolledBackAtMs as number, + }), + }); +} diff --git a/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyCoordinator.ts b/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyCoordinator.ts new file mode 100644 index 00000000..1d92bad5 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyCoordinator.ts @@ -0,0 +1,927 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { applyReconciliationAutomationDecision } from '@qinglong/local-admin/reconciliation-automation-decision'; +import { readPrivateLocalCommandFile } from '@qinglong/local-command-file'; +import { + establishAuthenticatedLocalCommand, + type AuthenticatedLocalCommand, +} from '@qinglong/local-owner-console/authenticated-command'; +import { openLocalSqliteAuthenticationReadDatabase } from '@qinglong/local-sqlite/authentication-read'; +import { + createLocalSqliteRolloutBackup, + inspectLocalSqliteSnapshot, + restoreLocalSqliteSnapshot, +} from '@qinglong/local-sqlite/rollout-safety'; +import type { SecurityPrincipal } from '@qinglong/runtime-core/security'; + +import { currentIdentity } from '../../../foundation/contract'; +import { LocalDeploymentConfigurationError } from '../../../foundation/error'; +import { + ensurePrivateDirectory, + preflightPublishedFile, + publishExactFile, + validatePrivateDirectory, +} from '../../../foundation/files'; +import { + advanceLocalCutoverInstanceHead, + readLocalCutoverInstanceHead, + type LocalCutoverInstanceHead, +} from '../../../cutover/instanceLineage'; +import { + inspectLocalReconciliationSealedBundle, + withLocalReconciliationSealedDatabaseAsync, +} from '../../sealed-bundle/reader'; +import { readLocalReconciliationCaptureIntent } from '../../preparation'; +import { proveLocalReconciliationStoppedState } from '../../stoppedProof'; +import { LocalReconciliationReviewIssuerKeyringFileProvider } from '../../review/issuerKeyring'; +import { + normalizeLocalReconciliationAutomationApplyCommand, + normalizeLocalReconciliationAutomationApplyRollbackCommand, + normalizeLocalReconciliationAutomationApplyVerifyCommand, + type LocalReconciliationAutomationApplyCommand, + type LocalReconciliationAutomationApplyOptions, + type LocalReconciliationAutomationApplyResult, +} from './applyContract'; +import { + buildLocalReconciliationAutomationApplyIntent, + buildLocalReconciliationAutomationApplyReceipt, + buildLocalReconciliationAutomationRollbackReceipt, + localReconciliationAutomationApplyEvidenceContents, + normalizeLocalReconciliationAutomationApplyIntent, + normalizeLocalReconciliationAutomationApplyReceipt, + normalizeLocalReconciliationAutomationRollbackReceipt, + type LocalReconciliationAutomationApplyIntent, + type LocalReconciliationAutomationApplyReceipt, +} from './applyEvidence'; +import { readLocalReconciliationAutomationDecisionTerminal } from './decisionCoordinator'; +import { createLocalReconciliationAutomationRequirementFactory } from './planReader'; + +const MAX_AUTHENTICATION_AGE_MS = 5 * 60 * 1_000; + +interface ApplyPaths { + root: string; + backupRoot: string; + intent: string; + backup: string; + receipt: string; + rollback: string; + restoreStage: string; + replaced: string; +} + +type AuthenticationDatabase = Awaited< + ReturnType +>; + +export interface LocalReconciliationAutomationApplyDependencies { + readonly openAuthenticationDatabase?: typeof openLocalSqliteAuthenticationReadDatabase; + readonly authenticate?: typeof establishAuthenticatedLocalCommand; + readonly createBackup?: typeof createLocalSqliteRolloutBackup; + readonly inspectSnapshot?: typeof inspectLocalSqliteSnapshot; + readonly restoreSnapshot?: typeof restoreLocalSqliteSnapshot; + readonly afterBackupPublished?: () => void; + readonly afterPreparedHead?: () => void; + readonly afterDatabaseCommit?: () => void; + readonly afterReceiptPublished?: () => void; + readonly afterAppliedHead?: () => void; + readonly afterRestore?: () => void; + readonly afterRollbackReceipt?: () => void; +} + +function fail(message: string, cause?: unknown): never { + throw new LocalDeploymentConfigurationError( + `reconciliation automation apply ${message}`, + { cause }, + ); +} + +function paths(root: string, automationId: string): Readonly { + const selected = path.join(root, automationId); + const backupRoot = path.join(selected, 'backup'); + return Object.freeze({ + root: selected, + backupRoot, + intent: path.join(selected, 'intent.json'), + backup: path.join(backupRoot, 'before.sqlite'), + receipt: path.join(selected, 'receipt.json'), + rollback: path.join(selected, 'rollback.json'), + restoreStage: path.join(backupRoot, 'restore-stage.sqlite'), + replaced: path.join(backupRoot, 'replaced.sqlite'), + }); +} + +function decisionOptions( + options: Readonly, +) { + return Object.freeze({ + deploymentRoot: options.deploymentRoot, + applicationRoot: options.applicationRoot, + automationRoot: options.automationRoot, + automationDecisionRoot: options.automationDecisionRoot, + allowRootService: options.allowRootService, + }); +} + +function readIntent( + selected: Readonly, +): Readonly { + return normalizeLocalReconciliationAutomationApplyIntent( + readPrivateLocalCommandFile(selected.intent), + ); +} + +function readReceipt( + selected: Readonly, +): Readonly { + return normalizeLocalReconciliationAutomationApplyReceipt( + readPrivateLocalCommandFile(selected.receipt), + ); +} + +function validateCatalog(selected: Readonly): void { + const allowed = new Set([ + 'backup', + 'intent.json', + 'receipt.json', + 'rollback.json', + '.intent.json.ql3-deploy-stage', + '.receipt.json.ql3-deploy-stage', + '.rollback.json.ql3-deploy-stage', + ]); + for (const entry of fs.readdirSync(selected.root, { withFileTypes: true })) { + if (!allowed.has(entry.name) || entry.isSymbolicLink()) + fail('apply root contains unknown material'); + } + const backupAllowed = new Set([ + 'before.sqlite', + '.before.sqlite.ql3-backup-stage', + 'restore-stage.sqlite', + 'replaced.sqlite', + ]); + for (const entry of fs.readdirSync(selected.backupRoot, { + withFileTypes: true, + })) { + if (!backupAllowed.has(entry.name) || entry.isSymbolicLink()) + fail('backup root contains unknown material'); + } +} + +function advance( + intent: Readonly, + uid: number, + state: + | 'reconciliation_automation_apply_prepared' + | 'reconciliation_automation_applied' + | 'reconciliation_automation_rolled_back', + digest: string, + atMs: number, +): Readonly { + return advanceLocalCutoverInstanceHead( + { + options: { deploymentRoot: intent.command.options.deploymentRoot }, + request: { + cutoverId: intent.cutoverId, + profile: intent.profile, + instanceId: intent.instanceId, + expectedActivationDigest: intent.activationDigest, + requestedAtMs: atMs, + }, + }, + uid, + state, + intent.generation, + digest, + ); +} + +function result( + operation: LocalReconciliationAutomationApplyResult['operation'], + status: LocalReconciliationAutomationApplyResult['status'], + receipt: Readonly, + intent: Readonly, + head: Readonly, +): Readonly { + return Object.freeze({ + schemaVersion: 1, + operation, + status, + state: head.state as + | 'reconciliation_automation_applied' + | 'reconciliation_automation_rolled_back', + decisionId: receipt.decisionId, + automationId: receipt.automationId, + applyDigest: receipt.applyDigest, + publicationDigest: receipt.publicationDigest, + adoptedTaskCount: receipt.adoptedTaskCount, + adoptedTriggerCount: receipt.adoptedTriggerCount, + backupSha256: intent.backup.sha256, + instanceHeadDigest: head.headDigest, + }); +} + +function assertReviewer( + authenticated: Readonly, + reviewer: Readonly, + atMs: number, +): void { + const principal = authenticated.principal; + if ( + reviewer.subject.type !== 'user' || + principal.subject.type !== 'user' || + reviewer.subject.id !== principal.subject.id || + principal.assurance !== 'local_console' || + principal.authenticatedAtMs > atMs || + atMs - principal.authenticatedAtMs > MAX_AUTHENTICATION_AGE_MS || + principal.expiresAtMs <= atMs + ) + fail('current reviewer authentication is not strong or identical'); +} + +async function authenticate( + options: Readonly, + atMs: number, + profile: 'edge' | 'standalone', + reviewer: Readonly, + dependencies: LocalReconciliationAutomationApplyDependencies, +): Promise< + Readonly<{ + authenticated: Readonly; + database: AuthenticationDatabase; + }> +> { + const open = + dependencies.openAuthenticationDatabase ?? + openLocalSqliteAuthenticationReadDatabase; + const database = await open({ + databasePath: options.targetDatabasePath, + profile, + ...(options.busyTimeoutMs === undefined + ? {} + : { busyTimeoutMs: options.busyTimeoutMs }), + }); + try { + const selected = await ( + dependencies.authenticate ?? establishAuthenticatedLocalCommand + )(database, { + deploymentRoot: options.deploymentRoot, + databasePath: options.targetDatabasePath, + ownerPepperKeyringDirectory: options.ownerPepperKeyringDirectory, + credentialFilePath: options.credentialFilePath, + authenticationNamespace: 'local_reconciliation_automation_apply', + now: () => atMs, + }); + assertReviewer(selected, reviewer, atMs); + return Object.freeze({ authenticated: selected, database }); + } catch (error) { + await database.close(); + throw error; + } +} + +function exactBundle( + intent: Readonly, + captureRoot: string, + captureId: string, + expectedBundleDigest: string, + expectedFingerprintDigest: string, + uid: number, +): void { + const current = inspectLocalReconciliationSealedBundle( + captureRoot, + captureId, + uid, + ); + if ( + current.receipt.bundleDigest !== expectedBundleDigest || + current.fingerprintDigest !== expectedFingerprintDigest || + intent.command.request.expectedDecisionDigest.length !== 64 + ) + fail('sealed source authority drifted'); +} + +function verifyIntentCommand( + intent: Readonly, + command: Readonly, +): void { + if ( + intent.command.request.automationId !== command.request.automationId || + intent.command.request.decisionId !== command.request.decisionId || + intent.command.request.expectedDecisionDigest !== + command.request.expectedDecisionDigest || + intent.command.request.mutationId !== command.request.mutationId || + intent.command.request.requestId !== command.request.requestId || + intent.command.request.appliedAtMs !== command.request.appliedAtMs || + intent.command.request.expectedHeadDigest !== + command.request.expectedHeadDigest || + JSON.stringify(intent.command.options) !== JSON.stringify(command.options) + ) + fail('apply command is not an exact replay'); +} + +export async function applyLocalReconciliationAutomation( + value: unknown, + dependencies: LocalReconciliationAutomationApplyDependencies = {}, +): Promise> { + const command = normalizeLocalReconciliationAutomationApplyCommand(value); + const uid = currentIdentity().uid; + for (const [directory, label] of [ + [command.options.deploymentRoot, 'deploymentRoot'], + [command.options.applicationRoot, 'applicationRoot'], + [command.options.automationRoot, 'automationRoot'], + [command.options.automationDecisionRoot, 'automationDecisionRoot'], + [command.options.automationApplyRoot, 'automationApplyRoot'], + ] as const) + validatePrivateDirectory(directory, uid, label); + const selected = paths( + command.options.automationApplyRoot, + command.request.automationId, + ); + ensurePrivateDirectory(selected.root, uid, 'automation apply root'); + ensurePrivateDirectory( + selected.backupRoot, + uid, + 'automation apply backup root', + ); + validateCatalog(selected); + + const terminal = await readLocalReconciliationAutomationDecisionTerminal( + decisionOptions(command.options), + command.request.automationId, + uid, + ); + if ( + terminal.receipt.decisionId !== command.request.decisionId || + terminal.receipt.decisionDigest !== command.request.expectedDecisionDigest + ) + fail('apply command is detached from signed decision'); + const capture = readLocalReconciliationCaptureIntent( + terminal.plan.intent.command.options.captureRoot, + terminal.plan.intent.command.request.captureId, + ); + if ( + capture.command.request.targetDatabasePath !== + command.options.targetDatabasePath + ) { + fail('target database is detached from stopped capture'); + } + + let intent: Readonly; + const recoveringPreparedIntent = fs.existsSync(selected.intent); + let head = readLocalCutoverInstanceHead( + command.options.deploymentRoot, + terminal.intent.instanceId, + uid, + ); + if (fs.existsSync(selected.intent)) { + intent = readIntent(selected); + verifyIntentCommand(intent, command); + } else { + if ( + head.state !== 'reconciliation_automation_reviewed' || + head.headDigest !== command.request.expectedHeadDigest || + head.sourceRecordDigest !== terminal.receipt.decisionDigest + ) + fail('apply lost reviewed head compare-and-swap'); + const before = proveLocalReconciliationStoppedState(capture.command, uid); + const backup = await ( + dependencies.createBackup ?? createLocalSqliteRolloutBackup + )({ + databasePath: command.options.targetDatabasePath, + backupPath: selected.backup, + profile: terminal.intent.profile, + ...(command.options.busyTimeoutMs === undefined + ? {} + : { busyTimeoutMs: command.options.busyTimeoutMs }), + }); + const after = proveLocalReconciliationStoppedState(capture.command, uid); + if (after.proofDigest !== before.proofDigest) + fail('stopped target drifted across backup'); + dependencies.afterBackupPublished?.(); + intent = buildLocalReconciliationAutomationApplyIntent({ + command, + instanceId: terminal.intent.instanceId, + cutoverId: terminal.intent.cutoverId, + activationDigest: terminal.intent.activationDigest, + profile: terminal.intent.profile, + projectId: terminal.intent.projectId, + generation: terminal.intent.generation, + stoppedProofDigest: before.proofDigest, + backup, + }); + publishExactFile( + selected.intent, + localReconciliationAutomationApplyEvidenceContents(intent), + 0o600, + uid, + 'automation apply intent', + ); + head = advance( + intent, + uid, + 'reconciliation_automation_apply_prepared', + intent.preparationDigest, + command.request.appliedAtMs, + ); + dependencies.afterPreparedHead?.(); + } + + if (fs.existsSync(selected.receipt)) { + const receipt = readReceipt(selected); + const current = await ( + dependencies.inspectSnapshot ?? inspectLocalSqliteSnapshot + )({ + databasePath: command.options.targetDatabasePath, + profile: intent.profile, + }); + if (current.sha256 !== receipt.targetAfter.sha256) + fail('terminal apply target drifted'); + if ( + head.state === 'reconciliation_automation_apply_prepared' && + head.sourceRecordDigest === intent.preparationDigest + ) { + head = advance( + intent, + uid, + 'reconciliation_automation_applied', + receipt.applyDigest, + receipt.appliedAtMs, + ); + dependencies.afterAppliedHead?.(); + } else if ( + head.state !== 'reconciliation_automation_applied' || + receipt.applyDigest !== head.sourceRecordDigest + ) + fail('terminal apply receipt drifted'); + return result(command.operation, 'existing', receipt, intent, head); + } + if ( + head.state !== 'reconciliation_automation_apply_prepared' || + head.sourceRecordDigest !== intent.preparationDigest + ) + fail('apply lost prepared head compare-and-swap'); + if (!recoveringPreparedIntent) { + const stopped = proveLocalReconciliationStoppedState(capture.command, uid); + if (stopped.proofDigest !== intent.stoppedProofDigest) + fail('stopped proof drifted before write'); + } + + const authenticatedScope = await authenticate( + command.options, + command.request.appliedAtMs, + intent.profile, + terminal.reviewer, + dependencies, + ); + let publication; + try { + const openRequirements = + createLocalReconciliationAutomationRequirementFactory( + terminal.context.automation.planPath, + terminal.context.automation.receipt, + uid, + ); + const keyProvider = new LocalReconciliationReviewIssuerKeyringFileProvider( + terminal.context.application.intent.command.options.issuerKeyringPath, + ); + const applied = await withLocalReconciliationSealedDatabaseAsync( + terminal.plan.bundle, + 'legacy', + uid, + {}, + async (sourceClient) => + applyReconciliationAutomationDecision({ + authorizationPath: terminal.authorizationPath, + decisionId: command.request.decisionId, + profile: intent.profile, + automationPlanDigest: terminal.receipt.automationPlanDigest, + inventoryDigest: terminal.receipt.legacyInventoryDigest, + sourceClient, + timezone: terminal.intent.legacyTimezone, + keyProvider, + observedAtMs: command.request.appliedAtMs, + openRequirements, + allowedModes: [0o400], + allowedParentModes: [0o500], + targetPath: command.options.targetDatabasePath, + projectId: intent.projectId, + mutationId: command.request.mutationId, + requestId: command.request.requestId, + confirmSourceIdentity() { + exactBundle( + intent, + terminal.plan.bundle.captureRoot, + terminal.plan.receipt.captureId, + terminal.plan.bundle.receipt.bundleDigest, + terminal.plan.bundle.fingerprintDigest, + uid, + ); + const currentHead = readLocalCutoverInstanceHead( + command.options.deploymentRoot, + intent.instanceId, + uid, + ); + if ( + currentHead.state !== + 'reconciliation_automation_apply_prepared' || + currentHead.sourceRecordDigest !== intent.preparationDigest + ) + fail('apply lost write head authority'); + }, + async confirmReviewerAuthority(reviewer) { + assertReviewer( + authenticatedScope.authenticated, + reviewer, + command.request.appliedAtMs, + ); + await authenticatedScope.authenticated.confirm(); + }, + }), + ); + if (applied === null) fail('sealed Legacy source requires manual handling'); + publication = applied; + } finally { + await authenticatedScope.database.close(); + } + dependencies.afterDatabaseCommit?.(); + const targetAfter = await ( + dependencies.inspectSnapshot ?? inspectLocalSqliteSnapshot + )({ + databasePath: command.options.targetDatabasePath, + profile: intent.profile, + }); + const receipt = buildLocalReconciliationAutomationApplyReceipt({ + decisionId: command.request.decisionId, + automationId: command.request.automationId, + mutationId: command.request.mutationId, + preparationDigest: intent.preparationDigest, + preparedHeadDigest: head.headDigest, + publicationDigest: publication.adoption.publicationDigest, + adoptedTaskCount: publication.adoption.adoptedTaskCount, + adoptedTriggerCount: publication.adoption.adoptedTriggerCount, + skippedCount: publication.adoption.skippedCount, + targetAfter, + appliedAtMs: command.request.appliedAtMs, + }); + preflightPublishedFile( + selected.receipt, + localReconciliationAutomationApplyEvidenceContents(receipt), + 0o600, + uid, + 'automation apply receipt', + ); + publishExactFile( + selected.receipt, + localReconciliationAutomationApplyEvidenceContents(receipt), + 0o600, + uid, + 'automation apply receipt', + ); + dependencies.afterReceiptPublished?.(); + head = advance( + intent, + uid, + 'reconciliation_automation_applied', + receipt.applyDigest, + command.request.appliedAtMs, + ); + dependencies.afterAppliedHead?.(); + return result( + command.operation, + publication.status === 'existing' ? 'existing' : 'applied', + receipt, + intent, + head, + ); +} + +export async function verifyLocalReconciliationAutomationApply( + value: unknown, +): Promise> { + const command = + normalizeLocalReconciliationAutomationApplyVerifyCommand(value); + const uid = currentIdentity().uid; + const selected = paths( + command.options.automationApplyRoot, + command.request.automationId, + ); + validatePrivateDirectory( + command.options.automationApplyRoot, + uid, + 'automationApplyRoot', + ); + validatePrivateDirectory(selected.root, uid, 'automation apply root'); + validatePrivateDirectory( + selected.backupRoot, + uid, + 'automation apply backup root', + ); + validateCatalog(selected); + const intent = readIntent(selected); + const receipt = readReceipt(selected); + if ( + intent.command.request.decisionId !== command.request.decisionId || + receipt.decisionId !== command.request.decisionId || + receipt.automationId !== command.request.automationId || + receipt.applyDigest !== command.request.expectedApplyDigest + ) + fail('verify command is detached from apply receipt'); + await readLocalReconciliationAutomationDecisionTerminal( + decisionOptions(command.options), + command.request.automationId, + uid, + ); + const head = readLocalCutoverInstanceHead( + command.options.deploymentRoot, + intent.instanceId, + uid, + ); + if (head.state === 'reconciliation_automation_applied') { + if (head.sourceRecordDigest !== receipt.applyDigest) + fail('applied head drifted'); + const current = await inspectLocalSqliteSnapshot({ + databasePath: command.options.targetDatabasePath, + profile: intent.profile, + }); + if (current.sha256 !== receipt.targetAfter.sha256) + fail('applied target drifted'); + return result(command.operation, 'verified', receipt, intent, head); + } + if ( + head.state === 'reconciliation_automation_rolled_back' && + fs.existsSync(selected.rollback) + ) { + const rollback = normalizeLocalReconciliationAutomationRollbackReceipt( + readPrivateLocalCommandFile(selected.rollback), + ); + if ( + rollback.applyDigest !== receipt.applyDigest || + head.sourceRecordDigest !== rollback.rollbackDigest + ) + fail('rollback head drifted'); + const current = await inspectLocalSqliteSnapshot({ + databasePath: command.options.targetDatabasePath, + profile: intent.profile, + }); + if ( + current.sha256 !== intent.backup.sha256 || + current.sha256 !== rollback.restored.sha256 + ) + fail('rolled-back target drifted'); + return result(command.operation, 'verified', receipt, intent, head); + } + return fail('apply head is not terminal'); +} + +export async function rollbackLocalReconciliationAutomationApply( + value: unknown, + dependencies: LocalReconciliationAutomationApplyDependencies = {}, +): Promise> { + const command = + normalizeLocalReconciliationAutomationApplyRollbackCommand(value); + const uid = currentIdentity().uid; + const selected = paths( + command.options.automationApplyRoot, + command.request.automationId, + ); + validatePrivateDirectory( + command.options.automationApplyRoot, + uid, + 'automationApplyRoot', + ); + validatePrivateDirectory(selected.root, uid, 'automation apply root'); + validatePrivateDirectory( + selected.backupRoot, + uid, + 'automation apply backup root', + ); + validateCatalog(selected); + const intent = readIntent(selected); + const receipt = readReceipt(selected); + if ( + receipt.decisionId !== command.request.decisionId || + receipt.automationId !== command.request.automationId || + receipt.applyDigest !== command.request.expectedApplyDigest + ) + fail('rollback command is detached from apply receipt'); + const terminal = await readLocalReconciliationAutomationDecisionTerminal( + decisionOptions(command.options), + command.request.automationId, + uid, + ); + let head = readLocalCutoverInstanceHead( + command.options.deploymentRoot, + intent.instanceId, + uid, + ); + if (fs.existsSync(selected.rollback)) { + const rollback = normalizeLocalReconciliationAutomationRollbackReceipt( + readPrivateLocalCommandFile(selected.rollback), + ); + const current = await ( + dependencies.inspectSnapshot ?? inspectLocalSqliteSnapshot + )({ + databasePath: command.options.targetDatabasePath, + profile: intent.profile, + }); + if (current.sha256 !== rollback.restored.sha256) + fail('rollback replay target drifted'); + if ( + head.state === 'reconciliation_automation_applied' && + head.sourceRecordDigest === receipt.applyDigest + ) { + head = advance( + intent, + uid, + 'reconciliation_automation_rolled_back', + rollback.rollbackDigest, + rollback.rolledBackAtMs, + ); + } else if ( + head.state !== 'reconciliation_automation_rolled_back' || + head.sourceRecordDigest !== rollback.rollbackDigest + ) + fail('rollback replay drifted'); + return result(command.operation, 'existing', receipt, intent, head); + } + if ( + head.state !== 'reconciliation_automation_applied' || + head.headDigest !== command.request.expectedHeadDigest || + head.sourceRecordDigest !== receipt.applyDigest + ) + fail('rollback lost applied head compare-and-swap'); + const current = await ( + dependencies.inspectSnapshot ?? inspectLocalSqliteSnapshot + )({ + databasePath: command.options.targetDatabasePath, + profile: intent.profile, + }); + if (current.sha256 === intent.backup.sha256) { + const rollback = buildLocalReconciliationAutomationRollbackReceipt({ + decisionId: receipt.decisionId, + automationId: receipt.automationId, + applyDigest: receipt.applyDigest, + restored: current, + rolledBackAtMs: command.request.rolledBackAtMs, + }); + publishExactFile( + selected.rollback, + localReconciliationAutomationApplyEvidenceContents(rollback), + 0o600, + uid, + 'automation rollback receipt', + ); + dependencies.afterRollbackReceipt?.(); + head = advance( + intent, + uid, + 'reconciliation_automation_rolled_back', + rollback.rollbackDigest, + command.request.rolledBackAtMs, + ); + return result(command.operation, 'existing', receipt, intent, head); + } + if (current.sha256 !== receipt.targetAfter.sha256) + fail('rollback current target drifted'); + + const authenticatedScope = await authenticate( + command.options, + command.request.rolledBackAtMs, + intent.profile, + terminal.reviewer, + dependencies, + ); + try { + const openRequirements = + createLocalReconciliationAutomationRequirementFactory( + terminal.context.automation.planPath, + terminal.context.automation.receipt, + uid, + ); + const keyProvider = new LocalReconciliationReviewIssuerKeyringFileProvider( + terminal.context.application.intent.command.options.issuerKeyringPath, + ); + const replay = await withLocalReconciliationSealedDatabaseAsync( + terminal.plan.bundle, + 'legacy', + uid, + {}, + async (sourceClient) => + applyReconciliationAutomationDecision({ + authorizationPath: terminal.authorizationPath, + decisionId: receipt.decisionId, + profile: intent.profile, + automationPlanDigest: terminal.receipt.automationPlanDigest, + inventoryDigest: terminal.receipt.legacyInventoryDigest, + sourceClient, + timezone: terminal.intent.legacyTimezone, + keyProvider, + observedAtMs: intent.command.request.appliedAtMs, + openRequirements, + allowedModes: [0o400], + allowedParentModes: [0o500], + targetPath: command.options.targetDatabasePath, + projectId: intent.projectId, + mutationId: intent.command.request.mutationId, + requestId: intent.command.request.requestId, + confirmSourceIdentity() { + const currentHead = readLocalCutoverInstanceHead( + command.options.deploymentRoot, + intent.instanceId, + uid, + ); + if ( + currentHead.state !== 'reconciliation_automation_applied' || + currentHead.sourceRecordDigest !== receipt.applyDigest + ) + fail('rollback lost current policy fence'); + }, + async confirmReviewerAuthority(reviewer) { + assertReviewer( + authenticatedScope.authenticated, + reviewer, + command.request.rolledBackAtMs, + ); + await authenticatedScope.authenticated.confirm(); + }, + }), + ); + if ( + replay === null || + replay.status !== 'existing' || + replay.adoption.publicationDigest !== receipt.publicationDigest + ) + fail('rollback could not verify current publication'); + } finally { + await authenticatedScope.database.close(); + } + const restored = await ( + dependencies.restoreSnapshot ?? restoreLocalSqliteSnapshot + )({ + databasePath: command.options.targetDatabasePath, + sourceSnapshotPath: selected.backup, + restoreStagePath: selected.restoreStage, + replacedDatabasePath: selected.replaced, + expectedCurrentSha256: receipt.targetAfter.sha256, + expectedSourceSha256: intent.backup.sha256, + preserveDatabaseIdentity: true, + profile: intent.profile, + ...(command.options.busyTimeoutMs === undefined + ? {} + : { busyTimeoutMs: command.options.busyTimeoutMs }), + }); + dependencies.afterRestore?.(); + const rollback = buildLocalReconciliationAutomationRollbackReceipt({ + decisionId: receipt.decisionId, + automationId: receipt.automationId, + applyDigest: receipt.applyDigest, + restored: Object.freeze({ + contractVersion: restored.contractVersion, + sha256: restored.sha256, + bytes: restored.bytes, + pageCount: restored.pageCount, + pageSize: restored.pageSize, + }), + rolledBackAtMs: command.request.rolledBackAtMs, + }); + publishExactFile( + selected.rollback, + localReconciliationAutomationApplyEvidenceContents(rollback), + 0o600, + uid, + 'automation rollback receipt', + ); + dependencies.afterRollbackReceipt?.(); + head = advance( + intent, + uid, + 'reconciliation_automation_rolled_back', + rollback.rollbackDigest, + command.request.rolledBackAtMs, + ); + return result(command.operation, 'rolled_back', receipt, intent, head); +} + +export async function applyLocalReconciliationAutomationCommandFile( + filePath: string, +): Promise> { + return applyLocalReconciliationAutomation( + readPrivateLocalCommandFile(filePath), + ); +} + +export async function verifyLocalReconciliationAutomationApplyCommandFile( + filePath: string, +): Promise> { + return verifyLocalReconciliationAutomationApply( + readPrivateLocalCommandFile(filePath), + ); +} + +export async function rollbackLocalReconciliationAutomationApplyCommandFile( + filePath: string, +): Promise> { + return rollbackLocalReconciliationAutomationApply( + readPrivateLocalCommandFile(filePath), + ); +} diff --git a/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyEvidence.ts b/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyEvidence.ts new file mode 100644 index 00000000..c7d1a044 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/applyEvidence.ts @@ -0,0 +1,351 @@ +import type { + LocalSqliteRolloutBackupEvidence, + LocalSqliteSnapshotEvidence, +} from '@qinglong/local-sqlite/rollout-safety'; + +import { LocalDeploymentConfigurationError } from '../../../foundation/error'; +import { cutoverDigest } from '../../../cutover/targetEvidence'; +import { + normalizeLocalReconciliationAutomationApplyCommand, + type LocalReconciliationAutomationApplyCommand, +} from './applyContract'; + +const INTENT_SCHEMA = 'qinglong3-local-reconciliation-automation-apply-intent'; +const RECEIPT_SCHEMA = + 'qinglong3-local-reconciliation-automation-apply-receipt'; +const ROLLBACK_SCHEMA = + 'qinglong3-local-reconciliation-automation-rollback-receipt'; +const DIGEST = /^[0-9a-f]{64}$/; + +export interface LocalReconciliationAutomationApplyIntent { + readonly schema: typeof INTENT_SCHEMA; + readonly schemaVersion: 1; + readonly command: Readonly; + readonly instanceId: string; + readonly cutoverId: string; + readonly activationDigest: string; + readonly profile: 'edge' | 'standalone'; + readonly projectId: string; + readonly generation: number; + readonly stoppedProofDigest: string; + readonly backup: Readonly; + readonly preparationDigest: string; +} + +export interface LocalReconciliationAutomationApplyReceipt { + readonly schema: typeof RECEIPT_SCHEMA; + readonly schemaVersion: 1; + readonly state: 'reconciliation_automation_applied'; + readonly decisionId: string; + readonly automationId: string; + readonly mutationId: string; + readonly preparationDigest: string; + readonly preparedHeadDigest: string; + readonly publicationDigest: string; + readonly adoptedTaskCount: number; + readonly adoptedTriggerCount: number; + readonly skippedCount: number; + readonly targetAfter: Readonly; + readonly appliedAtMs: number; + readonly applyDigest: string; +} + +export interface LocalReconciliationAutomationRollbackReceipt { + readonly schema: typeof ROLLBACK_SCHEMA; + readonly schemaVersion: 1; + readonly state: 'reconciliation_automation_rolled_back'; + readonly decisionId: string; + readonly automationId: string; + readonly applyDigest: string; + readonly restored: Readonly; + readonly rolledBackAtMs: number; + readonly rollbackDigest: string; +} + +function fail(message: string): never { + throw new LocalDeploymentConfigurationError( + `reconciliation automation apply evidence ${message}`, + ); +} + +function exact( + value: unknown, + keys: readonly string[], + label: string, +): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) + fail(`${label} must be an object`); + const record = value as Record; + const actual = Object.keys(record).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) + fail(`${label} shape is invalid`); + return record; +} + +function validDigest(value: unknown): value is string { + return typeof value === 'string' && DIGEST.test(value); +} + +function snapshot( + value: unknown, + label: string, +): Readonly { + const selected = exact( + value, + ['bytes', 'contractVersion', 'pageCount', 'pageSize', 'sha256'], + label, + ); + if ( + !validDigest(selected.sha256) || + ![selected.bytes, selected.pageCount, selected.pageSize].every( + (item) => Number.isSafeInteger(item) && (item as number) > 0, + ) || + typeof selected.contractVersion !== 'number' + ) + fail(`${label} is invalid`); + return Object.freeze({ + contractVersion: selected.contractVersion, + sha256: selected.sha256, + bytes: selected.bytes, + pageCount: selected.pageCount, + pageSize: selected.pageSize, + }) as unknown as Readonly; +} + +function backup(value: unknown): Readonly { + const selected = exact( + value, + [ + 'bytes', + 'contractVersion', + 'pageCount', + 'pageSize', + 'sha256', + 'status', + 'writeContractVersion', + ], + 'backup', + ); + const base = snapshot( + { + bytes: selected.bytes, + contractVersion: selected.contractVersion, + pageCount: selected.pageCount, + pageSize: selected.pageSize, + sha256: selected.sha256, + }, + 'backup snapshot', + ); + if ( + (selected.status !== 'prepared' && selected.status !== 'existing') || + typeof selected.writeContractVersion !== 'number' + ) + fail('backup is invalid'); + return Object.freeze({ + status: selected.status, + writeContractVersion: selected.writeContractVersion, + ...base, + }) as Readonly; +} + +export function buildLocalReconciliationAutomationApplyIntent( + input: Omit< + LocalReconciliationAutomationApplyIntent, + 'schema' | 'schemaVersion' | 'preparationDigest' + >, +): Readonly { + const payload = Object.freeze({ + schema: INTENT_SCHEMA, + schemaVersion: 1 as const, + ...input, + }); + return Object.freeze({ + ...payload, + preparationDigest: cutoverDigest(payload), + }); +} + +export function normalizeLocalReconciliationAutomationApplyIntent( + value: unknown, +): Readonly { + const selected = exact( + value, + [ + 'activationDigest', + 'backup', + 'command', + 'cutoverId', + 'generation', + 'instanceId', + 'preparationDigest', + 'profile', + 'projectId', + 'schema', + 'schemaVersion', + 'stoppedProofDigest', + ], + 'intent', + ); + const { preparationDigest, ...raw } = selected; + const command = normalizeLocalReconciliationAutomationApplyCommand( + selected.command, + ); + const normalized = Object.freeze({ + ...raw, + command, + backup: backup(selected.backup), + }); + if ( + selected.schema !== INTENT_SCHEMA || + selected.schemaVersion !== 1 || + !validDigest(preparationDigest) || + cutoverDigest(raw) !== preparationDigest || + cutoverDigest(normalized) !== preparationDigest || + !validDigest(selected.activationDigest) || + !validDigest(selected.stoppedProofDigest) || + (selected.profile !== 'edge' && selected.profile !== 'standalone') || + !Number.isSafeInteger(selected.generation) || + (selected.generation as number) < 1 || + ![selected.instanceId, selected.cutoverId, selected.projectId].every( + (item) => typeof item === 'string' && item.length > 0, + ) + ) + fail('intent binding is invalid'); + return Object.freeze({ + ...normalized, + preparationDigest, + }) as unknown as Readonly; +} + +export function buildLocalReconciliationAutomationApplyReceipt( + input: Omit< + LocalReconciliationAutomationApplyReceipt, + 'schema' | 'schemaVersion' | 'state' | 'applyDigest' + >, +): Readonly { + const payload = Object.freeze({ + schema: RECEIPT_SCHEMA, + schemaVersion: 1 as const, + state: 'reconciliation_automation_applied' as const, + ...input, + }); + return Object.freeze({ ...payload, applyDigest: cutoverDigest(payload) }); +} + +export function normalizeLocalReconciliationAutomationApplyReceipt( + value: unknown, +): Readonly { + const selected = exact( + value, + [ + 'adoptedTaskCount', + 'adoptedTriggerCount', + 'appliedAtMs', + 'applyDigest', + 'automationId', + 'decisionId', + 'mutationId', + 'preparationDigest', + 'preparedHeadDigest', + 'publicationDigest', + 'schema', + 'schemaVersion', + 'skippedCount', + 'state', + 'targetAfter', + ], + 'receipt', + ); + const { applyDigest, ...payload } = selected; + const targetAfter = snapshot(selected.targetAfter, 'targetAfter'); + if ( + selected.schema !== RECEIPT_SCHEMA || + selected.schemaVersion !== 1 || + selected.state !== 'reconciliation_automation_applied' || + !validDigest(applyDigest) || + cutoverDigest(payload) !== applyDigest || + ![ + selected.preparationDigest, + selected.preparedHeadDigest, + selected.publicationDigest, + ].every(validDigest) || + ![ + selected.adoptedTaskCount, + selected.adoptedTriggerCount, + selected.skippedCount, + selected.appliedAtMs, + ].every((item) => Number.isSafeInteger(item) && (item as number) >= 0) + ) + fail('receipt binding is invalid'); + return Object.freeze({ + ...payload, + targetAfter, + applyDigest, + }) as unknown as Readonly; +} + +export function buildLocalReconciliationAutomationRollbackReceipt( + input: Omit< + LocalReconciliationAutomationRollbackReceipt, + 'schema' | 'schemaVersion' | 'state' | 'rollbackDigest' + >, +): Readonly { + const payload = Object.freeze({ + schema: ROLLBACK_SCHEMA, + schemaVersion: 1 as const, + state: 'reconciliation_automation_rolled_back' as const, + ...input, + }); + return Object.freeze({ ...payload, rollbackDigest: cutoverDigest(payload) }); +} + +export function normalizeLocalReconciliationAutomationRollbackReceipt( + value: unknown, +): Readonly { + const selected = exact( + value, + [ + 'applyDigest', + 'automationId', + 'decisionId', + 'restored', + 'rollbackDigest', + 'rolledBackAtMs', + 'schema', + 'schemaVersion', + 'state', + ], + 'rollback receipt', + ); + const { rollbackDigest, ...payload } = selected; + const restored = snapshot(selected.restored, 'restored'); + if ( + selected.schema !== ROLLBACK_SCHEMA || + selected.schemaVersion !== 1 || + selected.state !== 'reconciliation_automation_rolled_back' || + !validDigest(selected.applyDigest) || + !validDigest(rollbackDigest) || + cutoverDigest(payload) !== rollbackDigest || + !Number.isSafeInteger(selected.rolledBackAtMs) || + (selected.rolledBackAtMs as number) < 0 + ) + fail('rollback receipt binding is invalid'); + return Object.freeze({ + ...payload, + restored, + rollbackDigest, + }) as unknown as Readonly; +} + +export function localReconciliationAutomationApplyEvidenceContents( + value: + | LocalReconciliationAutomationApplyIntent + | LocalReconciliationAutomationApplyReceipt + | LocalReconciliationAutomationRollbackReceipt, +): string { + return `${JSON.stringify(value, null, 2)}\n`; +} diff --git a/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/decisionCoordinator.ts b/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/decisionCoordinator.ts index ba646e59..d2b21739 100644 --- a/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/decisionCoordinator.ts +++ b/packages/ql3-local-owner-cli/src/deployment/reconciliation/application/automation/decisionCoordinator.ts @@ -13,6 +13,7 @@ import { type AuthenticatedLocalCommand, } from '@qinglong/local-owner-console/authenticated-command'; import { openLocalSqliteAuthenticationReadDatabase } from '@qinglong/local-sqlite/authentication-read'; +import type { SecurityPrincipal } from '@qinglong/runtime-core/security'; import { currentIdentity } from '../../../foundation/contract'; import { LocalDeploymentConfigurationError } from '../../../foundation/error'; @@ -103,6 +104,15 @@ export interface LocalReconciliationAutomationDecisionDependencies readonly afterHeadAdvanced?: () => void; } +export interface LocalReconciliationAutomationDecisionTerminal { + readonly intent: Readonly; + readonly receipt: Readonly; + readonly authorizationPath: string; + readonly context: Readonly; + readonly plan: ReturnType; + readonly reviewer: Readonly; +} + function configurationError(message: string, cause?: unknown): never { throw new LocalDeploymentConfigurationError( `reconciliation automation decision ${message}`, @@ -149,17 +159,17 @@ function validateDirectory( return mode; } -function validateCatalog(selected: Readonly, terminal: boolean): void { +function validateCatalog( + selected: Readonly, + terminal: boolean, +): void { const allowed = new Set([ 'authorization.ndjson', 'intent.json', 'receipt.json', 'staging', ...(!terminal - ? [ - '.intent.json.ql3-deploy-stage', - '.receipt.json.ql3-deploy-stage', - ] + ? ['.intent.json.ql3-deploy-stage', '.receipt.json.ql3-deploy-stage'] : []), ]); for (const entry of fs.readdirSync(selected.root, { withFileTypes: true })) { @@ -242,7 +252,9 @@ function terminalJson( } async function context( - options: Readonly, + options: Readonly< + LocalReconciliationAutomationDecisionPrepareCommand['options'] + >, automationId: string, uid: number, ): Promise> { @@ -412,7 +424,7 @@ export async function prepareLocalReconciliationAutomationDecision( ); if ( current.automation.receipt.automationPlanDigest !== - command.request.expectedAutomationPlanDigest + command.request.expectedAutomationPlanDigest ) { configurationError('expected automation plan digest drifted'); } @@ -438,7 +450,8 @@ export async function prepareLocalReconciliationAutomationDecision( if ( (head.state === 'reconciliation_automation_planned' && (head.headDigest !== command.request.expectedHeadDigest || - head.sourceRecordDigest !== intent.command.request.expectedAutomationPlanDigest)) || + head.sourceRecordDigest !== + intent.command.request.expectedAutomationPlanDigest)) || (head.state === 'reconciliation_automation_decision_prepared' && head.sourceRecordDigest !== intent.preparationDigest) || (head.state !== 'reconciliation_automation_planned' && @@ -453,7 +466,8 @@ export async function prepareLocalReconciliationAutomationDecision( ensurePrivateDirectory(selected.root, uid, 'automationDecisionDirectory'); ensurePrivateDirectory(selected.staging, uid, 'automationDecisionStaging'); validateCatalog(selected, false); - const contents = localReconciliationAutomationDecisionEvidenceContents(intent); + const contents = + localReconciliationAutomationDecisionEvidenceContents(intent); preflightPublishedFile( selected.intent, contents, @@ -505,7 +519,9 @@ function validateCommitBinding( function strongReviewer( authenticated: Readonly, - original: Readonly, + original: Readonly< + DecisionContext['application']['review']['authorization']['header']['reviewer'] + >, committedAtMs: number, ) { const principal = authenticated.principal; @@ -598,15 +614,16 @@ async function authorization( ); if ( capture.command.request.targetDatabasePath !== - command.options.targetDatabasePath + command.options.targetDatabasePath ) { configurationError('authentication database is detached from capture'); } - const openRequirements = createLocalReconciliationAutomationRequirementFactory( - selected.automation.planPath, - selected.automation.receipt, - uid, - ); + const openRequirements = + createLocalReconciliationAutomationRequirementFactory( + selected.automation.planPath, + selected.automation.receipt, + uid, + ); const keyringPath = selected.application.intent.command.options.issuerKeyringPath; ensureLocalReconciliationReviewIssuerKeyring(keyringPath); @@ -774,11 +791,12 @@ async function verifyPublication( allowedParentModes: readonly (0o500 | 0o700)[], ): Promise> { const terminal = planTerminal(selected); - const openRequirements = createLocalReconciliationAutomationRequirementFactory( - selected.automation.planPath, - selected.automation.receipt, - uid, - ); + const openRequirements = + createLocalReconciliationAutomationRequirementFactory( + selected.automation.planPath, + selected.automation.receipt, + uid, + ); const keyProvider = new LocalReconciliationReviewIssuerKeyringFileProvider( selected.application.intent.command.options.issuerKeyringPath, ); @@ -862,8 +880,7 @@ export async function commitLocalReconciliationAutomationDecision( receipt.preparedHeadDigest !== command.request.expectedHeadDigest || receipt.issuedAtMs !== command.request.committedAtMs || receipt.expiresAtMs !== - command.request.committedAtMs + - command.request.authorizationLifetimeMs + command.request.committedAtMs + command.request.authorizationLifetimeMs ) { configurationError('terminal receipt is not an exact command replay'); } @@ -1013,6 +1030,58 @@ export async function verifyLocalReconciliationAutomationDecision( return terminalResult(command.operation, 'verified', receipt, head); } +export async function readLocalReconciliationAutomationDecisionTerminal( + options: Readonly< + LocalReconciliationAutomationDecisionPrepareCommand['options'] + >, + automationId: string, + uid: number, + dependencies: LocalReconciliationAutomationDecisionDependencies = {}, +): Promise> { + for (const [directory, label] of [ + [options.deploymentRoot, 'deploymentRoot'], + [options.applicationRoot, 'applicationRoot'], + [options.automationRoot, 'automationRoot'], + [options.automationDecisionRoot, 'automationDecisionRoot'], + ] as const) { + validatePrivateDirectory(directory, uid, label); + } + const selectedPaths = paths(options.automationDecisionRoot, automationId); + validateDirectory(selectedPaths.root, uid, [0o500], 'decision root'); + validateDirectory(selectedPaths.staging, uid, [0o500], 'decision staging'); + validateCatalog(selectedPaths, true); + const intent = readIntent(selectedPaths, uid, [0o400]); + const receipt = readReceipt(selectedPaths, uid, [0o400]); + if ( + intent.command.request.automationId !== automationId || + receipt.automationId !== automationId || + receipt.decisionId !== intent.command.request.decisionId + ) { + configurationError('terminal decision identity drifted'); + } + const selected = await context(options, automationId, uid); + intentBinding(intent, selected); + const publication = await verifyPublication( + selectedPaths, + intent, + selected, + receipt, + dependencies, + uid, + [0o400], + [0o500], + ); + validateReceiptBinding(receipt, intent, publication); + return Object.freeze({ + intent, + receipt, + authorizationPath: selectedPaths.authorization, + context: selected, + plan: planTerminal(selected), + reviewer: publication.authorization.receipt.reviewer, + }); +} + export function prepareLocalReconciliationAutomationDecisionCommandFile( filePath: string, dependencies: LocalReconciliationAutomationDecisionDependencies = {}, diff --git a/packages/ql3-local-owner-cli/test/reconciliationCapturePrepare.test.cjs b/packages/ql3-local-owner-cli/test/reconciliationCapturePrepare.test.cjs index 6066dc46..da8bf51b 100644 --- a/packages/ql3-local-owner-cli/test/reconciliationCapturePrepare.test.cjs +++ b/packages/ql3-local-owner-cli/test/reconciliationCapturePrepare.test.cjs @@ -11,17 +11,21 @@ const { commitLocalReconciliationCapture, commitLocalReconciliationApplication, commitLocalReconciliationAutomationDecision, + applyLocalReconciliationAutomation, commitLocalReconciliationPlan, commitLocalReconciliationReview, prepareLocalReconciliationCapture, prepareLocalReconciliationApplication, prepareLocalReconciliationAutomationDecision, + readLocalReconciliationAutomationDecisionTerminal, + rollbackLocalReconciliationAutomationApply, planLocalReconciliationAutomation, prepareLocalReconciliationPlan, prepareLocalReconciliationReview, verifyLocalReconciliationCapture, verifyLocalReconciliationApplication, verifyLocalReconciliationAutomationDecision, + verifyLocalReconciliationAutomationApply, verifyLocalReconciliationAutomationPlan, verifyLocalReconciliationPlan, verifyLocalReconciliationReview, @@ -703,6 +707,41 @@ function automationDatabaseInitializer() { }; } +function automationReadyDatabaseInitializer() { + const initializeLegacy = automationDatabaseInitializer(); + return (paths) => { + initializeLegacy(paths); + fs.truncateSync(paths.targetDatabasePath, 0); + const migration = spawnSync( + process.execPath, + [ + '-e', + `require('@qinglong/local-sqlite/migration') + .migrateLocalSqlitePath({ databasePath: process.argv[1], profile: 'edge' }) + .catch((error) => { console.error(error); process.exitCode = 1; });`, + paths.targetDatabasePath, + ], + { encoding: 'utf8', cwd: path.join(__dirname, '..') }, + ); + assert.equal(migration.status, 0, migration.stderr); + fs.chmodSync(paths.targetDatabasePath, 0o600); + const target = new DatabaseSync(paths.targetDatabasePath); + target.exec(` + INSERT INTO "QingLong3ProjectRoleBindings" ( + "project_id", "subject_type", "subject_id", "version", "state", + "role", "mutation_id", "changed_by_type", "changed_by_id", + "created_at_ms" + ) VALUES ( + 'default', 'user', 'review-owner', 1, 'active', 'owner', + 'automation-apply-owner-binding', 'user', 'review-owner', 1 + ); + PRAGMA wal_checkpoint(TRUNCATE); + PRAGMA journal_mode=DELETE; + `); + target.close(); + }; +} + function mutateAutomationTarget({ targetDatabasePath }, occupied = false) { const target = new DatabaseSync(targetDatabasePath); if (occupied) { @@ -719,6 +758,13 @@ function mutateAutomationTarget({ targetDatabasePath }, occupied = false) { return Object.freeze({}); } +function mutateReadyAutomationTarget({ targetDatabasePath }) { + const target = new DatabaseSync(targetDatabasePath); + target.exec('PRAGMA user_version=1'); + target.close(); + return Object.freeze({}); +} + function mutatePlanningTarget({ targetDatabasePath }) { const target = new DatabaseSync(targetDatabasePath); target.exec('INSERT INTO "QingLong3TaskDefinitions" (id) VALUES (1)'); @@ -1080,9 +1126,14 @@ async function plannedAutomationFixture(t, options = {}) { applicationId: options.applicationId, reviewSuffix: `automation-decision-${suffix}`, createDefaultSidecars: false, - initializeDatabases: automationDatabaseInitializer(), + initializeDatabases: + options.readyTarget === true + ? automationReadyDatabaseInitializer() + : automationDatabaseInitializer(), mutateTarget(paths) { - return mutateAutomationTarget(paths, options.occupied === true); + return options.readyTarget === true + ? mutateReadyAutomationTarget(paths) + : mutateAutomationTarget(paths, options.occupied === true); }, mutateDecisions(records) { const selected = records.find( @@ -1094,8 +1145,10 @@ async function plannedAutomationFixture(t, options = {}) { record.disposition === 'exclude_legacy', ); assert.ok(selected); - selected.disposition = options.occupied === true ? 'retain_both' : 'adopt_legacy'; - selected.reason = options.occupied === true ? 'preserve_both' : 'prefer_legacy'; + selected.disposition = + options.occupied === true ? 'retain_both' : 'adopt_legacy'; + selected.reason = + options.occupied === true ? 'preserve_both' : 'prefer_legacy'; }, }); const preparedApplication = await prepareLocalReconciliationApplication( @@ -1242,7 +1295,8 @@ function automationDecisionCommitFixture( options: { ...prepared.commandOptions, targetDatabasePath: state.targetDatabasePath, - ownerPepperKeyringDirectory: state.command.options.ownerPepperKeyringDirectory, + ownerPepperKeyringDirectory: + state.command.options.ownerPepperKeyringDirectory, credentialFilePath: state.command.options.credentialFilePath, }, request: { @@ -2885,7 +2939,10 @@ test('automation adapter builds a sealed row plan with bounded conflict evidence const application = await commitLocalReconciliationApplication( applicationCommitCommand(state, prepared), ); - const automationRoot = path.join(path.dirname(state.captureRoot), 'automation-root'); + const automationRoot = path.join( + path.dirname(state.captureRoot), + 'automation-root', + ); fs.mkdirSync(automationRoot, { mode: 0o700 }); const targetBefore = fs.readFileSync(state.targetDatabasePath); const command = { @@ -2944,7 +3001,10 @@ test('automation adapter builds a sealed row plan with bounded conflict evidence assert.equal(planText.includes('legacy-cron:1'), true); assert.equal(planText.includes('review-owner'), false); assert.equal(planText.includes('"requirement":"review_adopt"'), true); - assert.equal(fs.readFileSync(state.targetDatabasePath).equals(targetBefore), true); + assert.equal( + fs.readFileSync(state.targetDatabasePath).equals(targetBefore), + true, + ); const replay = await planLocalReconciliationAutomation(command); assert.equal(replay.status, 'existing'); @@ -2994,6 +3054,7 @@ test('automation decision reauthenticates the same reviewer, seals exact row dec reviewId: '00000000-0000-4000-8000-000000000462', applicationId: '00000000-0000-4000-8000-000000000463', automationId: '00000000-0000-4000-8000-000000000464', + readyTarget: true, }); assert.equal(state.planRows[0].requirement, 'review_adopt'); const decisionId = '019b0000-0000-7000-8000-000000000461'; @@ -3067,11 +3128,28 @@ test('automation decision reauthenticates the same reviewer, seals exact row dec verifyCommand, ); assert.equal(verified.status, 'verified'); - assert.equal(verified.signedDecisionSetDigest, committed.signedDecisionSetDigest); + assert.equal( + verified.signedDecisionSetDigest, + committed.signedDecisionSetDigest, + ); const serialized = JSON.stringify(verified); assert.equal(serialized.includes('review-owner'), false); assert.equal(serialized.includes(state.planRows[0].sourceDigest), false); assert.equal(serialized.includes(review.filePath), false); + const terminal = await readLocalReconciliationAutomationDecisionTerminal( + prepareCommand.options, + state.automationCommand.request.automationId, + process.getuid(), + ); + assert.equal(terminal.receipt.decisionDigest, committed.decisionDigest); + assert.equal( + terminal.context.application.plan.applicationPlanDigest, + state.application.applicationPlanDigest, + ); + assert.equal( + terminal.authorizationPath.endsWith('/authorization.ndjson'), + true, + ); const commandPath = path.join( state.deploymentRoot, 'automation-decision-verify.json', @@ -3093,6 +3171,249 @@ test('automation decision reauthenticates the same reviewer, seals exact row dec assert.equal(JSON.parse(cli.stdout).status, 'verified'); assert.equal(cli.stdout.includes('review-owner'), false); assert.equal(cli.stdout.includes(state.planRows[0].sourceDigest), false); + + const automationApplyRoot = path.join( + path.dirname(state.captureRoot), + 'automation-apply-signed-success', + ); + fs.mkdirSync(automationApplyRoot, { mode: 0o700 }); + const appliedAtMs = commit.command.request.committedAtMs + 1; + const targetIdentity = fs.statSync(state.targetDatabasePath); + const applyOptions = { + ...prepareCommand.options, + automationApplyRoot, + targetDatabasePath: state.targetDatabasePath, + ownerPepperKeyringDirectory: + state.command.options.ownerPepperKeyringDirectory, + credentialFilePath: state.command.options.credentialFilePath, + }; + const applyCommand = { + schemaVersion: 1, + operation: 'local.deployment.reconciliation.automation.apply', + options: applyOptions, + request: { + decisionId, + automationId: state.automationCommand.request.automationId, + expectedDecisionDigest: committed.decisionDigest, + expectedHeadDigest: committed.instanceHeadDigest, + mutationId: '00000000-0000-4000-8000-000000000465', + requestId: 'automation-apply-signed-success', + appliedAtMs, + }, + }; + const applyDependencies = { + async openAuthenticationDatabase() { + return { async close() {} }; + }, + async authenticate(_database, options) { + assert.equal( + options.authenticationNamespace, + 'local_reconciliation_automation_apply', + ); + const authenticatedAtMs = options.now(); + return { + principal: { + subject: { type: 'user', id: 'review-owner' }, + authenticationId: 'local_reconciliation_automation_apply:test', + authenticatedAtMs, + expiresAtMs: authenticatedAtMs + 60 * 60 * 1_000, + assurance: 'local_console', + }, + databaseFence: { + credentialId: 'review-owner', + credentialVersion: 1, + pepperKeyId: 'review-owner-v1', + pepperVersion: 1, + }, + async confirm() {}, + }; + }, + }; + const liveApplyEvidence = readTargetDataReconciliationEvidenceForPaths( + { + profile: state.captureCommand.request.profile, + activationPath: state.captureCommand.request.activationPath, + legacySourcePath: state.captureCommand.request.legacySourcePath, + targetDatabasePath: state.targetDatabasePath, + expectedActivationDigest: + state.captureCommand.request.expectedActivationDigest, + }, + process.getuid(), + ); + assert.equal( + liveApplyEvidence.disposition, + 'reconciliation_required', + JSON.stringify({ + liveApplyEvidence, + activation: JSON.parse( + fs.readFileSync(state.captureCommand.request.activationPath, 'utf8'), + ), + target: { + ...fs.statSync(state.targetDatabasePath), + mode: fs.statSync(state.targetDatabasePath).mode & 0o777, + realpath: fs.realpathSync(state.targetDatabasePath), + sidecars: ['-wal', '-shm', '-journal'].map((suffix) => + fs.existsSync(`${state.targetDatabasePath}${suffix}`), + ), + }, + }), + ); + for (const boundary of ['afterBackupPublished', 'afterPreparedHead']) { + await assert.rejects( + applyLocalReconciliationAutomation(applyCommand, { + ...applyDependencies, + [boundary]() { + throw new Error(`automation apply ${boundary} response loss`); + }, + }), + new RegExp(`automation apply ${boundary} response loss`), + ); + } + await assert.rejects( + applyLocalReconciliationAutomation(applyCommand, { + ...applyDependencies, + async authenticate(database, options) { + const authenticated = await applyDependencies.authenticate( + database, + options, + ); + return { + ...authenticated, + principal: { + ...authenticated.principal, + subject: { type: 'user', id: 'another-owner' }, + }, + }; + }, + }), + /current reviewer authentication is not strong or identical/, + ); + for (const boundary of ['afterDatabaseCommit', 'afterReceiptPublished']) { + await assert.rejects( + applyLocalReconciliationAutomation(applyCommand, { + ...applyDependencies, + [boundary]() { + throw new Error(`automation apply ${boundary} response loss`); + }, + }), + new RegExp(`automation apply ${boundary} response loss`), + ); + } + const applied = await applyLocalReconciliationAutomation( + applyCommand, + applyDependencies, + ); + assert.equal(applied.status, 'existing'); + assert.equal(applied.state, 'reconciliation_automation_applied'); + assert.equal(applied.adoptedTaskCount, 1); + assert.equal(fs.statSync(state.targetDatabasePath).ino, targetIdentity.ino); + const applyReplay = await applyLocalReconciliationAutomation( + applyCommand, + applyDependencies, + ); + assert.equal(applyReplay.status, 'existing'); + const applyVerified = await verifyLocalReconciliationAutomationApply({ + schemaVersion: 1, + operation: 'local.deployment.reconciliation.automation.apply.verify', + options: applyOptions, + request: { + decisionId, + automationId: state.automationCommand.request.automationId, + expectedApplyDigest: applied.applyDigest, + }, + }); + assert.equal(applyVerified.status, 'verified'); + const rollbackCommand = { + schemaVersion: 1, + operation: 'local.deployment.reconciliation.automation.apply.rollback', + options: applyOptions, + request: { + decisionId, + automationId: state.automationCommand.request.automationId, + expectedApplyDigest: applied.applyDigest, + expectedHeadDigest: applied.instanceHeadDigest, + rolledBackAtMs: appliedAtMs + 1, + }, + }; + for (const boundary of ['afterRestore', 'afterRollbackReceipt']) { + await assert.rejects( + rollbackLocalReconciliationAutomationApply(rollbackCommand, { + ...applyDependencies, + [boundary]() { + throw new Error(`automation rollback ${boundary} response loss`); + }, + }), + new RegExp(`automation rollback ${boundary} response loss`), + ); + } + const rolledBack = await rollbackLocalReconciliationAutomationApply( + rollbackCommand, + applyDependencies, + ); + assert.equal(rolledBack.status, 'existing'); + assert.equal(rolledBack.state, 'reconciliation_automation_rolled_back'); + assert.equal(fs.statSync(state.targetDatabasePath).ino, targetIdentity.ino); + assert.equal( + ( + await rollbackLocalReconciliationAutomationApply( + rollbackCommand, + applyDependencies, + ) + ).status, + 'existing', + ); + const rollbackVerified = await verifyLocalReconciliationAutomationApply({ + schemaVersion: 1, + operation: 'local.deployment.reconciliation.automation.apply.verify', + options: applyOptions, + request: { + decisionId, + automationId: state.automationCommand.request.automationId, + expectedApplyDigest: applied.applyDigest, + }, + }); + assert.equal(rollbackVerified.status, 'verified'); + assert.equal(rollbackVerified.state, 'reconciliation_automation_rolled_back'); + const applyVerifyPath = path.join( + state.deploymentRoot, + 'automation-apply-verify.json', + ); + fs.writeFileSync( + applyVerifyPath, + `${JSON.stringify({ + schemaVersion: 1, + operation: 'local.deployment.reconciliation.automation.apply.verify', + options: applyOptions, + request: { + decisionId, + automationId: state.automationCommand.request.automationId, + expectedApplyDigest: applied.applyDigest, + }, + })}\n`, + { mode: 0o600 }, + ); + const applyCli = spawnSync( + process.execPath, + [ + path.join(__dirname, '../dist/deployment/localDeploymentCli.js'), + 'reconciliation-automation-apply-verify', + '--command-file', + applyVerifyPath, + ], + { encoding: 'utf8' }, + ); + assert.equal(applyCli.status, 0, applyCli.stderr); + assert.equal(JSON.parse(applyCli.stdout).status, 'verified'); + const restored = new DatabaseSync(state.targetDatabasePath, { + readOnly: true, + }); + assert.equal( + restored + .prepare('SELECT COUNT(*) AS count FROM "QingLong3LegacyAdoptions"') + .get().count, + 0, + ); + restored.close(); }); test('automation decision rejects conflict adoption, another reviewer and weak assurance', async (t) => { @@ -3294,8 +3615,7 @@ test('automation decision verification rejects sealed authorization and plan dri suffix: 'authorization-drift', automationId: '00000000-0000-4000-8000-000000000475', }); - const authorizationDecisionId = - '019b0000-0000-7000-8000-000000000475'; + const authorizationDecisionId = '019b0000-0000-7000-8000-000000000475'; const authorizationReview = automationDecisionReviewFile( authorizationState, authorizationDecisionId, @@ -3483,7 +3803,9 @@ test('automation row plan fails closed to manual review on a target task collisi }); test('automation row planner makes an empty table no-effect and a missing timezone manual', (t) => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-automation-row-unit-')); + const root = fs.mkdtempSync( + path.join(os.tmpdir(), 'ql3-automation-row-unit-'), + ); t.after(() => removeFixtureRoot(root)); const legacySourcePath = path.join(root, 'legacy.sqlite'); const recoveryPath = path.join(root, 'recovery.sqlite'); @@ -3548,7 +3870,10 @@ test('automation row planner makes an empty table no-effect and a missing timezo manual = writeLocalReconciliationAutomationPlan({ descriptor: manualDescriptor, maxBytes: 64 * 1024, - header: { ...header, automationId: '00000000-0000-4000-8000-00000000043b' }, + header: { + ...header, + automationId: '00000000-0000-4000-8000-00000000043b', + }, legacy, target, }); @@ -3639,7 +3964,13 @@ test('automation row planning replays every publication boundary and rejects dri ['seal', '000000000449', '00000000044a', '00000000044b', '00000000044c'], ['head', '00000000044d', '00000000044e', '00000000044f', '000000000450'], ]; - for (const [window, planTail, reviewTail, applicationTail, automationTail] of windows) { + for (const [ + window, + planTail, + reviewTail, + applicationTail, + automationTail, + ] of windows) { const state = await reviewedApplicationFixture(t, { planId: `00000000-0000-4000-8000-${planTail}`, reviewId: `00000000-0000-4000-8000-${reviewTail}`, diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index ad301f17..b27e04b9 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -207,10 +207,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: localOwnerCli.rootSourceFileRoles, }, { - sourceFiles: 165, + sourceFiles: 168, rootSourceFiles: 1, rootSourceLines: 50, - nestedSourceFiles: 164, + nestedSourceFiles: 167, rootSourceFileRoles: { 'cli.ts': 'binary_entry' }, }, );