mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(local): fence reconciliation completion
This commit is contained in:
whyour
@@ -37,6 +37,7 @@ export type LocalCutoverInstanceHeadState =
|
||||
| 'reconciliation_automation_apply_prepared'
|
||||
| 'reconciliation_automation_applied'
|
||||
| 'reconciliation_automation_rolled_back'
|
||||
| 'reconciliation_completed'
|
||||
| 'rollback_prepared'
|
||||
| 'legacy_restart_requested'
|
||||
| 'legacy_running'
|
||||
@@ -182,6 +183,7 @@ function parseHead(value: unknown): Readonly<LocalCutoverInstanceHead> {
|
||||
head.state !== 'reconciliation_automation_apply_prepared' &&
|
||||
head.state !== 'reconciliation_automation_applied' &&
|
||||
head.state !== 'reconciliation_automation_rolled_back' &&
|
||||
head.state !== 'reconciliation_completed' &&
|
||||
head.state !== 'rollback_prepared' &&
|
||||
head.state !== 'legacy_restart_requested' &&
|
||||
head.state !== 'legacy_running' &&
|
||||
@@ -362,6 +364,7 @@ export function advanceLocalCutoverInstanceHead(
|
||||
| 'reconciliation_automation_apply_prepared'
|
||||
| 'reconciliation_automation_applied'
|
||||
| 'reconciliation_automation_rolled_back'
|
||||
| 'reconciliation_completed'
|
||||
| 'rollback_prepared'
|
||||
| 'legacy_restart_requested'
|
||||
| 'legacy_running'
|
||||
@@ -414,6 +417,7 @@ export function advanceLocalCutoverInstanceHead(
|
||||
current.state === 'reconciliation_automation_apply_prepared' ||
|
||||
current.state === 'reconciliation_automation_applied' ||
|
||||
current.state === 'reconciliation_automation_rolled_back' ||
|
||||
current.state === 'reconciliation_completed' ||
|
||||
current.state === 'legacy_restart_requested' ||
|
||||
current.state === 'legacy_running' ||
|
||||
current.state === 'legacy_ready')
|
||||
@@ -427,7 +431,8 @@ export function advanceLocalCutoverInstanceHead(
|
||||
(state === 'legacy_stopped' && current.state === 'legacy_stop_requested') ||
|
||||
(state === 'target_active' &&
|
||||
(current.state === 'legacy_stopped' ||
|
||||
current.state === 'target_active')) ||
|
||||
current.state === 'target_active' ||
|
||||
current.state === 'reconciliation_completed')) ||
|
||||
(state === 'target_stopped' && current.state === 'target_active') ||
|
||||
(state === 'reconciliation_capture_prepared' &&
|
||||
current.state === 'target_stopped') ||
|
||||
@@ -457,6 +462,9 @@ export function advanceLocalCutoverInstanceHead(
|
||||
current.state === 'reconciliation_automation_apply_prepared') ||
|
||||
(state === 'reconciliation_automation_rolled_back' &&
|
||||
current.state === 'reconciliation_automation_applied') ||
|
||||
(state === 'reconciliation_completed' &&
|
||||
(current.state === 'reconciliation_application_planned' ||
|
||||
current.state === 'reconciliation_automation_applied')) ||
|
||||
(state === 'rollback_prepared' && current.state === 'target_stopped') ||
|
||||
(state === 'legacy_restart_requested' &&
|
||||
current.state === 'rollback_prepared') ||
|
||||
@@ -497,6 +505,7 @@ export function assertLocalCutoverTargetHead(
|
||||
head.activationDigest !== identity.request.expectedActivationDigest ||
|
||||
(head.state !== 'legacy_stopped' &&
|
||||
head.state !== 'target_active' &&
|
||||
head.state !== 'reconciliation_completed' &&
|
||||
head.state !== 'manual_required')
|
||||
) {
|
||||
configurationError(
|
||||
|
||||
@@ -154,6 +154,12 @@ import {
|
||||
verifyLocalReconciliationAutomationApply,
|
||||
verifyLocalReconciliationAutomationApplyCommandFile,
|
||||
} from './reconciliation/application/automation/applyCoordinator';
|
||||
import {
|
||||
completeLocalReconciliation,
|
||||
completeLocalReconciliationCommandFile,
|
||||
verifyLocalReconciliationCompletion,
|
||||
verifyLocalReconciliationCompletionCommandFile,
|
||||
} from './reconciliation/completion/coordinator';
|
||||
|
||||
export {
|
||||
commitLocalReconciliationPlan,
|
||||
@@ -193,8 +199,29 @@ export {
|
||||
verifyLocalReconciliationAutomationApplyCommandFile,
|
||||
rollbackLocalReconciliationAutomationApply,
|
||||
rollbackLocalReconciliationAutomationApplyCommandFile,
|
||||
completeLocalReconciliation,
|
||||
completeLocalReconciliationCommandFile,
|
||||
verifyLocalReconciliationCompletion,
|
||||
verifyLocalReconciliationCompletionCommandFile,
|
||||
};
|
||||
|
||||
export {
|
||||
normalizeLocalReconciliationCompleteCommand,
|
||||
normalizeLocalReconciliationCompletionVerifyCommand,
|
||||
type LocalReconciliationCompleteCommand,
|
||||
type LocalReconciliationCompletionAutomationBinding,
|
||||
type LocalReconciliationCompletionAutomationOptions,
|
||||
type LocalReconciliationCompletionOptions,
|
||||
type LocalReconciliationCompletionResult,
|
||||
type LocalReconciliationCompletionVerifyCommand,
|
||||
} from './reconciliation/completion/contract';
|
||||
export { type LocalReconciliationCompletionDependencies } from './reconciliation/completion/coordinator';
|
||||
export {
|
||||
normalizeLocalReconciliationCompletionReceipt,
|
||||
type LocalReconciliationCompletionDomainEvidence,
|
||||
type LocalReconciliationCompletionReceipt,
|
||||
} from './reconciliation/completion/evidence';
|
||||
|
||||
export {
|
||||
normalizeLocalReconciliationAutomationApplyCommand,
|
||||
normalizeLocalReconciliationAutomationApplyRollbackCommand,
|
||||
|
||||
@@ -33,6 +33,8 @@ import {
|
||||
applyLocalReconciliationAutomationCommandFile,
|
||||
verifyLocalReconciliationAutomationApplyCommandFile,
|
||||
rollbackLocalReconciliationAutomationApplyCommandFile,
|
||||
completeLocalReconciliationCommandFile,
|
||||
verifyLocalReconciliationCompletionCommandFile,
|
||||
writeLocalReconciliationReviewDiagnosticsCommandFile,
|
||||
prepareLocalDeploymentCommandFile,
|
||||
proveLocalDeploymentLegacyReadinessCommandFile,
|
||||
@@ -48,7 +50,7 @@ import {
|
||||
} from './localDeployment';
|
||||
|
||||
const USAGE =
|
||||
'Usage: ql3-local-deploy <prepare|adopted-prepare|adopted-verify|status|service-intent-prepare|service-outcome-consume|service-cutover-consume|service-legacy-rollback-prepare|service-legacy-rollback-authorize|service-legacy-rollback-consume|cutover-legacy-stop|cutover-target-start|cutover-target-restart|cutover-target-stop|cutover-legacy-rollback-prepare|cutover-legacy-rollback-commit|cutover-legacy-readiness-probe|cutover-manual-diagnose|cutover-manual-resolution-prepare|cutover-manual-resolution-commit|reconciliation-capture-prepare|reconciliation-capture-commit|reconciliation-capture-verify|reconciliation-plan-prepare|reconciliation-plan-commit|reconciliation-plan-verify|reconciliation-review-prepare|reconciliation-review-diagnostics|reconciliation-review-commit|reconciliation-review-verify|reconciliation-application-prepare|reconciliation-application-commit|reconciliation-application-verify|reconciliation-automation-plan|reconciliation-automation-verify|reconciliation-automation-decision-prepare|reconciliation-automation-decision-commit|reconciliation-automation-decision-verify|reconciliation-automation-apply|reconciliation-automation-apply-verify|reconciliation-automation-apply-rollback|compose-revision|compose-preflight|compose-apply|compose-restore-prepare|compose-restore-commit|compose-evidence-collect-prepare|compose-evidence-collect-commit> --command-file /absolute/private-command.json';
|
||||
'Usage: ql3-local-deploy <prepare|adopted-prepare|adopted-verify|status|service-intent-prepare|service-outcome-consume|service-cutover-consume|service-legacy-rollback-prepare|service-legacy-rollback-authorize|service-legacy-rollback-consume|cutover-legacy-stop|cutover-target-start|cutover-target-restart|cutover-target-stop|cutover-legacy-rollback-prepare|cutover-legacy-rollback-commit|cutover-legacy-readiness-probe|cutover-manual-diagnose|cutover-manual-resolution-prepare|cutover-manual-resolution-commit|reconciliation-capture-prepare|reconciliation-capture-commit|reconciliation-capture-verify|reconciliation-plan-prepare|reconciliation-plan-commit|reconciliation-plan-verify|reconciliation-review-prepare|reconciliation-review-diagnostics|reconciliation-review-commit|reconciliation-review-verify|reconciliation-application-prepare|reconciliation-application-commit|reconciliation-application-verify|reconciliation-automation-plan|reconciliation-automation-verify|reconciliation-automation-decision-prepare|reconciliation-automation-decision-commit|reconciliation-automation-decision-verify|reconciliation-automation-apply|reconciliation-automation-apply-verify|reconciliation-automation-apply-rollback|reconciliation-complete|reconciliation-complete-verify|compose-revision|compose-preflight|compose-apply|compose-restore-prepare|compose-restore-commit|compose-evidence-collect-prepare|compose-evidence-collect-commit> --command-file /absolute/private-command.json';
|
||||
|
||||
async function main(argv: readonly string[]): Promise<void> {
|
||||
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
|
||||
@@ -98,6 +100,8 @@ async function main(argv: readonly string[]): Promise<void> {
|
||||
argv[0] !== 'reconciliation-automation-apply' &&
|
||||
argv[0] !== 'reconciliation-automation-apply-verify' &&
|
||||
argv[0] !== 'reconciliation-automation-apply-rollback' &&
|
||||
argv[0] !== 'reconciliation-complete' &&
|
||||
argv[0] !== 'reconciliation-complete-verify' &&
|
||||
argv[0] !== 'compose-revision' &&
|
||||
argv[0] !== 'compose-preflight' &&
|
||||
argv[0] !== 'compose-apply' &&
|
||||
@@ -210,6 +214,10 @@ async function main(argv: readonly string[]): Promise<void> {
|
||||
? verifyLocalReconciliationAutomationApplyCommandFile(argv[2]!)
|
||||
: argv[0] === 'reconciliation-automation-apply-rollback'
|
||||
? rollbackLocalReconciliationAutomationApplyCommandFile(argv[2]!)
|
||||
: argv[0] === 'reconciliation-complete'
|
||||
? completeLocalReconciliationCommandFile(argv[2]!)
|
||||
: argv[0] === 'reconciliation-complete-verify'
|
||||
? verifyLocalReconciliationCompletionCommandFile(argv[2]!)
|
||||
: argv[0] === 'compose-revision'
|
||||
? switchLocalDeploymentComposeRevisionCommandFile(argv[2]!)
|
||||
: argv[0] === 'compose-preflight'
|
||||
|
||||
+49
@@ -503,6 +503,55 @@ export function validateLocalReconciliationAutomationAppliedStorage(
|
||||
validateBackup(selected, intent, uid, [0o400]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects the rollback copy only after the cross-domain completion head is
|
||||
* durable. The operation accepts its own partially collected layout so a
|
||||
* response loss or process crash can be replayed without restoring authority.
|
||||
*/
|
||||
export function collectLocalReconciliationAutomationCompletedStorage(
|
||||
selected: Readonly<LocalReconciliationAutomationApplyPaths>,
|
||||
intent: Readonly<LocalReconciliationAutomationApplyIntent>,
|
||||
uid: number,
|
||||
): void {
|
||||
directoryMode(selected.root, uid, [0o500], 'root');
|
||||
directoryMode(selected.backupRoot, uid, [0o700, 0o500], 'backup root');
|
||||
directoryMode(
|
||||
selected.rollbackRoot,
|
||||
uid,
|
||||
[0o700, 0o500],
|
||||
'rollback work root',
|
||||
);
|
||||
validateLocalReconciliationAutomationApplyCatalog(selected);
|
||||
emptyDirectory(selected.rollbackRoot, 'rollback work root');
|
||||
stableJson(selected.intent, uid, [0o400], 'intent', [1n]);
|
||||
stableJson(selected.receipt, uid, [0o400], 'receipt', [1n]);
|
||||
if (fs.existsSync(selected.backup)) {
|
||||
validateBackup(selected, intent, uid, [0o400]);
|
||||
if ((fs.statSync(selected.backupRoot).mode & 0o777) !== 0o700) {
|
||||
fs.chmodSync(selected.backupRoot, 0o700);
|
||||
syncDirectory(selected.root);
|
||||
}
|
||||
unlinkIfPresent(selected.backup);
|
||||
}
|
||||
sealDirectory(selected.backupRoot, uid, 'backup root');
|
||||
sealDirectory(selected.rollbackRoot, uid, 'rollback work root');
|
||||
validateLocalReconciliationAutomationCompletedStorage(selected, uid);
|
||||
}
|
||||
|
||||
export function validateLocalReconciliationAutomationCompletedStorage(
|
||||
selected: Readonly<LocalReconciliationAutomationApplyPaths>,
|
||||
uid: number,
|
||||
): void {
|
||||
directoryMode(selected.root, uid, [0o500], 'root');
|
||||
directoryMode(selected.backupRoot, uid, [0o500], 'backup root');
|
||||
directoryMode(selected.rollbackRoot, uid, [0o500], 'rollback work root');
|
||||
validateLocalReconciliationAutomationApplyCatalog(selected);
|
||||
emptyDirectory(selected.backupRoot, 'backup root');
|
||||
emptyDirectory(selected.rollbackRoot, 'rollback work root');
|
||||
stableJson(selected.intent, uid, [0o400], 'intent', [1n]);
|
||||
stableJson(selected.receipt, uid, [0o400], 'receipt', [1n]);
|
||||
}
|
||||
|
||||
export function prepareLocalReconciliationAutomationRollbackSource(
|
||||
selected: Readonly<LocalReconciliationAutomationApplyPaths>,
|
||||
intent: Readonly<LocalReconciliationAutomationApplyIntent>,
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
import path from 'node:path';
|
||||
|
||||
import { currentIdentity } from '../../foundation/contract';
|
||||
import { LocalDeploymentConfigurationError } from '../../foundation/error';
|
||||
|
||||
const DIGEST = /^[0-9a-f]{64}$/;
|
||||
const UUID_V4 =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const UUID_V7 =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const SAFE_PATH = /^\/[A-Za-z0-9._/@-]+$/;
|
||||
|
||||
export interface LocalReconciliationCompletionAutomationOptions {
|
||||
readonly automationRoot: string;
|
||||
readonly automationDecisionRoot: string;
|
||||
readonly automationApplyRoot: string;
|
||||
readonly targetDatabasePath: string;
|
||||
}
|
||||
|
||||
export interface LocalReconciliationCompletionOptions {
|
||||
readonly deploymentRoot: string;
|
||||
readonly applicationRoot: string;
|
||||
readonly completionRoot: string;
|
||||
readonly automation: Readonly<LocalReconciliationCompletionAutomationOptions> | null;
|
||||
readonly allowRootService: boolean;
|
||||
}
|
||||
|
||||
export interface LocalReconciliationCompletionAutomationBinding {
|
||||
readonly automationId: string;
|
||||
readonly decisionId: string;
|
||||
readonly expectedApplyDigest: string;
|
||||
}
|
||||
|
||||
export interface LocalReconciliationCompleteCommand {
|
||||
readonly schemaVersion: 1;
|
||||
readonly operation: 'local.deployment.reconciliation.complete';
|
||||
readonly options: Readonly<LocalReconciliationCompletionOptions>;
|
||||
readonly request: Readonly<{
|
||||
completionId: string;
|
||||
applicationId: string;
|
||||
expectedApplicationPlanDigest: string;
|
||||
expectedHeadDigest: string;
|
||||
automation: Readonly<LocalReconciliationCompletionAutomationBinding> | null;
|
||||
completedAtMs: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface LocalReconciliationCompletionVerifyCommand {
|
||||
readonly schemaVersion: 1;
|
||||
readonly operation: 'local.deployment.reconciliation.complete.verify';
|
||||
readonly options: Readonly<LocalReconciliationCompletionOptions>;
|
||||
readonly request: Readonly<{
|
||||
completionId: string;
|
||||
applicationId: string;
|
||||
expectedCompletionDigest: string;
|
||||
automation: Readonly<LocalReconciliationCompletionAutomationBinding> | null;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface LocalReconciliationCompletionResult {
|
||||
readonly schemaVersion: 1;
|
||||
readonly operation:
|
||||
| LocalReconciliationCompleteCommand['operation']
|
||||
| LocalReconciliationCompletionVerifyCommand['operation'];
|
||||
readonly status: 'completed' | 'existing' | 'verified';
|
||||
readonly state: 'reconciliation_completed';
|
||||
readonly completionId: string;
|
||||
readonly applicationId: string;
|
||||
readonly completionDigest: string;
|
||||
readonly domainCount: 8;
|
||||
readonly adapterCount: 0 | 1;
|
||||
readonly instanceHeadDigest: string;
|
||||
}
|
||||
|
||||
function fail(message: string): never {
|
||||
throw new LocalDeploymentConfigurationError(
|
||||
`reconciliation completion ${message}`,
|
||||
);
|
||||
}
|
||||
|
||||
function record(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
fail(`${label} must be an object`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function exact(
|
||||
value: Record<string, unknown>,
|
||||
keys: readonly string[],
|
||||
label: string,
|
||||
): void {
|
||||
const actual = Object.keys(value).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
fail(`${label} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function safePath(value: unknown, label: string): string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
!path.isAbsolute(value) ||
|
||||
path.parse(value).root === value ||
|
||||
path.normalize(value) !== value ||
|
||||
value.includes('\0') ||
|
||||
value.includes('//') ||
|
||||
!SAFE_PATH.test(value) ||
|
||||
Buffer.byteLength(value, 'utf8') > 4_096
|
||||
) {
|
||||
fail(`${label} must be a safe non-root absolute path`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function overlaps(left: string, right: string): boolean {
|
||||
const relative = path.relative(left, right);
|
||||
return (
|
||||
relative === '' ||
|
||||
(!relative.startsWith('..') && !path.isAbsolute(relative))
|
||||
);
|
||||
}
|
||||
|
||||
function identifier(value: unknown, pattern: RegExp, label: string): string {
|
||||
if (typeof value !== 'string' || !pattern.test(value)) {
|
||||
fail(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function digest(value: unknown, label: string): string {
|
||||
return identifier(value, DIGEST, label);
|
||||
}
|
||||
|
||||
function normalizeAutomationOptions(
|
||||
value: unknown,
|
||||
): Readonly<LocalReconciliationCompletionAutomationOptions> | null {
|
||||
if (value === null) return null;
|
||||
const selected = record(value, 'automation options');
|
||||
exact(
|
||||
selected,
|
||||
[
|
||||
'automationApplyRoot',
|
||||
'automationDecisionRoot',
|
||||
'automationRoot',
|
||||
'targetDatabasePath',
|
||||
],
|
||||
'automation options',
|
||||
);
|
||||
return Object.freeze({
|
||||
automationRoot: safePath(selected.automationRoot, 'automationRoot'),
|
||||
automationDecisionRoot: safePath(
|
||||
selected.automationDecisionRoot,
|
||||
'automationDecisionRoot',
|
||||
),
|
||||
automationApplyRoot: safePath(
|
||||
selected.automationApplyRoot,
|
||||
'automationApplyRoot',
|
||||
),
|
||||
targetDatabasePath: safePath(
|
||||
selected.targetDatabasePath,
|
||||
'targetDatabasePath',
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeOptions(
|
||||
value: unknown,
|
||||
): Readonly<LocalReconciliationCompletionOptions> {
|
||||
const selected = record(value, 'options');
|
||||
exact(
|
||||
selected,
|
||||
[
|
||||
'allowRootService',
|
||||
'applicationRoot',
|
||||
'automation',
|
||||
'completionRoot',
|
||||
'deploymentRoot',
|
||||
],
|
||||
'options',
|
||||
);
|
||||
if (
|
||||
typeof selected.allowRootService !== 'boolean' ||
|
||||
(currentIdentity().uid === 0) !== selected.allowRootService
|
||||
) {
|
||||
fail('command identity is invalid');
|
||||
}
|
||||
const automation = normalizeAutomationOptions(selected.automation);
|
||||
const normalized = Object.freeze({
|
||||
deploymentRoot: safePath(selected.deploymentRoot, 'deploymentRoot'),
|
||||
applicationRoot: safePath(selected.applicationRoot, 'applicationRoot'),
|
||||
completionRoot: safePath(selected.completionRoot, 'completionRoot'),
|
||||
automation,
|
||||
allowRootService: selected.allowRootService,
|
||||
}) as Readonly<LocalReconciliationCompletionOptions>;
|
||||
const roots = [
|
||||
normalized.deploymentRoot,
|
||||
normalized.applicationRoot,
|
||||
normalized.completionRoot,
|
||||
...(automation === null
|
||||
? []
|
||||
: [
|
||||
automation.automationRoot,
|
||||
automation.automationDecisionRoot,
|
||||
automation.automationApplyRoot,
|
||||
]),
|
||||
];
|
||||
for (let left = 0; left < roots.length; left += 1) {
|
||||
for (let right = left + 1; right < roots.length; right += 1) {
|
||||
if (
|
||||
overlaps(roots[left]!, roots[right]!) ||
|
||||
overlaps(roots[right]!, roots[left]!)
|
||||
) {
|
||||
fail('authority roots overlap');
|
||||
}
|
||||
}
|
||||
}
|
||||
if (
|
||||
automation !== null &&
|
||||
roots.some(
|
||||
(root) =>
|
||||
overlaps(root, automation.targetDatabasePath) ||
|
||||
overlaps(automation.targetDatabasePath, root),
|
||||
)
|
||||
) {
|
||||
fail('targetDatabasePath overlaps an authority root');
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function normalizeAutomationBinding(
|
||||
value: unknown,
|
||||
): Readonly<LocalReconciliationCompletionAutomationBinding> | null {
|
||||
if (value === null) return null;
|
||||
const selected = record(value, 'automation binding');
|
||||
exact(
|
||||
selected,
|
||||
['automationId', 'decisionId', 'expectedApplyDigest'],
|
||||
'automation binding',
|
||||
);
|
||||
return Object.freeze({
|
||||
automationId: identifier(selected.automationId, UUID_V4, 'automationId'),
|
||||
decisionId: identifier(selected.decisionId, UUID_V7, 'decisionId'),
|
||||
expectedApplyDigest: digest(
|
||||
selected.expectedApplyDigest,
|
||||
'expectedApplyDigest',
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function command(value: unknown, operation: string) {
|
||||
const selected = record(value, 'command');
|
||||
exact(
|
||||
selected,
|
||||
['operation', 'options', 'request', 'schemaVersion'],
|
||||
'command',
|
||||
);
|
||||
if (selected.schemaVersion !== 1 || selected.operation !== operation) {
|
||||
fail('command version or operation is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
options: normalizeOptions(selected.options),
|
||||
request: record(selected.request, 'request'),
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeLocalReconciliationCompleteCommand(
|
||||
value: unknown,
|
||||
): Readonly<LocalReconciliationCompleteCommand> {
|
||||
const selected = command(value, 'local.deployment.reconciliation.complete');
|
||||
exact(
|
||||
selected.request,
|
||||
[
|
||||
'applicationId',
|
||||
'automation',
|
||||
'completedAtMs',
|
||||
'completionId',
|
||||
'expectedApplicationPlanDigest',
|
||||
'expectedHeadDigest',
|
||||
],
|
||||
'request',
|
||||
);
|
||||
if (
|
||||
!Number.isSafeInteger(selected.request.completedAtMs) ||
|
||||
(selected.request.completedAtMs as number) < 0
|
||||
) {
|
||||
fail('completedAtMs is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation: 'local.deployment.reconciliation.complete',
|
||||
options: selected.options,
|
||||
request: Object.freeze({
|
||||
completionId: identifier(
|
||||
selected.request.completionId,
|
||||
UUID_V4,
|
||||
'completionId',
|
||||
),
|
||||
applicationId: identifier(
|
||||
selected.request.applicationId,
|
||||
UUID_V4,
|
||||
'applicationId',
|
||||
),
|
||||
expectedApplicationPlanDigest: digest(
|
||||
selected.request.expectedApplicationPlanDigest,
|
||||
'expectedApplicationPlanDigest',
|
||||
),
|
||||
expectedHeadDigest: digest(
|
||||
selected.request.expectedHeadDigest,
|
||||
'expectedHeadDigest',
|
||||
),
|
||||
automation: normalizeAutomationBinding(selected.request.automation),
|
||||
completedAtMs: selected.request.completedAtMs as number,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeLocalReconciliationCompletionVerifyCommand(
|
||||
value: unknown,
|
||||
): Readonly<LocalReconciliationCompletionVerifyCommand> {
|
||||
const selected = command(
|
||||
value,
|
||||
'local.deployment.reconciliation.complete.verify',
|
||||
);
|
||||
exact(
|
||||
selected.request,
|
||||
['applicationId', 'automation', 'completionId', 'expectedCompletionDigest'],
|
||||
'request',
|
||||
);
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation: 'local.deployment.reconciliation.complete.verify',
|
||||
options: selected.options,
|
||||
request: Object.freeze({
|
||||
completionId: identifier(
|
||||
selected.request.completionId,
|
||||
UUID_V4,
|
||||
'completionId',
|
||||
),
|
||||
applicationId: identifier(
|
||||
selected.request.applicationId,
|
||||
UUID_V4,
|
||||
'applicationId',
|
||||
),
|
||||
expectedCompletionDigest: digest(
|
||||
selected.request.expectedCompletionDigest,
|
||||
'expectedCompletionDigest',
|
||||
),
|
||||
automation: normalizeAutomationBinding(selected.request.automation),
|
||||
}),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,754 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import { readPrivateLocalCommandFile } from '@qinglong/local-command-file';
|
||||
import { inspectLocalSqliteSnapshot } from '@qinglong/local-sqlite/rollout-safety';
|
||||
|
||||
import { currentIdentity } from '../../foundation/contract';
|
||||
import { LocalDeploymentConfigurationError } from '../../foundation/error';
|
||||
import {
|
||||
preflightPublishedFile,
|
||||
publishExactFile,
|
||||
validatePrivateDirectory,
|
||||
} from '../../foundation/files';
|
||||
import {
|
||||
advanceLocalCutoverInstanceHead,
|
||||
readLocalCutoverInstanceHead,
|
||||
type LocalCutoverInstanceHead,
|
||||
} from '../../cutover/instanceLineage';
|
||||
import { readLocalReconciliationApplicationTerminal } from '../application/coordinator';
|
||||
import type { LocalReconciliationApplicationTerminal } from '../application/coordinator';
|
||||
import {
|
||||
collectLocalReconciliationAutomationCompletedStorage,
|
||||
localReconciliationAutomationApplyPaths,
|
||||
readLocalReconciliationAutomationApplyIntent,
|
||||
readLocalReconciliationAutomationApplyReceipt,
|
||||
validateLocalReconciliationAutomationAppliedStorage,
|
||||
validateLocalReconciliationAutomationApplyCatalog,
|
||||
validateLocalReconciliationAutomationApplyLayout,
|
||||
validateLocalReconciliationAutomationCompletedStorage,
|
||||
} from '../application/automation/applyStorage';
|
||||
import type {
|
||||
LocalReconciliationAutomationApplyIntent,
|
||||
LocalReconciliationAutomationApplyReceipt,
|
||||
} from '../application/automation/applyEvidence';
|
||||
import { readLocalReconciliationAutomationDecisionTerminal } from '../application/automation/decisionCoordinator';
|
||||
import type { LocalReconciliationCompletionDomainEvidence } from './evidence';
|
||||
import {
|
||||
buildLocalReconciliationCompletionReceipt,
|
||||
localReconciliationCompletionReceiptContents,
|
||||
normalizeLocalReconciliationCompletionReceipt,
|
||||
type LocalReconciliationCompletionReceipt,
|
||||
} from './evidence';
|
||||
import {
|
||||
normalizeLocalReconciliationCompleteCommand,
|
||||
normalizeLocalReconciliationCompletionVerifyCommand,
|
||||
type LocalReconciliationCompleteCommand,
|
||||
type LocalReconciliationCompletionOptions,
|
||||
type LocalReconciliationCompletionResult,
|
||||
type LocalReconciliationCompletionVerifyCommand,
|
||||
} from './contract';
|
||||
|
||||
const MAX_COMPLETIONS = 64;
|
||||
const MAX_RECEIPT_BYTES = 64 * 1024;
|
||||
|
||||
interface CompletionPaths {
|
||||
readonly root: string;
|
||||
readonly receipt: string;
|
||||
}
|
||||
|
||||
interface AutomationProof {
|
||||
readonly intent: Readonly<LocalReconciliationAutomationApplyIntent>;
|
||||
readonly receipt: Readonly<LocalReconciliationAutomationApplyReceipt>;
|
||||
readonly paths: ReturnType<typeof localReconciliationAutomationApplyPaths>;
|
||||
}
|
||||
|
||||
export interface LocalReconciliationCompletionDependencies {
|
||||
readonly inspectSnapshot?: typeof inspectLocalSqliteSnapshot;
|
||||
readonly afterReceiptPublished?: () => void;
|
||||
readonly afterTerminalSealed?: () => void;
|
||||
readonly afterHeadAdvanced?: () => void;
|
||||
readonly afterBackupCollected?: () => void;
|
||||
}
|
||||
|
||||
function fail(message: string, cause?: unknown): never {
|
||||
throw new LocalDeploymentConfigurationError(
|
||||
`reconciliation completion ${message}`,
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
|
||||
function completionPaths(
|
||||
completionRoot: string,
|
||||
completionId: string,
|
||||
): Readonly<CompletionPaths> {
|
||||
const root = path.join(completionRoot, completionId);
|
||||
return Object.freeze({ root, receipt: path.join(root, 'receipt.json') });
|
||||
}
|
||||
|
||||
function validateDirectory(
|
||||
directory: string,
|
||||
uid: number,
|
||||
modes: readonly number[],
|
||||
label: string,
|
||||
): number {
|
||||
let stat: fs.Stats;
|
||||
try {
|
||||
stat = fs.lstatSync(directory);
|
||||
} catch (error) {
|
||||
return fail(`${label} is unavailable`, error);
|
||||
}
|
||||
const mode = stat.mode & 0o777;
|
||||
if (
|
||||
!stat.isDirectory() ||
|
||||
stat.isSymbolicLink() ||
|
||||
stat.uid !== uid ||
|
||||
!modes.includes(mode) ||
|
||||
fs.realpathSync(directory) !== directory
|
||||
) {
|
||||
fail(`${label} identity is invalid`);
|
||||
}
|
||||
return mode;
|
||||
}
|
||||
|
||||
function validateCatalog(selected: Readonly<CompletionPaths>, sealed: boolean) {
|
||||
const allowed = new Set([
|
||||
'receipt.json',
|
||||
...(!sealed ? ['.receipt.json.ql3-deploy-stage'] : []),
|
||||
]);
|
||||
for (const entry of fs.readdirSync(selected.root, { withFileTypes: true })) {
|
||||
if (!allowed.has(entry.name) || entry.isSymbolicLink()) {
|
||||
fail('receipt catalog contains unknown material');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function ensureCompletionDirectory(
|
||||
completionRoot: string,
|
||||
completionId: string,
|
||||
uid: number,
|
||||
): Readonly<CompletionPaths> {
|
||||
const selected = completionPaths(completionRoot, completionId);
|
||||
const entries = fs.readdirSync(completionRoot, { withFileTypes: true });
|
||||
if (entries.some((entry) => !entry.isDirectory() || entry.isSymbolicLink())) {
|
||||
fail('completion catalog contains drift');
|
||||
}
|
||||
if (entries.length >= MAX_COMPLETIONS && !fs.existsSync(selected.root)) {
|
||||
fail('completion retention limit is reached');
|
||||
}
|
||||
try {
|
||||
fs.mkdirSync(selected.root, { mode: 0o700 });
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') {
|
||||
fail('receipt directory cannot be created', error);
|
||||
}
|
||||
}
|
||||
validateDirectory(selected.root, uid, [0o700, 0o500], 'receipt directory');
|
||||
validateCatalog(
|
||||
selected,
|
||||
(fs.statSync(selected.root).mode & 0o777) === 0o500,
|
||||
);
|
||||
return selected;
|
||||
}
|
||||
|
||||
function stableReceipt(
|
||||
selected: Readonly<CompletionPaths>,
|
||||
uid: number,
|
||||
modes: readonly number[],
|
||||
): Readonly<LocalReconciliationCompletionReceipt> {
|
||||
let descriptor: number | undefined;
|
||||
let bytes: Buffer | undefined;
|
||||
try {
|
||||
const before = fs.lstatSync(selected.receipt, { bigint: true });
|
||||
if (
|
||||
!before.isFile() ||
|
||||
before.isSymbolicLink() ||
|
||||
Number(before.uid) !== uid ||
|
||||
!modes.includes(Number(before.mode) & 0o777) ||
|
||||
before.nlink !== 1n ||
|
||||
before.size < 2n ||
|
||||
before.size > BigInt(MAX_RECEIPT_BYTES)
|
||||
) {
|
||||
fail('receipt identity is invalid');
|
||||
}
|
||||
descriptor = fs.openSync(
|
||||
selected.receipt,
|
||||
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
const opened = fs.fstatSync(descriptor, { bigint: true });
|
||||
if (
|
||||
!opened.isFile() ||
|
||||
opened.dev !== before.dev ||
|
||||
opened.ino !== before.ino ||
|
||||
opened.size !== before.size ||
|
||||
opened.mtimeNs !== before.mtimeNs ||
|
||||
opened.ctimeNs !== before.ctimeNs ||
|
||||
opened.mode !== before.mode ||
|
||||
opened.uid !== before.uid ||
|
||||
opened.nlink !== before.nlink
|
||||
) {
|
||||
fail('receipt changed while opening');
|
||||
}
|
||||
bytes = Buffer.alloc(Number(opened.size));
|
||||
let offset = 0;
|
||||
while (offset < bytes.length) {
|
||||
const count = fs.readSync(
|
||||
descriptor,
|
||||
bytes,
|
||||
offset,
|
||||
bytes.length - offset,
|
||||
offset,
|
||||
);
|
||||
if (count < 1) fail('receipt read stalled');
|
||||
offset += count;
|
||||
}
|
||||
const after = fs.fstatSync(descriptor, { bigint: true });
|
||||
const pathAfter = fs.lstatSync(selected.receipt, { bigint: true });
|
||||
if (
|
||||
after.dev !== opened.dev ||
|
||||
after.ino !== opened.ino ||
|
||||
after.size !== opened.size ||
|
||||
after.mtimeNs !== opened.mtimeNs ||
|
||||
after.ctimeNs !== opened.ctimeNs ||
|
||||
after.mode !== opened.mode ||
|
||||
after.uid !== opened.uid ||
|
||||
after.nlink !== opened.nlink ||
|
||||
pathAfter.dev !== before.dev ||
|
||||
pathAfter.ino !== before.ino ||
|
||||
pathAfter.size !== before.size ||
|
||||
pathAfter.mtimeNs !== before.mtimeNs ||
|
||||
pathAfter.ctimeNs !== before.ctimeNs ||
|
||||
pathAfter.mode !== before.mode ||
|
||||
pathAfter.uid !== before.uid ||
|
||||
pathAfter.nlink !== before.nlink
|
||||
) {
|
||||
fail('receipt drifted while reading');
|
||||
}
|
||||
return normalizeLocalReconciliationCompletionReceipt(
|
||||
JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalDeploymentConfigurationError) throw error;
|
||||
return fail('receipt cannot be read', error);
|
||||
} finally {
|
||||
bytes?.fill(0);
|
||||
if (descriptor !== undefined) fs.closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
function sealCompletion(
|
||||
selected: Readonly<CompletionPaths>,
|
||||
uid: number,
|
||||
): void {
|
||||
validateDirectory(selected.root, uid, [0o700, 0o500], 'receipt directory');
|
||||
stableReceipt(selected, uid, [0o600, 0o400]);
|
||||
let descriptor = fs.openSync(
|
||||
selected.receipt,
|
||||
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
try {
|
||||
const opened = fs.fstatSync(descriptor);
|
||||
if (
|
||||
!opened.isFile() ||
|
||||
opened.uid !== uid ||
|
||||
opened.nlink !== 1 ||
|
||||
![0o600, 0o400].includes(opened.mode & 0o777)
|
||||
) {
|
||||
fail('receipt cannot be sealed');
|
||||
}
|
||||
if ((opened.mode & 0o777) !== 0o400) {
|
||||
fs.fchmodSync(descriptor, 0o400);
|
||||
fs.fsyncSync(descriptor);
|
||||
}
|
||||
} finally {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
const rootMode = validateDirectory(
|
||||
selected.root,
|
||||
uid,
|
||||
[0o700, 0o500],
|
||||
'receipt directory',
|
||||
);
|
||||
descriptor = fs.openSync(selected.root, fs.constants.O_RDONLY);
|
||||
try {
|
||||
if (rootMode !== 0o500) fs.fchmodSync(descriptor, 0o500);
|
||||
fs.fsyncSync(descriptor);
|
||||
} finally {
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
validateCatalog(selected, true);
|
||||
stableReceipt(selected, uid, [0o400]);
|
||||
}
|
||||
|
||||
function validateApplication(
|
||||
terminal: Readonly<LocalReconciliationApplicationTerminal>,
|
||||
applicationId: string,
|
||||
applicationPlanDigest: string,
|
||||
deploymentRoot: string,
|
||||
applicationRoot: string,
|
||||
): void {
|
||||
if (
|
||||
terminal.intent.command.request.applicationId !== applicationId ||
|
||||
terminal.plan.applicationId !== applicationId ||
|
||||
terminal.plan.applicationPlanDigest !== applicationPlanDigest ||
|
||||
terminal.intent.command.options.deploymentRoot !== deploymentRoot ||
|
||||
terminal.intent.command.options.applicationRoot !== applicationRoot
|
||||
) {
|
||||
fail('application authority is detached');
|
||||
}
|
||||
}
|
||||
|
||||
async function automationProof(
|
||||
command: Readonly<LocalReconciliationCompleteCommand>,
|
||||
terminal: Readonly<LocalReconciliationApplicationTerminal>,
|
||||
uid: number,
|
||||
dependencies: LocalReconciliationCompletionDependencies,
|
||||
): Promise<Readonly<AutomationProof> | null> {
|
||||
const automationDomain = terminal.plan.domains.find(
|
||||
(domain) => domain.domain === 'automation',
|
||||
);
|
||||
if (!automationDomain) fail('automation domain is absent');
|
||||
if (automationDomain.action === 'no_effect') {
|
||||
if (
|
||||
command.options.automation !== null ||
|
||||
command.request.automation !== null
|
||||
) {
|
||||
fail('no-effect completion must not carry automation authority');
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
automationDomain.action !== 'adapter_required' ||
|
||||
command.options.automation === null ||
|
||||
command.request.automation === null
|
||||
) {
|
||||
fail('automation domain is not terminally provable');
|
||||
}
|
||||
const options = command.options.automation;
|
||||
const binding = command.request.automation;
|
||||
for (const [directory, label] of [
|
||||
[options.automationRoot, 'automationRoot'],
|
||||
[options.automationDecisionRoot, 'automationDecisionRoot'],
|
||||
[options.automationApplyRoot, 'automationApplyRoot'],
|
||||
] as const) {
|
||||
validatePrivateDirectory(directory, uid, label);
|
||||
}
|
||||
const decision = await readLocalReconciliationAutomationDecisionTerminal(
|
||||
{
|
||||
deploymentRoot: command.options.deploymentRoot,
|
||||
applicationRoot: command.options.applicationRoot,
|
||||
automationRoot: options.automationRoot,
|
||||
automationDecisionRoot: options.automationDecisionRoot,
|
||||
allowRootService: command.options.allowRootService,
|
||||
},
|
||||
binding.automationId,
|
||||
uid,
|
||||
);
|
||||
if (
|
||||
decision.receipt.decisionId !== binding.decisionId ||
|
||||
decision.context.application.plan.applicationPlanDigest !==
|
||||
terminal.plan.applicationPlanDigest
|
||||
) {
|
||||
fail('automation decision is detached from application authority');
|
||||
}
|
||||
const selected = localReconciliationAutomationApplyPaths(
|
||||
options.automationApplyRoot,
|
||||
binding.automationId,
|
||||
);
|
||||
validateLocalReconciliationAutomationApplyLayout(selected, uid);
|
||||
validateLocalReconciliationAutomationApplyCatalog(selected);
|
||||
const intent = readLocalReconciliationAutomationApplyIntent(selected, uid);
|
||||
const receipt = readLocalReconciliationAutomationApplyReceipt(selected, uid);
|
||||
if (
|
||||
intent.command.options.deploymentRoot !== command.options.deploymentRoot ||
|
||||
intent.command.options.applicationRoot !==
|
||||
command.options.applicationRoot ||
|
||||
intent.command.options.automationRoot !== options.automationRoot ||
|
||||
intent.command.options.automationDecisionRoot !==
|
||||
options.automationDecisionRoot ||
|
||||
intent.command.options.automationApplyRoot !==
|
||||
options.automationApplyRoot ||
|
||||
intent.command.options.targetDatabasePath !== options.targetDatabasePath ||
|
||||
intent.command.request.automationId !== binding.automationId ||
|
||||
intent.command.request.decisionId !== binding.decisionId ||
|
||||
receipt.automationId !== binding.automationId ||
|
||||
receipt.decisionId !== binding.decisionId ||
|
||||
receipt.applyDigest !== binding.expectedApplyDigest ||
|
||||
receipt.preparationDigest !== intent.preparationDigest
|
||||
) {
|
||||
fail('automation apply evidence is detached');
|
||||
}
|
||||
const current = await (
|
||||
dependencies.inspectSnapshot ?? inspectLocalSqliteSnapshot
|
||||
)({
|
||||
databasePath: options.targetDatabasePath,
|
||||
profile: intent.profile,
|
||||
});
|
||||
if (current.sha256 !== receipt.targetAfter.sha256) {
|
||||
fail('automation target drifted after apply');
|
||||
}
|
||||
return Object.freeze({ intent, receipt, paths: selected });
|
||||
}
|
||||
|
||||
function domainEvidence(
|
||||
terminal: Readonly<LocalReconciliationApplicationTerminal>,
|
||||
automation: Readonly<AutomationProof> | null,
|
||||
): readonly Readonly<LocalReconciliationCompletionDomainEvidence>[] {
|
||||
return Object.freeze(
|
||||
terminal.plan.domains.map((domain) => {
|
||||
if (domain.action === 'no_effect') {
|
||||
return Object.freeze({
|
||||
domain: domain.domain,
|
||||
action: 'no_effect' as const,
|
||||
evidenceKind: 'application_summary' as const,
|
||||
evidenceDigest: domain.summaryDigest,
|
||||
});
|
||||
}
|
||||
if (
|
||||
domain.domain === 'automation' &&
|
||||
domain.action === 'adapter_required' &&
|
||||
automation !== null
|
||||
) {
|
||||
return Object.freeze({
|
||||
domain: domain.domain,
|
||||
action: 'adapter_required' as const,
|
||||
evidenceKind: 'automation_apply' as const,
|
||||
evidenceDigest: automation.receipt.applyDigest,
|
||||
});
|
||||
}
|
||||
return fail(`${domain.domain} is not terminally reconciled`);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function validateReceiptBinding(
|
||||
receipt: Readonly<LocalReconciliationCompletionReceipt>,
|
||||
terminal: Readonly<LocalReconciliationApplicationTerminal>,
|
||||
expectedDomains: readonly Readonly<LocalReconciliationCompletionDomainEvidence>[],
|
||||
completionId: string,
|
||||
applicationId: string,
|
||||
): void {
|
||||
if (
|
||||
receipt.completionId !== completionId ||
|
||||
receipt.applicationId !== applicationId ||
|
||||
receipt.profile !== terminal.intent.profile ||
|
||||
receipt.instanceId !== terminal.intent.instanceId ||
|
||||
receipt.cutoverId !== terminal.intent.cutoverId ||
|
||||
receipt.generation !== terminal.intent.generation ||
|
||||
receipt.activationDigest !== terminal.intent.activationDigest ||
|
||||
receipt.applicationPlanDigest !== terminal.plan.applicationPlanDigest ||
|
||||
JSON.stringify(receipt.domains) !== JSON.stringify(expectedDomains)
|
||||
) {
|
||||
fail('receipt is detached from terminal domain evidence');
|
||||
}
|
||||
}
|
||||
|
||||
function advanceCompletedHead(
|
||||
terminal: Readonly<LocalReconciliationApplicationTerminal>,
|
||||
receipt: Readonly<LocalReconciliationCompletionReceipt>,
|
||||
uid: number,
|
||||
): Readonly<LocalCutoverInstanceHead> {
|
||||
return advanceLocalCutoverInstanceHead(
|
||||
{
|
||||
options: {
|
||||
deploymentRoot: terminal.intent.command.options.deploymentRoot,
|
||||
},
|
||||
request: {
|
||||
cutoverId: terminal.intent.cutoverId,
|
||||
profile: terminal.intent.profile,
|
||||
instanceId: terminal.intent.instanceId,
|
||||
expectedActivationDigest: terminal.intent.activationDigest,
|
||||
requestedAtMs: receipt.completedAtMs,
|
||||
},
|
||||
},
|
||||
uid,
|
||||
'reconciliation_completed',
|
||||
terminal.intent.generation,
|
||||
receipt.completionDigest,
|
||||
);
|
||||
}
|
||||
|
||||
function result(
|
||||
operation: LocalReconciliationCompletionResult['operation'],
|
||||
status: LocalReconciliationCompletionResult['status'],
|
||||
receipt: Readonly<LocalReconciliationCompletionReceipt>,
|
||||
head: Readonly<LocalCutoverInstanceHead>,
|
||||
): Readonly<LocalReconciliationCompletionResult> {
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation,
|
||||
status,
|
||||
state: 'reconciliation_completed',
|
||||
completionId: receipt.completionId,
|
||||
applicationId: receipt.applicationId,
|
||||
completionDigest: receipt.completionDigest,
|
||||
domainCount: 8,
|
||||
adapterCount: receipt.adapterCount,
|
||||
instanceHeadDigest: head.headDigest,
|
||||
});
|
||||
}
|
||||
|
||||
function assertSourceHead(
|
||||
head: Readonly<LocalCutoverInstanceHead>,
|
||||
expectedHeadDigest: string,
|
||||
automation: Readonly<AutomationProof> | null,
|
||||
): void {
|
||||
const expectedState =
|
||||
automation === null
|
||||
? 'reconciliation_application_planned'
|
||||
: 'reconciliation_automation_applied';
|
||||
const expectedSource =
|
||||
automation === null ? undefined : automation.receipt.applyDigest;
|
||||
if (
|
||||
head.headDigest !== expectedHeadDigest ||
|
||||
head.state !== expectedState ||
|
||||
(expectedSource !== undefined && head.sourceRecordDigest !== expectedSource)
|
||||
) {
|
||||
fail('completion lost source head compare-and-swap');
|
||||
}
|
||||
}
|
||||
|
||||
export async function completeLocalReconciliation(
|
||||
value: unknown,
|
||||
dependencies: LocalReconciliationCompletionDependencies = {},
|
||||
): Promise<Readonly<LocalReconciliationCompletionResult>> {
|
||||
const command = normalizeLocalReconciliationCompleteCommand(value);
|
||||
const uid = currentIdentity().uid;
|
||||
for (const [directory, label] of [
|
||||
[command.options.deploymentRoot, 'deploymentRoot'],
|
||||
[command.options.applicationRoot, 'applicationRoot'],
|
||||
[command.options.completionRoot, 'completionRoot'],
|
||||
] as const) {
|
||||
validatePrivateDirectory(directory, uid, label);
|
||||
}
|
||||
const terminal = await readLocalReconciliationApplicationTerminal(
|
||||
command.options.applicationRoot,
|
||||
command.request.applicationId,
|
||||
uid,
|
||||
);
|
||||
validateApplication(
|
||||
terminal,
|
||||
command.request.applicationId,
|
||||
command.request.expectedApplicationPlanDigest,
|
||||
command.options.deploymentRoot,
|
||||
command.options.applicationRoot,
|
||||
);
|
||||
const automation = await automationProof(
|
||||
command,
|
||||
terminal,
|
||||
uid,
|
||||
dependencies,
|
||||
);
|
||||
const domains = domainEvidence(terminal, automation);
|
||||
const selected = ensureCompletionDirectory(
|
||||
command.options.completionRoot,
|
||||
command.request.completionId,
|
||||
uid,
|
||||
);
|
||||
let receipt: Readonly<LocalReconciliationCompletionReceipt>;
|
||||
let status: 'completed' | 'existing' = 'completed';
|
||||
let head = readLocalCutoverInstanceHead(
|
||||
command.options.deploymentRoot,
|
||||
terminal.intent.instanceId,
|
||||
uid,
|
||||
);
|
||||
if (fs.existsSync(selected.receipt)) {
|
||||
status = 'existing';
|
||||
receipt = stableReceipt(selected, uid, [0o600, 0o400]);
|
||||
validateReceiptBinding(
|
||||
receipt,
|
||||
terminal,
|
||||
domains,
|
||||
command.request.completionId,
|
||||
command.request.applicationId,
|
||||
);
|
||||
if (
|
||||
receipt.sourceHeadDigest !== command.request.expectedHeadDigest ||
|
||||
receipt.completedAtMs !== command.request.completedAtMs
|
||||
) {
|
||||
fail('completion command is not an exact replay');
|
||||
}
|
||||
} else {
|
||||
assertSourceHead(head, command.request.expectedHeadDigest, automation);
|
||||
const adapterCount = domains.filter(
|
||||
(domain) => domain.action === 'adapter_required',
|
||||
).length as 0 | 1;
|
||||
const latestEvidenceAtMs = Math.max(
|
||||
terminal.plan.committedAtMs,
|
||||
automation?.receipt.appliedAtMs ?? 0,
|
||||
);
|
||||
if (command.request.completedAtMs < latestEvidenceAtMs) {
|
||||
fail('completion timestamp precedes terminal evidence');
|
||||
}
|
||||
receipt = buildLocalReconciliationCompletionReceipt({
|
||||
completionId: command.request.completionId,
|
||||
applicationId: command.request.applicationId,
|
||||
profile: terminal.intent.profile,
|
||||
instanceId: terminal.intent.instanceId,
|
||||
cutoverId: terminal.intent.cutoverId,
|
||||
generation: terminal.intent.generation,
|
||||
activationDigest: terminal.intent.activationDigest,
|
||||
applicationPlanDigest: terminal.plan.applicationPlanDigest,
|
||||
sourceHeadDigest: head.headDigest,
|
||||
domains,
|
||||
adapterCount,
|
||||
completedAtMs: command.request.completedAtMs,
|
||||
});
|
||||
const serialized = localReconciliationCompletionReceiptContents(receipt);
|
||||
preflightPublishedFile(
|
||||
selected.receipt,
|
||||
serialized,
|
||||
0o600,
|
||||
uid,
|
||||
'reconciliation completion receipt',
|
||||
);
|
||||
publishExactFile(
|
||||
selected.receipt,
|
||||
serialized,
|
||||
0o600,
|
||||
uid,
|
||||
'reconciliation completion receipt',
|
||||
);
|
||||
dependencies.afterReceiptPublished?.();
|
||||
}
|
||||
sealCompletion(selected, uid);
|
||||
dependencies.afterTerminalSealed?.();
|
||||
head = readLocalCutoverInstanceHead(
|
||||
command.options.deploymentRoot,
|
||||
terminal.intent.instanceId,
|
||||
uid,
|
||||
);
|
||||
if (head.state !== 'reconciliation_completed') {
|
||||
assertSourceHead(head, receipt.sourceHeadDigest, automation);
|
||||
if (automation !== null) {
|
||||
validateLocalReconciliationAutomationAppliedStorage(
|
||||
automation.paths,
|
||||
automation.intent,
|
||||
uid,
|
||||
);
|
||||
}
|
||||
head = advanceCompletedHead(terminal, receipt, uid);
|
||||
} else if (head.sourceRecordDigest !== receipt.completionDigest) {
|
||||
fail('completed head is detached from receipt');
|
||||
}
|
||||
dependencies.afterHeadAdvanced?.();
|
||||
if (automation !== null) {
|
||||
collectLocalReconciliationAutomationCompletedStorage(
|
||||
automation.paths,
|
||||
automation.intent,
|
||||
uid,
|
||||
);
|
||||
dependencies.afterBackupCollected?.();
|
||||
}
|
||||
return result(command.operation, status, receipt, head);
|
||||
}
|
||||
|
||||
export async function verifyLocalReconciliationCompletion(
|
||||
value: unknown,
|
||||
dependencies: LocalReconciliationCompletionDependencies = {},
|
||||
): Promise<Readonly<LocalReconciliationCompletionResult>> {
|
||||
const command = normalizeLocalReconciliationCompletionVerifyCommand(value);
|
||||
const uid = currentIdentity().uid;
|
||||
for (const [directory, label] of [
|
||||
[command.options.deploymentRoot, 'deploymentRoot'],
|
||||
[command.options.applicationRoot, 'applicationRoot'],
|
||||
[command.options.completionRoot, 'completionRoot'],
|
||||
] as const) {
|
||||
validatePrivateDirectory(directory, uid, label);
|
||||
}
|
||||
const selected = completionPaths(
|
||||
command.options.completionRoot,
|
||||
command.request.completionId,
|
||||
);
|
||||
validateDirectory(selected.root, uid, [0o500], 'receipt directory');
|
||||
validateCatalog(selected, true);
|
||||
const receipt = stableReceipt(selected, uid, [0o400]);
|
||||
if (
|
||||
receipt.applicationId !== command.request.applicationId ||
|
||||
receipt.completionDigest !== command.request.expectedCompletionDigest
|
||||
) {
|
||||
fail('verify command is detached from receipt');
|
||||
}
|
||||
const terminal = await readLocalReconciliationApplicationTerminal(
|
||||
command.options.applicationRoot,
|
||||
command.request.applicationId,
|
||||
uid,
|
||||
);
|
||||
validateApplication(
|
||||
terminal,
|
||||
command.request.applicationId,
|
||||
receipt.applicationPlanDigest,
|
||||
command.options.deploymentRoot,
|
||||
command.options.applicationRoot,
|
||||
);
|
||||
const syntheticCompleteCommand = Object.freeze({
|
||||
schemaVersion: 1 as const,
|
||||
operation: 'local.deployment.reconciliation.complete' as const,
|
||||
options: command.options,
|
||||
request: Object.freeze({
|
||||
completionId: receipt.completionId,
|
||||
applicationId: receipt.applicationId,
|
||||
expectedApplicationPlanDigest: receipt.applicationPlanDigest,
|
||||
expectedHeadDigest: receipt.sourceHeadDigest,
|
||||
automation: command.request.automation,
|
||||
completedAtMs: receipt.completedAtMs,
|
||||
}),
|
||||
});
|
||||
const automation = await automationProof(
|
||||
syntheticCompleteCommand,
|
||||
terminal,
|
||||
uid,
|
||||
dependencies,
|
||||
);
|
||||
const domains = domainEvidence(terminal, automation);
|
||||
validateReceiptBinding(
|
||||
receipt,
|
||||
terminal,
|
||||
domains,
|
||||
command.request.completionId,
|
||||
command.request.applicationId,
|
||||
);
|
||||
const head = readLocalCutoverInstanceHead(
|
||||
command.options.deploymentRoot,
|
||||
terminal.intent.instanceId,
|
||||
uid,
|
||||
);
|
||||
if (
|
||||
head.state !== 'reconciliation_completed' ||
|
||||
head.sourceRecordDigest !== receipt.completionDigest
|
||||
) {
|
||||
fail('completion receipt is detached from instance head');
|
||||
}
|
||||
if (automation !== null) {
|
||||
validateLocalReconciliationAutomationCompletedStorage(
|
||||
automation.paths,
|
||||
uid,
|
||||
);
|
||||
}
|
||||
return result(command.operation, 'verified', receipt, head);
|
||||
}
|
||||
|
||||
export function completeLocalReconciliationCommandFile(
|
||||
filePath: string,
|
||||
dependencies: LocalReconciliationCompletionDependencies = {},
|
||||
) {
|
||||
return completeLocalReconciliation(
|
||||
readPrivateLocalCommandFile(filePath),
|
||||
dependencies,
|
||||
);
|
||||
}
|
||||
|
||||
export function verifyLocalReconciliationCompletionCommandFile(
|
||||
filePath: string,
|
||||
dependencies: LocalReconciliationCompletionDependencies = {},
|
||||
) {
|
||||
return verifyLocalReconciliationCompletion(
|
||||
readPrivateLocalCommandFile(filePath),
|
||||
dependencies,
|
||||
);
|
||||
}
|
||||
|
||||
export type {
|
||||
LocalReconciliationCompletionOptions,
|
||||
LocalReconciliationCompletionVerifyCommand,
|
||||
};
|
||||
@@ -0,0 +1,195 @@
|
||||
import { LocalDeploymentConfigurationError } from '../../foundation/error';
|
||||
import { cutoverDigest } from '../../cutover/targetEvidence';
|
||||
import {
|
||||
LOCAL_RECONCILIATION_PLAN_DOMAINS,
|
||||
type LocalReconciliationPlanDomain,
|
||||
} from '../planning/contract';
|
||||
|
||||
const RECEIPT_SCHEMA = 'qinglong3-local-reconciliation-completion-receipt';
|
||||
const DIGEST = /^[0-9a-f]{64}$/;
|
||||
const UUID_V4 =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
|
||||
export interface LocalReconciliationCompletionDomainEvidence {
|
||||
readonly domain: LocalReconciliationPlanDomain;
|
||||
readonly action: 'no_effect' | 'adapter_required';
|
||||
readonly evidenceKind: 'application_summary' | 'automation_apply';
|
||||
readonly evidenceDigest: string;
|
||||
}
|
||||
|
||||
export interface LocalReconciliationCompletionReceipt {
|
||||
readonly schema: typeof RECEIPT_SCHEMA;
|
||||
readonly schemaVersion: 1;
|
||||
readonly state: 'reconciliation_completed';
|
||||
readonly completionId: string;
|
||||
readonly applicationId: string;
|
||||
readonly profile: 'edge' | 'standalone';
|
||||
readonly instanceId: string;
|
||||
readonly cutoverId: string;
|
||||
readonly generation: number;
|
||||
readonly activationDigest: string;
|
||||
readonly applicationPlanDigest: string;
|
||||
readonly sourceHeadDigest: string;
|
||||
readonly domains: readonly Readonly<LocalReconciliationCompletionDomainEvidence>[];
|
||||
readonly adapterCount: 0 | 1;
|
||||
readonly completedAtMs: number;
|
||||
readonly completionDigest: string;
|
||||
}
|
||||
|
||||
function fail(message: string): never {
|
||||
throw new LocalDeploymentConfigurationError(
|
||||
`reconciliation completion evidence ${message}`,
|
||||
);
|
||||
}
|
||||
|
||||
function exact(
|
||||
value: unknown,
|
||||
keys: readonly string[],
|
||||
label: string,
|
||||
): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
fail(`${label} must be an object`);
|
||||
}
|
||||
const selected = value as Record<string, unknown>;
|
||||
const actual = Object.keys(selected).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
fail(`${label} shape is invalid`);
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
function domainEvidence(
|
||||
value: unknown,
|
||||
expectedDomain: LocalReconciliationPlanDomain,
|
||||
): Readonly<LocalReconciliationCompletionDomainEvidence> {
|
||||
const selected = exact(
|
||||
value,
|
||||
['action', 'domain', 'evidenceDigest', 'evidenceKind'],
|
||||
'domain evidence',
|
||||
);
|
||||
const noEffect =
|
||||
selected.action === 'no_effect' &&
|
||||
selected.evidenceKind === 'application_summary';
|
||||
const automation =
|
||||
expectedDomain === 'automation' &&
|
||||
selected.action === 'adapter_required' &&
|
||||
selected.evidenceKind === 'automation_apply';
|
||||
if (
|
||||
selected.domain !== expectedDomain ||
|
||||
(!noEffect && !automation) ||
|
||||
typeof selected.evidenceDigest !== 'string' ||
|
||||
!DIGEST.test(selected.evidenceDigest)
|
||||
) {
|
||||
fail('domain evidence is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
domain: expectedDomain,
|
||||
action: selected.action,
|
||||
evidenceKind: selected.evidenceKind,
|
||||
evidenceDigest: selected.evidenceDigest,
|
||||
}) as Readonly<LocalReconciliationCompletionDomainEvidence>;
|
||||
}
|
||||
|
||||
export function buildLocalReconciliationCompletionReceipt(
|
||||
input: Omit<
|
||||
LocalReconciliationCompletionReceipt,
|
||||
'schema' | 'schemaVersion' | 'state' | 'completionDigest'
|
||||
>,
|
||||
): Readonly<LocalReconciliationCompletionReceipt> {
|
||||
const payload = Object.freeze({
|
||||
schema: RECEIPT_SCHEMA,
|
||||
schemaVersion: 1 as const,
|
||||
state: 'reconciliation_completed' as const,
|
||||
...input,
|
||||
});
|
||||
return Object.freeze({
|
||||
...payload,
|
||||
completionDigest: cutoverDigest(payload),
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeLocalReconciliationCompletionReceipt(
|
||||
value: unknown,
|
||||
): Readonly<LocalReconciliationCompletionReceipt> {
|
||||
const selected = exact(
|
||||
value,
|
||||
[
|
||||
'activationDigest',
|
||||
'adapterCount',
|
||||
'applicationId',
|
||||
'applicationPlanDigest',
|
||||
'completedAtMs',
|
||||
'completionDigest',
|
||||
'completionId',
|
||||
'cutoverId',
|
||||
'domains',
|
||||
'generation',
|
||||
'instanceId',
|
||||
'profile',
|
||||
'schema',
|
||||
'schemaVersion',
|
||||
'sourceHeadDigest',
|
||||
'state',
|
||||
],
|
||||
'receipt',
|
||||
);
|
||||
if (!Array.isArray(selected.domains) || selected.domains.length !== 8) {
|
||||
fail('receipt domain catalog is invalid');
|
||||
}
|
||||
const rawDomains = selected.domains as unknown[];
|
||||
const domains = Object.freeze(
|
||||
LOCAL_RECONCILIATION_PLAN_DOMAINS.map((domain, index) =>
|
||||
domainEvidence(rawDomains[index], domain),
|
||||
),
|
||||
);
|
||||
const adapterCount = domains.filter(
|
||||
(domain) => domain.action === 'adapter_required',
|
||||
).length;
|
||||
const { completionDigest, ...raw } = selected;
|
||||
const normalized = Object.freeze({ ...raw, domains });
|
||||
if (
|
||||
selected.schema !== RECEIPT_SCHEMA ||
|
||||
selected.schemaVersion !== 1 ||
|
||||
selected.state !== 'reconciliation_completed' ||
|
||||
typeof selected.completionId !== 'string' ||
|
||||
!UUID_V4.test(selected.completionId) ||
|
||||
typeof selected.applicationId !== 'string' ||
|
||||
!UUID_V4.test(selected.applicationId) ||
|
||||
(selected.profile !== 'edge' && selected.profile !== 'standalone') ||
|
||||
typeof selected.instanceId !== 'string' ||
|
||||
selected.instanceId.length < 1 ||
|
||||
typeof selected.cutoverId !== 'string' ||
|
||||
selected.cutoverId.length < 1 ||
|
||||
!Number.isSafeInteger(selected.generation) ||
|
||||
(selected.generation as number) < 1 ||
|
||||
![
|
||||
selected.activationDigest,
|
||||
selected.applicationPlanDigest,
|
||||
selected.sourceHeadDigest,
|
||||
completionDigest,
|
||||
].every(
|
||||
(candidate) => typeof candidate === 'string' && DIGEST.test(candidate),
|
||||
) ||
|
||||
(selected.adapterCount !== 0 && selected.adapterCount !== 1) ||
|
||||
selected.adapterCount !== adapterCount ||
|
||||
!Number.isSafeInteger(selected.completedAtMs) ||
|
||||
(selected.completedAtMs as number) < 0 ||
|
||||
cutoverDigest(normalized) !== completionDigest
|
||||
) {
|
||||
fail('receipt binding is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
...normalized,
|
||||
completionDigest,
|
||||
}) as unknown as Readonly<LocalReconciliationCompletionReceipt>;
|
||||
}
|
||||
|
||||
export function localReconciliationCompletionReceiptContents(
|
||||
receipt: Readonly<LocalReconciliationCompletionReceipt>,
|
||||
): string {
|
||||
return `${JSON.stringify(receipt, null, 2)}\n`;
|
||||
}
|
||||
@@ -11,6 +11,7 @@ const {
|
||||
commitLocalReconciliationCapture,
|
||||
commitLocalReconciliationApplication,
|
||||
commitLocalReconciliationAutomationDecision,
|
||||
completeLocalReconciliation,
|
||||
applyLocalReconciliationAutomation,
|
||||
commitLocalReconciliationPlan,
|
||||
commitLocalReconciliationReview,
|
||||
@@ -27,6 +28,7 @@ const {
|
||||
verifyLocalReconciliationAutomationDecision,
|
||||
verifyLocalReconciliationAutomationApply,
|
||||
verifyLocalReconciliationAutomationPlan,
|
||||
verifyLocalReconciliationCompletion,
|
||||
verifyLocalReconciliationPlan,
|
||||
verifyLocalReconciliationReview,
|
||||
writeLocalReconciliationReviewDiagnostics,
|
||||
@@ -42,6 +44,7 @@ const {
|
||||
} = require('@qinglong/local-sqlite/data-directory-application-commit');
|
||||
const {
|
||||
advanceLocalCutoverInstanceHead,
|
||||
assertLocalCutoverTargetHead,
|
||||
claimLocalCutoverInstance,
|
||||
readLocalCutoverInstanceHead,
|
||||
} = require('../dist/deployment/cutover/instanceLineage.js');
|
||||
@@ -1356,6 +1359,108 @@ function automationDecisionCommitFixture(
|
||||
};
|
||||
}
|
||||
|
||||
async function appliedAutomationFixture(t, options = {}) {
|
||||
const state = await plannedAutomationFixture(t, {
|
||||
suffix: options.suffix ?? 'completion-applied',
|
||||
planId: options.planId ?? '00000000-0000-4000-8000-000000000481',
|
||||
reviewId: options.reviewId ?? '00000000-0000-4000-8000-000000000482',
|
||||
applicationId:
|
||||
options.applicationId ?? '00000000-0000-4000-8000-000000000483',
|
||||
automationId:
|
||||
options.automationId ?? '00000000-0000-4000-8000-000000000484',
|
||||
readyTarget: true,
|
||||
});
|
||||
assert.equal(state.application.outcome, 'adapter_and_manual_required');
|
||||
const decisionId =
|
||||
options.decisionId ?? '019b0000-0000-7000-8000-000000000481';
|
||||
const review = automationDecisionReviewFile(
|
||||
state,
|
||||
decisionId,
|
||||
'adopt',
|
||||
'reviewed_lossless',
|
||||
options.suffix ?? 'completion-applied',
|
||||
);
|
||||
const prepareCommand = automationDecisionPrepareCommand(state, decisionId);
|
||||
const prepared = await prepareLocalReconciliationAutomationDecision(
|
||||
prepareCommand,
|
||||
);
|
||||
const commit = automationDecisionCommitFixture(
|
||||
state,
|
||||
{ result: prepared, commandOptions: prepareCommand.options },
|
||||
review.filePath,
|
||||
);
|
||||
const decision = await commitLocalReconciliationAutomationDecision(
|
||||
commit.command,
|
||||
commit.dependencies,
|
||||
);
|
||||
const automationApplyRoot = path.join(
|
||||
path.dirname(state.captureRoot),
|
||||
`automation-apply-${options.suffix ?? 'completion-applied'}`,
|
||||
);
|
||||
fs.mkdirSync(automationApplyRoot, { mode: 0o700 });
|
||||
const applyOptions = {
|
||||
...prepareCommand.options,
|
||||
automationApplyRoot,
|
||||
targetDatabasePath: state.targetDatabasePath,
|
||||
ownerPepperKeyringDirectory:
|
||||
state.command.options.ownerPepperKeyringDirectory,
|
||||
credentialFilePath: state.command.options.credentialFilePath,
|
||||
};
|
||||
const appliedAtMs = commit.command.request.committedAtMs + 1;
|
||||
const applyCommand = {
|
||||
schemaVersion: 1,
|
||||
operation: 'local.deployment.reconciliation.automation.apply',
|
||||
options: applyOptions,
|
||||
request: {
|
||||
decisionId,
|
||||
automationId: state.automationCommand.request.automationId,
|
||||
expectedDecisionDigest: decision.decisionDigest,
|
||||
expectedHeadDigest: decision.instanceHeadDigest,
|
||||
mutationId: options.mutationId ?? '00000000-0000-4000-8000-000000000485',
|
||||
requestId: `automation-apply-${options.suffix ?? 'completion-applied'}`,
|
||||
appliedAtMs,
|
||||
},
|
||||
};
|
||||
const applyDependencies = {
|
||||
async openAuthenticationDatabase() {
|
||||
return { async close() {} };
|
||||
},
|
||||
async authenticate(_database, authenticateOptions) {
|
||||
const authenticatedAtMs = authenticateOptions.now();
|
||||
return {
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'review-owner' },
|
||||
authenticationId: 'local_reconciliation_automation_apply:test',
|
||||
authenticatedAtMs,
|
||||
expiresAtMs: authenticatedAtMs + 60 * 60 * 1_000,
|
||||
assurance: 'local_console',
|
||||
},
|
||||
databaseFence: {
|
||||
credentialId: 'review-owner',
|
||||
credentialVersion: 1,
|
||||
pepperKeyId: 'review-owner-v1',
|
||||
pepperVersion: 1,
|
||||
},
|
||||
async confirm() {},
|
||||
};
|
||||
},
|
||||
};
|
||||
const applied = await applyLocalReconciliationAutomation(
|
||||
applyCommand,
|
||||
applyDependencies,
|
||||
);
|
||||
return {
|
||||
...state,
|
||||
decisionId,
|
||||
decision,
|
||||
automationApplyRoot,
|
||||
applyOptions,
|
||||
applyCommand,
|
||||
applyDependencies,
|
||||
applied,
|
||||
};
|
||||
}
|
||||
|
||||
function dockerReadSealedSqlite(assetsDirectory, mode) {
|
||||
const source =
|
||||
mode === 'main_only_immutable'
|
||||
@@ -2908,6 +3013,237 @@ test('application coordinator plans eight content-free domains and verifies with
|
||||
assert.equal(cli.stdout.includes('Crontabs'), false);
|
||||
});
|
||||
|
||||
test('completion fence seals all eight no-effect domains before target restart', async (t) => {
|
||||
const state = await reviewedApplicationFixture(t, {
|
||||
planId: '00000000-0000-4000-8000-000000000421',
|
||||
reviewId: '00000000-0000-4000-8000-000000000422',
|
||||
applicationId: '00000000-0000-4000-8000-000000000423',
|
||||
reviewSuffix: 'completion-no-effect',
|
||||
createDefaultSidecars: false,
|
||||
initializeDatabases: automationDatabaseInitializer(),
|
||||
mutateTarget(paths) {
|
||||
return mutateAutomationTarget(paths);
|
||||
},
|
||||
});
|
||||
const prepared = await prepareLocalReconciliationApplication(
|
||||
state.prepareApplicationCommand,
|
||||
);
|
||||
const application = await commitLocalReconciliationApplication(
|
||||
applicationCommitCommand(state, prepared),
|
||||
);
|
||||
assert.equal(application.outcome, 'no_effect_ready');
|
||||
const completionRoot = path.join(
|
||||
path.dirname(state.captureRoot),
|
||||
'completion-no-effect',
|
||||
);
|
||||
fs.mkdirSync(completionRoot, { mode: 0o700 });
|
||||
const command = {
|
||||
schemaVersion: 1,
|
||||
operation: 'local.deployment.reconciliation.complete',
|
||||
options: {
|
||||
deploymentRoot: state.deploymentRoot,
|
||||
applicationRoot: state.applicationRoot,
|
||||
completionRoot,
|
||||
automation: null,
|
||||
allowRootService: rootAcknowledgement(),
|
||||
},
|
||||
request: {
|
||||
completionId: '00000000-0000-4000-8000-000000000424',
|
||||
applicationId: state.prepareApplicationCommand.request.applicationId,
|
||||
expectedApplicationPlanDigest: application.applicationPlanDigest,
|
||||
expectedHeadDigest: application.instanceHeadDigest,
|
||||
automation: null,
|
||||
completedAtMs: state.prepareApplicationCommand.request.preparedAtMs + 4,
|
||||
},
|
||||
};
|
||||
for (const boundary of [
|
||||
'afterReceiptPublished',
|
||||
'afterTerminalSealed',
|
||||
'afterHeadAdvanced',
|
||||
]) {
|
||||
await assert.rejects(
|
||||
completeLocalReconciliation(command, {
|
||||
[boundary]() {
|
||||
throw new Error(`completion ${boundary} response loss`);
|
||||
},
|
||||
}),
|
||||
new RegExp(`completion ${boundary} response loss`),
|
||||
);
|
||||
}
|
||||
const completed = await completeLocalReconciliation(command);
|
||||
assert.equal(completed.status, 'existing');
|
||||
assert.equal(completed.state, 'reconciliation_completed');
|
||||
assert.equal(completed.domainCount, 8);
|
||||
assert.equal(completed.adapterCount, 0);
|
||||
const completionDirectory = path.join(
|
||||
completionRoot,
|
||||
command.request.completionId,
|
||||
);
|
||||
assert.deepEqual(fs.readdirSync(completionDirectory), ['receipt.json']);
|
||||
assert.equal(fs.statSync(completionDirectory).mode & 0o777, 0o500);
|
||||
assert.equal(
|
||||
fs.statSync(path.join(completionDirectory, 'receipt.json')).mode & 0o777,
|
||||
0o400,
|
||||
);
|
||||
const receipt = JSON.parse(
|
||||
fs.readFileSync(path.join(completionDirectory, 'receipt.json'), 'utf8'),
|
||||
);
|
||||
assert.equal(receipt.domains.length, 8);
|
||||
assert.equal(
|
||||
receipt.domains.every(
|
||||
(domain) =>
|
||||
domain.action === 'no_effect' &&
|
||||
domain.evidenceKind === 'application_summary',
|
||||
),
|
||||
true,
|
||||
);
|
||||
const verifyCommand = {
|
||||
schemaVersion: 1,
|
||||
operation: 'local.deployment.reconciliation.complete.verify',
|
||||
options: command.options,
|
||||
request: {
|
||||
completionId: command.request.completionId,
|
||||
applicationId: command.request.applicationId,
|
||||
expectedCompletionDigest: completed.completionDigest,
|
||||
automation: null,
|
||||
},
|
||||
};
|
||||
const verified = await verifyLocalReconciliationCompletion(verifyCommand);
|
||||
assert.equal(verified.status, 'verified');
|
||||
const commandPath = path.join(state.deploymentRoot, 'completion-verify.json');
|
||||
fs.writeFileSync(commandPath, `${JSON.stringify(verifyCommand)}\n`, {
|
||||
mode: 0o600,
|
||||
});
|
||||
const cli = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
path.join(__dirname, '../dist/deployment/localDeploymentCli.js'),
|
||||
'reconciliation-complete-verify',
|
||||
'--command-file',
|
||||
commandPath,
|
||||
],
|
||||
{ encoding: 'utf8' },
|
||||
);
|
||||
assert.equal(cli.status, 0, cli.stderr);
|
||||
assert.equal(JSON.parse(cli.stdout).status, 'verified');
|
||||
assert.equal(cli.stdout.includes(completionRoot), false);
|
||||
const identity = {
|
||||
options: { deploymentRoot: state.deploymentRoot },
|
||||
request: {
|
||||
cutoverId: state.captureCommand.request.cutoverId,
|
||||
profile: state.captureCommand.request.profile,
|
||||
instanceId: state.captureCommand.request.instanceId,
|
||||
expectedActivationDigest:
|
||||
state.captureCommand.request.expectedActivationDigest,
|
||||
requestedAtMs: command.request.completedAtMs + 1,
|
||||
},
|
||||
};
|
||||
const restartHead = assertLocalCutoverTargetHead(identity, state.uid);
|
||||
assert.equal(restartHead.state, 'reconciliation_completed');
|
||||
const activeHead = advanceLocalCutoverInstanceHead(
|
||||
identity,
|
||||
state.uid,
|
||||
'target_active',
|
||||
2,
|
||||
'e'.repeat(64),
|
||||
);
|
||||
assert.equal(activeHead.state, 'target_active');
|
||||
assert.equal(activeHead.generation, 2);
|
||||
});
|
||||
|
||||
test('completion fence retains automation rollback backup while other domains remain manual', async (t) => {
|
||||
const state = await appliedAutomationFixture(t, {
|
||||
suffix: 'completion-fence',
|
||||
planId: '00000000-0000-4000-8000-000000000491',
|
||||
reviewId: '00000000-0000-4000-8000-000000000492',
|
||||
applicationId: '00000000-0000-4000-8000-000000000493',
|
||||
automationId: '00000000-0000-4000-8000-000000000494',
|
||||
decisionId: '019b0000-0000-7000-8000-000000000491',
|
||||
mutationId: '00000000-0000-4000-8000-000000000495',
|
||||
});
|
||||
const completionRoot = path.join(
|
||||
path.dirname(state.captureRoot),
|
||||
'completion-automation-applied',
|
||||
);
|
||||
fs.mkdirSync(completionRoot, { mode: 0o700 });
|
||||
const automation = {
|
||||
automationId: state.automationCommand.request.automationId,
|
||||
decisionId: state.decisionId,
|
||||
expectedApplyDigest: state.applied.applyDigest,
|
||||
};
|
||||
const completionOptions = {
|
||||
deploymentRoot: state.deploymentRoot,
|
||||
applicationRoot: state.applicationRoot,
|
||||
completionRoot,
|
||||
automation: {
|
||||
automationRoot: state.automationRoot,
|
||||
automationDecisionRoot: state.automationDecisionRoot,
|
||||
automationApplyRoot: state.automationApplyRoot,
|
||||
targetDatabasePath: state.targetDatabasePath,
|
||||
},
|
||||
allowRootService: rootAcknowledgement(),
|
||||
};
|
||||
const command = {
|
||||
schemaVersion: 1,
|
||||
operation: 'local.deployment.reconciliation.complete',
|
||||
options: completionOptions,
|
||||
request: {
|
||||
completionId: '00000000-0000-4000-8000-000000000496',
|
||||
applicationId: state.application.applicationId,
|
||||
expectedApplicationPlanDigest: state.application.applicationPlanDigest,
|
||||
expectedHeadDigest: state.applied.instanceHeadDigest,
|
||||
automation,
|
||||
completedAtMs: state.applyCommand.request.appliedAtMs + 1,
|
||||
},
|
||||
};
|
||||
const applyRoot = path.join(
|
||||
state.automationApplyRoot,
|
||||
state.automationCommand.request.automationId,
|
||||
);
|
||||
const backupRoot = path.join(applyRoot, 'backup');
|
||||
const backupPath = path.join(backupRoot, 'before.sqlite');
|
||||
const rollbackRoot = path.join(applyRoot, 'rollback-work');
|
||||
assert.equal(fs.existsSync(backupPath), true);
|
||||
await assert.rejects(
|
||||
completeLocalReconciliation(command),
|
||||
/secret_and_config is not terminally reconciled/,
|
||||
);
|
||||
assert.equal(fs.existsSync(backupPath), true);
|
||||
assert.deepEqual(fs.readdirSync(backupRoot), ['before.sqlite']);
|
||||
assert.deepEqual(fs.readdirSync(rollbackRoot), []);
|
||||
assert.equal(fs.statSync(backupRoot).mode & 0o777, 0o500);
|
||||
assert.equal(fs.statSync(rollbackRoot).mode & 0o777, 0o700);
|
||||
assert.equal(
|
||||
fs.existsSync(path.join(completionRoot, command.request.completionId)),
|
||||
false,
|
||||
);
|
||||
const head = readLocalCutoverInstanceHead(
|
||||
state.deploymentRoot,
|
||||
state.captureCommand.request.instanceId,
|
||||
state.uid,
|
||||
);
|
||||
assert.equal(head.state, 'reconciliation_automation_applied');
|
||||
assert.equal(head.sourceRecordDigest, state.applied.applyDigest);
|
||||
assert.throws(
|
||||
() =>
|
||||
assertLocalCutoverTargetHead(
|
||||
{
|
||||
options: { deploymentRoot: state.deploymentRoot },
|
||||
request: {
|
||||
cutoverId: state.captureCommand.request.cutoverId,
|
||||
profile: state.captureCommand.request.profile,
|
||||
instanceId: state.captureCommand.request.instanceId,
|
||||
expectedActivationDigest:
|
||||
state.captureCommand.request.expectedActivationDigest,
|
||||
requestedAtMs: command.request.completedAtMs + 1,
|
||||
},
|
||||
},
|
||||
state.uid,
|
||||
),
|
||||
/not bound to the instance lineage head/,
|
||||
);
|
||||
});
|
||||
|
||||
test('automation adapter builds a sealed row plan with bounded conflict evidence', async (t) => {
|
||||
const state = await reviewedApplicationFixture(t, {
|
||||
planId: '00000000-0000-4000-8000-000000000431',
|
||||
|
||||
Reference in New Issue
Block a user