feat(ql3): capture stopped reconciliation bundles

This commit is contained in:
whyour
2026-08-21 16:52:22 +08:00
parent a789c4a4d6
commit c341a92a18
16 changed files with 2837 additions and 33 deletions
@@ -97,6 +97,12 @@ import {
prepareLocalReconciliationCapture,
prepareLocalReconciliationCaptureCommandFile,
} from './reconciliation/preparation';
import {
commitLocalReconciliationCapture,
commitLocalReconciliationCaptureCommandFile,
verifyLocalReconciliationCapture,
verifyLocalReconciliationCaptureCommandFile,
} from './reconciliation/bundle';
export {
LocalDeploymentConfigurationError,
@@ -128,14 +134,29 @@ export {
} from './foundation/contract';
export {
normalizeLocalReconciliationCapturePrepareCommand,
normalizeLocalReconciliationCaptureCommitCommand,
normalizeLocalReconciliationCaptureVerifyCommand,
type LocalReconciliationCaptureCommitCommand,
type LocalReconciliationCapturePrepareCommand,
type LocalReconciliationCapturePrepareResult,
type LocalReconciliationCaptureTerminalResult,
type LocalReconciliationCaptureVerifyCommand,
type LocalReconciliationStoppedAuthority,
} from './reconciliation/contract';
export {
localReconciliationCaptureDirectory,
localReconciliationCaptureIntentPath,
normalizeLocalReconciliationCaptureIntent,
readLocalReconciliationCaptureIntent,
type LocalReconciliationCaptureIntent,
} from './reconciliation/preparation';
export {
normalizeLocalReconciliationCaptureManifest,
normalizeLocalReconciliationCaptureReceipt,
type LocalReconciliationCaptureDependencies,
type LocalReconciliationCaptureManifest,
type LocalReconciliationCaptureReceipt,
} from './reconciliation/bundle';
export {
prepareLocalDeploymentAdoptedBundle,
runLocalDeploymentAdoptedBundleCommandFile,
@@ -241,6 +262,10 @@ export {
consumeLocalServiceManagerLegacyRollbackCommandFile,
prepareLocalReconciliationCapture,
prepareLocalReconciliationCaptureCommandFile,
commitLocalReconciliationCapture,
commitLocalReconciliationCaptureCommandFile,
verifyLocalReconciliationCapture,
verifyLocalReconciliationCaptureCommandFile,
};
export {
localServiceManagerIntentDigest,
@@ -14,6 +14,8 @@ import {
prepareLocalServiceManagerIntentCommandFile,
prepareLocalServiceManagerLegacyRollbackCommandFile,
prepareLocalReconciliationCaptureCommandFile,
commitLocalReconciliationCaptureCommandFile,
verifyLocalReconciliationCaptureCommandFile,
prepareLocalDeploymentCommandFile,
proveLocalDeploymentLegacyReadinessCommandFile,
restoreLocalDeploymentComposeCommitCommandFile,
@@ -28,7 +30,7 @@ import {
} from './localDeployment';
const USAGE =
'Usage: ql3-local-deploy <prepare|adopted-prepare|adopted-verify|status|service-intent-prepare|service-outcome-consume|service-cutover-consume|service-legacy-rollback-prepare|service-legacy-rollback-authorize|service-legacy-rollback-consume|cutover-legacy-stop|cutover-target-start|cutover-target-restart|cutover-target-stop|cutover-legacy-rollback-prepare|cutover-legacy-rollback-commit|cutover-legacy-readiness-probe|cutover-manual-diagnose|cutover-manual-resolution-prepare|cutover-manual-resolution-commit|reconciliation-capture-prepare|compose-revision|compose-preflight|compose-apply|compose-restore-prepare|compose-restore-commit|compose-evidence-collect-prepare|compose-evidence-collect-commit> --command-file /absolute/private-command.json';
'Usage: ql3-local-deploy <prepare|adopted-prepare|adopted-verify|status|service-intent-prepare|service-outcome-consume|service-cutover-consume|service-legacy-rollback-prepare|service-legacy-rollback-authorize|service-legacy-rollback-consume|cutover-legacy-stop|cutover-target-start|cutover-target-restart|cutover-target-stop|cutover-legacy-rollback-prepare|cutover-legacy-rollback-commit|cutover-legacy-readiness-probe|cutover-manual-diagnose|cutover-manual-resolution-prepare|cutover-manual-resolution-commit|reconciliation-capture-prepare|reconciliation-capture-commit|reconciliation-capture-verify|compose-revision|compose-preflight|compose-apply|compose-restore-prepare|compose-restore-commit|compose-evidence-collect-prepare|compose-evidence-collect-commit> --command-file /absolute/private-command.json';
async function main(argv: readonly string[]): Promise<void> {
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
@@ -58,6 +60,8 @@ async function main(argv: readonly string[]): Promise<void> {
argv[0] !== 'cutover-manual-resolution-prepare' &&
argv[0] !== 'cutover-manual-resolution-commit' &&
argv[0] !== 'reconciliation-capture-prepare' &&
argv[0] !== 'reconciliation-capture-commit' &&
argv[0] !== 'reconciliation-capture-verify' &&
argv[0] !== 'compose-revision' &&
argv[0] !== 'compose-preflight' &&
argv[0] !== 'compose-apply' &&
@@ -130,6 +134,10 @@ async function main(argv: readonly string[]): Promise<void> {
)
: argv[0] === 'reconciliation-capture-prepare'
? prepareLocalReconciliationCaptureCommandFile(argv[2]!)
: argv[0] === 'reconciliation-capture-commit'
? commitLocalReconciliationCaptureCommandFile(argv[2]!)
: argv[0] === 'reconciliation-capture-verify'
? verifyLocalReconciliationCaptureCommandFile(argv[2]!)
: argv[0] === 'compose-revision'
? switchLocalDeploymentComposeRevisionCommandFile(argv[2]!)
: argv[0] === 'compose-preflight'
@@ -0,0 +1,792 @@
import fs from 'node:fs';
import path from 'node:path';
import { readPrivateLocalCommandFile } from '@qinglong/local-command-file';
import { currentIdentity } from '../foundation/contract';
import { LocalDeploymentConfigurationError } from '../foundation/error';
import {
ensurePrivateDirectory,
publishExactFile,
validatePrivateDirectory,
} from '../foundation/files';
import {
advanceLocalCutoverInstanceHead,
readLocalCutoverInstanceHead,
type LocalCutoverInstanceHead,
} from '../cutover/instanceLineage';
import { cutoverDigest } from '../cutover/targetEvidence';
import {
normalizeLocalReconciliationCaptureCommitCommand,
normalizeLocalReconciliationCaptureVerifyCommand,
type LocalReconciliationCaptureCommitCommand,
type LocalReconciliationCaptureTerminalResult,
} from './contract';
import {
localReconciliationCaptureDirectory,
readLocalReconciliationCaptureIntent,
type LocalReconciliationCaptureIntent,
} from './preparation';
import { proveLocalReconciliationLineage } from './lineageProof';
import { proveLocalReconciliationStoppedState } from './stoppedProof';
import {
copyLocalReconciliationAsset,
localReconciliationCaptureAssetPlan,
verifyLocalReconciliationPublishedAsset,
verifyLocalReconciliationSidecarPlan,
verifyLocalReconciliationSourceSnapshot,
type LocalReconciliationCapturedAsset,
type LocalReconciliationStableCopyDependencies,
} from './stableCopy';
const MANIFEST_SCHEMA = 'qinglong3-local-reconciliation-capture-manifest';
const RECEIPT_SCHEMA = 'qinglong3-local-reconciliation-capture-receipt';
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
const LOGICAL_NAMES = [
'target-main',
'target-wal',
'target-shm',
'target-journal',
'legacy-main',
'legacy-wal',
'legacy-shm',
'legacy-journal',
'recovery-main',
] as const;
export interface LocalReconciliationCaptureManifest {
readonly schema: typeof MANIFEST_SCHEMA;
readonly schemaVersion: 1;
readonly state: 'reconciliation_captured';
readonly captureId: string;
readonly profile: 'edge' | 'standalone';
readonly preparationDigest: string;
readonly stoppedRecordDigest: string;
readonly stoppedProofDigest: string;
readonly reconciliationEvidenceDigest: string;
readonly lineageProjectionDigest: string;
readonly preparedHeadDigest: string;
readonly committedAtMs: number;
readonly assets: readonly Readonly<LocalReconciliationCapturedAsset>[];
readonly totalBytes: number;
readonly manifestDigest: string;
}
export interface LocalReconciliationCaptureReceipt {
readonly schema: typeof RECEIPT_SCHEMA;
readonly schemaVersion: 1;
readonly state: 'reconciliation_captured';
readonly captureId: string;
readonly profile: 'edge' | 'standalone';
readonly preparationDigest: string;
readonly manifestDigest: string;
readonly assetCount: number;
readonly totalBytes: number;
readonly committedAtMs: number;
readonly bundleDigest: string;
}
export interface LocalReconciliationCaptureDependencies {
readonly stableCopy?: LocalReconciliationStableCopyDependencies;
readonly afterAssetPublished?: (
logicalName: LocalReconciliationCapturedAsset['logicalName'],
) => void;
readonly afterManifestPublished?: () => void;
readonly afterReceiptPublished?: () => void;
}
function configurationError(message: string, cause?: unknown): never {
throw new LocalDeploymentConfigurationError(message, { cause });
}
function object(value: unknown, label: string): Record<string, unknown> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
(Object.getPrototypeOf(value) !== Object.prototype &&
Object.getPrototypeOf(value) !== null)
) {
configurationError(`${label} must be an object`);
}
return value as Record<string, unknown>;
}
function exact(
value: Record<string, unknown>,
keys: readonly string[],
label: string,
): void {
const actual = Object.keys(value).sort();
const expected = [...keys].sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index])
) {
configurationError(`${label} shape is invalid`);
}
}
function capturePaths(
captureRoot: string,
captureId: string,
): Readonly<{
root: string;
staging: string;
assets: string;
manifest: string;
receipt: string;
}> {
const root = localReconciliationCaptureDirectory(captureRoot, captureId);
return Object.freeze({
root,
staging: path.join(root, 'staging'),
assets: path.join(root, 'assets'),
manifest: path.join(root, 'manifest.json'),
receipt: path.join(root, 'receipt.json'),
});
}
function validateIntentBinding(
intent: Readonly<LocalReconciliationCaptureIntent>,
command: Readonly<LocalReconciliationCaptureCommitCommand>,
): void {
if (
intent.command.options.deploymentRoot !== command.options.deploymentRoot ||
intent.command.options.captureRoot !== command.options.captureRoot ||
intent.command.options.allowRootService !==
command.options.allowRootService ||
intent.command.request.captureId !== command.request.captureId ||
intent.preparationDigest !== command.request.expectedPreparationDigest ||
command.request.committedAtMs < intent.command.request.preparedAtMs
) {
configurationError('capture commit is not bound to its exact preparation');
}
}
function validateHeadIdentity(
head: Readonly<LocalCutoverInstanceHead>,
intent: Readonly<LocalReconciliationCaptureIntent>,
): void {
if (
head.profile !== intent.command.request.profile ||
head.cutoverId !== intent.command.request.cutoverId ||
head.activationDigest !== intent.command.request.expectedActivationDigest ||
head.generation !== intent.command.request.generation
) {
configurationError('capture instance head identity drifted');
}
}
function normalizeAsset(
value: unknown,
): Readonly<LocalReconciliationCapturedAsset> {
const asset = object(value, 'capture manifest asset');
exact(
asset,
['bytes', 'logicalName', 'sha256', 'sourceIdentityDigest'],
'capture manifest asset',
);
if (
typeof asset.logicalName !== 'string' ||
!LOGICAL_NAMES.includes(
asset.logicalName as (typeof LOGICAL_NAMES)[number],
) ||
!Number.isSafeInteger(asset.bytes) ||
(asset.bytes as number) < 0 ||
typeof asset.sha256 !== 'string' ||
!DIGEST_PATTERN.test(asset.sha256) ||
typeof asset.sourceIdentityDigest !== 'string' ||
!DIGEST_PATTERN.test(asset.sourceIdentityDigest)
) {
configurationError('capture manifest asset drifted');
}
return Object.freeze({
logicalName:
asset.logicalName as LocalReconciliationCapturedAsset['logicalName'],
bytes: asset.bytes as number,
sha256: asset.sha256,
sourceIdentityDigest: asset.sourceIdentityDigest,
});
}
export function normalizeLocalReconciliationCaptureManifest(
value: unknown,
): Readonly<LocalReconciliationCaptureManifest> {
const manifest = object(value, 'reconciliation capture manifest');
exact(
manifest,
[
'assets',
'captureId',
'committedAtMs',
'lineageProjectionDigest',
'manifestDigest',
'preparationDigest',
'preparedHeadDigest',
'profile',
'reconciliationEvidenceDigest',
'schema',
'schemaVersion',
'state',
'stoppedProofDigest',
'stoppedRecordDigest',
'totalBytes',
],
'reconciliation capture manifest',
);
if (!Array.isArray(manifest.assets)) {
configurationError('capture manifest assets must be an array');
}
const assets = Object.freeze(manifest.assets.map(normalizeAsset));
const names = assets.map((asset) => asset.logicalName);
const order = names.map((name) => LOGICAL_NAMES.indexOf(name));
const { manifestDigest, ...payload } = manifest;
if (
manifest.schema !== MANIFEST_SCHEMA ||
manifest.schemaVersion !== 1 ||
manifest.state !== 'reconciliation_captured' ||
typeof manifest.captureId !== 'string' ||
(manifest.profile !== 'edge' && manifest.profile !== 'standalone') ||
assets.length < 3 ||
assets.length > LOGICAL_NAMES.length ||
new Set(names).size !== names.length ||
order.some(
(index, position) => position > 0 && index <= order[position - 1]!,
) ||
names[0] !== 'target-main' ||
!names.includes('legacy-main') ||
names.at(-1) !== 'recovery-main' ||
!Number.isSafeInteger(manifest.totalBytes) ||
(manifest.totalBytes as number) < 1 ||
assets.reduce((total, asset) => total + asset.bytes, 0) !==
manifest.totalBytes ||
!Number.isSafeInteger(manifest.committedAtMs) ||
(manifest.committedAtMs as number) < 0 ||
[
manifest.preparationDigest,
manifest.stoppedRecordDigest,
manifest.stoppedProofDigest,
manifest.reconciliationEvidenceDigest,
manifest.lineageProjectionDigest,
manifest.preparedHeadDigest,
manifestDigest,
].some(
(candidate) =>
typeof candidate !== 'string' || !DIGEST_PATTERN.test(candidate),
) ||
cutoverDigest(payload) !== manifestDigest
) {
configurationError('reconciliation capture manifest drifted');
}
return Object.freeze({
...(manifest as unknown as LocalReconciliationCaptureManifest),
assets,
});
}
export function normalizeLocalReconciliationCaptureReceipt(
value: unknown,
): Readonly<LocalReconciliationCaptureReceipt> {
const receipt = object(value, 'reconciliation capture receipt');
exact(
receipt,
[
'assetCount',
'bundleDigest',
'captureId',
'committedAtMs',
'manifestDigest',
'preparationDigest',
'profile',
'schema',
'schemaVersion',
'state',
'totalBytes',
],
'reconciliation capture receipt',
);
const { bundleDigest, ...payload } = receipt;
if (
receipt.schema !== RECEIPT_SCHEMA ||
receipt.schemaVersion !== 1 ||
receipt.state !== 'reconciliation_captured' ||
typeof receipt.captureId !== 'string' ||
(receipt.profile !== 'edge' && receipt.profile !== 'standalone') ||
!Number.isSafeInteger(receipt.assetCount) ||
(receipt.assetCount as number) < 3 ||
(receipt.assetCount as number) > LOGICAL_NAMES.length ||
!Number.isSafeInteger(receipt.totalBytes) ||
(receipt.totalBytes as number) < 1 ||
!Number.isSafeInteger(receipt.committedAtMs) ||
(receipt.committedAtMs as number) < 0 ||
[receipt.preparationDigest, receipt.manifestDigest, bundleDigest].some(
(candidate) =>
typeof candidate !== 'string' || !DIGEST_PATTERN.test(candidate),
) ||
cutoverDigest(payload) !== bundleDigest
) {
configurationError('reconciliation capture receipt drifted');
}
return receipt as unknown as Readonly<LocalReconciliationCaptureReceipt>;
}
function manifestContents(
manifest: Readonly<LocalReconciliationCaptureManifest>,
): string {
return `${JSON.stringify(manifest, null, 2)}\n`;
}
function receiptContents(
receipt: Readonly<LocalReconciliationCaptureReceipt>,
): string {
return `${JSON.stringify(receipt, null, 2)}\n`;
}
function captureReceipt(
manifest: Readonly<LocalReconciliationCaptureManifest>,
): Readonly<LocalReconciliationCaptureReceipt> {
const payload = Object.freeze({
schema: RECEIPT_SCHEMA,
schemaVersion: 1 as const,
state: 'reconciliation_captured' as const,
captureId: manifest.captureId,
profile: manifest.profile,
preparationDigest: manifest.preparationDigest,
manifestDigest: manifest.manifestDigest,
assetCount: manifest.assets.length,
totalBytes: manifest.totalBytes,
committedAtMs: manifest.committedAtMs,
});
return Object.freeze({
...payload,
bundleDigest: cutoverDigest(payload),
});
}
function validateTerminalCatalog(
paths: ReturnType<typeof capturePaths>,
terminal: boolean,
): void {
const allowedRoot = new Set([
'assets',
'intent.json',
'manifest.json',
'receipt.json',
'staging',
...(!terminal
? ['.manifest.json.ql3-deploy-stage', '.receipt.json.ql3-deploy-stage']
: []),
]);
for (const entry of fs.readdirSync(paths.root, { withFileTypes: true })) {
if (!allowedRoot.has(entry.name) || entry.isSymbolicLink()) {
configurationError('capture bundle root contains unknown material');
}
}
if (fs.readdirSync(paths.staging).length !== 0) {
configurationError('capture staging root contains unknown material');
}
const allowedAssets = new Set<string>(LOGICAL_NAMES);
if (!terminal) {
for (const name of LOGICAL_NAMES) {
allowedAssets.add(`.${name}.ql3-capture-stage`);
}
}
for (const entry of fs.readdirSync(paths.assets, { withFileTypes: true })) {
if (
!entry.isFile() ||
entry.isSymbolicLink() ||
!allowedAssets.has(entry.name)
) {
configurationError('capture assets contain unknown material');
}
}
}
function readTerminal(
paths: ReturnType<typeof capturePaths>,
uid: number,
): Readonly<{
manifest: Readonly<LocalReconciliationCaptureManifest>;
receipt: Readonly<LocalReconciliationCaptureReceipt>;
}> {
validatePrivateDirectory(paths.root, uid, 'captureDirectory');
validatePrivateDirectory(paths.staging, uid, 'captureStagingDirectory');
validatePrivateDirectory(paths.assets, uid, 'captureAssetsDirectory');
const manifest = normalizeLocalReconciliationCaptureManifest(
readPrivateLocalCommandFile(paths.manifest),
);
const receipt = normalizeLocalReconciliationCaptureReceipt(
readPrivateLocalCommandFile(paths.receipt),
);
if (
receipt.captureId !== manifest.captureId ||
receipt.profile !== manifest.profile ||
receipt.preparationDigest !== manifest.preparationDigest ||
receipt.manifestDigest !== manifest.manifestDigest ||
receipt.assetCount !== manifest.assets.length ||
receipt.totalBytes !== manifest.totalBytes
) {
configurationError('capture terminal receipt is detached from manifest');
}
for (const asset of manifest.assets) {
verifyLocalReconciliationPublishedAsset(asset, paths.assets, uid);
}
validateTerminalCatalog(paths, true);
return Object.freeze({ manifest, receipt });
}
function readPublishedManifest(
paths: ReturnType<typeof capturePaths>,
uid: number,
): Readonly<LocalReconciliationCaptureManifest> {
validatePrivateDirectory(paths.root, uid, 'captureDirectory');
validatePrivateDirectory(paths.staging, uid, 'captureStagingDirectory');
validatePrivateDirectory(paths.assets, uid, 'captureAssetsDirectory');
const manifest = normalizeLocalReconciliationCaptureManifest(
readPrivateLocalCommandFile(paths.manifest),
);
for (const asset of manifest.assets) {
verifyLocalReconciliationPublishedAsset(asset, paths.assets, uid);
}
validateTerminalCatalog(paths, false);
return manifest;
}
function terminalResult(
operation: LocalReconciliationCaptureTerminalResult['operation'],
status: LocalReconciliationCaptureTerminalResult['status'],
terminal: ReturnType<typeof readTerminal>,
head: Readonly<LocalCutoverInstanceHead>,
): Readonly<LocalReconciliationCaptureTerminalResult> {
return Object.freeze({
schemaVersion: 1 as const,
operation,
status,
state: 'reconciliation_captured' as const,
captureId: terminal.receipt.captureId,
bundleDigest: terminal.receipt.bundleDigest,
profile: terminal.receipt.profile,
assetCount: terminal.receipt.assetCount,
totalBytes: terminal.receipt.totalBytes,
instanceHeadDigest: head.headDigest,
});
}
function advanceCapturedHead(
intent: Readonly<LocalReconciliationCaptureIntent>,
uid: number,
committedAtMs: number,
bundleDigest: string,
): Readonly<LocalCutoverInstanceHead> {
return advanceLocalCutoverInstanceHead(
{
options: {
deploymentRoot: intent.command.options.deploymentRoot,
},
request: {
cutoverId: intent.command.request.cutoverId,
profile: intent.command.request.profile,
instanceId: intent.command.request.instanceId,
expectedActivationDigest:
intent.command.request.expectedActivationDigest,
requestedAtMs: committedAtMs,
},
},
uid,
'reconciliation_captured',
intent.command.request.generation,
bundleDigest,
);
}
export function commitLocalReconciliationCapture(
input: unknown,
dependencies: LocalReconciliationCaptureDependencies = {},
): Readonly<LocalReconciliationCaptureTerminalResult> {
const command = normalizeLocalReconciliationCaptureCommitCommand(input);
const identity = currentIdentity();
validatePrivateDirectory(
command.options.deploymentRoot,
identity.uid,
'deploymentRoot',
);
validatePrivateDirectory(
command.options.captureRoot,
identity.uid,
'captureRoot',
);
const intent = readLocalReconciliationCaptureIntent(
command.options.captureRoot,
command.request.captureId,
);
validateIntentBinding(intent, command);
const paths = capturePaths(
command.options.captureRoot,
command.request.captureId,
);
validatePrivateDirectory(paths.root, identity.uid, 'captureDirectory');
validatePrivateDirectory(
paths.staging,
identity.uid,
'captureStagingDirectory',
);
const head = readLocalCutoverInstanceHead(
command.options.deploymentRoot,
intent.command.request.instanceId,
identity.uid,
);
validateHeadIdentity(head, intent);
if (fs.existsSync(paths.receipt)) {
const terminal = readTerminal(paths, identity.uid);
if (
terminal.receipt.preparationDigest !== intent.preparationDigest ||
terminal.receipt.committedAtMs !== command.request.committedAtMs ||
(head.state !== 'reconciliation_capture_prepared' &&
head.state !== 'reconciliation_captured') ||
(head.state === 'reconciliation_capture_prepared' &&
head.sourceRecordDigest !== intent.preparationDigest) ||
(head.state === 'reconciliation_captured' &&
head.sourceRecordDigest !== terminal.receipt.bundleDigest)
) {
configurationError('terminal capture lost its instance head binding');
}
const terminalHead =
head.state === 'reconciliation_captured'
? head
: advanceCapturedHead(
intent,
identity.uid,
terminal.receipt.committedAtMs,
terminal.receipt.bundleDigest,
);
return terminalResult(
command.operation,
head.state === 'reconciliation_captured' ? 'existing' : 'prepared',
terminal,
terminalHead,
);
}
if (
head.state !== 'reconciliation_capture_prepared' ||
head.sourceRecordDigest !== intent.preparationDigest
) {
configurationError('capture commit lost the prepared instance head fence');
}
if (fs.existsSync(paths.manifest)) {
const manifest = readPublishedManifest(paths, identity.uid);
if (
manifest.captureId !== command.request.captureId ||
manifest.profile !== intent.command.request.profile ||
manifest.preparationDigest !== intent.preparationDigest ||
manifest.stoppedRecordDigest !==
intent.command.request.expectedStoppedRecordDigest ||
manifest.stoppedProofDigest !== intent.stoppedProofDigest ||
manifest.reconciliationEvidenceDigest !==
intent.reconciliationEvidenceDigest ||
manifest.lineageProjectionDigest !== intent.lineage.projectionDigest ||
manifest.preparedHeadDigest !== head.headDigest ||
manifest.committedAtMs !== command.request.committedAtMs
) {
configurationError('published capture manifest lost its preparation');
}
const receipt = captureReceipt(manifest);
publishExactFile(
paths.receipt,
receiptContents(receipt),
0o600,
identity.uid,
'reconciliation capture receipt',
);
dependencies.afterReceiptPublished?.();
const terminal = readTerminal(paths, identity.uid);
const terminalHead = advanceCapturedHead(
intent,
identity.uid,
manifest.committedAtMs,
receipt.bundleDigest,
);
return terminalResult(
command.operation,
'prepared',
terminal,
terminalHead,
);
}
const stoppedBefore = proveLocalReconciliationStoppedState(
intent.command,
identity.uid,
);
const lineageBefore = proveLocalReconciliationLineage(
intent.command,
identity.uid,
);
if (
stoppedBefore.proofDigest !== intent.stoppedProofDigest ||
stoppedBefore.reconciliationEvidenceDigest !==
intent.reconciliationEvidenceDigest ||
lineageBefore.projectionDigest !== intent.lineage.projectionDigest
) {
configurationError('capture source lineage drifted before copy');
}
ensurePrivateDirectory(paths.assets, identity.uid, 'captureAssetsDirectory');
validateTerminalCatalog(paths, false);
const plan = localReconciliationCaptureAssetPlan(intent);
const copied = plan.assets.map((asset) => {
const result = copyLocalReconciliationAsset(
asset,
paths.assets,
identity.uid,
dependencies.stableCopy,
);
dependencies.afterAssetPublished?.(asset.logicalName);
return result;
});
for (const result of copied) {
verifyLocalReconciliationSourceSnapshot(result.sourceSnapshot);
}
verifyLocalReconciliationSidecarPlan(
intent,
plan.targetSidecars,
plan.legacySidecars,
);
const recovery = copied.find(
(result) => result.manifest.logicalName === 'recovery-main',
);
if (
recovery === undefined ||
recovery.manifest.sha256 !== intent.lineage.recoverySha256
) {
configurationError('captured recovery database drifted from activation');
}
const stoppedAfter = proveLocalReconciliationStoppedState(
intent.command,
identity.uid,
);
const lineageAfter = proveLocalReconciliationLineage(
intent.command,
identity.uid,
);
if (
stoppedAfter.proofDigest !== stoppedBefore.proofDigest ||
lineageAfter.projectionDigest !== lineageBefore.projectionDigest
) {
configurationError('capture source lineage changed during copy');
}
const assets = Object.freeze(copied.map((result) => result.manifest));
const totalBytes = assets.reduce((total, asset) => total + asset.bytes, 0);
const manifestPayload = Object.freeze({
schema: MANIFEST_SCHEMA,
schemaVersion: 1 as const,
state: 'reconciliation_captured' as const,
captureId: command.request.captureId,
profile: intent.command.request.profile,
preparationDigest: intent.preparationDigest,
stoppedRecordDigest: intent.command.request.expectedStoppedRecordDigest,
stoppedProofDigest: intent.stoppedProofDigest,
reconciliationEvidenceDigest: intent.reconciliationEvidenceDigest,
lineageProjectionDigest: intent.lineage.projectionDigest,
preparedHeadDigest: head.headDigest,
committedAtMs: command.request.committedAtMs,
assets,
totalBytes,
});
const manifest: Readonly<LocalReconciliationCaptureManifest> = Object.freeze({
...manifestPayload,
manifestDigest: cutoverDigest(manifestPayload),
});
publishExactFile(
paths.manifest,
manifestContents(manifest),
0o600,
identity.uid,
'reconciliation capture manifest',
);
dependencies.afterManifestPublished?.();
const receipt = captureReceipt(manifest);
publishExactFile(
paths.receipt,
receiptContents(receipt),
0o600,
identity.uid,
'reconciliation capture receipt',
);
dependencies.afterReceiptPublished?.();
const terminal = readTerminal(paths, identity.uid);
const terminalHead = advanceCapturedHead(
intent,
identity.uid,
command.request.committedAtMs,
receipt.bundleDigest,
);
return terminalResult(command.operation, 'prepared', terminal, terminalHead);
}
export function verifyLocalReconciliationCapture(
input: unknown,
): Readonly<LocalReconciliationCaptureTerminalResult> {
const command = normalizeLocalReconciliationCaptureVerifyCommand(input);
const identity = currentIdentity();
validatePrivateDirectory(
command.options.deploymentRoot,
identity.uid,
'deploymentRoot',
);
validatePrivateDirectory(
command.options.captureRoot,
identity.uid,
'captureRoot',
);
const intent = readLocalReconciliationCaptureIntent(
command.options.captureRoot,
command.request.captureId,
);
if (
intent.command.options.deploymentRoot !== command.options.deploymentRoot ||
intent.command.options.captureRoot !== command.options.captureRoot ||
intent.command.options.allowRootService !==
command.options.allowRootService ||
intent.command.request.captureId !== command.request.captureId
) {
configurationError('capture verify is detached from preparation');
}
const terminal = readTerminal(
capturePaths(command.options.captureRoot, command.request.captureId),
identity.uid,
);
if (
terminal.receipt.bundleDigest !== command.request.expectedBundleDigest ||
terminal.receipt.preparationDigest !== intent.preparationDigest
) {
configurationError('capture verify expected bundle drifted');
}
const head = readLocalCutoverInstanceHead(
command.options.deploymentRoot,
intent.command.request.instanceId,
identity.uid,
);
validateHeadIdentity(head, intent);
if (
head.state !== 'reconciliation_captured' ||
head.sourceRecordDigest !== terminal.receipt.bundleDigest
) {
configurationError('capture verify lost the terminal instance head');
}
return terminalResult(command.operation, 'verified', terminal, head);
}
export function commitLocalReconciliationCaptureCommandFile(
filePath: string,
): Readonly<LocalReconciliationCaptureTerminalResult> {
return commitLocalReconciliationCapture(
readPrivateLocalCommandFile(filePath),
);
}
export function verifyLocalReconciliationCaptureCommandFile(
filePath: string,
): Readonly<LocalReconciliationCaptureTerminalResult> {
return verifyLocalReconciliationCapture(
readPrivateLocalCommandFile(filePath),
);
}
@@ -9,9 +9,7 @@ const UUID_V4_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
const MAX_PATH_BYTES = 4_096;
export type LocalReconciliationStoppedAuthority =
| 'docker'
| 'service-manager';
export type LocalReconciliationStoppedAuthority = 'docker' | 'service-manager';
export interface LocalReconciliationCapturePrepareCommand {
readonly schemaVersion: 1;
@@ -28,6 +26,7 @@ export interface LocalReconciliationCapturePrepareCommand {
instanceId: string;
cutoverId: string;
generation: number;
applicationConfigPath: string;
activationPath: string;
legacySourcePath: string;
targetDatabasePath: string;
@@ -49,6 +48,42 @@ export interface LocalReconciliationCapturePrepareResult {
readonly instanceHeadDigest: string;
}
export interface LocalReconciliationCaptureCommitCommand {
readonly schemaVersion: 1;
readonly operation: 'local.deployment.reconciliation.capture.commit';
readonly options: LocalReconciliationCapturePrepareCommand['options'];
readonly request: Readonly<{
captureId: string;
expectedPreparationDigest: string;
committedAtMs: number;
}>;
}
export interface LocalReconciliationCaptureVerifyCommand {
readonly schemaVersion: 1;
readonly operation: 'local.deployment.reconciliation.capture.verify';
readonly options: LocalReconciliationCapturePrepareCommand['options'];
readonly request: Readonly<{
captureId: string;
expectedBundleDigest: string;
}>;
}
export interface LocalReconciliationCaptureTerminalResult {
readonly schemaVersion: 1;
readonly operation:
| 'local.deployment.reconciliation.capture.commit'
| 'local.deployment.reconciliation.capture.verify';
readonly status: 'prepared' | 'existing' | 'verified';
readonly state: 'reconciliation_captured';
readonly captureId: string;
readonly bundleDigest: string;
readonly profile: 'edge' | 'standalone';
readonly assetCount: number;
readonly totalBytes: number;
readonly instanceHeadDigest: string;
}
function configurationError(message: string): never {
throw new LocalDeploymentConfigurationError(message);
}
@@ -124,6 +159,7 @@ export function normalizeLocalReconciliationCapturePrepareCommand(
request,
[
'activationPath',
'applicationConfigPath',
'captureId',
'cutoverId',
'expectedActivationDigest',
@@ -143,8 +179,7 @@ export function normalizeLocalReconciliationCapturePrepareCommand(
const identity = currentIdentity();
if (
command.schemaVersion !== 1 ||
command.operation !==
'local.deployment.reconciliation.capture.prepare' ||
command.operation !== 'local.deployment.reconciliation.capture.prepare' ||
typeof options.allowRootService !== 'boolean' ||
(identity.uid === 0) !== options.allowRootService ||
typeof request.captureId !== 'string' ||
@@ -175,8 +210,7 @@ export function normalizeLocalReconciliationCapturePrepareCommand(
}
return Object.freeze({
schemaVersion: 1 as const,
operation:
'local.deployment.reconciliation.capture.prepare' as const,
operation: 'local.deployment.reconciliation.capture.prepare' as const,
options: Object.freeze({
deploymentRoot,
captureRoot,
@@ -189,6 +223,10 @@ export function normalizeLocalReconciliationCapturePrepareCommand(
instanceId: request.instanceId,
cutoverId: request.cutoverId,
generation: request.generation as number,
applicationConfigPath: safeAbsolutePath(
request.applicationConfigPath,
'applicationConfigPath',
),
activationPath: safeAbsolutePath(
request.activationPath,
'activationPath',
@@ -218,3 +256,109 @@ export function normalizeLocalReconciliationCapturePrepareCommand(
}),
});
}
function normalizeTerminalOptions(value: unknown): Readonly<{
deploymentRoot: string;
captureRoot: string;
allowRootService: boolean;
}> {
const options = object(value, 'options');
exact(
options,
['allowRootService', 'captureRoot', 'deploymentRoot'],
'options',
);
const identity = currentIdentity();
if (
typeof options.allowRootService !== 'boolean' ||
(identity.uid === 0) !== options.allowRootService
) {
configurationError('capture command identity is invalid');
}
const deploymentRoot = safeAbsolutePath(
options.deploymentRoot,
'deploymentRoot',
);
const captureRoot = safeAbsolutePath(options.captureRoot, 'captureRoot');
if (captureRoot === deploymentRoot) {
configurationError('captureRoot must be distinct from deploymentRoot');
}
return Object.freeze({
deploymentRoot,
captureRoot,
allowRootService: options.allowRootService,
});
}
export function normalizeLocalReconciliationCaptureCommitCommand(
value: unknown,
): Readonly<LocalReconciliationCaptureCommitCommand> {
const command = object(value, 'reconciliation capture commit command');
exact(
command,
['operation', 'options', 'request', 'schemaVersion'],
'command',
);
const request = object(command.request, 'request');
exact(
request,
['captureId', 'committedAtMs', 'expectedPreparationDigest'],
'request',
);
if (
command.schemaVersion !== 1 ||
command.operation !== 'local.deployment.reconciliation.capture.commit' ||
typeof request.captureId !== 'string' ||
!UUID_V4_PATTERN.test(request.captureId) ||
!Number.isSafeInteger(request.committedAtMs) ||
(request.committedAtMs as number) < 0
) {
configurationError('reconciliation capture commit command is invalid');
}
return Object.freeze({
schemaVersion: 1 as const,
operation: 'local.deployment.reconciliation.capture.commit' as const,
options: normalizeTerminalOptions(command.options),
request: Object.freeze({
captureId: request.captureId,
expectedPreparationDigest: digest(
request.expectedPreparationDigest,
'expectedPreparationDigest',
),
committedAtMs: request.committedAtMs as number,
}),
});
}
export function normalizeLocalReconciliationCaptureVerifyCommand(
value: unknown,
): Readonly<LocalReconciliationCaptureVerifyCommand> {
const command = object(value, 'reconciliation capture verify command');
exact(
command,
['operation', 'options', 'request', 'schemaVersion'],
'command',
);
const request = object(command.request, 'request');
exact(request, ['captureId', 'expectedBundleDigest'], 'request');
if (
command.schemaVersion !== 1 ||
command.operation !== 'local.deployment.reconciliation.capture.verify' ||
typeof request.captureId !== 'string' ||
!UUID_V4_PATTERN.test(request.captureId)
) {
configurationError('reconciliation capture verify command is invalid');
}
return Object.freeze({
schemaVersion: 1 as const,
operation: 'local.deployment.reconciliation.capture.verify' as const,
options: normalizeTerminalOptions(command.options),
request: Object.freeze({
captureId: request.captureId,
expectedBundleDigest: digest(
request.expectedBundleDigest,
'expectedBundleDigest',
),
}),
});
}
@@ -0,0 +1,466 @@
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { LocalDeploymentConfigurationError } from '../foundation/error';
import { cutoverDigest } from '../cutover/targetEvidence';
import type { LocalReconciliationCapturePrepareCommand } from './contract';
const MAX_LINEAGE_FILE_BYTES = 64 * 1024;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
const UUID_V4_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
export interface LocalReconciliationLineageProjection {
readonly applicationConfigDigest: string;
readonly activationDigest: string;
readonly adoptionManifestDigest: string;
readonly commitmentDigest: string;
readonly legacyDataApplicationCommitDigest: string;
readonly legacyDataApplicationReceiptDigest: string;
readonly adoptedBundleDigest: string;
readonly recoverySha256: string;
readonly projectionDigest: string;
}
interface PrivateJsonMaterial {
readonly value: unknown;
readonly sha256: string;
}
interface LocalDataApplicationEvidence {
readonly profile: 'edge' | 'standalone';
readonly commitDigest: string;
readonly receiptDigest: string;
}
function configurationError(message: string, cause?: unknown): never {
throw new LocalDeploymentConfigurationError(message, { cause });
}
function object(value: unknown, label: string): Record<string, unknown> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
(Object.getPrototypeOf(value) !== Object.prototype &&
Object.getPrototypeOf(value) !== null)
) {
configurationError(`${label} must be an object`);
}
return value as Record<string, unknown>;
}
function exact(
value: Record<string, unknown>,
keys: readonly string[],
label: string,
): void {
const actual = Object.keys(value).sort();
const expected = [...keys].sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index])
) {
configurationError(`${label} shape is invalid`);
}
}
function sameStat(left: fs.BigIntStats, right: fs.BigIntStats): boolean {
return (
left.dev === right.dev &&
left.ino === right.ino &&
left.uid === right.uid &&
left.gid === right.gid &&
left.mode === right.mode &&
left.nlink === right.nlink &&
left.size === right.size &&
left.mtimeNs === right.mtimeNs &&
left.ctimeNs === right.ctimeNs
);
}
function privateJsonMaterial(
filePath: string,
uid: number,
label: string,
): Readonly<PrivateJsonMaterial> {
let descriptor: number | undefined;
let bytes: Buffer | undefined;
try {
const pathStat = fs.lstatSync(filePath, { bigint: true });
descriptor = fs.openSync(
filePath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
const opened = fs.fstatSync(descriptor, { bigint: true });
if (
!pathStat.isFile() ||
pathStat.isSymbolicLink() ||
!sameStat(pathStat, opened) ||
opened.uid !== BigInt(uid) ||
opened.nlink !== 1n ||
(opened.mode & 0o077n) !== 0n ||
opened.size < 2n ||
opened.size > BigInt(MAX_LINEAGE_FILE_BYTES) ||
fs.realpathSync(filePath) !== filePath
) {
configurationError(`${label} identity is invalid`);
}
bytes = Buffer.allocUnsafe(Number(opened.size));
let offset = 0;
while (offset < bytes.byteLength) {
const count = fs.readSync(
descriptor,
bytes,
offset,
bytes.byteLength - offset,
null,
);
if (count === 0) break;
offset += count;
}
const after = fs.fstatSync(descriptor, { bigint: true });
if (offset !== bytes.byteLength || !sameStat(opened, after)) {
configurationError(`${label} changed while reading`);
}
let value: unknown;
try {
value = JSON.parse(
new TextDecoder('utf-8', { fatal: true }).decode(bytes),
) as unknown;
} catch (error) {
configurationError(`${label} is not valid UTF-8 JSON`, error);
}
return Object.freeze({
value,
sha256: crypto.createHash('sha256').update(bytes).digest('hex'),
});
} catch (error) {
if (error instanceof LocalDeploymentConfigurationError) throw error;
return configurationError(`${label} cannot be read`, error);
} finally {
bytes?.fill(0);
if (descriptor !== undefined) fs.closeSync(descriptor);
}
}
function documentDigest(
value: unknown,
digestKey: string,
label: string,
): Readonly<{ document: Record<string, unknown>; digest: string }> {
const document = object(value, label);
const claimed = document[digestKey];
const payload = { ...document };
delete payload[digestKey];
if (
typeof claimed !== 'string' ||
!DIGEST_PATTERN.test(claimed) ||
cutoverDigest(payload) !== claimed
) {
configurationError(`${label} digest drifted`);
}
return Object.freeze({ document, digest: claimed });
}
function textDigest(value: string): string {
return crypto.createHash('sha256').update(value, 'utf8').digest('hex');
}
function normalizeLocalDataApplicationEvidence(
value: unknown,
): Readonly<LocalDataApplicationEvidence> {
const commit = object(value, 'legacy data application commitment');
exact(
commit,
[
'commitDigest',
'committedAtMs',
'environmentSecretCount',
'kind',
'modelDigest',
'mutationId',
'profile',
'projectIdDigest',
'publicationDigest',
'receiptDigest',
'reclamation',
'schemaVersion',
'secretCount',
'sourceStageManifestDigest',
'sshSecretCount',
'state',
'transformationDigest',
],
'legacy data application commitment',
);
const reclamation = object(
commit.reclamation,
'legacy data application reclamation',
);
exact(
reclamation,
['modelRemoved', 'physicalErasureGuaranteed', 'plaintextFilesRemoved'],
'legacy data application reclamation',
);
const digests = [
commit.projectIdDigest,
commit.sourceStageManifestDigest,
commit.transformationDigest,
commit.modelDigest,
commit.publicationDigest,
commit.receiptDigest,
];
if (
commit.schemaVersion !== 1 ||
commit.kind !== 'qinglong3-legacy-data-directory-application' ||
commit.state !== 'committed' ||
typeof commit.mutationId !== 'string' ||
!UUID_V4_PATTERN.test(commit.mutationId) ||
(commit.profile !== 'edge' && commit.profile !== 'standalone') ||
digests.some(
(candidate) =>
typeof candidate !== 'string' || !DIGEST_PATTERN.test(candidate),
) ||
!Number.isSafeInteger(commit.secretCount) ||
(commit.secretCount as number) < 0 ||
!Number.isSafeInteger(commit.environmentSecretCount) ||
(commit.environmentSecretCount as number) < 0 ||
!Number.isSafeInteger(commit.sshSecretCount) ||
(commit.sshSecretCount as number) < 0 ||
(commit.secretCount as number) !==
(commit.environmentSecretCount as number) +
(commit.sshSecretCount as number) ||
!Number.isSafeInteger(commit.committedAtMs) ||
(commit.committedAtMs as number) < 0 ||
reclamation.modelRemoved !== true ||
reclamation.plaintextFilesRemoved !== true ||
reclamation.physicalErasureGuaranteed !== false
) {
configurationError('legacy data application commitment values are invalid');
}
const payload = {
schemaVersion: 1,
kind: 'qinglong3-legacy-data-directory-application',
state: 'committed',
mutationId: commit.mutationId,
profile: commit.profile,
projectIdDigest: commit.projectIdDigest,
sourceStageManifestDigest: commit.sourceStageManifestDigest,
transformationDigest: commit.transformationDigest,
modelDigest: commit.modelDigest,
publicationDigest: commit.publicationDigest,
receiptDigest: commit.receiptDigest,
secretCount: commit.secretCount,
environmentSecretCount: commit.environmentSecretCount,
sshSecretCount: commit.sshSecretCount,
committedAtMs: commit.committedAtMs,
reclamation: {
modelRemoved: true,
plaintextFilesRemoved: true,
physicalErasureGuaranteed: false,
},
};
if (
typeof commit.commitDigest !== 'string' ||
!DIGEST_PATTERN.test(commit.commitDigest) ||
textDigest(JSON.stringify(payload)) !== commit.commitDigest
) {
configurationError('legacy data application commitment digest drifted');
}
return Object.freeze({
profile: commit.profile,
commitDigest: commit.commitDigest,
receiptDigest: commit.receiptDigest as string,
});
}
export function proveLocalReconciliationLineage(
command: Readonly<LocalReconciliationCapturePrepareCommand>,
uid: number,
): Readonly<LocalReconciliationLineageProjection> {
if (
command.request.applicationConfigPath !==
path.join(command.options.deploymentRoot, 'local-application.json')
) {
configurationError('application configuration path is not authoritative');
}
const applicationMaterial = privateJsonMaterial(
command.request.applicationConfigPath,
uid,
'application configuration',
);
const application = object(
applicationMaterial.value,
'application configuration',
);
const storage = object(application.storage, 'application storage');
const cutover = object(application.cutover, 'application cutover');
const dataApplication = object(
application.legacyDataApplication,
'legacy data application',
);
exact(
dataApplication,
['commitPath', 'expectedCommitDigest', 'expectedReceiptDigest'],
'legacy data application',
);
const commitmentPath = path.join(
command.options.deploymentRoot,
'service',
'cutovers',
command.request.cutoverId,
'0002-legacy-stopped.json',
);
if (
application.schema !== 'qinglong/local-application-process@v4' ||
application.profile !== command.request.profile ||
application.instanceId !== command.request.instanceId ||
storage.mode !== 'adopted' ||
storage.sourcePath !== command.request.legacySourcePath ||
storage.targetPath !== command.request.targetDatabasePath ||
storage.recoveryPath !== command.request.recoveryPath ||
storage.activationPath !== command.request.activationPath ||
storage.expectedActivationDigest !==
command.request.expectedActivationDigest ||
cutover.cutoverId !== command.request.cutoverId ||
cutover.commitmentPath !== commitmentPath ||
typeof cutover.expectedCommitmentDigest !== 'string' ||
!DIGEST_PATTERN.test(cutover.expectedCommitmentDigest) ||
typeof storage.manifestPath !== 'string' ||
!path.isAbsolute(storage.manifestPath) ||
path.normalize(storage.manifestPath) !== storage.manifestPath ||
typeof dataApplication.commitPath !== 'string' ||
!path.isAbsolute(dataApplication.commitPath) ||
path.normalize(dataApplication.commitPath) !== dataApplication.commitPath ||
typeof dataApplication.expectedCommitDigest !== 'string' ||
!DIGEST_PATTERN.test(dataApplication.expectedCommitDigest) ||
typeof dataApplication.expectedReceiptDigest !== 'string' ||
!DIGEST_PATTERN.test(dataApplication.expectedReceiptDigest)
) {
configurationError('application lineage binding drifted');
}
const activation = documentDigest(
privateJsonMaterial(command.request.activationPath, uid, 'activation')
.value,
'activationDigest',
'activation',
);
if (
activation.document.schemaVersion !== 1 ||
activation.document.kind !== 'qinglong3-local-sqlite-activation' ||
activation.document.state !== 'prepared' ||
activation.document.profile !== command.request.profile ||
activation.digest !== command.request.expectedActivationDigest ||
activation.document.sourcePathDigest !==
textDigest(command.request.legacySourcePath) ||
activation.document.targetPathDigest !==
textDigest(command.request.targetDatabasePath) ||
typeof activation.document.adoptionManifestDigest !== 'string' ||
!DIGEST_PATTERN.test(activation.document.adoptionManifestDigest) ||
typeof activation.document.recoverySha256 !== 'string' ||
!DIGEST_PATTERN.test(activation.document.recoverySha256)
) {
configurationError('activation lineage drifted');
}
const manifest = documentDigest(
privateJsonMaterial(
storage.manifestPath as string,
uid,
'adoption manifest',
).value,
'manifestDigest',
'adoption manifest',
);
if (manifest.digest !== activation.document.adoptionManifestDigest) {
configurationError('adoption manifest lineage drifted');
}
const commitment = documentDigest(
privateJsonMaterial(commitmentPath, uid, 'legacy silence commitment').value,
'commitmentDigest',
'legacy silence commitment',
);
if (
commitment.document.schemaVersion !== 1 ||
commitment.document.kind !== 'qinglong3-local-legacy-silence-commitment' ||
commitment.document.state !== 'legacy_stopped' ||
commitment.document.cutoverId !== command.request.cutoverId ||
commitment.document.profile !== command.request.profile ||
commitment.document.instanceId !== command.request.instanceId ||
commitment.document.activationDigest !== activation.digest ||
commitment.digest !== cutover.expectedCommitmentDigest
) {
configurationError('legacy silence lineage drifted');
}
const dataCommit = normalizeLocalDataApplicationEvidence(
privateJsonMaterial(
dataApplication.commitPath as string,
uid,
'legacy data application commitment',
).value,
);
if (
dataCommit.profile !== command.request.profile ||
dataCommit.commitDigest !== dataApplication.expectedCommitDigest ||
dataCommit.receiptDigest !== dataApplication.expectedReceiptDigest
) {
configurationError('legacy data application receipt drifted');
}
const adoptedBundle = documentDigest(
privateJsonMaterial(
path.join(
command.options.deploymentRoot,
'service',
'adopted-bundle.json',
),
uid,
'adopted bundle receipt',
).value,
'bundleDigest',
'adopted bundle receipt',
);
if (
adoptedBundle.document.schemaVersion !== 1 ||
adoptedBundle.document.kind !==
'qinglong3-local-adopted-deployment-bundle' ||
adoptedBundle.document.state !== 'prepared' ||
adoptedBundle.document.profile !== command.request.profile ||
adoptedBundle.document.instanceId !== command.request.instanceId ||
adoptedBundle.document.cutoverId !== command.request.cutoverId ||
adoptedBundle.document.applicationConfigDigest !==
applicationMaterial.sha256 ||
adoptedBundle.document.activationDigest !== activation.digest ||
adoptedBundle.document.commitmentDigest !== commitment.digest ||
adoptedBundle.document.legacyDataApplicationCommitDigest !==
dataCommit.commitDigest ||
adoptedBundle.document.legacyDataApplicationReceiptDigest !==
dataCommit.receiptDigest ||
adoptedBundle.document.manifestDigest !== manifest.digest ||
adoptedBundle.document.sourcePathDigest !==
textDigest(command.request.legacySourcePath) ||
adoptedBundle.document.recoverySha256 !== activation.document.recoverySha256
) {
configurationError('adopted bundle lineage drifted');
}
const payload = Object.freeze({
applicationConfigDigest: applicationMaterial.sha256,
activationDigest: activation.digest,
adoptionManifestDigest: manifest.digest,
commitmentDigest: commitment.digest,
legacyDataApplicationCommitDigest: dataCommit.commitDigest,
legacyDataApplicationReceiptDigest: dataCommit.receiptDigest,
adoptedBundleDigest: adoptedBundle.digest,
recoverySha256: activation.document.recoverySha256 as string,
});
return Object.freeze({
...payload,
projectionDigest: cutoverDigest(payload),
});
}
@@ -21,16 +21,22 @@ import {
type LocalReconciliationCapturePrepareResult,
} from './contract';
import { proveLocalReconciliationStoppedState } from './stoppedProof';
import {
proveLocalReconciliationLineage,
type LocalReconciliationLineageProjection,
} from './lineageProof';
const INTENT_SCHEMA = 'qinglong3-local-reconciliation-capture-intent';
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
interface LocalReconciliationCaptureIntent {
export interface LocalReconciliationCaptureIntent {
readonly schema: typeof INTENT_SCHEMA;
readonly schemaVersion: 1;
readonly state: 'reconciliation_capture_prepared';
readonly command: Readonly<LocalReconciliationCapturePrepareCommand>;
readonly stoppedProofDigest: string;
readonly reconciliationEvidenceDigest: string;
readonly lineage: Readonly<LocalReconciliationLineageProjection>;
readonly preparationDigest: string;
}
@@ -38,6 +44,34 @@ function configurationError(message: string): never {
throw new LocalDeploymentConfigurationError(message);
}
function object(value: unknown, label: string): Record<string, unknown> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
(Object.getPrototypeOf(value) !== Object.prototype &&
Object.getPrototypeOf(value) !== null)
) {
configurationError(`${label} must be an object`);
}
return value as Record<string, unknown>;
}
function exact(
value: Record<string, unknown>,
keys: readonly string[],
label: string,
): void {
const actual = Object.keys(value).sort();
const expected = [...keys].sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index])
) {
configurationError(`${label} shape is invalid`);
}
}
export function localReconciliationCaptureDirectory(
captureRoot: string,
captureId: string,
@@ -55,7 +89,92 @@ export function localReconciliationCaptureIntentPath(
);
}
function intentContents(intent: Readonly<LocalReconciliationCaptureIntent>): string {
export function normalizeLocalReconciliationCaptureIntent(
value: unknown,
): Readonly<LocalReconciliationCaptureIntent> {
const intent = object(value, 'reconciliation capture intent');
exact(
intent,
[
'command',
'lineage',
'preparationDigest',
'reconciliationEvidenceDigest',
'schema',
'schemaVersion',
'state',
'stoppedProofDigest',
],
'reconciliation capture intent',
);
const command = normalizeLocalReconciliationCapturePrepareCommand(
intent.command,
);
const lineage = object(intent.lineage, 'reconciliation lineage projection');
exact(
lineage,
[
'activationDigest',
'adoptedBundleDigest',
'adoptionManifestDigest',
'applicationConfigDigest',
'commitmentDigest',
'legacyDataApplicationCommitDigest',
'legacyDataApplicationReceiptDigest',
'projectionDigest',
'recoverySha256',
],
'reconciliation lineage projection',
);
const { projectionDigest, ...lineagePayload } = lineage;
const { preparationDigest, ...payload } = intent;
if (
intent.schema !== INTENT_SCHEMA ||
intent.schemaVersion !== 1 ||
intent.state !== 'reconciliation_capture_prepared' ||
typeof intent.stoppedProofDigest !== 'string' ||
!DIGEST_PATTERN.test(intent.stoppedProofDigest) ||
typeof intent.reconciliationEvidenceDigest !== 'string' ||
!DIGEST_PATTERN.test(intent.reconciliationEvidenceDigest) ||
Object.values(lineage).some(
(candidate) =>
typeof candidate !== 'string' || !DIGEST_PATTERN.test(candidate),
) ||
cutoverDigest(lineagePayload) !== projectionDigest ||
typeof preparationDigest !== 'string' ||
!DIGEST_PATTERN.test(preparationDigest) ||
cutoverDigest(payload) !== preparationDigest
) {
configurationError('reconciliation capture intent drifted');
}
return Object.freeze({
schema: INTENT_SCHEMA,
schemaVersion: 1 as const,
state: 'reconciliation_capture_prepared' as const,
command,
stoppedProofDigest: intent.stoppedProofDigest,
reconciliationEvidenceDigest: intent.reconciliationEvidenceDigest,
lineage: Object.freeze(
lineage,
) as unknown as Readonly<LocalReconciliationLineageProjection>,
preparationDigest,
});
}
export function readLocalReconciliationCaptureIntent(
captureRoot: string,
captureId: string,
): Readonly<LocalReconciliationCaptureIntent> {
return normalizeLocalReconciliationCaptureIntent(
readPrivateLocalCommandFile(
localReconciliationCaptureIntentPath(captureRoot, captureId),
),
);
}
function intentContents(
intent: Readonly<LocalReconciliationCaptureIntent>,
): string {
return `${JSON.stringify(intent, null, 2)}\n`;
}
@@ -96,6 +215,7 @@ export function prepareLocalReconciliationCapture(
);
}
const proof = proveLocalReconciliationStoppedState(command, identity.uid);
const lineage = proveLocalReconciliationLineage(command, identity.uid);
const payload = Object.freeze({
schema: INTENT_SCHEMA,
schemaVersion: 1 as const,
@@ -103,6 +223,7 @@ export function prepareLocalReconciliationCapture(
command,
stoppedProofDigest: proof.proofDigest,
reconciliationEvidenceDigest: proof.reconciliationEvidenceDigest,
lineage,
});
const intent: Readonly<LocalReconciliationCaptureIntent> = Object.freeze({
...payload,
@@ -177,5 +298,7 @@ export function prepareLocalReconciliationCapture(
export function prepareLocalReconciliationCaptureCommandFile(
filePath: string,
): Readonly<LocalReconciliationCapturePrepareResult> {
return prepareLocalReconciliationCapture(readPrivateLocalCommandFile(filePath));
return prepareLocalReconciliationCapture(
readPrivateLocalCommandFile(filePath),
);
}
@@ -0,0 +1,551 @@
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { LocalDeploymentConfigurationError } from '../foundation/error';
import { syncPublishedDirectory } from '../foundation/files';
import { cutoverDigest } from '../cutover/targetEvidence';
import type { LocalReconciliationCaptureIntent } from './preparation';
const COPY_BUFFER_BYTES = 64 * 1024;
const SIDECAR_SUFFIXES = ['-wal', '-shm', '-journal'] as const;
export interface LocalReconciliationCaptureSourceAsset {
readonly logicalName:
| 'target-main'
| 'target-wal'
| 'target-shm'
| 'target-journal'
| 'legacy-main'
| 'legacy-wal'
| 'legacy-shm'
| 'legacy-journal'
| 'recovery-main';
readonly sourcePath: string;
readonly requireNonEmpty: boolean;
}
export interface LocalReconciliationCapturedAsset {
readonly logicalName: LocalReconciliationCaptureSourceAsset['logicalName'];
readonly bytes: number;
readonly sha256: string;
readonly sourceIdentityDigest: string;
}
export interface LocalReconciliationStableCopyResult {
readonly manifest: Readonly<LocalReconciliationCapturedAsset>;
readonly sourceSnapshot: Readonly<{
path: string;
device: string;
inode: string;
uid: number;
gid: number;
mode: number;
links: number;
bytes: number;
modifiedAtNs: string;
changedAtNs: string;
}>;
}
export interface LocalReconciliationStableCopyDependencies {
readonly write?: (
descriptor: number,
buffer: Buffer,
offset: number,
length: number,
position: number,
) => number;
readonly unlink?: (filePath: string) => void;
}
function configurationError(message: string, cause?: unknown): never {
throw new LocalDeploymentConfigurationError(message, { cause });
}
function sidecarExists(filePath: string): boolean {
try {
fs.lstatSync(filePath);
return true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
return configurationError('SQLite sidecar cannot be inspected', error);
}
}
export function localReconciliationSidecarSnapshot(
mainPath: string,
): readonly boolean[] {
return Object.freeze(
SIDECAR_SUFFIXES.map((suffix) => sidecarExists(`${mainPath}${suffix}`)),
);
}
export function localReconciliationCaptureAssetPlan(
intent: Readonly<LocalReconciliationCaptureIntent>,
): Readonly<{
assets: readonly Readonly<LocalReconciliationCaptureSourceAsset>[];
targetSidecars: readonly boolean[];
legacySidecars: readonly boolean[];
}> {
const targetSidecars = localReconciliationSidecarSnapshot(
intent.command.request.targetDatabasePath,
);
const legacySidecars = localReconciliationSidecarSnapshot(
intent.command.request.legacySourcePath,
);
const assets: LocalReconciliationCaptureSourceAsset[] = [
{
logicalName: 'target-main',
sourcePath: intent.command.request.targetDatabasePath,
requireNonEmpty: true,
},
...SIDECAR_SUFFIXES.flatMap((suffix, index) =>
targetSidecars[index]
? [
{
logicalName: `target-${suffix.slice(
1,
)}` as LocalReconciliationCaptureSourceAsset['logicalName'],
sourcePath: `${intent.command.request.targetDatabasePath}${suffix}`,
requireNonEmpty: false,
},
]
: [],
),
{
logicalName: 'legacy-main',
sourcePath: intent.command.request.legacySourcePath,
requireNonEmpty: true,
},
...SIDECAR_SUFFIXES.flatMap((suffix, index) =>
legacySidecars[index]
? [
{
logicalName: `legacy-${suffix.slice(
1,
)}` as LocalReconciliationCaptureSourceAsset['logicalName'],
sourcePath: `${intent.command.request.legacySourcePath}${suffix}`,
requireNonEmpty: false,
},
]
: [],
),
{
logicalName: 'recovery-main',
sourcePath: intent.command.request.recoveryPath,
requireNonEmpty: true,
},
];
return Object.freeze({
assets: Object.freeze(assets.map((asset) => Object.freeze(asset))),
targetSidecars,
legacySidecars,
});
}
function sameStat(left: fs.BigIntStats, right: fs.BigIntStats): boolean {
return (
left.dev === right.dev &&
left.ino === right.ino &&
left.uid === right.uid &&
left.gid === right.gid &&
left.mode === right.mode &&
left.nlink === right.nlink &&
left.size === right.size &&
left.mtimeNs === right.mtimeNs &&
left.ctimeNs === right.ctimeNs
);
}
function validateSource(
filePath: string,
stat: fs.BigIntStats,
uid: number,
requireNonEmpty: boolean,
): void {
if (
!stat.isFile() ||
stat.isSymbolicLink() ||
stat.uid !== BigInt(uid) ||
stat.nlink !== 1n ||
(stat.mode & 0o077n) !== 0n ||
(requireNonEmpty && stat.size < 1n) ||
stat.size > BigInt(Number.MAX_SAFE_INTEGER) ||
fs.realpathSync(filePath) !== filePath
) {
configurationError('capture source identity is invalid');
}
}
function sourceSnapshot(
filePath: string,
stat: fs.BigIntStats,
): LocalReconciliationStableCopyResult['sourceSnapshot'] {
return Object.freeze({
path: filePath,
device: stat.dev.toString(),
inode: stat.ino.toString(),
uid: Number(stat.uid),
gid: Number(stat.gid),
mode: Number(stat.mode),
links: Number(stat.nlink),
bytes: Number(stat.size),
modifiedAtNs: stat.mtimeNs.toString(),
changedAtNs: stat.ctimeNs.toString(),
});
}
function hashDescriptor(
descriptor: number,
bytes: number,
buffer: Buffer,
): string {
const hash = crypto.createHash('sha256');
let offset = 0;
while (offset < bytes) {
const count = fs.readSync(
descriptor,
buffer,
0,
Math.min(buffer.byteLength, bytes - offset),
offset,
);
if (count < 1) configurationError('capture file read stalled');
hash.update(buffer.subarray(0, count));
offset += count;
}
return hash.digest('hex');
}
function hashPrefix(descriptor: number, bytes: number, buffer: Buffer): string {
return hashDescriptor(descriptor, bytes, buffer);
}
function validateOutputDescriptor(
descriptor: number,
uid: number,
allowedLinks: readonly bigint[],
): fs.BigIntStats {
const stat = fs.fstatSync(descriptor, { bigint: true });
if (
!stat.isFile() ||
stat.uid !== BigInt(uid) ||
(stat.mode & 0o777n) !== 0o600n ||
!allowedLinks.includes(stat.nlink) ||
stat.size > BigInt(Number.MAX_SAFE_INTEGER)
) {
configurationError('capture output identity is invalid');
}
return stat;
}
function verifyPublishedAsset(
targetPath: string,
uid: number,
expectedBytes: number,
expectedSha256: string,
buffer: Buffer,
allowedLinks: readonly bigint[] = [1n],
): void {
let descriptor: number | undefined;
try {
const pathStat = fs.lstatSync(targetPath, { bigint: true });
descriptor = fs.openSync(
targetPath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
const opened = validateOutputDescriptor(descriptor, uid, allowedLinks);
if (
pathStat.isSymbolicLink() ||
!sameStat(pathStat, opened) ||
Number(opened.size) !== expectedBytes ||
fs.realpathSync(targetPath) !== targetPath ||
hashDescriptor(descriptor, expectedBytes, buffer) !== expectedSha256 ||
!sameStat(opened, fs.fstatSync(descriptor, { bigint: true }))
) {
configurationError('published capture asset drifted');
}
} catch (error) {
if (error instanceof LocalDeploymentConfigurationError) throw error;
configurationError('published capture asset cannot be verified', error);
} finally {
if (descriptor !== undefined) fs.closeSync(descriptor);
}
}
function appendSourceToStage(
sourceDescriptor: number,
stageDescriptor: number,
start: number,
total: number,
buffer: Buffer,
write: NonNullable<LocalReconciliationStableCopyDependencies['write']>,
): void {
let sourceOffset = start;
while (sourceOffset < total) {
const count = fs.readSync(
sourceDescriptor,
buffer,
0,
Math.min(buffer.byteLength, total - sourceOffset),
sourceOffset,
);
if (count < 1) configurationError('capture source read stalled');
let written = 0;
while (written < count) {
const countWritten = write(
stageDescriptor,
buffer,
written,
count - written,
sourceOffset + written,
);
if (countWritten < 1) configurationError('capture stage write stalled');
written += countWritten;
}
sourceOffset += count;
}
}
export function copyLocalReconciliationAsset(
asset: Readonly<LocalReconciliationCaptureSourceAsset>,
assetsDirectory: string,
uid: number,
dependencies: LocalReconciliationStableCopyDependencies = {},
): Readonly<LocalReconciliationStableCopyResult> {
const targetPath = path.join(assetsDirectory, asset.logicalName);
const stagePath = path.join(
assetsDirectory,
`.${asset.logicalName}.ql3-capture-stage`,
);
const buffer = Buffer.allocUnsafe(COPY_BUFFER_BYTES);
const unlink = dependencies.unlink ?? fs.unlinkSync;
let sourceDescriptor: number | undefined;
let stageDescriptor: number | undefined;
let createdStage = false;
try {
const pathStat = fs.lstatSync(asset.sourcePath, { bigint: true });
sourceDescriptor = fs.openSync(
asset.sourcePath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
const opened = fs.fstatSync(sourceDescriptor, { bigint: true });
validateSource(asset.sourcePath, pathStat, uid, asset.requireNonEmpty);
if (!sameStat(pathStat, opened)) {
configurationError('capture source changed while opening');
}
const bytes = Number(opened.size);
const sha256 = hashDescriptor(sourceDescriptor, bytes, buffer);
if (!sameStat(opened, fs.fstatSync(sourceDescriptor, { bigint: true }))) {
configurationError('capture source changed while hashing');
}
const identityPayload = Object.freeze({
pathDigest: crypto
.createHash('sha256')
.update(asset.sourcePath, 'utf8')
.digest('hex'),
device: opened.dev.toString(),
inode: opened.ino.toString(),
uid: Number(opened.uid),
gid: Number(opened.gid),
mode: Number(opened.mode),
links: Number(opened.nlink),
bytes,
modifiedAtNs: opened.mtimeNs.toString(),
changedAtNs: opened.ctimeNs.toString(),
});
const manifest = Object.freeze({
logicalName: asset.logicalName,
bytes,
sha256,
sourceIdentityDigest: cutoverDigest(identityPayload),
});
if (fs.existsSync(targetPath)) {
if (fs.existsSync(stagePath)) {
const target = fs.lstatSync(targetPath, { bigint: true });
const stage = fs.lstatSync(stagePath, { bigint: true });
if (
target.isSymbolicLink() ||
stage.isSymbolicLink() ||
target.nlink !== 2n ||
stage.nlink !== 2n ||
target.dev !== stage.dev ||
target.ino !== stage.ino ||
fs.realpathSync(stagePath) !== stagePath
) {
configurationError('linked capture stage identity drifted');
}
verifyPublishedAsset(targetPath, uid, bytes, sha256, buffer, [2n]);
verifyPublishedAsset(stagePath, uid, bytes, sha256, buffer, [2n]);
unlink(stagePath);
syncPublishedDirectory(assetsDirectory);
}
verifyPublishedAsset(targetPath, uid, bytes, sha256, buffer);
return Object.freeze({
manifest,
sourceSnapshot: sourceSnapshot(asset.sourcePath, opened),
});
}
try {
stageDescriptor = fs.openSync(
stagePath,
fs.constants.O_RDWR |
fs.constants.O_CREAT |
fs.constants.O_EXCL |
(fs.constants.O_NOFOLLOW ?? 0),
0o600,
);
createdStage = true;
fs.fchmodSync(stageDescriptor, 0o600);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
stageDescriptor = fs.openSync(
stagePath,
fs.constants.O_RDWR | (fs.constants.O_NOFOLLOW ?? 0),
);
}
const stage = validateOutputDescriptor(stageDescriptor, uid, [1n]);
const stagedBytes = Number(stage.size);
if (stagedBytes > bytes) {
configurationError('capture stage exceeds its exact source');
}
if (stagedBytes > 0) {
const stagedHash = hashDescriptor(stageDescriptor, stagedBytes, buffer);
const sourcePrefixHash = hashPrefix(
sourceDescriptor,
stagedBytes,
buffer,
);
if (stagedHash !== sourcePrefixHash) {
configurationError('capture stage does not match its exact source');
}
}
appendSourceToStage(
sourceDescriptor,
stageDescriptor,
stagedBytes,
bytes,
buffer,
dependencies.write ?? fs.writeSync,
);
fs.fsyncSync(stageDescriptor);
const completed = validateOutputDescriptor(stageDescriptor, uid, [1n]);
if (
Number(completed.size) !== bytes ||
hashDescriptor(stageDescriptor, bytes, buffer) !== sha256 ||
!sameStat(opened, fs.fstatSync(sourceDescriptor, { bigint: true }))
) {
configurationError('capture stage or source drifted');
}
fs.closeSync(stageDescriptor);
stageDescriptor = undefined;
try {
fs.linkSync(stagePath, targetPath);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
}
syncPublishedDirectory(assetsDirectory);
const linked = fs.lstatSync(stagePath, { bigint: true });
const target = fs.lstatSync(targetPath, { bigint: true });
if (
linked.nlink !== 2n ||
linked.dev !== target.dev ||
linked.ino !== target.ino
) {
configurationError('capture asset publication identity drifted');
}
unlink(stagePath);
syncPublishedDirectory(assetsDirectory);
verifyPublishedAsset(targetPath, uid, bytes, sha256, buffer);
return Object.freeze({
manifest,
sourceSnapshot: sourceSnapshot(asset.sourcePath, opened),
});
} catch (error) {
if (stageDescriptor !== undefined) {
fs.closeSync(stageDescriptor);
stageDescriptor = undefined;
}
if (createdStage) {
try {
unlink(stagePath);
syncPublishedDirectory(assetsDirectory);
} catch {
// A cleanup failure leaves an exact-prefix stage for bounded replay.
}
}
if (error instanceof LocalDeploymentConfigurationError) throw error;
return configurationError('capture asset cannot be published', error);
} finally {
buffer.fill(0);
if (stageDescriptor !== undefined) fs.closeSync(stageDescriptor);
if (sourceDescriptor !== undefined) fs.closeSync(sourceDescriptor);
}
}
export function verifyLocalReconciliationSourceSnapshot(
snapshot: LocalReconciliationStableCopyResult['sourceSnapshot'],
): void {
try {
const stat = fs.lstatSync(snapshot.path, { bigint: true });
if (
!stat.isFile() ||
stat.isSymbolicLink() ||
stat.dev.toString() !== snapshot.device ||
stat.ino.toString() !== snapshot.inode ||
Number(stat.uid) !== snapshot.uid ||
Number(stat.gid) !== snapshot.gid ||
Number(stat.mode) !== snapshot.mode ||
Number(stat.nlink) !== snapshot.links ||
Number(stat.size) !== snapshot.bytes ||
stat.mtimeNs.toString() !== snapshot.modifiedAtNs ||
stat.ctimeNs.toString() !== snapshot.changedAtNs ||
fs.realpathSync(snapshot.path) !== snapshot.path
) {
configurationError('capture source changed before bundle publication');
}
} catch (error) {
if (error instanceof LocalDeploymentConfigurationError) throw error;
configurationError('capture source cannot be reverified', error);
}
}
export function verifyLocalReconciliationSidecarPlan(
intent: Readonly<LocalReconciliationCaptureIntent>,
targetSidecars: readonly boolean[],
legacySidecars: readonly boolean[],
): void {
const currentTarget = localReconciliationSidecarSnapshot(
intent.command.request.targetDatabasePath,
);
const currentLegacy = localReconciliationSidecarSnapshot(
intent.command.request.legacySourcePath,
);
if (
targetSidecars.some((value, index) => value !== currentTarget[index]) ||
legacySidecars.some((value, index) => value !== currentLegacy[index])
) {
configurationError('SQLite sidecar set changed during capture');
}
}
export function verifyLocalReconciliationPublishedAsset(
asset: Readonly<LocalReconciliationCapturedAsset>,
assetsDirectory: string,
uid: number,
): void {
const buffer = Buffer.allocUnsafe(COPY_BUFFER_BYTES);
try {
verifyPublishedAsset(
path.join(assetsDirectory, asset.logicalName),
uid,
asset.bytes,
asset.sha256,
buffer,
);
} finally {
buffer.fill(0);
}
}
@@ -746,14 +746,18 @@ function replayResult(
intent.instanceId,
currentIdentity().uid,
);
const stoppedCaptureProgress =
record.state === 'target_stopped' &&
(head.state === 'reconciliation_capture_prepared' ||
head.state === 'reconciliation_captured');
if (
record.actionId !== intent.actionId ||
record.intentDigest !== intent.intentDigest ||
record.evidence.managerOutcomeDigest !== outcome.outcomeDigest ||
head.cutoverId !== intent.lineage.cutoverId ||
head.generation !== intent.lineage.generation ||
head.state !== record.state ||
head.sourceRecordDigest !== record.recordDigest
(!stoppedCaptureProgress && head.state !== record.state) ||
(!stoppedCaptureProgress && head.sourceRecordDigest !== record.recordDigest)
) {
configurationError('service manager cutover replay drifted');
}
@@ -7,8 +7,13 @@ const { spawnSync } = require('node:child_process');
const { test } = require('node:test');
const {
commitLocalReconciliationCapture,
prepareLocalReconciliationCapture,
verifyLocalReconciliationCapture,
} = require('../dist/deployment/localDeployment.js');
const {
createLocalDataDirectoryApplicationCommit,
} = require('@qinglong/local-sqlite/data-directory-application-commit');
const {
advanceLocalCutoverInstanceHead,
claimLocalCutoverInstance,
@@ -39,7 +44,11 @@ function rootAcknowledgement() {
function fixture(
t,
{ reconciliationRequired = true, stoppedAuthority = 'docker' } = {},
{
reconciliationRequired = true,
stoppedAuthority = 'docker',
mutateTarget,
} = {},
) {
const root = fs.realpathSync(
fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-reconciliation-capture-')),
@@ -65,10 +74,25 @@ function fixture(
const legacySourcePath = path.join(root, 'database.sqlite');
const targetDatabasePath = path.join(root, 'database.ql3.sqlite');
const recoveryPath = path.join(root, 'database.recovery.sqlite');
const manifestPath = path.join(root, 'adoption-manifest.json');
const activationPath = path.join(root, 'activation.json');
const applicationConfigPath = path.join(
deploymentRoot,
'local-application.json',
);
fs.writeFileSync(legacySourcePath, 'legacy-source\n', { mode: 0o600 });
fs.writeFileSync(targetDatabasePath, 'target-initial\n', { mode: 0o600 });
fs.writeFileSync(recoveryPath, 'legacy-source\n', { mode: 0o600 });
const manifestPayload = {
schemaVersion: 1,
kind: 'qinglong3-local-sqlite-adoption-manifest-fixture',
};
const manifestDigest = digest(manifestPayload);
fs.writeFileSync(
manifestPath,
`${JSON.stringify({ ...manifestPayload, manifestDigest })}\n`,
{ mode: 0o600 },
);
const targetStat = fs.statSync(targetDatabasePath, { bigint: true });
const activationPayload = {
schemaVersion: 1,
@@ -76,7 +100,7 @@ function fixture(
state: 'prepared',
profile: 'edge',
createdAtMs: 1_000,
adoptionManifestDigest: '1'.repeat(64),
adoptionManifestDigest: manifestDigest,
planDigest: '2'.repeat(64),
sourcePathDigest: crypto
.createHash('sha256')
@@ -107,9 +131,150 @@ function fixture(
`${JSON.stringify({ ...activationPayload, activationDigest })}\n`,
{ mode: 0o600 },
);
let mutationEvidence = Object.freeze({});
if (reconciliationRequired) {
fs.writeFileSync(targetDatabasePath, 'target-mutated\n', { mode: 0o600 });
mutationEvidence =
mutateTarget?.({ root, targetDatabasePath }) ?? Object.freeze({});
if (mutateTarget === undefined) {
fs.writeFileSync(targetDatabasePath, 'target-mutated\n', {
mode: 0o600,
});
}
fs.writeFileSync(`${targetDatabasePath}-wal`, 'target-wal-facts\n', {
mode: 0o600,
});
fs.writeFileSync(`${legacySourcePath}-journal`, 'legacy-journal-state\n', {
mode: 0o600,
});
}
const commitmentPayload = {
schemaVersion: 1,
kind: 'qinglong3-local-legacy-silence-commitment',
state: 'legacy_stopped',
cutoverId,
profile: 'edge',
instanceId: 'edge-router-1',
activationDigest,
requestedAtMs: 1_100,
observedAtMs: 1_200,
previousRecordDigest: '1'.repeat(64),
controller: {
kind: 'docker',
endpointDigest: '2'.repeat(64),
legacyContainerId: '3'.repeat(64),
legacyContainerIdentityDigest: '4'.repeat(64),
legacySourceBindingDigest: '5'.repeat(64),
},
};
const commitmentDigest = digest(commitmentPayload);
const commitmentPath = path.join(journal, '0002-legacy-stopped.json');
fs.writeFileSync(
commitmentPath,
`${JSON.stringify({ ...commitmentPayload, commitmentDigest })}\n`,
{ mode: 0o600 },
);
const dataCommit = createLocalDataDirectoryApplicationCommit({
mutationId: '00000000-0000-4000-8000-000000000301',
projectId: 'project-edge-router-1',
profile: 'edge',
sourceStageManifestDigest: '6'.repeat(64),
transformationDigest: '7'.repeat(64),
modelDigest: '8'.repeat(64),
publicationDigest: '9'.repeat(64),
receiptDigest: 'a'.repeat(64),
committedAtMs: 1_300,
receipt: {
secretCount: 0,
environmentSecretCount: 0,
sshSecretCount: 0,
},
});
const dataCommitPath = path.join(root, 'legacy-data-commit.json');
fs.writeFileSync(dataCommitPath, `${JSON.stringify(dataCommit)}\n`, {
mode: 0o600,
});
const application = {
schema: 'qinglong/local-application-process@v4',
instanceId: 'edge-router-1',
profile: 'edge',
storage: {
mode: 'adopted',
sourcePath: legacySourcePath,
targetPath: targetDatabasePath,
recoveryPath,
manifestPath,
activationPath,
expectedActivationDigest: activationDigest,
},
runtime: {
receiptRoot: path.join(deploymentRoot, 'receipts'),
artifactRoot: path.join(deploymentRoot, 'artifacts'),
secretKeyringPath: path.join(deploymentRoot, 'local-secret-keyring.json'),
},
pluginPackages: {
stagingRoot: path.join(deploymentRoot, 'plugin-staging'),
activationRoot: path.join(deploymentRoot, 'plugin-activation'),
recoverySource: { mode: 'disabled' },
pageSize: 4,
maxPages: 4,
taskPublicationPageSize: 4,
taskPublicationMaxPages: 4,
},
ai: { deployment: 'excluded' },
cutover: {
cutoverId,
commitmentPath,
expectedCommitmentDigest: commitmentDigest,
},
legacyDataApplication: {
commitPath: dataCommitPath,
expectedCommitDigest: dataCommit.commitDigest,
expectedReceiptDigest: dataCommit.receiptDigest,
},
};
const applicationContents = `${JSON.stringify(application, null, 2)}\n`;
fs.writeFileSync(applicationConfigPath, applicationContents, { mode: 0o600 });
const adoptedBundlePayload = {
schemaVersion: 1,
kind: 'qinglong3-local-adopted-deployment-bundle',
state: 'prepared',
bundleId: '00000000-0000-4000-8000-000000000d88',
preparedAtMs: 1_400,
profile: 'edge',
instanceId: 'edge-router-1',
cutoverId,
serviceKind: 'compose',
deploymentRootDigest: crypto
.createHash('sha256')
.update(deploymentRoot, 'utf8')
.digest('hex'),
sourcePathDigest: crypto
.createHash('sha256')
.update(legacySourcePath, 'utf8')
.digest('hex'),
applicationConfigDigest: crypto
.createHash('sha256')
.update(applicationContents, 'utf8')
.digest('hex'),
serviceDescriptorDigest: 'b'.repeat(64),
composeSelectionDigest: 'c'.repeat(64),
activationDigest,
commitmentDigest,
legacyDataApplicationCommitDigest: dataCommit.commitDigest,
legacyDataApplicationReceiptDigest: dataCommit.receiptDigest,
manifestDigest,
sourceSha256: activationPayload.sourceSha256,
recoverySha256: activationPayload.recoverySha256,
targetIdentityDigest: 'd'.repeat(64),
};
fs.writeFileSync(
path.join(serviceRoot, 'adopted-bundle.json'),
`${JSON.stringify({
...adoptedBundlePayload,
bundleDigest: digest(adoptedBundlePayload),
})}\n`,
{ mode: 0o600 },
);
const identity = {
options: { deploymentRoot },
request: {
@@ -166,7 +331,8 @@ function fixture(
targetStoppedEvidence(
{
activeRecordDigest: '5'.repeat(64),
targetContainerIdentityDigest: '7'.repeat(64),
targetContainerIdentityDigest:
mutationEvidence.targetContainerIdentityDigest ?? '7'.repeat(64),
targetApplicationBindingDigest: '8'.repeat(64),
startupReceiptDigest: '9'.repeat(64),
},
@@ -237,6 +403,7 @@ function fixture(
instanceId: 'edge-router-1',
cutoverId,
generation: 1,
applicationConfigPath,
activationPath,
legacySourcePath,
targetDatabasePath,
@@ -247,7 +414,17 @@ function fixture(
preparedAtMs: 4_000,
},
};
return { command, deploymentRoot, captureRoot, identity, stoppedHead, uid };
return {
command,
deploymentRoot,
captureRoot,
identity,
stoppedHead,
uid,
legacySourcePath,
targetDatabasePath,
recoveryPath,
};
}
test('capture prepare establishes one replayable reconciliation fence', (t) => {
@@ -351,6 +528,422 @@ test('capture prepare CLI consumes a private command and emits no paths', (t) =>
assert.equal(output.state, 'reconciliation_capture_prepared');
assert.equal(output.captureId, state.command.request.captureId);
assert.equal(result.stdout.includes(state.captureRoot), false);
assert.equal(result.stdout.includes(state.command.request.targetDatabasePath), false);
assert.equal(
result.stdout.includes(state.command.request.targetDatabasePath),
false,
);
assert.equal(result.stderr, '');
});
function preparedCapture(t, options) {
const state = fixture(t, options);
const prepared = prepareLocalReconciliationCapture(state.command);
const commitCommand = {
schemaVersion: 1,
operation: 'local.deployment.reconciliation.capture.commit',
options: state.command.options,
request: {
captureId: state.command.request.captureId,
expectedPreparationDigest: prepared.preparationDigest,
committedAtMs: 5_000,
},
};
return { ...state, prepared, commitCommand };
}
function capturePath(state, name) {
return path.join(state.captureRoot, state.command.request.captureId, name);
}
test('commit captures main, sidecars and recovery then verifies without sources', (t) => {
const state = preparedCapture(t);
fs.writeFileSync(`${state.targetDatabasePath}.unrelated`, 'ignored\n', {
mode: 0o600,
});
const committed = commitLocalReconciliationCapture(state.commitCommand);
assert.equal(committed.status, 'prepared');
assert.equal(committed.state, 'reconciliation_captured');
assert.equal(committed.assetCount, 5);
const manifest = JSON.parse(
fs.readFileSync(capturePath(state, 'manifest.json'), 'utf8'),
);
assert.deepEqual(
manifest.assets.map((asset) => asset.logicalName),
[
'target-main',
'target-wal',
'legacy-main',
'legacy-journal',
'recovery-main',
],
);
const manifestText = fs.readFileSync(
capturePath(state, 'manifest.json'),
'utf8',
);
assert.equal(manifestText.includes(state.captureRoot), false);
assert.equal(manifestText.includes(state.targetDatabasePath), false);
assert.equal(manifestText.includes(state.legacySourcePath), false);
assert.equal(
fs.readFileSync(capturePath(state, 'assets/target-main'), 'utf8'),
'target-mutated\n',
);
assert.equal(
fs.readFileSync(capturePath(state, 'assets/target-wal'), 'utf8'),
'target-wal-facts\n',
);
const head = readLocalCutoverInstanceHead(
state.deploymentRoot,
state.command.request.instanceId,
state.uid,
);
assert.equal(head.state, 'reconciliation_captured');
assert.equal(head.sourceRecordDigest, committed.bundleDigest);
const verifyCommand = {
schemaVersion: 1,
operation: 'local.deployment.reconciliation.capture.verify',
options: state.command.options,
request: {
captureId: state.command.request.captureId,
expectedBundleDigest: committed.bundleDigest,
},
};
assert.equal(
verifyLocalReconciliationCapture(verifyCommand).status,
'verified',
);
fs.unlinkSync(state.targetDatabasePath);
fs.unlinkSync(state.legacySourcePath);
fs.unlinkSync(state.recoveryPath);
assert.equal(
verifyLocalReconciliationCapture(verifyCommand).bundleDigest,
committed.bundleDigest,
);
assert.equal(
commitLocalReconciliationCapture(state.commitCommand).status,
'existing',
);
const driftedCommit = structuredClone(state.commitCommand);
driftedCommit.request.committedAtMs += 1;
assert.throws(
() => commitLocalReconciliationCapture(driftedCommit),
/instance head binding/,
);
});
test('commit resumes after an asset publication crash without replacement', (t) => {
const state = preparedCapture(t);
let failed = false;
assert.throws(
() =>
commitLocalReconciliationCapture(state.commitCommand, {
afterAssetPublished(logicalName) {
if (!failed && logicalName === 'target-main') {
failed = true;
throw new Error('asset crash');
}
},
}),
/asset crash/,
);
const targetAsset = capturePath(state, 'assets/target-main');
const before = fs.statSync(targetAsset, { bigint: true });
assert.equal(fs.existsSync(capturePath(state, 'manifest.json')), false);
const committed = commitLocalReconciliationCapture(state.commitCommand);
const after = fs.statSync(targetAsset, { bigint: true });
assert.equal(after.ino, before.ino);
assert.equal(committed.status, 'prepared');
});
test('commit resumes after manifest and receipt crash windows', (t) => {
const manifestState = preparedCapture(t);
assert.throws(
() =>
commitLocalReconciliationCapture(manifestState.commitCommand, {
afterManifestPublished() {
throw new Error('manifest crash');
},
}),
/manifest crash/,
);
assert.equal(
fs.existsSync(capturePath(manifestState, 'manifest.json')),
true,
);
assert.equal(
fs.existsSync(capturePath(manifestState, 'receipt.json')),
false,
);
fs.unlinkSync(manifestState.targetDatabasePath);
fs.unlinkSync(manifestState.legacySourcePath);
fs.unlinkSync(manifestState.recoveryPath);
assert.equal(
commitLocalReconciliationCapture(manifestState.commitCommand).state,
'reconciliation_captured',
);
const receiptState = preparedCapture(t, {
stoppedAuthority: 'service-manager',
});
assert.throws(
() =>
commitLocalReconciliationCapture(receiptState.commitCommand, {
afterReceiptPublished() {
throw new Error('receipt crash');
},
}),
/receipt crash/,
);
const preparedHead = readLocalCutoverInstanceHead(
receiptState.deploymentRoot,
receiptState.command.request.instanceId,
receiptState.uid,
);
assert.equal(preparedHead.state, 'reconciliation_capture_prepared');
fs.unlinkSync(receiptState.targetDatabasePath);
fs.unlinkSync(receiptState.legacySourcePath);
fs.unlinkSync(receiptState.recoveryPath);
const resumed = commitLocalReconciliationCapture(receiptState.commitCommand);
assert.equal(resumed.status, 'prepared');
assert.equal(resumed.state, 'reconciliation_captured');
});
test('hard-link publication replay removes only the exact retained stage', (t) => {
const state = preparedCapture(t);
const cleanupError = Object.assign(new Error('stage cleanup unavailable'), {
code: 'EIO',
});
assert.throws(
() =>
commitLocalReconciliationCapture(state.commitCommand, {
stableCopy: {
unlink() {
throw cleanupError;
},
},
}),
/capture asset cannot be published/,
);
const target = capturePath(state, 'assets/target-main');
const stage = capturePath(state, 'assets/.target-main.ql3-capture-stage');
const targetBefore = fs.statSync(target, { bigint: true });
const stageBefore = fs.statSync(stage, { bigint: true });
assert.equal(targetBefore.ino, stageBefore.ino);
assert.equal(targetBefore.nlink, 2n);
const committed = commitLocalReconciliationCapture(state.commitCommand);
assert.equal(committed.state, 'reconciliation_captured');
assert.equal(fs.existsSync(stage), false);
assert.equal(fs.statSync(target, { bigint: true }).ino, targetBefore.ino);
});
test('ENOSPC cleans a new stage and exact replay completes', (t) => {
const state = preparedCapture(t);
const noSpace = Object.assign(new Error('no space'), { code: 'ENOSPC' });
assert.throws(
() =>
commitLocalReconciliationCapture(state.commitCommand, {
stableCopy: {
write() {
throw noSpace;
},
},
}),
/capture asset cannot be published/,
);
const assetsRoot = capturePath(state, 'assets');
assert.deepEqual(fs.readdirSync(assetsRoot), []);
const head = readLocalCutoverInstanceHead(
state.deploymentRoot,
state.command.request.instanceId,
state.uid,
);
assert.equal(head.state, 'reconciliation_capture_prepared');
assert.equal(
commitLocalReconciliationCapture(state.commitCommand).state,
'reconciliation_captured',
);
});
test('a cleanup-resistant partial stage resumes only from its exact prefix', (t) => {
const state = preparedCapture(t);
let writes = 0;
assert.throws(
() =>
commitLocalReconciliationCapture(state.commitCommand, {
stableCopy: {
write(descriptor, buffer, offset, length, position) {
writes += 1;
if (writes === 1) {
return fs.writeSync(
descriptor,
buffer,
offset,
Math.min(4, length),
position,
);
}
throw Object.assign(new Error('no space'), { code: 'ENOSPC' });
},
unlink() {
throw Object.assign(new Error('cleanup unavailable'), {
code: 'EIO',
});
},
},
}),
/capture asset cannot be published/,
);
const stage = capturePath(state, 'assets/.target-main.ql3-capture-stage');
assert.equal(fs.statSync(stage).size, 4);
assert.equal(
fs
.readFileSync(stage)
.equals(fs.readFileSync(state.targetDatabasePath).subarray(0, 4)),
true,
);
assert.equal(
commitLocalReconciliationCapture(state.commitCommand).state,
'reconciliation_captured',
);
assert.equal(fs.existsSync(stage), false);
});
test('sidecar set drift prevents manifest publication and remains replayable', (t) => {
const state = preparedCapture(t);
let changed = false;
const unexpectedSidecar = `${state.targetDatabasePath}-shm`;
assert.throws(
() =>
commitLocalReconciliationCapture(state.commitCommand, {
afterAssetPublished() {
if (!changed) {
changed = true;
fs.writeFileSync(unexpectedSidecar, 'late-sidecar\n', {
mode: 0o600,
});
}
},
}),
/sidecar set changed/,
);
assert.equal(fs.existsSync(capturePath(state, 'manifest.json')), false);
fs.unlinkSync(unexpectedSidecar);
assert.equal(
commitLocalReconciliationCapture(state.commitCommand).state,
'reconciliation_captured',
);
});
test('terminal verify rejects asset drift and CLI output remains content-free', (t) => {
const state = preparedCapture(t);
const committed = commitLocalReconciliationCapture(state.commitCommand);
const verifyCommand = {
schemaVersion: 1,
operation: 'local.deployment.reconciliation.capture.verify',
options: state.command.options,
request: {
captureId: state.command.request.captureId,
expectedBundleDigest: committed.bundleDigest,
},
};
const verifyPath = path.join(state.deploymentRoot, 'verify-command.json');
fs.writeFileSync(verifyPath, `${JSON.stringify(verifyCommand)}\n`, {
mode: 0o600,
});
const result = spawnSync(
process.execPath,
[
path.join(__dirname, '../dist/deployment/localDeploymentCli.js'),
'reconciliation-capture-verify',
'--command-file',
verifyPath,
],
{ encoding: 'utf8' },
);
assert.equal(result.status, 0, result.stderr);
assert.equal(result.stdout.includes(state.captureRoot), false);
assert.equal(result.stdout.includes(state.targetDatabasePath), false);
fs.writeFileSync(capturePath(state, 'assets/target-main'), 'drift\n');
assert.throws(
() => verifyLocalReconciliationCapture(verifyCommand),
/asset drifted/,
);
});
test(
'real stopped Docker target produces an independently verified bundle',
{ skip: process.env.QL3_RECONCILIATION_DOCKER_GATE !== '1' },
(t) => {
const containerName = `ql3-reconciliation-${process.pid}-${Date.now()}`;
t.after(() => {
spawnSync('docker', ['rm', '--force', containerName], {
encoding: 'utf8',
});
});
const state = preparedCapture(t, {
mutateTarget({ root }) {
const created = spawnSync(
'docker',
[
'create',
'--name',
containerName,
'--mount',
`type=bind,source=${root},target=/capture-fixture`,
'node:24-bookworm-slim',
'node',
'-e',
"require('node:fs').writeFileSync('/capture-fixture/database.ql3.sqlite','target-docker-mutated\\n')",
],
{ encoding: 'utf8' },
);
assert.equal(created.status, 0, created.stderr);
const containerId = created.stdout.trim();
assert.match(containerId, /^[0-9a-f]{64}$/);
const started = spawnSync(
'docker',
['start', '--attach', containerName],
{
encoding: 'utf8',
},
);
assert.equal(started.status, 0, started.stderr);
const inspected = spawnSync(
'docker',
[
'inspect',
'--format',
'{{.State.Running}} {{.State.Status}}',
containerName,
],
{ encoding: 'utf8' },
);
assert.equal(inspected.status, 0, inspected.stderr);
assert.equal(inspected.stdout.trim(), 'false exited');
return Object.freeze({
targetContainerIdentityDigest: crypto
.createHash('sha256')
.update(containerId, 'utf8')
.digest('hex'),
});
},
});
const committed = commitLocalReconciliationCapture(state.commitCommand);
assert.equal(
fs.readFileSync(capturePath(state, 'assets/target-main'), 'utf8'),
'target-docker-mutated\n',
);
const verified = verifyLocalReconciliationCapture({
schemaVersion: 1,
operation: 'local.deployment.reconciliation.capture.verify',
options: state.command.options,
request: {
captureId: state.command.request.captureId,
expectedBundleDigest: committed.bundleDigest,
},
});
assert.equal(verified.status, 'verified');
assert.equal(verified.bundleDigest, committed.bundleDigest);
},
);
@@ -704,6 +704,40 @@ test('stop advances only after the exact receipted process identity disappears',
),
);
assert.equal(record.evidence.shutdownReceiptDigest, shutdownReceiptDigest);
const lineageIdentity = {
options: { deploymentRoot: state.root },
request: {
cutoverId: state.cutoverId,
profile: 'edge',
instanceId: 'edge-router-1',
expectedActivationDigest: state.activationDigest,
requestedAtMs: 1786416000400,
},
};
advanceLocalCutoverInstanceHead(
lineageIdentity,
process.getuid(),
'reconciliation_capture_prepared',
1,
'e'.repeat(64),
);
const preparedReplay = await consumeLocalServiceManagerCutoverOutcome(
consumeCommand(state, stopped),
{ procRoot: state.procRoot },
);
assert.equal(preparedReplay.status, 'existing');
advanceLocalCutoverInstanceHead(
lineageIdentity,
process.getuid(),
'reconciliation_captured',
1,
'f'.repeat(64),
);
const capturedReplay = await consumeLocalServiceManagerCutoverOutcome(
consumeCommand(state, stopped),
{ procRoot: state.procRoot },
);
assert.equal(capturedReplay.status, 'existing');
});
test('prepares and exactly replays lossless service-manager legacy rollback evidence', async (t) => {