open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-20 16:07:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): preserve adopted compose lineage

Browse Source
This commit is contained in:
whyour
2026-08-21 15:18:07 +08:00
parent 190117a662
commit 4f6be5bc2c
18 changed files with 2192 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-local-owner-cli/src/deployment/adopted-bundle/adoptedBundle.ts
+53 -3
View File
@@ -237,7 +237,32 @@ export function prepareLocalDeploymentAdoptedBundle(
[material.paths.pluginActivation, 'pluginActivationRoot'],
...(material.composeSelection === null
? []
: ([[material.paths.composeRevisions, 'composeRevisionRoot']] as const)),
: ([
[material.paths.composeRevisions, 'composeRevisionRoot'],
[material.paths.composeRollouts, 'composeRolloutRoot'],
[material.paths.composeRolloutBackups, 'composeRolloutBackupRoot'],
[material.paths.composeRestores, 'composeRestoreRoot'],
[
material.paths.composeRestoreSafeguards,
'composeRestoreSafeguardRoot',
],
[
material.paths.composeEvidenceCollections,
'composeEvidenceCollectionRoot',
],
[
material.paths.composeCollectedEvidence,
'composeCollectedEvidenceRoot',
],
[
material.paths.composeCollectedRolloutBackups,
'composeCollectedRolloutBackupRoot',
],
[
material.paths.composeCollectedRestoreSafeguards,
'composeCollectedRestoreSafeguardRoot',
],
] as const)),
] as const;
const directoryStatuses = directoryPaths.map(([directory, label]) =>
ensurePrivateDirectory(directory, identity.uid, label),
@@ -352,7 +377,32 @@ export function verifyLocalDeploymentAdoptedBundle(
[material.paths.pluginActivation, 'pluginActivationRoot'],
...(material.composeSelection === null
? []
: ([[material.paths.composeRevisions, 'composeRevisionRoot']] as const)),
: ([
[material.paths.composeRevisions, 'composeRevisionRoot'],
[material.paths.composeRollouts, 'composeRolloutRoot'],
[material.paths.composeRolloutBackups, 'composeRolloutBackupRoot'],
[material.paths.composeRestores, 'composeRestoreRoot'],
[
material.paths.composeRestoreSafeguards,
'composeRestoreSafeguardRoot',
],
[
material.paths.composeEvidenceCollections,
'composeEvidenceCollectionRoot',
],
[
material.paths.composeCollectedEvidence,
'composeCollectedEvidenceRoot',
],
[
material.paths.composeCollectedRolloutBackups,
'composeCollectedRolloutBackupRoot',
],
[
material.paths.composeCollectedRestoreSafeguards,
'composeCollectedRestoreSafeguardRoot',
],
] as const)),
] as const) {
validatePrivateDirectory(directory, identity.uid, label);
}
@@ -420,7 +470,7 @@ export function verifyLocalDeploymentAdoptedBundle(
}),
directories: Object.freeze({
created: 0,
existing: material.composeSelection === null ? 4 : 5,
existing: material.composeSelection === null ? 4 : 13,
}),
});
}
packages/ql3-local-owner-cli/src/deployment/adopted-bundle/material.ts
+24
View File
@@ -34,6 +34,14 @@ export interface LocalDeploymentAdoptedBundlePaths {
readonly composeSelection: string;
readonly composeRevisions: string;
readonly composeRevision: string;
readonly composeRollouts: string;
readonly composeRolloutBackups: string;
readonly composeRestores: string;
readonly composeRestoreSafeguards: string;
readonly composeEvidenceCollections: string;
readonly composeCollectedEvidence: string;
readonly composeCollectedRolloutBackups: string;
readonly composeCollectedRestoreSafeguards: string;
}
export interface LocalDeploymentAdoptedEvidence {
@@ -316,6 +324,22 @@ export function adoptedBundlePaths(
composeSelection: path.join(service, 'compose.image.yaml'),
composeRevisions: path.join(service, 'revisions'),
composeRevision: path.join(service, 'revisions', '1.yaml'),
composeRollouts: path.join(service, 'rollouts'),
composeRolloutBackups: path.join(service, 'rollout-backups'),
composeRestores: path.join(service, 'restores'),
composeRestoreSafeguards: path.join(service, 'restore-safeguards'),
composeEvidenceCollections: path.join(service, 'evidence-collections'),
composeCollectedEvidence: path.join(service, 'collected-evidence'),
composeCollectedRolloutBackups: path.join(
service,
'collected-evidence',
'rollout-backups',
),
composeCollectedRestoreSafeguards: path.join(
service,
'collected-evidence',
'restore-safeguards',
),
});
}
packages/ql3-local-owner-cli/src/deployment/compose/composeApply.ts
+89 -21
View File
@@ -12,7 +12,10 @@ import {
type LocalSqliteRolloutBackupOptions,
} from '@qinglong/local-sqlite/rollout-safety';
import { preflightLocalDeploymentCompose } from './composePreflight';
import {
preflightLocalDeploymentCompose,
type LocalDeploymentComposePreflightDependencies,
} from './composePreflight';
import { evidenceDigest, inspectCollectedEvidence } from './composeEvidence';
import {
inspectActiveComposeImageSelection,
@@ -43,8 +46,15 @@ import {
deploymentPaths,
type LocalDeploymentPaths,
} from '../foundation/render';
import {
assertLocalDeploymentComposeLineageReceipt,
inspectLocalDeploymentComposeLineage,
normalizeLocalDeploymentComposeLineageReceipt,
type LocalDeploymentComposeLineage,
type LocalDeploymentComposeLineageReceipt,
} from './composeLineage';
const RECEIPT_SCHEMA = 'qinglong/local-compose-rollout-receipt@v2';
const RECEIPT_SCHEMA = 'qinglong/local-compose-rollout-receipt@v3';
const CONTAINER_ID_PATTERN = /^[0-9a-f]{12,64}$/;
const UUID_V4_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
@@ -59,6 +69,7 @@ export interface LocalDeploymentComposeApplyDependencies {
readonly createBackup?: typeof createLocalSqliteRolloutBackup;
readonly inspectBackup?: typeof inspectLocalSqliteRolloutBackup;
readonly openChangeObserver?: typeof openLocalSqliteChangeObserver;
readonly auditSqlite?: LocalDeploymentComposePreflightDependencies['auditSqlite'];
}
interface ActiveEvidence {
@@ -88,6 +99,7 @@ interface RolloutReceipt {
readonly attemptedGeneration: number;
readonly recordedAtMs: number;
readonly healthEventDigest: string | null;
readonly lineage: Readonly<LocalDeploymentComposeLineageReceipt>;
readonly sqlite: Readonly<RolloutSqliteReceipt>;
readonly result: Readonly<LocalDeploymentComposeApplyResult>;
}
@@ -181,6 +193,7 @@ function inspectRunningContainer(
output: string,
containerId: string,
selection: Readonly<ComposeImageSelection>,
lineage: Readonly<LocalDeploymentComposeLineage>,
): boolean {
let value: unknown;
try {
@@ -207,6 +220,19 @@ function inspectRunningContainer(
};
}
| undefined;
const labels = candidate?.Config?.Labels;
const lineageLabelsValid =
lineage.mode === 'fresh' ||
(labels?.['io.qinglong.deployment.mode'] === 'adopted' &&
labels['io.qinglong.deployment.profile'] === lineage.profile &&
labels['io.qinglong.deployment.instance'] === lineage.instanceId &&
labels['io.qinglong.deployment.bundle'] === lineage.bundleId &&
labels['io.qinglong.application.config'] ===
lineage.receipt.applicationConfigDigest &&
labels['io.qinglong.data.commit'] ===
lineage.receipt.legacyDataApplicationCommitDigest &&
labels['io.qinglong.data.receipt'] ===
lineage.receipt.legacyDataApplicationReceiptDigest);
return (
(candidate?.Id === containerId ||
(typeof candidate?.Id === 'string' &&
@@ -214,10 +240,10 @@ function inspectRunningContainer(
candidate?.State?.Running === true &&
candidate.State.Status === 'running' &&
candidate.Config?.Image === selection.image &&
candidate.Config.Labels?.['io.qinglong.deployment.generation'] ===
labels?.['io.qinglong.deployment.generation'] ===
String(selection.generation) &&
candidate.Config.Labels?.['io.qinglong.deployment.mutation'] ===
selection.mutationId &&
labels?.['io.qinglong.deployment.mutation'] === selection.mutationId &&
lineageLabelsValid &&
candidate.HostConfig?.ReadonlyRootfs === true &&
candidate.HostConfig.NetworkMode === 'none' &&
candidate.HostConfig.Privileged !== true
@@ -260,6 +286,7 @@ async function observeActive(
paths: Readonly<LocalDeploymentPaths>,
selection: Readonly<ComposeImageSelection>,
profile: LocalDeploymentProfile,
lineage: Readonly<LocalDeploymentComposeLineage>,
dependencies: Required<
Pick<LocalDeploymentComposeApplyDependencies, 'runDocker' | 'now' | 'wait'>
>,
@@ -285,6 +312,7 @@ async function observeActive(
]),
containerId,
selection,
lineage,
);
if (running) {
const evidence = activeEventEvidence(
@@ -321,9 +349,10 @@ function backupOptions(
command: Readonly<LocalDeploymentComposeApplyCommand>,
paths: Readonly<LocalDeploymentPaths>,
profile: LocalDeploymentProfile,
databasePath: string,
): Readonly<LocalSqliteRolloutBackupOptions> {
return Object.freeze({
databasePath: paths.database,
databasePath,
backupPath: backupPathFor(paths, command.request.rolloutId),
profile,
});
@@ -389,6 +418,7 @@ async function applyAndObserveCandidate(
paths: Readonly<LocalDeploymentPaths>,
selection: Readonly<ComposeImageSelection>,
profile: LocalDeploymentProfile,
lineage: Readonly<LocalDeploymentComposeLineage>,
dependencies: Required<
Pick<
LocalDeploymentComposeApplyDependencies,
@@ -402,23 +432,26 @@ async function applyAndObserveCandidate(
}>
> {
const observer = dependencies.openChangeObserver({
databasePath: paths.database,
databasePath: lineage.databasePath,
profile,
});
let evidence: Readonly<ActiveEvidence> | null = null;
try {
try {
applyActiveSelection(command, paths, dependencies.runDocker);
evidence = await observeActive(
command,
paths,
selection,
profile,
dependencies,
);
} catch {
evidence = null;
// Docker may lose the response after creating the exact candidate. The
// bounded observation below is the recovery authority; do not issue a
// second `compose up` in the same attempt.
}
evidence = await observeActive(
command,
paths,
selection,
profile,
lineage,
dependencies,
);
let writeObservation: SqliteWriteObservation;
try {
writeObservation = observer.changed() ? 'changed' : 'unchanged';
@@ -467,6 +500,7 @@ async function readReceipt(
expectedGeneration: number,
paths: Readonly<LocalDeploymentPaths>,
inspectBackup: typeof inspectLocalSqliteRolloutBackup,
expectedLineage: Readonly<LocalDeploymentComposeLineage>,
): Promise<Readonly<LocalDeploymentComposeApplyResult> | null> {
if (!fs.existsSync(filePath)) return null;
const stat = fs.lstatSync(filePath);
@@ -496,11 +530,15 @@ async function readReceipt(
const healthKeys = Object.keys(receipt.result?.health ?? {}).sort();
const serviceKeys = Object.keys(receipt.result?.service ?? {}).sort();
const sqliteKeys = Object.keys(receipt.sqlite ?? {}).sort();
const lineage = normalizeLocalDeploymentComposeLineageReceipt(
receipt.lineage,
);
const backupKeys = Object.keys(receipt.sqlite?.backup ?? {}).sort();
const expectedReceiptKeys = [
'attemptedGeneration',
'commandDigest',
'healthEventDigest',
'lineage',
'recordedAtMs',
'result',
'rolloutId',
@@ -546,6 +584,7 @@ async function readReceipt(
attemptedGeneration: receipt.attemptedGeneration,
recordedAtMs: receipt.recordedAtMs,
healthEventDigest: receipt.healthEventDigest,
lineage,
sqlite: {
contractVersion: receipt.sqlite?.contractVersion,
writeContractVersion: receipt.sqlite?.writeContractVersion,
@@ -636,11 +675,15 @@ async function readReceipt(
) {
configurationError('compose rollout receipt drifted');
}
assertLocalDeploymentComposeLineageReceipt(
receipt.lineage,
expectedLineage.receipt,
);
if (receipt.sqlite.backup !== null) {
const backupPath = backupPathFor(paths, rolloutId);
if (fs.existsSync(backupPath)) {
const inspected = await inspectBackup({
databasePath: paths.database,
databasePath: expectedLineage.databasePath,
backupPath,
profile: receipt.result.profile,
});
@@ -698,6 +741,7 @@ function publishReceipt(
rolloutResult: Readonly<LocalDeploymentComposeApplyResult>,
evidence: Readonly<ActiveEvidence> | null,
sqlite: Readonly<RolloutSqliteReceipt>,
lineage: Readonly<LocalDeploymentComposeLineage>,
uid: number,
): void {
publishExactFile(
@@ -709,6 +753,7 @@ function publishReceipt(
attemptedGeneration: command.request.expectedGeneration,
recordedAtMs: command.request.failureRollbackChangedAtMs,
healthEventDigest: evidence?.digest ?? null,
lineage: lineage.receipt,
sqlite,
result: rolloutResult,
}),
@@ -763,6 +808,12 @@ export async function applyLocalDeploymentCompose(
identity.uid,
'composeRolloutBackupRoot',
);
const lineage = inspectLocalDeploymentComposeLineage(
command.options.deploymentRoot,
identity.uid,
identity.gid,
command.options.allowRootService,
);
if (fs.existsSync(paths.composeEvidenceCollectionLock)) {
configurationError('compose rollout is fenced by an evidence collection');
}
@@ -785,6 +836,7 @@ export async function applyLocalDeploymentCompose(
dependencies.inspectBackup ?? inspectLocalSqliteRolloutBackup;
const openChangeObserver =
dependencies.openChangeObserver ?? openLocalSqliteChangeObserver;
const auditSqlite = dependencies.auditSqlite;
const receiptPath = path.join(
paths.composeRollouts,
`${command.request.rolloutId}.json`,
@@ -798,6 +850,7 @@ export async function applyLocalDeploymentCompose(
command.request.expectedGeneration,
paths,
inspectBackup,
lineage,
);
if (replay) {
releaseLock(paths.composeRolloutLock, intent, identity.uid);
@@ -851,11 +904,11 @@ export async function applyLocalDeploymentCompose(
expectedGeneration: command.request.expectedGeneration,
},
},
{ runDocker, validateSocket },
{ runDocker, validateSocket, auditSqlite },
);
if (selection.previousGeneration >= 1) {
rolloutBackup = await createBackup(
backupOptions(command, paths, preflight.profile),
backupOptions(command, paths, preflight.profile, lineage.databasePath),
);
}
const candidate = await applyAndObserveCandidate(
@@ -863,6 +916,7 @@ export async function applyLocalDeploymentCompose(
paths,
selection,
preflight.profile,
lineage,
{ runDocker, now, wait, openChangeObserver },
);
const evidence = candidate.evidence;
@@ -880,6 +934,7 @@ export async function applyLocalDeploymentCompose(
active,
evidence,
sqliteReceipt(rolloutBackup, writeObservation),
lineage,
identity.uid,
);
releaseLock(paths.composeRolloutLock, intent, identity.uid);
@@ -899,6 +954,7 @@ export async function applyLocalDeploymentCompose(
stopped,
null,
sqliteReceipt(null, writeObservation),
lineage,
identity.uid,
);
releaseLock(paths.composeRolloutLock, intent, identity.uid);
@@ -936,19 +992,30 @@ export async function applyLocalDeploymentCompose(
options: command.options,
request: { expectedGeneration: selection.generation },
},
{ runDocker, validateSocket },
{ runDocker, validateSocket, auditSqlite },
);
if (command.request.expectedGeneration > 1 && rolloutBackup === null) {
rolloutBackup = await inspectBackup(
backupOptions(command, paths, rollbackPreflight.profile),
backupOptions(
command,
paths,
rollbackPreflight.profile,
lineage.databasePath,
),
);
}
applyActiveSelection(rollbackCommand, paths, runDocker);
try {
applyActiveSelection(rollbackCommand, paths, runDocker);
} catch {
// As with the candidate, inspect the exact generation after a lost Docker
// response instead of blindly recreating the rollback container.
}
const rollbackEvidence = await observeActive(
rollbackCommand,
paths,
selection,
rollbackPreflight.profile,
lineage,
{ runDocker, now, wait },
);
if (!rollbackEvidence) {
@@ -968,6 +1035,7 @@ export async function applyLocalDeploymentCompose(
rolledBack,
rollbackEvidence,
sqliteReceipt(rolloutBackup, writeObservation),
lineage,
identity.uid,
);
releaseLock(paths.composeRolloutLock, intent, identity.uid);
packages/ql3-local-owner-cli/src/deployment/compose/composeEvidenceCollection.ts
+61 -36
View File
@@ -37,11 +37,18 @@ import {
deploymentPaths,
type LocalDeploymentPaths,
} from '../foundation/render';
import {
assertLocalDeploymentComposeLineageReceipt,
inspectLocalDeploymentComposeLineage,
normalizeLocalDeploymentComposeLineageReceipt,
type LocalDeploymentComposeLineage,
type LocalDeploymentComposeLineageReceipt,
} from './composeLineage';
const PREPARE_SCHEMA = 'qinglong/local-compose-evidence-collection-prepare@v1';
const COMMIT_SCHEMA = 'qinglong/local-compose-evidence-collection-commit@v1';
const ROLLOUT_RECEIPT_SCHEMA = 'qinglong/local-compose-rollout-receipt@v2';
const RESTORE_COMMIT_SCHEMA = 'qinglong/local-compose-restore-commit@v1';
const PREPARE_SCHEMA = 'qinglong/local-compose-evidence-collection-prepare@v2';
const COMMIT_SCHEMA = 'qinglong/local-compose-evidence-collection-commit@v2';
const ROLLOUT_RECEIPT_SCHEMA = 'qinglong/local-compose-rollout-receipt@v3';
const RESTORE_COMMIT_SCHEMA = 'qinglong/local-compose-restore-commit@v2';
const UUID_V4_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
const SHA256_PATTERN = /^[0-9a-f]{64}$/;
@@ -61,6 +68,7 @@ interface CollectionPrepareReceipt {
readonly collectionId: string;
readonly generation: number;
readonly profile: LocalDeploymentProfile;
readonly lineage: Readonly<LocalDeploymentComposeLineageReceipt>;
readonly recordedAtMs: number;
readonly items: readonly Readonly<CollectionItem>[];
}
@@ -72,6 +80,7 @@ interface CollectionCommitReceipt {
readonly collectionId: string;
readonly generation: number;
readonly profile: LocalDeploymentProfile;
readonly lineage: Readonly<LocalDeploymentComposeLineageReceipt>;
readonly recordedAtMs: number;
readonly items: readonly Readonly<CollectionItem>[];
readonly bytes: number;
@@ -174,31 +183,11 @@ function exactSnapshot(
}
}
function readProfile(
paths: Readonly<LocalDeploymentPaths>,
uid: number,
): LocalDeploymentProfile {
const value = readJson(
boundedPrivateFile(
paths.applicationConfig,
uid,
'application configuration',
),
'application configuration',
);
if (
value.schema !== 'qinglong/local-application-process@v2' ||
(value.profile !== 'edge' && value.profile !== 'standalone')
) {
configurationError('application configuration collection binding drifted');
}
return value.profile;
}
function rolloutItem(
paths: Readonly<LocalDeploymentPaths>,
uid: number,
rolloutId: string,
lineage: Readonly<LocalDeploymentComposeLineage>,
): Readonly<CollectionItem> {
const receiptContents = boundedPrivateFile(
path.join(paths.composeRollouts, `${rolloutId}.json`),
@@ -210,6 +199,7 @@ function rolloutItem(
const sqlite = receipt.sqlite as Record<string, unknown> | undefined;
const backup = sqlite?.backup as Record<string, unknown> | undefined;
const result = receipt.result as Record<string, unknown> | undefined;
assertLocalDeploymentComposeLineageReceipt(receipt.lineage, lineage.receipt);
if (
receipt.schema !== ROLLOUT_RECEIPT_SCHEMA ||
receipt.rolloutId !== rolloutId ||
@@ -243,6 +233,7 @@ function restoreItem(
paths: Readonly<LocalDeploymentPaths>,
uid: number,
restoreId: string,
lineage: Readonly<LocalDeploymentComposeLineage>,
): Readonly<CollectionItem> {
const receiptContents = boundedPrivateFile(
path.join(paths.composeRestores, `${restoreId}.commit.json`),
@@ -251,6 +242,7 @@ function restoreItem(
[1, 2],
);
const receipt = readJson(receiptContents, 'compose restore commit receipt');
assertLocalDeploymentComposeLineageReceipt(receipt.lineage, lineage.receipt);
if (
receipt.schema !== RESTORE_COMMIT_SCHEMA ||
receipt.restoreId !== restoreId ||
@@ -348,6 +340,7 @@ async function prepareItems(
paths: Readonly<LocalDeploymentPaths>,
uid: number,
profile: LocalDeploymentProfile,
lineage: Readonly<LocalDeploymentComposeLineage>,
): Promise<readonly Readonly<CollectionItem>[]> {
const rolloutCatalog = catalogIds(
paths.composeRolloutBackups,
@@ -359,14 +352,14 @@ async function prepareItems(
);
const rolloutItems = await Promise.all(
rolloutCatalog.map(async (id) => {
const item = rolloutItem(paths, uid, id);
const item = rolloutItem(paths, uid, id, lineage);
await inspectPhysicalItem(paths, profile, item);
return item;
}),
);
const restoreItems = await Promise.all(
restoreCatalog.map(async (id) => {
const item = restoreItem(paths, uid, id);
const item = restoreItem(paths, uid, id, lineage);
await inspectPhysicalItem(paths, profile, item);
return item;
}),
@@ -491,6 +484,7 @@ function inspectPrepareReceipt(
paths: Readonly<LocalDeploymentPaths>,
uid: number,
command: Readonly<LocalDeploymentComposeEvidenceCollectionPrepareCommand>,
expectedLineage: Readonly<LocalDeploymentComposeLineage>,
): Readonly<CollectionPrepareReceipt> | null {
const filePath = prepareReceiptPath(paths, command.request.collectionId);
if (!fs.existsSync(filePath)) return null;
@@ -505,7 +499,12 @@ function inspectPrepareReceipt(
'compose evidence collection prepare receipt',
);
const items = parseItems(value.items);
const receipt = { ...value, items } as unknown as CollectionPrepareReceipt;
const lineage = normalizeLocalDeploymentComposeLineageReceipt(value.lineage);
const receipt = {
...value,
lineage,
items,
} as unknown as CollectionPrepareReceipt;
if (
Object.keys(value).sort().join(',') !==
[
@@ -513,6 +512,7 @@ function inspectPrepareReceipt(
'commandDigest',
'generation',
'items',
'lineage',
'profile',
'recordedAtMs',
'schema',
@@ -529,6 +529,10 @@ function inspectPrepareReceipt(
) {
configurationError('compose evidence collection prepare receipt drifted');
}
assertLocalDeploymentComposeLineageReceipt(
receipt.lineage,
expectedLineage.receipt,
);
publishExactFile(
filePath,
contents,
@@ -543,6 +547,7 @@ function inspectCommitReceipt(
paths: Readonly<LocalDeploymentPaths>,
uid: number,
command: Readonly<LocalDeploymentComposeEvidenceCollectionCommand>,
expectedLineage: Readonly<LocalDeploymentComposeLineage>,
): Readonly<CollectionCommitReceipt> | null {
const filePath = commitReceiptPath(paths, command.request.collectionId);
if (!fs.existsSync(filePath)) return null;
@@ -557,7 +562,12 @@ function inspectCommitReceipt(
'compose evidence collection commit receipt',
);
const items = parseItems(value.items);
const receipt = { ...value, items } as unknown as CollectionCommitReceipt;
const lineage = normalizeLocalDeploymentComposeLineageReceipt(value.lineage);
const receipt = {
...value,
lineage,
items,
} as unknown as CollectionCommitReceipt;
if (
Object.keys(value).sort().join(',') !==
[
@@ -566,6 +576,7 @@ function inspectCommitReceipt(
'commandDigest',
'generation',
'items',
'lineage',
'prepareReceiptDigest',
'profile',
'recordedAtMs',
@@ -590,6 +601,10 @@ function inspectCommitReceipt(
) {
configurationError('compose evidence collection commit receipt drifted');
}
assertLocalDeploymentComposeLineageReceipt(
receipt.lineage,
expectedLineage.receipt,
);
publishExactFile(
filePath,
contents,
@@ -658,8 +673,9 @@ async function prepareCollection(
command: Readonly<LocalDeploymentComposeEvidenceCollectionPrepareCommand>,
paths: Readonly<LocalDeploymentPaths>,
uid: number,
lineage: Readonly<LocalDeploymentComposeLineage>,
): Promise<Readonly<LocalDeploymentComposeEvidenceCollectionResult>> {
const replay = inspectPrepareReceipt(paths, uid, command);
const replay = inspectPrepareReceipt(paths, uid, command, lineage);
if (replay) return result(command, 'existing', replay.profile, replay.items);
assertOperationsIdle(paths);
const selection = inspectActiveComposeImageSelection(
@@ -672,8 +688,8 @@ async function prepareCollection(
'active compose generation does not match evidence collection',
);
}
const profile = readProfile(paths, uid);
const items = await prepareItems(command, paths, uid, profile);
const profile = lineage.profile;
const items = await prepareItems(command, paths, uid, profile, lineage);
if (items.some((item) => item.recordedAtMs > command.request.preparedAtMs)) {
configurationError(
'compose evidence collection precedes retained evidence',
@@ -699,6 +715,7 @@ async function prepareCollection(
collectionId: command.request.collectionId,
generation: command.request.expectedGeneration,
profile,
lineage: lineage.receipt,
recordedAtMs: command.request.preparedAtMs,
items,
});
@@ -826,13 +843,14 @@ async function commitCollection(
command: Readonly<LocalDeploymentComposeEvidenceCollectionCommand>,
paths: Readonly<LocalDeploymentPaths>,
uid: number,
lineage: Readonly<LocalDeploymentComposeLineage>,
): Promise<Readonly<LocalDeploymentComposeEvidenceCollectionResult>> {
if (
command.operation !== 'local.deployment.compose.evidence-collection.commit'
) {
configurationError('compose evidence collection commit command is invalid');
}
const existing = inspectCommitReceipt(paths, uid, command);
const existing = inspectCommitReceipt(paths, uid, command, lineage);
if (existing) {
const prepareContents = boundedPrivateFile(
prepareReceiptPath(paths, command.request.collectionId),
@@ -875,7 +893,7 @@ async function commitCollection(
return result(command, 'existing', existing.profile, existing.items);
}
const prepared = prepareCommandFromLock(command, paths, uid);
const prepareReceipt = inspectPrepareReceipt(paths, uid, prepared);
const prepareReceipt = inspectPrepareReceipt(paths, uid, prepared, lineage);
if (!prepareReceipt) {
configurationError(
'compose evidence collection prepare receipt is unavailable',
@@ -911,6 +929,7 @@ async function commitCollection(
collectionId: command.request.collectionId,
generation: command.request.expectedGeneration,
profile: prepareReceipt.profile,
lineage: lineage.receipt,
recordedAtMs: command.request.committedAtMs,
items: prepareReceipt.items,
bytes: prepareReceipt.items.reduce(
@@ -992,10 +1011,16 @@ export async function collectLocalDeploymentComposeEvidence(
identity.uid,
'composeCollectedRestoreSafeguardRoot',
);
const lineage = inspectLocalDeploymentComposeLineage(
command.options.deploymentRoot,
identity.uid,
identity.gid,
command.options.allowRootService,
);
return command.operation ===
'local.deployment.compose.evidence-collection.prepare'
? prepareCollection(command, paths, identity.uid)
: commitCollection(command, paths, identity.uid);
? prepareCollection(command, paths, identity.uid, lineage)
: commitCollection(command, paths, identity.uid, lineage);
}
export function collectLocalDeploymentComposeEvidencePrepareCommandFile(
packages/ql3-local-owner-cli/src/deployment/compose/composeLineage.ts
+794
View File
@@ -0,0 +1,794 @@
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { cutoverDigest } from '../cutover/targetEvidence';
import type {
LocalDeploymentPrepareCommand,
LocalDeploymentProfile,
} from '../foundation/contract';
import { LocalDeploymentConfigurationError } from '../foundation/error';
import {
applicationConfiguration,
deploymentPaths,
descriptor,
} from '../foundation/render';
import type { NormalizedLocalDeploymentAdoptedBundleCommand } from '../adopted-bundle/contract';
import {
renderLocalDeploymentAdoptedBundleMaterial,
verifyLocalDeploymentAdoptedEvidence,
type LocalDeploymentAdoptedBundleReceipt,
} from '../adopted-bundle/material';
import { inspectComposeImageSelectionGeneration } from './composeRevision';
const APPLICATION_SCHEMA_V2 = 'qinglong/local-application-process@v2';
const APPLICATION_SCHEMA_V4 = 'qinglong/local-application-process@v4';
const LINEAGE_SCHEMA = 'qinglong/local-compose-deployment-lineage@v1';
const MAX_PRIVATE_FILE_BYTES = 64 * 1024;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
const UUID_V4_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
export interface LocalDeploymentComposeLineageReceipt {
readonly schema: typeof LINEAGE_SCHEMA;
readonly mode: 'fresh' | 'adopted';
readonly applicationConfigDigest: string;
readonly adoptedBundleDigest: string | null;
readonly activationDigest: string | null;
readonly commitmentDigest: string | null;
readonly legacyDataApplicationCommitDigest: string | null;
readonly legacyDataApplicationReceiptDigest: string | null;
}
export interface LocalDeploymentComposeLineage {
readonly mode: 'fresh' | 'adopted';
readonly instanceId: string;
readonly profile: LocalDeploymentProfile;
readonly busyTimeoutMs?: number;
readonly databasePath: string;
readonly sourcePath: string | null;
readonly bundleId: string | null;
readonly receipt: Readonly<LocalDeploymentComposeLineageReceipt>;
}
interface ParsedAdoptedApplication {
readonly instanceId: string;
readonly profile: LocalDeploymentProfile;
readonly busyTimeoutMs?: number;
readonly storage: Readonly<{
sourcePath: string;
targetPath: string;
recoveryPath: string;
manifestPath: string;
activationPath: string;
expectedActivationDigest: string;
}>;
readonly cutover: Readonly<{
cutoverId: string;
commitmentPath: string;
expectedCommitmentDigest: string;
}>;
readonly legacyDataApplication: Readonly<{
commitPath: string;
expectedCommitDigest: string;
expectedReceiptDigest: string;
}>;
}
function configurationError(message: string, cause?: unknown): never {
throw new LocalDeploymentConfigurationError(message, { cause });
}
function object(value: unknown, label: string): Record<string, unknown> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
(Object.getPrototypeOf(value) !== Object.prototype &&
Object.getPrototypeOf(value) !== null)
) {
configurationError(`${label} must be an object`);
}
return value as Record<string, unknown>;
}
function exact(
value: Readonly<Record<string, unknown>>,
keys: readonly string[],
label: string,
): void {
const actual = Object.keys(value).sort();
const expected = [...keys].sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index])
) {
configurationError(`${label} shape is invalid`);
}
}
function sha256(value: string | Buffer): string {
return crypto.createHash('sha256').update(value).digest('hex');
}
function digest(value: unknown, label: string): string {
if (typeof value !== 'string' || !DIGEST_PATTERN.test(value)) {
configurationError(`${label} is invalid`);
}
return value;
}
function absolutePath(value: unknown, label: string): string {
if (
typeof value !== 'string' ||
!path.isAbsolute(value) ||
path.normalize(value) !== value ||
path.parse(value).root === value ||
value.includes('\0') ||
value.includes('//') ||
Buffer.byteLength(value, 'utf8') > 4_096
) {
configurationError(`${label} is not a canonical bounded absolute path`);
}
return value;
}
function strictDescendant(root: string, candidate: string): boolean {
const relative = path.relative(root, candidate);
return (
relative.length > 0 &&
relative !== '..' &&
!relative.startsWith(`..${path.sep}`) &&
!path.isAbsolute(relative)
);
}
function readExactPrivateFile(
filePath: string,
uid: number,
gid: number,
label: string,
): string {
let descriptor: number | undefined;
let bytes: Buffer | undefined;
try {
const before = fs.lstatSync(filePath, { bigint: true });
descriptor = fs.openSync(
filePath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
const opened = fs.fstatSync(descriptor, { bigint: true });
if (
!opened.isFile() ||
opened.isSymbolicLink() ||
opened.dev !== before.dev ||
opened.ino !== before.ino ||
Number(opened.uid) !== uid ||
Number(opened.gid) !== gid ||
(Number(opened.mode) & 0o777) !== 0o600 ||
opened.nlink !== 1n ||
opened.size < 2n ||
opened.size > BigInt(MAX_PRIVATE_FILE_BYTES) ||
fs.realpathSync(filePath) !== filePath
) {
configurationError(`${label} identity is invalid`);
}
bytes = Buffer.alloc(Number(opened.size));
let offset = 0;
while (offset < bytes.byteLength) {
const count = fs.readSync(
descriptor,
bytes,
offset,
bytes.byteLength - offset,
offset,
);
if (count === 0) break;
offset += count;
}
const after = fs.fstatSync(descriptor, { bigint: true });
if (
offset !== bytes.byteLength ||
after.dev !== opened.dev ||
after.ino !== opened.ino ||
after.size !== opened.size ||
after.mtimeNs !== opened.mtimeNs ||
after.ctimeNs !== opened.ctimeNs ||
after.uid !== opened.uid ||
after.gid !== opened.gid ||
after.mode !== opened.mode ||
after.nlink !== opened.nlink
) {
configurationError(`${label} changed while reading`);
}
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
} catch (error) {
if (error instanceof LocalDeploymentConfigurationError) throw error;
return configurationError(`${label} cannot be read`, error);
} finally {
bytes?.fill(0);
if (descriptor !== undefined) fs.closeSync(descriptor);
}
}
function verifyExactPrivateFile(
filePath: string,
expected: string,
uid: number,
gid: number,
label: string,
): void {
if (readExactPrivateFile(filePath, uid, gid, label) !== expected) {
configurationError(`${label} content drifted`);
}
}
function parseJson(contents: string, label: string): Record<string, unknown> {
try {
return object(JSON.parse(contents), label);
} catch (error) {
if (error instanceof LocalDeploymentConfigurationError) throw error;
return configurationError(`${label} is invalid`, error);
}
}
function optionalBusyTimeout(value: unknown): number | undefined {
if (value === undefined) return undefined;
if (
!Number.isSafeInteger(value) ||
(value as number) < 100 ||
(value as number) > 30_000
) {
configurationError('application busyTimeoutMs is invalid');
}
return value as number;
}
function parseFreshApplication(
value: Readonly<Record<string, unknown>>,
): Readonly<{
instanceId: string;
profile: LocalDeploymentProfile;
databasePath: string;
busyTimeoutMs?: number;
}> {
const storage = object(value.storage, 'application storage');
if (
typeof value.instanceId !== 'string' ||
(value.profile !== 'edge' && value.profile !== 'standalone') ||
storage.mode !== 'fresh'
) {
configurationError('application configuration is not a fresh Compose v2');
}
const databasePath = absolutePath(
storage.databasePath,
'application databasePath',
);
const busyTimeoutMs = optionalBusyTimeout(storage.busyTimeoutMs);
return Object.freeze({
instanceId: value.instanceId,
profile: value.profile,
databasePath,
...(busyTimeoutMs === undefined ? {} : { busyTimeoutMs }),
});
}
function parseAdoptedApplication(
value: Readonly<Record<string, unknown>>,
deploymentRoot: string,
): Readonly<ParsedAdoptedApplication> {
exact(
value,
[
'ai',
'cutover',
'instanceId',
'legacyDataApplication',
'pluginPackages',
'profile',
'runtime',
'schema',
'storage',
],
'adopted application configuration',
);
if (
typeof value.instanceId !== 'string' ||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value.instanceId) ||
(value.profile !== 'edge' && value.profile !== 'standalone')
) {
configurationError('adopted application identity is invalid');
}
const storage = object(value.storage, 'adopted storage');
const optionalStorageKeys = Object.hasOwn(storage, 'busyTimeoutMs')
? ['busyTimeoutMs']
: [];
exact(
storage,
[
'activationPath',
'expectedActivationDigest',
'manifestPath',
'mode',
'recoveryPath',
'sourcePath',
'targetPath',
...optionalStorageKeys,
],
'adopted storage',
);
const runtime = object(value.runtime, 'adopted runtime');
exact(
runtime,
['artifactRoot', 'receiptRoot', 'secretKeyringPath'],
'adopted runtime',
);
const pluginPackages = object(
value.pluginPackages,
'adopted plugin packages',
);
exact(
pluginPackages,
[
'activationRoot',
'maxPages',
'pageSize',
'recoverySource',
'stagingRoot',
'taskPublicationMaxPages',
'taskPublicationPageSize',
],
'adopted plugin packages',
);
const recoverySource = object(
pluginPackages.recoverySource,
'plugin recovery source',
);
exact(recoverySource, ['mode'], 'plugin recovery source');
const ai = object(value.ai, 'adopted AI');
exact(ai, ['deployment'], 'adopted AI');
const cutover = object(value.cutover, 'adopted cutover');
exact(
cutover,
['commitmentPath', 'cutoverId', 'expectedCommitmentDigest'],
'adopted cutover',
);
const dataApplication = object(
value.legacyDataApplication,
'legacy data application',
);
exact(
dataApplication,
['commitPath', 'expectedCommitDigest', 'expectedReceiptDigest'],
'legacy data application',
);
const sourcePath = absolutePath(storage.sourcePath, 'sourcePath');
const authorityPaths = {
targetPath: absolutePath(storage.targetPath, 'targetPath'),
recoveryPath: absolutePath(storage.recoveryPath, 'recoveryPath'),
manifestPath: absolutePath(storage.manifestPath, 'manifestPath'),
activationPath: absolutePath(storage.activationPath, 'activationPath'),
commitmentPath: absolutePath(cutover.commitmentPath, 'commitmentPath'),
commitPath: absolutePath(dataApplication.commitPath, 'commitPath'),
};
const paths = deploymentPaths(deploymentRoot);
const pageSize = value.profile === 'edge' ? 4 : 16;
if (
storage.mode !== 'adopted' ||
strictDescendant(deploymentRoot, sourcePath) ||
Object.values(authorityPaths).some(
(candidate) => !strictDescendant(deploymentRoot, candidate),
) ||
new Set([sourcePath, ...Object.values(authorityPaths)]).size !== 7 ||
runtime.receiptRoot !== paths.receipts ||
runtime.artifactRoot !== paths.artifacts ||
runtime.secretKeyringPath !== paths.localSecretKeyring ||
pluginPackages.stagingRoot !== paths.pluginStaging ||
pluginPackages.activationRoot !== paths.pluginActivation ||
recoverySource.mode !== 'disabled' ||
pluginPackages.pageSize !== pageSize ||
pluginPackages.maxPages !== pageSize ||
pluginPackages.taskPublicationPageSize !== pageSize ||
pluginPackages.taskPublicationMaxPages !== pageSize ||
ai.deployment !== 'excluded' ||
typeof cutover.cutoverId !== 'string' ||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(cutover.cutoverId)
) {
configurationError('adopted application authority binding drifted');
}
const busyTimeoutMs = optionalBusyTimeout(storage.busyTimeoutMs);
return Object.freeze({
instanceId: value.instanceId,
profile: value.profile,
...(busyTimeoutMs === undefined ? {} : { busyTimeoutMs }),
storage: Object.freeze({
sourcePath,
targetPath: authorityPaths.targetPath,
recoveryPath: authorityPaths.recoveryPath,
manifestPath: authorityPaths.manifestPath,
activationPath: authorityPaths.activationPath,
expectedActivationDigest: digest(
storage.expectedActivationDigest,
'expectedActivationDigest',
),
}),
cutover: Object.freeze({
cutoverId: cutover.cutoverId,
commitmentPath: authorityPaths.commitmentPath,
expectedCommitmentDigest: digest(
cutover.expectedCommitmentDigest,
'expectedCommitmentDigest',
),
}),
legacyDataApplication: Object.freeze({
commitPath: authorityPaths.commitPath,
expectedCommitDigest: digest(
dataApplication.expectedCommitDigest,
'expectedCommitDigest',
),
expectedReceiptDigest: digest(
dataApplication.expectedReceiptDigest,
'expectedReceiptDigest',
),
}),
});
}
function parseBundleReceipt(
contents: string,
): Readonly<LocalDeploymentAdoptedBundleReceipt> {
const value = parseJson(contents, 'adopted bundle receipt');
exact(
value,
[
'activationDigest',
'applicationConfigDigest',
'bundleDigest',
'bundleId',
'commitmentDigest',
'composeSelectionDigest',
'cutoverId',
'deploymentRootDigest',
'instanceId',
'kind',
'legacyDataApplicationCommitDigest',
'legacyDataApplicationReceiptDigest',
'manifestDigest',
'preparedAtMs',
'profile',
'recoverySha256',
'schemaVersion',
'serviceDescriptorDigest',
'serviceKind',
'sourcePathDigest',
'sourceSha256',
'state',
'targetIdentityDigest',
],
'adopted bundle receipt',
);
const { bundleDigest, ...payload } = value;
if (
value.schemaVersion !== 1 ||
value.kind !== 'qinglong3-local-adopted-deployment-bundle' ||
value.state !== 'prepared' ||
typeof value.bundleId !== 'string' ||
!UUID_V4_PATTERN.test(value.bundleId) ||
!Number.isSafeInteger(value.preparedAtMs) ||
(value.preparedAtMs as number) < 0 ||
(value.profile !== 'edge' && value.profile !== 'standalone') ||
typeof value.instanceId !== 'string' ||
typeof value.cutoverId !== 'string' ||
value.serviceKind !== 'compose' ||
Object.entries(value).some(([key, candidate]) =>
key.endsWith('Digest')
? typeof candidate !== 'string' || !DIGEST_PATTERN.test(candidate)
: false,
) ||
typeof value.sourceSha256 !== 'string' ||
!DIGEST_PATTERN.test(value.sourceSha256) ||
typeof value.recoverySha256 !== 'string' ||
!DIGEST_PATTERN.test(value.recoverySha256) ||
cutoverDigest(payload) !== bundleDigest
) {
configurationError('adopted bundle receipt drifted');
}
return value as unknown as Readonly<LocalDeploymentAdoptedBundleReceipt>;
}
function lineageReceipt(input: {
readonly mode: 'fresh' | 'adopted';
readonly applicationConfigDigest: string;
readonly adoptedBundleDigest?: string;
readonly activationDigest?: string;
readonly commitmentDigest?: string;
readonly legacyDataApplicationCommitDigest?: string;
readonly legacyDataApplicationReceiptDigest?: string;
}): Readonly<LocalDeploymentComposeLineageReceipt> {
const adopted = input.mode === 'adopted';
return Object.freeze({
schema: LINEAGE_SCHEMA,
mode: input.mode,
applicationConfigDigest: input.applicationConfigDigest,
adoptedBundleDigest: adopted ? input.adoptedBundleDigest! : null,
activationDigest: adopted ? input.activationDigest! : null,
commitmentDigest: adopted ? input.commitmentDigest! : null,
legacyDataApplicationCommitDigest: adopted
? input.legacyDataApplicationCommitDigest!
: null,
legacyDataApplicationReceiptDigest: adopted
? input.legacyDataApplicationReceiptDigest!
: null,
});
}
export function inspectLocalDeploymentComposeLineage(
deploymentRoot: string,
uid: number,
gid: number,
allowRootService: boolean,
): Readonly<LocalDeploymentComposeLineage> {
const paths = deploymentPaths(deploymentRoot);
const applicationContents = readExactPrivateFile(
paths.applicationConfig,
uid,
gid,
'application configuration',
);
const application = parseJson(
applicationContents,
'application configuration',
);
if (application.schema === APPLICATION_SCHEMA_V2) {
const fresh = parseFreshApplication(application);
const syntheticPrepare: Readonly<LocalDeploymentPrepareCommand> = {
schemaVersion: 1,
operation: 'local.deployment.prepare',
options: {
deploymentRoot,
profile: fresh.profile,
instanceId: fresh.instanceId,
...(fresh.busyTimeoutMs === undefined
? {}
: { busyTimeoutMs: fresh.busyTimeoutMs }),
service: {
kind: 'compose',
releaseSelection: {
path: paths.composeSelection,
expectedSelectionDigest: 'sha256:'.concat('0'.repeat(64)),
},
allowRootService,
},
},
request: {
ownerPepperKeyId: 'lineage',
registerMutationId: '00000000-0000-4000-8000-000000000001',
activateMutationId: '00000000-0000-4000-8000-000000000002',
registeredAtMs: 0,
activatedAtMs: 0,
},
};
const expectedApplication = applicationConfiguration(
syntheticPrepare,
paths,
);
if (applicationContents !== expectedApplication) {
configurationError('fresh application configuration drifted');
}
const expectedDescriptor = descriptor(
syntheticPrepare,
paths.applicationConfig,
uid,
gid,
);
verifyExactPrivateFile(
path.join(paths.service, expectedDescriptor.fileName),
expectedDescriptor.contents,
uid,
gid,
'fresh Compose descriptor',
);
return Object.freeze({
mode: 'fresh' as const,
instanceId: fresh.instanceId,
profile: fresh.profile,
...(fresh.busyTimeoutMs === undefined
? {}
: { busyTimeoutMs: fresh.busyTimeoutMs }),
databasePath: paths.database,
sourcePath: null,
bundleId: null,
receipt: lineageReceipt({
mode: 'fresh',
applicationConfigDigest: sha256(applicationContents),
}),
});
}
if (application.schema !== APPLICATION_SCHEMA_V4) {
configurationError('Compose requires an Application v2 or adopted v4');
}
const adopted = parseAdoptedApplication(application, deploymentRoot);
const bundleContents = readExactPrivateFile(
path.join(paths.service, 'adopted-bundle.json'),
uid,
gid,
'adopted bundle receipt',
);
const bundle = parseBundleReceipt(bundleContents);
if (
bundle.profile !== adopted.profile ||
bundle.instanceId !== adopted.instanceId ||
bundle.cutoverId !== adopted.cutover.cutoverId ||
bundle.deploymentRootDigest !== sha256(deploymentRoot) ||
bundle.sourcePathDigest !== sha256(adopted.storage.sourcePath) ||
bundle.applicationConfigDigest !== sha256(applicationContents) ||
bundle.activationDigest !== adopted.storage.expectedActivationDigest ||
bundle.commitmentDigest !== adopted.cutover.expectedCommitmentDigest ||
bundle.legacyDataApplicationCommitDigest !==
adopted.legacyDataApplication.expectedCommitDigest ||
bundle.legacyDataApplicationReceiptDigest !==
adopted.legacyDataApplication.expectedReceiptDigest
) {
configurationError('adopted bundle application binding drifted');
}
const initialSelection = inspectComposeImageSelectionGeneration(
paths.composeRevisions,
1,
uid,
);
const command: Readonly<NormalizedLocalDeploymentAdoptedBundleCommand> =
Object.freeze({
schemaVersion: 1 as const,
operation: 'local.deployment.adopted.verify' as const,
options: Object.freeze({
deploymentRoot,
profile: adopted.profile,
instanceId: adopted.instanceId,
...(adopted.busyTimeoutMs === undefined
? {}
: { busyTimeoutMs: adopted.busyTimeoutMs }),
service: Object.freeze({
kind: 'compose' as const,
allowRootService,
releaseSelection: Object.freeze({
path: paths.composeSelection,
expectedSelectionDigest: initialSelection.selectionDigest,
authority: initialSelection,
}),
}),
}),
request: Object.freeze({
bundleId: bundle.bundleId,
preparedAtMs: bundle.preparedAtMs,
cutoverId: adopted.cutover.cutoverId,
storage: adopted.storage,
cutover: Object.freeze({
commitmentPath: adopted.cutover.commitmentPath,
expectedCommitmentDigest: adopted.cutover.expectedCommitmentDigest,
}),
legacyDataApplication: adopted.legacyDataApplication,
}),
});
const evidence = verifyLocalDeploymentAdoptedEvidence(command, uid, gid);
const material = renderLocalDeploymentAdoptedBundleMaterial(
command,
evidence,
uid,
gid,
);
verifyExactPrivateFile(
material.paths.applicationConfig,
material.applicationConfig,
uid,
gid,
'adopted application configuration',
);
verifyExactPrivateFile(
material.paths.descriptor,
material.descriptor.contents,
uid,
gid,
'adopted Compose descriptor',
);
verifyExactPrivateFile(
material.paths.bundleReceipt,
material.receiptContents,
uid,
gid,
'adopted bundle receipt',
);
if (material.composeSelection === null) {
configurationError('adopted Compose selection is unavailable');
}
verifyExactPrivateFile(
material.paths.composeRevision,
material.composeSelection,
uid,
gid,
'initial adopted Compose revision',
);
return Object.freeze({
mode: 'adopted' as const,
instanceId: adopted.instanceId,
profile: adopted.profile,
...(adopted.busyTimeoutMs === undefined
? {}
: { busyTimeoutMs: adopted.busyTimeoutMs }),
databasePath: adopted.storage.targetPath,
sourcePath: adopted.storage.sourcePath,
bundleId: bundle.bundleId,
receipt: lineageReceipt({
mode: 'adopted',
applicationConfigDigest: bundle.applicationConfigDigest,
adoptedBundleDigest: bundle.bundleDigest,
activationDigest: bundle.activationDigest,
commitmentDigest: bundle.commitmentDigest,
legacyDataApplicationCommitDigest:
bundle.legacyDataApplicationCommitDigest,
legacyDataApplicationReceiptDigest:
bundle.legacyDataApplicationReceiptDigest,
}),
});
}
export function normalizeLocalDeploymentComposeLineageReceipt(
value: unknown,
): Readonly<LocalDeploymentComposeLineageReceipt> {
const receipt = object(value, 'Compose deployment lineage');
exact(
receipt,
[
'activationDigest',
'adoptedBundleDigest',
'applicationConfigDigest',
'commitmentDigest',
'legacyDataApplicationCommitDigest',
'legacyDataApplicationReceiptDigest',
'mode',
'schema',
],
'Compose deployment lineage',
);
const adopted = receipt.mode === 'adopted';
if (
receipt.schema !== LINEAGE_SCHEMA ||
(receipt.mode !== 'fresh' && !adopted) ||
typeof receipt.applicationConfigDigest !== 'string' ||
!DIGEST_PATTERN.test(receipt.applicationConfigDigest) ||
[
receipt.adoptedBundleDigest,
receipt.activationDigest,
receipt.commitmentDigest,
receipt.legacyDataApplicationCommitDigest,
receipt.legacyDataApplicationReceiptDigest,
].some((candidate) =>
adopted
? typeof candidate !== 'string' || !DIGEST_PATTERN.test(candidate)
: candidate !== null,
)
) {
configurationError('Compose deployment lineage is invalid');
}
return Object.freeze({
schema: LINEAGE_SCHEMA,
mode: receipt.mode,
applicationConfigDigest: receipt.applicationConfigDigest,
adoptedBundleDigest: receipt.adoptedBundleDigest,
activationDigest: receipt.activationDigest,
commitmentDigest: receipt.commitmentDigest,
legacyDataApplicationCommitDigest:
receipt.legacyDataApplicationCommitDigest,
legacyDataApplicationReceiptDigest:
receipt.legacyDataApplicationReceiptDigest,
}) as Readonly<LocalDeploymentComposeLineageReceipt>;
}
export function assertLocalDeploymentComposeLineageReceipt(
value: unknown,
expected: Readonly<LocalDeploymentComposeLineageReceipt>,
): void {
const actual = normalizeLocalDeploymentComposeLineageReceipt(value);
if (JSON.stringify(actual) !== JSON.stringify(expected)) {
configurationError('Compose deployment lineage drifted');
}
}
packages/ql3-local-owner-cli/src/deployment/compose/composePreflight.ts
+138 -137
View File
@@ -1,4 +1,3 @@
import fs from 'node:fs';
import path from 'node:path';
import { readPrivateLocalCommandFile } from '@qinglong/local-command-file';
@@ -31,6 +30,10 @@ import {
deploymentPaths,
descriptor,
} from '../foundation/render';
import {
inspectLocalDeploymentComposeLineage,
type LocalDeploymentComposeLineage,
} from './composeLineage';
const CONTAINER_ROOT = '/var/lib/qinglong3';
const MAX_DOCKER_OUTPUT_BYTES = 256 * 1024;
@@ -46,87 +49,10 @@ export interface LocalDeploymentComposePreflightDependencies {
readonly validateSocket?: (socketPath: string, uid: number) => void;
}
interface ApplicationIdentity {
readonly instanceId: string;
readonly profile: LocalDeploymentProfile;
readonly busyTimeoutMs?: number;
}
function configurationError(message: string, cause?: unknown): never {
throw new LocalDeploymentConfigurationError(message, { cause });
}
function readApplicationIdentity(
filePath: string,
uid: number,
): Readonly<ApplicationIdentity> {
let stat: fs.Stats;
try {
stat = fs.lstatSync(filePath);
} catch (error) {
configurationError('application configuration is unavailable', error);
}
if (
!stat.isFile() ||
stat.isSymbolicLink() ||
stat.uid !== uid ||
(stat.mode & 0o777) !== 0o600 ||
stat.nlink !== 1 ||
stat.size < 2 ||
stat.size > 64 * 1024
) {
configurationError('application configuration identity is invalid');
}
let value: unknown;
try {
value = JSON.parse(fs.readFileSync(filePath, 'utf8'));
} catch (error) {
configurationError('application configuration is invalid', error);
}
const candidate = value as {
readonly schema?: unknown;
readonly instanceId?: unknown;
readonly profile?: unknown;
readonly storage?: {
readonly mode?: unknown;
readonly databasePath?: unknown;
readonly busyTimeoutMs?: unknown;
};
};
if (
!candidate ||
typeof candidate !== 'object' ||
Array.isArray(candidate) ||
candidate.schema !== 'qinglong/local-application-process@v2' ||
typeof candidate.instanceId !== 'string' ||
(candidate.profile !== 'edge' && candidate.profile !== 'standalone') ||
!candidate.storage ||
candidate.storage.mode !== 'fresh' ||
candidate.storage.databasePath !== `${CONTAINER_ROOT}/qinglong3.sqlite`
) {
configurationError('application configuration is not a fresh Compose v2');
}
const busyTimeoutMs =
candidate.storage.busyTimeoutMs === undefined
? undefined
: candidate.storage.busyTimeoutMs;
if (
busyTimeoutMs !== undefined &&
(!Number.isSafeInteger(busyTimeoutMs) ||
(busyTimeoutMs as number) < 100 ||
(busyTimeoutMs as number) > 30_000)
) {
configurationError('application busyTimeoutMs is invalid');
}
return Object.freeze({
instanceId: candidate.instanceId,
profile: candidate.profile,
...(busyTimeoutMs === undefined
? {}
: { busyTimeoutMs: busyTimeoutMs as number }),
});
}
function parseJson(value: string, label: string): unknown {
if (
Buffer.byteLength(value, 'utf8') < 2 ||
@@ -157,6 +83,7 @@ function inspectImage(
image: string,
profile: LocalDeploymentProfile,
sqliteContractVersion: number,
applicationConfigVersion: '2' | '4',
): 'amd64' | 'arm64' {
const parsed = parseJson(output, 'Docker image inspect') as unknown[];
if (!Array.isArray(parsed) || parsed.length !== 1) {
@@ -194,6 +121,10 @@ function inspectImage(
const minimum = safeLabel(labels, 'io.qinglong.local.sqlite-contract-min');
const maximum = safeLabel(labels, 'io.qinglong.local.sqlite-contract-max');
const profiles = safeLabel(labels, 'io.qinglong.profile').split(',');
const applicationConfigs = safeLabel(
labels,
'io.qinglong.local.application-config',
).split(',');
if (
!/^[1-9][0-9]{0,3}$/.test(minimum) ||
!/^[1-9][0-9]{0,3}$/.test(maximum) ||
@@ -201,7 +132,7 @@ function inspectImage(
Number(maximum) < sqliteContractVersion ||
safeLabel(labels, 'io.qinglong.local.sqlite-write-contract') !==
String(sqliteContractVersion) ||
safeLabel(labels, 'io.qinglong.local.application-config') !== '2' ||
!applicationConfigs.includes(applicationConfigVersion) ||
safeLabel(labels, 'io.qinglong.local.compose-selection') !== '1' ||
safeLabel(labels, 'io.qinglong.ai') !== 'excluded' ||
!profiles.includes(profile) ||
@@ -233,10 +164,15 @@ function inspectComposeConfig(
image: string;
generation: number;
mutationId: string;
releaseSelectionDigest: string;
releaseSetDigest: string;
catalogManifestDigest: string;
catalogConsumptionReportDigest: string;
deploymentRoot: string;
uid: number;
gid: number;
profile: LocalDeploymentProfile;
lineage: Readonly<LocalDeploymentComposeLineage>;
}>,
): void {
const config = parseJson(output, 'Docker Compose config') as {
@@ -267,7 +203,8 @@ function inspectComposeConfig(
service.user !== `${expected.uid}:${expected.gid}` ||
service.read_only !== true ||
service.network_mode !== 'none' ||
service.restart !== 'unless-stopped' ||
service.restart !==
(expected.lineage.mode === 'fresh' ? 'unless-stopped' : 'no') ||
Number(service.mem_limit) !== expectedMemory ||
Number(service.pids_limit) !== expectedPids ||
service.privileged === true ||
@@ -279,7 +216,9 @@ function inspectComposeConfig(
!exactArray(service.security_opt, ['no-new-privileges:true']) ||
!exactArray(service.command, [
'--config',
`${CONTAINER_ROOT}/local-application.json`,
expected.lineage.mode === 'fresh'
? `${CONTAINER_ROOT}/local-application.json`
: path.join(expected.deploymentRoot, 'local-application.json'),
])
) {
configurationError('Docker Compose service contract is invalid');
@@ -287,9 +226,44 @@ function inspectComposeConfig(
const labels = service.labels as
| Readonly<Record<string, unknown>>
| undefined;
const expectedLabels =
expected.lineage.mode === 'fresh'
? {
'io.qinglong.deployment.generation': String(expected.generation),
'io.qinglong.deployment.mutation': expected.mutationId,
'io.qinglong.release.selection': expected.releaseSelectionDigest,
'io.qinglong.release.set': expected.releaseSetDigest,
'io.qinglong.release.catalog-manifest':
expected.catalogManifestDigest,
'io.qinglong.release.catalog-report':
expected.catalogConsumptionReportDigest,
}
: {
'io.qinglong.deployment.mode': 'adopted',
'io.qinglong.deployment.profile': expected.profile,
'io.qinglong.deployment.instance': expected.lineage.instanceId,
'io.qinglong.deployment.bundle': expected.lineage.bundleId!,
'io.qinglong.application.config':
expected.lineage.receipt.applicationConfigDigest,
'io.qinglong.data.commit':
expected.lineage.receipt.legacyDataApplicationCommitDigest!,
'io.qinglong.data.receipt':
expected.lineage.receipt.legacyDataApplicationReceiptDigest!,
'io.qinglong.deployment.generation': String(expected.generation),
'io.qinglong.deployment.mutation': expected.mutationId,
'io.qinglong.release.selection': expected.releaseSelectionDigest,
'io.qinglong.release.set': expected.releaseSetDigest,
'io.qinglong.release.catalog-manifest':
expected.catalogManifestDigest,
'io.qinglong.release.catalog-report':
expected.catalogConsumptionReportDigest,
};
if (
!labels ||
Object.keys(labels).length !== 2 ||
Object.keys(labels).length !== Object.keys(expectedLabels).length ||
Object.entries(expectedLabels).some(
([key, value]) => labels[key] !== value,
) ||
labels['io.qinglong.deployment.generation'] !==
String(expected.generation) ||
labels['io.qinglong.deployment.mutation'] !== expected.mutationId
@@ -297,15 +271,32 @@ function inspectComposeConfig(
configurationError('Docker Compose deployment labels are invalid');
}
const volumes = service.volumes as readonly unknown[] | undefined;
const volume = volumes?.[0] as Readonly<Record<string, unknown>> | undefined;
const tmpfs = service.tmpfs as readonly unknown[] | undefined;
const expectedVolumes =
expected.lineage.mode === 'fresh'
? [
{
type: 'bind',
source: expected.deploymentRoot,
target: CONTAINER_ROOT,
},
]
: [
{
type: 'bind',
source: expected.deploymentRoot,
target: expected.deploymentRoot,
},
{
type: 'bind',
source: expected.lineage.sourcePath,
target: expected.lineage.sourcePath,
read_only: true,
},
];
if (
!Array.isArray(volumes) ||
volumes.length !== 1 ||
!volume ||
volume.type !== 'bind' ||
volume.source !== expected.deploymentRoot ||
volume.target !== CONTAINER_ROOT ||
JSON.stringify(volumes) !== JSON.stringify(expectedVolumes) ||
!Array.isArray(tmpfs) ||
tmpfs.length !== 1 ||
typeof tmpfs[0] !== 'string' ||
@@ -353,62 +344,66 @@ export async function preflightLocalDeploymentCompose(
'active compose generation does not match expectedGeneration',
);
}
const application = readApplicationIdentity(
paths.applicationConfig,
identity.uid,
);
const syntheticPrepare: Readonly<LocalDeploymentPrepareCommand> = {
schemaVersion: 1,
operation: 'local.deployment.prepare',
options: {
deploymentRoot: command.options.deploymentRoot,
profile: application.profile,
instanceId: application.instanceId,
...(application.busyTimeoutMs === undefined
? {}
: { busyTimeoutMs: application.busyTimeoutMs }),
service: {
kind: 'compose',
releaseSelection: {
path: paths.composeSelection,
expectedSelectionDigest: selection.selectionDigest,
},
allowRootService: command.options.allowRootService,
},
},
request: {
ownerPepperKeyId: 'preflight',
registerMutationId: '00000000-0000-4000-8000-000000000001',
activateMutationId: '00000000-0000-4000-8000-000000000002',
registeredAtMs: 0,
activatedAtMs: 0,
},
};
preflightPublishedFile(
paths.applicationConfig,
applicationConfiguration(syntheticPrepare, paths),
0o600,
identity.uid,
'application configuration',
);
const expectedDescriptor = descriptor(
syntheticPrepare,
paths.applicationConfig,
const application = inspectLocalDeploymentComposeLineage(
command.options.deploymentRoot,
identity.uid,
identity.gid,
command.options.allowRootService,
);
preflightPublishedFile(
path.join(paths.service, expectedDescriptor.fileName),
expectedDescriptor.contents,
expectedDescriptor.mode,
identity.uid,
'service descriptor',
);
if (application.mode === 'fresh') {
const syntheticPrepare: Readonly<LocalDeploymentPrepareCommand> = {
schemaVersion: 1,
operation: 'local.deployment.prepare',
options: {
deploymentRoot: command.options.deploymentRoot,
profile: application.profile,
instanceId: application.instanceId,
...(application.busyTimeoutMs === undefined
? {}
: { busyTimeoutMs: application.busyTimeoutMs }),
service: {
kind: 'compose',
releaseSelection: {
path: paths.composeSelection,
expectedSelectionDigest: selection.selectionDigest,
},
allowRootService: command.options.allowRootService,
},
},
request: {
ownerPepperKeyId: 'preflight',
registerMutationId: '00000000-0000-4000-8000-000000000001',
activateMutationId: '00000000-0000-4000-8000-000000000002',
registeredAtMs: 0,
activatedAtMs: 0,
},
};
preflightPublishedFile(
paths.applicationConfig,
applicationConfiguration(syntheticPrepare, paths),
0o600,
identity.uid,
'application configuration',
);
const expectedDescriptor = descriptor(
syntheticPrepare,
paths.applicationConfig,
identity.uid,
identity.gid,
);
preflightPublishedFile(
path.join(paths.service, expectedDescriptor.fileName),
expectedDescriptor.contents,
expectedDescriptor.mode,
identity.uid,
'service descriptor',
);
}
const auditSqlite =
dependencies.auditSqlite ?? inspectLocalSqliteReadinessPath;
const sqlite = await auditSqlite({
databasePath: paths.database,
databasePath: application.databasePath,
profile: application.profile,
...(application.busyTimeoutMs === undefined
? {}
@@ -429,6 +424,7 @@ export async function preflightLocalDeploymentCompose(
selection.image,
application.profile,
sqlite.contractVersion,
application.mode === 'fresh' ? '2' : '4',
);
const composeOutput = runDocker({
executable: command.options.dockerExecutable,
@@ -451,10 +447,15 @@ export async function preflightLocalDeploymentCompose(
image: selection.image,
generation: selection.generation,
mutationId: selection.mutationId,
releaseSelectionDigest: selection.selectionDigest,
releaseSetDigest: selection.releaseSetDigest,
catalogManifestDigest: selection.catalogManifestDigest,
catalogConsumptionReportDigest: selection.catalogConsumptionReportDigest,
deploymentRoot: command.options.deploymentRoot,
uid: identity.uid,
gid: identity.gid,
profile: application.profile,
lineage: application,
});
return Object.freeze({
packages/ql3-local-owner-cli/src/deployment/compose/composeRestore.ts
+69 -55
View File
@@ -49,9 +49,16 @@ import {
deploymentPaths,
type LocalDeploymentPaths,
} from '../foundation/render';
import {
assertLocalDeploymentComposeLineageReceipt,
inspectLocalDeploymentComposeLineage,
normalizeLocalDeploymentComposeLineageReceipt,
type LocalDeploymentComposeLineage,
type LocalDeploymentComposeLineageReceipt,
} from './composeLineage';
const PREPARE_SCHEMA = 'qinglong/local-compose-restore-prepare@v1';
const COMMIT_SCHEMA = 'qinglong/local-compose-restore-commit@v1';
const PREPARE_SCHEMA = 'qinglong/local-compose-restore-prepare@v2';
const COMMIT_SCHEMA = 'qinglong/local-compose-restore-commit@v2';
const UUID_V4_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
const SHA256_PATTERN = /^[0-9a-f]{64}$/;
@@ -68,6 +75,7 @@ interface RestorePrepareReceipt {
readonly generation: number;
readonly recordedAtMs: number;
readonly profile: LocalDeploymentProfile;
readonly lineage: Readonly<LocalDeploymentComposeLineageReceipt>;
readonly current: Readonly<LocalSqliteSnapshotEvidence>;
readonly source: Readonly<LocalSqliteSnapshotEvidence>;
readonly safeguard: Readonly<LocalSqliteSnapshotEvidence>;
@@ -82,6 +90,7 @@ interface RestoreCommitReceipt {
readonly generation: number;
readonly recordedAtMs: number;
readonly profile: LocalDeploymentProfile;
readonly lineage: Readonly<LocalDeploymentComposeLineageReceipt>;
readonly source: Readonly<LocalSqliteSnapshotEvidence>;
readonly safeguard: Readonly<LocalSqliteSnapshotEvidence>;
readonly restored: Readonly<LocalSqliteSnapshotEvidence>;
@@ -211,40 +220,6 @@ function boundedReceipt(
return fs.readFileSync(filePath, 'utf8');
}
function readApplicationProfile(
paths: Readonly<LocalDeploymentPaths>,
uid: number,
): LocalDeploymentProfile {
const contents = boundedReceipt(
paths.applicationConfig,
uid,
'application configuration',
);
let value: unknown;
try {
value = JSON.parse(contents);
} catch (error) {
configurationError('application configuration is invalid', error);
}
const candidate = value as {
readonly schema?: unknown;
readonly profile?: unknown;
readonly storage?: {
readonly mode?: unknown;
readonly databasePath?: unknown;
};
};
if (
candidate?.schema !== 'qinglong/local-application-process@v2' ||
(candidate.profile !== 'edge' && candidate.profile !== 'standalone') ||
candidate.storage?.mode !== 'fresh' ||
candidate.storage.databasePath !== '/var/lib/qinglong3/qinglong3.sqlite'
) {
configurationError('application configuration restore binding drifted');
}
return candidate.profile;
}
function composeArgs(
paths: Readonly<LocalDeploymentPaths>,
args: readonly string[],
@@ -280,12 +255,17 @@ function stopAndVerifyCompose(
paths: Readonly<LocalDeploymentPaths>,
runDocker: LocalDeploymentDockerRunner,
): void {
docker(
command,
runDocker,
composeArgs(paths, ['stop', '--timeout', '30', 'qinglong3']),
45_000,
);
try {
docker(
command,
runDocker,
composeArgs(paths, ['stop', '--timeout', '30', 'qinglong3']),
45_000,
);
} catch {
// A lost Docker response is reconciled by the inspect-only observation
// below. The restore ceremony never repeats `compose stop` blindly.
}
const containerId = docker(
command,
runDocker,
@@ -418,6 +398,7 @@ async function readRestoreState(
command: Readonly<LocalDeploymentComposeRestorePrepareCommand>,
paths: Readonly<LocalDeploymentPaths>,
uid: number,
lineage: Readonly<LocalDeploymentComposeLineage>,
): Promise<Readonly<RestoreState>> {
const failedIntent = boundedReceipt(
paths.composeRolloutLock,
@@ -478,7 +459,7 @@ async function readRestoreState(
) {
configurationError('compose restore recovery generation drifted');
}
const profile = readApplicationProfile(paths, uid);
const profile = lineage.profile;
const sourcePath = path.join(
paths.composeRolloutBackups,
`${command.request.sourceRolloutId}.sqlite`,
@@ -503,6 +484,7 @@ async function inspectPrepareReceipt(
expectedCommandDigest: string,
command: Readonly<LocalDeploymentComposeRestorePrepareCommand>,
paths: Readonly<LocalDeploymentPaths>,
expectedLineage: Readonly<LocalDeploymentComposeLineage>,
): Promise<Readonly<RestorePrepareReceipt> | null> {
if (!fs.existsSync(filePath)) return null;
const contents = boundedReceipt(
@@ -518,6 +500,9 @@ async function inspectPrepareReceipt(
configurationError('compose restore prepare receipt is invalid', error);
}
const receipt = value as RestorePrepareReceipt;
const lineage = normalizeLocalDeploymentComposeLineageReceipt(
receipt.lineage,
);
if (
!receipt ||
typeof receipt !== 'object' ||
@@ -528,6 +513,7 @@ async function inspectPrepareReceipt(
'current',
'failedApplyCommandDigest',
'generation',
'lineage',
'profile',
'recordedAtMs',
'restoreId',
@@ -550,10 +536,12 @@ async function inspectPrepareReceipt(
!snapshotIsValid(receipt.source) ||
!snapshotIsValid(receipt.safeguard) ||
receipt.current.sha256 === receipt.source.sha256 ||
JSON.stringify(receipt.lineage) !== JSON.stringify(lineage) ||
contents !== receiptContents(receipt)
) {
configurationError('compose restore prepare receipt drifted');
}
assertLocalDeploymentComposeLineageReceipt(lineage, expectedLineage.receipt);
const source = await inspectLocalSqliteSnapshot({
databasePath: path.join(
paths.composeRolloutBackups,
@@ -586,6 +574,7 @@ function inspectCommitReceipt(
uid: number,
expectedCommandDigest: string,
command: Readonly<LocalDeploymentComposeRestoreCommand>,
expectedLineage: Readonly<LocalDeploymentComposeLineage>,
): Readonly<RestoreCommitReceipt> | null {
if (!fs.existsSync(filePath)) return null;
const contents = boundedReceipt(
@@ -601,6 +590,9 @@ function inspectCommitReceipt(
configurationError('compose restore commit receipt is invalid', error);
}
const receipt = value as RestoreCommitReceipt;
const lineage = normalizeLocalDeploymentComposeLineageReceipt(
receipt.lineage,
);
if (
!receipt ||
typeof receipt !== 'object' ||
@@ -609,6 +601,7 @@ function inspectCommitReceipt(
[
'commandDigest',
'generation',
'lineage',
'prepareReceiptDigest',
'profile',
'recordedAtMs',
@@ -636,10 +629,12 @@ function inspectCommitReceipt(
!snapshotIsValid(receipt.safeguard) ||
!snapshotIsValid(receipt.restored) ||
receipt.source.sha256 !== receipt.restored.sha256 ||
JSON.stringify(receipt.lineage) !== JSON.stringify(lineage) ||
contents !== receiptContents(receipt)
) {
configurationError('compose restore commit receipt drifted');
}
assertLocalDeploymentComposeLineageReceipt(lineage, expectedLineage.receipt);
publishExactFile(
filePath,
contents,
@@ -671,6 +666,7 @@ async function prepareRestore(
command: Readonly<LocalDeploymentComposeRestorePrepareCommand>,
paths: Readonly<LocalDeploymentPaths>,
dependencies: LocalDeploymentComposeRestoreDependencies,
lineage: Readonly<LocalDeploymentComposeLineage>,
): Promise<Readonly<LocalDeploymentComposeRestoreResult>> {
const identity = currentIdentity();
if (fs.existsSync(commitReceiptPath(paths, command.request.restoreId))) {
@@ -680,7 +676,7 @@ async function prepareRestore(
dependencies.validateSocket ?? validateLocalDeploymentDockerSocket;
validateSocket(command.options.dockerSocketPath, identity.uid);
const runDocker = dependencies.runDocker ?? runLocalDeploymentDockerCommand;
const state = await readRestoreState(command, paths, identity.uid);
const state = await readRestoreState(command, paths, identity.uid, lineage);
const intent = `${JSON.stringify(command, null, 2)}\n`;
publishExactFile(
paths.composeRestoreLock,
@@ -701,10 +697,11 @@ async function prepareRestore(
commandDigest(command),
command,
paths,
lineage,
);
if (replay) {
const current = await inspectLocalSqliteSnapshot({
databasePath: paths.database,
databasePath: lineage.databasePath,
profile: replay.profile,
});
exactSnapshot(replay.current, current, 'compose restore current database');
@@ -717,7 +714,7 @@ async function prepareRestore(
}
const checkpoint = dependencies.checkpoint ?? checkpointLocalSqliteForRestore;
const current = await checkpoint({
databasePath: paths.database,
databasePath: lineage.databasePath,
profile: state.profile,
});
if (current.sha256 === state.source.sha256) {
@@ -728,7 +725,7 @@ async function prepareRestore(
const createSafeguard =
dependencies.createSafeguard ?? createLocalSqliteRolloutBackup;
const safeguard = await createSafeguard({
databasePath: paths.database,
databasePath: lineage.databasePath,
backupPath: safeguardPath(paths, command.request.restoreId),
profile: state.profile,
});
@@ -741,6 +738,7 @@ async function prepareRestore(
generation: command.request.expectedGeneration,
recordedAtMs: command.request.preparedAtMs,
profile: state.profile,
lineage: lineage.receipt,
current: snapshotReceipt(current),
source: snapshotReceipt(state.source),
safeguard: snapshotReceipt(safeguard),
@@ -764,6 +762,7 @@ async function commitRestore(
command: Readonly<LocalDeploymentComposeRestoreCommand>,
paths: Readonly<LocalDeploymentPaths>,
dependencies: LocalDeploymentComposeRestoreDependencies,
lineage: Readonly<LocalDeploymentComposeLineage>,
): Promise<Readonly<LocalDeploymentComposeRestoreResult>> {
if (command.operation !== 'local.deployment.compose.restore.commit') {
configurationError('compose restore commit command is invalid');
@@ -775,6 +774,7 @@ async function commitRestore(
identity.uid,
commandDigest(command),
command,
lineage,
);
if (replay) {
const sourcePath = path.join(
@@ -945,11 +945,17 @@ async function commitRestore(
commandDigest(prepareCommand),
prepareCommand,
paths,
lineage,
);
if (!prepareReceipt) {
configurationError('compose restore prepare receipt is unavailable');
}
const state = await readRestoreState(prepareCommand, paths, identity.uid);
const state = await readRestoreState(
prepareCommand,
paths,
identity.uid,
lineage,
);
if (
state.failedCommandDigest !== prepareReceipt.failedApplyCommandDigest ||
state.source.sha256 !== prepareReceipt.source.sha256
@@ -961,11 +967,11 @@ async function commitRestore(
validateSocket(command.options.dockerSocketPath, identity.uid);
const runDocker = dependencies.runDocker ?? runLocalDeploymentDockerCommand;
stopAndVerifyCompose(command, paths, runDocker);
if (fs.existsSync(paths.database)) {
if (fs.existsSync(lineage.databasePath)) {
const checkpoint =
dependencies.checkpoint ?? checkpointLocalSqliteForRestore;
const current = await checkpoint({
databasePath: paths.database,
databasePath: lineage.databasePath,
profile: prepareReceipt.profile,
});
if (
@@ -980,12 +986,13 @@ async function commitRestore(
const restoreSnapshot =
dependencies.restoreSnapshot ?? restoreLocalSqliteSnapshot;
const restored: Readonly<LocalSqliteRestoreEvidence> = await restoreSnapshot({
databasePath: paths.database,
databasePath: lineage.databasePath,
profile: prepareReceipt.profile,
preserveDatabaseIdentity: lineage.mode === 'adopted',
sourceSnapshotPath: state.sourcePath,
restoreStagePath: path.join(
path.dirname(paths.database),
`.${path.basename(paths.database)}.${
path.dirname(lineage.databasePath),
`.${path.basename(lineage.databasePath)}.${
command.request.restoreId
}.restore-stage`,
),
@@ -1011,6 +1018,7 @@ async function commitRestore(
generation: command.request.expectedGeneration,
recordedAtMs: command.request.committedAtMs,
profile: prepareReceipt.profile,
lineage: lineage.receipt,
source: prepareReceipt.source,
safeguard: prepareReceipt.safeguard,
restored: snapshotReceipt(restored),
@@ -1073,13 +1081,19 @@ export async function restoreLocalDeploymentCompose(
identity.uid,
'composeRestoreSafeguardRoot',
);
const lineage = inspectLocalDeploymentComposeLineage(
command.options.deploymentRoot,
identity.uid,
identity.gid,
command.options.allowRootService,
);
if (fs.existsSync(paths.composeEvidenceCollectionLock)) {
configurationError('compose restore is fenced by an evidence collection');
}
preflightRestoreCatalog(paths, command.request.restoreId);
return command.operation === 'local.deployment.compose.restore.prepare'
? prepareRestore(command, paths, dependencies)
: commitRestore(command, paths, dependencies);
? prepareRestore(command, paths, dependencies, lineage)
: commitRestore(command, paths, dependencies, lineage);
}
export function restoreLocalDeploymentComposeCommandFile(
packages/ql3-local-owner-cli/test/adoptedDeploymentBundle.test.cjs
+603 -3
View File
@@ -4,16 +4,26 @@ const crypto = require('node:crypto');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { DatabaseSync } = require('node:sqlite');
const { test } = require('node:test');
const {
LocalDeploymentConfigurationError,
applyLocalDeploymentCompose,
collectLocalDeploymentComposeEvidence,
prepareLocalDeploymentAdoptedBundle,
preflightLocalDeploymentCompose,
restoreLocalDeploymentCompose,
switchLocalDeploymentComposeRevision,
verifyLocalDeploymentAdoptedBundle,
} = require('../dist/deployment/localDeployment.js');
const {
createLocalDataDirectoryApplicationCommit,
} = require('@qinglong/local-sqlite/data-directory-application-commit');
const { migrateLocalSqlitePath } = require('@qinglong/local-sqlite/migration');
const {
createLocalSqliteRolloutBackup,
} = require('@qinglong/local-sqlite/rollout-safety');
const {
normalizeLocalApplicationProcessConfig,
} = require('../../ql3-local-application/dist/production-process/processConfig.js');
@@ -42,8 +52,8 @@ function writePrivate(filePath, value) {
);
}
function releaseSelection(managementRoot) {
const image = `ghcr.io/example/qinglong3-local-application@sha256:${'a'.repeat(
function releaseSelection(managementRoot, marker = 'a') {
const image = `ghcr.io/example/qinglong3-local-application@sha256:${marker.repeat(
64,
)}`;
const releaseSetDigest = prefixedDigest(`release-set:${image}`);
@@ -86,7 +96,10 @@ function releaseSelection(managementRoot) {
},
};
const selectionDigest = prefixedDigest(JSON.stringify(unsigned));
const filePath = path.join(managementRoot, 'release-selection.json');
const filePath = path.join(
managementRoot,
`release-selection-${marker}.json`,
);
writePrivate(filePath, { ...unsigned, selectionDigest });
return {
path: filePath,
@@ -247,13 +260,373 @@ function fixture(t, kind) {
};
return {
root,
managementRoot,
command,
commitPath,
commitmentPath,
sourcePath,
targetPath,
recoveryPath,
manifestPath,
activationPath,
};
}
async function prepareRuntimeComposeFixture(t) {
const state = fixture(t, 'compose');
fs.unlinkSync(state.targetPath);
await migrateLocalSqlitePath({
databasePath: state.targetPath,
profile: 'edge',
busyTimeoutMs: 100,
});
const target = fs.statSync(state.targetPath, { bigint: true });
const activation = JSON.parse(fs.readFileSync(state.activationPath, 'utf8'));
delete activation.activationDigest;
activation.targetSha256 = hexDigest(fs.readFileSync(state.targetPath));
activation.targetDevice = target.dev.toString();
activation.targetInode = target.ino.toString();
const activationDigest = canonicalDigest(activation);
writePrivate(state.activationPath, { ...activation, activationDigest });
const commitment = JSON.parse(fs.readFileSync(state.commitmentPath, 'utf8'));
delete commitment.commitmentDigest;
commitment.activationDigest = activationDigest;
const commitmentDigest = canonicalDigest(commitment);
writePrivate(state.commitmentPath, { ...commitment, commitmentDigest });
state.command.request.storage.expectedActivationDigest = activationDigest;
state.command.request.cutover.expectedCommitmentDigest = commitmentDigest;
prepareLocalDeploymentAdoptedBundle(state.command);
return state;
}
function selectedComposeGeneration(state) {
const contents = fs.readFileSync(
path.join(state.root, 'service/compose.image.yaml'),
'utf8',
);
return {
generation: Number(/^ generation: ([0-9]+)$/m.exec(contents)[1]),
mutationId: /^ mutation_id: ([0-9a-f-]+)$/m.exec(contents)[1],
image: /^ image: ([^\n]+)$/m.exec(contents)[1],
releaseSelectionDigest: /^ release_selection_digest: ([^\n]+)$/m.exec(
contents,
)[1],
releaseSetDigest: /^ release_set_digest: ([^\n]+)$/m.exec(contents)[1],
catalogManifestDigest: /^ catalog_manifest_digest: ([^\n]+)$/m.exec(
contents,
)[1],
catalogConsumptionReportDigest:
/^ catalog_consumption_report_digest: ([^\n]+)$/m.exec(contents)[1],
};
}
function adoptedDockerHarness(state, options = {}) {
const calls = [];
const containerId = '1'.repeat(64);
const bundle = JSON.parse(
fs.readFileSync(path.join(state.root, 'service/adopted-bundle.json')),
);
const compose = fs.readFileSync(
path.join(state.root, 'service/compose.yaml'),
'utf8',
);
const projectName = /^name: ([a-z0-9_-]+)$/m.exec(compose)[1];
let running = false;
let loseNextUpResponse = options.loseNextUpResponse === true;
const labels = (selected) => ({
'io.qinglong.deployment.mode': 'adopted',
'io.qinglong.deployment.profile': 'edge',
'io.qinglong.deployment.instance': 'edge-router-1',
'io.qinglong.deployment.bundle': bundle.bundleId,
'io.qinglong.application.config': bundle.applicationConfigDigest,
'io.qinglong.data.commit': bundle.legacyDataApplicationCommitDigest,
'io.qinglong.data.receipt': bundle.legacyDataApplicationReceiptDigest,
'io.qinglong.deployment.generation': String(selected.generation),
'io.qinglong.deployment.mutation': selected.mutationId,
'io.qinglong.release.selection': selected.releaseSelectionDigest,
'io.qinglong.release.set': selected.releaseSetDigest,
'io.qinglong.release.catalog-manifest': selected.catalogManifestDigest,
'io.qinglong.release.catalog-report':
selected.catalogConsumptionReportDigest,
});
const runDocker = ({ args }) => {
calls.push(args);
const selected = selectedComposeGeneration(state);
if (args[0] === 'image') {
return JSON.stringify([
{
Id: `sha256:${'2'.repeat(64)}`,
RepoDigests: [selected.image],
Architecture: 'arm64',
Os: 'linux',
Config: {
User: '65532:65532',
Entrypoint: [
'node',
'/opt/qinglong/node_modules/@qinglong/local-application/dist/cli.js',
],
Labels: {
'io.qinglong.local.sqlite-contract-min': '50',
'io.qinglong.local.sqlite-contract-max': '50',
'io.qinglong.local.sqlite-write-contract': '50',
'io.qinglong.local.application-config': '2,3,4',
'io.qinglong.local.compose-selection': '1',
'io.qinglong.ai': 'excluded',
'io.qinglong.profile': 'edge,standalone',
'org.opencontainers.image.source':
'https://github.com/whyour/qinglong',
'org.opencontainers.image.revision': '3'.repeat(40),
'org.opencontainers.image.version': '3.0.0-alpha.0',
},
},
},
]);
}
if (args[0] === 'compose' && args.includes('config')) {
const volumes = [
{ type: 'bind', source: state.root, target: state.root },
{
type: 'bind',
source: state.sourcePath,
target: state.sourcePath,
read_only: true,
},
];
if (options.driftMount === true) volumes[0].target = '/var/lib/qinglong3';
return JSON.stringify({
name: projectName,
services: {
qinglong3: {
image: selected.image,
user: `${process.getuid()}:${process.getgid()}`,
read_only: true,
network_mode: 'none',
restart: 'no',
mem_limit: 128 * 1024 * 1024,
pids_limit: 64,
cap_drop: ['ALL'],
security_opt: ['no-new-privileges:true'],
command: [
'--config',
path.join(state.root, 'local-application.json'),
],
labels: labels(selected),
volumes,
tmpfs: ['/tmp:rw,noexec,nosuid,nodev,size=16m'],
},
},
});
}
if (args[0] === 'compose' && args.includes('up')) {
running = true;
if (loseNextUpResponse) {
loseNextUpResponse = false;
throw new Error('injected Docker response loss');
}
return '';
}
if (args[0] === 'compose' && args.includes('stop')) {
running = false;
return '';
}
if (args[0] === 'compose' && args.includes('ps')) return `${containerId}\n`;
if (args[0] === 'container' && args[1] === 'inspect') {
return JSON.stringify([
{
Id: containerId,
State: { Running: running, Status: running ? 'running' : 'exited' },
Config: { Image: selected.image, Labels: labels(selected) },
HostConfig: {
ReadonlyRootfs: true,
NetworkMode: 'none',
Privileged: false,
},
},
]);
}
if (args[0] === 'container' && args[1] === 'logs') {
return `${JSON.stringify({
schemaVersion: 1,
component: 'qinglong3-local-application',
level: 'info',
event: 'active',
profile: 'edge',
aiStatus: 'deployment_excluded',
instanceId: 'edge-router-1',
})}\n`;
}
return '';
};
return {
calls,
runDocker,
now: () => 10_000,
wait: async () => {},
};
}
function composeApplyCommand(state, generation, suffix) {
return {
schemaVersion: 1,
operation: 'local.deployment.compose.apply',
options: {
deploymentRoot: state.root,
dockerExecutable: fs.realpathSync(process.execPath),
dockerSocketPath: path.join(state.managementRoot, 'docker.sock'),
allowRootService: rootAcknowledgement(),
},
request: {
expectedGeneration: generation,
rolloutId: `00000000-0000-4000-8000-000000000d${suffix}`,
startedAtMs: 1786416000500,
failureRollbackMutationId: `00000000-0000-4000-8000-000000000e${suffix}`,
failureRollbackChangedAtMs: 1786416000501,
},
};
}
function composePreflightCommand(state, generation) {
return {
schemaVersion: 1,
operation: 'local.deployment.compose.preflight',
options: {
deploymentRoot: state.root,
dockerExecutable: fs.realpathSync(process.execPath),
dockerSocketPath: path.join(state.managementRoot, 'docker.sock'),
allowRootService: rootAcknowledgement(),
},
request: { expectedGeneration: generation },
};
}
function composeRevisionCommand(state, operation, request) {
return {
schemaVersion: 1,
operation,
options: {
deploymentRoot: state.root,
allowRootService: rootAcknowledgement(),
},
request,
};
}
function composeRestoreCommands(state, failedCommand, suffix) {
const options = {
deploymentRoot: state.root,
dockerExecutable: fs.realpathSync(process.execPath),
dockerSocketPath: path.join(state.managementRoot, 'docker.sock'),
allowRootService: rootAcknowledgement(),
};
const restoreId = `00000000-0000-4000-8000-000000000d${suffix}`;
return {
prepare: {
schemaVersion: 1,
operation: 'local.deployment.compose.restore.prepare',
options,
request: {
expectedGeneration: failedCommand.request.expectedGeneration + 1,
restoreId,
sourceRolloutId: failedCommand.request.rolloutId,
preparedAtMs: 1786416000600,
},
},
commit: {
schemaVersion: 1,
operation: 'local.deployment.compose.restore.commit',
options,
request: {
expectedGeneration: failedCommand.request.expectedGeneration + 1,
restoreId,
committedAtMs: 1786416000601,
},
},
};
}
function composeEvidenceCollectionCommands(
state,
generation,
restoreId,
suffix,
) {
const options = {
deploymentRoot: state.root,
allowRootService: rootAcknowledgement(),
};
const collectionId = `00000000-0000-4000-8000-000000000d${suffix}`;
return {
prepare: {
schemaVersion: 1,
operation: 'local.deployment.compose.evidence-collection.prepare',
options,
request: {
expectedGeneration: generation,
collectionId,
rolloutIds: [],
restoreIds: [restoreId],
preparedAtMs: 1786416000700,
},
},
commit: {
schemaVersion: 1,
operation: 'local.deployment.compose.evidence-collection.commit',
options,
request: {
expectedGeneration: generation,
collectionId,
committedAtMs: 1786416000701,
},
},
};
}
async function createFailedRestoreState(state, currentGeneration, suffix) {
const attemptedGeneration = currentGeneration + 1;
const upgradeMutationId = `00000000-0000-4000-8000-000000000a${suffix}`;
await switchLocalDeploymentComposeRevision(
composeRevisionCommand(state, 'local.deployment.compose.upgrade', {
expectedGeneration: currentGeneration,
releaseSelection: releaseSelection(
state.managementRoot,
String(attemptedGeneration),
),
mutationId: upgradeMutationId,
changedAtMs: 1786416000400 + attemptedGeneration,
}),
);
const failedCommand = composeApplyCommand(state, attemptedGeneration, suffix);
const intent = `${JSON.stringify(failedCommand, null, 2)}\n`;
fs.writeFileSync(
path.join(state.root, 'service/.compose-rollout.lock'),
intent,
{ mode: 0o600 },
);
await createLocalSqliteRolloutBackup({
databasePath: state.targetPath,
backupPath: path.join(
state.root,
'service',
'rollout-backups',
`${failedCommand.request.rolloutId}.sqlite`,
),
profile: 'edge',
});
const writer = new DatabaseSync(state.targetPath);
writer.exec(`PRAGMA user_version = ${900 + attemptedGeneration}`);
writer.close();
await switchLocalDeploymentComposeRevision(
composeRevisionCommand(state, 'local.deployment.compose.rollback', {
expectedGeneration: attemptedGeneration,
targetGeneration: currentGeneration,
mutationId: failedCommand.request.failureRollbackMutationId,
changedAtMs: failedCommand.request.failureRollbackChangedAtMs,
}),
intent,
);
return failedCommand;
}
for (const kind of ['systemd', 'openrc', 'compose']) {
test(`prepares and verifies an exact adopted ${kind} bundle without activation`, (t) => {
const state = fixture(t, kind);
@@ -406,6 +779,233 @@ test('requires the legacy source to be the only authority outside deployment roo
);
});
test('preflights adopted Compose identity mounts and rejects mount drift', async (t) => {
const state = await prepareRuntimeComposeFixture(t);
const harness = adoptedDockerHarness(state);
const ready = await preflightLocalDeploymentCompose(
composePreflightCommand(state, 1),
{
runDocker: harness.runDocker,
validateSocket() {},
},
);
assert.equal(ready.status, 'ready');
assert.equal(ready.profile, 'edge');
assert.equal(ready.sqlite.contractVersion, 50);
await assert.rejects(
preflightLocalDeploymentCompose(composePreflightCommand(state, 1), {
runDocker: adoptedDockerHarness(state, { driftMount: true }).runDocker,
validateSocket() {},
}),
LocalDeploymentConfigurationError,
);
});
const dockerComposeAvailable =
spawnSync('docker', ['compose', 'version'], { encoding: 'utf8' }).status ===
0;
test(
'accepts the real Docker Compose adopted config projection',
{ skip: !dockerComposeAvailable },
async (t) => {
const state = await prepareRuntimeComposeFixture(t);
const harness = adoptedDockerHarness(state);
const runDocker = ({ args, ...request }) => {
if (args[0] === 'compose' && args.includes('config')) {
const result = spawnSync('docker', args, { encoding: 'utf8' });
assert.equal(result.status, 0, result.stderr);
return result.stdout;
}
return harness.runDocker({ args, ...request });
};
const ready = await preflightLocalDeploymentCompose(
composePreflightCommand(state, 1),
{ runDocker, validateSocket() {} },
);
assert.equal(ready.status, 'ready');
},
);
test('recovers adopted Compose up response loss and binds the rollout receipt', async (t) => {
const state = await prepareRuntimeComposeFixture(t);
const harness = adoptedDockerHarness(state, { loseNextUpResponse: true });
const command = composeApplyCommand(state, 1, '89');
const dependencies = {
runDocker: harness.runDocker,
validateSocket() {},
now: harness.now,
wait: harness.wait,
};
const applied = await applyLocalDeploymentCompose(command, dependencies);
assert.equal(applied.status, 'active');
assert.equal(
harness.calls.filter((args) => args[0] === 'compose' && args.includes('up'))
.length,
1,
);
const receipt = JSON.parse(
fs.readFileSync(
path.join(
state.root,
'service',
'rollouts',
`${command.request.rolloutId}.json`,
),
'utf8',
),
);
assert.equal(receipt.schema, 'qinglong/local-compose-rollout-receipt@v3');
assert.equal(receipt.lineage.mode, 'adopted');
assert.equal(
receipt.lineage.legacyDataApplicationCommitDigest,
state.command.request.legacyDataApplication.expectedCommitDigest,
);
assert.equal(
receipt.lineage.legacyDataApplicationReceiptDigest,
state.command.request.legacyDataApplication.expectedReceiptDigest,
);
const replay = await applyLocalDeploymentCompose(command, dependencies);
assert.equal(replay.status, 'active');
assert.equal(
harness.calls.filter((args) => args[0] === 'compose' && args.includes('up'))
.length,
1,
);
const commitment = JSON.parse(fs.readFileSync(state.commitmentPath, 'utf8'));
writePrivate(state.commitmentPath, {
...commitment,
observedAtMs: commitment.observedAtMs + 1,
});
await assert.rejects(
applyLocalDeploymentCompose(command, dependencies),
LocalDeploymentConfigurationError,
);
assert.equal(
harness.calls.filter((args) => args[0] === 'compose' && args.includes('up'))
.length,
1,
);
});
test('preserves adopted restore identity and carries lineage through evidence collection', async (t) => {
const state = await prepareRuntimeComposeFixture(t);
const activated = fs.statSync(state.targetPath, { bigint: true });
const failedCommand = await createFailedRestoreState(state, 1, '90');
const restore = composeRestoreCommands(state, failedCommand, '91');
const harness = adoptedDockerHarness(state);
const dependencies = {
runDocker: harness.runDocker,
validateSocket() {},
};
const prepared = await restoreLocalDeploymentCompose(
restore.prepare,
dependencies,
);
assert.equal(prepared.status, 'prepared');
const committed = await restoreLocalDeploymentCompose(
restore.commit,
dependencies,
);
assert.equal(committed.status, 'restored');
const restored = fs.statSync(state.targetPath, { bigint: true });
assert.equal(restored.dev, activated.dev);
assert.equal(restored.ino, activated.ino);
const commitPath = path.join(
state.root,
'service',
'restores',
`${restore.commit.request.restoreId}.commit.json`,
);
const restoreReceipt = JSON.parse(fs.readFileSync(commitPath, 'utf8'));
assert.equal(
restoreReceipt.schema,
'qinglong/local-compose-restore-commit@v2',
);
assert.equal(restoreReceipt.lineage.mode, 'adopted');
assert.equal(
restoreReceipt.lineage.legacyDataApplicationReceiptDigest,
state.command.request.legacyDataApplication.expectedReceiptDigest,
);
const applyHarness = adoptedDockerHarness(state);
await applyLocalDeploymentCompose(failedCommand, {
runDocker: applyHarness.runDocker,
validateSocket() {},
now: applyHarness.now,
wait: applyHarness.wait,
});
const secondRestoreId = '00000000-0000-4000-8000-000000000d92';
const secondSafeguard = await createLocalSqliteRolloutBackup({
databasePath: state.targetPath,
backupPath: path.join(
state.root,
'service',
'restore-safeguards',
`${secondRestoreId}.sqlite`,
),
profile: 'edge',
});
const secondReceipt = {
...restoreReceipt,
commandDigest: 'd'.repeat(64),
restoreId: secondRestoreId,
recordedAtMs: restoreReceipt.recordedAtMs + 1,
safeguard: {
contractVersion: secondSafeguard.contractVersion,
sha256: secondSafeguard.sha256,
bytes: secondSafeguard.bytes,
pageCount: secondSafeguard.pageCount,
pageSize: secondSafeguard.pageSize,
},
};
writePrivate(
path.join(
state.root,
'service',
'restores',
`${secondRestoreId}.commit.json`,
),
`${JSON.stringify(secondReceipt, null, 2)}\n`,
);
const collection = composeEvidenceCollectionCommands(
state,
3,
restore.commit.request.restoreId,
'93',
);
const collectionPrepared = await collectLocalDeploymentComposeEvidence(
collection.prepare,
);
assert.equal(collectionPrepared.status, 'prepared');
const collectionCommitted = await collectLocalDeploymentComposeEvidence(
collection.commit,
);
assert.equal(collectionCommitted.status, 'collected');
const collectionReceipt = JSON.parse(
fs.readFileSync(
path.join(
state.root,
'service',
'evidence-collections',
`${collection.commit.request.collectionId}.commit.json`,
),
'utf8',
),
);
assert.equal(
collectionReceipt.schema,
'qinglong/local-compose-evidence-collection-commit@v2',
);
assert.deepEqual(collectionReceipt.lineage, restoreReceipt.lineage);
const replay = await restoreLocalDeploymentCompose(
restore.commit,
dependencies,
);
assert.equal(replay.status, 'existing');
assert.equal(replay.sqlite.safeguard, 'collected');
});
test('exposes separate exact prepare and verify CLI operations', (t) => {
const state = fixture(t, 'systemd');
const cli = path.resolve(
packages/ql3-local-owner-cli/test/localDeployment.test.cjs
+33
View File
@@ -450,6 +450,15 @@ function composeDockerHarness(
generation: Number(/^ generation: ([0-9]+)$/m.exec(source)[1]),
mutationId: /^ mutation_id: ([0-9a-f-]+)$/m.exec(source)[1],
image: /^ image: ([^\n]+)$/m.exec(source)[1],
releaseSelectionDigest: /^ release_selection_digest: ([^\n]+)$/m.exec(
source,
)[1],
releaseSetDigest: /^ release_set_digest: ([^\n]+)$/m.exec(source)[1],
catalogManifestDigest: /^ catalog_manifest_digest: ([^\n]+)$/m.exec(
source,
)[1],
catalogConsumptionReportDigest:
/^ catalog_consumption_report_digest: ([^\n]+)$/m.exec(source)[1],
};
};
const composeSource = fs.readFileSync(
@@ -457,6 +466,12 @@ function composeDockerHarness(
'utf8',
);
const projectName = /^name: ([a-z0-9_-]+)$/m.exec(composeSource)[1];
const release = JSON.parse(
fs.readFileSync(
state.command.options.service.releaseSelection.path,
'utf8',
),
);
let running = true;
const runDocker = ({ args }) => {
calls.push(args);
@@ -509,6 +524,12 @@ function composeDockerHarness(
labels: {
'io.qinglong.deployment.generation': String(selected.generation),
'io.qinglong.deployment.mutation': selected.mutationId,
'io.qinglong.release.selection': selected.releaseSelectionDigest,
'io.qinglong.release.set': selected.releaseSetDigest,
'io.qinglong.release.catalog-manifest':
selected.catalogManifestDigest,
'io.qinglong.release.catalog-report':
selected.catalogConsumptionReportDigest,
},
volumes: [
{
@@ -1137,6 +1158,12 @@ test('preflights exact local image, Compose merge and SQLite capability', async
await prepareLocalDeployment(state.command);
const dockerSocketPath = path.join(state.managementRoot, 'docker.sock');
const image = state.composeImage;
const release = JSON.parse(
fs.readFileSync(
state.command.options.service.releaseSelection.path,
'utf8',
),
);
const composeSource = fs.readFileSync(
path.join(state.deploymentRoot, 'service', 'compose.yaml'),
'utf8',
@@ -1206,6 +1233,12 @@ test('preflights exact local image, Compose merge and SQLite capability', async
'io.qinglong.deployment.generation': '1',
'io.qinglong.deployment.mutation':
state.command.request.activateMutationId,
'io.qinglong.release.selection': release.selectionDigest,
'io.qinglong.release.set': release.releaseSetDigest,
'io.qinglong.release.catalog-manifest':
release.catalog.manifestDigest,
'io.qinglong.release.catalog-report':
release.catalog.consumptionReportDigest,
},
volumes: [
{
packages/ql3-local-sqlite/src/readiness/rolloutSafety.ts
+248
View File
@@ -49,6 +49,7 @@ export interface LocalSqliteRestoreOptions extends LocalSqliteDatabaseOptions {
readonly replacedDatabasePath: string;
readonly expectedCurrentSha256: string;
readonly expectedSourceSha256: string;
readonly preserveDatabaseIdentity?: boolean;
}
export interface LocalSqliteRestoreEvidence
@@ -58,6 +59,7 @@ export interface LocalSqliteRestoreEvidence
export interface LocalSqliteRestoreDependencies {
readonly copySnapshot?: (sourcePath: string, targetPath: string) => void;
readonly rewriteSnapshot?: (sourcePath: string, targetPath: string) => void;
}
export interface LocalSqliteChangeObserver {
@@ -582,6 +584,239 @@ function sameSnapshot(
);
}
function rewriteSnapshotFilePreservingIdentity(
sourcePath: string,
targetPath: string,
): void {
const targetIdentity = fs.lstatSync(targetPath, { bigint: true });
let sourceDescriptor: number | undefined;
let targetDescriptor: number | undefined;
const buffer = Buffer.allocUnsafe(64 * 1024);
try {
sourceDescriptor = fs.openSync(
sourcePath,
fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW,
);
targetDescriptor = fs.openSync(
targetPath,
fs.constants.O_WRONLY | fs.constants.O_TRUNC | fs.constants.O_NOFOLLOW,
);
const openedTarget = fs.fstatSync(targetDescriptor, { bigint: true });
if (
!openedTarget.isFile() ||
openedTarget.dev !== targetIdentity.dev ||
openedTarget.ino !== targetIdentity.ino ||
openedTarget.nlink !== 1n
) {
configurationError('identity-preserving restore target drifted');
}
for (;;) {
const count = fs.readSync(
sourceDescriptor,
buffer,
0,
buffer.byteLength,
null,
);
if (count === 0) break;
let offset = 0;
while (offset < count) {
const written = fs.writeSync(
targetDescriptor,
buffer,
offset,
count - offset,
null,
);
if (written < 1) {
configurationError('identity-preserving restore write stalled');
}
offset += written;
}
}
fs.fsyncSync(targetDescriptor);
const writtenTarget = fs.fstatSync(targetDescriptor, { bigint: true });
if (
writtenTarget.dev !== targetIdentity.dev ||
writtenTarget.ino !== targetIdentity.ino ||
writtenTarget.nlink !== 1n
) {
configurationError('identity-preserving restore target changed');
}
} finally {
buffer.fill(0);
if (targetDescriptor !== undefined) fs.closeSync(targetDescriptor);
if (sourceDescriptor !== undefined) fs.closeSync(sourceDescriptor);
}
}
async function restoreSnapshotPreservingIdentity(
options: Readonly<LocalSqliteRestoreOptions>,
dependencies: LocalSqliteRestoreDependencies,
input: Readonly<{
uid: number;
databasePath: string;
sourceSnapshotPath: string;
restoreStagePath: string;
replacedDatabasePath: string;
source: Readonly<LocalSqliteSnapshotEvidence>;
}>,
): Promise<Readonly<LocalSqliteRestoreEvidence>> {
if (!fs.existsSync(input.databasePath)) {
configurationError(
'identity-preserving restore requires the activated database inode',
);
}
const targetIdentity = fs.lstatSync(input.databasePath);
if (
!targetIdentity.isFile() ||
targetIdentity.isSymbolicLink() ||
targetIdentity.uid !== input.uid ||
(targetIdentity.mode & 0o777) !== 0o600 ||
targetIdentity.nlink !== 1 ||
fs.realpathSync(input.databasePath) !== input.databasePath
) {
configurationError('identity-preserving restore database is invalid');
}
let current: Readonly<LocalSqliteSnapshotEvidence> | undefined;
try {
current = await inspectSnapshotFile(
input.databasePath,
options.profile,
input.uid,
'restore current database',
);
} catch (error) {
if (
!fs.existsSync(input.restoreStagePath) ||
!fs.existsSync(input.replacedDatabasePath)
) {
throw error;
}
}
const restoredAtEntry =
current !== undefined && sameSnapshot(current, input.source);
if (
current !== undefined &&
!restoredAtEntry &&
current.sha256 !== options.expectedCurrentSha256
) {
configurationError('restore current database drifted');
}
if (!fs.existsSync(input.restoreStagePath) && !restoredAtEntry) {
try {
(
dependencies.copySnapshot ??
((sourcePath: string, targetPath: string) =>
fs.copyFileSync(sourcePath, targetPath, fs.constants.COPYFILE_EXCL))
)(input.sourceSnapshotPath, input.restoreStagePath);
fs.chmodSync(input.restoreStagePath, 0o600);
syncFile(input.restoreStagePath);
syncDirectory(path.dirname(input.restoreStagePath));
} catch (error) {
if (fs.existsSync(input.restoreStagePath)) {
try {
fs.unlinkSync(input.restoreStagePath);
syncDirectory(path.dirname(input.restoreStagePath));
} catch {
// A deterministic stage remains fail-closed for exact replay.
}
}
configurationError('restore stage could not be created', error);
}
}
if (fs.existsSync(input.restoreStagePath)) {
const staged = await inspectSnapshotFile(
input.restoreStagePath,
options.profile,
input.uid,
'restore stage',
);
if (!sameSnapshot(staged, input.source)) {
configurationError('restore stage drifted');
}
}
if (!restoredAtEntry && !fs.existsSync(input.replacedDatabasePath)) {
if (current === undefined) {
configurationError('restore replacement evidence is unavailable');
}
try {
fs.copyFileSync(
input.databasePath,
input.replacedDatabasePath,
fs.constants.COPYFILE_EXCL,
);
fs.chmodSync(input.replacedDatabasePath, 0o600);
syncFile(input.replacedDatabasePath);
syncDirectory(path.dirname(input.replacedDatabasePath));
} catch (error) {
configurationError(
'restore replacement evidence cannot be created',
error,
);
}
}
if (fs.existsSync(input.replacedDatabasePath)) {
const replaced = await inspectSnapshotFile(
input.replacedDatabasePath,
options.profile,
input.uid,
'replaced database',
);
if (replaced.sha256 !== options.expectedCurrentSha256) {
configurationError('replaced database evidence drifted');
}
}
if (!restoredAtEntry) {
try {
(dependencies.rewriteSnapshot ?? rewriteSnapshotFilePreservingIdentity)(
input.restoreStagePath,
input.databasePath,
);
} catch (error) {
configurationError(
'identity-preserving restore write could not complete',
error,
);
}
}
const afterIdentity = validateDatabaseFile(
input.databasePath,
input.uid,
'identity-preserving restored database',
);
if (
afterIdentity.dev !== targetIdentity.dev ||
afterIdentity.ino !== targetIdentity.ino
) {
configurationError('identity-preserving restore changed the target inode');
}
const restored = await inspectSnapshotFile(
input.databasePath,
options.profile,
input.uid,
'restored database',
);
if (!sameSnapshot(restored, input.source)) {
configurationError('restored database drifted');
}
for (const evidencePath of [
input.restoreStagePath,
input.replacedDatabasePath,
]) {
if (!fs.existsSync(evidencePath)) continue;
fs.unlinkSync(evidencePath);
syncDirectory(path.dirname(evidencePath));
}
return Object.freeze({
status: restoredAtEntry ? ('existing' as const) : ('restored' as const),
...restored,
});
}
export async function restoreLocalSqliteSnapshot(
options: Readonly<LocalSqliteRestoreOptions>,
dependencies: LocalSqliteRestoreDependencies = {},
@@ -608,6 +843,8 @@ export async function restoreLocalSqliteSnapshot(
!DIGEST_PATTERN.test(options.expectedCurrentSha256) ||
!DIGEST_PATTERN.test(options.expectedSourceSha256) ||
options.expectedCurrentSha256 === options.expectedSourceSha256 ||
(options.preserveDatabaseIdentity !== undefined &&
typeof options.preserveDatabaseIdentity !== 'boolean') ||
new Set([
databasePath,
sourceSnapshotPath,
@@ -641,6 +878,17 @@ export async function restoreLocalSqliteSnapshot(
}
assertNoRestoreSidecars(databasePath, uid);
if (options.preserveDatabaseIdentity === true) {
return restoreSnapshotPreservingIdentity(options, dependencies, {
uid,
databasePath,
sourceSnapshotPath,
restoreStagePath,
replacedDatabasePath,
source,
});
}
let restoredAtEntry = false;
if (fs.existsSync(databasePath)) {
const current = await inspectSnapshotFile(
packages/ql3-local-sqlite/test/rolloutSafety.test.cjs
+66
View File
@@ -210,6 +210,72 @@ test('converges the moved-current restore window and cleans ENOSPC stage', async
assert.equal(fs.existsSync(replacedDatabasePath), false);
});
test('preserves an activated inode and recovers an ENOSPC partial rewrite', async (t) => {
const state = fixture(t);
await migrateLocalSqlitePath(state);
const activated = fs.statSync(state.databasePath, { bigint: true });
const source = await createLocalSqliteRolloutBackup(state);
const writer = new DatabaseSync(state.databasePath);
writer.exec('PRAGMA user_version = 29');
writer.close();
const current = await checkpointLocalSqliteForRestore(state);
const restoreStagePath = path.join(state.root, '.identity.restore-stage');
const replacedDatabasePath = path.join(
path.dirname(state.backupPath),
'identity.replaced.sqlite',
);
const restoreOptions = {
databasePath: state.databasePath,
profile: state.profile,
sourceSnapshotPath: state.backupPath,
restoreStagePath,
replacedDatabasePath,
expectedCurrentSha256: current.sha256,
expectedSourceSha256: source.sha256,
preserveDatabaseIdentity: true,
};
await assert.rejects(
restoreLocalSqliteSnapshot(restoreOptions, {
rewriteSnapshot(_sourcePath, targetPath) {
const descriptor = fs.openSync(
targetPath,
fs.constants.O_WRONLY |
fs.constants.O_TRUNC |
fs.constants.O_NOFOLLOW,
);
try {
fs.writeSync(descriptor, Buffer.from('partial'));
fs.fsyncSync(descriptor);
} finally {
fs.closeSync(descriptor);
}
throw Object.assign(new Error('injected identity restore ENOSPC'), {
code: 'ENOSPC',
});
},
}),
/identity-preserving restore write could not complete/,
);
assert.equal(fs.existsSync(restoreStagePath), true);
assert.equal(fs.existsSync(replacedDatabasePath), true);
const partial = fs.statSync(state.databasePath, { bigint: true });
assert.equal(partial.dev, activated.dev);
assert.equal(partial.ino, activated.ino);
const recovered = await restoreLocalSqliteSnapshot(restoreOptions);
assert.equal(recovered.status, 'restored');
assert.equal(recovered.sha256, source.sha256);
const restored = fs.statSync(state.databasePath, { bigint: true });
assert.equal(restored.dev, activated.dev);
assert.equal(restored.ino, activated.ino);
assert.equal(fs.existsSync(restoreStagePath), false);
assert.equal(fs.existsSync(replacedDatabasePath), false);
assert.equal(
(await restoreLocalSqliteSnapshot(restoreOptions)).status,
'existing',
);
});
test('rollout safety subpath excludes DDL and mutable repositories', () => {
const script = `
const safety = require(${JSON.stringify(