diff --git a/.github/workflows/ql3-ci.yml b/.github/workflows/ql3-ci.yml index c99f972d..24e0874d 100644 --- a/.github/workflows/ql3-ci.yml +++ b/.github/workflows/ql3-ci.yml @@ -383,7 +383,7 @@ jobs: - name: Verify non-root identity and architecture env: IMAGE: qinglong3-local-application:ci-${{ matrix.image_arch }} - EXPECTED: ${{ matrix.image_arch }} 65532:65532 2 44 44 44 1 + EXPECTED: ${{ matrix.image_arch }} 65532:65532 2,3,4 50 50 50 1 run: | set -euo pipefail actual="$(docker image inspect --format '{{.Architecture}} {{.Config.User}} {{index .Config.Labels "io.qinglong.local.application-config"}} {{index .Config.Labels "io.qinglong.local.sqlite-contract-min"}} {{index .Config.Labels "io.qinglong.local.sqlite-contract-max"}} {{index .Config.Labels "io.qinglong.local.sqlite-write-contract"}} {{index .Config.Labels "io.qinglong.local.compose-selection"}}' "${IMAGE}")" diff --git a/deploy/containers/ql3-local-application/Dockerfile b/deploy/containers/ql3-local-application/Dockerfile index 0d715904..4811889c 100644 --- a/deploy/containers/ql3-local-application/Dockerfile +++ b/deploy/containers/ql3-local-application/Dockerfile @@ -142,10 +142,10 @@ LABEL org.opencontainers.image.title="QingLong 3.0 Local Application" \ org.opencontainers.image.version="3.0.0-alpha.0" \ io.qinglong.profile="edge,standalone" \ io.qinglong.ai="excluded" \ - io.qinglong.local.application-config="2" \ - io.qinglong.local.sqlite-contract-min="44" \ - io.qinglong.local.sqlite-contract-max="44" \ - io.qinglong.local.sqlite-write-contract="44" \ + io.qinglong.local.application-config="2,3,4" \ + io.qinglong.local.sqlite-contract-min="50" \ + io.qinglong.local.sqlite-contract-max="50" \ + io.qinglong.local.sqlite-write-contract="50" \ io.qinglong.local.compose-selection="1" ENV NODE_ENV=production diff --git a/packages/ql3-local-owner-cli/src/deployment/adopted-bundle/adoptedBundle.ts b/packages/ql3-local-owner-cli/src/deployment/adopted-bundle/adoptedBundle.ts index 503f6b5e..e6f3b238 100644 --- a/packages/ql3-local-owner-cli/src/deployment/adopted-bundle/adoptedBundle.ts +++ b/packages/ql3-local-owner-cli/src/deployment/adopted-bundle/adoptedBundle.ts @@ -237,7 +237,32 @@ export function prepareLocalDeploymentAdoptedBundle( [material.paths.pluginActivation, 'pluginActivationRoot'], ...(material.composeSelection === null ? [] - : ([[material.paths.composeRevisions, 'composeRevisionRoot']] as const)), + : ([ + [material.paths.composeRevisions, 'composeRevisionRoot'], + [material.paths.composeRollouts, 'composeRolloutRoot'], + [material.paths.composeRolloutBackups, 'composeRolloutBackupRoot'], + [material.paths.composeRestores, 'composeRestoreRoot'], + [ + material.paths.composeRestoreSafeguards, + 'composeRestoreSafeguardRoot', + ], + [ + material.paths.composeEvidenceCollections, + 'composeEvidenceCollectionRoot', + ], + [ + material.paths.composeCollectedEvidence, + 'composeCollectedEvidenceRoot', + ], + [ + material.paths.composeCollectedRolloutBackups, + 'composeCollectedRolloutBackupRoot', + ], + [ + material.paths.composeCollectedRestoreSafeguards, + 'composeCollectedRestoreSafeguardRoot', + ], + ] as const)), ] as const; const directoryStatuses = directoryPaths.map(([directory, label]) => ensurePrivateDirectory(directory, identity.uid, label), @@ -352,7 +377,32 @@ export function verifyLocalDeploymentAdoptedBundle( [material.paths.pluginActivation, 'pluginActivationRoot'], ...(material.composeSelection === null ? [] - : ([[material.paths.composeRevisions, 'composeRevisionRoot']] as const)), + : ([ + [material.paths.composeRevisions, 'composeRevisionRoot'], + [material.paths.composeRollouts, 'composeRolloutRoot'], + [material.paths.composeRolloutBackups, 'composeRolloutBackupRoot'], + [material.paths.composeRestores, 'composeRestoreRoot'], + [ + material.paths.composeRestoreSafeguards, + 'composeRestoreSafeguardRoot', + ], + [ + material.paths.composeEvidenceCollections, + 'composeEvidenceCollectionRoot', + ], + [ + material.paths.composeCollectedEvidence, + 'composeCollectedEvidenceRoot', + ], + [ + material.paths.composeCollectedRolloutBackups, + 'composeCollectedRolloutBackupRoot', + ], + [ + material.paths.composeCollectedRestoreSafeguards, + 'composeCollectedRestoreSafeguardRoot', + ], + ] as const)), ] as const) { validatePrivateDirectory(directory, identity.uid, label); } @@ -420,7 +470,7 @@ export function verifyLocalDeploymentAdoptedBundle( }), directories: Object.freeze({ created: 0, - existing: material.composeSelection === null ? 4 : 5, + existing: material.composeSelection === null ? 4 : 13, }), }); } diff --git a/packages/ql3-local-owner-cli/src/deployment/adopted-bundle/material.ts b/packages/ql3-local-owner-cli/src/deployment/adopted-bundle/material.ts index 0e7b4a84..c89b8501 100644 --- a/packages/ql3-local-owner-cli/src/deployment/adopted-bundle/material.ts +++ b/packages/ql3-local-owner-cli/src/deployment/adopted-bundle/material.ts @@ -34,6 +34,14 @@ export interface LocalDeploymentAdoptedBundlePaths { readonly composeSelection: string; readonly composeRevisions: string; readonly composeRevision: string; + readonly composeRollouts: string; + readonly composeRolloutBackups: string; + readonly composeRestores: string; + readonly composeRestoreSafeguards: string; + readonly composeEvidenceCollections: string; + readonly composeCollectedEvidence: string; + readonly composeCollectedRolloutBackups: string; + readonly composeCollectedRestoreSafeguards: string; } export interface LocalDeploymentAdoptedEvidence { @@ -316,6 +324,22 @@ export function adoptedBundlePaths( composeSelection: path.join(service, 'compose.image.yaml'), composeRevisions: path.join(service, 'revisions'), composeRevision: path.join(service, 'revisions', '1.yaml'), + composeRollouts: path.join(service, 'rollouts'), + composeRolloutBackups: path.join(service, 'rollout-backups'), + composeRestores: path.join(service, 'restores'), + composeRestoreSafeguards: path.join(service, 'restore-safeguards'), + composeEvidenceCollections: path.join(service, 'evidence-collections'), + composeCollectedEvidence: path.join(service, 'collected-evidence'), + composeCollectedRolloutBackups: path.join( + service, + 'collected-evidence', + 'rollout-backups', + ), + composeCollectedRestoreSafeguards: path.join( + service, + 'collected-evidence', + 'restore-safeguards', + ), }); } diff --git a/packages/ql3-local-owner-cli/src/deployment/compose/composeApply.ts b/packages/ql3-local-owner-cli/src/deployment/compose/composeApply.ts index 323ef707..d9c34ec5 100644 --- a/packages/ql3-local-owner-cli/src/deployment/compose/composeApply.ts +++ b/packages/ql3-local-owner-cli/src/deployment/compose/composeApply.ts @@ -12,7 +12,10 @@ import { type LocalSqliteRolloutBackupOptions, } from '@qinglong/local-sqlite/rollout-safety'; -import { preflightLocalDeploymentCompose } from './composePreflight'; +import { + preflightLocalDeploymentCompose, + type LocalDeploymentComposePreflightDependencies, +} from './composePreflight'; import { evidenceDigest, inspectCollectedEvidence } from './composeEvidence'; import { inspectActiveComposeImageSelection, @@ -43,8 +46,15 @@ import { deploymentPaths, type LocalDeploymentPaths, } from '../foundation/render'; +import { + assertLocalDeploymentComposeLineageReceipt, + inspectLocalDeploymentComposeLineage, + normalizeLocalDeploymentComposeLineageReceipt, + type LocalDeploymentComposeLineage, + type LocalDeploymentComposeLineageReceipt, +} from './composeLineage'; -const RECEIPT_SCHEMA = 'qinglong/local-compose-rollout-receipt@v2'; +const RECEIPT_SCHEMA = 'qinglong/local-compose-rollout-receipt@v3'; const CONTAINER_ID_PATTERN = /^[0-9a-f]{12,64}$/; const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; @@ -59,6 +69,7 @@ export interface LocalDeploymentComposeApplyDependencies { readonly createBackup?: typeof createLocalSqliteRolloutBackup; readonly inspectBackup?: typeof inspectLocalSqliteRolloutBackup; readonly openChangeObserver?: typeof openLocalSqliteChangeObserver; + readonly auditSqlite?: LocalDeploymentComposePreflightDependencies['auditSqlite']; } interface ActiveEvidence { @@ -88,6 +99,7 @@ interface RolloutReceipt { readonly attemptedGeneration: number; readonly recordedAtMs: number; readonly healthEventDigest: string | null; + readonly lineage: Readonly; readonly sqlite: Readonly; readonly result: Readonly; } @@ -181,6 +193,7 @@ function inspectRunningContainer( output: string, containerId: string, selection: Readonly, + lineage: Readonly, ): boolean { let value: unknown; try { @@ -207,6 +220,19 @@ function inspectRunningContainer( }; } | undefined; + const labels = candidate?.Config?.Labels; + const lineageLabelsValid = + lineage.mode === 'fresh' || + (labels?.['io.qinglong.deployment.mode'] === 'adopted' && + labels['io.qinglong.deployment.profile'] === lineage.profile && + labels['io.qinglong.deployment.instance'] === lineage.instanceId && + labels['io.qinglong.deployment.bundle'] === lineage.bundleId && + labels['io.qinglong.application.config'] === + lineage.receipt.applicationConfigDigest && + labels['io.qinglong.data.commit'] === + lineage.receipt.legacyDataApplicationCommitDigest && + labels['io.qinglong.data.receipt'] === + lineage.receipt.legacyDataApplicationReceiptDigest); return ( (candidate?.Id === containerId || (typeof candidate?.Id === 'string' && @@ -214,10 +240,10 @@ function inspectRunningContainer( candidate?.State?.Running === true && candidate.State.Status === 'running' && candidate.Config?.Image === selection.image && - candidate.Config.Labels?.['io.qinglong.deployment.generation'] === + labels?.['io.qinglong.deployment.generation'] === String(selection.generation) && - candidate.Config.Labels?.['io.qinglong.deployment.mutation'] === - selection.mutationId && + labels?.['io.qinglong.deployment.mutation'] === selection.mutationId && + lineageLabelsValid && candidate.HostConfig?.ReadonlyRootfs === true && candidate.HostConfig.NetworkMode === 'none' && candidate.HostConfig.Privileged !== true @@ -260,6 +286,7 @@ async function observeActive( paths: Readonly, selection: Readonly, profile: LocalDeploymentProfile, + lineage: Readonly, dependencies: Required< Pick >, @@ -285,6 +312,7 @@ async function observeActive( ]), containerId, selection, + lineage, ); if (running) { const evidence = activeEventEvidence( @@ -321,9 +349,10 @@ function backupOptions( command: Readonly, paths: Readonly, profile: LocalDeploymentProfile, + databasePath: string, ): Readonly { return Object.freeze({ - databasePath: paths.database, + databasePath, backupPath: backupPathFor(paths, command.request.rolloutId), profile, }); @@ -389,6 +418,7 @@ async function applyAndObserveCandidate( paths: Readonly, selection: Readonly, profile: LocalDeploymentProfile, + lineage: Readonly, dependencies: Required< Pick< LocalDeploymentComposeApplyDependencies, @@ -402,23 +432,26 @@ async function applyAndObserveCandidate( }> > { const observer = dependencies.openChangeObserver({ - databasePath: paths.database, + databasePath: lineage.databasePath, profile, }); let evidence: Readonly | null = null; try { try { applyActiveSelection(command, paths, dependencies.runDocker); - evidence = await observeActive( - command, - paths, - selection, - profile, - dependencies, - ); } catch { - evidence = null; + // Docker may lose the response after creating the exact candidate. The + // bounded observation below is the recovery authority; do not issue a + // second `compose up` in the same attempt. } + evidence = await observeActive( + command, + paths, + selection, + profile, + lineage, + dependencies, + ); let writeObservation: SqliteWriteObservation; try { writeObservation = observer.changed() ? 'changed' : 'unchanged'; @@ -467,6 +500,7 @@ async function readReceipt( expectedGeneration: number, paths: Readonly, inspectBackup: typeof inspectLocalSqliteRolloutBackup, + expectedLineage: Readonly, ): Promise | null> { if (!fs.existsSync(filePath)) return null; const stat = fs.lstatSync(filePath); @@ -496,11 +530,15 @@ async function readReceipt( const healthKeys = Object.keys(receipt.result?.health ?? {}).sort(); const serviceKeys = Object.keys(receipt.result?.service ?? {}).sort(); const sqliteKeys = Object.keys(receipt.sqlite ?? {}).sort(); + const lineage = normalizeLocalDeploymentComposeLineageReceipt( + receipt.lineage, + ); const backupKeys = Object.keys(receipt.sqlite?.backup ?? {}).sort(); const expectedReceiptKeys = [ 'attemptedGeneration', 'commandDigest', 'healthEventDigest', + 'lineage', 'recordedAtMs', 'result', 'rolloutId', @@ -546,6 +584,7 @@ async function readReceipt( attemptedGeneration: receipt.attemptedGeneration, recordedAtMs: receipt.recordedAtMs, healthEventDigest: receipt.healthEventDigest, + lineage, sqlite: { contractVersion: receipt.sqlite?.contractVersion, writeContractVersion: receipt.sqlite?.writeContractVersion, @@ -636,11 +675,15 @@ async function readReceipt( ) { configurationError('compose rollout receipt drifted'); } + assertLocalDeploymentComposeLineageReceipt( + receipt.lineage, + expectedLineage.receipt, + ); if (receipt.sqlite.backup !== null) { const backupPath = backupPathFor(paths, rolloutId); if (fs.existsSync(backupPath)) { const inspected = await inspectBackup({ - databasePath: paths.database, + databasePath: expectedLineage.databasePath, backupPath, profile: receipt.result.profile, }); @@ -698,6 +741,7 @@ function publishReceipt( rolloutResult: Readonly, evidence: Readonly | null, sqlite: Readonly, + lineage: Readonly, uid: number, ): void { publishExactFile( @@ -709,6 +753,7 @@ function publishReceipt( attemptedGeneration: command.request.expectedGeneration, recordedAtMs: command.request.failureRollbackChangedAtMs, healthEventDigest: evidence?.digest ?? null, + lineage: lineage.receipt, sqlite, result: rolloutResult, }), @@ -763,6 +808,12 @@ export async function applyLocalDeploymentCompose( identity.uid, 'composeRolloutBackupRoot', ); + const lineage = inspectLocalDeploymentComposeLineage( + command.options.deploymentRoot, + identity.uid, + identity.gid, + command.options.allowRootService, + ); if (fs.existsSync(paths.composeEvidenceCollectionLock)) { configurationError('compose rollout is fenced by an evidence collection'); } @@ -785,6 +836,7 @@ export async function applyLocalDeploymentCompose( dependencies.inspectBackup ?? inspectLocalSqliteRolloutBackup; const openChangeObserver = dependencies.openChangeObserver ?? openLocalSqliteChangeObserver; + const auditSqlite = dependencies.auditSqlite; const receiptPath = path.join( paths.composeRollouts, `${command.request.rolloutId}.json`, @@ -798,6 +850,7 @@ export async function applyLocalDeploymentCompose( command.request.expectedGeneration, paths, inspectBackup, + lineage, ); if (replay) { releaseLock(paths.composeRolloutLock, intent, identity.uid); @@ -851,11 +904,11 @@ export async function applyLocalDeploymentCompose( expectedGeneration: command.request.expectedGeneration, }, }, - { runDocker, validateSocket }, + { runDocker, validateSocket, auditSqlite }, ); if (selection.previousGeneration >= 1) { rolloutBackup = await createBackup( - backupOptions(command, paths, preflight.profile), + backupOptions(command, paths, preflight.profile, lineage.databasePath), ); } const candidate = await applyAndObserveCandidate( @@ -863,6 +916,7 @@ export async function applyLocalDeploymentCompose( paths, selection, preflight.profile, + lineage, { runDocker, now, wait, openChangeObserver }, ); const evidence = candidate.evidence; @@ -880,6 +934,7 @@ export async function applyLocalDeploymentCompose( active, evidence, sqliteReceipt(rolloutBackup, writeObservation), + lineage, identity.uid, ); releaseLock(paths.composeRolloutLock, intent, identity.uid); @@ -899,6 +954,7 @@ export async function applyLocalDeploymentCompose( stopped, null, sqliteReceipt(null, writeObservation), + lineage, identity.uid, ); releaseLock(paths.composeRolloutLock, intent, identity.uid); @@ -936,19 +992,30 @@ export async function applyLocalDeploymentCompose( options: command.options, request: { expectedGeneration: selection.generation }, }, - { runDocker, validateSocket }, + { runDocker, validateSocket, auditSqlite }, ); if (command.request.expectedGeneration > 1 && rolloutBackup === null) { rolloutBackup = await inspectBackup( - backupOptions(command, paths, rollbackPreflight.profile), + backupOptions( + command, + paths, + rollbackPreflight.profile, + lineage.databasePath, + ), ); } - applyActiveSelection(rollbackCommand, paths, runDocker); + try { + applyActiveSelection(rollbackCommand, paths, runDocker); + } catch { + // As with the candidate, inspect the exact generation after a lost Docker + // response instead of blindly recreating the rollback container. + } const rollbackEvidence = await observeActive( rollbackCommand, paths, selection, rollbackPreflight.profile, + lineage, { runDocker, now, wait }, ); if (!rollbackEvidence) { @@ -968,6 +1035,7 @@ export async function applyLocalDeploymentCompose( rolledBack, rollbackEvidence, sqliteReceipt(rolloutBackup, writeObservation), + lineage, identity.uid, ); releaseLock(paths.composeRolloutLock, intent, identity.uid); diff --git a/packages/ql3-local-owner-cli/src/deployment/compose/composeEvidenceCollection.ts b/packages/ql3-local-owner-cli/src/deployment/compose/composeEvidenceCollection.ts index a8f372a7..eb0b5aa0 100644 --- a/packages/ql3-local-owner-cli/src/deployment/compose/composeEvidenceCollection.ts +++ b/packages/ql3-local-owner-cli/src/deployment/compose/composeEvidenceCollection.ts @@ -37,11 +37,18 @@ import { deploymentPaths, type LocalDeploymentPaths, } from '../foundation/render'; +import { + assertLocalDeploymentComposeLineageReceipt, + inspectLocalDeploymentComposeLineage, + normalizeLocalDeploymentComposeLineageReceipt, + type LocalDeploymentComposeLineage, + type LocalDeploymentComposeLineageReceipt, +} from './composeLineage'; -const PREPARE_SCHEMA = 'qinglong/local-compose-evidence-collection-prepare@v1'; -const COMMIT_SCHEMA = 'qinglong/local-compose-evidence-collection-commit@v1'; -const ROLLOUT_RECEIPT_SCHEMA = 'qinglong/local-compose-rollout-receipt@v2'; -const RESTORE_COMMIT_SCHEMA = 'qinglong/local-compose-restore-commit@v1'; +const PREPARE_SCHEMA = 'qinglong/local-compose-evidence-collection-prepare@v2'; +const COMMIT_SCHEMA = 'qinglong/local-compose-evidence-collection-commit@v2'; +const ROLLOUT_RECEIPT_SCHEMA = 'qinglong/local-compose-rollout-receipt@v3'; +const RESTORE_COMMIT_SCHEMA = 'qinglong/local-compose-restore-commit@v2'; const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; const SHA256_PATTERN = /^[0-9a-f]{64}$/; @@ -61,6 +68,7 @@ interface CollectionPrepareReceipt { readonly collectionId: string; readonly generation: number; readonly profile: LocalDeploymentProfile; + readonly lineage: Readonly; readonly recordedAtMs: number; readonly items: readonly Readonly[]; } @@ -72,6 +80,7 @@ interface CollectionCommitReceipt { readonly collectionId: string; readonly generation: number; readonly profile: LocalDeploymentProfile; + readonly lineage: Readonly; readonly recordedAtMs: number; readonly items: readonly Readonly[]; readonly bytes: number; @@ -174,31 +183,11 @@ function exactSnapshot( } } -function readProfile( - paths: Readonly, - uid: number, -): LocalDeploymentProfile { - const value = readJson( - boundedPrivateFile( - paths.applicationConfig, - uid, - 'application configuration', - ), - 'application configuration', - ); - if ( - value.schema !== 'qinglong/local-application-process@v2' || - (value.profile !== 'edge' && value.profile !== 'standalone') - ) { - configurationError('application configuration collection binding drifted'); - } - return value.profile; -} - function rolloutItem( paths: Readonly, uid: number, rolloutId: string, + lineage: Readonly, ): Readonly { const receiptContents = boundedPrivateFile( path.join(paths.composeRollouts, `${rolloutId}.json`), @@ -210,6 +199,7 @@ function rolloutItem( const sqlite = receipt.sqlite as Record | undefined; const backup = sqlite?.backup as Record | undefined; const result = receipt.result as Record | undefined; + assertLocalDeploymentComposeLineageReceipt(receipt.lineage, lineage.receipt); if ( receipt.schema !== ROLLOUT_RECEIPT_SCHEMA || receipt.rolloutId !== rolloutId || @@ -243,6 +233,7 @@ function restoreItem( paths: Readonly, uid: number, restoreId: string, + lineage: Readonly, ): Readonly { const receiptContents = boundedPrivateFile( path.join(paths.composeRestores, `${restoreId}.commit.json`), @@ -251,6 +242,7 @@ function restoreItem( [1, 2], ); const receipt = readJson(receiptContents, 'compose restore commit receipt'); + assertLocalDeploymentComposeLineageReceipt(receipt.lineage, lineage.receipt); if ( receipt.schema !== RESTORE_COMMIT_SCHEMA || receipt.restoreId !== restoreId || @@ -348,6 +340,7 @@ async function prepareItems( paths: Readonly, uid: number, profile: LocalDeploymentProfile, + lineage: Readonly, ): Promise[]> { const rolloutCatalog = catalogIds( paths.composeRolloutBackups, @@ -359,14 +352,14 @@ async function prepareItems( ); const rolloutItems = await Promise.all( rolloutCatalog.map(async (id) => { - const item = rolloutItem(paths, uid, id); + const item = rolloutItem(paths, uid, id, lineage); await inspectPhysicalItem(paths, profile, item); return item; }), ); const restoreItems = await Promise.all( restoreCatalog.map(async (id) => { - const item = restoreItem(paths, uid, id); + const item = restoreItem(paths, uid, id, lineage); await inspectPhysicalItem(paths, profile, item); return item; }), @@ -491,6 +484,7 @@ function inspectPrepareReceipt( paths: Readonly, uid: number, command: Readonly, + expectedLineage: Readonly, ): Readonly | null { const filePath = prepareReceiptPath(paths, command.request.collectionId); if (!fs.existsSync(filePath)) return null; @@ -505,7 +499,12 @@ function inspectPrepareReceipt( 'compose evidence collection prepare receipt', ); const items = parseItems(value.items); - const receipt = { ...value, items } as unknown as CollectionPrepareReceipt; + const lineage = normalizeLocalDeploymentComposeLineageReceipt(value.lineage); + const receipt = { + ...value, + lineage, + items, + } as unknown as CollectionPrepareReceipt; if ( Object.keys(value).sort().join(',') !== [ @@ -513,6 +512,7 @@ function inspectPrepareReceipt( 'commandDigest', 'generation', 'items', + 'lineage', 'profile', 'recordedAtMs', 'schema', @@ -529,6 +529,10 @@ function inspectPrepareReceipt( ) { configurationError('compose evidence collection prepare receipt drifted'); } + assertLocalDeploymentComposeLineageReceipt( + receipt.lineage, + expectedLineage.receipt, + ); publishExactFile( filePath, contents, @@ -543,6 +547,7 @@ function inspectCommitReceipt( paths: Readonly, uid: number, command: Readonly, + expectedLineage: Readonly, ): Readonly | null { const filePath = commitReceiptPath(paths, command.request.collectionId); if (!fs.existsSync(filePath)) return null; @@ -557,7 +562,12 @@ function inspectCommitReceipt( 'compose evidence collection commit receipt', ); const items = parseItems(value.items); - const receipt = { ...value, items } as unknown as CollectionCommitReceipt; + const lineage = normalizeLocalDeploymentComposeLineageReceipt(value.lineage); + const receipt = { + ...value, + lineage, + items, + } as unknown as CollectionCommitReceipt; if ( Object.keys(value).sort().join(',') !== [ @@ -566,6 +576,7 @@ function inspectCommitReceipt( 'commandDigest', 'generation', 'items', + 'lineage', 'prepareReceiptDigest', 'profile', 'recordedAtMs', @@ -590,6 +601,10 @@ function inspectCommitReceipt( ) { configurationError('compose evidence collection commit receipt drifted'); } + assertLocalDeploymentComposeLineageReceipt( + receipt.lineage, + expectedLineage.receipt, + ); publishExactFile( filePath, contents, @@ -658,8 +673,9 @@ async function prepareCollection( command: Readonly, paths: Readonly, uid: number, + lineage: Readonly, ): Promise> { - const replay = inspectPrepareReceipt(paths, uid, command); + const replay = inspectPrepareReceipt(paths, uid, command, lineage); if (replay) return result(command, 'existing', replay.profile, replay.items); assertOperationsIdle(paths); const selection = inspectActiveComposeImageSelection( @@ -672,8 +688,8 @@ async function prepareCollection( 'active compose generation does not match evidence collection', ); } - const profile = readProfile(paths, uid); - const items = await prepareItems(command, paths, uid, profile); + const profile = lineage.profile; + const items = await prepareItems(command, paths, uid, profile, lineage); if (items.some((item) => item.recordedAtMs > command.request.preparedAtMs)) { configurationError( 'compose evidence collection precedes retained evidence', @@ -699,6 +715,7 @@ async function prepareCollection( collectionId: command.request.collectionId, generation: command.request.expectedGeneration, profile, + lineage: lineage.receipt, recordedAtMs: command.request.preparedAtMs, items, }); @@ -826,13 +843,14 @@ async function commitCollection( command: Readonly, paths: Readonly, uid: number, + lineage: Readonly, ): Promise> { if ( command.operation !== 'local.deployment.compose.evidence-collection.commit' ) { configurationError('compose evidence collection commit command is invalid'); } - const existing = inspectCommitReceipt(paths, uid, command); + const existing = inspectCommitReceipt(paths, uid, command, lineage); if (existing) { const prepareContents = boundedPrivateFile( prepareReceiptPath(paths, command.request.collectionId), @@ -875,7 +893,7 @@ async function commitCollection( return result(command, 'existing', existing.profile, existing.items); } const prepared = prepareCommandFromLock(command, paths, uid); - const prepareReceipt = inspectPrepareReceipt(paths, uid, prepared); + const prepareReceipt = inspectPrepareReceipt(paths, uid, prepared, lineage); if (!prepareReceipt) { configurationError( 'compose evidence collection prepare receipt is unavailable', @@ -911,6 +929,7 @@ async function commitCollection( collectionId: command.request.collectionId, generation: command.request.expectedGeneration, profile: prepareReceipt.profile, + lineage: lineage.receipt, recordedAtMs: command.request.committedAtMs, items: prepareReceipt.items, bytes: prepareReceipt.items.reduce( @@ -992,10 +1011,16 @@ export async function collectLocalDeploymentComposeEvidence( identity.uid, 'composeCollectedRestoreSafeguardRoot', ); + const lineage = inspectLocalDeploymentComposeLineage( + command.options.deploymentRoot, + identity.uid, + identity.gid, + command.options.allowRootService, + ); return command.operation === 'local.deployment.compose.evidence-collection.prepare' - ? prepareCollection(command, paths, identity.uid) - : commitCollection(command, paths, identity.uid); + ? prepareCollection(command, paths, identity.uid, lineage) + : commitCollection(command, paths, identity.uid, lineage); } export function collectLocalDeploymentComposeEvidencePrepareCommandFile( diff --git a/packages/ql3-local-owner-cli/src/deployment/compose/composeLineage.ts b/packages/ql3-local-owner-cli/src/deployment/compose/composeLineage.ts new file mode 100644 index 00000000..513d1a66 --- /dev/null +++ b/packages/ql3-local-owner-cli/src/deployment/compose/composeLineage.ts @@ -0,0 +1,794 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { cutoverDigest } from '../cutover/targetEvidence'; +import type { + LocalDeploymentPrepareCommand, + LocalDeploymentProfile, +} from '../foundation/contract'; +import { LocalDeploymentConfigurationError } from '../foundation/error'; +import { + applicationConfiguration, + deploymentPaths, + descriptor, +} from '../foundation/render'; +import type { NormalizedLocalDeploymentAdoptedBundleCommand } from '../adopted-bundle/contract'; +import { + renderLocalDeploymentAdoptedBundleMaterial, + verifyLocalDeploymentAdoptedEvidence, + type LocalDeploymentAdoptedBundleReceipt, +} from '../adopted-bundle/material'; +import { inspectComposeImageSelectionGeneration } from './composeRevision'; + +const APPLICATION_SCHEMA_V2 = 'qinglong/local-application-process@v2'; +const APPLICATION_SCHEMA_V4 = 'qinglong/local-application-process@v4'; +const LINEAGE_SCHEMA = 'qinglong/local-compose-deployment-lineage@v1'; +const MAX_PRIVATE_FILE_BYTES = 64 * 1024; +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const UUID_V4_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +export interface LocalDeploymentComposeLineageReceipt { + readonly schema: typeof LINEAGE_SCHEMA; + readonly mode: 'fresh' | 'adopted'; + readonly applicationConfigDigest: string; + readonly adoptedBundleDigest: string | null; + readonly activationDigest: string | null; + readonly commitmentDigest: string | null; + readonly legacyDataApplicationCommitDigest: string | null; + readonly legacyDataApplicationReceiptDigest: string | null; +} + +export interface LocalDeploymentComposeLineage { + readonly mode: 'fresh' | 'adopted'; + readonly instanceId: string; + readonly profile: LocalDeploymentProfile; + readonly busyTimeoutMs?: number; + readonly databasePath: string; + readonly sourcePath: string | null; + readonly bundleId: string | null; + readonly receipt: Readonly; +} + +interface ParsedAdoptedApplication { + readonly instanceId: string; + readonly profile: LocalDeploymentProfile; + readonly busyTimeoutMs?: number; + readonly storage: Readonly<{ + sourcePath: string; + targetPath: string; + recoveryPath: string; + manifestPath: string; + activationPath: string; + expectedActivationDigest: string; + }>; + readonly cutover: Readonly<{ + cutoverId: string; + commitmentPath: string; + expectedCommitmentDigest: string; + }>; + readonly legacyDataApplication: Readonly<{ + commitPath: string; + expectedCommitDigest: string; + expectedReceiptDigest: string; + }>; +} + +function configurationError(message: string, cause?: unknown): never { + throw new LocalDeploymentConfigurationError(message, { cause }); +} + +function object(value: unknown, label: string): Record { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + (Object.getPrototypeOf(value) !== Object.prototype && + Object.getPrototypeOf(value) !== null) + ) { + configurationError(`${label} must be an object`); + } + return value as Record; +} + +function exact( + value: Readonly>, + keys: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + configurationError(`${label} shape is invalid`); + } +} + +function sha256(value: string | Buffer): string { + return crypto.createHash('sha256').update(value).digest('hex'); +} + +function digest(value: unknown, label: string): string { + if (typeof value !== 'string' || !DIGEST_PATTERN.test(value)) { + configurationError(`${label} is invalid`); + } + return value; +} + +function absolutePath(value: unknown, label: string): string { + if ( + typeof value !== 'string' || + !path.isAbsolute(value) || + path.normalize(value) !== value || + path.parse(value).root === value || + value.includes('\0') || + value.includes('//') || + Buffer.byteLength(value, 'utf8') > 4_096 + ) { + configurationError(`${label} is not a canonical bounded absolute path`); + } + return value; +} + +function strictDescendant(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return ( + relative.length > 0 && + relative !== '..' && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) + ); +} + +function readExactPrivateFile( + filePath: string, + uid: number, + gid: number, + label: string, +): string { + let descriptor: number | undefined; + let bytes: Buffer | undefined; + try { + const before = fs.lstatSync(filePath, { bigint: true }); + descriptor = fs.openSync( + filePath, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + const opened = fs.fstatSync(descriptor, { bigint: true }); + if ( + !opened.isFile() || + opened.isSymbolicLink() || + opened.dev !== before.dev || + opened.ino !== before.ino || + Number(opened.uid) !== uid || + Number(opened.gid) !== gid || + (Number(opened.mode) & 0o777) !== 0o600 || + opened.nlink !== 1n || + opened.size < 2n || + opened.size > BigInt(MAX_PRIVATE_FILE_BYTES) || + fs.realpathSync(filePath) !== filePath + ) { + configurationError(`${label} identity is invalid`); + } + bytes = Buffer.alloc(Number(opened.size)); + let offset = 0; + while (offset < bytes.byteLength) { + const count = fs.readSync( + descriptor, + bytes, + offset, + bytes.byteLength - offset, + offset, + ); + if (count === 0) break; + offset += count; + } + const after = fs.fstatSync(descriptor, { bigint: true }); + if ( + offset !== bytes.byteLength || + after.dev !== opened.dev || + after.ino !== opened.ino || + after.size !== opened.size || + after.mtimeNs !== opened.mtimeNs || + after.ctimeNs !== opened.ctimeNs || + after.uid !== opened.uid || + after.gid !== opened.gid || + after.mode !== opened.mode || + after.nlink !== opened.nlink + ) { + configurationError(`${label} changed while reading`); + } + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch (error) { + if (error instanceof LocalDeploymentConfigurationError) throw error; + return configurationError(`${label} cannot be read`, error); + } finally { + bytes?.fill(0); + if (descriptor !== undefined) fs.closeSync(descriptor); + } +} + +function verifyExactPrivateFile( + filePath: string, + expected: string, + uid: number, + gid: number, + label: string, +): void { + if (readExactPrivateFile(filePath, uid, gid, label) !== expected) { + configurationError(`${label} content drifted`); + } +} + +function parseJson(contents: string, label: string): Record { + try { + return object(JSON.parse(contents), label); + } catch (error) { + if (error instanceof LocalDeploymentConfigurationError) throw error; + return configurationError(`${label} is invalid`, error); + } +} + +function optionalBusyTimeout(value: unknown): number | undefined { + if (value === undefined) return undefined; + if ( + !Number.isSafeInteger(value) || + (value as number) < 100 || + (value as number) > 30_000 + ) { + configurationError('application busyTimeoutMs is invalid'); + } + return value as number; +} + +function parseFreshApplication( + value: Readonly>, +): Readonly<{ + instanceId: string; + profile: LocalDeploymentProfile; + databasePath: string; + busyTimeoutMs?: number; +}> { + const storage = object(value.storage, 'application storage'); + if ( + typeof value.instanceId !== 'string' || + (value.profile !== 'edge' && value.profile !== 'standalone') || + storage.mode !== 'fresh' + ) { + configurationError('application configuration is not a fresh Compose v2'); + } + const databasePath = absolutePath( + storage.databasePath, + 'application databasePath', + ); + const busyTimeoutMs = optionalBusyTimeout(storage.busyTimeoutMs); + return Object.freeze({ + instanceId: value.instanceId, + profile: value.profile, + databasePath, + ...(busyTimeoutMs === undefined ? {} : { busyTimeoutMs }), + }); +} + +function parseAdoptedApplication( + value: Readonly>, + deploymentRoot: string, +): Readonly { + exact( + value, + [ + 'ai', + 'cutover', + 'instanceId', + 'legacyDataApplication', + 'pluginPackages', + 'profile', + 'runtime', + 'schema', + 'storage', + ], + 'adopted application configuration', + ); + if ( + typeof value.instanceId !== 'string' || + !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value.instanceId) || + (value.profile !== 'edge' && value.profile !== 'standalone') + ) { + configurationError('adopted application identity is invalid'); + } + const storage = object(value.storage, 'adopted storage'); + const optionalStorageKeys = Object.hasOwn(storage, 'busyTimeoutMs') + ? ['busyTimeoutMs'] + : []; + exact( + storage, + [ + 'activationPath', + 'expectedActivationDigest', + 'manifestPath', + 'mode', + 'recoveryPath', + 'sourcePath', + 'targetPath', + ...optionalStorageKeys, + ], + 'adopted storage', + ); + const runtime = object(value.runtime, 'adopted runtime'); + exact( + runtime, + ['artifactRoot', 'receiptRoot', 'secretKeyringPath'], + 'adopted runtime', + ); + const pluginPackages = object( + value.pluginPackages, + 'adopted plugin packages', + ); + exact( + pluginPackages, + [ + 'activationRoot', + 'maxPages', + 'pageSize', + 'recoverySource', + 'stagingRoot', + 'taskPublicationMaxPages', + 'taskPublicationPageSize', + ], + 'adopted plugin packages', + ); + const recoverySource = object( + pluginPackages.recoverySource, + 'plugin recovery source', + ); + exact(recoverySource, ['mode'], 'plugin recovery source'); + const ai = object(value.ai, 'adopted AI'); + exact(ai, ['deployment'], 'adopted AI'); + const cutover = object(value.cutover, 'adopted cutover'); + exact( + cutover, + ['commitmentPath', 'cutoverId', 'expectedCommitmentDigest'], + 'adopted cutover', + ); + const dataApplication = object( + value.legacyDataApplication, + 'legacy data application', + ); + exact( + dataApplication, + ['commitPath', 'expectedCommitDigest', 'expectedReceiptDigest'], + 'legacy data application', + ); + const sourcePath = absolutePath(storage.sourcePath, 'sourcePath'); + const authorityPaths = { + targetPath: absolutePath(storage.targetPath, 'targetPath'), + recoveryPath: absolutePath(storage.recoveryPath, 'recoveryPath'), + manifestPath: absolutePath(storage.manifestPath, 'manifestPath'), + activationPath: absolutePath(storage.activationPath, 'activationPath'), + commitmentPath: absolutePath(cutover.commitmentPath, 'commitmentPath'), + commitPath: absolutePath(dataApplication.commitPath, 'commitPath'), + }; + const paths = deploymentPaths(deploymentRoot); + const pageSize = value.profile === 'edge' ? 4 : 16; + if ( + storage.mode !== 'adopted' || + strictDescendant(deploymentRoot, sourcePath) || + Object.values(authorityPaths).some( + (candidate) => !strictDescendant(deploymentRoot, candidate), + ) || + new Set([sourcePath, ...Object.values(authorityPaths)]).size !== 7 || + runtime.receiptRoot !== paths.receipts || + runtime.artifactRoot !== paths.artifacts || + runtime.secretKeyringPath !== paths.localSecretKeyring || + pluginPackages.stagingRoot !== paths.pluginStaging || + pluginPackages.activationRoot !== paths.pluginActivation || + recoverySource.mode !== 'disabled' || + pluginPackages.pageSize !== pageSize || + pluginPackages.maxPages !== pageSize || + pluginPackages.taskPublicationPageSize !== pageSize || + pluginPackages.taskPublicationMaxPages !== pageSize || + ai.deployment !== 'excluded' || + typeof cutover.cutoverId !== 'string' || + !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(cutover.cutoverId) + ) { + configurationError('adopted application authority binding drifted'); + } + const busyTimeoutMs = optionalBusyTimeout(storage.busyTimeoutMs); + return Object.freeze({ + instanceId: value.instanceId, + profile: value.profile, + ...(busyTimeoutMs === undefined ? {} : { busyTimeoutMs }), + storage: Object.freeze({ + sourcePath, + targetPath: authorityPaths.targetPath, + recoveryPath: authorityPaths.recoveryPath, + manifestPath: authorityPaths.manifestPath, + activationPath: authorityPaths.activationPath, + expectedActivationDigest: digest( + storage.expectedActivationDigest, + 'expectedActivationDigest', + ), + }), + cutover: Object.freeze({ + cutoverId: cutover.cutoverId, + commitmentPath: authorityPaths.commitmentPath, + expectedCommitmentDigest: digest( + cutover.expectedCommitmentDigest, + 'expectedCommitmentDigest', + ), + }), + legacyDataApplication: Object.freeze({ + commitPath: authorityPaths.commitPath, + expectedCommitDigest: digest( + dataApplication.expectedCommitDigest, + 'expectedCommitDigest', + ), + expectedReceiptDigest: digest( + dataApplication.expectedReceiptDigest, + 'expectedReceiptDigest', + ), + }), + }); +} + +function parseBundleReceipt( + contents: string, +): Readonly { + const value = parseJson(contents, 'adopted bundle receipt'); + exact( + value, + [ + 'activationDigest', + 'applicationConfigDigest', + 'bundleDigest', + 'bundleId', + 'commitmentDigest', + 'composeSelectionDigest', + 'cutoverId', + 'deploymentRootDigest', + 'instanceId', + 'kind', + 'legacyDataApplicationCommitDigest', + 'legacyDataApplicationReceiptDigest', + 'manifestDigest', + 'preparedAtMs', + 'profile', + 'recoverySha256', + 'schemaVersion', + 'serviceDescriptorDigest', + 'serviceKind', + 'sourcePathDigest', + 'sourceSha256', + 'state', + 'targetIdentityDigest', + ], + 'adopted bundle receipt', + ); + const { bundleDigest, ...payload } = value; + if ( + value.schemaVersion !== 1 || + value.kind !== 'qinglong3-local-adopted-deployment-bundle' || + value.state !== 'prepared' || + typeof value.bundleId !== 'string' || + !UUID_V4_PATTERN.test(value.bundleId) || + !Number.isSafeInteger(value.preparedAtMs) || + (value.preparedAtMs as number) < 0 || + (value.profile !== 'edge' && value.profile !== 'standalone') || + typeof value.instanceId !== 'string' || + typeof value.cutoverId !== 'string' || + value.serviceKind !== 'compose' || + Object.entries(value).some(([key, candidate]) => + key.endsWith('Digest') + ? typeof candidate !== 'string' || !DIGEST_PATTERN.test(candidate) + : false, + ) || + typeof value.sourceSha256 !== 'string' || + !DIGEST_PATTERN.test(value.sourceSha256) || + typeof value.recoverySha256 !== 'string' || + !DIGEST_PATTERN.test(value.recoverySha256) || + cutoverDigest(payload) !== bundleDigest + ) { + configurationError('adopted bundle receipt drifted'); + } + return value as unknown as Readonly; +} + +function lineageReceipt(input: { + readonly mode: 'fresh' | 'adopted'; + readonly applicationConfigDigest: string; + readonly adoptedBundleDigest?: string; + readonly activationDigest?: string; + readonly commitmentDigest?: string; + readonly legacyDataApplicationCommitDigest?: string; + readonly legacyDataApplicationReceiptDigest?: string; +}): Readonly { + const adopted = input.mode === 'adopted'; + return Object.freeze({ + schema: LINEAGE_SCHEMA, + mode: input.mode, + applicationConfigDigest: input.applicationConfigDigest, + adoptedBundleDigest: adopted ? input.adoptedBundleDigest! : null, + activationDigest: adopted ? input.activationDigest! : null, + commitmentDigest: adopted ? input.commitmentDigest! : null, + legacyDataApplicationCommitDigest: adopted + ? input.legacyDataApplicationCommitDigest! + : null, + legacyDataApplicationReceiptDigest: adopted + ? input.legacyDataApplicationReceiptDigest! + : null, + }); +} + +export function inspectLocalDeploymentComposeLineage( + deploymentRoot: string, + uid: number, + gid: number, + allowRootService: boolean, +): Readonly { + const paths = deploymentPaths(deploymentRoot); + const applicationContents = readExactPrivateFile( + paths.applicationConfig, + uid, + gid, + 'application configuration', + ); + const application = parseJson( + applicationContents, + 'application configuration', + ); + if (application.schema === APPLICATION_SCHEMA_V2) { + const fresh = parseFreshApplication(application); + const syntheticPrepare: Readonly = { + schemaVersion: 1, + operation: 'local.deployment.prepare', + options: { + deploymentRoot, + profile: fresh.profile, + instanceId: fresh.instanceId, + ...(fresh.busyTimeoutMs === undefined + ? {} + : { busyTimeoutMs: fresh.busyTimeoutMs }), + service: { + kind: 'compose', + releaseSelection: { + path: paths.composeSelection, + expectedSelectionDigest: 'sha256:'.concat('0'.repeat(64)), + }, + allowRootService, + }, + }, + request: { + ownerPepperKeyId: 'lineage', + registerMutationId: '00000000-0000-4000-8000-000000000001', + activateMutationId: '00000000-0000-4000-8000-000000000002', + registeredAtMs: 0, + activatedAtMs: 0, + }, + }; + const expectedApplication = applicationConfiguration( + syntheticPrepare, + paths, + ); + if (applicationContents !== expectedApplication) { + configurationError('fresh application configuration drifted'); + } + const expectedDescriptor = descriptor( + syntheticPrepare, + paths.applicationConfig, + uid, + gid, + ); + verifyExactPrivateFile( + path.join(paths.service, expectedDescriptor.fileName), + expectedDescriptor.contents, + uid, + gid, + 'fresh Compose descriptor', + ); + return Object.freeze({ + mode: 'fresh' as const, + instanceId: fresh.instanceId, + profile: fresh.profile, + ...(fresh.busyTimeoutMs === undefined + ? {} + : { busyTimeoutMs: fresh.busyTimeoutMs }), + databasePath: paths.database, + sourcePath: null, + bundleId: null, + receipt: lineageReceipt({ + mode: 'fresh', + applicationConfigDigest: sha256(applicationContents), + }), + }); + } + if (application.schema !== APPLICATION_SCHEMA_V4) { + configurationError('Compose requires an Application v2 or adopted v4'); + } + const adopted = parseAdoptedApplication(application, deploymentRoot); + const bundleContents = readExactPrivateFile( + path.join(paths.service, 'adopted-bundle.json'), + uid, + gid, + 'adopted bundle receipt', + ); + const bundle = parseBundleReceipt(bundleContents); + if ( + bundle.profile !== adopted.profile || + bundle.instanceId !== adopted.instanceId || + bundle.cutoverId !== adopted.cutover.cutoverId || + bundle.deploymentRootDigest !== sha256(deploymentRoot) || + bundle.sourcePathDigest !== sha256(adopted.storage.sourcePath) || + bundle.applicationConfigDigest !== sha256(applicationContents) || + bundle.activationDigest !== adopted.storage.expectedActivationDigest || + bundle.commitmentDigest !== adopted.cutover.expectedCommitmentDigest || + bundle.legacyDataApplicationCommitDigest !== + adopted.legacyDataApplication.expectedCommitDigest || + bundle.legacyDataApplicationReceiptDigest !== + adopted.legacyDataApplication.expectedReceiptDigest + ) { + configurationError('adopted bundle application binding drifted'); + } + const initialSelection = inspectComposeImageSelectionGeneration( + paths.composeRevisions, + 1, + uid, + ); + const command: Readonly = + Object.freeze({ + schemaVersion: 1 as const, + operation: 'local.deployment.adopted.verify' as const, + options: Object.freeze({ + deploymentRoot, + profile: adopted.profile, + instanceId: adopted.instanceId, + ...(adopted.busyTimeoutMs === undefined + ? {} + : { busyTimeoutMs: adopted.busyTimeoutMs }), + service: Object.freeze({ + kind: 'compose' as const, + allowRootService, + releaseSelection: Object.freeze({ + path: paths.composeSelection, + expectedSelectionDigest: initialSelection.selectionDigest, + authority: initialSelection, + }), + }), + }), + request: Object.freeze({ + bundleId: bundle.bundleId, + preparedAtMs: bundle.preparedAtMs, + cutoverId: adopted.cutover.cutoverId, + storage: adopted.storage, + cutover: Object.freeze({ + commitmentPath: adopted.cutover.commitmentPath, + expectedCommitmentDigest: adopted.cutover.expectedCommitmentDigest, + }), + legacyDataApplication: adopted.legacyDataApplication, + }), + }); + const evidence = verifyLocalDeploymentAdoptedEvidence(command, uid, gid); + const material = renderLocalDeploymentAdoptedBundleMaterial( + command, + evidence, + uid, + gid, + ); + verifyExactPrivateFile( + material.paths.applicationConfig, + material.applicationConfig, + uid, + gid, + 'adopted application configuration', + ); + verifyExactPrivateFile( + material.paths.descriptor, + material.descriptor.contents, + uid, + gid, + 'adopted Compose descriptor', + ); + verifyExactPrivateFile( + material.paths.bundleReceipt, + material.receiptContents, + uid, + gid, + 'adopted bundle receipt', + ); + if (material.composeSelection === null) { + configurationError('adopted Compose selection is unavailable'); + } + verifyExactPrivateFile( + material.paths.composeRevision, + material.composeSelection, + uid, + gid, + 'initial adopted Compose revision', + ); + return Object.freeze({ + mode: 'adopted' as const, + instanceId: adopted.instanceId, + profile: adopted.profile, + ...(adopted.busyTimeoutMs === undefined + ? {} + : { busyTimeoutMs: adopted.busyTimeoutMs }), + databasePath: adopted.storage.targetPath, + sourcePath: adopted.storage.sourcePath, + bundleId: bundle.bundleId, + receipt: lineageReceipt({ + mode: 'adopted', + applicationConfigDigest: bundle.applicationConfigDigest, + adoptedBundleDigest: bundle.bundleDigest, + activationDigest: bundle.activationDigest, + commitmentDigest: bundle.commitmentDigest, + legacyDataApplicationCommitDigest: + bundle.legacyDataApplicationCommitDigest, + legacyDataApplicationReceiptDigest: + bundle.legacyDataApplicationReceiptDigest, + }), + }); +} + +export function normalizeLocalDeploymentComposeLineageReceipt( + value: unknown, +): Readonly { + const receipt = object(value, 'Compose deployment lineage'); + exact( + receipt, + [ + 'activationDigest', + 'adoptedBundleDigest', + 'applicationConfigDigest', + 'commitmentDigest', + 'legacyDataApplicationCommitDigest', + 'legacyDataApplicationReceiptDigest', + 'mode', + 'schema', + ], + 'Compose deployment lineage', + ); + const adopted = receipt.mode === 'adopted'; + if ( + receipt.schema !== LINEAGE_SCHEMA || + (receipt.mode !== 'fresh' && !adopted) || + typeof receipt.applicationConfigDigest !== 'string' || + !DIGEST_PATTERN.test(receipt.applicationConfigDigest) || + [ + receipt.adoptedBundleDigest, + receipt.activationDigest, + receipt.commitmentDigest, + receipt.legacyDataApplicationCommitDigest, + receipt.legacyDataApplicationReceiptDigest, + ].some((candidate) => + adopted + ? typeof candidate !== 'string' || !DIGEST_PATTERN.test(candidate) + : candidate !== null, + ) + ) { + configurationError('Compose deployment lineage is invalid'); + } + return Object.freeze({ + schema: LINEAGE_SCHEMA, + mode: receipt.mode, + applicationConfigDigest: receipt.applicationConfigDigest, + adoptedBundleDigest: receipt.adoptedBundleDigest, + activationDigest: receipt.activationDigest, + commitmentDigest: receipt.commitmentDigest, + legacyDataApplicationCommitDigest: + receipt.legacyDataApplicationCommitDigest, + legacyDataApplicationReceiptDigest: + receipt.legacyDataApplicationReceiptDigest, + }) as Readonly; +} + +export function assertLocalDeploymentComposeLineageReceipt( + value: unknown, + expected: Readonly, +): void { + const actual = normalizeLocalDeploymentComposeLineageReceipt(value); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + configurationError('Compose deployment lineage drifted'); + } +} diff --git a/packages/ql3-local-owner-cli/src/deployment/compose/composePreflight.ts b/packages/ql3-local-owner-cli/src/deployment/compose/composePreflight.ts index 9e17cf62..d56cdc0f 100644 --- a/packages/ql3-local-owner-cli/src/deployment/compose/composePreflight.ts +++ b/packages/ql3-local-owner-cli/src/deployment/compose/composePreflight.ts @@ -1,4 +1,3 @@ -import fs from 'node:fs'; import path from 'node:path'; import { readPrivateLocalCommandFile } from '@qinglong/local-command-file'; @@ -31,6 +30,10 @@ import { deploymentPaths, descriptor, } from '../foundation/render'; +import { + inspectLocalDeploymentComposeLineage, + type LocalDeploymentComposeLineage, +} from './composeLineage'; const CONTAINER_ROOT = '/var/lib/qinglong3'; const MAX_DOCKER_OUTPUT_BYTES = 256 * 1024; @@ -46,87 +49,10 @@ export interface LocalDeploymentComposePreflightDependencies { readonly validateSocket?: (socketPath: string, uid: number) => void; } -interface ApplicationIdentity { - readonly instanceId: string; - readonly profile: LocalDeploymentProfile; - readonly busyTimeoutMs?: number; -} - function configurationError(message: string, cause?: unknown): never { throw new LocalDeploymentConfigurationError(message, { cause }); } -function readApplicationIdentity( - filePath: string, - uid: number, -): Readonly { - let stat: fs.Stats; - try { - stat = fs.lstatSync(filePath); - } catch (error) { - configurationError('application configuration is unavailable', error); - } - if ( - !stat.isFile() || - stat.isSymbolicLink() || - stat.uid !== uid || - (stat.mode & 0o777) !== 0o600 || - stat.nlink !== 1 || - stat.size < 2 || - stat.size > 64 * 1024 - ) { - configurationError('application configuration identity is invalid'); - } - let value: unknown; - try { - value = JSON.parse(fs.readFileSync(filePath, 'utf8')); - } catch (error) { - configurationError('application configuration is invalid', error); - } - const candidate = value as { - readonly schema?: unknown; - readonly instanceId?: unknown; - readonly profile?: unknown; - readonly storage?: { - readonly mode?: unknown; - readonly databasePath?: unknown; - readonly busyTimeoutMs?: unknown; - }; - }; - if ( - !candidate || - typeof candidate !== 'object' || - Array.isArray(candidate) || - candidate.schema !== 'qinglong/local-application-process@v2' || - typeof candidate.instanceId !== 'string' || - (candidate.profile !== 'edge' && candidate.profile !== 'standalone') || - !candidate.storage || - candidate.storage.mode !== 'fresh' || - candidate.storage.databasePath !== `${CONTAINER_ROOT}/qinglong3.sqlite` - ) { - configurationError('application configuration is not a fresh Compose v2'); - } - const busyTimeoutMs = - candidate.storage.busyTimeoutMs === undefined - ? undefined - : candidate.storage.busyTimeoutMs; - if ( - busyTimeoutMs !== undefined && - (!Number.isSafeInteger(busyTimeoutMs) || - (busyTimeoutMs as number) < 100 || - (busyTimeoutMs as number) > 30_000) - ) { - configurationError('application busyTimeoutMs is invalid'); - } - return Object.freeze({ - instanceId: candidate.instanceId, - profile: candidate.profile, - ...(busyTimeoutMs === undefined - ? {} - : { busyTimeoutMs: busyTimeoutMs as number }), - }); -} - function parseJson(value: string, label: string): unknown { if ( Buffer.byteLength(value, 'utf8') < 2 || @@ -157,6 +83,7 @@ function inspectImage( image: string, profile: LocalDeploymentProfile, sqliteContractVersion: number, + applicationConfigVersion: '2' | '4', ): 'amd64' | 'arm64' { const parsed = parseJson(output, 'Docker image inspect') as unknown[]; if (!Array.isArray(parsed) || parsed.length !== 1) { @@ -194,6 +121,10 @@ function inspectImage( const minimum = safeLabel(labels, 'io.qinglong.local.sqlite-contract-min'); const maximum = safeLabel(labels, 'io.qinglong.local.sqlite-contract-max'); const profiles = safeLabel(labels, 'io.qinglong.profile').split(','); + const applicationConfigs = safeLabel( + labels, + 'io.qinglong.local.application-config', + ).split(','); if ( !/^[1-9][0-9]{0,3}$/.test(minimum) || !/^[1-9][0-9]{0,3}$/.test(maximum) || @@ -201,7 +132,7 @@ function inspectImage( Number(maximum) < sqliteContractVersion || safeLabel(labels, 'io.qinglong.local.sqlite-write-contract') !== String(sqliteContractVersion) || - safeLabel(labels, 'io.qinglong.local.application-config') !== '2' || + !applicationConfigs.includes(applicationConfigVersion) || safeLabel(labels, 'io.qinglong.local.compose-selection') !== '1' || safeLabel(labels, 'io.qinglong.ai') !== 'excluded' || !profiles.includes(profile) || @@ -233,10 +164,15 @@ function inspectComposeConfig( image: string; generation: number; mutationId: string; + releaseSelectionDigest: string; + releaseSetDigest: string; + catalogManifestDigest: string; + catalogConsumptionReportDigest: string; deploymentRoot: string; uid: number; gid: number; profile: LocalDeploymentProfile; + lineage: Readonly; }>, ): void { const config = parseJson(output, 'Docker Compose config') as { @@ -267,7 +203,8 @@ function inspectComposeConfig( service.user !== `${expected.uid}:${expected.gid}` || service.read_only !== true || service.network_mode !== 'none' || - service.restart !== 'unless-stopped' || + service.restart !== + (expected.lineage.mode === 'fresh' ? 'unless-stopped' : 'no') || Number(service.mem_limit) !== expectedMemory || Number(service.pids_limit) !== expectedPids || service.privileged === true || @@ -279,7 +216,9 @@ function inspectComposeConfig( !exactArray(service.security_opt, ['no-new-privileges:true']) || !exactArray(service.command, [ '--config', - `${CONTAINER_ROOT}/local-application.json`, + expected.lineage.mode === 'fresh' + ? `${CONTAINER_ROOT}/local-application.json` + : path.join(expected.deploymentRoot, 'local-application.json'), ]) ) { configurationError('Docker Compose service contract is invalid'); @@ -287,9 +226,44 @@ function inspectComposeConfig( const labels = service.labels as | Readonly> | undefined; + const expectedLabels = + expected.lineage.mode === 'fresh' + ? { + 'io.qinglong.deployment.generation': String(expected.generation), + 'io.qinglong.deployment.mutation': expected.mutationId, + 'io.qinglong.release.selection': expected.releaseSelectionDigest, + 'io.qinglong.release.set': expected.releaseSetDigest, + 'io.qinglong.release.catalog-manifest': + expected.catalogManifestDigest, + 'io.qinglong.release.catalog-report': + expected.catalogConsumptionReportDigest, + } + : { + 'io.qinglong.deployment.mode': 'adopted', + 'io.qinglong.deployment.profile': expected.profile, + 'io.qinglong.deployment.instance': expected.lineage.instanceId, + 'io.qinglong.deployment.bundle': expected.lineage.bundleId!, + 'io.qinglong.application.config': + expected.lineage.receipt.applicationConfigDigest, + 'io.qinglong.data.commit': + expected.lineage.receipt.legacyDataApplicationCommitDigest!, + 'io.qinglong.data.receipt': + expected.lineage.receipt.legacyDataApplicationReceiptDigest!, + 'io.qinglong.deployment.generation': String(expected.generation), + 'io.qinglong.deployment.mutation': expected.mutationId, + 'io.qinglong.release.selection': expected.releaseSelectionDigest, + 'io.qinglong.release.set': expected.releaseSetDigest, + 'io.qinglong.release.catalog-manifest': + expected.catalogManifestDigest, + 'io.qinglong.release.catalog-report': + expected.catalogConsumptionReportDigest, + }; if ( !labels || - Object.keys(labels).length !== 2 || + Object.keys(labels).length !== Object.keys(expectedLabels).length || + Object.entries(expectedLabels).some( + ([key, value]) => labels[key] !== value, + ) || labels['io.qinglong.deployment.generation'] !== String(expected.generation) || labels['io.qinglong.deployment.mutation'] !== expected.mutationId @@ -297,15 +271,32 @@ function inspectComposeConfig( configurationError('Docker Compose deployment labels are invalid'); } const volumes = service.volumes as readonly unknown[] | undefined; - const volume = volumes?.[0] as Readonly> | undefined; const tmpfs = service.tmpfs as readonly unknown[] | undefined; + const expectedVolumes = + expected.lineage.mode === 'fresh' + ? [ + { + type: 'bind', + source: expected.deploymentRoot, + target: CONTAINER_ROOT, + }, + ] + : [ + { + type: 'bind', + source: expected.deploymentRoot, + target: expected.deploymentRoot, + }, + { + type: 'bind', + source: expected.lineage.sourcePath, + target: expected.lineage.sourcePath, + read_only: true, + }, + ]; if ( !Array.isArray(volumes) || - volumes.length !== 1 || - !volume || - volume.type !== 'bind' || - volume.source !== expected.deploymentRoot || - volume.target !== CONTAINER_ROOT || + JSON.stringify(volumes) !== JSON.stringify(expectedVolumes) || !Array.isArray(tmpfs) || tmpfs.length !== 1 || typeof tmpfs[0] !== 'string' || @@ -353,62 +344,66 @@ export async function preflightLocalDeploymentCompose( 'active compose generation does not match expectedGeneration', ); } - const application = readApplicationIdentity( - paths.applicationConfig, - identity.uid, - ); - const syntheticPrepare: Readonly = { - schemaVersion: 1, - operation: 'local.deployment.prepare', - options: { - deploymentRoot: command.options.deploymentRoot, - profile: application.profile, - instanceId: application.instanceId, - ...(application.busyTimeoutMs === undefined - ? {} - : { busyTimeoutMs: application.busyTimeoutMs }), - service: { - kind: 'compose', - releaseSelection: { - path: paths.composeSelection, - expectedSelectionDigest: selection.selectionDigest, - }, - allowRootService: command.options.allowRootService, - }, - }, - request: { - ownerPepperKeyId: 'preflight', - registerMutationId: '00000000-0000-4000-8000-000000000001', - activateMutationId: '00000000-0000-4000-8000-000000000002', - registeredAtMs: 0, - activatedAtMs: 0, - }, - }; - preflightPublishedFile( - paths.applicationConfig, - applicationConfiguration(syntheticPrepare, paths), - 0o600, - identity.uid, - 'application configuration', - ); - const expectedDescriptor = descriptor( - syntheticPrepare, - paths.applicationConfig, + const application = inspectLocalDeploymentComposeLineage( + command.options.deploymentRoot, identity.uid, identity.gid, + command.options.allowRootService, ); - preflightPublishedFile( - path.join(paths.service, expectedDescriptor.fileName), - expectedDescriptor.contents, - expectedDescriptor.mode, - identity.uid, - 'service descriptor', - ); + if (application.mode === 'fresh') { + const syntheticPrepare: Readonly = { + schemaVersion: 1, + operation: 'local.deployment.prepare', + options: { + deploymentRoot: command.options.deploymentRoot, + profile: application.profile, + instanceId: application.instanceId, + ...(application.busyTimeoutMs === undefined + ? {} + : { busyTimeoutMs: application.busyTimeoutMs }), + service: { + kind: 'compose', + releaseSelection: { + path: paths.composeSelection, + expectedSelectionDigest: selection.selectionDigest, + }, + allowRootService: command.options.allowRootService, + }, + }, + request: { + ownerPepperKeyId: 'preflight', + registerMutationId: '00000000-0000-4000-8000-000000000001', + activateMutationId: '00000000-0000-4000-8000-000000000002', + registeredAtMs: 0, + activatedAtMs: 0, + }, + }; + preflightPublishedFile( + paths.applicationConfig, + applicationConfiguration(syntheticPrepare, paths), + 0o600, + identity.uid, + 'application configuration', + ); + const expectedDescriptor = descriptor( + syntheticPrepare, + paths.applicationConfig, + identity.uid, + identity.gid, + ); + preflightPublishedFile( + path.join(paths.service, expectedDescriptor.fileName), + expectedDescriptor.contents, + expectedDescriptor.mode, + identity.uid, + 'service descriptor', + ); + } const auditSqlite = dependencies.auditSqlite ?? inspectLocalSqliteReadinessPath; const sqlite = await auditSqlite({ - databasePath: paths.database, + databasePath: application.databasePath, profile: application.profile, ...(application.busyTimeoutMs === undefined ? {} @@ -429,6 +424,7 @@ export async function preflightLocalDeploymentCompose( selection.image, application.profile, sqlite.contractVersion, + application.mode === 'fresh' ? '2' : '4', ); const composeOutput = runDocker({ executable: command.options.dockerExecutable, @@ -451,10 +447,15 @@ export async function preflightLocalDeploymentCompose( image: selection.image, generation: selection.generation, mutationId: selection.mutationId, + releaseSelectionDigest: selection.selectionDigest, + releaseSetDigest: selection.releaseSetDigest, + catalogManifestDigest: selection.catalogManifestDigest, + catalogConsumptionReportDigest: selection.catalogConsumptionReportDigest, deploymentRoot: command.options.deploymentRoot, uid: identity.uid, gid: identity.gid, profile: application.profile, + lineage: application, }); return Object.freeze({ diff --git a/packages/ql3-local-owner-cli/src/deployment/compose/composeRestore.ts b/packages/ql3-local-owner-cli/src/deployment/compose/composeRestore.ts index c2e4160b..042252c8 100644 --- a/packages/ql3-local-owner-cli/src/deployment/compose/composeRestore.ts +++ b/packages/ql3-local-owner-cli/src/deployment/compose/composeRestore.ts @@ -49,9 +49,16 @@ import { deploymentPaths, type LocalDeploymentPaths, } from '../foundation/render'; +import { + assertLocalDeploymentComposeLineageReceipt, + inspectLocalDeploymentComposeLineage, + normalizeLocalDeploymentComposeLineageReceipt, + type LocalDeploymentComposeLineage, + type LocalDeploymentComposeLineageReceipt, +} from './composeLineage'; -const PREPARE_SCHEMA = 'qinglong/local-compose-restore-prepare@v1'; -const COMMIT_SCHEMA = 'qinglong/local-compose-restore-commit@v1'; +const PREPARE_SCHEMA = 'qinglong/local-compose-restore-prepare@v2'; +const COMMIT_SCHEMA = 'qinglong/local-compose-restore-commit@v2'; const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; const SHA256_PATTERN = /^[0-9a-f]{64}$/; @@ -68,6 +75,7 @@ interface RestorePrepareReceipt { readonly generation: number; readonly recordedAtMs: number; readonly profile: LocalDeploymentProfile; + readonly lineage: Readonly; readonly current: Readonly; readonly source: Readonly; readonly safeguard: Readonly; @@ -82,6 +90,7 @@ interface RestoreCommitReceipt { readonly generation: number; readonly recordedAtMs: number; readonly profile: LocalDeploymentProfile; + readonly lineage: Readonly; readonly source: Readonly; readonly safeguard: Readonly; readonly restored: Readonly; @@ -211,40 +220,6 @@ function boundedReceipt( return fs.readFileSync(filePath, 'utf8'); } -function readApplicationProfile( - paths: Readonly, - uid: number, -): LocalDeploymentProfile { - const contents = boundedReceipt( - paths.applicationConfig, - uid, - 'application configuration', - ); - let value: unknown; - try { - value = JSON.parse(contents); - } catch (error) { - configurationError('application configuration is invalid', error); - } - const candidate = value as { - readonly schema?: unknown; - readonly profile?: unknown; - readonly storage?: { - readonly mode?: unknown; - readonly databasePath?: unknown; - }; - }; - if ( - candidate?.schema !== 'qinglong/local-application-process@v2' || - (candidate.profile !== 'edge' && candidate.profile !== 'standalone') || - candidate.storage?.mode !== 'fresh' || - candidate.storage.databasePath !== '/var/lib/qinglong3/qinglong3.sqlite' - ) { - configurationError('application configuration restore binding drifted'); - } - return candidate.profile; -} - function composeArgs( paths: Readonly, args: readonly string[], @@ -280,12 +255,17 @@ function stopAndVerifyCompose( paths: Readonly, runDocker: LocalDeploymentDockerRunner, ): void { - docker( - command, - runDocker, - composeArgs(paths, ['stop', '--timeout', '30', 'qinglong3']), - 45_000, - ); + try { + docker( + command, + runDocker, + composeArgs(paths, ['stop', '--timeout', '30', 'qinglong3']), + 45_000, + ); + } catch { + // A lost Docker response is reconciled by the inspect-only observation + // below. The restore ceremony never repeats `compose stop` blindly. + } const containerId = docker( command, runDocker, @@ -418,6 +398,7 @@ async function readRestoreState( command: Readonly, paths: Readonly, uid: number, + lineage: Readonly, ): Promise> { const failedIntent = boundedReceipt( paths.composeRolloutLock, @@ -478,7 +459,7 @@ async function readRestoreState( ) { configurationError('compose restore recovery generation drifted'); } - const profile = readApplicationProfile(paths, uid); + const profile = lineage.profile; const sourcePath = path.join( paths.composeRolloutBackups, `${command.request.sourceRolloutId}.sqlite`, @@ -503,6 +484,7 @@ async function inspectPrepareReceipt( expectedCommandDigest: string, command: Readonly, paths: Readonly, + expectedLineage: Readonly, ): Promise | null> { if (!fs.existsSync(filePath)) return null; const contents = boundedReceipt( @@ -518,6 +500,9 @@ async function inspectPrepareReceipt( configurationError('compose restore prepare receipt is invalid', error); } const receipt = value as RestorePrepareReceipt; + const lineage = normalizeLocalDeploymentComposeLineageReceipt( + receipt.lineage, + ); if ( !receipt || typeof receipt !== 'object' || @@ -528,6 +513,7 @@ async function inspectPrepareReceipt( 'current', 'failedApplyCommandDigest', 'generation', + 'lineage', 'profile', 'recordedAtMs', 'restoreId', @@ -550,10 +536,12 @@ async function inspectPrepareReceipt( !snapshotIsValid(receipt.source) || !snapshotIsValid(receipt.safeguard) || receipt.current.sha256 === receipt.source.sha256 || + JSON.stringify(receipt.lineage) !== JSON.stringify(lineage) || contents !== receiptContents(receipt) ) { configurationError('compose restore prepare receipt drifted'); } + assertLocalDeploymentComposeLineageReceipt(lineage, expectedLineage.receipt); const source = await inspectLocalSqliteSnapshot({ databasePath: path.join( paths.composeRolloutBackups, @@ -586,6 +574,7 @@ function inspectCommitReceipt( uid: number, expectedCommandDigest: string, command: Readonly, + expectedLineage: Readonly, ): Readonly | null { if (!fs.existsSync(filePath)) return null; const contents = boundedReceipt( @@ -601,6 +590,9 @@ function inspectCommitReceipt( configurationError('compose restore commit receipt is invalid', error); } const receipt = value as RestoreCommitReceipt; + const lineage = normalizeLocalDeploymentComposeLineageReceipt( + receipt.lineage, + ); if ( !receipt || typeof receipt !== 'object' || @@ -609,6 +601,7 @@ function inspectCommitReceipt( [ 'commandDigest', 'generation', + 'lineage', 'prepareReceiptDigest', 'profile', 'recordedAtMs', @@ -636,10 +629,12 @@ function inspectCommitReceipt( !snapshotIsValid(receipt.safeguard) || !snapshotIsValid(receipt.restored) || receipt.source.sha256 !== receipt.restored.sha256 || + JSON.stringify(receipt.lineage) !== JSON.stringify(lineage) || contents !== receiptContents(receipt) ) { configurationError('compose restore commit receipt drifted'); } + assertLocalDeploymentComposeLineageReceipt(lineage, expectedLineage.receipt); publishExactFile( filePath, contents, @@ -671,6 +666,7 @@ async function prepareRestore( command: Readonly, paths: Readonly, dependencies: LocalDeploymentComposeRestoreDependencies, + lineage: Readonly, ): Promise> { const identity = currentIdentity(); if (fs.existsSync(commitReceiptPath(paths, command.request.restoreId))) { @@ -680,7 +676,7 @@ async function prepareRestore( dependencies.validateSocket ?? validateLocalDeploymentDockerSocket; validateSocket(command.options.dockerSocketPath, identity.uid); const runDocker = dependencies.runDocker ?? runLocalDeploymentDockerCommand; - const state = await readRestoreState(command, paths, identity.uid); + const state = await readRestoreState(command, paths, identity.uid, lineage); const intent = `${JSON.stringify(command, null, 2)}\n`; publishExactFile( paths.composeRestoreLock, @@ -701,10 +697,11 @@ async function prepareRestore( commandDigest(command), command, paths, + lineage, ); if (replay) { const current = await inspectLocalSqliteSnapshot({ - databasePath: paths.database, + databasePath: lineage.databasePath, profile: replay.profile, }); exactSnapshot(replay.current, current, 'compose restore current database'); @@ -717,7 +714,7 @@ async function prepareRestore( } const checkpoint = dependencies.checkpoint ?? checkpointLocalSqliteForRestore; const current = await checkpoint({ - databasePath: paths.database, + databasePath: lineage.databasePath, profile: state.profile, }); if (current.sha256 === state.source.sha256) { @@ -728,7 +725,7 @@ async function prepareRestore( const createSafeguard = dependencies.createSafeguard ?? createLocalSqliteRolloutBackup; const safeguard = await createSafeguard({ - databasePath: paths.database, + databasePath: lineage.databasePath, backupPath: safeguardPath(paths, command.request.restoreId), profile: state.profile, }); @@ -741,6 +738,7 @@ async function prepareRestore( generation: command.request.expectedGeneration, recordedAtMs: command.request.preparedAtMs, profile: state.profile, + lineage: lineage.receipt, current: snapshotReceipt(current), source: snapshotReceipt(state.source), safeguard: snapshotReceipt(safeguard), @@ -764,6 +762,7 @@ async function commitRestore( command: Readonly, paths: Readonly, dependencies: LocalDeploymentComposeRestoreDependencies, + lineage: Readonly, ): Promise> { if (command.operation !== 'local.deployment.compose.restore.commit') { configurationError('compose restore commit command is invalid'); @@ -775,6 +774,7 @@ async function commitRestore( identity.uid, commandDigest(command), command, + lineage, ); if (replay) { const sourcePath = path.join( @@ -945,11 +945,17 @@ async function commitRestore( commandDigest(prepareCommand), prepareCommand, paths, + lineage, ); if (!prepareReceipt) { configurationError('compose restore prepare receipt is unavailable'); } - const state = await readRestoreState(prepareCommand, paths, identity.uid); + const state = await readRestoreState( + prepareCommand, + paths, + identity.uid, + lineage, + ); if ( state.failedCommandDigest !== prepareReceipt.failedApplyCommandDigest || state.source.sha256 !== prepareReceipt.source.sha256 @@ -961,11 +967,11 @@ async function commitRestore( validateSocket(command.options.dockerSocketPath, identity.uid); const runDocker = dependencies.runDocker ?? runLocalDeploymentDockerCommand; stopAndVerifyCompose(command, paths, runDocker); - if (fs.existsSync(paths.database)) { + if (fs.existsSync(lineage.databasePath)) { const checkpoint = dependencies.checkpoint ?? checkpointLocalSqliteForRestore; const current = await checkpoint({ - databasePath: paths.database, + databasePath: lineage.databasePath, profile: prepareReceipt.profile, }); if ( @@ -980,12 +986,13 @@ async function commitRestore( const restoreSnapshot = dependencies.restoreSnapshot ?? restoreLocalSqliteSnapshot; const restored: Readonly = await restoreSnapshot({ - databasePath: paths.database, + databasePath: lineage.databasePath, profile: prepareReceipt.profile, + preserveDatabaseIdentity: lineage.mode === 'adopted', sourceSnapshotPath: state.sourcePath, restoreStagePath: path.join( - path.dirname(paths.database), - `.${path.basename(paths.database)}.${ + path.dirname(lineage.databasePath), + `.${path.basename(lineage.databasePath)}.${ command.request.restoreId }.restore-stage`, ), @@ -1011,6 +1018,7 @@ async function commitRestore( generation: command.request.expectedGeneration, recordedAtMs: command.request.committedAtMs, profile: prepareReceipt.profile, + lineage: lineage.receipt, source: prepareReceipt.source, safeguard: prepareReceipt.safeguard, restored: snapshotReceipt(restored), @@ -1073,13 +1081,19 @@ export async function restoreLocalDeploymentCompose( identity.uid, 'composeRestoreSafeguardRoot', ); + const lineage = inspectLocalDeploymentComposeLineage( + command.options.deploymentRoot, + identity.uid, + identity.gid, + command.options.allowRootService, + ); if (fs.existsSync(paths.composeEvidenceCollectionLock)) { configurationError('compose restore is fenced by an evidence collection'); } preflightRestoreCatalog(paths, command.request.restoreId); return command.operation === 'local.deployment.compose.restore.prepare' - ? prepareRestore(command, paths, dependencies) - : commitRestore(command, paths, dependencies); + ? prepareRestore(command, paths, dependencies, lineage) + : commitRestore(command, paths, dependencies, lineage); } export function restoreLocalDeploymentComposeCommandFile( diff --git a/packages/ql3-local-owner-cli/test/adoptedDeploymentBundle.test.cjs b/packages/ql3-local-owner-cli/test/adoptedDeploymentBundle.test.cjs index 205baf0f..d9330780 100644 --- a/packages/ql3-local-owner-cli/test/adoptedDeploymentBundle.test.cjs +++ b/packages/ql3-local-owner-cli/test/adoptedDeploymentBundle.test.cjs @@ -4,16 +4,26 @@ const crypto = require('node:crypto'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); const { test } = require('node:test'); const { LocalDeploymentConfigurationError, + applyLocalDeploymentCompose, + collectLocalDeploymentComposeEvidence, prepareLocalDeploymentAdoptedBundle, + preflightLocalDeploymentCompose, + restoreLocalDeploymentCompose, + switchLocalDeploymentComposeRevision, verifyLocalDeploymentAdoptedBundle, } = require('../dist/deployment/localDeployment.js'); const { createLocalDataDirectoryApplicationCommit, } = require('@qinglong/local-sqlite/data-directory-application-commit'); +const { migrateLocalSqlitePath } = require('@qinglong/local-sqlite/migration'); +const { + createLocalSqliteRolloutBackup, +} = require('@qinglong/local-sqlite/rollout-safety'); const { normalizeLocalApplicationProcessConfig, } = require('../../ql3-local-application/dist/production-process/processConfig.js'); @@ -42,8 +52,8 @@ function writePrivate(filePath, value) { ); } -function releaseSelection(managementRoot) { - const image = `ghcr.io/example/qinglong3-local-application@sha256:${'a'.repeat( +function releaseSelection(managementRoot, marker = 'a') { + const image = `ghcr.io/example/qinglong3-local-application@sha256:${marker.repeat( 64, )}`; const releaseSetDigest = prefixedDigest(`release-set:${image}`); @@ -86,7 +96,10 @@ function releaseSelection(managementRoot) { }, }; const selectionDigest = prefixedDigest(JSON.stringify(unsigned)); - const filePath = path.join(managementRoot, 'release-selection.json'); + const filePath = path.join( + managementRoot, + `release-selection-${marker}.json`, + ); writePrivate(filePath, { ...unsigned, selectionDigest }); return { path: filePath, @@ -247,13 +260,373 @@ function fixture(t, kind) { }; return { root, + managementRoot, command, commitPath, commitmentPath, sourcePath, + targetPath, + recoveryPath, + manifestPath, + activationPath, }; } +async function prepareRuntimeComposeFixture(t) { + const state = fixture(t, 'compose'); + fs.unlinkSync(state.targetPath); + await migrateLocalSqlitePath({ + databasePath: state.targetPath, + profile: 'edge', + busyTimeoutMs: 100, + }); + const target = fs.statSync(state.targetPath, { bigint: true }); + const activation = JSON.parse(fs.readFileSync(state.activationPath, 'utf8')); + delete activation.activationDigest; + activation.targetSha256 = hexDigest(fs.readFileSync(state.targetPath)); + activation.targetDevice = target.dev.toString(); + activation.targetInode = target.ino.toString(); + const activationDigest = canonicalDigest(activation); + writePrivate(state.activationPath, { ...activation, activationDigest }); + const commitment = JSON.parse(fs.readFileSync(state.commitmentPath, 'utf8')); + delete commitment.commitmentDigest; + commitment.activationDigest = activationDigest; + const commitmentDigest = canonicalDigest(commitment); + writePrivate(state.commitmentPath, { ...commitment, commitmentDigest }); + state.command.request.storage.expectedActivationDigest = activationDigest; + state.command.request.cutover.expectedCommitmentDigest = commitmentDigest; + prepareLocalDeploymentAdoptedBundle(state.command); + return state; +} + +function selectedComposeGeneration(state) { + const contents = fs.readFileSync( + path.join(state.root, 'service/compose.image.yaml'), + 'utf8', + ); + return { + generation: Number(/^ generation: ([0-9]+)$/m.exec(contents)[1]), + mutationId: /^ mutation_id: ([0-9a-f-]+)$/m.exec(contents)[1], + image: /^ image: ([^\n]+)$/m.exec(contents)[1], + releaseSelectionDigest: /^ release_selection_digest: ([^\n]+)$/m.exec( + contents, + )[1], + releaseSetDigest: /^ release_set_digest: ([^\n]+)$/m.exec(contents)[1], + catalogManifestDigest: /^ catalog_manifest_digest: ([^\n]+)$/m.exec( + contents, + )[1], + catalogConsumptionReportDigest: + /^ catalog_consumption_report_digest: ([^\n]+)$/m.exec(contents)[1], + }; +} + +function adoptedDockerHarness(state, options = {}) { + const calls = []; + const containerId = '1'.repeat(64); + const bundle = JSON.parse( + fs.readFileSync(path.join(state.root, 'service/adopted-bundle.json')), + ); + const compose = fs.readFileSync( + path.join(state.root, 'service/compose.yaml'), + 'utf8', + ); + const projectName = /^name: ([a-z0-9_-]+)$/m.exec(compose)[1]; + let running = false; + let loseNextUpResponse = options.loseNextUpResponse === true; + const labels = (selected) => ({ + 'io.qinglong.deployment.mode': 'adopted', + 'io.qinglong.deployment.profile': 'edge', + 'io.qinglong.deployment.instance': 'edge-router-1', + 'io.qinglong.deployment.bundle': bundle.bundleId, + 'io.qinglong.application.config': bundle.applicationConfigDigest, + 'io.qinglong.data.commit': bundle.legacyDataApplicationCommitDigest, + 'io.qinglong.data.receipt': bundle.legacyDataApplicationReceiptDigest, + 'io.qinglong.deployment.generation': String(selected.generation), + 'io.qinglong.deployment.mutation': selected.mutationId, + 'io.qinglong.release.selection': selected.releaseSelectionDigest, + 'io.qinglong.release.set': selected.releaseSetDigest, + 'io.qinglong.release.catalog-manifest': selected.catalogManifestDigest, + 'io.qinglong.release.catalog-report': + selected.catalogConsumptionReportDigest, + }); + const runDocker = ({ args }) => { + calls.push(args); + const selected = selectedComposeGeneration(state); + if (args[0] === 'image') { + return JSON.stringify([ + { + Id: `sha256:${'2'.repeat(64)}`, + RepoDigests: [selected.image], + Architecture: 'arm64', + Os: 'linux', + Config: { + User: '65532:65532', + Entrypoint: [ + 'node', + '/opt/qinglong/node_modules/@qinglong/local-application/dist/cli.js', + ], + Labels: { + 'io.qinglong.local.sqlite-contract-min': '50', + 'io.qinglong.local.sqlite-contract-max': '50', + 'io.qinglong.local.sqlite-write-contract': '50', + 'io.qinglong.local.application-config': '2,3,4', + 'io.qinglong.local.compose-selection': '1', + 'io.qinglong.ai': 'excluded', + 'io.qinglong.profile': 'edge,standalone', + 'org.opencontainers.image.source': + 'https://github.com/whyour/qinglong', + 'org.opencontainers.image.revision': '3'.repeat(40), + 'org.opencontainers.image.version': '3.0.0-alpha.0', + }, + }, + }, + ]); + } + if (args[0] === 'compose' && args.includes('config')) { + const volumes = [ + { type: 'bind', source: state.root, target: state.root }, + { + type: 'bind', + source: state.sourcePath, + target: state.sourcePath, + read_only: true, + }, + ]; + if (options.driftMount === true) volumes[0].target = '/var/lib/qinglong3'; + return JSON.stringify({ + name: projectName, + services: { + qinglong3: { + image: selected.image, + user: `${process.getuid()}:${process.getgid()}`, + read_only: true, + network_mode: 'none', + restart: 'no', + mem_limit: 128 * 1024 * 1024, + pids_limit: 64, + cap_drop: ['ALL'], + security_opt: ['no-new-privileges:true'], + command: [ + '--config', + path.join(state.root, 'local-application.json'), + ], + labels: labels(selected), + volumes, + tmpfs: ['/tmp:rw,noexec,nosuid,nodev,size=16m'], + }, + }, + }); + } + if (args[0] === 'compose' && args.includes('up')) { + running = true; + if (loseNextUpResponse) { + loseNextUpResponse = false; + throw new Error('injected Docker response loss'); + } + return ''; + } + if (args[0] === 'compose' && args.includes('stop')) { + running = false; + return ''; + } + if (args[0] === 'compose' && args.includes('ps')) return `${containerId}\n`; + if (args[0] === 'container' && args[1] === 'inspect') { + return JSON.stringify([ + { + Id: containerId, + State: { Running: running, Status: running ? 'running' : 'exited' }, + Config: { Image: selected.image, Labels: labels(selected) }, + HostConfig: { + ReadonlyRootfs: true, + NetworkMode: 'none', + Privileged: false, + }, + }, + ]); + } + if (args[0] === 'container' && args[1] === 'logs') { + return `${JSON.stringify({ + schemaVersion: 1, + component: 'qinglong3-local-application', + level: 'info', + event: 'active', + profile: 'edge', + aiStatus: 'deployment_excluded', + instanceId: 'edge-router-1', + })}\n`; + } + return ''; + }; + return { + calls, + runDocker, + now: () => 10_000, + wait: async () => {}, + }; +} + +function composeApplyCommand(state, generation, suffix) { + return { + schemaVersion: 1, + operation: 'local.deployment.compose.apply', + options: { + deploymentRoot: state.root, + dockerExecutable: fs.realpathSync(process.execPath), + dockerSocketPath: path.join(state.managementRoot, 'docker.sock'), + allowRootService: rootAcknowledgement(), + }, + request: { + expectedGeneration: generation, + rolloutId: `00000000-0000-4000-8000-000000000d${suffix}`, + startedAtMs: 1786416000500, + failureRollbackMutationId: `00000000-0000-4000-8000-000000000e${suffix}`, + failureRollbackChangedAtMs: 1786416000501, + }, + }; +} + +function composePreflightCommand(state, generation) { + return { + schemaVersion: 1, + operation: 'local.deployment.compose.preflight', + options: { + deploymentRoot: state.root, + dockerExecutable: fs.realpathSync(process.execPath), + dockerSocketPath: path.join(state.managementRoot, 'docker.sock'), + allowRootService: rootAcknowledgement(), + }, + request: { expectedGeneration: generation }, + }; +} + +function composeRevisionCommand(state, operation, request) { + return { + schemaVersion: 1, + operation, + options: { + deploymentRoot: state.root, + allowRootService: rootAcknowledgement(), + }, + request, + }; +} + +function composeRestoreCommands(state, failedCommand, suffix) { + const options = { + deploymentRoot: state.root, + dockerExecutable: fs.realpathSync(process.execPath), + dockerSocketPath: path.join(state.managementRoot, 'docker.sock'), + allowRootService: rootAcknowledgement(), + }; + const restoreId = `00000000-0000-4000-8000-000000000d${suffix}`; + return { + prepare: { + schemaVersion: 1, + operation: 'local.deployment.compose.restore.prepare', + options, + request: { + expectedGeneration: failedCommand.request.expectedGeneration + 1, + restoreId, + sourceRolloutId: failedCommand.request.rolloutId, + preparedAtMs: 1786416000600, + }, + }, + commit: { + schemaVersion: 1, + operation: 'local.deployment.compose.restore.commit', + options, + request: { + expectedGeneration: failedCommand.request.expectedGeneration + 1, + restoreId, + committedAtMs: 1786416000601, + }, + }, + }; +} + +function composeEvidenceCollectionCommands( + state, + generation, + restoreId, + suffix, +) { + const options = { + deploymentRoot: state.root, + allowRootService: rootAcknowledgement(), + }; + const collectionId = `00000000-0000-4000-8000-000000000d${suffix}`; + return { + prepare: { + schemaVersion: 1, + operation: 'local.deployment.compose.evidence-collection.prepare', + options, + request: { + expectedGeneration: generation, + collectionId, + rolloutIds: [], + restoreIds: [restoreId], + preparedAtMs: 1786416000700, + }, + }, + commit: { + schemaVersion: 1, + operation: 'local.deployment.compose.evidence-collection.commit', + options, + request: { + expectedGeneration: generation, + collectionId, + committedAtMs: 1786416000701, + }, + }, + }; +} + +async function createFailedRestoreState(state, currentGeneration, suffix) { + const attemptedGeneration = currentGeneration + 1; + const upgradeMutationId = `00000000-0000-4000-8000-000000000a${suffix}`; + await switchLocalDeploymentComposeRevision( + composeRevisionCommand(state, 'local.deployment.compose.upgrade', { + expectedGeneration: currentGeneration, + releaseSelection: releaseSelection( + state.managementRoot, + String(attemptedGeneration), + ), + mutationId: upgradeMutationId, + changedAtMs: 1786416000400 + attemptedGeneration, + }), + ); + const failedCommand = composeApplyCommand(state, attemptedGeneration, suffix); + const intent = `${JSON.stringify(failedCommand, null, 2)}\n`; + fs.writeFileSync( + path.join(state.root, 'service/.compose-rollout.lock'), + intent, + { mode: 0o600 }, + ); + await createLocalSqliteRolloutBackup({ + databasePath: state.targetPath, + backupPath: path.join( + state.root, + 'service', + 'rollout-backups', + `${failedCommand.request.rolloutId}.sqlite`, + ), + profile: 'edge', + }); + const writer = new DatabaseSync(state.targetPath); + writer.exec(`PRAGMA user_version = ${900 + attemptedGeneration}`); + writer.close(); + await switchLocalDeploymentComposeRevision( + composeRevisionCommand(state, 'local.deployment.compose.rollback', { + expectedGeneration: attemptedGeneration, + targetGeneration: currentGeneration, + mutationId: failedCommand.request.failureRollbackMutationId, + changedAtMs: failedCommand.request.failureRollbackChangedAtMs, + }), + intent, + ); + return failedCommand; +} + for (const kind of ['systemd', 'openrc', 'compose']) { test(`prepares and verifies an exact adopted ${kind} bundle without activation`, (t) => { const state = fixture(t, kind); @@ -406,6 +779,233 @@ test('requires the legacy source to be the only authority outside deployment roo ); }); +test('preflights adopted Compose identity mounts and rejects mount drift', async (t) => { + const state = await prepareRuntimeComposeFixture(t); + const harness = adoptedDockerHarness(state); + const ready = await preflightLocalDeploymentCompose( + composePreflightCommand(state, 1), + { + runDocker: harness.runDocker, + validateSocket() {}, + }, + ); + assert.equal(ready.status, 'ready'); + assert.equal(ready.profile, 'edge'); + assert.equal(ready.sqlite.contractVersion, 50); + await assert.rejects( + preflightLocalDeploymentCompose(composePreflightCommand(state, 1), { + runDocker: adoptedDockerHarness(state, { driftMount: true }).runDocker, + validateSocket() {}, + }), + LocalDeploymentConfigurationError, + ); +}); + +const dockerComposeAvailable = + spawnSync('docker', ['compose', 'version'], { encoding: 'utf8' }).status === + 0; + +test( + 'accepts the real Docker Compose adopted config projection', + { skip: !dockerComposeAvailable }, + async (t) => { + const state = await prepareRuntimeComposeFixture(t); + const harness = adoptedDockerHarness(state); + const runDocker = ({ args, ...request }) => { + if (args[0] === 'compose' && args.includes('config')) { + const result = spawnSync('docker', args, { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + return result.stdout; + } + return harness.runDocker({ args, ...request }); + }; + const ready = await preflightLocalDeploymentCompose( + composePreflightCommand(state, 1), + { runDocker, validateSocket() {} }, + ); + assert.equal(ready.status, 'ready'); + }, +); + +test('recovers adopted Compose up response loss and binds the rollout receipt', async (t) => { + const state = await prepareRuntimeComposeFixture(t); + const harness = adoptedDockerHarness(state, { loseNextUpResponse: true }); + const command = composeApplyCommand(state, 1, '89'); + const dependencies = { + runDocker: harness.runDocker, + validateSocket() {}, + now: harness.now, + wait: harness.wait, + }; + const applied = await applyLocalDeploymentCompose(command, dependencies); + assert.equal(applied.status, 'active'); + assert.equal( + harness.calls.filter((args) => args[0] === 'compose' && args.includes('up')) + .length, + 1, + ); + const receipt = JSON.parse( + fs.readFileSync( + path.join( + state.root, + 'service', + 'rollouts', + `${command.request.rolloutId}.json`, + ), + 'utf8', + ), + ); + assert.equal(receipt.schema, 'qinglong/local-compose-rollout-receipt@v3'); + assert.equal(receipt.lineage.mode, 'adopted'); + assert.equal( + receipt.lineage.legacyDataApplicationCommitDigest, + state.command.request.legacyDataApplication.expectedCommitDigest, + ); + assert.equal( + receipt.lineage.legacyDataApplicationReceiptDigest, + state.command.request.legacyDataApplication.expectedReceiptDigest, + ); + const replay = await applyLocalDeploymentCompose(command, dependencies); + assert.equal(replay.status, 'active'); + assert.equal( + harness.calls.filter((args) => args[0] === 'compose' && args.includes('up')) + .length, + 1, + ); + const commitment = JSON.parse(fs.readFileSync(state.commitmentPath, 'utf8')); + writePrivate(state.commitmentPath, { + ...commitment, + observedAtMs: commitment.observedAtMs + 1, + }); + await assert.rejects( + applyLocalDeploymentCompose(command, dependencies), + LocalDeploymentConfigurationError, + ); + assert.equal( + harness.calls.filter((args) => args[0] === 'compose' && args.includes('up')) + .length, + 1, + ); +}); + +test('preserves adopted restore identity and carries lineage through evidence collection', async (t) => { + const state = await prepareRuntimeComposeFixture(t); + const activated = fs.statSync(state.targetPath, { bigint: true }); + const failedCommand = await createFailedRestoreState(state, 1, '90'); + const restore = composeRestoreCommands(state, failedCommand, '91'); + const harness = adoptedDockerHarness(state); + const dependencies = { + runDocker: harness.runDocker, + validateSocket() {}, + }; + const prepared = await restoreLocalDeploymentCompose( + restore.prepare, + dependencies, + ); + assert.equal(prepared.status, 'prepared'); + const committed = await restoreLocalDeploymentCompose( + restore.commit, + dependencies, + ); + assert.equal(committed.status, 'restored'); + const restored = fs.statSync(state.targetPath, { bigint: true }); + assert.equal(restored.dev, activated.dev); + assert.equal(restored.ino, activated.ino); + const commitPath = path.join( + state.root, + 'service', + 'restores', + `${restore.commit.request.restoreId}.commit.json`, + ); + const restoreReceipt = JSON.parse(fs.readFileSync(commitPath, 'utf8')); + assert.equal( + restoreReceipt.schema, + 'qinglong/local-compose-restore-commit@v2', + ); + assert.equal(restoreReceipt.lineage.mode, 'adopted'); + assert.equal( + restoreReceipt.lineage.legacyDataApplicationReceiptDigest, + state.command.request.legacyDataApplication.expectedReceiptDigest, + ); + + const applyHarness = adoptedDockerHarness(state); + await applyLocalDeploymentCompose(failedCommand, { + runDocker: applyHarness.runDocker, + validateSocket() {}, + now: applyHarness.now, + wait: applyHarness.wait, + }); + const secondRestoreId = '00000000-0000-4000-8000-000000000d92'; + const secondSafeguard = await createLocalSqliteRolloutBackup({ + databasePath: state.targetPath, + backupPath: path.join( + state.root, + 'service', + 'restore-safeguards', + `${secondRestoreId}.sqlite`, + ), + profile: 'edge', + }); + const secondReceipt = { + ...restoreReceipt, + commandDigest: 'd'.repeat(64), + restoreId: secondRestoreId, + recordedAtMs: restoreReceipt.recordedAtMs + 1, + safeguard: { + contractVersion: secondSafeguard.contractVersion, + sha256: secondSafeguard.sha256, + bytes: secondSafeguard.bytes, + pageCount: secondSafeguard.pageCount, + pageSize: secondSafeguard.pageSize, + }, + }; + writePrivate( + path.join( + state.root, + 'service', + 'restores', + `${secondRestoreId}.commit.json`, + ), + `${JSON.stringify(secondReceipt, null, 2)}\n`, + ); + const collection = composeEvidenceCollectionCommands( + state, + 3, + restore.commit.request.restoreId, + '93', + ); + const collectionPrepared = await collectLocalDeploymentComposeEvidence( + collection.prepare, + ); + assert.equal(collectionPrepared.status, 'prepared'); + const collectionCommitted = await collectLocalDeploymentComposeEvidence( + collection.commit, + ); + assert.equal(collectionCommitted.status, 'collected'); + const collectionReceipt = JSON.parse( + fs.readFileSync( + path.join( + state.root, + 'service', + 'evidence-collections', + `${collection.commit.request.collectionId}.commit.json`, + ), + 'utf8', + ), + ); + assert.equal( + collectionReceipt.schema, + 'qinglong/local-compose-evidence-collection-commit@v2', + ); + assert.deepEqual(collectionReceipt.lineage, restoreReceipt.lineage); + const replay = await restoreLocalDeploymentCompose( + restore.commit, + dependencies, + ); + assert.equal(replay.status, 'existing'); + assert.equal(replay.sqlite.safeguard, 'collected'); +}); + test('exposes separate exact prepare and verify CLI operations', (t) => { const state = fixture(t, 'systemd'); const cli = path.resolve( diff --git a/packages/ql3-local-owner-cli/test/localDeployment.test.cjs b/packages/ql3-local-owner-cli/test/localDeployment.test.cjs index 8a95cbcd..dd1505b3 100644 --- a/packages/ql3-local-owner-cli/test/localDeployment.test.cjs +++ b/packages/ql3-local-owner-cli/test/localDeployment.test.cjs @@ -450,6 +450,15 @@ function composeDockerHarness( generation: Number(/^ generation: ([0-9]+)$/m.exec(source)[1]), mutationId: /^ mutation_id: ([0-9a-f-]+)$/m.exec(source)[1], image: /^ image: ([^\n]+)$/m.exec(source)[1], + releaseSelectionDigest: /^ release_selection_digest: ([^\n]+)$/m.exec( + source, + )[1], + releaseSetDigest: /^ release_set_digest: ([^\n]+)$/m.exec(source)[1], + catalogManifestDigest: /^ catalog_manifest_digest: ([^\n]+)$/m.exec( + source, + )[1], + catalogConsumptionReportDigest: + /^ catalog_consumption_report_digest: ([^\n]+)$/m.exec(source)[1], }; }; const composeSource = fs.readFileSync( @@ -457,6 +466,12 @@ function composeDockerHarness( 'utf8', ); const projectName = /^name: ([a-z0-9_-]+)$/m.exec(composeSource)[1]; + const release = JSON.parse( + fs.readFileSync( + state.command.options.service.releaseSelection.path, + 'utf8', + ), + ); let running = true; const runDocker = ({ args }) => { calls.push(args); @@ -509,6 +524,12 @@ function composeDockerHarness( labels: { 'io.qinglong.deployment.generation': String(selected.generation), 'io.qinglong.deployment.mutation': selected.mutationId, + 'io.qinglong.release.selection': selected.releaseSelectionDigest, + 'io.qinglong.release.set': selected.releaseSetDigest, + 'io.qinglong.release.catalog-manifest': + selected.catalogManifestDigest, + 'io.qinglong.release.catalog-report': + selected.catalogConsumptionReportDigest, }, volumes: [ { @@ -1137,6 +1158,12 @@ test('preflights exact local image, Compose merge and SQLite capability', async await prepareLocalDeployment(state.command); const dockerSocketPath = path.join(state.managementRoot, 'docker.sock'); const image = state.composeImage; + const release = JSON.parse( + fs.readFileSync( + state.command.options.service.releaseSelection.path, + 'utf8', + ), + ); const composeSource = fs.readFileSync( path.join(state.deploymentRoot, 'service', 'compose.yaml'), 'utf8', @@ -1206,6 +1233,12 @@ test('preflights exact local image, Compose merge and SQLite capability', async 'io.qinglong.deployment.generation': '1', 'io.qinglong.deployment.mutation': state.command.request.activateMutationId, + 'io.qinglong.release.selection': release.selectionDigest, + 'io.qinglong.release.set': release.releaseSetDigest, + 'io.qinglong.release.catalog-manifest': + release.catalog.manifestDigest, + 'io.qinglong.release.catalog-report': + release.catalog.consumptionReportDigest, }, volumes: [ { diff --git a/packages/ql3-local-sqlite/src/readiness/rolloutSafety.ts b/packages/ql3-local-sqlite/src/readiness/rolloutSafety.ts index 3cf0b294..23c84206 100644 --- a/packages/ql3-local-sqlite/src/readiness/rolloutSafety.ts +++ b/packages/ql3-local-sqlite/src/readiness/rolloutSafety.ts @@ -49,6 +49,7 @@ export interface LocalSqliteRestoreOptions extends LocalSqliteDatabaseOptions { readonly replacedDatabasePath: string; readonly expectedCurrentSha256: string; readonly expectedSourceSha256: string; + readonly preserveDatabaseIdentity?: boolean; } export interface LocalSqliteRestoreEvidence @@ -58,6 +59,7 @@ export interface LocalSqliteRestoreEvidence export interface LocalSqliteRestoreDependencies { readonly copySnapshot?: (sourcePath: string, targetPath: string) => void; + readonly rewriteSnapshot?: (sourcePath: string, targetPath: string) => void; } export interface LocalSqliteChangeObserver { @@ -582,6 +584,239 @@ function sameSnapshot( ); } +function rewriteSnapshotFilePreservingIdentity( + sourcePath: string, + targetPath: string, +): void { + const targetIdentity = fs.lstatSync(targetPath, { bigint: true }); + let sourceDescriptor: number | undefined; + let targetDescriptor: number | undefined; + const buffer = Buffer.allocUnsafe(64 * 1024); + try { + sourceDescriptor = fs.openSync( + sourcePath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW, + ); + targetDescriptor = fs.openSync( + targetPath, + fs.constants.O_WRONLY | fs.constants.O_TRUNC | fs.constants.O_NOFOLLOW, + ); + const openedTarget = fs.fstatSync(targetDescriptor, { bigint: true }); + if ( + !openedTarget.isFile() || + openedTarget.dev !== targetIdentity.dev || + openedTarget.ino !== targetIdentity.ino || + openedTarget.nlink !== 1n + ) { + configurationError('identity-preserving restore target drifted'); + } + for (;;) { + const count = fs.readSync( + sourceDescriptor, + buffer, + 0, + buffer.byteLength, + null, + ); + if (count === 0) break; + let offset = 0; + while (offset < count) { + const written = fs.writeSync( + targetDescriptor, + buffer, + offset, + count - offset, + null, + ); + if (written < 1) { + configurationError('identity-preserving restore write stalled'); + } + offset += written; + } + } + fs.fsyncSync(targetDescriptor); + const writtenTarget = fs.fstatSync(targetDescriptor, { bigint: true }); + if ( + writtenTarget.dev !== targetIdentity.dev || + writtenTarget.ino !== targetIdentity.ino || + writtenTarget.nlink !== 1n + ) { + configurationError('identity-preserving restore target changed'); + } + } finally { + buffer.fill(0); + if (targetDescriptor !== undefined) fs.closeSync(targetDescriptor); + if (sourceDescriptor !== undefined) fs.closeSync(sourceDescriptor); + } +} + +async function restoreSnapshotPreservingIdentity( + options: Readonly, + dependencies: LocalSqliteRestoreDependencies, + input: Readonly<{ + uid: number; + databasePath: string; + sourceSnapshotPath: string; + restoreStagePath: string; + replacedDatabasePath: string; + source: Readonly; + }>, +): Promise> { + if (!fs.existsSync(input.databasePath)) { + configurationError( + 'identity-preserving restore requires the activated database inode', + ); + } + const targetIdentity = fs.lstatSync(input.databasePath); + if ( + !targetIdentity.isFile() || + targetIdentity.isSymbolicLink() || + targetIdentity.uid !== input.uid || + (targetIdentity.mode & 0o777) !== 0o600 || + targetIdentity.nlink !== 1 || + fs.realpathSync(input.databasePath) !== input.databasePath + ) { + configurationError('identity-preserving restore database is invalid'); + } + let current: Readonly | undefined; + try { + current = await inspectSnapshotFile( + input.databasePath, + options.profile, + input.uid, + 'restore current database', + ); + } catch (error) { + if ( + !fs.existsSync(input.restoreStagePath) || + !fs.existsSync(input.replacedDatabasePath) + ) { + throw error; + } + } + const restoredAtEntry = + current !== undefined && sameSnapshot(current, input.source); + if ( + current !== undefined && + !restoredAtEntry && + current.sha256 !== options.expectedCurrentSha256 + ) { + configurationError('restore current database drifted'); + } + + if (!fs.existsSync(input.restoreStagePath) && !restoredAtEntry) { + try { + ( + dependencies.copySnapshot ?? + ((sourcePath: string, targetPath: string) => + fs.copyFileSync(sourcePath, targetPath, fs.constants.COPYFILE_EXCL)) + )(input.sourceSnapshotPath, input.restoreStagePath); + fs.chmodSync(input.restoreStagePath, 0o600); + syncFile(input.restoreStagePath); + syncDirectory(path.dirname(input.restoreStagePath)); + } catch (error) { + if (fs.existsSync(input.restoreStagePath)) { + try { + fs.unlinkSync(input.restoreStagePath); + syncDirectory(path.dirname(input.restoreStagePath)); + } catch { + // A deterministic stage remains fail-closed for exact replay. + } + } + configurationError('restore stage could not be created', error); + } + } + if (fs.existsSync(input.restoreStagePath)) { + const staged = await inspectSnapshotFile( + input.restoreStagePath, + options.profile, + input.uid, + 'restore stage', + ); + if (!sameSnapshot(staged, input.source)) { + configurationError('restore stage drifted'); + } + } + + if (!restoredAtEntry && !fs.existsSync(input.replacedDatabasePath)) { + if (current === undefined) { + configurationError('restore replacement evidence is unavailable'); + } + try { + fs.copyFileSync( + input.databasePath, + input.replacedDatabasePath, + fs.constants.COPYFILE_EXCL, + ); + fs.chmodSync(input.replacedDatabasePath, 0o600); + syncFile(input.replacedDatabasePath); + syncDirectory(path.dirname(input.replacedDatabasePath)); + } catch (error) { + configurationError( + 'restore replacement evidence cannot be created', + error, + ); + } + } + if (fs.existsSync(input.replacedDatabasePath)) { + const replaced = await inspectSnapshotFile( + input.replacedDatabasePath, + options.profile, + input.uid, + 'replaced database', + ); + if (replaced.sha256 !== options.expectedCurrentSha256) { + configurationError('replaced database evidence drifted'); + } + } + + if (!restoredAtEntry) { + try { + (dependencies.rewriteSnapshot ?? rewriteSnapshotFilePreservingIdentity)( + input.restoreStagePath, + input.databasePath, + ); + } catch (error) { + configurationError( + 'identity-preserving restore write could not complete', + error, + ); + } + } + const afterIdentity = validateDatabaseFile( + input.databasePath, + input.uid, + 'identity-preserving restored database', + ); + if ( + afterIdentity.dev !== targetIdentity.dev || + afterIdentity.ino !== targetIdentity.ino + ) { + configurationError('identity-preserving restore changed the target inode'); + } + const restored = await inspectSnapshotFile( + input.databasePath, + options.profile, + input.uid, + 'restored database', + ); + if (!sameSnapshot(restored, input.source)) { + configurationError('restored database drifted'); + } + for (const evidencePath of [ + input.restoreStagePath, + input.replacedDatabasePath, + ]) { + if (!fs.existsSync(evidencePath)) continue; + fs.unlinkSync(evidencePath); + syncDirectory(path.dirname(evidencePath)); + } + return Object.freeze({ + status: restoredAtEntry ? ('existing' as const) : ('restored' as const), + ...restored, + }); +} + export async function restoreLocalSqliteSnapshot( options: Readonly, dependencies: LocalSqliteRestoreDependencies = {}, @@ -608,6 +843,8 @@ export async function restoreLocalSqliteSnapshot( !DIGEST_PATTERN.test(options.expectedCurrentSha256) || !DIGEST_PATTERN.test(options.expectedSourceSha256) || options.expectedCurrentSha256 === options.expectedSourceSha256 || + (options.preserveDatabaseIdentity !== undefined && + typeof options.preserveDatabaseIdentity !== 'boolean') || new Set([ databasePath, sourceSnapshotPath, @@ -641,6 +878,17 @@ export async function restoreLocalSqliteSnapshot( } assertNoRestoreSidecars(databasePath, uid); + if (options.preserveDatabaseIdentity === true) { + return restoreSnapshotPreservingIdentity(options, dependencies, { + uid, + databasePath, + sourceSnapshotPath, + restoreStagePath, + replacedDatabasePath, + source, + }); + } + let restoredAtEntry = false; if (fs.existsSync(databasePath)) { const current = await inspectSnapshotFile( diff --git a/packages/ql3-local-sqlite/test/rolloutSafety.test.cjs b/packages/ql3-local-sqlite/test/rolloutSafety.test.cjs index 388c522f..22791bad 100644 --- a/packages/ql3-local-sqlite/test/rolloutSafety.test.cjs +++ b/packages/ql3-local-sqlite/test/rolloutSafety.test.cjs @@ -210,6 +210,72 @@ test('converges the moved-current restore window and cleans ENOSPC stage', async assert.equal(fs.existsSync(replacedDatabasePath), false); }); +test('preserves an activated inode and recovers an ENOSPC partial rewrite', async (t) => { + const state = fixture(t); + await migrateLocalSqlitePath(state); + const activated = fs.statSync(state.databasePath, { bigint: true }); + const source = await createLocalSqliteRolloutBackup(state); + const writer = new DatabaseSync(state.databasePath); + writer.exec('PRAGMA user_version = 29'); + writer.close(); + const current = await checkpointLocalSqliteForRestore(state); + const restoreStagePath = path.join(state.root, '.identity.restore-stage'); + const replacedDatabasePath = path.join( + path.dirname(state.backupPath), + 'identity.replaced.sqlite', + ); + const restoreOptions = { + databasePath: state.databasePath, + profile: state.profile, + sourceSnapshotPath: state.backupPath, + restoreStagePath, + replacedDatabasePath, + expectedCurrentSha256: current.sha256, + expectedSourceSha256: source.sha256, + preserveDatabaseIdentity: true, + }; + await assert.rejects( + restoreLocalSqliteSnapshot(restoreOptions, { + rewriteSnapshot(_sourcePath, targetPath) { + const descriptor = fs.openSync( + targetPath, + fs.constants.O_WRONLY | + fs.constants.O_TRUNC | + fs.constants.O_NOFOLLOW, + ); + try { + fs.writeSync(descriptor, Buffer.from('partial')); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + throw Object.assign(new Error('injected identity restore ENOSPC'), { + code: 'ENOSPC', + }); + }, + }), + /identity-preserving restore write could not complete/, + ); + assert.equal(fs.existsSync(restoreStagePath), true); + assert.equal(fs.existsSync(replacedDatabasePath), true); + const partial = fs.statSync(state.databasePath, { bigint: true }); + assert.equal(partial.dev, activated.dev); + assert.equal(partial.ino, activated.ino); + + const recovered = await restoreLocalSqliteSnapshot(restoreOptions); + assert.equal(recovered.status, 'restored'); + assert.equal(recovered.sha256, source.sha256); + const restored = fs.statSync(state.databasePath, { bigint: true }); + assert.equal(restored.dev, activated.dev); + assert.equal(restored.ino, activated.ino); + assert.equal(fs.existsSync(restoreStagePath), false); + assert.equal(fs.existsSync(replacedDatabasePath), false); + assert.equal( + (await restoreLocalSqliteSnapshot(restoreOptions)).status, + 'existing', + ); +}); + test('rollout safety subpath excludes DDL and mutable repositories', () => { const script = ` const safety = require(${JSON.stringify( diff --git a/scripts/ql3-cluster-oci-layout-audit.cjs b/scripts/ql3-cluster-oci-layout-audit.cjs index 607dad66..52105b54 100644 --- a/scripts/ql3-cluster-oci-layout-audit.cjs +++ b/scripts/ql3-cluster-oci-layout-audit.cjs @@ -222,7 +222,7 @@ function expectedImageConfig(architecture, revision, image) { WorkingDir: '/opt/qinglong', Labels: { 'io.qinglong.ai': 'excluded', - 'io.qinglong.local.application-config': '2', + 'io.qinglong.local.application-config': '2,3,4', 'io.qinglong.local.compose-selection': '1', 'io.qinglong.local.sqlite-contract-max': '41', 'io.qinglong.local.sqlite-contract-min': '41', diff --git a/scripts/ql3-local-image-audit.cjs b/scripts/ql3-local-image-audit.cjs index f89b284f..747c1c31 100644 --- a/scripts/ql3-local-image-audit.cjs +++ b/scripts/ql3-local-image-audit.cjs @@ -246,10 +246,10 @@ function auditDockerfile(contents, findings) { ) || !contents.includes('io.qinglong.ai="excluded"') || !contents.includes('io.qinglong.profile="edge,standalone"') || - !contents.includes('io.qinglong.local.application-config="2"') || - !contents.includes('io.qinglong.local.sqlite-contract-min="44"') || - !contents.includes('io.qinglong.local.sqlite-contract-max="44"') || - !contents.includes('io.qinglong.local.sqlite-write-contract="44"') || + !contents.includes('io.qinglong.local.application-config="2,3,4"') || + !contents.includes('io.qinglong.local.sqlite-contract-min="50"') || + !contents.includes('io.qinglong.local.sqlite-contract-max="50"') || + !contents.includes('io.qinglong.local.sqlite-write-contract="50"') || !contents.includes('io.qinglong.local.compose-selection="1"') ) { addFinding(findings, 'RUNTIME_IDENTITY_OR_LABEL_DRIFT'); diff --git a/test/back/ql3ClusterOciLayoutAudit.test.cjs b/test/back/ql3ClusterOciLayoutAudit.test.cjs index 3c66824f..ecb55d71 100644 --- a/test/back/ql3ClusterOciLayoutAudit.test.cjs +++ b/test/back/ql3ClusterOciLayoutAudit.test.cjs @@ -136,7 +136,7 @@ function createFixture(t, options = {}) { ...(isLocal ? { 'io.qinglong.ai': 'excluded', - 'io.qinglong.local.application-config': '2', + 'io.qinglong.local.application-config': '2,3,4', 'io.qinglong.local.compose-selection': '1', 'io.qinglong.local.sqlite-contract-max': '41', 'io.qinglong.local.sqlite-contract-min': '41', diff --git a/test/back/ql3LocalImageAudit.test.cjs b/test/back/ql3LocalImageAudit.test.cjs index f52bf9d9..08320311 100644 --- a/test/back/ql3LocalImageAudit.test.cjs +++ b/test/back/ql3LocalImageAudit.test.cjs @@ -156,7 +156,7 @@ test('rejects removal of the SQLite rollout compatibility labels', () => { const dockerfilePath = path.join(current.target, 'Dockerfile'); const dockerfile = fs .readFileSync(dockerfilePath, 'utf8') - .replace(' io.qinglong.local.sqlite-write-contract="44" \\\n', ''); + .replace(' io.qinglong.local.sqlite-write-contract="50" \\\n', ''); fs.writeFileSync(dockerfilePath, dockerfile); const report = auditLocalImageContract(current.root); assert.equal(report.compatible, false); diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index e7c5ccbc..d4c63c16 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -207,10 +207,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: localOwnerCli.rootSourceFileRoles, }, { - sourceFiles: 134, + sourceFiles: 135, rootSourceFiles: 1, rootSourceLines: 50, - nestedSourceFiles: 133, + nestedSourceFiles: 134, rootSourceFileRoles: { 'cli.ts': 'binary_entry' }, }, );