diff --git a/packages/ql3-local-application/src/production-process/legacyDataApplicationCommitment.ts b/packages/ql3-local-application/src/production-process/legacyDataApplicationCommitment.ts new file mode 100644 index 00000000..c1f6626d --- /dev/null +++ b/packages/ql3-local-application/src/production-process/legacyDataApplicationCommitment.ts @@ -0,0 +1,160 @@ +import fs from 'node:fs'; + +import { + LocalDataDirectoryApplicationCommitError, + normalizeLocalDataDirectoryApplicationCommit, + type LocalDataDirectoryApplicationCommit, +} from '@qinglong/local-sqlite/data-directory-application-commit'; + +import { + LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4, + type LocalApplicationProcessConfig, +} from './processConfig'; + +const MAX_COMMIT_BYTES = 64 * 1024; + +export class LocalApplicationLegacyDataCommitmentError extends Error { + readonly code = 'QL3_LOCAL_APPLICATION_LEGACY_DATA_COMMITMENT_INVALID'; + + constructor(message: string, options?: ErrorOptions) { + super( + `Local application legacy data commitment is invalid: ${message}`, + options, + ); + this.name = 'LocalApplicationLegacyDataCommitmentError'; + } +} + +function currentIdentity(): Readonly<{ uid: number; gid: number }> { + if ( + typeof process.getuid !== 'function' || + typeof process.geteuid !== 'function' || + typeof process.getgid !== 'function' || + typeof process.getegid !== 'function' || + process.getuid() !== process.geteuid() || + process.getgid() !== process.getegid() + ) { + throw new LocalApplicationLegacyDataCommitmentError( + 'real and effective POSIX identities must match', + ); + } + return Object.freeze({ uid: process.getuid(), gid: process.getgid() }); +} + +function readCommit( + filePath: string, +): Readonly { + const identity = currentIdentity(); + let descriptor: number | undefined; + let bytes: Buffer | undefined; + try { + const before = fs.lstatSync(filePath, { bigint: true }); + descriptor = fs.openSync( + filePath, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + const opened = fs.fstatSync(descriptor, { bigint: true }); + if ( + !opened.isFile() || + opened.isSymbolicLink() || + opened.dev !== before.dev || + opened.ino !== before.ino || + Number(opened.uid) !== identity.uid || + Number(opened.gid) !== identity.gid || + (Number(opened.mode) & 0o777) !== 0o600 || + opened.nlink !== 1n || + opened.size < 2n || + opened.size > BigInt(MAX_COMMIT_BYTES) + ) { + throw new LocalApplicationLegacyDataCommitmentError( + 'commit file identity is invalid', + ); + } + bytes = Buffer.alloc(Number(opened.size)); + let offset = 0; + while (offset < bytes.byteLength) { + const count = fs.readSync( + descriptor, + bytes, + offset, + bytes.byteLength - offset, + offset, + ); + if (count === 0) break; + offset += count; + } + const after = fs.fstatSync(descriptor, { bigint: true }); + if ( + offset !== bytes.byteLength || + after.dev !== opened.dev || + after.ino !== opened.ino || + after.size !== opened.size || + after.mtimeNs !== opened.mtimeNs || + after.ctimeNs !== opened.ctimeNs || + after.uid !== opened.uid || + after.gid !== opened.gid || + after.mode !== opened.mode || + after.nlink !== opened.nlink + ) { + throw new LocalApplicationLegacyDataCommitmentError( + 'commit file identity changed while reading', + ); + } + let value: unknown; + try { + value = JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(bytes), + ); + } catch (error) { + throw new LocalApplicationLegacyDataCommitmentError( + 'commit file is not canonical UTF-8 JSON', + { cause: error }, + ); + } + try { + return normalizeLocalDataDirectoryApplicationCommit(value); + } catch (error) { + if (error instanceof LocalDataDirectoryApplicationCommitError) { + throw new LocalApplicationLegacyDataCommitmentError( + 'commit document is invalid', + { cause: error }, + ); + } + throw error; + } + } catch (error) { + if (error instanceof LocalApplicationLegacyDataCommitmentError) throw error; + throw new LocalApplicationLegacyDataCommitmentError( + 'commit file cannot be read', + { cause: error }, + ); + } finally { + bytes?.fill(0); + if (descriptor !== undefined) fs.closeSync(descriptor); + } +} + +export function verifyLocalApplicationLegacyDataCommitment( + config: Readonly, +): Readonly | undefined { + if (config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4) { + return undefined; + } + const binding = config.legacyDataApplication; + if (binding === undefined) { + throw new LocalApplicationLegacyDataCommitmentError( + 'v4 binding is unavailable', + ); + } + const commit = readCommit(binding.commitPath); + if ( + commit.profile !== config.profile || + commit.commitDigest !== binding.expectedCommitDigest || + commit.receiptDigest !== binding.expectedReceiptDigest + ) { + throw new LocalApplicationLegacyDataCommitmentError( + 'commit does not match the application binding', + ); + } + return commit; +} diff --git a/packages/ql3-local-application/src/production-process/processApplication.ts b/packages/ql3-local-application/src/production-process/processApplication.ts index 97a2ba33..f433bf9c 100644 --- a/packages/ql3-local-application/src/production-process/processApplication.ts +++ b/packages/ql3-local-application/src/production-process/processApplication.ts @@ -19,6 +19,7 @@ import { type LocalApplicationProcessConfig, } from './processConfig'; import { verifyLocalApplicationCutoverCommitment } from './cutoverCommitment'; +import { verifyLocalApplicationLegacyDataCommitment } from './legacyDataApplicationCommitment'; import { recordLocalApplicationShutdownReceipt } from './shutdownReceipt'; import { recordLocalApplicationStartupReceipt } from './startupReceipt'; @@ -221,6 +222,7 @@ export async function runProductionLocalApplicationProcess( throw new TypeError('Local application process options are invalid'); } const config = loadLocalApplicationProcessConfig(options.configFilePath); + verifyLocalApplicationLegacyDataCommitment(config); verifyLocalApplicationCutoverCommitment(config); const selectedAi = aiOptions(config, options); const start = options.start ?? defaultStarter; diff --git a/packages/ql3-local-application/src/production-process/processConfig.ts b/packages/ql3-local-application/src/production-process/processConfig.ts index 3819b8d5..d992fc26 100644 --- a/packages/ql3-local-application/src/production-process/processConfig.ts +++ b/packages/ql3-local-application/src/production-process/processConfig.ts @@ -2,9 +2,7 @@ import path from 'node:path'; import { readPrivateLocalCommandFile } from '@qinglong/local-command-file'; import { MAX_PLUGIN_PACKAGE_INSTALL_RECOVERY_PAGE_SIZE } from '@qinglong/runtime-core/plugin-package-install'; -import { - MAX_PLUGIN_PACKAGE_RECOVERY_PAGES, -} from '@qinglong/runtime-core/plugin-package-recovery'; +import { MAX_PLUGIN_PACKAGE_RECOVERY_PAGES } from '@qinglong/runtime-core/plugin-package-recovery'; import { MAX_PLUGIN_PACKAGE_TASK_PUBLICATION_RECOVERY_PAGES, MAX_PLUGIN_PACKAGE_TASK_PUBLICATION_RECOVERY_PAGE_SIZE, @@ -18,6 +16,8 @@ export const LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V2 = 'qinglong/local-application-process@v2' as const; export const LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 = 'qinglong/local-application-process@v3' as const; +export const LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4 = + 'qinglong/local-application-process@v4' as const; const MAX_PATH_BYTES = 4_096; const INSTANCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; @@ -56,6 +56,12 @@ export interface LocalApplicationProcessCutoverConfig { readonly expectedCommitmentDigest: string; } +export interface LocalApplicationProcessLegacyDataApplicationConfig { + readonly commitPath: string; + readonly expectedCommitDigest: string; + readonly expectedReceiptDigest: string; +} + export type LocalApplicationProcessPluginPackageRecoverySourceConfig = | Readonly<{ mode: 'disabled' }> | Readonly<{ @@ -89,7 +95,8 @@ export interface LocalApplicationProcessConfig { readonly schema: | typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA | typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V2 - | typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3; + | typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 + | typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4; readonly instanceId: string; readonly profile: LocalApplicationProfile; readonly storage: Readonly; @@ -97,13 +104,17 @@ export interface LocalApplicationProcessConfig { readonly pluginPackages: Readonly; readonly ai: LocalApplicationProcessAiConfig; readonly cutover?: Readonly; + readonly legacyDataApplication?: Readonly; } export class LocalApplicationProcessConfigError extends TypeError { readonly code = 'QL3_LOCAL_APPLICATION_PROCESS_CONFIG_INVALID'; constructor(message: string, options?: ErrorOptions) { - super(`Local application process configuration is invalid: ${message}`, options); + super( + `Local application process configuration is invalid: ${message}`, + options, + ); this.name = 'LocalApplicationProcessConfigError'; } } @@ -175,7 +186,8 @@ function storageConfig( schema: | typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA | typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V2 - | typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3, + | typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 + | typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4, ): Readonly { const storage = record(value, 'storage'); if (schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA) { @@ -218,9 +230,7 @@ function storageConfig( 'recoveryPath', 'sourcePath', 'targetPath', - ...(schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA - ? ['mode'] - : []), + ...(schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA ? ['mode'] : []), ...optionalKeys, ], 'storage', @@ -297,6 +307,32 @@ function cutoverConfig( }); } +function legacyDataApplicationConfig( + value: unknown, +): Readonly { + const application = record(value, 'legacyDataApplication'); + exactKeys( + application, + ['commitPath', 'expectedCommitDigest', 'expectedReceiptDigest'], + 'legacyDataApplication', + ); + if ( + typeof application.expectedCommitDigest !== 'string' || + !DIGEST_PATTERN.test(application.expectedCommitDigest) || + typeof application.expectedReceiptDigest !== 'string' || + !DIGEST_PATTERN.test(application.expectedReceiptDigest) + ) { + throw new LocalApplicationProcessConfigError( + 'legacyDataApplication digest is invalid', + ); + } + return Object.freeze({ + commitPath: absolutePath(application.commitPath, 'commitPath'), + expectedCommitDigest: application.expectedCommitDigest, + expectedReceiptDigest: application.expectedReceiptDigest, + }); +} + function runtimeConfig( value: unknown, ): Readonly { @@ -498,7 +534,8 @@ export function normalizeLocalApplicationProcessConfig( if ( config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA && config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V2 && - config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 + config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 && + config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4 ) { throw new LocalApplicationProcessConfigError('schema is invalid'); } @@ -506,10 +543,14 @@ export function normalizeLocalApplicationProcessConfig( config, [ 'ai', - ...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 + ...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 || + config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4 ? ['cutover'] : []), 'instanceId', + ...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4 + ? ['legacyDataApplication'] + : []), 'pluginPackages', 'profile', 'runtime', @@ -528,7 +569,8 @@ export function normalizeLocalApplicationProcessConfig( throw new LocalApplicationProcessConfigError('profile is invalid'); } if ( - config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 && + (config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 || + config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4) && record(config.storage, 'storage').mode !== 'adopted' ) { throw new LocalApplicationProcessConfigError( @@ -543,9 +585,17 @@ export function normalizeLocalApplicationProcessConfig( runtime: runtimeConfig(config.runtime), pluginPackages: pluginPackageConfig(config.pluginPackages), ai: aiConfig(config.ai), - ...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 + ...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 || + config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4 ? { cutover: cutoverConfig(config.cutover) } : {}), + ...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4 + ? { + legacyDataApplication: legacyDataApplicationConfig( + config.legacyDataApplication, + ), + } + : {}), } as const; const authorityPaths = [ ...(normalized.storage.mode === 'fresh' @@ -565,8 +615,10 @@ export function normalizeLocalApplicationProcessConfig( ...(normalized.cutover === undefined ? [] : [normalized.cutover.commitmentPath]), - ...(normalized.pluginPackages.recoverySource.mode === - 'materialized_catalog' + ...(normalized.legacyDataApplication === undefined + ? [] + : [normalized.legacyDataApplication.commitPath]), + ...(normalized.pluginPackages.recoverySource.mode === 'materialized_catalog' ? [ normalized.pluginPackages.recoverySource.catalogRoot, normalized.pluginPackages.recoverySource.bundleRoot, diff --git a/packages/ql3-local-application/test/legacyDataApplicationCommitment.test.cjs b/packages/ql3-local-application/test/legacyDataApplicationCommitment.test.cjs new file mode 100644 index 00000000..a85531ff --- /dev/null +++ b/packages/ql3-local-application/test/legacyDataApplicationCommitment.test.cjs @@ -0,0 +1,204 @@ +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { test } = require('node:test'); + +const { + createLocalDataDirectoryApplicationCommit, +} = require('@qinglong/local-sqlite/data-directory-application-commit'); +const { + LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3, + LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4, + LocalApplicationProcessConfigError, + normalizeLocalApplicationProcessConfig, +} = require('../dist/production-process/processConfig.js'); +const { + LocalApplicationLegacyDataCommitmentError, + verifyLocalApplicationLegacyDataCommitment, +} = require('../dist/production-process/legacyDataApplicationCommitment.js'); +const { + runProductionLocalApplicationProcess, +} = require('../dist/production-process/processApplication.js'); + +function temporaryRoot(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-data-commitment-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return root; +} + +function commitDocument(profile = 'edge') { + return createLocalDataDirectoryApplicationCommit({ + mutationId: '00000000-0000-4000-8000-000000000001', + projectId: 'project-edge-1', + profile, + sourceStageManifestDigest: '1'.repeat(64), + transformationDigest: '2'.repeat(64), + modelDigest: '3'.repeat(64), + publicationDigest: '4'.repeat(64), + receiptDigest: '5'.repeat(64), + committedAtMs: 1_000, + receipt: { + secretCount: 2, + environmentSecretCount: 1, + sshSecretCount: 1, + }, + }); +} + +function v4Config(root, commit) { + const cutoverPayload = { + schemaVersion: 1, + kind: 'qinglong3-local-legacy-silence-commitment', + state: 'legacy_stopped', + cutoverId: 'cutover-data-1', + profile: 'edge', + instanceId: 'edge-router-1', + activationDigest: 'a'.repeat(64), + previousRecordDigest: 'b'.repeat(64), + requestedAtMs: 1_000, + observedAtMs: 1_000, + controller: { + kind: 'docker', + endpointDigest: 'c'.repeat(64), + legacyContainerId: 'd'.repeat(64), + legacyContainerIdentityDigest: 'e'.repeat(64), + legacySourceBindingDigest: 'f'.repeat(64), + }, + }; + return { + schema: LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4, + instanceId: 'edge-router-1', + profile: 'edge', + storage: { + mode: 'adopted', + sourcePath: path.join(root, 'legacy.sqlite'), + targetPath: path.join(root, 'qinglong3.sqlite'), + recoveryPath: path.join(root, 'recovery.sqlite'), + manifestPath: path.join(root, 'manifest.json'), + activationPath: path.join(root, 'activation.json'), + expectedActivationDigest: 'a'.repeat(64), + busyTimeoutMs: 100, + }, + runtime: { + receiptRoot: path.join(root, 'receipts'), + artifactRoot: path.join(root, 'artifacts'), + secretKeyringPath: path.join(root, 'secret-keyring.json'), + }, + pluginPackages: { + stagingRoot: path.join(root, 'plugin-staging'), + activationRoot: path.join(root, 'plugin-activation'), + recoverySource: { mode: 'disabled' }, + pageSize: 4, + maxPages: 4, + taskPublicationPageSize: 4, + taskPublicationMaxPages: 4, + }, + ai: { deployment: 'excluded' }, + cutover: { + cutoverId: cutoverPayload.cutoverId, + commitmentPath: path.join(root, 'legacy-stopped.json'), + expectedCommitmentDigest: crypto + .createHash('sha256') + .update(JSON.stringify(cutoverPayload), 'utf8') + .digest('hex'), + }, + legacyDataApplication: { + commitPath: path.join(root, 'commit.json'), + expectedCommitDigest: commit.commitDigest, + expectedReceiptDigest: commit.receiptDigest, + }, + }; +} + +function writeCommit(config, commit) { + fs.writeFileSync( + config.legacyDataApplication.commitPath, + `${JSON.stringify(commit)}\n`, + { mode: 0o600 }, + ); +} + +test('v4 config binds and verifies a canonical committed data application', (t) => { + const root = temporaryRoot(t); + const commit = commitDocument(); + const config = v4Config(root, commit); + writeCommit(config, commit); + const normalized = normalizeLocalApplicationProcessConfig(config); + assert.equal(normalized.schema, LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4); + assert.deepEqual( + verifyLocalApplicationLegacyDataCommitment(normalized), + commit, + ); +}); + +test('v4 config rejects fresh storage and missing receipt binding', (t) => { + const root = temporaryRoot(t); + const commit = commitDocument(); + const config = v4Config(root, commit); + assert.throws( + () => + normalizeLocalApplicationProcessConfig({ + ...config, + storage: { + mode: 'fresh', + databasePath: path.join(root, 'fresh.sqlite'), + }, + }), + LocalApplicationProcessConfigError, + ); + const { legacyDataApplication, ...missing } = config; + assert.ok(legacyDataApplication); + assert.throws( + () => normalizeLocalApplicationProcessConfig(missing), + LocalApplicationProcessConfigError, + ); +}); + +test('commit and receipt drift fail before signal or storage authority', async (t) => { + const root = temporaryRoot(t); + const commit = commitDocument(); + const config = v4Config(root, commit); + writeCommit(config, commit); + config.legacyDataApplication.expectedReceiptDigest = '0'.repeat(64); + const configPath = path.join(root, 'local-application.json'); + fs.writeFileSync(configPath, `${JSON.stringify(config)}\n`, { mode: 0o600 }); + let subscriptions = 0; + let starts = 0; + await assert.rejects( + () => + runProductionLocalApplicationProcess({ + configFilePath: configPath, + signals: { + subscribe() { + subscriptions += 1; + return () => undefined; + }, + }, + emit() {}, + async start() { + starts += 1; + throw new Error('must not start'); + }, + }), + (error) => + error instanceof LocalApplicationLegacyDataCommitmentError && + error.code === 'QL3_LOCAL_APPLICATION_LEGACY_DATA_COMMITMENT_INVALID', + ); + assert.equal(subscriptions, 0); + assert.equal(starts, 0); +}); + +test('v3 remains SQLite-only and does not claim a data application receipt', (t) => { + const root = temporaryRoot(t); + const commit = commitDocument(); + const config = v4Config(root, commit); + const { legacyDataApplication, ...withoutDataApplication } = config; + assert.ok(legacyDataApplication); + const v3 = normalizeLocalApplicationProcessConfig({ + ...withoutDataApplication, + schema: LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3, + }); + assert.equal(verifyLocalApplicationLegacyDataCommitment(v3), undefined); +}); diff --git a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/application/cleanup.ts b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/application/cleanup.ts index fccc4ba7..c39fd3ce 100644 --- a/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/application/cleanup.ts +++ b/packages/ql3-local-owner-cli/src/lifecycle/data-directory-adoption/application/cleanup.ts @@ -2,7 +2,9 @@ import fs from 'node:fs'; import path from 'node:path'; import { + createLocalDataDirectoryApplicationCommit, createLocalDataDirectorySourceNameDigest, + type LocalDataDirectoryApplicationCommit, type LocalDataDirectoryAdoptionRecord, } from '@qinglong/local-sqlite/data-directory-adoption'; @@ -13,7 +15,6 @@ import { sortedNames, syncDirectory, } from '../filesystem'; -import { sha256Text } from '../manifest'; import { readStablePrivateUtf8File, writePrivateJson, @@ -29,34 +30,6 @@ const MODEL_NAME = 'model'; const MAX_JSON_BYTES = 1024 * 1024; const ZERO_CHUNK = Buffer.alloc(64 * 1024); -interface LocalDataDirectoryApplicationCommitPayload { - readonly schemaVersion: 1; - readonly kind: 'qinglong3-legacy-data-directory-application'; - readonly state: 'committed'; - readonly mutationId: string; - readonly profile: 'edge' | 'standalone'; - readonly projectIdDigest: string; - readonly sourceStageManifestDigest: string; - readonly transformationDigest: string; - readonly modelDigest: string; - readonly publicationDigest: string; - readonly receiptDigest: string; - readonly secretCount: number; - readonly environmentSecretCount: number; - readonly sshSecretCount: number; - readonly committedAtMs: number; - readonly reclamation: Readonly<{ - modelRemoved: true; - plaintextFilesRemoved: true; - physicalErasureGuaranteed: false; - }>; -} - -export interface LocalDataDirectoryApplicationCommit - extends LocalDataDirectoryApplicationCommitPayload { - readonly commitDigest: string; -} - function exactKeys(value: object, expected: readonly string[]): boolean { const actual = Object.keys(value).sort(); const canonical = [...expected].sort(); @@ -66,37 +39,6 @@ function exactKeys(value: object, expected: readonly string[]): boolean { ); } -function expectedCommit( - adoption: Readonly, -): Readonly { - const payload: LocalDataDirectoryApplicationCommitPayload = { - schemaVersion: 1, - kind: 'qinglong3-legacy-data-directory-application', - state: 'committed', - mutationId: adoption.mutationId, - profile: adoption.profile, - projectIdDigest: sha256Text(adoption.projectId), - sourceStageManifestDigest: adoption.sourceStageManifestDigest, - transformationDigest: adoption.transformationDigest, - modelDigest: adoption.modelDigest, - publicationDigest: adoption.publicationDigest, - receiptDigest: adoption.receiptDigest, - secretCount: adoption.receipt.secretCount, - environmentSecretCount: adoption.receipt.environmentSecretCount, - sshSecretCount: adoption.receipt.sshSecretCount, - committedAtMs: adoption.committedAtMs, - reclamation: Object.freeze({ - modelRemoved: true, - plaintextFilesRemoved: true, - physicalErasureGuaranteed: false, - }), - }; - return Object.freeze({ - ...payload, - commitDigest: sha256Text(JSON.stringify(payload)), - }); -} - function readJson(filePath: string, uid: number, label: string): unknown { try { return JSON.parse( @@ -167,7 +109,7 @@ function verifyCommit( authority.uid, 'application commit', ); - const expected = expectedCommit(adoption); + const expected = createLocalDataDirectoryApplicationCommit(adoption); if (!sameJson(actual, expected)) { throw new LocalDataDirectoryAdoptionConfigurationError( 'application commit does not match the durable database receipt', @@ -415,7 +357,10 @@ export function reclaimCommittedTransformationModel(options: { const commitPath = path.join(root, APPLICATION_COMMIT_NAME); if (!fs.existsSync(commitPath)) { - writePrivateJson(commitPath, expectedCommit(adoption)); + writePrivateJson( + commitPath, + createLocalDataDirectoryApplicationCommit(adoption), + ); syncDirectory(root); } const commit = verifyCommit(authority, adoption); diff --git a/packages/ql3-local-sqlite/package.json b/packages/ql3-local-sqlite/package.json index 8e32d133..fc4680bf 100644 --- a/packages/ql3-local-sqlite/package.json +++ b/packages/ql3-local-sqlite/package.json @@ -80,6 +80,11 @@ "require": "./dist/adoption/data-directory/dataDirectoryAdoptionDatabase.js", "default": "./dist/adoption/data-directory/dataDirectoryAdoptionDatabase.js" }, + "./data-directory-application-commit": { + "types": "./dist/adoption/data-directory/applicationCommit.d.ts", + "require": "./dist/adoption/data-directory/applicationCommit.js", + "default": "./dist/adoption/data-directory/applicationCommit.js" + }, "./plugin-package-install": { "types": "./dist/plugin-package/pluginPackageInstallRepository.d.ts", "require": "./dist/plugin-package/pluginPackageInstallRepository.js", diff --git a/packages/ql3-local-sqlite/src/adoption/data-directory/applicationCommit.ts b/packages/ql3-local-sqlite/src/adoption/data-directory/applicationCommit.ts new file mode 100644 index 00000000..0fae810a --- /dev/null +++ b/packages/ql3-local-sqlite/src/adoption/data-directory/applicationCommit.ts @@ -0,0 +1,230 @@ +import { createHash } from 'node:crypto'; + +import type { LocalDataDirectoryAdoptionRecord } from './dataDirectoryAdoptionDatabase'; + +const DIGEST_PATTERN = /^[0-9a-f]{64}$/; +const UUID_V4_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +export const LOCAL_DATA_DIRECTORY_APPLICATION_COMMIT_KIND = + 'qinglong3-legacy-data-directory-application' as const; + +export interface LocalDataDirectoryApplicationCommitPayload { + readonly schemaVersion: 1; + readonly kind: typeof LOCAL_DATA_DIRECTORY_APPLICATION_COMMIT_KIND; + readonly state: 'committed'; + readonly mutationId: string; + readonly profile: 'edge' | 'standalone'; + readonly projectIdDigest: string; + readonly sourceStageManifestDigest: string; + readonly transformationDigest: string; + readonly modelDigest: string; + readonly publicationDigest: string; + readonly receiptDigest: string; + readonly secretCount: number; + readonly environmentSecretCount: number; + readonly sshSecretCount: number; + readonly committedAtMs: number; + readonly reclamation: Readonly<{ + modelRemoved: true; + plaintextFilesRemoved: true; + physicalErasureGuaranteed: false; + }>; +} + +export interface LocalDataDirectoryApplicationCommit + extends LocalDataDirectoryApplicationCommitPayload { + readonly commitDigest: string; +} + +export class LocalDataDirectoryApplicationCommitError extends TypeError { + readonly code = 'LOCAL_DATA_DIRECTORY_APPLICATION_COMMIT_INVALID'; + + constructor(message: string) { + super(`Local data directory application commit is invalid: ${message}`); + this.name = 'LocalDataDirectoryApplicationCommitError'; + } +} + +function digest(value: string): string { + return createHash('sha256').update(value, 'utf8').digest('hex'); +} + +function record(value: unknown, label: string): Record { + if ( + !value || + typeof value !== 'object' || + Array.isArray(value) || + (Object.getPrototypeOf(value) !== Object.prototype && + Object.getPrototypeOf(value) !== null) + ) { + throw new LocalDataDirectoryApplicationCommitError( + `${label} must be an object`, + ); + } + return value as Record; +} + +function exact( + value: Record, + keys: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + throw new LocalDataDirectoryApplicationCommitError( + `${label} shape is invalid`, + ); + } +} + +function count(value: unknown, label: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new LocalDataDirectoryApplicationCommitError(`${label} is invalid`); + } + return value as number; +} + +export function localDataDirectoryApplicationCommitDigest( + payload: Readonly, +): string { + return digest(JSON.stringify(payload)); +} + +export function normalizeLocalDataDirectoryApplicationCommit( + value: unknown, +): Readonly { + const commit = record(value, 'commit'); + exact( + commit, + [ + 'commitDigest', + 'committedAtMs', + 'environmentSecretCount', + 'kind', + 'modelDigest', + 'mutationId', + 'profile', + 'projectIdDigest', + 'publicationDigest', + 'receiptDigest', + 'reclamation', + 'schemaVersion', + 'secretCount', + 'sourceStageManifestDigest', + 'sshSecretCount', + 'state', + 'transformationDigest', + ], + 'commit', + ); + const reclamation = record(commit.reclamation, 'reclamation'); + exact( + reclamation, + ['modelRemoved', 'physicalErasureGuaranteed', 'plaintextFilesRemoved'], + 'reclamation', + ); + const secretCount = count(commit.secretCount, 'secretCount'); + const environmentSecretCount = count( + commit.environmentSecretCount, + 'environmentSecretCount', + ); + const sshSecretCount = count(commit.sshSecretCount, 'sshSecretCount'); + const digestValues = [ + commit.projectIdDigest, + commit.sourceStageManifestDigest, + commit.transformationDigest, + commit.modelDigest, + commit.publicationDigest, + commit.receiptDigest, + ]; + if ( + commit.schemaVersion !== 1 || + commit.kind !== LOCAL_DATA_DIRECTORY_APPLICATION_COMMIT_KIND || + commit.state !== 'committed' || + typeof commit.mutationId !== 'string' || + !UUID_V4_PATTERN.test(commit.mutationId) || + (commit.profile !== 'edge' && commit.profile !== 'standalone') || + digestValues.some( + (candidate) => + typeof candidate !== 'string' || !DIGEST_PATTERN.test(candidate), + ) || + secretCount !== environmentSecretCount + sshSecretCount || + !Number.isSafeInteger(commit.committedAtMs) || + (commit.committedAtMs as number) < 0 || + reclamation.modelRemoved !== true || + reclamation.plaintextFilesRemoved !== true || + reclamation.physicalErasureGuaranteed !== false + ) { + throw new LocalDataDirectoryApplicationCommitError( + 'commit values are invalid', + ); + } + const payload: LocalDataDirectoryApplicationCommitPayload = { + schemaVersion: 1, + kind: LOCAL_DATA_DIRECTORY_APPLICATION_COMMIT_KIND, + state: 'committed', + mutationId: commit.mutationId, + profile: commit.profile, + projectIdDigest: commit.projectIdDigest as string, + sourceStageManifestDigest: commit.sourceStageManifestDigest as string, + transformationDigest: commit.transformationDigest as string, + modelDigest: commit.modelDigest as string, + publicationDigest: commit.publicationDigest as string, + receiptDigest: commit.receiptDigest as string, + secretCount, + environmentSecretCount, + sshSecretCount, + committedAtMs: commit.committedAtMs as number, + reclamation: Object.freeze({ + modelRemoved: true, + plaintextFilesRemoved: true, + physicalErasureGuaranteed: false, + }), + }; + if ( + typeof commit.commitDigest !== 'string' || + !DIGEST_PATTERN.test(commit.commitDigest) || + localDataDirectoryApplicationCommitDigest(payload) !== commit.commitDigest + ) { + throw new LocalDataDirectoryApplicationCommitError( + 'commit digest is invalid', + ); + } + return Object.freeze({ ...payload, commitDigest: commit.commitDigest }); +} + +export function createLocalDataDirectoryApplicationCommit( + adoption: Readonly, +): Readonly { + const payload: LocalDataDirectoryApplicationCommitPayload = { + schemaVersion: 1, + kind: LOCAL_DATA_DIRECTORY_APPLICATION_COMMIT_KIND, + state: 'committed', + mutationId: adoption.mutationId, + profile: adoption.profile, + projectIdDigest: digest(adoption.projectId), + sourceStageManifestDigest: adoption.sourceStageManifestDigest, + transformationDigest: adoption.transformationDigest, + modelDigest: adoption.modelDigest, + publicationDigest: adoption.publicationDigest, + receiptDigest: adoption.receiptDigest, + secretCount: adoption.receipt.secretCount, + environmentSecretCount: adoption.receipt.environmentSecretCount, + sshSecretCount: adoption.receipt.sshSecretCount, + committedAtMs: adoption.committedAtMs, + reclamation: Object.freeze({ + modelRemoved: true, + plaintextFilesRemoved: true, + physicalErasureGuaranteed: false, + }), + }; + return Object.freeze({ + ...payload, + commitDigest: localDataDirectoryApplicationCommitDigest(payload), + }); +} diff --git a/packages/ql3-local-sqlite/src/adoption/data-directory/dataDirectoryAdoptionDatabase.ts b/packages/ql3-local-sqlite/src/adoption/data-directory/dataDirectoryAdoptionDatabase.ts index 002778ac..684fbb81 100644 --- a/packages/ql3-local-sqlite/src/adoption/data-directory/dataDirectoryAdoptionDatabase.ts +++ b/packages/ql3-local-sqlite/src/adoption/data-directory/dataDirectoryAdoptionDatabase.ts @@ -34,6 +34,8 @@ import { type LocalSqliteProfile, } from '../../storage/config'; +export * from './applicationCommit'; + const DIGEST_PATTERN = /^[0-9a-f]{64}$/; const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; diff --git a/packages/ql3-local-sqlite/test/dataDirectoryApplicationCommit.test.cjs b/packages/ql3-local-sqlite/test/dataDirectoryApplicationCommit.test.cjs new file mode 100644 index 00000000..08bf3e88 --- /dev/null +++ b/packages/ql3-local-sqlite/test/dataDirectoryApplicationCommit.test.cjs @@ -0,0 +1,60 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); + +const { + LocalDataDirectoryApplicationCommitError, + createLocalDataDirectoryApplicationCommit, + normalizeLocalDataDirectoryApplicationCommit, +} = require('../dist/adoption/data-directory/dataDirectoryAdoptionDatabase.js'); + +function adoptionRecord() { + return { + mutationId: '00000000-0000-4000-8000-000000000001', + projectId: 'project-edge-1', + profile: 'edge', + sourceStageManifestDigest: '1'.repeat(64), + transformationDigest: '2'.repeat(64), + modelDigest: '3'.repeat(64), + publicationDigest: '4'.repeat(64), + receiptDigest: '5'.repeat(64), + committedAtMs: 1_000, + receipt: { + secretCount: 3, + environmentSecretCount: 2, + sshSecretCount: 1, + }, + }; +} + +test('canonical data application commit is exact and replayable', () => { + const commit = createLocalDataDirectoryApplicationCommit(adoptionRecord()); + assert.equal(commit.kind, 'qinglong3-legacy-data-directory-application'); + assert.equal(commit.state, 'committed'); + assert.equal(commit.profile, 'edge'); + assert.equal(commit.receiptDigest, '5'.repeat(64)); + assert.equal(commit.commitDigest.length, 64); + assert.deepEqual( + normalizeLocalDataDirectoryApplicationCommit( + JSON.parse(JSON.stringify(commit)), + ), + commit, + ); +}); + +test('data application commit rejects shape, count, and digest drift', () => { + const commit = createLocalDataDirectoryApplicationCommit(adoptionRecord()); + for (const drift of [ + { ...commit, unexpected: true }, + { ...commit, secretCount: 4 }, + { ...commit, receiptDigest: '0'.repeat(64) }, + { + ...commit, + reclamation: { ...commit.reclamation, physicalErasureGuaranteed: true }, + }, + ]) { + assert.throws( + () => normalizeLocalDataDirectoryApplicationCommit(drift), + LocalDataDirectoryApplicationCommitError, + ); + } +}); diff --git a/scripts/ql3-cluster-dependency-audit.cjs b/scripts/ql3-cluster-dependency-audit.cjs index 2e6b7d04..801c42bb 100644 --- a/scripts/ql3-cluster-dependency-audit.cjs +++ b/scripts/ql3-cluster-dependency-audit.cjs @@ -2649,6 +2649,14 @@ function auditSourceImports(root, packagePath, findings) { ) { continue; } + if ( + packagePath === 'packages/ql3-local-application' && + path.relative(packageDirectory, filePath) === + 'src/production-process/legacyDataApplicationCommitment.ts' && + specifier === '@qinglong/local-sqlite/data-directory-application-commit' + ) { + continue; + } if ( packagePath === 'packages/ql3-local-owner-cli' && path.relative(packageDirectory, filePath) === diff --git a/test/back/ql3ClusterDependencyAudit.test.cjs b/test/back/ql3ClusterDependencyAudit.test.cjs index f2df099c..a031c5b0 100644 --- a/test/back/ql3ClusterDependencyAudit.test.cjs +++ b/test/back/ql3ClusterDependencyAudit.test.cjs @@ -2477,6 +2477,47 @@ test('local application imports only reviewed execution subpaths and process bou ); }); +test('local application receives only the pure data application commit codec', (t) => { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), 'ql3-local-data-commit-codec-boundary-'), + ); + const sourceDirectory = path.join( + root, + 'packages/ql3-local-application/src/production-process', + ); + fs.mkdirSync(sourceDirectory, { recursive: true }); + fs.writeFileSync( + path.join(sourceDirectory, 'legacyDataApplicationCommitment.ts'), + [ + "import { normalize } from '@qinglong/local-sqlite/data-directory-application-commit';", + "import { mutate } from '@qinglong/local-sqlite/data-directory-adoption';", + ].join('\n'), + ); + fs.writeFileSync( + path.join(sourceDirectory, 'neighbor.ts'), + "import { normalize } from '@qinglong/local-sqlite/data-directory-application-commit';", + ); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + + const findings = []; + auditSourceImports(root, 'packages/ql3-local-application', findings); + assert.deepEqual( + findings.map(({ code, file, specifier }) => ({ code, file, specifier })), + [ + { + code: 'FORBIDDEN_PACKAGE_SOURCE_IMPORT', + file: 'packages/ql3-local-application/src/production-process/legacyDataApplicationCommitment.ts', + specifier: '@qinglong/local-sqlite/data-directory-adoption', + }, + { + code: 'FORBIDDEN_PACKAGE_SOURCE_IMPORT', + file: 'packages/ql3-local-application/src/production-process/neighbor.ts', + specifier: '@qinglong/local-sqlite/data-directory-application-commit', + }, + ], + ); +}); + test('local AI application imports only the reviewed dynamic composition subpaths', (t) => { const root = fs.mkdtempSync( path.join(os.tmpdir(), 'ql3-local-ai-application-boundary-'), diff --git a/test/back/ql3PackageBoundaryAudit.test.cjs b/test/back/ql3PackageBoundaryAudit.test.cjs index 5021f9d8..6d059c6c 100644 --- a/test/back/ql3PackageBoundaryAudit.test.cjs +++ b/test/back/ql3PackageBoundaryAudit.test.cjs @@ -540,10 +540,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', ( rootSourceFileRoles: localSqlite.rootSourceFileRoles, }, { - sourceFiles: 201, + sourceFiles: 202, rootSourceFiles: 1, rootSourceLines: 31, - nestedSourceFiles: 200, + nestedSourceFiles: 201, rootSourceFileRoles: { 'index.ts': 'public_export' }, }, );