open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-22 19:29:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): gate startup on legacy data receipt

Browse Source
This commit is contained in:
whyour
2026-08-21 12:49:45 +08:00
parent 3a725cc8e8
commit f1fc316be1
12 changed files with 788 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-local-application/src/production-process/legacyDataApplicationCommitment.ts
+160
View File
@@ -0,0 +1,160 @@
import fs from 'node:fs';
import {
LocalDataDirectoryApplicationCommitError,
normalizeLocalDataDirectoryApplicationCommit,
type LocalDataDirectoryApplicationCommit,
} from '@qinglong/local-sqlite/data-directory-application-commit';
import {
LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4,
type LocalApplicationProcessConfig,
} from './processConfig';
const MAX_COMMIT_BYTES = 64 * 1024;
export class LocalApplicationLegacyDataCommitmentError extends Error {
readonly code = 'QL3_LOCAL_APPLICATION_LEGACY_DATA_COMMITMENT_INVALID';
constructor(message: string, options?: ErrorOptions) {
super(
`Local application legacy data commitment is invalid: ${message}`,
options,
);
this.name = 'LocalApplicationLegacyDataCommitmentError';
}
}
function currentIdentity(): Readonly<{ uid: number; gid: number }> {
if (
typeof process.getuid !== 'function' ||
typeof process.geteuid !== 'function' ||
typeof process.getgid !== 'function' ||
typeof process.getegid !== 'function' ||
process.getuid() !== process.geteuid() ||
process.getgid() !== process.getegid()
) {
throw new LocalApplicationLegacyDataCommitmentError(
'real and effective POSIX identities must match',
);
}
return Object.freeze({ uid: process.getuid(), gid: process.getgid() });
}
function readCommit(
filePath: string,
): Readonly<LocalDataDirectoryApplicationCommit> {
const identity = currentIdentity();
let descriptor: number | undefined;
let bytes: Buffer | undefined;
try {
const before = fs.lstatSync(filePath, { bigint: true });
descriptor = fs.openSync(
filePath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
const opened = fs.fstatSync(descriptor, { bigint: true });
if (
!opened.isFile() ||
opened.isSymbolicLink() ||
opened.dev !== before.dev ||
opened.ino !== before.ino ||
Number(opened.uid) !== identity.uid ||
Number(opened.gid) !== identity.gid ||
(Number(opened.mode) & 0o777) !== 0o600 ||
opened.nlink !== 1n ||
opened.size < 2n ||
opened.size > BigInt(MAX_COMMIT_BYTES)
) {
throw new LocalApplicationLegacyDataCommitmentError(
'commit file identity is invalid',
);
}
bytes = Buffer.alloc(Number(opened.size));
let offset = 0;
while (offset < bytes.byteLength) {
const count = fs.readSync(
descriptor,
bytes,
offset,
bytes.byteLength - offset,
offset,
);
if (count === 0) break;
offset += count;
}
const after = fs.fstatSync(descriptor, { bigint: true });
if (
offset !== bytes.byteLength ||
after.dev !== opened.dev ||
after.ino !== opened.ino ||
after.size !== opened.size ||
after.mtimeNs !== opened.mtimeNs ||
after.ctimeNs !== opened.ctimeNs ||
after.uid !== opened.uid ||
after.gid !== opened.gid ||
after.mode !== opened.mode ||
after.nlink !== opened.nlink
) {
throw new LocalApplicationLegacyDataCommitmentError(
'commit file identity changed while reading',
);
}
let value: unknown;
try {
value = JSON.parse(
new TextDecoder('utf-8', { fatal: true }).decode(bytes),
);
} catch (error) {
throw new LocalApplicationLegacyDataCommitmentError(
'commit file is not canonical UTF-8 JSON',
{ cause: error },
);
}
try {
return normalizeLocalDataDirectoryApplicationCommit(value);
} catch (error) {
if (error instanceof LocalDataDirectoryApplicationCommitError) {
throw new LocalApplicationLegacyDataCommitmentError(
'commit document is invalid',
{ cause: error },
);
}
throw error;
}
} catch (error) {
if (error instanceof LocalApplicationLegacyDataCommitmentError) throw error;
throw new LocalApplicationLegacyDataCommitmentError(
'commit file cannot be read',
{ cause: error },
);
} finally {
bytes?.fill(0);
if (descriptor !== undefined) fs.closeSync(descriptor);
}
}
export function verifyLocalApplicationLegacyDataCommitment(
config: Readonly<LocalApplicationProcessConfig>,
): Readonly<LocalDataDirectoryApplicationCommit> | undefined {
if (config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4) {
return undefined;
}
const binding = config.legacyDataApplication;
if (binding === undefined) {
throw new LocalApplicationLegacyDataCommitmentError(
'v4 binding is unavailable',
);
}
const commit = readCommit(binding.commitPath);
if (
commit.profile !== config.profile ||
commit.commitDigest !== binding.expectedCommitDigest ||
commit.receiptDigest !== binding.expectedReceiptDigest
) {
throw new LocalApplicationLegacyDataCommitmentError(
'commit does not match the application binding',
);
}
return commit;
}
packages/ql3-local-application/src/production-process/processApplication.ts
+2
View File
@@ -19,6 +19,7 @@ import {
type LocalApplicationProcessConfig,
} from './processConfig';
import { verifyLocalApplicationCutoverCommitment } from './cutoverCommitment';
import { verifyLocalApplicationLegacyDataCommitment } from './legacyDataApplicationCommitment';
import { recordLocalApplicationShutdownReceipt } from './shutdownReceipt';
import { recordLocalApplicationStartupReceipt } from './startupReceipt';
@@ -221,6 +222,7 @@ export async function runProductionLocalApplicationProcess(
throw new TypeError('Local application process options are invalid');
}
const config = loadLocalApplicationProcessConfig(options.configFilePath);
verifyLocalApplicationLegacyDataCommitment(config);
verifyLocalApplicationCutoverCommitment(config);
const selectedAi = aiOptions(config, options);
const start = options.start ?? defaultStarter;
packages/ql3-local-application/src/production-process/processConfig.ts
+67 -15
View File
@@ -2,9 +2,7 @@ import path from 'node:path';
import { readPrivateLocalCommandFile } from '@qinglong/local-command-file';
import { MAX_PLUGIN_PACKAGE_INSTALL_RECOVERY_PAGE_SIZE } from '@qinglong/runtime-core/plugin-package-install';
import {
MAX_PLUGIN_PACKAGE_RECOVERY_PAGES,
} from '@qinglong/runtime-core/plugin-package-recovery';
import { MAX_PLUGIN_PACKAGE_RECOVERY_PAGES } from '@qinglong/runtime-core/plugin-package-recovery';
import {
MAX_PLUGIN_PACKAGE_TASK_PUBLICATION_RECOVERY_PAGES,
MAX_PLUGIN_PACKAGE_TASK_PUBLICATION_RECOVERY_PAGE_SIZE,
@@ -18,6 +16,8 @@ export const LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V2 =
'qinglong/local-application-process@v2' as const;
export const LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 =
'qinglong/local-application-process@v3' as const;
export const LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4 =
'qinglong/local-application-process@v4' as const;
const MAX_PATH_BYTES = 4_096;
const INSTANCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
@@ -56,6 +56,12 @@ export interface LocalApplicationProcessCutoverConfig {
readonly expectedCommitmentDigest: string;
}
export interface LocalApplicationProcessLegacyDataApplicationConfig {
readonly commitPath: string;
readonly expectedCommitDigest: string;
readonly expectedReceiptDigest: string;
}
export type LocalApplicationProcessPluginPackageRecoverySourceConfig =
| Readonly<{ mode: 'disabled' }>
| Readonly<{
@@ -89,7 +95,8 @@ export interface LocalApplicationProcessConfig {
readonly schema:
| typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA
| typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V2
| typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3;
| typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3
| typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4;
readonly instanceId: string;
readonly profile: LocalApplicationProfile;
readonly storage: Readonly<LocalApplicationProcessStorageConfig>;
@@ -97,13 +104,17 @@ export interface LocalApplicationProcessConfig {
readonly pluginPackages: Readonly<LocalApplicationProcessPluginPackageConfig>;
readonly ai: LocalApplicationProcessAiConfig;
readonly cutover?: Readonly<LocalApplicationProcessCutoverConfig>;
readonly legacyDataApplication?: Readonly<LocalApplicationProcessLegacyDataApplicationConfig>;
}
export class LocalApplicationProcessConfigError extends TypeError {
readonly code = 'QL3_LOCAL_APPLICATION_PROCESS_CONFIG_INVALID';
constructor(message: string, options?: ErrorOptions) {
super(`Local application process configuration is invalid: ${message}`, options);
super(
`Local application process configuration is invalid: ${message}`,
options,
);
this.name = 'LocalApplicationProcessConfigError';
}
}
@@ -175,7 +186,8 @@ function storageConfig(
schema:
| typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA
| typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V2
| typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3,
| typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3
| typeof LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4,
): Readonly<LocalApplicationProcessStorageConfig> {
const storage = record(value, 'storage');
if (schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA) {
@@ -218,9 +230,7 @@ function storageConfig(
'recoveryPath',
'sourcePath',
'targetPath',
...(schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA
? ['mode']
: []),
...(schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA ? ['mode'] : []),
...optionalKeys,
],
'storage',
@@ -297,6 +307,32 @@ function cutoverConfig(
});
}
function legacyDataApplicationConfig(
value: unknown,
): Readonly<LocalApplicationProcessLegacyDataApplicationConfig> {
const application = record(value, 'legacyDataApplication');
exactKeys(
application,
['commitPath', 'expectedCommitDigest', 'expectedReceiptDigest'],
'legacyDataApplication',
);
if (
typeof application.expectedCommitDigest !== 'string' ||
!DIGEST_PATTERN.test(application.expectedCommitDigest) ||
typeof application.expectedReceiptDigest !== 'string' ||
!DIGEST_PATTERN.test(application.expectedReceiptDigest)
) {
throw new LocalApplicationProcessConfigError(
'legacyDataApplication digest is invalid',
);
}
return Object.freeze({
commitPath: absolutePath(application.commitPath, 'commitPath'),
expectedCommitDigest: application.expectedCommitDigest,
expectedReceiptDigest: application.expectedReceiptDigest,
});
}
function runtimeConfig(
value: unknown,
): Readonly<LocalApplicationProcessRuntimeConfig> {
@@ -498,7 +534,8 @@ export function normalizeLocalApplicationProcessConfig(
if (
config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA &&
config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V2 &&
config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3
config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 &&
config.schema !== LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4
) {
throw new LocalApplicationProcessConfigError('schema is invalid');
}
@@ -506,10 +543,14 @@ export function normalizeLocalApplicationProcessConfig(
config,
[
'ai',
...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3
...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 ||
config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4
? ['cutover']
: []),
'instanceId',
...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4
? ['legacyDataApplication']
: []),
'pluginPackages',
'profile',
'runtime',
@@ -528,7 +569,8 @@ export function normalizeLocalApplicationProcessConfig(
throw new LocalApplicationProcessConfigError('profile is invalid');
}
if (
config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 &&
(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 ||
config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4) &&
record(config.storage, 'storage').mode !== 'adopted'
) {
throw new LocalApplicationProcessConfigError(
@@ -543,9 +585,17 @@ export function normalizeLocalApplicationProcessConfig(
runtime: runtimeConfig(config.runtime),
pluginPackages: pluginPackageConfig(config.pluginPackages),
ai: aiConfig(config.ai),
...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3
...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V3 ||
config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4
? { cutover: cutoverConfig(config.cutover) }
: {}),
...(config.schema === LOCAL_APPLICATION_PROCESS_CONFIG_SCHEMA_V4
? {
legacyDataApplication: legacyDataApplicationConfig(
config.legacyDataApplication,
),
}
: {}),
} as const;
const authorityPaths = [
...(normalized.storage.mode === 'fresh'
@@ -565,8 +615,10 @@ export function normalizeLocalApplicationProcessConfig(
...(normalized.cutover === undefined
? []
: [normalized.cutover.commitmentPath]),
...(normalized.pluginPackages.recoverySource.mode ===
'materialized_catalog'
...(normalized.legacyDataApplication === undefined
? []
: [normalized.legacyDataApplication.commitPath]),
...(normalized.pluginPackages.recoverySource.mode === 'materialized_catalog'
? [
normalized.pluginPackages.recoverySource.catalogRoot,
normalized.pluginPackages.recoverySource.bundleRoot,