feat(ql3): stage legacy data directory

This commit is contained in:
whyour
2026-08-21 10:09:34 +08:00
parent 878a360b09
commit 19bb09faa3
14 changed files with 2139 additions and 26 deletions
@@ -2,7 +2,7 @@ import { readPrivateLocalCommandFile } from '@qinglong/local-command-file';
import { runLegacyCrontabAdoptionCommandFile } from './adoption';
import { isLocalDataDirectoryAdoptionOperation } from './data-directory-adoption/contract';
import type { LocalDataDirectoryAdoptionInspectResult } from './data-directory-adoption/inventory';
import type { LocalDataDirectoryAdoptionProductCommandResult } from './data-directory-adoption/command';
import {
isLocalSqliteAdoptionProductOperation,
type LocalSqliteAdoptionProductOperation,
@@ -12,7 +12,7 @@ import type { LocalSqliteAdoptionProductCommandResult } from './sqlite-adoption/
export type LocalAdoptionProductCommandResult =
| Awaited<ReturnType<typeof runLegacyCrontabAdoptionCommandFile>>
| LocalSqliteAdoptionProductCommandResult
| LocalDataDirectoryAdoptionInspectResult;
| LocalDataDirectoryAdoptionProductCommandResult;
function operation(value: unknown): unknown {
return value && typeof value === 'object' && !Array.isArray(value)
@@ -32,10 +32,10 @@ export async function runLocalAdoptionProductCommandFile(
}
const selected = operation(candidate);
if (isLocalDataDirectoryAdoptionOperation(selected)) {
const { inspectLocalDataDirectoryAdoption } = await import(
'./data-directory-adoption/inventory.js'
const { runLocalDataDirectoryAdoptionProductCommand } = await import(
'./data-directory-adoption/command.js'
);
return inspectLocalDataDirectoryAdoption(candidate);
return runLocalDataDirectoryAdoptionProductCommand(candidate);
}
if (!isLocalSqliteAdoptionProductOperation(selected)) {
return runLegacyCrontabAdoptionCommandFile(commandFilePath);
@@ -0,0 +1,31 @@
import {
LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION,
LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION,
normalizeLocalDataDirectoryAdoptionCommand,
} from './contract';
import {
inspectLocalDataDirectoryAdoption,
type LocalDataDirectoryAdoptionInspectResult,
} from './inventory';
import {
stageLocalDataDirectoryAdoption,
verifyLocalDataDirectoryAdoption,
type LocalDataDirectoryAdoptionMutationResult,
} from './staging';
export type LocalDataDirectoryAdoptionProductCommandResult =
| LocalDataDirectoryAdoptionInspectResult
| LocalDataDirectoryAdoptionMutationResult;
export async function runLocalDataDirectoryAdoptionProductCommand(
value: unknown,
): Promise<Readonly<LocalDataDirectoryAdoptionProductCommandResult>> {
const command = normalizeLocalDataDirectoryAdoptionCommand(value);
if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION) {
return inspectLocalDataDirectoryAdoption(command);
}
if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION) {
return stageLocalDataDirectoryAdoption(command);
}
return verifyLocalDataDirectoryAdoption(command);
}
@@ -1,9 +1,19 @@
import path from 'node:path';
const MAX_PATH_BYTES = 4_096;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
export const LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION =
'local-data-directory.adoption.inspect' as const;
export const LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION =
'local-data-directory.adoption.stage' as const;
export const LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION =
'local-data-directory.adoption.verify' as const;
export type LocalDataDirectoryAdoptionOperation =
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION;
export interface InspectLocalDataDirectoryAdoptionCommand {
readonly schemaVersion: 1;
@@ -14,6 +24,44 @@ export interface InspectLocalDataDirectoryAdoptionCommand {
};
}
export interface LocalDataDirectoryAdoptionSqliteBinding {
readonly sourcePath: string;
readonly targetPath: string;
readonly recoveryPath: string;
readonly manifestPath: string;
readonly activationPath: string;
readonly expectedActivationDigest: string;
}
interface LocalDataDirectoryAdoptionMutationOptions {
readonly deploymentRoot: string;
readonly dataRoot: string;
readonly stagingRoot: string;
readonly profile: 'edge' | 'standalone';
readonly sqlite: LocalDataDirectoryAdoptionSqliteBinding;
}
export interface StageLocalDataDirectoryAdoptionCommand {
readonly schemaVersion: 1;
readonly operation: typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION;
readonly options: LocalDataDirectoryAdoptionMutationOptions & {
readonly expectedPlanDigest: string;
};
}
export interface VerifyLocalDataDirectoryAdoptionCommand {
readonly schemaVersion: 1;
readonly operation: typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION;
readonly options: LocalDataDirectoryAdoptionMutationOptions & {
readonly expectedManifestDigest: string;
};
}
export type LocalDataDirectoryAdoptionCommand =
| InspectLocalDataDirectoryAdoptionCommand
| StageLocalDataDirectoryAdoptionCommand
| VerifyLocalDataDirectoryAdoptionCommand;
export class LocalDataDirectoryAdoptionConfigurationError extends TypeError {
readonly code = 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID';
@@ -45,13 +93,77 @@ function normalizedAbsolutePath(value: unknown): value is string {
export function isLocalDataDirectoryAdoptionOperation(
value: unknown,
): value is typeof LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION {
return value === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION;
): value is LocalDataDirectoryAdoptionOperation {
return (
value === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION ||
value === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION ||
value === LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION
);
}
export function normalizeInspectLocalDataDirectoryAdoptionCommand(
function normalizeSqliteBinding(
value: unknown,
): Readonly<InspectLocalDataDirectoryAdoptionCommand> {
): Readonly<LocalDataDirectoryAdoptionSqliteBinding> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!exactKeys(value, [
'activationPath',
'expectedActivationDigest',
'manifestPath',
'recoveryPath',
'sourcePath',
'targetPath',
])
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation binding shape is invalid',
);
}
const binding = value as Record<string, unknown>;
for (const key of [
'activationPath',
'manifestPath',
'recoveryPath',
'sourcePath',
'targetPath',
]) {
if (!normalizedAbsolutePath(binding[key])) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation binding path is invalid',
);
}
}
if (
typeof binding.expectedActivationDigest !== 'string' ||
!DIGEST_PATTERN.test(binding.expectedActivationDigest)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation digest is invalid',
);
}
if (
new Set(
[
binding.activationPath,
binding.manifestPath,
binding.recoveryPath,
binding.sourcePath,
binding.targetPath,
].map((candidate) => path.resolve(candidate as string)),
).size !== 5
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation paths must be distinct',
);
}
return Object.freeze(value as LocalDataDirectoryAdoptionSqliteBinding);
}
export function normalizeLocalDataDirectoryAdoptionCommand(
value: unknown,
): Readonly<LocalDataDirectoryAdoptionCommand> {
if (
!value ||
typeof value !== 'object' ||
@@ -68,15 +180,35 @@ export function normalizeInspectLocalDataDirectoryAdoptionCommand(
!isLocalDataDirectoryAdoptionOperation(candidate.operation) ||
!candidate.options ||
typeof candidate.options !== 'object' ||
Array.isArray(candidate.options) ||
!exactKeys(candidate.options, ['dataRoot', 'profile'])
Array.isArray(candidate.options)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'command value is invalid',
);
}
const options = candidate.options as Record<string, unknown>;
const expectedKeys =
candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION
? ['dataRoot', 'profile']
: candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
? [
'dataRoot',
'deploymentRoot',
'expectedPlanDigest',
'profile',
'sqlite',
'stagingRoot',
]
: [
'dataRoot',
'deploymentRoot',
'expectedManifestDigest',
'profile',
'sqlite',
'stagingRoot',
];
if (
!exactKeys(options, expectedKeys) ||
!normalizedAbsolutePath(options.dataRoot) ||
(options.profile !== 'edge' && options.profile !== 'standalone')
) {
@@ -84,5 +216,37 @@ export function normalizeInspectLocalDataDirectoryAdoptionCommand(
'command options are invalid',
);
}
return Object.freeze(value as InspectLocalDataDirectoryAdoptionCommand);
if (candidate.operation !== LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION) {
if (
!normalizedAbsolutePath(options.deploymentRoot) ||
!normalizedAbsolutePath(options.stagingRoot)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'adoption root path is invalid',
);
}
normalizeSqliteBinding(options.sqlite);
const digest =
candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
? options.expectedPlanDigest
: options.expectedManifestDigest;
if (typeof digest !== 'string' || !DIGEST_PATTERN.test(digest)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'reviewed adoption digest is invalid',
);
}
}
return Object.freeze(value as LocalDataDirectoryAdoptionCommand);
}
export function normalizeInspectLocalDataDirectoryAdoptionCommand(
value: unknown,
): Readonly<InspectLocalDataDirectoryAdoptionCommand> {
const command = normalizeLocalDataDirectoryAdoptionCommand(value);
if (command.operation !== LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'inspection operation is invalid',
);
}
return command;
}
@@ -0,0 +1,430 @@
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import {
LocalDataDirectoryAdoptionConfigurationError,
type StageLocalDataDirectoryAdoptionCommand,
type VerifyLocalDataDirectoryAdoptionCommand,
} from './contract';
const HASH_BUFFER_BYTES = 64 * 1024;
const MAX_RELATIVE_PATH_BYTES = 4_096;
export interface RootAuthority {
readonly uid: number;
readonly deploymentRoot: string;
readonly dataRoot: string;
readonly stagingRoot: string;
}
export interface CopyBudget {
readonly maxEntries: number;
readonly maxHashedBytes: number;
readonly maxFileBytes: number;
readonly maxDepth: number;
}
export interface MutableCopyBudget {
entries: number;
bytes: number;
}
function currentUid(): number {
if (
typeof process.getuid !== 'function' ||
typeof process.geteuid !== 'function' ||
process.getuid() !== process.geteuid()
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'real and effective POSIX users must match',
);
}
return process.getuid();
}
function inside(root: string, candidate: string): boolean {
const relative = path.relative(root, candidate);
return (
relative !== '' &&
relative !== '..' &&
!relative.startsWith(`..${path.sep}`) &&
!path.isAbsolute(relative)
);
}
export function assertPrivateDirectory(
directoryPath: string,
uid: number,
label: string,
): fs.BigIntStats {
let stat: fs.BigIntStats;
try {
stat = fs.lstatSync(directoryPath, { bigint: true });
} catch (error) {
throw new LocalDataDirectoryAdoptionConfigurationError(
`${label} is unavailable`,
error,
);
}
if (
!stat.isDirectory() ||
stat.isSymbolicLink() ||
stat.uid !== BigInt(uid) ||
(stat.mode & 0o777n) !== 0o700n ||
fs.realpathSync(directoryPath) !== directoryPath
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
`${label} must be an owner-controlled 0700 canonical directory`,
);
}
return stat;
}
function assertMissing(candidate: string, label: string): void {
try {
fs.lstatSync(candidate);
} catch (error) {
if (
error &&
typeof error === 'object' &&
'code' in error &&
error.code === 'ENOENT'
) {
return;
}
throw new LocalDataDirectoryAdoptionConfigurationError(
`${label} cannot be inspected`,
error,
);
}
throw new LocalDataDirectoryAdoptionConfigurationError(
`${label} must not already exist`,
);
}
export function rootAuthority(
options:
| StageLocalDataDirectoryAdoptionCommand['options']
| VerifyLocalDataDirectoryAdoptionCommand['options'],
requireMissing: boolean,
): Readonly<RootAuthority> {
const uid = currentUid();
assertPrivateDirectory(options.deploymentRoot, uid, 'deploymentRoot');
if (
!inside(options.deploymentRoot, options.stagingRoot) ||
options.dataRoot === options.stagingRoot ||
inside(options.dataRoot, options.stagingRoot) ||
inside(options.stagingRoot, options.dataRoot)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'stagingRoot must be isolated inside deploymentRoot',
);
}
const stagingParent = path.dirname(options.stagingRoot);
if (
stagingParent !== options.deploymentRoot &&
!inside(options.deploymentRoot, stagingParent)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'stagingRoot parent must remain inside deploymentRoot',
);
}
assertPrivateDirectory(stagingParent, uid, 'stagingRoot parent');
const expectedSource = path.join(options.dataRoot, 'db', 'database.sqlite');
if (options.sqlite.sourcePath !== expectedSource) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation source must be the reviewed primary database',
);
}
for (const candidate of [
options.sqlite.targetPath,
options.sqlite.recoveryPath,
options.sqlite.manifestPath,
options.sqlite.activationPath,
]) {
if (candidate === options.dataRoot || inside(options.dataRoot, candidate)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite adoption evidence must remain outside dataRoot',
);
}
}
if (requireMissing) assertMissing(options.stagingRoot, 'stagingRoot');
else assertPrivateDirectory(options.stagingRoot, uid, 'stagingRoot');
return Object.freeze({
uid,
deploymentRoot: options.deploymentRoot,
dataRoot: options.dataRoot,
stagingRoot: options.stagingRoot,
});
}
export function sameStat(left: fs.BigIntStats, right: fs.BigIntStats): boolean {
return (
left.dev === right.dev &&
left.ino === right.ino &&
left.mode === right.mode &&
left.nlink === right.nlink &&
left.uid === right.uid &&
left.size === right.size &&
left.mtimeNs === right.mtimeNs &&
left.ctimeNs === right.ctimeNs
);
}
export function sortedNames(directoryPath: string): readonly string[] {
return fs
.readdirSync(directoryPath)
.sort((left, right) =>
Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')),
);
}
function assertRelativePath(value: string): void {
if (
value.length < 1 ||
path.isAbsolute(value) ||
value === '..' ||
value.startsWith(`..${path.sep}`) ||
Buffer.byteLength(value, 'utf8') > MAX_RELATIVE_PATH_BYTES
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'payload relative path is invalid or too long',
);
}
}
export function syncDirectory(directoryPath: string): void {
const descriptor = fs.openSync(directoryPath, fs.constants.O_RDONLY);
try {
fs.fsyncSync(descriptor);
} finally {
fs.closeSync(descriptor);
}
}
export function writeExclusiveJson(filePath: string, value: object): void {
const descriptor = fs.openSync(filePath, 'wx', 0o600);
try {
const bytes = Buffer.from(`${JSON.stringify(value)}\n`, 'utf8');
let offset = 0;
while (offset < bytes.length) {
offset += fs.writeSync(
descriptor,
bytes,
offset,
bytes.length - offset,
null,
);
}
fs.fsyncSync(descriptor);
} finally {
fs.closeSync(descriptor);
}
}
function copyStableFile(
sourcePath: string,
destinationPath: string,
expected: fs.BigIntStats,
): void {
const source = fs.openSync(
sourcePath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
let destination: number | undefined;
const buffer = Buffer.allocUnsafe(HASH_BUFFER_BYTES);
try {
const before = fs.fstatSync(source, { bigint: true });
if (!before.isFile() || !sameStat(expected, before)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source file identity changed before staging',
);
}
destination = fs.openSync(destinationPath, 'wx', 0o600);
for (;;) {
const count = fs.readSync(source, buffer, 0, buffer.length, null);
if (count === 0) break;
let offset = 0;
while (offset < count) {
offset += fs.writeSync(
destination,
buffer,
offset,
count - offset,
null,
);
}
}
fs.fsyncSync(destination);
if (!sameStat(before, fs.fstatSync(source, { bigint: true }))) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source file changed during staging',
);
}
} finally {
buffer.fill(0);
if (destination !== undefined) fs.closeSync(destination);
fs.closeSync(source);
}
}
function shouldExcludeDatabaseEntry(
category: string,
relative: string,
): boolean {
return (
category === 'db' &&
/^(?:database\.sqlite|database\.sqlite-(?:wal|shm|journal))$/.test(relative)
);
}
export function copyCategory(
sourceRoot: string,
destinationRoot: string,
category: string,
uid: number,
limits: Readonly<CopyBudget>,
shared: MutableCopyBudget,
): void {
const sourceCategory = path.join(sourceRoot, category);
let categoryStat: fs.BigIntStats;
try {
categoryStat = fs.lstatSync(sourceCategory, { bigint: true });
} catch (error) {
if (
error &&
typeof error === 'object' &&
'code' in error &&
error.code === 'ENOENT'
) {
return;
}
throw error;
}
if (
!categoryStat.isDirectory() ||
categoryStat.isSymbolicLink() ||
categoryStat.uid !== BigInt(uid) ||
(categoryStat.mode & 0o022n) !== 0n
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source category identity is unsafe',
);
}
const destinationCategory = path.join(destinationRoot, category);
fs.mkdirSync(destinationCategory, { mode: 0o700 });
const visit = (
sourceDirectory: string,
destinationDirectory: string,
expectedDirectory: fs.BigIntStats,
depth: number,
): void => {
if (depth > limits.maxDepth) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload depth exceeds the Profile budget',
);
}
for (const name of sortedNames(sourceDirectory)) {
const sourceEntry = path.join(sourceDirectory, name);
const categoryRelative = path.relative(sourceCategory, sourceEntry);
assertRelativePath(categoryRelative);
if (shouldExcludeDatabaseEntry(category, categoryRelative)) continue;
shared.entries += 1;
if (shared.entries > limits.maxEntries) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload entry count exceeds the Profile budget',
);
}
const destinationEntry = path.join(destinationDirectory, name);
const stat = fs.lstatSync(sourceEntry, { bigint: true });
if (
stat.isSymbolicLink() ||
stat.uid !== BigInt(uid) ||
(stat.mode & 0o022n) !== 0n
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload entry identity is unsafe',
);
}
if (stat.isDirectory()) {
fs.mkdirSync(destinationEntry, { mode: 0o700 });
visit(sourceEntry, destinationEntry, stat, depth + 1);
syncDirectory(destinationEntry);
} else if (stat.isFile() && stat.nlink === 1n) {
if (
stat.size < 0n ||
stat.size > BigInt(limits.maxFileBytes) ||
stat.size > BigInt(Number.MAX_SAFE_INTEGER)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload file exceeds the Profile budget',
);
}
const bytes = Number(stat.size);
if (
!Number.isSafeInteger(shared.bytes + bytes) ||
shared.bytes + bytes > limits.maxHashedBytes
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload bytes exceed the Profile budget',
);
}
shared.bytes += bytes;
copyStableFile(sourceEntry, destinationEntry, stat);
} else {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload entry kind is unsafe',
);
}
}
if (
!sameStat(
expectedDirectory,
fs.lstatSync(sourceDirectory, { bigint: true }),
)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source directory changed during staging',
);
}
};
visit(sourceCategory, destinationCategory, categoryStat, 1);
syncDirectory(destinationCategory);
}
export function stableFileDigest(
filePath: string,
expected: fs.BigIntStats,
): string {
const descriptor = fs.openSync(
filePath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
const buffer = Buffer.allocUnsafe(HASH_BUFFER_BYTES);
try {
const before = fs.fstatSync(descriptor, { bigint: true });
if (!before.isFile() || !sameStat(expected, before)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged file identity changed before verification',
);
}
const hash = crypto.createHash('sha256');
for (;;) {
const count = fs.readSync(descriptor, buffer, 0, buffer.length, null);
if (count === 0) break;
hash.update(buffer.subarray(0, count));
}
if (!sameStat(before, fs.fstatSync(descriptor, { bigint: true }))) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged file changed during verification',
);
}
return hash.digest('hex');
} finally {
buffer.fill(0);
fs.closeSync(descriptor);
}
}
@@ -0,0 +1,437 @@
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { LocalDataDirectoryAdoptionConfigurationError } from './contract';
import {
assertPrivateDirectory,
sameStat,
sortedNames,
stableFileDigest,
type RootAuthority,
} from './filesystem';
const MAX_MANIFEST_BYTES = 64 * 1024;
const MAX_RELATIVE_PATH_BYTES = 4_096;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
export const MANIFEST_NAME = 'manifest.json';
export const PAYLOAD_GROUPS = Object.freeze([
Object.freeze({
name: 'copy_reviewed' as const,
directoryName: 'copy-reviewed',
categories: Object.freeze(['scripts', 'upload'] as const),
}),
Object.freeze({
name: 'transform_input' as const,
directoryName: 'transform-input',
categories: Object.freeze(['config', 'db', 'ssh.d'] as const),
}),
]);
type PayloadGroupName = (typeof PAYLOAD_GROUPS)[number]['name'];
export interface LocalDataDirectoryPayloadEvidence {
readonly name: PayloadGroupName;
readonly categories: readonly string[];
readonly entries: number;
readonly directories: number;
readonly files: number;
readonly bytes: number;
readonly digest: string;
}
export interface LocalDataDirectoryAdoptionManifestPayload {
readonly schemaVersion: 1;
readonly kind: 'qinglong3-legacy-data-directory-adoption';
readonly state: 'staged';
readonly profile: 'edge' | 'standalone';
readonly createdAtMs: number;
readonly planDigest: string;
readonly sqliteActivationDigest: string;
readonly sqliteAdoptionManifestDigest: string;
readonly dataRootPathDigest: string;
readonly stagingRootPathDigest: string;
readonly payload: readonly LocalDataDirectoryPayloadEvidence[];
}
export interface LocalDataDirectoryAdoptionManifest
extends LocalDataDirectoryAdoptionManifestPayload {
readonly manifestDigest: string;
}
interface MutablePayloadSummary {
entries: number;
directories: number;
files: number;
bytes: number;
}
export function sha256Text(value: string): string {
return crypto.createHash('sha256').update(value, 'utf8').digest('hex');
}
function exactKeys(value: object, expected: readonly string[]): boolean {
const actual = Object.keys(value).sort();
const canonical = [...expected].sort();
return (
actual.length === canonical.length &&
actual.every((key, index) => key === canonical[index])
);
}
function assertRelativePath(value: string): void {
if (
value.length < 1 ||
path.isAbsolute(value) ||
value === '..' ||
value.startsWith(`..${path.sep}`) ||
Buffer.byteLength(value, 'utf8') > MAX_RELATIVE_PATH_BYTES
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'payload relative path is invalid or too long',
);
}
}
function payloadEvidence(
groupRoot: string,
group: (typeof PAYLOAD_GROUPS)[number],
uid: number,
): Readonly<LocalDataDirectoryPayloadEvidence> {
assertPrivateDirectory(groupRoot, uid, 'payload group');
const allowed = new Set<string>(group.categories);
const summary: MutablePayloadSummary = {
entries: 0,
directories: 0,
files: 0,
bytes: 0,
};
const hash = crypto.createHash('sha256');
const visit = (directoryPath: string, expected: fs.BigIntStats): void => {
for (const name of sortedNames(directoryPath)) {
const entryPath = path.join(directoryPath, name);
const relative = path.relative(groupRoot, entryPath);
assertRelativePath(relative);
if (directoryPath === groupRoot && !allowed.has(name)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload contains an unexpected category',
);
}
const stat = fs.lstatSync(entryPath, { bigint: true });
summary.entries += 1;
if (
stat.isSymbolicLink() ||
stat.uid !== BigInt(uid) ||
(stat.mode & 0o777n) !== (stat.isDirectory() ? 0o700n : 0o600n)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload identity or mode is invalid',
);
}
const canonicalRelative = relative.split(path.sep).join('/');
if (stat.isDirectory()) {
summary.directories += 1;
hash.update(
`${JSON.stringify({
relative: canonicalRelative,
kind: 'directory',
})}\n`,
'utf8',
);
visit(entryPath, stat);
} else if (stat.isFile() && stat.nlink === 1n) {
if (stat.size < 0n || stat.size > BigInt(Number.MAX_SAFE_INTEGER)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload file size is unsupported',
);
}
const bytes = Number(stat.size);
if (!Number.isSafeInteger(summary.bytes + bytes)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload byte total is unsupported',
);
}
summary.files += 1;
summary.bytes += bytes;
hash.update(
`${JSON.stringify({
relative: canonicalRelative,
kind: 'file',
bytes,
contentDigest: stableFileDigest(entryPath, stat),
})}\n`,
'utf8',
);
} else {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload entry kind is invalid',
);
}
}
if (!sameStat(expected, fs.lstatSync(directoryPath, { bigint: true }))) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload directory changed during verification',
);
}
};
visit(groupRoot, fs.lstatSync(groupRoot, { bigint: true }));
return Object.freeze({
name: group.name,
categories: group.categories,
...summary,
digest: hash.digest('hex'),
});
}
export function inspectPayload(
stagingRoot: string,
uid: number,
): readonly LocalDataDirectoryPayloadEvidence[] {
const payloadRoot = path.join(stagingRoot, 'payload');
const payloadRootBefore = assertPrivateDirectory(
payloadRoot,
uid,
'payload root',
);
const expectedGroupNames = PAYLOAD_GROUPS.map((group) => group.directoryName);
if (
JSON.stringify(sortedNames(payloadRoot)) !==
JSON.stringify(
[...expectedGroupNames].sort((left, right) =>
Buffer.compare(Buffer.from(left), Buffer.from(right)),
),
)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'payload group set is invalid',
);
}
const evidence = Object.freeze(
PAYLOAD_GROUPS.map((group) =>
payloadEvidence(path.join(payloadRoot, group.directoryName), group, uid),
),
);
if (
!sameStat(payloadRootBefore, fs.lstatSync(payloadRoot, { bigint: true }))
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'payload root changed during verification',
);
}
return evidence;
}
function parsePayloadEvidence(
value: unknown,
): LocalDataDirectoryPayloadEvidence {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!exactKeys(value, [
'bytes',
'categories',
'digest',
'directories',
'entries',
'files',
'name',
])
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'manifest payload evidence shape is invalid',
);
}
const candidate = value as Partial<LocalDataDirectoryPayloadEvidence>;
const group = PAYLOAD_GROUPS.find((entry) => entry.name === candidate.name);
if (
!group ||
JSON.stringify(candidate.categories) !== JSON.stringify(group.categories) ||
!DIGEST_PATTERN.test(candidate.digest ?? '') ||
![
candidate.entries,
candidate.directories,
candidate.files,
candidate.bytes,
].every(
(number) => Number.isSafeInteger(number) && (number as number) >= 0,
) ||
candidate.entries !==
(candidate.directories as number) + (candidate.files as number)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'manifest payload evidence is invalid',
);
}
return value as LocalDataDirectoryPayloadEvidence;
}
function parseManifest(value: unknown): LocalDataDirectoryAdoptionManifest {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!exactKeys(value, [
'createdAtMs',
'dataRootPathDigest',
'kind',
'manifestDigest',
'payload',
'planDigest',
'profile',
'schemaVersion',
'sqliteActivationDigest',
'sqliteAdoptionManifestDigest',
'stagingRootPathDigest',
'state',
])
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest shape is invalid',
);
}
const manifest = value as Partial<LocalDataDirectoryAdoptionManifest>;
if (
manifest.schemaVersion !== 1 ||
manifest.kind !== 'qinglong3-legacy-data-directory-adoption' ||
manifest.state !== 'staged' ||
(manifest.profile !== 'edge' && manifest.profile !== 'standalone') ||
!Number.isSafeInteger(manifest.createdAtMs) ||
(manifest.createdAtMs as number) < 0 ||
![
manifest.manifestDigest,
manifest.planDigest,
manifest.sqliteActivationDigest,
manifest.sqliteAdoptionManifestDigest,
manifest.dataRootPathDigest,
manifest.stagingRootPathDigest,
].every(
(digest) => typeof digest === 'string' && DIGEST_PATTERN.test(digest),
) ||
!Array.isArray(manifest.payload) ||
manifest.payload.length !== PAYLOAD_GROUPS.length
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest value is invalid',
);
}
const parsedPayload = manifest.payload.map(parsePayloadEvidence);
if (
JSON.stringify(parsedPayload.map((entry) => entry.name)) !==
JSON.stringify(PAYLOAD_GROUPS.map((entry) => entry.name))
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest payload order is invalid',
);
}
const { manifestDigest, ...payload } =
manifest as LocalDataDirectoryAdoptionManifest;
if (sha256Text(JSON.stringify(payload)) !== manifestDigest) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest digest does not match',
);
}
return manifest as LocalDataDirectoryAdoptionManifest;
}
function readManifest(
stagingRoot: string,
uid: number,
): Readonly<LocalDataDirectoryAdoptionManifest> {
const manifestPath = path.join(stagingRoot, MANIFEST_NAME);
const stat = fs.lstatSync(manifestPath, { bigint: true });
if (
!stat.isFile() ||
stat.isSymbolicLink() ||
stat.nlink !== 1n ||
stat.uid !== BigInt(uid) ||
(stat.mode & 0o777n) !== 0o600n ||
stat.size < 1n ||
stat.size > BigInt(MAX_MANIFEST_BYTES)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest identity, mode, or size is invalid',
);
}
const descriptor = fs.openSync(
manifestPath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
try {
const before = fs.fstatSync(descriptor, { bigint: true });
if (!sameStat(stat, before)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest identity changed before reading',
);
}
const content = fs.readFileSync(descriptor, 'utf8');
if (!sameStat(before, fs.fstatSync(descriptor, { bigint: true }))) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest changed while reading',
);
}
return parseManifest(JSON.parse(content));
} catch (error) {
if (error instanceof LocalDataDirectoryAdoptionConfigurationError) {
throw error;
}
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest JSON is invalid',
error,
);
} finally {
fs.closeSync(descriptor);
}
}
function assertCompleteRoot(stagingRoot: string): void {
if (
JSON.stringify(sortedNames(stagingRoot)) !==
JSON.stringify([MANIFEST_NAME, 'payload'].sort())
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging root is incomplete or contains unexpected entries',
);
}
}
export function verifyStaticStage(
authority: Readonly<RootAuthority>,
expectedManifestDigest: string,
): Readonly<LocalDataDirectoryAdoptionManifest> {
const rootBefore = assertPrivateDirectory(
authority.stagingRoot,
authority.uid,
'stagingRoot',
);
assertCompleteRoot(authority.stagingRoot);
const manifest = readManifest(authority.stagingRoot, authority.uid);
if (manifest.manifestDigest !== expectedManifestDigest) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest no longer matches the reviewed digest',
);
}
if (
manifest.dataRootPathDigest !== sha256Text(authority.dataRoot) ||
manifest.stagingRootPathDigest !== sha256Text(authority.stagingRoot)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest path binding is invalid',
);
}
const actualPayload = inspectPayload(authority.stagingRoot, authority.uid);
if (JSON.stringify(actualPayload) !== JSON.stringify(manifest.payload)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload no longer matches the manifest',
);
}
if (
!sameStat(rootBefore, fs.lstatSync(authority.stagingRoot, { bigint: true }))
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging root changed during verification',
);
}
return manifest;
}
@@ -0,0 +1,319 @@
import fs from 'node:fs';
import path from 'node:path';
import {
acquireLocalSqliteActivation,
type LocalSqliteActivationFence,
} from '@qinglong/local-admin/runtime';
import {
LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION,
LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION,
LocalDataDirectoryAdoptionConfigurationError,
type LocalDataDirectoryAdoptionSqliteBinding,
type StageLocalDataDirectoryAdoptionCommand,
type VerifyLocalDataDirectoryAdoptionCommand,
} from './contract';
import {
copyCategory,
rootAuthority,
syncDirectory,
writeExclusiveJson,
type MutableCopyBudget,
} from './filesystem';
import {
MANIFEST_NAME,
PAYLOAD_GROUPS,
inspectPayload,
sha256Text,
verifyStaticStage,
type LocalDataDirectoryAdoptionManifest,
type LocalDataDirectoryAdoptionManifestPayload,
type LocalDataDirectoryPayloadEvidence,
} from './manifest';
import {
inspectLocalDataDirectoryAdoption,
type LocalDataDirectoryAdoptionEvidence,
} from './inventory';
const INCOMPLETE_NAME = '.incomplete';
export type {
LocalDataDirectoryAdoptionManifest,
LocalDataDirectoryPayloadEvidence,
} from './manifest';
export interface LocalDataDirectoryAdoptionMutationResult {
readonly schemaVersion: 1;
readonly operation:
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION;
readonly status: 'staged' | 'verified';
readonly evidence: Readonly<{
profile: 'edge' | 'standalone';
createdAtMs: number;
planDigest: string;
manifestDigest: string;
sqliteActivationDigest: string;
sqliteAdoptionManifestDigest: string;
payload: readonly LocalDataDirectoryPayloadEvidence[];
}>;
}
function inspectPlan(
dataRoot: string,
profile: 'edge' | 'standalone',
): Readonly<LocalDataDirectoryAdoptionEvidence> {
return inspectLocalDataDirectoryAdoption({
schemaVersion: 1,
operation: 'local-data-directory.adoption.inspect',
options: { dataRoot, profile },
}).evidence;
}
function assertReviewablePlan(
plan: Readonly<LocalDataDirectoryAdoptionEvidence>,
expectedPlanDigest: string,
): void {
const database = plan.categories.find((category) => category.name === 'db');
if (
plan.planDigest !== expectedPlanDigest ||
plan.assessment !== 'reviewable' ||
plan.totalUnsafeEntries !== 0 ||
plan.unknownTopLevelEntries !== 0 ||
!database ||
database.primaryDatabaseFiles !== 1 ||
plan.categories.some((category) => category.activeSqliteSidecars !== 0)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'data directory no longer matches a reviewable migration plan',
);
}
}
async function acquireSqliteFence(
binding: Readonly<LocalDataDirectoryAdoptionSqliteBinding>,
profile: 'edge' | 'standalone',
): Promise<Readonly<LocalSqliteActivationFence>> {
const fence = await acquireLocalSqliteActivation({
sourcePath: binding.sourcePath,
targetPath: binding.targetPath,
recoveryPath: binding.recoveryPath,
manifestPath: binding.manifestPath,
activationPath: binding.activationPath,
expectedActivationDigest: binding.expectedActivationDigest,
});
if (fence.activation.profile !== profile) {
await fence.release();
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation profile does not match directory adoption',
);
}
return fence;
}
function manifestPayload(
command: Readonly<StageLocalDataDirectoryAdoptionCommand>,
plan: Readonly<LocalDataDirectoryAdoptionEvidence>,
fence: Readonly<LocalSqliteActivationFence>,
payload: readonly LocalDataDirectoryPayloadEvidence[],
): Readonly<LocalDataDirectoryAdoptionManifestPayload> {
const createdAtMs = Date.now();
if (!Number.isSafeInteger(createdAtMs) || createdAtMs < 0) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'system clock returned an invalid timestamp',
);
}
return Object.freeze({
schemaVersion: 1,
kind: 'qinglong3-legacy-data-directory-adoption',
state: 'staged',
profile: command.options.profile,
createdAtMs,
planDigest: plan.planDigest,
sqliteActivationDigest: fence.activation.activationDigest,
sqliteAdoptionManifestDigest: fence.adoption.manifestDigest,
dataRootPathDigest: sha256Text(command.options.dataRoot),
stagingRootPathDigest: sha256Text(command.options.stagingRoot),
payload,
});
}
function result(
operation:
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION,
status: 'staged' | 'verified',
manifest: Readonly<LocalDataDirectoryAdoptionManifest>,
): Readonly<LocalDataDirectoryAdoptionMutationResult> {
return Object.freeze({
schemaVersion: 1,
operation,
status,
evidence: Object.freeze({
profile: manifest.profile,
createdAtMs: manifest.createdAtMs,
planDigest: manifest.planDigest,
manifestDigest: manifest.manifestDigest,
sqliteActivationDigest: manifest.sqliteActivationDigest,
sqliteAdoptionManifestDigest: manifest.sqliteAdoptionManifestDigest,
payload: manifest.payload,
}),
});
}
export async function stageLocalDataDirectoryAdoption(
command: Readonly<StageLocalDataDirectoryAdoptionCommand>,
): Promise<Readonly<LocalDataDirectoryAdoptionMutationResult>> {
try {
const authority = rootAuthority(command.options, true);
const before = inspectPlan(
command.options.dataRoot,
command.options.profile,
);
assertReviewablePlan(before, command.options.expectedPlanDigest);
const fence = await acquireSqliteFence(
command.options.sqlite,
command.options.profile,
);
let payload: readonly LocalDataDirectoryPayloadEvidence[];
try {
fs.mkdirSync(command.options.stagingRoot, { mode: 0o700 });
writeExclusiveJson(
path.join(command.options.stagingRoot, INCOMPLETE_NAME),
{
schemaVersion: 1,
kind: 'qinglong3-legacy-data-directory-adoption-incomplete',
},
);
syncDirectory(command.options.stagingRoot);
syncDirectory(path.dirname(command.options.stagingRoot));
const payloadRoot = path.join(command.options.stagingRoot, 'payload');
fs.mkdirSync(payloadRoot, { mode: 0o700 });
const copyBudget: MutableCopyBudget = { entries: 0, bytes: 0 };
for (const group of PAYLOAD_GROUPS) {
const groupRoot = path.join(payloadRoot, group.directoryName);
fs.mkdirSync(groupRoot, { mode: 0o700 });
for (const category of group.categories) {
copyCategory(
command.options.dataRoot,
groupRoot,
category,
authority.uid,
before.budget,
copyBudget,
);
}
syncDirectory(groupRoot);
}
syncDirectory(payloadRoot);
fence.assertTargetIdentity();
payload = inspectPayload(command.options.stagingRoot, authority.uid);
} finally {
await fence.release();
}
const after = inspectPlan(
command.options.dataRoot,
command.options.profile,
);
assertReviewablePlan(after, command.options.expectedPlanDigest);
if (JSON.stringify(after) !== JSON.stringify(before)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'data directory changed during staging',
);
}
const manifestBase = manifestPayload(command, after, fence, payload);
const manifest = Object.freeze({
...manifestBase,
manifestDigest: sha256Text(JSON.stringify(manifestBase)),
});
writeExclusiveJson(
path.join(command.options.stagingRoot, MANIFEST_NAME),
manifest,
);
syncDirectory(command.options.stagingRoot);
fs.unlinkSync(path.join(command.options.stagingRoot, INCOMPLETE_NAME));
syncDirectory(command.options.stagingRoot);
const verified = verifyStaticStage(authority, manifest.manifestDigest);
return result(
LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION,
'staged',
verified,
);
} catch (error) {
if (error instanceof LocalDataDirectoryAdoptionConfigurationError) {
throw error;
}
throw new LocalDataDirectoryAdoptionConfigurationError(
'data directory staging failed',
error,
);
}
}
export async function verifyLocalDataDirectoryAdoption(
command: Readonly<VerifyLocalDataDirectoryAdoptionCommand>,
): Promise<Readonly<LocalDataDirectoryAdoptionMutationResult>> {
try {
const authority = rootAuthority(command.options, false);
const manifest = verifyStaticStage(
authority,
command.options.expectedManifestDigest,
);
if (
manifest.profile !== command.options.profile ||
manifest.sqliteActivationDigest !==
command.options.sqlite.expectedActivationDigest
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest authority binding is invalid',
);
}
const before = inspectPlan(
command.options.dataRoot,
command.options.profile,
);
assertReviewablePlan(before, manifest.planDigest);
const fence = await acquireSqliteFence(
command.options.sqlite,
command.options.profile,
);
try {
if (
fence.activation.activationDigest !== manifest.sqliteActivationDigest ||
fence.adoption.manifestDigest !== manifest.sqliteAdoptionManifestDigest
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation no longer matches the staging manifest',
);
}
verifyStaticStage(authority, command.options.expectedManifestDigest);
fence.assertTargetIdentity();
} finally {
await fence.release();
}
const after = inspectPlan(
command.options.dataRoot,
command.options.profile,
);
assertReviewablePlan(after, manifest.planDigest);
if (JSON.stringify(after) !== JSON.stringify(before)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'data directory changed during verification',
);
}
return result(
LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION,
'verified',
manifest,
);
} catch (error) {
if (error instanceof LocalDataDirectoryAdoptionConfigurationError) {
throw error;
}
throw new LocalDataDirectoryAdoptionConfigurationError(
'data directory verification failed',
error,
);
}
}