feat(ql3): stage legacy data directory

This commit is contained in:
whyour
2026-08-21 10:09:34 +08:00
parent 878a360b09
commit 19bb09faa3
14 changed files with 2139 additions and 26 deletions
@@ -2,7 +2,7 @@ import { readPrivateLocalCommandFile } from '@qinglong/local-command-file';
import { runLegacyCrontabAdoptionCommandFile } from './adoption';
import { isLocalDataDirectoryAdoptionOperation } from './data-directory-adoption/contract';
import type { LocalDataDirectoryAdoptionInspectResult } from './data-directory-adoption/inventory';
import type { LocalDataDirectoryAdoptionProductCommandResult } from './data-directory-adoption/command';
import {
isLocalSqliteAdoptionProductOperation,
type LocalSqliteAdoptionProductOperation,
@@ -12,7 +12,7 @@ import type { LocalSqliteAdoptionProductCommandResult } from './sqlite-adoption/
export type LocalAdoptionProductCommandResult =
| Awaited<ReturnType<typeof runLegacyCrontabAdoptionCommandFile>>
| LocalSqliteAdoptionProductCommandResult
| LocalDataDirectoryAdoptionInspectResult;
| LocalDataDirectoryAdoptionProductCommandResult;
function operation(value: unknown): unknown {
return value && typeof value === 'object' && !Array.isArray(value)
@@ -32,10 +32,10 @@ export async function runLocalAdoptionProductCommandFile(
}
const selected = operation(candidate);
if (isLocalDataDirectoryAdoptionOperation(selected)) {
const { inspectLocalDataDirectoryAdoption } = await import(
'./data-directory-adoption/inventory.js'
const { runLocalDataDirectoryAdoptionProductCommand } = await import(
'./data-directory-adoption/command.js'
);
return inspectLocalDataDirectoryAdoption(candidate);
return runLocalDataDirectoryAdoptionProductCommand(candidate);
}
if (!isLocalSqliteAdoptionProductOperation(selected)) {
return runLegacyCrontabAdoptionCommandFile(commandFilePath);
@@ -0,0 +1,31 @@
import {
LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION,
LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION,
normalizeLocalDataDirectoryAdoptionCommand,
} from './contract';
import {
inspectLocalDataDirectoryAdoption,
type LocalDataDirectoryAdoptionInspectResult,
} from './inventory';
import {
stageLocalDataDirectoryAdoption,
verifyLocalDataDirectoryAdoption,
type LocalDataDirectoryAdoptionMutationResult,
} from './staging';
export type LocalDataDirectoryAdoptionProductCommandResult =
| LocalDataDirectoryAdoptionInspectResult
| LocalDataDirectoryAdoptionMutationResult;
export async function runLocalDataDirectoryAdoptionProductCommand(
value: unknown,
): Promise<Readonly<LocalDataDirectoryAdoptionProductCommandResult>> {
const command = normalizeLocalDataDirectoryAdoptionCommand(value);
if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION) {
return inspectLocalDataDirectoryAdoption(command);
}
if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION) {
return stageLocalDataDirectoryAdoption(command);
}
return verifyLocalDataDirectoryAdoption(command);
}
@@ -1,9 +1,19 @@
import path from 'node:path';
const MAX_PATH_BYTES = 4_096;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
export const LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION =
'local-data-directory.adoption.inspect' as const;
export const LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION =
'local-data-directory.adoption.stage' as const;
export const LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION =
'local-data-directory.adoption.verify' as const;
export type LocalDataDirectoryAdoptionOperation =
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION;
export interface InspectLocalDataDirectoryAdoptionCommand {
readonly schemaVersion: 1;
@@ -14,6 +24,44 @@ export interface InspectLocalDataDirectoryAdoptionCommand {
};
}
export interface LocalDataDirectoryAdoptionSqliteBinding {
readonly sourcePath: string;
readonly targetPath: string;
readonly recoveryPath: string;
readonly manifestPath: string;
readonly activationPath: string;
readonly expectedActivationDigest: string;
}
interface LocalDataDirectoryAdoptionMutationOptions {
readonly deploymentRoot: string;
readonly dataRoot: string;
readonly stagingRoot: string;
readonly profile: 'edge' | 'standalone';
readonly sqlite: LocalDataDirectoryAdoptionSqliteBinding;
}
export interface StageLocalDataDirectoryAdoptionCommand {
readonly schemaVersion: 1;
readonly operation: typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION;
readonly options: LocalDataDirectoryAdoptionMutationOptions & {
readonly expectedPlanDigest: string;
};
}
export interface VerifyLocalDataDirectoryAdoptionCommand {
readonly schemaVersion: 1;
readonly operation: typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION;
readonly options: LocalDataDirectoryAdoptionMutationOptions & {
readonly expectedManifestDigest: string;
};
}
export type LocalDataDirectoryAdoptionCommand =
| InspectLocalDataDirectoryAdoptionCommand
| StageLocalDataDirectoryAdoptionCommand
| VerifyLocalDataDirectoryAdoptionCommand;
export class LocalDataDirectoryAdoptionConfigurationError extends TypeError {
readonly code = 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID';
@@ -45,13 +93,77 @@ function normalizedAbsolutePath(value: unknown): value is string {
export function isLocalDataDirectoryAdoptionOperation(
value: unknown,
): value is typeof LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION {
return value === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION;
): value is LocalDataDirectoryAdoptionOperation {
return (
value === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION ||
value === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION ||
value === LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION
);
}
export function normalizeInspectLocalDataDirectoryAdoptionCommand(
function normalizeSqliteBinding(
value: unknown,
): Readonly<InspectLocalDataDirectoryAdoptionCommand> {
): Readonly<LocalDataDirectoryAdoptionSqliteBinding> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!exactKeys(value, [
'activationPath',
'expectedActivationDigest',
'manifestPath',
'recoveryPath',
'sourcePath',
'targetPath',
])
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation binding shape is invalid',
);
}
const binding = value as Record<string, unknown>;
for (const key of [
'activationPath',
'manifestPath',
'recoveryPath',
'sourcePath',
'targetPath',
]) {
if (!normalizedAbsolutePath(binding[key])) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation binding path is invalid',
);
}
}
if (
typeof binding.expectedActivationDigest !== 'string' ||
!DIGEST_PATTERN.test(binding.expectedActivationDigest)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation digest is invalid',
);
}
if (
new Set(
[
binding.activationPath,
binding.manifestPath,
binding.recoveryPath,
binding.sourcePath,
binding.targetPath,
].map((candidate) => path.resolve(candidate as string)),
).size !== 5
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation paths must be distinct',
);
}
return Object.freeze(value as LocalDataDirectoryAdoptionSqliteBinding);
}
export function normalizeLocalDataDirectoryAdoptionCommand(
value: unknown,
): Readonly<LocalDataDirectoryAdoptionCommand> {
if (
!value ||
typeof value !== 'object' ||
@@ -68,15 +180,35 @@ export function normalizeInspectLocalDataDirectoryAdoptionCommand(
!isLocalDataDirectoryAdoptionOperation(candidate.operation) ||
!candidate.options ||
typeof candidate.options !== 'object' ||
Array.isArray(candidate.options) ||
!exactKeys(candidate.options, ['dataRoot', 'profile'])
Array.isArray(candidate.options)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'command value is invalid',
);
}
const options = candidate.options as Record<string, unknown>;
const expectedKeys =
candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION
? ['dataRoot', 'profile']
: candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
? [
'dataRoot',
'deploymentRoot',
'expectedPlanDigest',
'profile',
'sqlite',
'stagingRoot',
]
: [
'dataRoot',
'deploymentRoot',
'expectedManifestDigest',
'profile',
'sqlite',
'stagingRoot',
];
if (
!exactKeys(options, expectedKeys) ||
!normalizedAbsolutePath(options.dataRoot) ||
(options.profile !== 'edge' && options.profile !== 'standalone')
) {
@@ -84,5 +216,37 @@ export function normalizeInspectLocalDataDirectoryAdoptionCommand(
'command options are invalid',
);
}
return Object.freeze(value as InspectLocalDataDirectoryAdoptionCommand);
if (candidate.operation !== LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION) {
if (
!normalizedAbsolutePath(options.deploymentRoot) ||
!normalizedAbsolutePath(options.stagingRoot)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'adoption root path is invalid',
);
}
normalizeSqliteBinding(options.sqlite);
const digest =
candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
? options.expectedPlanDigest
: options.expectedManifestDigest;
if (typeof digest !== 'string' || !DIGEST_PATTERN.test(digest)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'reviewed adoption digest is invalid',
);
}
}
return Object.freeze(value as LocalDataDirectoryAdoptionCommand);
}
export function normalizeInspectLocalDataDirectoryAdoptionCommand(
value: unknown,
): Readonly<InspectLocalDataDirectoryAdoptionCommand> {
const command = normalizeLocalDataDirectoryAdoptionCommand(value);
if (command.operation !== LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'inspection operation is invalid',
);
}
return command;
}
@@ -0,0 +1,430 @@
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import {
LocalDataDirectoryAdoptionConfigurationError,
type StageLocalDataDirectoryAdoptionCommand,
type VerifyLocalDataDirectoryAdoptionCommand,
} from './contract';
const HASH_BUFFER_BYTES = 64 * 1024;
const MAX_RELATIVE_PATH_BYTES = 4_096;
export interface RootAuthority {
readonly uid: number;
readonly deploymentRoot: string;
readonly dataRoot: string;
readonly stagingRoot: string;
}
export interface CopyBudget {
readonly maxEntries: number;
readonly maxHashedBytes: number;
readonly maxFileBytes: number;
readonly maxDepth: number;
}
export interface MutableCopyBudget {
entries: number;
bytes: number;
}
function currentUid(): number {
if (
typeof process.getuid !== 'function' ||
typeof process.geteuid !== 'function' ||
process.getuid() !== process.geteuid()
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'real and effective POSIX users must match',
);
}
return process.getuid();
}
function inside(root: string, candidate: string): boolean {
const relative = path.relative(root, candidate);
return (
relative !== '' &&
relative !== '..' &&
!relative.startsWith(`..${path.sep}`) &&
!path.isAbsolute(relative)
);
}
export function assertPrivateDirectory(
directoryPath: string,
uid: number,
label: string,
): fs.BigIntStats {
let stat: fs.BigIntStats;
try {
stat = fs.lstatSync(directoryPath, { bigint: true });
} catch (error) {
throw new LocalDataDirectoryAdoptionConfigurationError(
`${label} is unavailable`,
error,
);
}
if (
!stat.isDirectory() ||
stat.isSymbolicLink() ||
stat.uid !== BigInt(uid) ||
(stat.mode & 0o777n) !== 0o700n ||
fs.realpathSync(directoryPath) !== directoryPath
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
`${label} must be an owner-controlled 0700 canonical directory`,
);
}
return stat;
}
function assertMissing(candidate: string, label: string): void {
try {
fs.lstatSync(candidate);
} catch (error) {
if (
error &&
typeof error === 'object' &&
'code' in error &&
error.code === 'ENOENT'
) {
return;
}
throw new LocalDataDirectoryAdoptionConfigurationError(
`${label} cannot be inspected`,
error,
);
}
throw new LocalDataDirectoryAdoptionConfigurationError(
`${label} must not already exist`,
);
}
export function rootAuthority(
options:
| StageLocalDataDirectoryAdoptionCommand['options']
| VerifyLocalDataDirectoryAdoptionCommand['options'],
requireMissing: boolean,
): Readonly<RootAuthority> {
const uid = currentUid();
assertPrivateDirectory(options.deploymentRoot, uid, 'deploymentRoot');
if (
!inside(options.deploymentRoot, options.stagingRoot) ||
options.dataRoot === options.stagingRoot ||
inside(options.dataRoot, options.stagingRoot) ||
inside(options.stagingRoot, options.dataRoot)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'stagingRoot must be isolated inside deploymentRoot',
);
}
const stagingParent = path.dirname(options.stagingRoot);
if (
stagingParent !== options.deploymentRoot &&
!inside(options.deploymentRoot, stagingParent)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'stagingRoot parent must remain inside deploymentRoot',
);
}
assertPrivateDirectory(stagingParent, uid, 'stagingRoot parent');
const expectedSource = path.join(options.dataRoot, 'db', 'database.sqlite');
if (options.sqlite.sourcePath !== expectedSource) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation source must be the reviewed primary database',
);
}
for (const candidate of [
options.sqlite.targetPath,
options.sqlite.recoveryPath,
options.sqlite.manifestPath,
options.sqlite.activationPath,
]) {
if (candidate === options.dataRoot || inside(options.dataRoot, candidate)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite adoption evidence must remain outside dataRoot',
);
}
}
if (requireMissing) assertMissing(options.stagingRoot, 'stagingRoot');
else assertPrivateDirectory(options.stagingRoot, uid, 'stagingRoot');
return Object.freeze({
uid,
deploymentRoot: options.deploymentRoot,
dataRoot: options.dataRoot,
stagingRoot: options.stagingRoot,
});
}
export function sameStat(left: fs.BigIntStats, right: fs.BigIntStats): boolean {
return (
left.dev === right.dev &&
left.ino === right.ino &&
left.mode === right.mode &&
left.nlink === right.nlink &&
left.uid === right.uid &&
left.size === right.size &&
left.mtimeNs === right.mtimeNs &&
left.ctimeNs === right.ctimeNs
);
}
export function sortedNames(directoryPath: string): readonly string[] {
return fs
.readdirSync(directoryPath)
.sort((left, right) =>
Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')),
);
}
function assertRelativePath(value: string): void {
if (
value.length < 1 ||
path.isAbsolute(value) ||
value === '..' ||
value.startsWith(`..${path.sep}`) ||
Buffer.byteLength(value, 'utf8') > MAX_RELATIVE_PATH_BYTES
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'payload relative path is invalid or too long',
);
}
}
export function syncDirectory(directoryPath: string): void {
const descriptor = fs.openSync(directoryPath, fs.constants.O_RDONLY);
try {
fs.fsyncSync(descriptor);
} finally {
fs.closeSync(descriptor);
}
}
export function writeExclusiveJson(filePath: string, value: object): void {
const descriptor = fs.openSync(filePath, 'wx', 0o600);
try {
const bytes = Buffer.from(`${JSON.stringify(value)}\n`, 'utf8');
let offset = 0;
while (offset < bytes.length) {
offset += fs.writeSync(
descriptor,
bytes,
offset,
bytes.length - offset,
null,
);
}
fs.fsyncSync(descriptor);
} finally {
fs.closeSync(descriptor);
}
}
function copyStableFile(
sourcePath: string,
destinationPath: string,
expected: fs.BigIntStats,
): void {
const source = fs.openSync(
sourcePath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
let destination: number | undefined;
const buffer = Buffer.allocUnsafe(HASH_BUFFER_BYTES);
try {
const before = fs.fstatSync(source, { bigint: true });
if (!before.isFile() || !sameStat(expected, before)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source file identity changed before staging',
);
}
destination = fs.openSync(destinationPath, 'wx', 0o600);
for (;;) {
const count = fs.readSync(source, buffer, 0, buffer.length, null);
if (count === 0) break;
let offset = 0;
while (offset < count) {
offset += fs.writeSync(
destination,
buffer,
offset,
count - offset,
null,
);
}
}
fs.fsyncSync(destination);
if (!sameStat(before, fs.fstatSync(source, { bigint: true }))) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source file changed during staging',
);
}
} finally {
buffer.fill(0);
if (destination !== undefined) fs.closeSync(destination);
fs.closeSync(source);
}
}
function shouldExcludeDatabaseEntry(
category: string,
relative: string,
): boolean {
return (
category === 'db' &&
/^(?:database\.sqlite|database\.sqlite-(?:wal|shm|journal))$/.test(relative)
);
}
export function copyCategory(
sourceRoot: string,
destinationRoot: string,
category: string,
uid: number,
limits: Readonly<CopyBudget>,
shared: MutableCopyBudget,
): void {
const sourceCategory = path.join(sourceRoot, category);
let categoryStat: fs.BigIntStats;
try {
categoryStat = fs.lstatSync(sourceCategory, { bigint: true });
} catch (error) {
if (
error &&
typeof error === 'object' &&
'code' in error &&
error.code === 'ENOENT'
) {
return;
}
throw error;
}
if (
!categoryStat.isDirectory() ||
categoryStat.isSymbolicLink() ||
categoryStat.uid !== BigInt(uid) ||
(categoryStat.mode & 0o022n) !== 0n
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source category identity is unsafe',
);
}
const destinationCategory = path.join(destinationRoot, category);
fs.mkdirSync(destinationCategory, { mode: 0o700 });
const visit = (
sourceDirectory: string,
destinationDirectory: string,
expectedDirectory: fs.BigIntStats,
depth: number,
): void => {
if (depth > limits.maxDepth) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload depth exceeds the Profile budget',
);
}
for (const name of sortedNames(sourceDirectory)) {
const sourceEntry = path.join(sourceDirectory, name);
const categoryRelative = path.relative(sourceCategory, sourceEntry);
assertRelativePath(categoryRelative);
if (shouldExcludeDatabaseEntry(category, categoryRelative)) continue;
shared.entries += 1;
if (shared.entries > limits.maxEntries) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload entry count exceeds the Profile budget',
);
}
const destinationEntry = path.join(destinationDirectory, name);
const stat = fs.lstatSync(sourceEntry, { bigint: true });
if (
stat.isSymbolicLink() ||
stat.uid !== BigInt(uid) ||
(stat.mode & 0o022n) !== 0n
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload entry identity is unsafe',
);
}
if (stat.isDirectory()) {
fs.mkdirSync(destinationEntry, { mode: 0o700 });
visit(sourceEntry, destinationEntry, stat, depth + 1);
syncDirectory(destinationEntry);
} else if (stat.isFile() && stat.nlink === 1n) {
if (
stat.size < 0n ||
stat.size > BigInt(limits.maxFileBytes) ||
stat.size > BigInt(Number.MAX_SAFE_INTEGER)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload file exceeds the Profile budget',
);
}
const bytes = Number(stat.size);
if (
!Number.isSafeInteger(shared.bytes + bytes) ||
shared.bytes + bytes > limits.maxHashedBytes
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload bytes exceed the Profile budget',
);
}
shared.bytes += bytes;
copyStableFile(sourceEntry, destinationEntry, stat);
} else {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source payload entry kind is unsafe',
);
}
}
if (
!sameStat(
expectedDirectory,
fs.lstatSync(sourceDirectory, { bigint: true }),
)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'source directory changed during staging',
);
}
};
visit(sourceCategory, destinationCategory, categoryStat, 1);
syncDirectory(destinationCategory);
}
export function stableFileDigest(
filePath: string,
expected: fs.BigIntStats,
): string {
const descriptor = fs.openSync(
filePath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
const buffer = Buffer.allocUnsafe(HASH_BUFFER_BYTES);
try {
const before = fs.fstatSync(descriptor, { bigint: true });
if (!before.isFile() || !sameStat(expected, before)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged file identity changed before verification',
);
}
const hash = crypto.createHash('sha256');
for (;;) {
const count = fs.readSync(descriptor, buffer, 0, buffer.length, null);
if (count === 0) break;
hash.update(buffer.subarray(0, count));
}
if (!sameStat(before, fs.fstatSync(descriptor, { bigint: true }))) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged file changed during verification',
);
}
return hash.digest('hex');
} finally {
buffer.fill(0);
fs.closeSync(descriptor);
}
}
@@ -0,0 +1,437 @@
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { LocalDataDirectoryAdoptionConfigurationError } from './contract';
import {
assertPrivateDirectory,
sameStat,
sortedNames,
stableFileDigest,
type RootAuthority,
} from './filesystem';
const MAX_MANIFEST_BYTES = 64 * 1024;
const MAX_RELATIVE_PATH_BYTES = 4_096;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
export const MANIFEST_NAME = 'manifest.json';
export const PAYLOAD_GROUPS = Object.freeze([
Object.freeze({
name: 'copy_reviewed' as const,
directoryName: 'copy-reviewed',
categories: Object.freeze(['scripts', 'upload'] as const),
}),
Object.freeze({
name: 'transform_input' as const,
directoryName: 'transform-input',
categories: Object.freeze(['config', 'db', 'ssh.d'] as const),
}),
]);
type PayloadGroupName = (typeof PAYLOAD_GROUPS)[number]['name'];
export interface LocalDataDirectoryPayloadEvidence {
readonly name: PayloadGroupName;
readonly categories: readonly string[];
readonly entries: number;
readonly directories: number;
readonly files: number;
readonly bytes: number;
readonly digest: string;
}
export interface LocalDataDirectoryAdoptionManifestPayload {
readonly schemaVersion: 1;
readonly kind: 'qinglong3-legacy-data-directory-adoption';
readonly state: 'staged';
readonly profile: 'edge' | 'standalone';
readonly createdAtMs: number;
readonly planDigest: string;
readonly sqliteActivationDigest: string;
readonly sqliteAdoptionManifestDigest: string;
readonly dataRootPathDigest: string;
readonly stagingRootPathDigest: string;
readonly payload: readonly LocalDataDirectoryPayloadEvidence[];
}
export interface LocalDataDirectoryAdoptionManifest
extends LocalDataDirectoryAdoptionManifestPayload {
readonly manifestDigest: string;
}
interface MutablePayloadSummary {
entries: number;
directories: number;
files: number;
bytes: number;
}
export function sha256Text(value: string): string {
return crypto.createHash('sha256').update(value, 'utf8').digest('hex');
}
function exactKeys(value: object, expected: readonly string[]): boolean {
const actual = Object.keys(value).sort();
const canonical = [...expected].sort();
return (
actual.length === canonical.length &&
actual.every((key, index) => key === canonical[index])
);
}
function assertRelativePath(value: string): void {
if (
value.length < 1 ||
path.isAbsolute(value) ||
value === '..' ||
value.startsWith(`..${path.sep}`) ||
Buffer.byteLength(value, 'utf8') > MAX_RELATIVE_PATH_BYTES
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'payload relative path is invalid or too long',
);
}
}
function payloadEvidence(
groupRoot: string,
group: (typeof PAYLOAD_GROUPS)[number],
uid: number,
): Readonly<LocalDataDirectoryPayloadEvidence> {
assertPrivateDirectory(groupRoot, uid, 'payload group');
const allowed = new Set<string>(group.categories);
const summary: MutablePayloadSummary = {
entries: 0,
directories: 0,
files: 0,
bytes: 0,
};
const hash = crypto.createHash('sha256');
const visit = (directoryPath: string, expected: fs.BigIntStats): void => {
for (const name of sortedNames(directoryPath)) {
const entryPath = path.join(directoryPath, name);
const relative = path.relative(groupRoot, entryPath);
assertRelativePath(relative);
if (directoryPath === groupRoot && !allowed.has(name)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload contains an unexpected category',
);
}
const stat = fs.lstatSync(entryPath, { bigint: true });
summary.entries += 1;
if (
stat.isSymbolicLink() ||
stat.uid !== BigInt(uid) ||
(stat.mode & 0o777n) !== (stat.isDirectory() ? 0o700n : 0o600n)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload identity or mode is invalid',
);
}
const canonicalRelative = relative.split(path.sep).join('/');
if (stat.isDirectory()) {
summary.directories += 1;
hash.update(
`${JSON.stringify({
relative: canonicalRelative,
kind: 'directory',
})}\n`,
'utf8',
);
visit(entryPath, stat);
} else if (stat.isFile() && stat.nlink === 1n) {
if (stat.size < 0n || stat.size > BigInt(Number.MAX_SAFE_INTEGER)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload file size is unsupported',
);
}
const bytes = Number(stat.size);
if (!Number.isSafeInteger(summary.bytes + bytes)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload byte total is unsupported',
);
}
summary.files += 1;
summary.bytes += bytes;
hash.update(
`${JSON.stringify({
relative: canonicalRelative,
kind: 'file',
bytes,
contentDigest: stableFileDigest(entryPath, stat),
})}\n`,
'utf8',
);
} else {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload entry kind is invalid',
);
}
}
if (!sameStat(expected, fs.lstatSync(directoryPath, { bigint: true }))) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload directory changed during verification',
);
}
};
visit(groupRoot, fs.lstatSync(groupRoot, { bigint: true }));
return Object.freeze({
name: group.name,
categories: group.categories,
...summary,
digest: hash.digest('hex'),
});
}
export function inspectPayload(
stagingRoot: string,
uid: number,
): readonly LocalDataDirectoryPayloadEvidence[] {
const payloadRoot = path.join(stagingRoot, 'payload');
const payloadRootBefore = assertPrivateDirectory(
payloadRoot,
uid,
'payload root',
);
const expectedGroupNames = PAYLOAD_GROUPS.map((group) => group.directoryName);
if (
JSON.stringify(sortedNames(payloadRoot)) !==
JSON.stringify(
[...expectedGroupNames].sort((left, right) =>
Buffer.compare(Buffer.from(left), Buffer.from(right)),
),
)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'payload group set is invalid',
);
}
const evidence = Object.freeze(
PAYLOAD_GROUPS.map((group) =>
payloadEvidence(path.join(payloadRoot, group.directoryName), group, uid),
),
);
if (
!sameStat(payloadRootBefore, fs.lstatSync(payloadRoot, { bigint: true }))
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'payload root changed during verification',
);
}
return evidence;
}
function parsePayloadEvidence(
value: unknown,
): LocalDataDirectoryPayloadEvidence {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!exactKeys(value, [
'bytes',
'categories',
'digest',
'directories',
'entries',
'files',
'name',
])
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'manifest payload evidence shape is invalid',
);
}
const candidate = value as Partial<LocalDataDirectoryPayloadEvidence>;
const group = PAYLOAD_GROUPS.find((entry) => entry.name === candidate.name);
if (
!group ||
JSON.stringify(candidate.categories) !== JSON.stringify(group.categories) ||
!DIGEST_PATTERN.test(candidate.digest ?? '') ||
![
candidate.entries,
candidate.directories,
candidate.files,
candidate.bytes,
].every(
(number) => Number.isSafeInteger(number) && (number as number) >= 0,
) ||
candidate.entries !==
(candidate.directories as number) + (candidate.files as number)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'manifest payload evidence is invalid',
);
}
return value as LocalDataDirectoryPayloadEvidence;
}
function parseManifest(value: unknown): LocalDataDirectoryAdoptionManifest {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!exactKeys(value, [
'createdAtMs',
'dataRootPathDigest',
'kind',
'manifestDigest',
'payload',
'planDigest',
'profile',
'schemaVersion',
'sqliteActivationDigest',
'sqliteAdoptionManifestDigest',
'stagingRootPathDigest',
'state',
])
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest shape is invalid',
);
}
const manifest = value as Partial<LocalDataDirectoryAdoptionManifest>;
if (
manifest.schemaVersion !== 1 ||
manifest.kind !== 'qinglong3-legacy-data-directory-adoption' ||
manifest.state !== 'staged' ||
(manifest.profile !== 'edge' && manifest.profile !== 'standalone') ||
!Number.isSafeInteger(manifest.createdAtMs) ||
(manifest.createdAtMs as number) < 0 ||
![
manifest.manifestDigest,
manifest.planDigest,
manifest.sqliteActivationDigest,
manifest.sqliteAdoptionManifestDigest,
manifest.dataRootPathDigest,
manifest.stagingRootPathDigest,
].every(
(digest) => typeof digest === 'string' && DIGEST_PATTERN.test(digest),
) ||
!Array.isArray(manifest.payload) ||
manifest.payload.length !== PAYLOAD_GROUPS.length
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest value is invalid',
);
}
const parsedPayload = manifest.payload.map(parsePayloadEvidence);
if (
JSON.stringify(parsedPayload.map((entry) => entry.name)) !==
JSON.stringify(PAYLOAD_GROUPS.map((entry) => entry.name))
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest payload order is invalid',
);
}
const { manifestDigest, ...payload } =
manifest as LocalDataDirectoryAdoptionManifest;
if (sha256Text(JSON.stringify(payload)) !== manifestDigest) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest digest does not match',
);
}
return manifest as LocalDataDirectoryAdoptionManifest;
}
function readManifest(
stagingRoot: string,
uid: number,
): Readonly<LocalDataDirectoryAdoptionManifest> {
const manifestPath = path.join(stagingRoot, MANIFEST_NAME);
const stat = fs.lstatSync(manifestPath, { bigint: true });
if (
!stat.isFile() ||
stat.isSymbolicLink() ||
stat.nlink !== 1n ||
stat.uid !== BigInt(uid) ||
(stat.mode & 0o777n) !== 0o600n ||
stat.size < 1n ||
stat.size > BigInt(MAX_MANIFEST_BYTES)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest identity, mode, or size is invalid',
);
}
const descriptor = fs.openSync(
manifestPath,
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
);
try {
const before = fs.fstatSync(descriptor, { bigint: true });
if (!sameStat(stat, before)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest identity changed before reading',
);
}
const content = fs.readFileSync(descriptor, 'utf8');
if (!sameStat(before, fs.fstatSync(descriptor, { bigint: true }))) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest changed while reading',
);
}
return parseManifest(JSON.parse(content));
} catch (error) {
if (error instanceof LocalDataDirectoryAdoptionConfigurationError) {
throw error;
}
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest JSON is invalid',
error,
);
} finally {
fs.closeSync(descriptor);
}
}
function assertCompleteRoot(stagingRoot: string): void {
if (
JSON.stringify(sortedNames(stagingRoot)) !==
JSON.stringify([MANIFEST_NAME, 'payload'].sort())
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging root is incomplete or contains unexpected entries',
);
}
}
export function verifyStaticStage(
authority: Readonly<RootAuthority>,
expectedManifestDigest: string,
): Readonly<LocalDataDirectoryAdoptionManifest> {
const rootBefore = assertPrivateDirectory(
authority.stagingRoot,
authority.uid,
'stagingRoot',
);
assertCompleteRoot(authority.stagingRoot);
const manifest = readManifest(authority.stagingRoot, authority.uid);
if (manifest.manifestDigest !== expectedManifestDigest) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest no longer matches the reviewed digest',
);
}
if (
manifest.dataRootPathDigest !== sha256Text(authority.dataRoot) ||
manifest.stagingRootPathDigest !== sha256Text(authority.stagingRoot)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest path binding is invalid',
);
}
const actualPayload = inspectPayload(authority.stagingRoot, authority.uid);
if (JSON.stringify(actualPayload) !== JSON.stringify(manifest.payload)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staged payload no longer matches the manifest',
);
}
if (
!sameStat(rootBefore, fs.lstatSync(authority.stagingRoot, { bigint: true }))
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging root changed during verification',
);
}
return manifest;
}
@@ -0,0 +1,319 @@
import fs from 'node:fs';
import path from 'node:path';
import {
acquireLocalSqliteActivation,
type LocalSqliteActivationFence,
} from '@qinglong/local-admin/runtime';
import {
LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION,
LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION,
LocalDataDirectoryAdoptionConfigurationError,
type LocalDataDirectoryAdoptionSqliteBinding,
type StageLocalDataDirectoryAdoptionCommand,
type VerifyLocalDataDirectoryAdoptionCommand,
} from './contract';
import {
copyCategory,
rootAuthority,
syncDirectory,
writeExclusiveJson,
type MutableCopyBudget,
} from './filesystem';
import {
MANIFEST_NAME,
PAYLOAD_GROUPS,
inspectPayload,
sha256Text,
verifyStaticStage,
type LocalDataDirectoryAdoptionManifest,
type LocalDataDirectoryAdoptionManifestPayload,
type LocalDataDirectoryPayloadEvidence,
} from './manifest';
import {
inspectLocalDataDirectoryAdoption,
type LocalDataDirectoryAdoptionEvidence,
} from './inventory';
const INCOMPLETE_NAME = '.incomplete';
export type {
LocalDataDirectoryAdoptionManifest,
LocalDataDirectoryPayloadEvidence,
} from './manifest';
export interface LocalDataDirectoryAdoptionMutationResult {
readonly schemaVersion: 1;
readonly operation:
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION;
readonly status: 'staged' | 'verified';
readonly evidence: Readonly<{
profile: 'edge' | 'standalone';
createdAtMs: number;
planDigest: string;
manifestDigest: string;
sqliteActivationDigest: string;
sqliteAdoptionManifestDigest: string;
payload: readonly LocalDataDirectoryPayloadEvidence[];
}>;
}
function inspectPlan(
dataRoot: string,
profile: 'edge' | 'standalone',
): Readonly<LocalDataDirectoryAdoptionEvidence> {
return inspectLocalDataDirectoryAdoption({
schemaVersion: 1,
operation: 'local-data-directory.adoption.inspect',
options: { dataRoot, profile },
}).evidence;
}
function assertReviewablePlan(
plan: Readonly<LocalDataDirectoryAdoptionEvidence>,
expectedPlanDigest: string,
): void {
const database = plan.categories.find((category) => category.name === 'db');
if (
plan.planDigest !== expectedPlanDigest ||
plan.assessment !== 'reviewable' ||
plan.totalUnsafeEntries !== 0 ||
plan.unknownTopLevelEntries !== 0 ||
!database ||
database.primaryDatabaseFiles !== 1 ||
plan.categories.some((category) => category.activeSqliteSidecars !== 0)
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'data directory no longer matches a reviewable migration plan',
);
}
}
async function acquireSqliteFence(
binding: Readonly<LocalDataDirectoryAdoptionSqliteBinding>,
profile: 'edge' | 'standalone',
): Promise<Readonly<LocalSqliteActivationFence>> {
const fence = await acquireLocalSqliteActivation({
sourcePath: binding.sourcePath,
targetPath: binding.targetPath,
recoveryPath: binding.recoveryPath,
manifestPath: binding.manifestPath,
activationPath: binding.activationPath,
expectedActivationDigest: binding.expectedActivationDigest,
});
if (fence.activation.profile !== profile) {
await fence.release();
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation profile does not match directory adoption',
);
}
return fence;
}
function manifestPayload(
command: Readonly<StageLocalDataDirectoryAdoptionCommand>,
plan: Readonly<LocalDataDirectoryAdoptionEvidence>,
fence: Readonly<LocalSqliteActivationFence>,
payload: readonly LocalDataDirectoryPayloadEvidence[],
): Readonly<LocalDataDirectoryAdoptionManifestPayload> {
const createdAtMs = Date.now();
if (!Number.isSafeInteger(createdAtMs) || createdAtMs < 0) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'system clock returned an invalid timestamp',
);
}
return Object.freeze({
schemaVersion: 1,
kind: 'qinglong3-legacy-data-directory-adoption',
state: 'staged',
profile: command.options.profile,
createdAtMs,
planDigest: plan.planDigest,
sqliteActivationDigest: fence.activation.activationDigest,
sqliteAdoptionManifestDigest: fence.adoption.manifestDigest,
dataRootPathDigest: sha256Text(command.options.dataRoot),
stagingRootPathDigest: sha256Text(command.options.stagingRoot),
payload,
});
}
function result(
operation:
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION,
status: 'staged' | 'verified',
manifest: Readonly<LocalDataDirectoryAdoptionManifest>,
): Readonly<LocalDataDirectoryAdoptionMutationResult> {
return Object.freeze({
schemaVersion: 1,
operation,
status,
evidence: Object.freeze({
profile: manifest.profile,
createdAtMs: manifest.createdAtMs,
planDigest: manifest.planDigest,
manifestDigest: manifest.manifestDigest,
sqliteActivationDigest: manifest.sqliteActivationDigest,
sqliteAdoptionManifestDigest: manifest.sqliteAdoptionManifestDigest,
payload: manifest.payload,
}),
});
}
export async function stageLocalDataDirectoryAdoption(
command: Readonly<StageLocalDataDirectoryAdoptionCommand>,
): Promise<Readonly<LocalDataDirectoryAdoptionMutationResult>> {
try {
const authority = rootAuthority(command.options, true);
const before = inspectPlan(
command.options.dataRoot,
command.options.profile,
);
assertReviewablePlan(before, command.options.expectedPlanDigest);
const fence = await acquireSqliteFence(
command.options.sqlite,
command.options.profile,
);
let payload: readonly LocalDataDirectoryPayloadEvidence[];
try {
fs.mkdirSync(command.options.stagingRoot, { mode: 0o700 });
writeExclusiveJson(
path.join(command.options.stagingRoot, INCOMPLETE_NAME),
{
schemaVersion: 1,
kind: 'qinglong3-legacy-data-directory-adoption-incomplete',
},
);
syncDirectory(command.options.stagingRoot);
syncDirectory(path.dirname(command.options.stagingRoot));
const payloadRoot = path.join(command.options.stagingRoot, 'payload');
fs.mkdirSync(payloadRoot, { mode: 0o700 });
const copyBudget: MutableCopyBudget = { entries: 0, bytes: 0 };
for (const group of PAYLOAD_GROUPS) {
const groupRoot = path.join(payloadRoot, group.directoryName);
fs.mkdirSync(groupRoot, { mode: 0o700 });
for (const category of group.categories) {
copyCategory(
command.options.dataRoot,
groupRoot,
category,
authority.uid,
before.budget,
copyBudget,
);
}
syncDirectory(groupRoot);
}
syncDirectory(payloadRoot);
fence.assertTargetIdentity();
payload = inspectPayload(command.options.stagingRoot, authority.uid);
} finally {
await fence.release();
}
const after = inspectPlan(
command.options.dataRoot,
command.options.profile,
);
assertReviewablePlan(after, command.options.expectedPlanDigest);
if (JSON.stringify(after) !== JSON.stringify(before)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'data directory changed during staging',
);
}
const manifestBase = manifestPayload(command, after, fence, payload);
const manifest = Object.freeze({
...manifestBase,
manifestDigest: sha256Text(JSON.stringify(manifestBase)),
});
writeExclusiveJson(
path.join(command.options.stagingRoot, MANIFEST_NAME),
manifest,
);
syncDirectory(command.options.stagingRoot);
fs.unlinkSync(path.join(command.options.stagingRoot, INCOMPLETE_NAME));
syncDirectory(command.options.stagingRoot);
const verified = verifyStaticStage(authority, manifest.manifestDigest);
return result(
LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION,
'staged',
verified,
);
} catch (error) {
if (error instanceof LocalDataDirectoryAdoptionConfigurationError) {
throw error;
}
throw new LocalDataDirectoryAdoptionConfigurationError(
'data directory staging failed',
error,
);
}
}
export async function verifyLocalDataDirectoryAdoption(
command: Readonly<VerifyLocalDataDirectoryAdoptionCommand>,
): Promise<Readonly<LocalDataDirectoryAdoptionMutationResult>> {
try {
const authority = rootAuthority(command.options, false);
const manifest = verifyStaticStage(
authority,
command.options.expectedManifestDigest,
);
if (
manifest.profile !== command.options.profile ||
manifest.sqliteActivationDigest !==
command.options.sqlite.expectedActivationDigest
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'staging manifest authority binding is invalid',
);
}
const before = inspectPlan(
command.options.dataRoot,
command.options.profile,
);
assertReviewablePlan(before, manifest.planDigest);
const fence = await acquireSqliteFence(
command.options.sqlite,
command.options.profile,
);
try {
if (
fence.activation.activationDigest !== manifest.sqliteActivationDigest ||
fence.adoption.manifestDigest !== manifest.sqliteAdoptionManifestDigest
) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'SQLite activation no longer matches the staging manifest',
);
}
verifyStaticStage(authority, command.options.expectedManifestDigest);
fence.assertTargetIdentity();
} finally {
await fence.release();
}
const after = inspectPlan(
command.options.dataRoot,
command.options.profile,
);
assertReviewablePlan(after, manifest.planDigest);
if (JSON.stringify(after) !== JSON.stringify(before)) {
throw new LocalDataDirectoryAdoptionConfigurationError(
'data directory changed during verification',
);
}
return result(
LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION,
'verified',
manifest,
);
} catch (error) {
if (error instanceof LocalDataDirectoryAdoptionConfigurationError) {
throw error;
}
throw new LocalDataDirectoryAdoptionConfigurationError(
'data directory verification failed',
error,
);
}
}
@@ -0,0 +1,481 @@
const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { DatabaseSync } = require('node:sqlite');
const { test } = require('node:test');
const BINARY = path.join(__dirname, '../dist/lifecycle/adoptionCli.js');
const DIRECTORY_INSPECT = 'local-data-directory.adoption.inspect';
const DIRECTORY_STAGE = 'local-data-directory.adoption.stage';
const DIRECTORY_VERIFY = 'local-data-directory.adoption.verify';
function privateDirectory(directoryPath) {
fs.mkdirSync(directoryPath, { recursive: true, mode: 0o700 });
fs.chmodSync(directoryPath, 0o700);
}
function privateFile(filePath, content) {
privateDirectory(path.dirname(filePath));
fs.writeFileSync(filePath, content, { mode: 0o600 });
fs.chmodSync(filePath, 0o600);
}
function createLegacyDatabase(sourcePath) {
const source = new DatabaseSync(sourcePath);
source.exec(`
CREATE TABLE "Crontabs" (
id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255),
command VARCHAR(255), schedule VARCHAR(255), timestamp VARCHAR(255),
saved TINYINT(1), status DECIMAL, isSystem DECIMAL, pid DECIMAL,
isDisabled DECIMAL, isPinned DECIMAL, log_path VARCHAR(255), labels JSON,
last_running_time DECIMAL, last_execution_time DECIMAL, sub_id DECIMAL,
extra_schedules JSON, task_before VARCHAR(255), task_after VARCHAR(255),
log_name VARCHAR(255), allow_multiple_instances DECIMAL,
work_dir VARCHAR(255), createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL
);
CREATE TABLE "Dependences" (
id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), type DECIMAL,
timestamp VARCHAR(255), status DECIMAL, log JSON, remark VARCHAR(255),
createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL
);
CREATE TABLE "Apps" (
id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), scopes JSON,
client_id VARCHAR(255), client_secret VARCHAR(255), tokens JSON,
createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL
);
CREATE TABLE "Auths" (
id INTEGER PRIMARY KEY AUTOINCREMENT, ip VARCHAR(255), type VARCHAR(255),
info JSON, createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL
);
CREATE TABLE "Envs" (
id INTEGER PRIMARY KEY AUTOINCREMENT, value VARCHAR(255),
timestamp VARCHAR(255), status DECIMAL, position DECIMAL,
name VARCHAR(255), remarks VARCHAR(255), isPinned DECIMAL, labels JSON,
createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL
);
CREATE TABLE "Subscriptions" (
id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), url VARCHAR(255),
schedule VARCHAR(255), interval_schedule JSON, type VARCHAR(255),
whitelist VARCHAR(255), blacklist VARCHAR(255), status DECIMAL,
dependences VARCHAR(255), extensions VARCHAR(255), sub_before VARCHAR(255),
sub_after VARCHAR(255), branch VARCHAR(255), pull_type VARCHAR(255),
pull_option JSON, pid DECIMAL, is_disabled DECIMAL, log_path VARCHAR(255),
schedule_type VARCHAR(255), alias VARCHAR(255), proxy VARCHAR(255),
autoAddCron DECIMAL, autoDelCron DECIMAL,
createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL
);
CREATE TABLE "CrontabViews" (
id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255), position DECIMAL,
isDisabled DECIMAL, filters JSON, sorts JSON, filterRelation VARCHAR(255),
type DECIMAL, createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL
);
CREATE TABLE "CrontabStats" (
id INTEGER PRIMARY KEY AUTOINCREMENT, ref_id DECIMAL NOT NULL,
date VARCHAR(255) NOT NULL, run_count DECIMAL, success_count DECIMAL,
fail_count DECIMAL, total_time DECIMAL, max_time DECIMAL,
createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL
);
CREATE TABLE "RunningInstances" (
id INTEGER PRIMARY KEY AUTOINCREMENT, cron_id DECIMAL NOT NULL,
run_id VARCHAR(36), attempt_id VARCHAR(36), pid DECIMAL,
log_path VARCHAR(255), started_at DECIMAL NOT NULL, finished_at DECIMAL,
status DECIMAL NOT NULL, exit_code DECIMAL,
createdAt DATETIME NOT NULL, updatedAt DATETIME NOT NULL
);
CREATE TABLE "PluginOwnedState" (
id INTEGER PRIMARY KEY, payload TEXT NOT NULL
);
INSERT INTO "Crontabs" (
id, name, command, schedule, status, isDisabled, isPinned,
createdAt, updatedAt
) VALUES (
1, 'Legacy task', 'task /scripts/legacy.sh', '0 0 * * *',
1, 0, 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
);
INSERT INTO "Envs" (
id, name, value, status, position, createdAt, updatedAt
) VALUES (
1, 'LEGACY_VALUE', 'preserved', 0, 100,
CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
);
INSERT INTO "PluginOwnedState" (id, payload)
VALUES (1, '{"preserved":true}');
`);
source.close();
fs.chmodSync(sourcePath, 0o600);
}
function fixture(t) {
const deploymentRoot = fs.realpathSync(
fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-directory-stage-')),
);
fs.chmodSync(deploymentRoot, 0o700);
t.after(() => fs.rmSync(deploymentRoot, { recursive: true, force: true }));
const value = {
deploymentRoot,
commandsDirectory: path.join(deploymentRoot, 'commands'),
artifactsDirectory: path.join(deploymentRoot, 'artifacts'),
stagingParent: path.join(deploymentRoot, 'staging'),
dataRoot: path.join(deploymentRoot, 'legacy-data'),
};
privateDirectory(value.commandsDirectory);
privateDirectory(value.artifactsDirectory);
privateDirectory(value.stagingParent);
privateDirectory(value.dataRoot);
Object.assign(value, {
sourcePath: path.join(value.dataRoot, 'db', 'database.sqlite'),
targetPath: path.join(value.artifactsDirectory, 'qinglong3.sqlite'),
recoveryPath: path.join(
value.artifactsDirectory,
'database.pre-ql3.sqlite',
),
sqliteManifestPath: path.join(
value.artifactsDirectory,
'qinglong3-sqlite-adoption.json',
),
activationPath: path.join(
value.artifactsDirectory,
'qinglong3-sqlite-activation.json',
),
stagingRoot: path.join(value.stagingParent, 'reviewed-data'),
});
privateDirectory(path.dirname(value.sourcePath));
createLegacyDatabase(value.sourcePath);
privateFile(path.join(value.dataRoot, 'db', 'keyv.sqlite'), 'legacy-keyv');
privateFile(path.join(value.dataRoot, 'config', 'config.sh'), 'export A=1\n');
privateFile(
path.join(value.dataRoot, 'scripts', 'jobs', 'example.sh'),
'echo qinglong\n',
);
privateFile(
path.join(value.dataRoot, 'upload', 'avatar.bin'),
Buffer.from([1, 2, 3]),
);
privateFile(
path.join(value.dataRoot, 'ssh.d', 'repository-key'),
'private-key',
);
privateFile(
path.join(value.dataRoot, 'repo', 'cache', 'ignored'),
'regenerate-me',
);
privateFile(
path.join(value.dataRoot, 'log', 'history', 'ignored'),
'retain-me',
);
return value;
}
function runRaw(value, name, operation, options) {
const commandPath = path.join(value.commandsDirectory, `${name}.json`);
fs.writeFileSync(
commandPath,
`${JSON.stringify({ schemaVersion: 1, operation, options })}\n`,
{ mode: 0o600 },
);
fs.chmodSync(commandPath, 0o600);
return spawnSync(
process.execPath,
[BINARY, 'run', '--command-file', commandPath],
{ encoding: 'utf8' },
);
}
function run(value, name, operation, options) {
const child = runRaw(value, name, operation, options);
assert.equal(child.status, 0, child.stderr);
assert.equal(child.stderr, '');
return { child, result: JSON.parse(child.stdout) };
}
function prepare(value) {
const base = { deploymentRoot: value.deploymentRoot, profile: 'edge' };
const sqlitePlan = run(
value,
'sqlite-inspect',
'local-sqlite.adoption.inspect',
{ ...base, sourcePath: value.sourcePath, legacyTimezone: 'UTC' },
).result;
run(value, 'sqlite-stage', 'local-sqlite.adoption.stage', {
...base,
sourcePath: value.sourcePath,
targetPath: value.targetPath,
recoveryPath: value.recoveryPath,
manifestPath: value.sqliteManifestPath,
expectedPlanDigest: sqlitePlan.evidence.planDigest,
legacyTimezone: 'UTC',
});
const sqliteVerified = run(
value,
'sqlite-verify',
'local-sqlite.adoption.verify',
{
...base,
targetPath: value.targetPath,
recoveryPath: value.recoveryPath,
manifestPath: value.sqliteManifestPath,
},
).result;
const activation = run(
value,
'sqlite-activate',
'local-sqlite.activation.prepare',
{
...base,
sourcePath: value.sourcePath,
targetPath: value.targetPath,
recoveryPath: value.recoveryPath,
manifestPath: value.sqliteManifestPath,
activationPath: value.activationPath,
expectedManifestDigest: sqliteVerified.evidence.manifestDigest,
},
).result;
const directoryPlan = run(value, 'directory-inspect', DIRECTORY_INSPECT, {
dataRoot: value.dataRoot,
profile: 'edge',
}).result;
return {
directoryPlanDigest: directoryPlan.evidence.planDigest,
activationDigest: activation.evidence.activationDigest,
};
}
function sqliteBinding(value, activationDigest) {
return {
sourcePath: value.sourcePath,
targetPath: value.targetPath,
recoveryPath: value.recoveryPath,
manifestPath: value.sqliteManifestPath,
activationPath: value.activationPath,
expectedActivationDigest: activationDigest,
};
}
function stageOptions(value, prepared) {
return {
deploymentRoot: value.deploymentRoot,
dataRoot: value.dataRoot,
stagingRoot: value.stagingRoot,
profile: 'edge',
expectedPlanDigest: prepared.directoryPlanDigest,
sqlite: sqliteBinding(value, prepared.activationDigest),
};
}
function verifyOptions(value, prepared, manifestDigest) {
return {
deploymentRoot: value.deploymentRoot,
dataRoot: value.dataRoot,
stagingRoot: value.stagingRoot,
profile: 'edge',
expectedManifestDigest: manifestDigest,
sqlite: sqliteBinding(value, prepared.activationDigest),
};
}
test('stages only reviewed payloads behind the real SQLite activation fence', (t) => {
const value = fixture(t);
const prepared = prepare(value);
const staged = run(
value,
'directory-stage',
DIRECTORY_STAGE,
stageOptions(value, prepared),
);
assert.equal(staged.result.status, 'staged');
assert.match(staged.result.evidence.manifestDigest, /^[0-9a-f]{64}$/);
assert.deepEqual(fs.readdirSync(value.stagingRoot).sort(), [
'manifest.json',
'payload',
]);
const expectedFiles = [
['payload', 'copy-reviewed', 'scripts', 'jobs', 'example.sh'],
['payload', 'copy-reviewed', 'upload', 'avatar.bin'],
['payload', 'transform-input', 'config', 'config.sh'],
['payload', 'transform-input', 'db', 'keyv.sqlite'],
['payload', 'transform-input', 'ssh.d', 'repository-key'],
];
for (const parts of expectedFiles) {
const filePath = path.join(value.stagingRoot, ...parts);
assert.equal(fs.statSync(filePath).mode & 0o777, 0o600);
}
assert.equal(
fs.existsSync(
path.join(
value.stagingRoot,
'payload',
'transform-input',
'db',
'database.sqlite',
),
),
false,
);
assert.equal(staged.child.stdout.includes(value.dataRoot), false);
assert.equal(staged.child.stdout.includes('example.sh'), false);
assert.equal(staged.child.stdout.includes('private-key'), false);
const verified = run(
value,
'directory-verify',
DIRECTORY_VERIFY,
verifyOptions(value, prepared, staged.result.evidence.manifestDigest),
).result;
assert.equal(verified.status, 'verified');
assert.deepEqual(verified.evidence, staged.result.evidence);
const replayed = run(
value,
'directory-verify-replay',
DIRECTORY_VERIFY,
verifyOptions(value, prepared, staged.result.evidence.manifestDigest),
).result;
assert.deepEqual(replayed, verified);
});
test('verification rejects staged payload and source drift', (t) => {
const value = fixture(t);
const prepared = prepare(value);
const staged = run(
value,
'stage-before-drift',
DIRECTORY_STAGE,
stageOptions(value, prepared),
).result;
const stagedScript = path.join(
value.stagingRoot,
'payload',
'copy-reviewed',
'scripts',
'jobs',
'example.sh',
);
fs.writeFileSync(stagedScript, 'tampered\n');
const targetDrift = runRaw(
value,
'verify-target-drift',
DIRECTORY_VERIFY,
verifyOptions(value, prepared, staged.evidence.manifestDigest),
);
assert.equal(targetDrift.status, 1);
assert.equal(
JSON.parse(targetDrift.stderr).code,
'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID',
);
fs.writeFileSync(stagedScript, 'echo qinglong\n');
fs.chmodSync(stagedScript, 0o600);
privateFile(
path.join(value.dataRoot, 'scripts', 'jobs', 'example.sh'),
'source-drift\n',
);
const sourceDrift = runRaw(
value,
'verify-source-drift',
DIRECTORY_VERIFY,
verifyOptions(value, prepared, staged.evidence.manifestDigest),
);
assert.equal(sourceDrift.status, 1);
assert.equal(
JSON.parse(sourceDrift.stderr).code,
'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID',
);
});
test('verification never follows a staged payload symlink', (t) => {
const value = fixture(t);
const prepared = prepare(value);
const staged = run(
value,
'stage-before-link',
DIRECTORY_STAGE,
stageOptions(value, prepared),
).result;
const stagedScript = path.join(
value.stagingRoot,
'payload',
'copy-reviewed',
'scripts',
'jobs',
'example.sh',
);
fs.unlinkSync(stagedScript);
fs.symlinkSync(value.sourcePath, stagedScript);
const child = runRaw(
value,
'verify-link',
DIRECTORY_VERIFY,
verifyOptions(value, prepared, staged.evidence.manifestDigest),
);
assert.equal(child.status, 1);
assert.equal(child.stdout, '');
assert.equal(
JSON.parse(child.stderr).code,
'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID',
);
});
test('staging is no-replace and fails before copying on activation drift', (t) => {
const value = fixture(t);
const prepared = prepare(value);
privateDirectory(value.stagingRoot);
privateFile(path.join(value.stagingRoot, '.incomplete'), 'crash-residue');
const residue = runRaw(
value,
'stage-residue',
DIRECTORY_STAGE,
stageOptions(value, prepared),
);
assert.equal(residue.status, 1);
assert.equal(
fs.readFileSync(path.join(value.stagingRoot, '.incomplete'), 'utf8'),
'crash-residue',
);
fs.rmSync(value.stagingRoot, { recursive: true });
const drifted = stageOptions(value, prepared);
drifted.sqlite.expectedActivationDigest = '0'.repeat(64);
const activationDrift = runRaw(
value,
'stage-activation-drift',
DIRECTORY_STAGE,
drifted,
);
assert.equal(activationDrift.status, 1);
assert.equal(fs.existsSync(value.stagingRoot), false);
});
test('widened directory staging commands fail closed before source access', (t) => {
const value = fixture(t);
const child = runRaw(value, 'widened-stage', DIRECTORY_STAGE, {
deploymentRoot: value.deploymentRoot,
dataRoot: path.join(value.deploymentRoot, 'missing-source'),
stagingRoot: value.stagingRoot,
profile: 'edge',
expectedPlanDigest: '0'.repeat(64),
sqlite: {
sourcePath: path.join(
value.deploymentRoot,
'missing-source',
'db',
'database.sqlite',
),
targetPath: path.join(value.artifactsDirectory, 'missing-target'),
recoveryPath: path.join(value.artifactsDirectory, 'missing-recovery'),
manifestPath: path.join(value.artifactsDirectory, 'missing-manifest'),
activationPath: path.join(value.artifactsDirectory, 'missing-activation'),
expectedActivationDigest: '0'.repeat(64),
},
extraAuthority: true,
});
assert.equal(child.status, 1);
assert.equal(child.stdout, '');
assert.equal(
JSON.parse(child.stderr).code,
'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID',
);
});