mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): prepare legacy data transformation
This commit is contained in:
whyour
@@ -1,6 +1,8 @@
|
||||
import {
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION,
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION,
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION,
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION,
|
||||
normalizeLocalDataDirectoryAdoptionCommand,
|
||||
} from './contract';
|
||||
import {
|
||||
@@ -12,10 +14,16 @@ import {
|
||||
verifyLocalDataDirectoryAdoption,
|
||||
type LocalDataDirectoryAdoptionMutationResult,
|
||||
} from './staging';
|
||||
import {
|
||||
transformLocalDataDirectoryAdoption,
|
||||
verifyLocalDataDirectoryAdoptionTransformation,
|
||||
type LocalDataDirectoryTransformationResult,
|
||||
} from './transformation/transformation';
|
||||
|
||||
export type LocalDataDirectoryAdoptionProductCommandResult =
|
||||
| LocalDataDirectoryAdoptionInspectResult
|
||||
| LocalDataDirectoryAdoptionMutationResult;
|
||||
| LocalDataDirectoryAdoptionMutationResult
|
||||
| LocalDataDirectoryTransformationResult;
|
||||
|
||||
export async function runLocalDataDirectoryAdoptionProductCommand(
|
||||
value: unknown,
|
||||
@@ -27,5 +35,11 @@ export async function runLocalDataDirectoryAdoptionProductCommand(
|
||||
if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION) {
|
||||
return stageLocalDataDirectoryAdoption(command);
|
||||
}
|
||||
return verifyLocalDataDirectoryAdoption(command);
|
||||
if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION) {
|
||||
return verifyLocalDataDirectoryAdoption(command);
|
||||
}
|
||||
if (command.operation === LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION) {
|
||||
return transformLocalDataDirectoryAdoption(command);
|
||||
}
|
||||
return verifyLocalDataDirectoryAdoptionTransformation(command);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import path from 'node:path';
|
||||
|
||||
const MAX_PATH_BYTES = 4_096;
|
||||
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
|
||||
const PROJECT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
|
||||
export const LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION =
|
||||
'local-data-directory.adoption.inspect' as const;
|
||||
@@ -9,11 +10,17 @@ export const LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION =
|
||||
'local-data-directory.adoption.stage' as const;
|
||||
export const LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION =
|
||||
'local-data-directory.adoption.verify' as const;
|
||||
export const LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION =
|
||||
'local-data-directory.adoption.transform' as const;
|
||||
export const LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION =
|
||||
'local-data-directory.adoption.transform.verify' as const;
|
||||
|
||||
export type LocalDataDirectoryAdoptionOperation =
|
||||
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION
|
||||
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
|
||||
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION;
|
||||
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION
|
||||
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION
|
||||
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION;
|
||||
|
||||
export interface InspectLocalDataDirectoryAdoptionCommand {
|
||||
readonly schemaVersion: 1;
|
||||
@@ -57,10 +64,33 @@ export interface VerifyLocalDataDirectoryAdoptionCommand {
|
||||
};
|
||||
}
|
||||
|
||||
interface LocalDataDirectoryAdoptionTransformationOptions
|
||||
extends LocalDataDirectoryAdoptionMutationOptions {
|
||||
readonly transformationRoot: string;
|
||||
readonly projectId: string;
|
||||
readonly expectedManifestDigest: string;
|
||||
}
|
||||
|
||||
export interface TransformLocalDataDirectoryAdoptionCommand {
|
||||
readonly schemaVersion: 1;
|
||||
readonly operation: typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION;
|
||||
readonly options: LocalDataDirectoryAdoptionTransformationOptions;
|
||||
}
|
||||
|
||||
export interface VerifyLocalDataDirectoryAdoptionTransformationCommand {
|
||||
readonly schemaVersion: 1;
|
||||
readonly operation: typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION;
|
||||
readonly options: LocalDataDirectoryAdoptionTransformationOptions & {
|
||||
readonly expectedTransformationDigest: string;
|
||||
};
|
||||
}
|
||||
|
||||
export type LocalDataDirectoryAdoptionCommand =
|
||||
| InspectLocalDataDirectoryAdoptionCommand
|
||||
| StageLocalDataDirectoryAdoptionCommand
|
||||
| VerifyLocalDataDirectoryAdoptionCommand;
|
||||
| VerifyLocalDataDirectoryAdoptionCommand
|
||||
| TransformLocalDataDirectoryAdoptionCommand
|
||||
| VerifyLocalDataDirectoryAdoptionTransformationCommand;
|
||||
|
||||
export class LocalDataDirectoryAdoptionConfigurationError extends TypeError {
|
||||
readonly code = 'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID';
|
||||
@@ -97,7 +127,9 @@ export function isLocalDataDirectoryAdoptionOperation(
|
||||
return (
|
||||
value === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION ||
|
||||
value === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION ||
|
||||
value === LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION
|
||||
value === LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION ||
|
||||
value === LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION ||
|
||||
value === LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION
|
||||
);
|
||||
}
|
||||
|
||||
@@ -187,26 +219,37 @@ export function normalizeLocalDataDirectoryAdoptionCommand(
|
||||
);
|
||||
}
|
||||
const options = candidate.options as Record<string, unknown>;
|
||||
const expectedKeys =
|
||||
candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION
|
||||
? ['dataRoot', 'profile']
|
||||
: candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
|
||||
? [
|
||||
'dataRoot',
|
||||
'deploymentRoot',
|
||||
'expectedPlanDigest',
|
||||
'profile',
|
||||
'sqlite',
|
||||
'stagingRoot',
|
||||
]
|
||||
: [
|
||||
'dataRoot',
|
||||
'deploymentRoot',
|
||||
'expectedManifestDigest',
|
||||
'profile',
|
||||
'sqlite',
|
||||
'stagingRoot',
|
||||
];
|
||||
let expectedKeys: readonly string[];
|
||||
if (candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_INSPECT_OPERATION) {
|
||||
expectedKeys = ['dataRoot', 'profile'];
|
||||
} else if (
|
||||
candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_STAGE_OPERATION
|
||||
) {
|
||||
expectedKeys = [
|
||||
'dataRoot',
|
||||
'deploymentRoot',
|
||||
'expectedPlanDigest',
|
||||
'profile',
|
||||
'sqlite',
|
||||
'stagingRoot',
|
||||
];
|
||||
} else {
|
||||
expectedKeys = [
|
||||
'dataRoot',
|
||||
'deploymentRoot',
|
||||
'expectedManifestDigest',
|
||||
'profile',
|
||||
'sqlite',
|
||||
'stagingRoot',
|
||||
...(candidate.operation === LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION
|
||||
? []
|
||||
: ['projectId', 'transformationRoot']),
|
||||
...(candidate.operation ===
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION
|
||||
? ['expectedTransformationDigest']
|
||||
: []),
|
||||
];
|
||||
}
|
||||
if (
|
||||
!exactKeys(options, expectedKeys) ||
|
||||
!normalizedAbsolutePath(options.dataRoot) ||
|
||||
@@ -235,6 +278,32 @@ export function normalizeLocalDataDirectoryAdoptionCommand(
|
||||
'reviewed adoption digest is invalid',
|
||||
);
|
||||
}
|
||||
if (
|
||||
candidate.operation ===
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION ||
|
||||
candidate.operation ===
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION
|
||||
) {
|
||||
if (
|
||||
!normalizedAbsolutePath(options.transformationRoot) ||
|
||||
typeof options.projectId !== 'string' ||
|
||||
!PROJECT_ID_PATTERN.test(options.projectId)
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation target binding is invalid',
|
||||
);
|
||||
}
|
||||
if (
|
||||
candidate.operation ===
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION &&
|
||||
(typeof options.expectedTransformationDigest !== 'string' ||
|
||||
!DIGEST_PATTERN.test(options.expectedTransformationDigest))
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation digest is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return Object.freeze(value as LocalDataDirectoryAdoptionCommand);
|
||||
}
|
||||
|
||||
+218
@@ -0,0 +1,218 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import { LocalDataDirectoryAdoptionConfigurationError } from '../contract';
|
||||
import { sha256Text } from '../manifest';
|
||||
import {
|
||||
optionalPrivateDirectory,
|
||||
readStablePrivateUtf8File,
|
||||
summarizePrivateTree,
|
||||
type PrivateTreeEvidence,
|
||||
} from './files';
|
||||
|
||||
const MAX_CONFIG_BYTES = 256 * 1024;
|
||||
const MAX_SECRET_BYTES = 16 * 1024;
|
||||
const NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/;
|
||||
const SAFE_UNQUOTED_PATTERN = /^[A-Za-z0-9_./:@%,+-]*$/;
|
||||
|
||||
export interface SecretImportDraft {
|
||||
readonly kind: 'environment' | 'ssh_private_key';
|
||||
readonly sourceName: string;
|
||||
readonly targetName: string;
|
||||
readonly value: string;
|
||||
}
|
||||
|
||||
export interface ConfigTransformationModel {
|
||||
readonly schema: 'qinglong/legacy-config-transformation@v1';
|
||||
readonly exportedEnvironment: readonly Readonly<{
|
||||
environmentName: string;
|
||||
targetSecretName: string;
|
||||
}>[];
|
||||
readonly retiredSettings: readonly Readonly<{
|
||||
name: string;
|
||||
valueDigest: string;
|
||||
}>[];
|
||||
readonly omittedEmptyExports: number;
|
||||
readonly duplicateAssignments: number;
|
||||
readonly unsupportedLines: number;
|
||||
readonly unsupportedLineDigest: string;
|
||||
readonly disabledAssetEntries: number;
|
||||
readonly activation: 'disabled';
|
||||
}
|
||||
|
||||
export interface ConfigTransformation {
|
||||
readonly source: Readonly<PrivateTreeEvidence> | null;
|
||||
readonly model: Readonly<ConfigTransformationModel>;
|
||||
readonly secrets: readonly Readonly<SecretImportDraft>[];
|
||||
readonly assessment: 'ready' | 'manual_required';
|
||||
}
|
||||
|
||||
function literal(value: string): string | null {
|
||||
if (value.length === 0) return '';
|
||||
if (value.startsWith("'") && value.endsWith("'")) {
|
||||
const inner = value.slice(1, -1);
|
||||
return inner.includes("'") ? null : inner;
|
||||
}
|
||||
if (value.startsWith('"') && value.endsWith('"')) {
|
||||
const inner = value.slice(1, -1);
|
||||
return /["`$\\]/.test(inner) ? null : inner;
|
||||
}
|
||||
return SAFE_UNQUOTED_PATTERN.test(value) ? value : null;
|
||||
}
|
||||
|
||||
function targetSecretName(name: string): string {
|
||||
return `legacy-env-${sha256Text(name).slice(0, 32)}`;
|
||||
}
|
||||
|
||||
function emptyModel(): Readonly<ConfigTransformationModel> {
|
||||
return Object.freeze({
|
||||
schema: 'qinglong/legacy-config-transformation@v1',
|
||||
exportedEnvironment: Object.freeze([]),
|
||||
retiredSettings: Object.freeze([]),
|
||||
omittedEmptyExports: 0,
|
||||
duplicateAssignments: 0,
|
||||
unsupportedLines: 0,
|
||||
unsupportedLineDigest: sha256Text(''),
|
||||
disabledAssetEntries: 0,
|
||||
activation: 'disabled',
|
||||
});
|
||||
}
|
||||
|
||||
export function transformLegacyConfig(
|
||||
categoryRoot: string,
|
||||
uid: number,
|
||||
): Readonly<ConfigTransformation> {
|
||||
if (
|
||||
!optionalPrivateDirectory(categoryRoot, uid, 'config transformation input')
|
||||
) {
|
||||
return Object.freeze({
|
||||
source: null,
|
||||
model: emptyModel(),
|
||||
secrets: Object.freeze([]),
|
||||
assessment: 'ready',
|
||||
});
|
||||
}
|
||||
const source = summarizePrivateTree(categoryRoot, uid);
|
||||
const configPath = path.join(categoryRoot, 'config.sh');
|
||||
let configExists = false;
|
||||
try {
|
||||
const stat = fs.lstatSync(configPath);
|
||||
configExists = stat.isFile() && !stat.isSymbolicLink();
|
||||
} catch (error) {
|
||||
if (
|
||||
!error ||
|
||||
typeof error !== 'object' ||
|
||||
!('code' in error) ||
|
||||
error.code !== 'ENOENT'
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (!configExists) {
|
||||
const model = Object.freeze({
|
||||
...emptyModel(),
|
||||
disabledAssetEntries: source.entries,
|
||||
});
|
||||
return Object.freeze({
|
||||
source,
|
||||
model,
|
||||
secrets: Object.freeze([]),
|
||||
assessment: source.entries === 0 ? 'ready' : 'manual_required',
|
||||
});
|
||||
}
|
||||
|
||||
const content = readStablePrivateUtf8File(
|
||||
configPath,
|
||||
uid,
|
||||
MAX_CONFIG_BYTES,
|
||||
'legacy config.sh',
|
||||
);
|
||||
const recognized = new Map<
|
||||
string,
|
||||
{ readonly exported: boolean; readonly value: string }
|
||||
>();
|
||||
const duplicates = new Set<string>();
|
||||
let unsupportedLines = 0;
|
||||
const unsupportedHash = crypto.createHash('sha256');
|
||||
const lines = content.split('\n');
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const line = lines[index]!.endsWith('\r')
|
||||
? lines[index]!.slice(0, -1)
|
||||
: lines[index]!;
|
||||
if (line.trim().length === 0 || line.trimStart().startsWith('#')) continue;
|
||||
const match =
|
||||
/^(?:(export)[ \t]+)?([A-Za-z_][A-Za-z0-9_]{0,127})=(.*)$/.exec(line);
|
||||
const parsed = match ? literal(match[3]!) : null;
|
||||
if (!match || !NAME_PATTERN.test(match[2]!) || parsed === null) {
|
||||
unsupportedLines += 1;
|
||||
unsupportedHash.update(`${index + 1}:${sha256Text(line)}\n`, 'utf8');
|
||||
continue;
|
||||
}
|
||||
const name = match[2]!;
|
||||
if (recognized.has(name)) duplicates.add(name);
|
||||
recognized.set(name, { exported: match[1] === 'export', value: parsed });
|
||||
}
|
||||
|
||||
const exportedEnvironment: Array<{
|
||||
environmentName: string;
|
||||
targetSecretName: string;
|
||||
}> = [];
|
||||
const retiredSettings: Array<{ name: string; valueDigest: string }> = [];
|
||||
const secrets: SecretImportDraft[] = [];
|
||||
let omittedEmptyExports = 0;
|
||||
for (const name of [...recognized.keys()].sort((left, right) =>
|
||||
Buffer.compare(Buffer.from(left), Buffer.from(right)),
|
||||
)) {
|
||||
const entry = recognized.get(name)!;
|
||||
if (duplicates.has(name)) continue;
|
||||
if (!entry.exported) {
|
||||
retiredSettings.push({ name, valueDigest: sha256Text(entry.value) });
|
||||
continue;
|
||||
}
|
||||
if (entry.value.length === 0) {
|
||||
omittedEmptyExports += 1;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
entry.value.includes('\0') ||
|
||||
Buffer.byteLength(entry.value, 'utf8') > MAX_SECRET_BYTES
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'legacy exported environment value exceeds the Secret budget',
|
||||
);
|
||||
}
|
||||
const targetName = targetSecretName(name);
|
||||
exportedEnvironment.push({
|
||||
environmentName: name,
|
||||
targetSecretName: targetName,
|
||||
});
|
||||
secrets.push({
|
||||
kind: 'environment',
|
||||
sourceName: name,
|
||||
targetName,
|
||||
value: entry.value,
|
||||
});
|
||||
}
|
||||
const disabledAssetEntries = Math.max(0, source.entries - 1);
|
||||
const model = Object.freeze({
|
||||
schema: 'qinglong/legacy-config-transformation@v1' as const,
|
||||
exportedEnvironment: Object.freeze(exportedEnvironment),
|
||||
retiredSettings: Object.freeze(retiredSettings),
|
||||
omittedEmptyExports,
|
||||
duplicateAssignments: duplicates.size,
|
||||
unsupportedLines,
|
||||
unsupportedLineDigest: unsupportedHash.digest('hex'),
|
||||
disabledAssetEntries,
|
||||
activation: 'disabled' as const,
|
||||
});
|
||||
return Object.freeze({
|
||||
source,
|
||||
model,
|
||||
secrets: Object.freeze(secrets),
|
||||
assessment:
|
||||
duplicates.size > 0 || unsupportedLines > 0 || disabledAssetEntries > 0
|
||||
? 'manual_required'
|
||||
: 'ready',
|
||||
});
|
||||
}
|
||||
+316
@@ -0,0 +1,316 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { TextDecoder } from 'node:util';
|
||||
|
||||
import { LocalDataDirectoryAdoptionConfigurationError } from '../contract';
|
||||
import {
|
||||
assertPrivateDirectory,
|
||||
rootAuthority,
|
||||
sameStat,
|
||||
sortedNames,
|
||||
stableFileDigest,
|
||||
syncDirectory,
|
||||
type RootAuthority,
|
||||
} from '../filesystem';
|
||||
import type {
|
||||
TransformLocalDataDirectoryAdoptionCommand,
|
||||
VerifyLocalDataDirectoryAdoptionTransformationCommand,
|
||||
} from '../contract';
|
||||
|
||||
const MAX_RELATIVE_PATH_BYTES = 4_096;
|
||||
|
||||
export interface TransformationAuthority extends RootAuthority {
|
||||
readonly transformationRoot: string;
|
||||
}
|
||||
|
||||
export interface PrivateTreeEvidence {
|
||||
readonly entries: number;
|
||||
readonly directories: number;
|
||||
readonly files: number;
|
||||
readonly bytes: number;
|
||||
readonly digest: string;
|
||||
}
|
||||
|
||||
function inside(root: string, candidate: string): boolean {
|
||||
const relative = path.relative(root, candidate);
|
||||
return (
|
||||
relative !== '' &&
|
||||
relative !== '..' &&
|
||||
!relative.startsWith(`..${path.sep}`) &&
|
||||
!path.isAbsolute(relative)
|
||||
);
|
||||
}
|
||||
|
||||
function assertMissing(candidate: string): void {
|
||||
try {
|
||||
fs.lstatSync(candidate);
|
||||
} catch (error) {
|
||||
if (
|
||||
error &&
|
||||
typeof error === 'object' &&
|
||||
'code' in error &&
|
||||
error.code === 'ENOENT'
|
||||
) {
|
||||
return;
|
||||
}
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformationRoot cannot be inspected',
|
||||
error,
|
||||
);
|
||||
}
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformationRoot must not already exist',
|
||||
);
|
||||
}
|
||||
|
||||
export function transformationAuthority(
|
||||
options:
|
||||
| TransformLocalDataDirectoryAdoptionCommand['options']
|
||||
| VerifyLocalDataDirectoryAdoptionTransformationCommand['options'],
|
||||
requireMissing: boolean,
|
||||
): Readonly<TransformationAuthority> {
|
||||
const stage = rootAuthority(
|
||||
{
|
||||
deploymentRoot: options.deploymentRoot,
|
||||
dataRoot: options.dataRoot,
|
||||
stagingRoot: options.stagingRoot,
|
||||
profile: options.profile,
|
||||
sqlite: options.sqlite,
|
||||
expectedManifestDigest: options.expectedManifestDigest,
|
||||
},
|
||||
false,
|
||||
);
|
||||
const target = options.transformationRoot;
|
||||
if (
|
||||
!inside(stage.deploymentRoot, target) ||
|
||||
target === stage.dataRoot ||
|
||||
target === stage.stagingRoot ||
|
||||
inside(stage.dataRoot, target) ||
|
||||
inside(target, stage.dataRoot) ||
|
||||
inside(stage.stagingRoot, target) ||
|
||||
inside(target, stage.stagingRoot)
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformationRoot must be isolated inside deploymentRoot',
|
||||
);
|
||||
}
|
||||
assertPrivateDirectory(
|
||||
path.dirname(target),
|
||||
stage.uid,
|
||||
'transformationRoot parent',
|
||||
);
|
||||
if (requireMissing) assertMissing(target);
|
||||
else assertPrivateDirectory(target, stage.uid, 'transformationRoot');
|
||||
return Object.freeze({ ...stage, transformationRoot: target });
|
||||
}
|
||||
|
||||
function assertRelativePath(value: string): void {
|
||||
if (
|
||||
value.length < 1 ||
|
||||
path.isAbsolute(value) ||
|
||||
value === '..' ||
|
||||
value.startsWith(`..${path.sep}`) ||
|
||||
Buffer.byteLength(value, 'utf8') > MAX_RELATIVE_PATH_BYTES
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation relative path is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function privateFileStat(
|
||||
filePath: string,
|
||||
uid: number,
|
||||
label: string,
|
||||
): fs.BigIntStats {
|
||||
const stat = fs.lstatSync(filePath, { bigint: true });
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.isSymbolicLink() ||
|
||||
stat.nlink !== 1n ||
|
||||
stat.uid !== BigInt(uid) ||
|
||||
(stat.mode & 0o777n) !== 0o600n
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
`${label} identity or mode is invalid`,
|
||||
);
|
||||
}
|
||||
return stat;
|
||||
}
|
||||
|
||||
export function optionalPrivateDirectory(
|
||||
directoryPath: string,
|
||||
uid: number,
|
||||
label: string,
|
||||
): fs.BigIntStats | null {
|
||||
try {
|
||||
return assertPrivateDirectory(directoryPath, uid, label);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof LocalDataDirectoryAdoptionConfigurationError &&
|
||||
error.cause &&
|
||||
typeof error.cause === 'object' &&
|
||||
'code' in error.cause &&
|
||||
error.cause.code === 'ENOENT'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function readStablePrivateUtf8File(
|
||||
filePath: string,
|
||||
uid: number,
|
||||
maximumBytes: number,
|
||||
label: string,
|
||||
): string {
|
||||
const expected = privateFileStat(filePath, uid, label);
|
||||
if (
|
||||
expected.size < 0n ||
|
||||
expected.size > BigInt(maximumBytes) ||
|
||||
expected.size > BigInt(Number.MAX_SAFE_INTEGER)
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
`${label} exceeds its byte budget`,
|
||||
);
|
||||
}
|
||||
const descriptor = fs.openSync(
|
||||
filePath,
|
||||
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
let bytes: Buffer | undefined;
|
||||
try {
|
||||
const before = fs.fstatSync(descriptor, { bigint: true });
|
||||
if (!sameStat(expected, before)) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
`${label} identity changed before reading`,
|
||||
);
|
||||
}
|
||||
bytes = fs.readFileSync(descriptor);
|
||||
if (!sameStat(before, fs.fstatSync(descriptor, { bigint: true }))) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
`${label} changed while reading`,
|
||||
);
|
||||
}
|
||||
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalDataDirectoryAdoptionConfigurationError) {
|
||||
throw error;
|
||||
}
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
`${label} is not valid UTF-8`,
|
||||
error,
|
||||
);
|
||||
} finally {
|
||||
bytes?.fill(0);
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
export function writePrivateJson(filePath: string, value: object): void {
|
||||
const descriptor = fs.openSync(filePath, 'wx', 0o600);
|
||||
const bytes = Buffer.from(`${JSON.stringify(value)}\n`, 'utf8');
|
||||
try {
|
||||
let offset = 0;
|
||||
while (offset < bytes.length) {
|
||||
offset += fs.writeSync(
|
||||
descriptor,
|
||||
bytes,
|
||||
offset,
|
||||
bytes.length - offset,
|
||||
null,
|
||||
);
|
||||
}
|
||||
fs.fsyncSync(descriptor);
|
||||
} finally {
|
||||
bytes.fill(0);
|
||||
fs.closeSync(descriptor);
|
||||
}
|
||||
}
|
||||
|
||||
export function summarizePrivateTree(
|
||||
root: string,
|
||||
uid: number,
|
||||
): Readonly<PrivateTreeEvidence> {
|
||||
const rootStat = assertPrivateDirectory(
|
||||
root,
|
||||
uid,
|
||||
'transformation input category',
|
||||
);
|
||||
let entries = 0;
|
||||
let directories = 0;
|
||||
let files = 0;
|
||||
let bytes = 0;
|
||||
const hash = crypto.createHash('sha256');
|
||||
const visit = (directoryPath: string, expected: fs.BigIntStats): void => {
|
||||
for (const name of sortedNames(directoryPath)) {
|
||||
const entryPath = path.join(directoryPath, name);
|
||||
const relative = path.relative(root, entryPath);
|
||||
assertRelativePath(relative);
|
||||
const stat = fs.lstatSync(entryPath, { bigint: true });
|
||||
entries += 1;
|
||||
if (
|
||||
stat.isSymbolicLink() ||
|
||||
stat.uid !== BigInt(uid) ||
|
||||
(stat.mode & 0o777n) !== (stat.isDirectory() ? 0o700n : 0o600n)
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation input identity or mode is invalid',
|
||||
);
|
||||
}
|
||||
const canonical = relative.split(path.sep).join('/');
|
||||
if (stat.isDirectory()) {
|
||||
directories += 1;
|
||||
hash.update(
|
||||
`${JSON.stringify({ relative: canonical, kind: 'directory' })}\n`,
|
||||
);
|
||||
visit(entryPath, stat);
|
||||
} else if (stat.isFile() && stat.nlink === 1n) {
|
||||
if (stat.size < 0n || stat.size > BigInt(Number.MAX_SAFE_INTEGER)) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation input file size is unsupported',
|
||||
);
|
||||
}
|
||||
const count = Number(stat.size);
|
||||
if (!Number.isSafeInteger(bytes + count)) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation input byte total is unsupported',
|
||||
);
|
||||
}
|
||||
bytes += count;
|
||||
files += 1;
|
||||
hash.update(
|
||||
`${JSON.stringify({
|
||||
relative: canonical,
|
||||
kind: 'file',
|
||||
bytes: count,
|
||||
contentDigest: stableFileDigest(entryPath, stat),
|
||||
})}\n`,
|
||||
);
|
||||
} else {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation input entry kind is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!sameStat(expected, fs.lstatSync(directoryPath, { bigint: true }))) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation input directory changed while reading',
|
||||
);
|
||||
}
|
||||
};
|
||||
visit(root, rootStat);
|
||||
return Object.freeze({
|
||||
entries,
|
||||
directories,
|
||||
files,
|
||||
bytes,
|
||||
digest: hash.digest('hex'),
|
||||
});
|
||||
}
|
||||
|
||||
export function finishPrivateDirectory(directoryPath: string): void {
|
||||
syncDirectory(directoryPath);
|
||||
}
|
||||
+351
@@ -0,0 +1,351 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
import { LocalDataDirectoryAdoptionConfigurationError } from '../contract';
|
||||
import { sameStat } from '../filesystem';
|
||||
import { sha256Text } from '../manifest';
|
||||
import {
|
||||
optionalPrivateDirectory,
|
||||
summarizePrivateTree,
|
||||
type PrivateTreeEvidence,
|
||||
} from './files';
|
||||
|
||||
const KNOWN_KEYS = Object.freeze([
|
||||
Object.freeze({
|
||||
key: 'keyv:authInfo',
|
||||
target: 'credential_reissue' as const,
|
||||
}),
|
||||
Object.freeze({
|
||||
key: 'keyv:apps',
|
||||
target: 'main_database_apps_reconciliation' as const,
|
||||
}),
|
||||
Object.freeze({
|
||||
key: 'keyv:lang',
|
||||
target: 'main_database_system_settings_reconciliation' as const,
|
||||
}),
|
||||
]);
|
||||
|
||||
type KeyvTarget = (typeof KNOWN_KEYS)[number]['target'];
|
||||
|
||||
export interface KeyvTransformationModel {
|
||||
readonly schema: 'qinglong/legacy-keyv-transformation@v1';
|
||||
readonly databasePresent: boolean;
|
||||
readonly integrity: 'absent' | 'ok';
|
||||
readonly mappings: readonly Readonly<{
|
||||
legacyKey: string;
|
||||
target: KeyvTarget;
|
||||
state: 'absent' | 'retired' | 'reconcile';
|
||||
valueDigest: string | null;
|
||||
}>[];
|
||||
readonly cachedLocale: 'zh' | 'en' | null;
|
||||
readonly unknownEntries: number;
|
||||
readonly unknownEntryDigest: string;
|
||||
readonly unknownSchemaObjects: number;
|
||||
readonly unknownSchemaDigest: string;
|
||||
readonly disabledAssetEntries: number;
|
||||
readonly activation: 'disabled';
|
||||
}
|
||||
|
||||
export interface KeyvTransformation {
|
||||
readonly source: Readonly<PrivateTreeEvidence> | null;
|
||||
readonly model: Readonly<KeyvTransformationModel>;
|
||||
readonly assessment: 'ready' | 'manual_required';
|
||||
}
|
||||
|
||||
function emptyModel(): Readonly<KeyvTransformationModel> {
|
||||
return Object.freeze({
|
||||
schema: 'qinglong/legacy-keyv-transformation@v1',
|
||||
databasePresent: false,
|
||||
integrity: 'absent',
|
||||
mappings: Object.freeze(
|
||||
KNOWN_KEYS.map((entry) =>
|
||||
Object.freeze({
|
||||
legacyKey: entry.key,
|
||||
target: entry.target,
|
||||
state: 'absent' as const,
|
||||
valueDigest: null,
|
||||
}),
|
||||
),
|
||||
),
|
||||
cachedLocale: null,
|
||||
unknownEntries: 0,
|
||||
unknownEntryDigest: sha256Text(''),
|
||||
unknownSchemaObjects: 0,
|
||||
unknownSchemaDigest: sha256Text(''),
|
||||
disabledAssetEntries: 0,
|
||||
activation: 'disabled',
|
||||
});
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const actual = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return (
|
||||
actual.length === canonical.length &&
|
||||
actual.every((key, index) => key === canonical[index])
|
||||
);
|
||||
}
|
||||
|
||||
function cachedLocale(value: string): 'zh' | 'en' | null {
|
||||
try {
|
||||
const envelope = JSON.parse(value) as unknown;
|
||||
if (
|
||||
!envelope ||
|
||||
typeof envelope !== 'object' ||
|
||||
Array.isArray(envelope) ||
|
||||
!exactKeys(envelope, ['expires', 'value'])
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const candidate = envelope as {
|
||||
readonly expires?: unknown;
|
||||
value?: unknown;
|
||||
};
|
||||
return candidate.expires === null &&
|
||||
(candidate.value === 'zh' || candidate.value === 'en')
|
||||
? candidate.value
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function transformLegacyKeyv(
|
||||
categoryRoot: string,
|
||||
uid: number,
|
||||
profile: 'edge' | 'standalone',
|
||||
): Readonly<KeyvTransformation> {
|
||||
if (
|
||||
!optionalPrivateDirectory(categoryRoot, uid, 'Keyv transformation input')
|
||||
) {
|
||||
return Object.freeze({
|
||||
source: null,
|
||||
model: emptyModel(),
|
||||
assessment: 'ready',
|
||||
});
|
||||
}
|
||||
const source = summarizePrivateTree(categoryRoot, uid);
|
||||
const databasePath = path.join(categoryRoot, 'keyv.sqlite');
|
||||
let expected: fs.BigIntStats;
|
||||
try {
|
||||
expected = fs.lstatSync(databasePath, { bigint: true });
|
||||
} catch (error) {
|
||||
if (
|
||||
error &&
|
||||
typeof error === 'object' &&
|
||||
'code' in error &&
|
||||
error.code === 'ENOENT'
|
||||
) {
|
||||
return Object.freeze({
|
||||
source,
|
||||
model: Object.freeze({
|
||||
...emptyModel(),
|
||||
disabledAssetEntries: source.entries,
|
||||
}),
|
||||
assessment: source.entries === 0 ? 'ready' : 'manual_required',
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (
|
||||
!expected.isFile() ||
|
||||
expected.isSymbolicLink() ||
|
||||
expected.nlink !== 1n ||
|
||||
expected.uid !== BigInt(uid) ||
|
||||
(expected.mode & 0o777n) !== 0o600n
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'staged Keyv database identity or mode is invalid',
|
||||
);
|
||||
}
|
||||
|
||||
const client = new DatabaseSync(databasePath, {
|
||||
allowExtension: false,
|
||||
defensive: true,
|
||||
enableDoubleQuotedStringLiterals: false,
|
||||
enableForeignKeyConstraints: true,
|
||||
readOnly: true,
|
||||
timeout: 5_000,
|
||||
});
|
||||
const values = new Map<string, string>();
|
||||
let unknownEntries = 0;
|
||||
const unknownEntryHash = crypto.createHash('sha256');
|
||||
let unknownSchemaObjects = 0;
|
||||
const unknownSchemaHash = crypto.createHash('sha256');
|
||||
try {
|
||||
client.enableDefensive(true);
|
||||
client.exec(
|
||||
`PRAGMA trusted_schema = OFF; PRAGMA query_only = ON; PRAGMA mmap_size = 0; PRAGMA cache_size = ${
|
||||
profile === 'edge' ? -2048 : -8192
|
||||
}`,
|
||||
);
|
||||
const integrity = client.prepare('PRAGMA integrity_check(1)').get() as
|
||||
| { readonly integrity_check?: unknown }
|
||||
| undefined;
|
||||
if (integrity?.integrity_check !== 'ok') {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'staged Keyv database integrity check failed',
|
||||
);
|
||||
}
|
||||
const tables = client.prepare(`PRAGMA table_list('keyv')`).all() as Array<{
|
||||
readonly schema?: unknown;
|
||||
readonly name?: unknown;
|
||||
readonly type?: unknown;
|
||||
readonly ncol?: unknown;
|
||||
readonly wr?: unknown;
|
||||
readonly strict?: unknown;
|
||||
}>;
|
||||
if (
|
||||
tables.length !== 1 ||
|
||||
tables[0]?.schema !== 'main' ||
|
||||
tables[0]?.name !== 'keyv' ||
|
||||
tables[0]?.type !== 'table' ||
|
||||
tables[0]?.ncol !== 2 ||
|
||||
tables[0]?.wr !== 0 ||
|
||||
tables[0]?.strict !== 0
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'staged Keyv database is not a reviewed ordinary table',
|
||||
);
|
||||
}
|
||||
const columns = client.prepare(`PRAGMA table_info('keyv')`).all() as Array<{
|
||||
readonly name?: unknown;
|
||||
readonly type?: unknown;
|
||||
readonly pk?: unknown;
|
||||
}>;
|
||||
if (
|
||||
columns.length !== 2 ||
|
||||
columns[0]?.name !== 'key' ||
|
||||
columns[0]?.type !== 'VARCHAR(255)' ||
|
||||
columns[0]?.pk !== 1 ||
|
||||
columns[1]?.name !== 'value' ||
|
||||
columns[1]?.type !== 'TEXT' ||
|
||||
columns[1]?.pk !== 0
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'staged Keyv database schema is not the reviewed v4 shape',
|
||||
);
|
||||
}
|
||||
const schema = client
|
||||
.prepare(
|
||||
`SELECT type, name, tbl_name AS tableName, sql
|
||||
FROM sqlite_schema
|
||||
WHERE name NOT LIKE 'sqlite_%'
|
||||
ORDER BY CAST(name AS BLOB)`,
|
||||
)
|
||||
.all() as Array<{
|
||||
readonly type?: unknown;
|
||||
readonly name?: unknown;
|
||||
readonly tableName?: unknown;
|
||||
readonly sql?: unknown;
|
||||
}>;
|
||||
for (const object of schema) {
|
||||
if (
|
||||
object.type === 'table' &&
|
||||
object.name === 'keyv' &&
|
||||
object.tableName === 'keyv'
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
unknownSchemaObjects += 1;
|
||||
unknownSchemaHash.update(
|
||||
`${sha256Text(JSON.stringify(object))}\n`,
|
||||
'utf8',
|
||||
);
|
||||
}
|
||||
const rowBudget = profile === 'edge' ? 256 : 2_048;
|
||||
const byteBudget = profile === 'edge' ? 4 * 1024 * 1024 : 16 * 1024 * 1024;
|
||||
let rows = 0;
|
||||
let bytes = 0;
|
||||
const statement = client.prepare(
|
||||
`SELECT key, value, length(CAST(value AS BLOB)) AS valueBytes
|
||||
FROM keyv
|
||||
ORDER BY CAST(key AS BLOB)`,
|
||||
);
|
||||
for (const row of statement.iterate() as Iterable<{
|
||||
readonly key?: unknown;
|
||||
readonly value?: unknown;
|
||||
readonly valueBytes?: unknown;
|
||||
}>) {
|
||||
rows += 1;
|
||||
if (
|
||||
rows > rowBudget ||
|
||||
typeof row.key !== 'string' ||
|
||||
Buffer.byteLength(row.key, 'utf8') > 255 ||
|
||||
typeof row.value !== 'string' ||
|
||||
!Number.isSafeInteger(row.valueBytes) ||
|
||||
(row.valueBytes as number) < 0 ||
|
||||
(row.valueBytes as number) > 1024 * 1024 ||
|
||||
!Number.isSafeInteger(bytes + (row.valueBytes as number)) ||
|
||||
bytes + (row.valueBytes as number) > byteBudget
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'staged Keyv data exceeds the Profile budget',
|
||||
);
|
||||
}
|
||||
bytes += row.valueBytes as number;
|
||||
if (KNOWN_KEYS.some((entry) => entry.key === row.key)) {
|
||||
values.set(row.key, row.value);
|
||||
} else {
|
||||
unknownEntries += 1;
|
||||
unknownEntryHash.update(
|
||||
`${sha256Text(row.key)}:${sha256Text(row.value)}\n`,
|
||||
'utf8',
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
client.close();
|
||||
}
|
||||
if (!sameStat(expected, fs.lstatSync(databasePath, { bigint: true }))) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'staged Keyv database changed while transforming',
|
||||
);
|
||||
}
|
||||
|
||||
const localeValue = values.get('keyv:lang');
|
||||
const locale = localeValue === undefined ? null : cachedLocale(localeValue);
|
||||
const mappings = Object.freeze(
|
||||
KNOWN_KEYS.map((entry) => {
|
||||
const value = values.get(entry.key);
|
||||
return Object.freeze({
|
||||
legacyKey: entry.key,
|
||||
target: entry.target,
|
||||
state:
|
||||
value === undefined
|
||||
? ('absent' as const)
|
||||
: entry.key === 'keyv:authInfo'
|
||||
? ('retired' as const)
|
||||
: ('reconcile' as const),
|
||||
valueDigest: value === undefined ? null : sha256Text(value),
|
||||
});
|
||||
}),
|
||||
);
|
||||
const disabledAssetEntries = Math.max(0, source.entries - 1);
|
||||
const model = Object.freeze({
|
||||
schema: 'qinglong/legacy-keyv-transformation@v1' as const,
|
||||
databasePresent: true,
|
||||
integrity: 'ok' as const,
|
||||
mappings,
|
||||
cachedLocale: locale,
|
||||
unknownEntries,
|
||||
unknownEntryDigest: unknownEntryHash.digest('hex'),
|
||||
unknownSchemaObjects,
|
||||
unknownSchemaDigest: unknownSchemaHash.digest('hex'),
|
||||
disabledAssetEntries,
|
||||
activation: 'disabled' as const,
|
||||
});
|
||||
return Object.freeze({
|
||||
source,
|
||||
model,
|
||||
assessment:
|
||||
unknownEntries > 0 ||
|
||||
unknownSchemaObjects > 0 ||
|
||||
disabledAssetEntries > 0 ||
|
||||
(localeValue !== undefined && locale === null)
|
||||
? 'manual_required'
|
||||
: 'ready',
|
||||
});
|
||||
}
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import { LocalDataDirectoryAdoptionConfigurationError } from '../contract';
|
||||
import { assertPrivateDirectory, sameStat, sortedNames } from '../filesystem';
|
||||
import { sha256Text } from '../manifest';
|
||||
import {
|
||||
readStablePrivateUtf8File,
|
||||
type TransformationAuthority,
|
||||
} from './files';
|
||||
import {
|
||||
verifyTransformationModel,
|
||||
type LocalDataDirectoryTransformationManifest,
|
||||
type TransformationModelEvidence,
|
||||
type TransformationSourceEvidence,
|
||||
} from './model';
|
||||
|
||||
export const TRANSFORMATION_MANIFEST_NAME = 'manifest.json';
|
||||
const MAX_MANIFEST_BYTES = 64 * 1024;
|
||||
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
|
||||
const SOURCE_NAMES = Object.freeze(['config', 'keyv', 'ssh'] as const);
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const actual = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return (
|
||||
actual.length === canonical.length &&
|
||||
actual.every((key, index) => key === canonical[index])
|
||||
);
|
||||
}
|
||||
|
||||
function safeCount(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && (value as number) >= 0;
|
||||
}
|
||||
|
||||
function treeEvidence(
|
||||
value: unknown,
|
||||
extraKeys: readonly string[],
|
||||
): value is Record<string, unknown> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!exactKeys(value, [
|
||||
'bytes',
|
||||
'digest',
|
||||
'directories',
|
||||
'entries',
|
||||
'files',
|
||||
...extraKeys,
|
||||
])
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const candidate = value as Record<string, unknown>;
|
||||
return (
|
||||
[
|
||||
candidate.entries,
|
||||
candidate.directories,
|
||||
candidate.files,
|
||||
candidate.bytes,
|
||||
...extraKeys.map((key) => candidate[key]),
|
||||
].every(safeCount) &&
|
||||
candidate.entries ===
|
||||
(candidate.directories as number) + (candidate.files as number) &&
|
||||
typeof candidate.digest === 'string' &&
|
||||
DIGEST_PATTERN.test(candidate.digest)
|
||||
);
|
||||
}
|
||||
|
||||
function sourceEvidence(
|
||||
value: unknown,
|
||||
expectedName: (typeof SOURCE_NAMES)[number],
|
||||
): value is TransformationSourceEvidence {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!exactKeys(value, [
|
||||
'assessment',
|
||||
'bytes',
|
||||
'digest',
|
||||
'directories',
|
||||
'entries',
|
||||
'files',
|
||||
'name',
|
||||
'present',
|
||||
])
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const candidate = value as Record<string, unknown>;
|
||||
return (
|
||||
treeEvidence(
|
||||
{
|
||||
bytes: candidate.bytes,
|
||||
digest: candidate.digest,
|
||||
directories: candidate.directories,
|
||||
entries: candidate.entries,
|
||||
files: candidate.files,
|
||||
},
|
||||
[],
|
||||
) &&
|
||||
candidate.name === expectedName &&
|
||||
typeof candidate.present === 'boolean' &&
|
||||
(candidate.assessment === 'ready' ||
|
||||
candidate.assessment === 'manual_required') &&
|
||||
(candidate.present || candidate.entries === 0)
|
||||
);
|
||||
}
|
||||
|
||||
function parseManifest(
|
||||
value: unknown,
|
||||
): LocalDataDirectoryTransformationManifest {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!exactKeys(value, [
|
||||
'assessment',
|
||||
'createdAtMs',
|
||||
'kind',
|
||||
'model',
|
||||
'profile',
|
||||
'projectIdDigest',
|
||||
'schemaVersion',
|
||||
'sourceStageManifestDigest',
|
||||
'sources',
|
||||
'state',
|
||||
'transformationDigest',
|
||||
'transformationRootPathDigest',
|
||||
])
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation manifest shape is invalid',
|
||||
);
|
||||
}
|
||||
const candidate = value as Record<string, unknown>;
|
||||
if (
|
||||
candidate.schemaVersion !== 1 ||
|
||||
candidate.kind !== 'qinglong3-legacy-data-directory-transformation' ||
|
||||
candidate.state !== 'prepared' ||
|
||||
(candidate.profile !== 'edge' && candidate.profile !== 'standalone') ||
|
||||
(candidate.assessment !== 'ready' &&
|
||||
candidate.assessment !== 'manual_required') ||
|
||||
!safeCount(candidate.createdAtMs) ||
|
||||
![
|
||||
candidate.projectIdDigest,
|
||||
candidate.sourceStageManifestDigest,
|
||||
candidate.transformationDigest,
|
||||
candidate.transformationRootPathDigest,
|
||||
].every(
|
||||
(digest) => typeof digest === 'string' && DIGEST_PATTERN.test(digest),
|
||||
) ||
|
||||
!Array.isArray(candidate.sources) ||
|
||||
candidate.sources.length !== SOURCE_NAMES.length ||
|
||||
!candidate.sources.every((entry, index) =>
|
||||
sourceEvidence(entry, SOURCE_NAMES[index]!),
|
||||
) ||
|
||||
!treeEvidence(candidate.model, [
|
||||
'environmentSecrets',
|
||||
'manualCategories',
|
||||
'sshSecrets',
|
||||
])
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation manifest value is invalid',
|
||||
);
|
||||
}
|
||||
const { transformationDigest, ...payload } =
|
||||
candidate as unknown as LocalDataDirectoryTransformationManifest;
|
||||
if (sha256Text(JSON.stringify(payload)) !== transformationDigest) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation manifest digest does not match',
|
||||
);
|
||||
}
|
||||
return candidate as unknown as LocalDataDirectoryTransformationManifest;
|
||||
}
|
||||
|
||||
function readManifest(
|
||||
root: string,
|
||||
uid: number,
|
||||
): Readonly<LocalDataDirectoryTransformationManifest> {
|
||||
try {
|
||||
return parseManifest(
|
||||
JSON.parse(
|
||||
readStablePrivateUtf8File(
|
||||
path.join(root, TRANSFORMATION_MANIFEST_NAME),
|
||||
uid,
|
||||
MAX_MANIFEST_BYTES,
|
||||
'transformation manifest',
|
||||
),
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalDataDirectoryAdoptionConfigurationError) {
|
||||
throw error;
|
||||
}
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation manifest JSON is invalid',
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function verifyStaticTransformation(options: {
|
||||
readonly authority: Readonly<TransformationAuthority>;
|
||||
readonly profile: 'edge' | 'standalone';
|
||||
readonly projectId: string;
|
||||
readonly sourceStageManifestDigest: string;
|
||||
readonly expectedTransformationDigest: string;
|
||||
}): Readonly<LocalDataDirectoryTransformationManifest> {
|
||||
const before = assertPrivateDirectory(
|
||||
options.authority.transformationRoot,
|
||||
options.authority.uid,
|
||||
'transformationRoot',
|
||||
);
|
||||
if (
|
||||
JSON.stringify(sortedNames(options.authority.transformationRoot)) !==
|
||||
JSON.stringify([TRANSFORMATION_MANIFEST_NAME, 'model'].sort())
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation root is incomplete or contains unexpected entries',
|
||||
);
|
||||
}
|
||||
const manifest = readManifest(
|
||||
options.authority.transformationRoot,
|
||||
options.authority.uid,
|
||||
);
|
||||
if (
|
||||
manifest.transformationDigest !== options.expectedTransformationDigest ||
|
||||
manifest.profile !== options.profile ||
|
||||
manifest.projectIdDigest !== sha256Text(options.projectId) ||
|
||||
manifest.sourceStageManifestDigest !== options.sourceStageManifestDigest ||
|
||||
manifest.transformationRootPathDigest !==
|
||||
sha256Text(options.authority.transformationRoot)
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation manifest authority binding is invalid',
|
||||
);
|
||||
}
|
||||
verifyTransformationModel({
|
||||
modelRoot: path.join(options.authority.transformationRoot, 'model'),
|
||||
uid: options.authority.uid,
|
||||
projectId: options.projectId,
|
||||
profile: options.profile,
|
||||
expected: manifest.model as Readonly<TransformationModelEvidence>,
|
||||
});
|
||||
if (
|
||||
!sameStat(
|
||||
before,
|
||||
fs.lstatSync(options.authority.transformationRoot, { bigint: true }),
|
||||
)
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation root changed during verification',
|
||||
);
|
||||
}
|
||||
return manifest;
|
||||
}
|
||||
+430
@@ -0,0 +1,430 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import { LocalDataDirectoryAdoptionConfigurationError } from '../contract';
|
||||
import { sortedNames, syncDirectory } from '../filesystem';
|
||||
import { sha256Text } from '../manifest';
|
||||
import type { ConfigTransformation, SecretImportDraft } from './config';
|
||||
import {
|
||||
readStablePrivateUtf8File,
|
||||
summarizePrivateTree,
|
||||
writePrivateJson,
|
||||
type PrivateTreeEvidence,
|
||||
} from './files';
|
||||
import type { KeyvTransformation } from './keyv';
|
||||
import type { SshTransformation } from './ssh';
|
||||
|
||||
const MAX_MODEL_FILE_BYTES = 1024 * 1024;
|
||||
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
|
||||
const SECRET_FILE_PATTERN = /^secret-values\/[0-9a-f]{64}\.json$/;
|
||||
|
||||
export interface TransformationSourceEvidence extends PrivateTreeEvidence {
|
||||
readonly name: 'config' | 'keyv' | 'ssh';
|
||||
readonly present: boolean;
|
||||
readonly assessment: 'ready' | 'manual_required';
|
||||
}
|
||||
|
||||
export interface TransformationModelEvidence extends PrivateTreeEvidence {
|
||||
readonly environmentSecrets: number;
|
||||
readonly sshSecrets: number;
|
||||
readonly manualCategories: number;
|
||||
}
|
||||
|
||||
export interface LocalDataDirectoryTransformationManifestPayload {
|
||||
readonly schemaVersion: 1;
|
||||
readonly kind: 'qinglong3-legacy-data-directory-transformation';
|
||||
readonly state: 'prepared';
|
||||
readonly profile: 'edge' | 'standalone';
|
||||
readonly createdAtMs: number;
|
||||
readonly projectIdDigest: string;
|
||||
readonly sourceStageManifestDigest: string;
|
||||
readonly transformationRootPathDigest: string;
|
||||
readonly assessment: 'ready' | 'manual_required';
|
||||
readonly sources: readonly TransformationSourceEvidence[];
|
||||
readonly model: Readonly<TransformationModelEvidence>;
|
||||
}
|
||||
|
||||
export interface LocalDataDirectoryTransformationManifest
|
||||
extends LocalDataDirectoryTransformationManifestPayload {
|
||||
readonly transformationDigest: string;
|
||||
}
|
||||
|
||||
interface SecretImportEntry {
|
||||
readonly kind: SecretImportDraft['kind'];
|
||||
readonly sourceName: string;
|
||||
readonly targetName: string;
|
||||
readonly expectedCurrentVersion: 0;
|
||||
readonly valueFile: string;
|
||||
readonly valueDigest: string;
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const actual = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return (
|
||||
actual.length === canonical.length &&
|
||||
actual.every((key, index) => key === canonical[index])
|
||||
);
|
||||
}
|
||||
|
||||
function emptyEvidence(): Readonly<PrivateTreeEvidence> {
|
||||
return Object.freeze({
|
||||
entries: 0,
|
||||
directories: 0,
|
||||
files: 0,
|
||||
bytes: 0,
|
||||
digest: sha256Text(''),
|
||||
});
|
||||
}
|
||||
|
||||
function sourceEvidence(
|
||||
name: TransformationSourceEvidence['name'],
|
||||
transformation:
|
||||
| Readonly<ConfigTransformation>
|
||||
| Readonly<KeyvTransformation>
|
||||
| Readonly<SshTransformation>,
|
||||
): Readonly<TransformationSourceEvidence> {
|
||||
return Object.freeze({
|
||||
name,
|
||||
present: transformation.source !== null,
|
||||
...(transformation.source ?? emptyEvidence()),
|
||||
assessment: transformation.assessment,
|
||||
});
|
||||
}
|
||||
|
||||
function secretId(entry: Readonly<SecretImportDraft>): string {
|
||||
return sha256Text(`${entry.kind}\0${entry.sourceName}\0${entry.targetName}`);
|
||||
}
|
||||
|
||||
export function writeTransformationModel(options: {
|
||||
readonly modelRoot: string;
|
||||
readonly uid: number;
|
||||
readonly projectId: string;
|
||||
readonly profile: 'edge' | 'standalone';
|
||||
readonly config: Readonly<ConfigTransformation>;
|
||||
readonly keyv: Readonly<KeyvTransformation>;
|
||||
readonly ssh: Readonly<SshTransformation>;
|
||||
}): Readonly<{
|
||||
sources: readonly TransformationSourceEvidence[];
|
||||
model: Readonly<TransformationModelEvidence>;
|
||||
assessment: 'ready' | 'manual_required';
|
||||
}> {
|
||||
fs.mkdirSync(options.modelRoot, { mode: 0o700 });
|
||||
const secretRoot = path.join(options.modelRoot, 'secret-values');
|
||||
fs.mkdirSync(secretRoot, { mode: 0o700 });
|
||||
const drafts = [...options.config.secrets, ...options.ssh.secrets].sort(
|
||||
(left, right) =>
|
||||
Buffer.compare(
|
||||
Buffer.from(`${left.kind}\0${left.sourceName}`, 'utf8'),
|
||||
Buffer.from(`${right.kind}\0${right.sourceName}`, 'utf8'),
|
||||
),
|
||||
);
|
||||
const maximumSecrets = options.profile === 'edge' ? 128 : 512;
|
||||
if (drafts.length > maximumSecrets) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation Secret count exceeds the Profile budget',
|
||||
);
|
||||
}
|
||||
const targets = new Set<string>();
|
||||
const files = new Set<string>();
|
||||
const imports: SecretImportEntry[] = [];
|
||||
for (const draft of drafts) {
|
||||
const id = secretId(draft);
|
||||
const relative = `secret-values/${id}.json`;
|
||||
if (targets.has(draft.targetName) || files.has(relative)) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation Secret identity collides',
|
||||
);
|
||||
}
|
||||
targets.add(draft.targetName);
|
||||
files.add(relative);
|
||||
writePrivateJson(path.join(options.modelRoot, relative), {
|
||||
schemaVersion: 1,
|
||||
kind: 'qinglong3-local-secret-value',
|
||||
value: draft.value,
|
||||
});
|
||||
imports.push({
|
||||
kind: draft.kind,
|
||||
sourceName: draft.sourceName,
|
||||
targetName: draft.targetName,
|
||||
expectedCurrentVersion: 0,
|
||||
valueFile: relative,
|
||||
valueDigest: sha256Text(draft.value),
|
||||
});
|
||||
}
|
||||
syncDirectory(secretRoot);
|
||||
writePrivateJson(
|
||||
path.join(options.modelRoot, 'config.json'),
|
||||
options.config.model,
|
||||
);
|
||||
writePrivateJson(
|
||||
path.join(options.modelRoot, 'keyv.json'),
|
||||
options.keyv.model,
|
||||
);
|
||||
writePrivateJson(path.join(options.modelRoot, 'ssh.json'), options.ssh.model);
|
||||
writePrivateJson(path.join(options.modelRoot, 'secret-imports.json'), {
|
||||
schema: 'qinglong/local-secret-import-plan@v1',
|
||||
projectId: options.projectId,
|
||||
state: 'prepared',
|
||||
imports,
|
||||
});
|
||||
const sources = Object.freeze([
|
||||
sourceEvidence('config', options.config),
|
||||
sourceEvidence('keyv', options.keyv),
|
||||
sourceEvidence('ssh', options.ssh),
|
||||
]);
|
||||
const manualCategories = sources.filter(
|
||||
({ assessment }) => assessment === 'manual_required',
|
||||
).length;
|
||||
writePrivateJson(path.join(options.modelRoot, 'manual-review.json'), {
|
||||
schema: 'qinglong/legacy-data-directory-manual-review@v1',
|
||||
required: manualCategories > 0,
|
||||
categories: sources.map(({ name, present, assessment, ...evidence }) => ({
|
||||
name,
|
||||
present,
|
||||
assessment,
|
||||
evidence,
|
||||
})),
|
||||
activation: 'disabled',
|
||||
});
|
||||
syncDirectory(options.modelRoot);
|
||||
const tree = summarizePrivateTree(options.modelRoot, options.uid);
|
||||
const environmentSecrets = imports.filter(
|
||||
({ kind }) => kind === 'environment',
|
||||
).length;
|
||||
const sshSecrets = imports.length - environmentSecrets;
|
||||
return Object.freeze({
|
||||
sources,
|
||||
model: Object.freeze({
|
||||
...tree,
|
||||
environmentSecrets,
|
||||
sshSecrets,
|
||||
manualCategories,
|
||||
}),
|
||||
assessment: manualCategories > 0 ? 'manual_required' : 'ready',
|
||||
});
|
||||
}
|
||||
|
||||
function readJson(filePath: string, uid: number): unknown {
|
||||
try {
|
||||
return JSON.parse(
|
||||
readStablePrivateUtf8File(
|
||||
filePath,
|
||||
uid,
|
||||
MAX_MODEL_FILE_BYTES,
|
||||
'transformation model file',
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalDataDirectoryAdoptionConfigurationError) {
|
||||
throw error;
|
||||
}
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation model JSON is invalid',
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertSchemaFile(
|
||||
filePath: string,
|
||||
uid: number,
|
||||
schema: string,
|
||||
): Record<string, unknown> {
|
||||
const value = readJson(filePath, uid);
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
(value as { readonly schema?: unknown }).schema !== schema ||
|
||||
(value as { readonly activation?: unknown }).activation !== 'disabled'
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation target model schema is invalid',
|
||||
);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function verifyTransformationModel(options: {
|
||||
readonly modelRoot: string;
|
||||
readonly uid: number;
|
||||
readonly projectId: string;
|
||||
readonly profile: 'edge' | 'standalone';
|
||||
readonly expected: Readonly<TransformationModelEvidence>;
|
||||
}): void {
|
||||
if (
|
||||
JSON.stringify(sortedNames(options.modelRoot)) !==
|
||||
JSON.stringify(
|
||||
[
|
||||
'config.json',
|
||||
'keyv.json',
|
||||
'manual-review.json',
|
||||
'secret-imports.json',
|
||||
'secret-values',
|
||||
'ssh.json',
|
||||
].sort(),
|
||||
)
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation model root contains unexpected entries',
|
||||
);
|
||||
}
|
||||
assertSchemaFile(
|
||||
path.join(options.modelRoot, 'config.json'),
|
||||
options.uid,
|
||||
'qinglong/legacy-config-transformation@v1',
|
||||
);
|
||||
assertSchemaFile(
|
||||
path.join(options.modelRoot, 'keyv.json'),
|
||||
options.uid,
|
||||
'qinglong/legacy-keyv-transformation@v1',
|
||||
);
|
||||
assertSchemaFile(
|
||||
path.join(options.modelRoot, 'ssh.json'),
|
||||
options.uid,
|
||||
'qinglong/legacy-ssh-transformation@v1',
|
||||
);
|
||||
const manual = assertSchemaFile(
|
||||
path.join(options.modelRoot, 'manual-review.json'),
|
||||
options.uid,
|
||||
'qinglong/legacy-data-directory-manual-review@v1',
|
||||
);
|
||||
if (
|
||||
!exactKeys(manual, ['activation', 'categories', 'required', 'schema']) ||
|
||||
typeof manual.required !== 'boolean' ||
|
||||
!Array.isArray(manual.categories) ||
|
||||
manual.categories.length !== 3
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'manual-review model is invalid',
|
||||
);
|
||||
}
|
||||
const plan = readJson(
|
||||
path.join(options.modelRoot, 'secret-imports.json'),
|
||||
options.uid,
|
||||
);
|
||||
if (
|
||||
!plan ||
|
||||
typeof plan !== 'object' ||
|
||||
Array.isArray(plan) ||
|
||||
!exactKeys(plan, ['imports', 'projectId', 'schema', 'state'])
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'Secret import plan shape is invalid',
|
||||
);
|
||||
}
|
||||
const candidate = plan as Record<string, unknown>;
|
||||
const maximumSecrets = options.profile === 'edge' ? 128 : 512;
|
||||
if (
|
||||
candidate.schema !== 'qinglong/local-secret-import-plan@v1' ||
|
||||
candidate.projectId !== options.projectId ||
|
||||
candidate.state !== 'prepared' ||
|
||||
!Array.isArray(candidate.imports) ||
|
||||
candidate.imports.length > maximumSecrets
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'Secret import plan value is invalid',
|
||||
);
|
||||
}
|
||||
const expectedFiles: string[] = [];
|
||||
const targets = new Set<string>();
|
||||
let environmentSecrets = 0;
|
||||
let sshSecrets = 0;
|
||||
for (const value of candidate.imports) {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!exactKeys(value, [
|
||||
'expectedCurrentVersion',
|
||||
'kind',
|
||||
'sourceName',
|
||||
'targetName',
|
||||
'valueDigest',
|
||||
'valueFile',
|
||||
])
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'Secret import entry shape is invalid',
|
||||
);
|
||||
}
|
||||
const entry = value as Record<string, unknown>;
|
||||
if (
|
||||
(entry.kind !== 'environment' && entry.kind !== 'ssh_private_key') ||
|
||||
typeof entry.sourceName !== 'string' ||
|
||||
typeof entry.targetName !== 'string' ||
|
||||
entry.expectedCurrentVersion !== 0 ||
|
||||
typeof entry.valueFile !== 'string' ||
|
||||
!SECRET_FILE_PATTERN.test(entry.valueFile) ||
|
||||
typeof entry.valueDigest !== 'string' ||
|
||||
!DIGEST_PATTERN.test(entry.valueDigest) ||
|
||||
targets.has(entry.targetName)
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'Secret import entry value is invalid',
|
||||
);
|
||||
}
|
||||
targets.add(entry.targetName);
|
||||
expectedFiles.push(path.basename(entry.valueFile));
|
||||
const secret = readJson(
|
||||
path.join(options.modelRoot, entry.valueFile),
|
||||
options.uid,
|
||||
);
|
||||
if (
|
||||
!secret ||
|
||||
typeof secret !== 'object' ||
|
||||
Array.isArray(secret) ||
|
||||
!exactKeys(secret, ['kind', 'schemaVersion', 'value'])
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'Secret value file shape is invalid',
|
||||
);
|
||||
}
|
||||
const secretValue = secret as Record<string, unknown>;
|
||||
if (
|
||||
secretValue.schemaVersion !== 1 ||
|
||||
secretValue.kind !== 'qinglong3-local-secret-value' ||
|
||||
typeof secretValue.value !== 'string' ||
|
||||
secretValue.value.includes('\0') ||
|
||||
Buffer.byteLength(secretValue.value, 'utf8') > 16 * 1024 ||
|
||||
sha256Text(secretValue.value) !== entry.valueDigest
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'Secret value file is invalid',
|
||||
);
|
||||
}
|
||||
if (entry.kind === 'environment') environmentSecrets += 1;
|
||||
else sshSecrets += 1;
|
||||
}
|
||||
expectedFiles.sort();
|
||||
if (
|
||||
JSON.stringify(
|
||||
sortedNames(path.join(options.modelRoot, 'secret-values')),
|
||||
) !== JSON.stringify(expectedFiles)
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'Secret value file set is invalid',
|
||||
);
|
||||
}
|
||||
const actual = summarizePrivateTree(options.modelRoot, options.uid);
|
||||
if (
|
||||
JSON.stringify({
|
||||
...actual,
|
||||
environmentSecrets,
|
||||
sshSecrets,
|
||||
manualCategories: (manual.categories as unknown[]).filter(
|
||||
(entry) =>
|
||||
!!entry &&
|
||||
typeof entry === 'object' &&
|
||||
!Array.isArray(entry) &&
|
||||
(entry as { readonly assessment?: unknown }).assessment ===
|
||||
'manual_required',
|
||||
).length,
|
||||
}) !== JSON.stringify(options.expected)
|
||||
) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'transformation model no longer matches the manifest',
|
||||
);
|
||||
}
|
||||
}
|
||||
+224
@@ -0,0 +1,224 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import { LocalDataDirectoryAdoptionConfigurationError } from '../contract';
|
||||
import { sortedNames } from '../filesystem';
|
||||
import { sha256Text } from '../manifest';
|
||||
import {
|
||||
optionalPrivateDirectory,
|
||||
readStablePrivateUtf8File,
|
||||
summarizePrivateTree,
|
||||
type PrivateTreeEvidence,
|
||||
} from './files';
|
||||
import type { SecretImportDraft } from './config';
|
||||
|
||||
const MAX_KEY_BYTES = 16 * 1024;
|
||||
const MAX_CONFIG_BYTES = 64 * 1024;
|
||||
const MAX_ALIAS_BYTES = 128;
|
||||
const KEY_KINDS = new Set([
|
||||
'OPENSSH PRIVATE KEY',
|
||||
'RSA PRIVATE KEY',
|
||||
'EC PRIVATE KEY',
|
||||
'PRIVATE KEY',
|
||||
]);
|
||||
|
||||
export interface SshTransformationModel {
|
||||
readonly schema: 'qinglong/legacy-ssh-transformation@v1';
|
||||
readonly bindings: readonly Readonly<{
|
||||
alias: string;
|
||||
legacyHostPattern: string;
|
||||
targetSecretName: string;
|
||||
legacyConfigDigest: string;
|
||||
legacyProxyCommandPresent: boolean;
|
||||
legacyHostKeyBypassPresent: boolean;
|
||||
hostKeyPolicy: 'operator_verification_required';
|
||||
activation: 'disabled';
|
||||
}>[];
|
||||
readonly manualEntries: number;
|
||||
readonly manualEntryDigest: string;
|
||||
readonly activation: 'disabled';
|
||||
}
|
||||
|
||||
export interface SshTransformation {
|
||||
readonly source: Readonly<PrivateTreeEvidence> | null;
|
||||
readonly model: Readonly<SshTransformationModel>;
|
||||
readonly secrets: readonly Readonly<SecretImportDraft>[];
|
||||
readonly assessment: 'ready' | 'manual_required';
|
||||
}
|
||||
|
||||
function targetSecretName(alias: string): string {
|
||||
return `legacy-ssh-${sha256Text(alias).slice(0, 32)}`;
|
||||
}
|
||||
|
||||
function privateKey(value: string): boolean {
|
||||
if (value.includes('\0')) return false;
|
||||
const lines = value.trimEnd().split('\n');
|
||||
if (lines.length < 3) return false;
|
||||
const begin = /^-----BEGIN ([A-Z0-9 ]+)-----$/.exec(lines[0]!);
|
||||
const end = /^-----END ([A-Z0-9 ]+)-----$/.exec(lines.at(-1)!);
|
||||
return !!begin && !!end && begin[1] === end[1] && KEY_KINDS.has(begin[1]!);
|
||||
}
|
||||
|
||||
function configBinding(
|
||||
value: string,
|
||||
alias: string,
|
||||
): Readonly<{
|
||||
hostPattern: string;
|
||||
proxyCommandPresent: boolean;
|
||||
hostKeyBypassPresent: boolean;
|
||||
}> | null {
|
||||
if (
|
||||
value.includes('\0') ||
|
||||
/[\u0001-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(value)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const lines = value.split(/\r?\n/);
|
||||
const hosts = lines
|
||||
.map((line) => /^Host[ \t]+([^\s]+)[ \t]*$/.exec(line)?.[1])
|
||||
.filter((entry): entry is string => entry !== undefined);
|
||||
const identities = lines
|
||||
.map((line) => /^[ \t]+IdentityFile[ \t]+(.+?)[ \t]*$/.exec(line)?.[1])
|
||||
.filter((entry): entry is string => entry !== undefined);
|
||||
if (
|
||||
hosts.length !== 1 ||
|
||||
identities.length !== 1 ||
|
||||
Buffer.byteLength(hosts[0]!, 'utf8') > 255 ||
|
||||
path.basename(identities[0]!) !== alias
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return Object.freeze({
|
||||
hostPattern: hosts[0]!,
|
||||
proxyCommandPresent: lines.some((line) =>
|
||||
/^[ \t]+ProxyCommand[ \t]+/.test(line),
|
||||
),
|
||||
hostKeyBypassPresent: lines.some((line) =>
|
||||
/^[ \t]+StrictHostKeyChecking[ \t]+no[ \t]*$/i.test(line),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function emptyModel(): Readonly<SshTransformationModel> {
|
||||
return Object.freeze({
|
||||
schema: 'qinglong/legacy-ssh-transformation@v1',
|
||||
bindings: Object.freeze([]),
|
||||
manualEntries: 0,
|
||||
manualEntryDigest: sha256Text(''),
|
||||
activation: 'disabled',
|
||||
});
|
||||
}
|
||||
|
||||
export function transformLegacySsh(
|
||||
categoryRoot: string,
|
||||
uid: number,
|
||||
): Readonly<SshTransformation> {
|
||||
if (
|
||||
!optionalPrivateDirectory(categoryRoot, uid, 'SSH transformation input')
|
||||
) {
|
||||
return Object.freeze({
|
||||
source: null,
|
||||
model: emptyModel(),
|
||||
secrets: Object.freeze([]),
|
||||
assessment: 'ready',
|
||||
});
|
||||
}
|
||||
const source = summarizePrivateTree(categoryRoot, uid);
|
||||
const names = sortedNames(categoryRoot);
|
||||
const nameSet = new Set(names);
|
||||
const consumed = new Set<string>();
|
||||
const bindings: Array<{
|
||||
alias: string;
|
||||
legacyHostPattern: string;
|
||||
targetSecretName: string;
|
||||
legacyConfigDigest: string;
|
||||
legacyProxyCommandPresent: boolean;
|
||||
legacyHostKeyBypassPresent: boolean;
|
||||
hostKeyPolicy: 'operator_verification_required';
|
||||
activation: 'disabled';
|
||||
}> = [];
|
||||
const secrets: SecretImportDraft[] = [];
|
||||
const manualHash = crypto.createHash('sha256');
|
||||
for (const configName of names.filter((name) => name.endsWith('.config'))) {
|
||||
const alias = configName.slice(0, -'.config'.length);
|
||||
if (
|
||||
alias.length === 0 ||
|
||||
Buffer.byteLength(alias, 'utf8') > MAX_ALIAS_BYTES ||
|
||||
/[\u0000-\u001f\u007f]/.test(alias) ||
|
||||
!nameSet.has(alias)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const keyPath = path.join(categoryRoot, alias);
|
||||
const configPath = path.join(categoryRoot, configName);
|
||||
let key: string;
|
||||
let config: string;
|
||||
try {
|
||||
key = readStablePrivateUtf8File(
|
||||
keyPath,
|
||||
uid,
|
||||
MAX_KEY_BYTES,
|
||||
'legacy SSH private key',
|
||||
);
|
||||
config = readStablePrivateUtf8File(
|
||||
configPath,
|
||||
uid,
|
||||
MAX_CONFIG_BYTES,
|
||||
'legacy SSH config',
|
||||
);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const parsed = configBinding(config, alias);
|
||||
if (!privateKey(key) || !parsed) continue;
|
||||
const secretName = targetSecretName(alias);
|
||||
bindings.push({
|
||||
alias,
|
||||
legacyHostPattern: parsed.hostPattern,
|
||||
targetSecretName: secretName,
|
||||
legacyConfigDigest: sha256Text(config),
|
||||
legacyProxyCommandPresent: parsed.proxyCommandPresent,
|
||||
legacyHostKeyBypassPresent: parsed.hostKeyBypassPresent,
|
||||
hostKeyPolicy: 'operator_verification_required',
|
||||
activation: 'disabled',
|
||||
});
|
||||
secrets.push({
|
||||
kind: 'ssh_private_key',
|
||||
sourceName: alias,
|
||||
targetName: secretName,
|
||||
value: key,
|
||||
});
|
||||
consumed.add(alias);
|
||||
consumed.add(configName);
|
||||
}
|
||||
for (const name of names) {
|
||||
if (consumed.has(name)) continue;
|
||||
const entryPath = path.join(categoryRoot, name);
|
||||
const stat = fs.lstatSync(entryPath, { bigint: true });
|
||||
manualHash.update(
|
||||
`${sha256Text(name)}:${stat.isDirectory() ? 'directory' : 'file'}\n`,
|
||||
'utf8',
|
||||
);
|
||||
}
|
||||
const after = summarizePrivateTree(categoryRoot, uid);
|
||||
if (JSON.stringify(after) !== JSON.stringify(source)) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'SSH transformation input changed while reading',
|
||||
);
|
||||
}
|
||||
const manualEntries = Math.max(0, source.entries - consumed.size);
|
||||
const model = Object.freeze({
|
||||
schema: 'qinglong/legacy-ssh-transformation@v1' as const,
|
||||
bindings: Object.freeze(bindings),
|
||||
manualEntries,
|
||||
manualEntryDigest: manualHash.digest('hex'),
|
||||
activation: 'disabled' as const,
|
||||
});
|
||||
return Object.freeze({
|
||||
source,
|
||||
model,
|
||||
secrets: Object.freeze(secrets),
|
||||
assessment: manualEntries > 0 ? 'manual_required' : 'ready',
|
||||
});
|
||||
}
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION,
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION,
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION,
|
||||
LocalDataDirectoryAdoptionConfigurationError,
|
||||
type TransformLocalDataDirectoryAdoptionCommand,
|
||||
type VerifyLocalDataDirectoryAdoptionCommand,
|
||||
type VerifyLocalDataDirectoryAdoptionTransformationCommand,
|
||||
} from '../contract';
|
||||
import { syncDirectory } from '../filesystem';
|
||||
import { sha256Text } from '../manifest';
|
||||
import { verifyLocalDataDirectoryAdoption } from '../staging';
|
||||
import { transformLegacyConfig } from './config';
|
||||
import {
|
||||
finishPrivateDirectory,
|
||||
transformationAuthority,
|
||||
writePrivateJson,
|
||||
} from './files';
|
||||
import { transformLegacyKeyv } from './keyv';
|
||||
import {
|
||||
TRANSFORMATION_MANIFEST_NAME,
|
||||
verifyStaticTransformation,
|
||||
} from './manifest';
|
||||
import {
|
||||
writeTransformationModel,
|
||||
type LocalDataDirectoryTransformationManifest,
|
||||
type LocalDataDirectoryTransformationManifestPayload,
|
||||
type TransformationModelEvidence,
|
||||
type TransformationSourceEvidence,
|
||||
} from './model';
|
||||
import { transformLegacySsh } from './ssh';
|
||||
|
||||
const INCOMPLETE_NAME = '.incomplete';
|
||||
|
||||
export interface LocalDataDirectoryTransformationResult {
|
||||
readonly schemaVersion: 1;
|
||||
readonly operation:
|
||||
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION
|
||||
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION;
|
||||
readonly status: 'prepared' | 'verified';
|
||||
readonly evidence: Readonly<{
|
||||
profile: 'edge' | 'standalone';
|
||||
createdAtMs: number;
|
||||
sourceStageManifestDigest: string;
|
||||
transformationDigest: string;
|
||||
assessment: 'ready' | 'manual_required';
|
||||
sources: readonly TransformationSourceEvidence[];
|
||||
model: Readonly<TransformationModelEvidence>;
|
||||
}>;
|
||||
}
|
||||
|
||||
type TransformationCommand =
|
||||
| Readonly<TransformLocalDataDirectoryAdoptionCommand>
|
||||
| Readonly<VerifyLocalDataDirectoryAdoptionTransformationCommand>;
|
||||
|
||||
async function verifySource(command: TransformationCommand) {
|
||||
const sourceCommand: VerifyLocalDataDirectoryAdoptionCommand = {
|
||||
schemaVersion: 1,
|
||||
operation: LOCAL_DATA_DIRECTORY_ADOPTION_VERIFY_OPERATION,
|
||||
options: {
|
||||
deploymentRoot: command.options.deploymentRoot,
|
||||
dataRoot: command.options.dataRoot,
|
||||
stagingRoot: command.options.stagingRoot,
|
||||
profile: command.options.profile,
|
||||
sqlite: command.options.sqlite,
|
||||
expectedManifestDigest: command.options.expectedManifestDigest,
|
||||
},
|
||||
};
|
||||
return verifyLocalDataDirectoryAdoption(sourceCommand);
|
||||
}
|
||||
|
||||
function unchangedSource(
|
||||
before: Awaited<ReturnType<typeof verifySource>>,
|
||||
after: Awaited<ReturnType<typeof verifySource>>,
|
||||
): void {
|
||||
if (JSON.stringify(before.evidence) !== JSON.stringify(after.evidence)) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'staged source changed during transformation',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function createdAtMs(): number {
|
||||
const value = Date.now();
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'system clock returned an invalid timestamp',
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function result(
|
||||
operation:
|
||||
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION
|
||||
| typeof LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION,
|
||||
status: 'prepared' | 'verified',
|
||||
manifest: Readonly<LocalDataDirectoryTransformationManifest>,
|
||||
): Readonly<LocalDataDirectoryTransformationResult> {
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation,
|
||||
status,
|
||||
evidence: Object.freeze({
|
||||
profile: manifest.profile,
|
||||
createdAtMs: manifest.createdAtMs,
|
||||
sourceStageManifestDigest: manifest.sourceStageManifestDigest,
|
||||
transformationDigest: manifest.transformationDigest,
|
||||
assessment: manifest.assessment,
|
||||
sources: manifest.sources,
|
||||
model: manifest.model,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export async function transformLocalDataDirectoryAdoption(
|
||||
command: Readonly<TransformLocalDataDirectoryAdoptionCommand>,
|
||||
): Promise<Readonly<LocalDataDirectoryTransformationResult>> {
|
||||
try {
|
||||
const authority = transformationAuthority(command.options, true);
|
||||
const before = await verifySource(command);
|
||||
fs.mkdirSync(authority.transformationRoot, { mode: 0o700 });
|
||||
writePrivateJson(path.join(authority.transformationRoot, INCOMPLETE_NAME), {
|
||||
schemaVersion: 1,
|
||||
kind: 'qinglong3-legacy-data-directory-transformation-incomplete',
|
||||
});
|
||||
finishPrivateDirectory(authority.transformationRoot);
|
||||
syncDirectory(path.dirname(authority.transformationRoot));
|
||||
|
||||
const inputRoot = path.join(
|
||||
authority.stagingRoot,
|
||||
'payload',
|
||||
'transform-input',
|
||||
);
|
||||
const config = transformLegacyConfig(
|
||||
path.join(inputRoot, 'config'),
|
||||
authority.uid,
|
||||
);
|
||||
const keyv = transformLegacyKeyv(
|
||||
path.join(inputRoot, 'db'),
|
||||
authority.uid,
|
||||
command.options.profile,
|
||||
);
|
||||
const ssh = transformLegacySsh(
|
||||
path.join(inputRoot, 'ssh.d'),
|
||||
authority.uid,
|
||||
);
|
||||
const prepared = writeTransformationModel({
|
||||
modelRoot: path.join(authority.transformationRoot, 'model'),
|
||||
uid: authority.uid,
|
||||
projectId: command.options.projectId,
|
||||
profile: command.options.profile,
|
||||
config,
|
||||
keyv,
|
||||
ssh,
|
||||
});
|
||||
const after = await verifySource(command);
|
||||
unchangedSource(before, after);
|
||||
|
||||
const payload: LocalDataDirectoryTransformationManifestPayload = {
|
||||
schemaVersion: 1,
|
||||
kind: 'qinglong3-legacy-data-directory-transformation',
|
||||
state: 'prepared',
|
||||
profile: command.options.profile,
|
||||
createdAtMs: createdAtMs(),
|
||||
projectIdDigest: sha256Text(command.options.projectId),
|
||||
sourceStageManifestDigest: command.options.expectedManifestDigest,
|
||||
transformationRootPathDigest: sha256Text(authority.transformationRoot),
|
||||
assessment: prepared.assessment,
|
||||
sources: prepared.sources,
|
||||
model: prepared.model,
|
||||
};
|
||||
const manifest: LocalDataDirectoryTransformationManifest = {
|
||||
...payload,
|
||||
transformationDigest: sha256Text(JSON.stringify(payload)),
|
||||
};
|
||||
writePrivateJson(
|
||||
path.join(authority.transformationRoot, TRANSFORMATION_MANIFEST_NAME),
|
||||
manifest,
|
||||
);
|
||||
finishPrivateDirectory(authority.transformationRoot);
|
||||
fs.unlinkSync(path.join(authority.transformationRoot, INCOMPLETE_NAME));
|
||||
finishPrivateDirectory(authority.transformationRoot);
|
||||
const verified = verifyStaticTransformation({
|
||||
authority,
|
||||
profile: command.options.profile,
|
||||
projectId: command.options.projectId,
|
||||
sourceStageManifestDigest: command.options.expectedManifestDigest,
|
||||
expectedTransformationDigest: manifest.transformationDigest,
|
||||
});
|
||||
return result(
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_OPERATION,
|
||||
'prepared',
|
||||
verified,
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalDataDirectoryAdoptionConfigurationError) {
|
||||
throw error;
|
||||
}
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'data directory transformation failed',
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyLocalDataDirectoryAdoptionTransformation(
|
||||
command: Readonly<VerifyLocalDataDirectoryAdoptionTransformationCommand>,
|
||||
): Promise<Readonly<LocalDataDirectoryTransformationResult>> {
|
||||
try {
|
||||
const authority = transformationAuthority(command.options, false);
|
||||
const before = await verifySource(command);
|
||||
const manifest = verifyStaticTransformation({
|
||||
authority,
|
||||
profile: command.options.profile,
|
||||
projectId: command.options.projectId,
|
||||
sourceStageManifestDigest: command.options.expectedManifestDigest,
|
||||
expectedTransformationDigest:
|
||||
command.options.expectedTransformationDigest,
|
||||
});
|
||||
const after = await verifySource(command);
|
||||
unchangedSource(before, after);
|
||||
verifyStaticTransformation({
|
||||
authority,
|
||||
profile: command.options.profile,
|
||||
projectId: command.options.projectId,
|
||||
sourceStageManifestDigest: command.options.expectedManifestDigest,
|
||||
expectedTransformationDigest:
|
||||
command.options.expectedTransformationDigest,
|
||||
});
|
||||
return result(
|
||||
LOCAL_DATA_DIRECTORY_ADOPTION_TRANSFORM_VERIFY_OPERATION,
|
||||
'verified',
|
||||
manifest,
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof LocalDataDirectoryAdoptionConfigurationError) {
|
||||
throw error;
|
||||
}
|
||||
throw new LocalDataDirectoryAdoptionConfigurationError(
|
||||
'data directory transformation verification failed',
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,9 @@ const BINARY = path.join(__dirname, '../dist/lifecycle/adoptionCli.js');
|
||||
const DIRECTORY_INSPECT = 'local-data-directory.adoption.inspect';
|
||||
const DIRECTORY_STAGE = 'local-data-directory.adoption.stage';
|
||||
const DIRECTORY_VERIFY = 'local-data-directory.adoption.verify';
|
||||
const DIRECTORY_TRANSFORM = 'local-data-directory.adoption.transform';
|
||||
const DIRECTORY_TRANSFORM_VERIFY =
|
||||
'local-data-directory.adoption.transform.verify';
|
||||
|
||||
function privateDirectory(directoryPath) {
|
||||
fs.mkdirSync(directoryPath, { recursive: true, mode: 0o700 });
|
||||
@@ -275,6 +278,99 @@ function verifyOptions(value, prepared, manifestDigest) {
|
||||
};
|
||||
}
|
||||
|
||||
function createKeyvDatabase(databasePath, values = {}) {
|
||||
fs.rmSync(databasePath, { force: true });
|
||||
const database = new DatabaseSync(databasePath);
|
||||
database.exec('CREATE TABLE keyv(key VARCHAR(255) PRIMARY KEY, value TEXT)');
|
||||
const insert = database.prepare('INSERT INTO keyv(key, value) VALUES (?, ?)');
|
||||
const entries = {
|
||||
'keyv:authInfo': {
|
||||
value: { token: values.authSecret ?? 'legacy-auth-token-never-carried' },
|
||||
expires: null,
|
||||
},
|
||||
'keyv:apps': { value: [{ id: 'legacy-app' }], expires: null },
|
||||
'keyv:lang': { value: 'en', expires: null },
|
||||
...(values.extra ?? {}),
|
||||
};
|
||||
for (const [key, value] of Object.entries(entries)) {
|
||||
insert.run(key, JSON.stringify(value));
|
||||
}
|
||||
database.close();
|
||||
fs.chmodSync(databasePath, 0o600);
|
||||
}
|
||||
|
||||
function configureTransformationInput(value) {
|
||||
const secrets = {
|
||||
environmentName: 'D386_API_TOKEN',
|
||||
environmentValue: 'd386-environment-secret',
|
||||
projectId: 'project-d386',
|
||||
sshAlias: 'repository-key',
|
||||
sshValue:
|
||||
'-----BEGIN OPENSSH PRIVATE KEY-----\nZDM4Ni1wcml2YXRlLWtleQ==\n-----END OPENSSH PRIVATE KEY-----\n',
|
||||
authValue: 'legacy-auth-token-never-carried',
|
||||
};
|
||||
privateFile(
|
||||
path.join(value.dataRoot, 'config', 'config.sh'),
|
||||
[
|
||||
`export ${secrets.environmentName}='${secrets.environmentValue}'`,
|
||||
'AutoStartBot=false',
|
||||
'export EMPTY_VALUE=',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
createKeyvDatabase(path.join(value.dataRoot, 'db', 'keyv.sqlite'), {
|
||||
authSecret: secrets.authValue,
|
||||
});
|
||||
privateFile(
|
||||
path.join(value.dataRoot, 'ssh.d', secrets.sshAlias),
|
||||
secrets.sshValue,
|
||||
);
|
||||
privateFile(
|
||||
path.join(value.dataRoot, 'ssh.d', `${secrets.sshAlias}.config`),
|
||||
[
|
||||
`Host ${secrets.sshAlias}`,
|
||||
` IdentityFile /root/.ssh/${secrets.sshAlias}`,
|
||||
' StrictHostKeyChecking no',
|
||||
' ProxyCommand nc -x legacy-proxy:1080 %h %p',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
value.transformationParent = path.join(
|
||||
value.deploymentRoot,
|
||||
'transformations',
|
||||
);
|
||||
value.transformationRoot = path.join(
|
||||
value.transformationParent,
|
||||
'reviewed-data-v1',
|
||||
);
|
||||
privateDirectory(value.transformationParent);
|
||||
return secrets;
|
||||
}
|
||||
|
||||
function stageForTransformation(value) {
|
||||
const prepared = prepare(value);
|
||||
const staged = run(
|
||||
value,
|
||||
'directory-stage-for-transformation',
|
||||
DIRECTORY_STAGE,
|
||||
stageOptions(value, prepared),
|
||||
).result;
|
||||
return { prepared, staged };
|
||||
}
|
||||
|
||||
function transformationOptions(value, prepared, staged, projectId) {
|
||||
return {
|
||||
deploymentRoot: value.deploymentRoot,
|
||||
dataRoot: value.dataRoot,
|
||||
stagingRoot: value.stagingRoot,
|
||||
transformationRoot: value.transformationRoot,
|
||||
projectId,
|
||||
profile: 'edge',
|
||||
expectedManifestDigest: staged.evidence.manifestDigest,
|
||||
sqlite: sqliteBinding(value, prepared.activationDigest),
|
||||
};
|
||||
}
|
||||
|
||||
test('stages only reviewed payloads behind the real SQLite activation fence', (t) => {
|
||||
const value = fixture(t);
|
||||
const prepared = prepare(value);
|
||||
@@ -479,3 +575,355 @@ test('widened directory staging commands fail closed before source access', (t)
|
||||
'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID',
|
||||
);
|
||||
});
|
||||
|
||||
test('prepares and exactly verifies a disabled versioned legacy transformation', (t) => {
|
||||
const value = fixture(t);
|
||||
const secrets = configureTransformationInput(value);
|
||||
const { prepared, staged } = stageForTransformation(value);
|
||||
const options = transformationOptions(
|
||||
value,
|
||||
prepared,
|
||||
staged,
|
||||
secrets.projectId,
|
||||
);
|
||||
const transformed = run(
|
||||
value,
|
||||
'directory-transform',
|
||||
DIRECTORY_TRANSFORM,
|
||||
options,
|
||||
);
|
||||
|
||||
assert.equal(transformed.result.status, 'prepared');
|
||||
assert.equal(transformed.result.evidence.assessment, 'ready');
|
||||
assert.match(
|
||||
transformed.result.evidence.transformationDigest,
|
||||
/^[0-9a-f]{64}$/,
|
||||
);
|
||||
for (const sensitive of [
|
||||
value.transformationRoot,
|
||||
secrets.projectId,
|
||||
secrets.environmentName,
|
||||
secrets.environmentValue,
|
||||
secrets.sshAlias,
|
||||
secrets.sshValue,
|
||||
secrets.authValue,
|
||||
]) {
|
||||
assert.equal(transformed.child.stdout.includes(sensitive), false);
|
||||
}
|
||||
assert.deepEqual(fs.readdirSync(value.transformationRoot).sort(), [
|
||||
'manifest.json',
|
||||
'model',
|
||||
]);
|
||||
const modelRoot = path.join(value.transformationRoot, 'model');
|
||||
const manifestText = fs.readFileSync(
|
||||
path.join(value.transformationRoot, 'manifest.json'),
|
||||
'utf8',
|
||||
);
|
||||
for (const sensitive of [
|
||||
secrets.projectId,
|
||||
secrets.environmentName,
|
||||
secrets.environmentValue,
|
||||
secrets.sshAlias,
|
||||
secrets.sshValue,
|
||||
secrets.authValue,
|
||||
]) {
|
||||
assert.equal(manifestText.includes(sensitive), false);
|
||||
}
|
||||
const config = JSON.parse(
|
||||
fs.readFileSync(path.join(modelRoot, 'config.json'), 'utf8'),
|
||||
);
|
||||
assert.deepEqual(
|
||||
config.exportedEnvironment.map((entry) => entry.environmentName),
|
||||
[secrets.environmentName],
|
||||
);
|
||||
assert.equal(config.retiredSettings[0].name, 'AutoStartBot');
|
||||
assert.equal(config.omittedEmptyExports, 1);
|
||||
assert.equal(config.activation, 'disabled');
|
||||
|
||||
const keyv = JSON.parse(
|
||||
fs.readFileSync(path.join(modelRoot, 'keyv.json'), 'utf8'),
|
||||
);
|
||||
assert.equal(keyv.integrity, 'ok');
|
||||
assert.equal(keyv.cachedLocale, 'en');
|
||||
assert.equal(
|
||||
keyv.mappings.find((entry) => entry.legacyKey === 'keyv:authInfo').state,
|
||||
'retired',
|
||||
);
|
||||
assert.equal(JSON.stringify(keyv).includes(secrets.authValue), false);
|
||||
|
||||
const ssh = JSON.parse(
|
||||
fs.readFileSync(path.join(modelRoot, 'ssh.json'), 'utf8'),
|
||||
);
|
||||
assert.equal(ssh.bindings[0].activation, 'disabled');
|
||||
assert.equal(ssh.bindings[0].hostKeyPolicy, 'operator_verification_required');
|
||||
assert.equal(ssh.bindings[0].legacyProxyCommandPresent, true);
|
||||
assert.equal(ssh.bindings[0].legacyHostKeyBypassPresent, true);
|
||||
assert.equal(JSON.stringify(ssh).includes('legacy-proxy'), false);
|
||||
assert.equal(JSON.stringify(ssh).includes('StrictHostKeyChecking'), false);
|
||||
|
||||
const importPlan = JSON.parse(
|
||||
fs.readFileSync(path.join(modelRoot, 'secret-imports.json'), 'utf8'),
|
||||
);
|
||||
assert.equal(importPlan.state, 'prepared');
|
||||
assert.equal(importPlan.projectId, secrets.projectId);
|
||||
assert.equal(importPlan.imports.length, 2);
|
||||
const secretValues = importPlan.imports.map((entry) => {
|
||||
const secretPath = path.join(modelRoot, entry.valueFile);
|
||||
assert.equal(fs.statSync(secretPath).mode & 0o777, 0o600);
|
||||
return JSON.parse(fs.readFileSync(secretPath, 'utf8')).value;
|
||||
});
|
||||
assert.deepEqual(
|
||||
secretValues.sort(),
|
||||
[secrets.environmentValue, secrets.sshValue].sort(),
|
||||
);
|
||||
const fullTarget =
|
||||
fs.readFileSync(path.join(modelRoot, 'secret-imports.json'), 'utf8') +
|
||||
fs.readFileSync(path.join(modelRoot, 'keyv.json'), 'utf8');
|
||||
assert.equal(fullTarget.includes(secrets.authValue), false);
|
||||
|
||||
const verified = run(
|
||||
value,
|
||||
'directory-transform-verify',
|
||||
DIRECTORY_TRANSFORM_VERIFY,
|
||||
{
|
||||
...options,
|
||||
expectedTransformationDigest:
|
||||
transformed.result.evidence.transformationDigest,
|
||||
},
|
||||
).result;
|
||||
assert.equal(verified.status, 'verified');
|
||||
assert.deepEqual(verified.evidence, transformed.result.evidence);
|
||||
const replayed = run(
|
||||
value,
|
||||
'directory-transform-verify-replay',
|
||||
DIRECTORY_TRANSFORM_VERIFY,
|
||||
{
|
||||
...options,
|
||||
expectedTransformationDigest:
|
||||
transformed.result.evidence.transformationDigest,
|
||||
},
|
||||
).result;
|
||||
assert.deepEqual(replayed, verified);
|
||||
});
|
||||
|
||||
test('transformation verification rejects target and current source drift', (t) => {
|
||||
const target = fixture(t);
|
||||
const targetSecrets = configureTransformationInput(target);
|
||||
const targetStage = stageForTransformation(target);
|
||||
const targetOptions = transformationOptions(
|
||||
target,
|
||||
targetStage.prepared,
|
||||
targetStage.staged,
|
||||
targetSecrets.projectId,
|
||||
);
|
||||
const transformed = run(
|
||||
target,
|
||||
'directory-transform-before-target-drift',
|
||||
DIRECTORY_TRANSFORM,
|
||||
targetOptions,
|
||||
).result;
|
||||
const secretFile = fs.readdirSync(
|
||||
path.join(target.transformationRoot, 'model', 'secret-values'),
|
||||
)[0];
|
||||
privateFile(
|
||||
path.join(target.transformationRoot, 'model', 'secret-values', secretFile),
|
||||
'{"schemaVersion":1,"kind":"qinglong3-local-secret-value","value":"tampered"}\n',
|
||||
);
|
||||
const targetDrift = runRaw(
|
||||
target,
|
||||
'directory-transform-target-drift',
|
||||
DIRECTORY_TRANSFORM_VERIFY,
|
||||
{
|
||||
...targetOptions,
|
||||
expectedTransformationDigest: transformed.evidence.transformationDigest,
|
||||
},
|
||||
);
|
||||
assert.equal(targetDrift.status, 1);
|
||||
assert.equal(
|
||||
JSON.parse(targetDrift.stderr).code,
|
||||
'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID',
|
||||
);
|
||||
|
||||
const source = fixture(t);
|
||||
const sourceSecrets = configureTransformationInput(source);
|
||||
const sourceStage = stageForTransformation(source);
|
||||
const sourceOptions = transformationOptions(
|
||||
source,
|
||||
sourceStage.prepared,
|
||||
sourceStage.staged,
|
||||
sourceSecrets.projectId,
|
||||
);
|
||||
const sourceTransformed = run(
|
||||
source,
|
||||
'directory-transform-before-source-drift',
|
||||
DIRECTORY_TRANSFORM,
|
||||
sourceOptions,
|
||||
).result;
|
||||
privateFile(
|
||||
path.join(source.dataRoot, 'config', 'config.sh'),
|
||||
'export D386_API_TOKEN=source-drift\n',
|
||||
);
|
||||
const sourceDrift = runRaw(
|
||||
source,
|
||||
'directory-transform-source-drift',
|
||||
DIRECTORY_TRANSFORM_VERIFY,
|
||||
{
|
||||
...sourceOptions,
|
||||
expectedTransformationDigest:
|
||||
sourceTransformed.evidence.transformationDigest,
|
||||
},
|
||||
);
|
||||
assert.equal(sourceDrift.status, 1);
|
||||
assert.equal(
|
||||
JSON.parse(sourceDrift.stderr).code,
|
||||
'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID',
|
||||
);
|
||||
});
|
||||
|
||||
test('unknown legacy behavior is retained as disabled manual-review evidence', (t) => {
|
||||
const value = fixture(t);
|
||||
const secrets = configureTransformationInput(value);
|
||||
privateFile(
|
||||
path.join(value.dataRoot, 'config', 'config.sh'),
|
||||
`export ${secrets.environmentName}=${secrets.environmentValue}\neval dangerous\n`,
|
||||
);
|
||||
const keyvPath = path.join(value.dataRoot, 'db', 'keyv.sqlite');
|
||||
const keyv = new DatabaseSync(keyvPath);
|
||||
keyv
|
||||
.prepare('INSERT INTO keyv(key, value) VALUES (?, ?)')
|
||||
.run('keyv:unknown', JSON.stringify({ value: 'retain', expires: null }));
|
||||
keyv.close();
|
||||
privateFile(path.join(value.dataRoot, 'ssh.d', 'unpaired-key'), 'manual');
|
||||
const { prepared, staged } = stageForTransformation(value);
|
||||
const transformed = run(
|
||||
value,
|
||||
'directory-transform-manual',
|
||||
DIRECTORY_TRANSFORM,
|
||||
transformationOptions(value, prepared, staged, secrets.projectId),
|
||||
).result;
|
||||
|
||||
assert.equal(transformed.evidence.assessment, 'manual_required');
|
||||
assert.equal(transformed.evidence.model.manualCategories, 3);
|
||||
const modelRoot = path.join(value.transformationRoot, 'model');
|
||||
assert.equal(
|
||||
JSON.parse(fs.readFileSync(path.join(modelRoot, 'config.json'), 'utf8'))
|
||||
.unsupportedLines,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
JSON.parse(fs.readFileSync(path.join(modelRoot, 'keyv.json'), 'utf8'))
|
||||
.unknownEntries,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
JSON.parse(fs.readFileSync(path.join(modelRoot, 'ssh.json'), 'utf8'))
|
||||
.manualEntries,
|
||||
1,
|
||||
);
|
||||
const manual = JSON.parse(
|
||||
fs.readFileSync(path.join(modelRoot, 'manual-review.json'), 'utf8'),
|
||||
);
|
||||
assert.equal(manual.required, true);
|
||||
assert.equal(manual.activation, 'disabled');
|
||||
});
|
||||
|
||||
test('edge Secret budget leaves no-replace recovery residue', (t) => {
|
||||
const value = fixture(t);
|
||||
const secrets = configureTransformationInput(value);
|
||||
privateFile(
|
||||
path.join(value.dataRoot, 'config', 'config.sh'),
|
||||
`${Array.from(
|
||||
{ length: 129 },
|
||||
(_, index) => `export D386_${index}=value`,
|
||||
).join('\n')}\n`,
|
||||
);
|
||||
const { prepared, staged } = stageForTransformation(value);
|
||||
const options = transformationOptions(
|
||||
value,
|
||||
prepared,
|
||||
staged,
|
||||
secrets.projectId,
|
||||
);
|
||||
const overBudget = runRaw(
|
||||
value,
|
||||
'directory-transform-over-budget',
|
||||
DIRECTORY_TRANSFORM,
|
||||
options,
|
||||
);
|
||||
assert.equal(overBudget.status, 1);
|
||||
assert.equal(
|
||||
JSON.parse(overBudget.stderr).code,
|
||||
'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID',
|
||||
);
|
||||
const marker = path.join(value.transformationRoot, '.incomplete');
|
||||
const residue = fs.readFileSync(marker, 'utf8');
|
||||
const replay = runRaw(
|
||||
value,
|
||||
'directory-transform-over-budget-replay',
|
||||
DIRECTORY_TRANSFORM,
|
||||
options,
|
||||
);
|
||||
assert.equal(replay.status, 1);
|
||||
assert.equal(fs.readFileSync(marker, 'utf8'), residue);
|
||||
});
|
||||
|
||||
test('widened transformation commands fail closed before source access', (t) => {
|
||||
const value = fixture(t);
|
||||
const missingData = path.join(value.deploymentRoot, 'missing-source');
|
||||
const child = runRaw(value, 'widened-transform', DIRECTORY_TRANSFORM, {
|
||||
deploymentRoot: value.deploymentRoot,
|
||||
dataRoot: missingData,
|
||||
stagingRoot: path.join(value.deploymentRoot, 'missing-stage'),
|
||||
transformationRoot: path.join(value.deploymentRoot, 'missing-transform'),
|
||||
projectId: 'project-d386',
|
||||
profile: 'edge',
|
||||
expectedManifestDigest: '0'.repeat(64),
|
||||
sqlite: {
|
||||
sourcePath: path.join(missingData, 'db', 'database.sqlite'),
|
||||
targetPath: path.join(value.artifactsDirectory, 'missing-target'),
|
||||
recoveryPath: path.join(value.artifactsDirectory, 'missing-recovery'),
|
||||
manifestPath: path.join(value.artifactsDirectory, 'missing-manifest'),
|
||||
activationPath: path.join(value.artifactsDirectory, 'missing-activation'),
|
||||
expectedActivationDigest: '0'.repeat(64),
|
||||
},
|
||||
extraAuthority: true,
|
||||
});
|
||||
assert.equal(child.status, 1);
|
||||
assert.equal(child.stdout, '');
|
||||
assert.equal(
|
||||
JSON.parse(child.stderr).code,
|
||||
'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID',
|
||||
);
|
||||
|
||||
const invalidProject = runRaw(
|
||||
value,
|
||||
'invalid-transform-project',
|
||||
DIRECTORY_TRANSFORM,
|
||||
{
|
||||
deploymentRoot: value.deploymentRoot,
|
||||
dataRoot: missingData,
|
||||
stagingRoot: path.join(value.deploymentRoot, 'missing-stage'),
|
||||
transformationRoot: path.join(value.deploymentRoot, 'missing-transform'),
|
||||
projectId: 'project with spaces',
|
||||
profile: 'edge',
|
||||
expectedManifestDigest: '0'.repeat(64),
|
||||
sqlite: {
|
||||
sourcePath: path.join(missingData, 'db', 'database.sqlite'),
|
||||
targetPath: path.join(value.artifactsDirectory, 'missing-target'),
|
||||
recoveryPath: path.join(value.artifactsDirectory, 'missing-recovery'),
|
||||
manifestPath: path.join(value.artifactsDirectory, 'missing-manifest'),
|
||||
activationPath: path.join(
|
||||
value.artifactsDirectory,
|
||||
'missing-activation',
|
||||
),
|
||||
expectedActivationDigest: '0'.repeat(64),
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(invalidProject.status, 1);
|
||||
assert.equal(invalidProject.stdout, '');
|
||||
assert.equal(
|
||||
JSON.parse(invalidProject.stderr).code,
|
||||
'LOCAL_DATA_DIRECTORY_ADOPTION_CONFIGURATION_INVALID',
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user