mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add frozen-admission cutover probe
This commit is contained in:
@@ -6,9 +6,10 @@ import {
|
||||
type LocalApplicationProcessSignal,
|
||||
type LocalApplicationProcessSignalSource,
|
||||
} from './production-process/processApplication';
|
||||
import { runProductionLocalApplicationCutoverProbe } from './production-process/cutoverProbeProcess';
|
||||
|
||||
const USAGE =
|
||||
'Usage: ql3-local-application --config /absolute/private-config.json';
|
||||
'Usage: ql3-local-application [--cutover-probe] --config /absolute/private-config.json';
|
||||
|
||||
const nodeSignals: LocalApplicationProcessSignalSource = Object.freeze({
|
||||
subscribe(
|
||||
@@ -53,9 +54,28 @@ function failureFact(error: unknown): Readonly<Record<string, unknown>> {
|
||||
});
|
||||
}
|
||||
|
||||
function configFileArgument(argv: readonly string[]): string | null {
|
||||
if (argv.length !== 2 || argv[0] !== '--config' || !argv[1]) return null;
|
||||
return argv[1];
|
||||
function configFileArgument(argv: readonly string[]): Readonly<{
|
||||
configFilePath: string;
|
||||
mode: 'application' | 'cutover_probe';
|
||||
}> | null {
|
||||
if (argv.length === 2 && argv[0] === '--config' && argv[1]) {
|
||||
return Object.freeze({
|
||||
configFilePath: argv[1],
|
||||
mode: 'application' as const,
|
||||
});
|
||||
}
|
||||
if (
|
||||
argv.length === 3 &&
|
||||
argv[0] === '--cutover-probe' &&
|
||||
argv[1] === '--config' &&
|
||||
argv[2]
|
||||
) {
|
||||
return Object.freeze({
|
||||
configFilePath: argv[2],
|
||||
mode: 'cutover_probe' as const,
|
||||
});
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function main(argv: readonly string[]): Promise<void> {
|
||||
@@ -63,8 +83,8 @@ async function main(argv: readonly string[]): Promise<void> {
|
||||
process.stdout.write(`${USAGE}\n`);
|
||||
return;
|
||||
}
|
||||
const configFilePath = configFileArgument(argv);
|
||||
if (configFilePath === null) {
|
||||
const command = configFileArgument(argv);
|
||||
if (command === null) {
|
||||
process.stderr.write(
|
||||
`${JSON.stringify({
|
||||
code: 'QL3_LOCAL_APPLICATION_CLI_USAGE_INVALID',
|
||||
@@ -75,11 +95,18 @@ async function main(argv: readonly string[]): Promise<void> {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const stopResult = await runProductionLocalApplicationProcess({
|
||||
configFilePath,
|
||||
signals: nodeSignals,
|
||||
emit,
|
||||
});
|
||||
const stopResult =
|
||||
command.mode === 'cutover_probe'
|
||||
? await runProductionLocalApplicationCutoverProbe({
|
||||
configFilePath: command.configFilePath,
|
||||
signals: nodeSignals,
|
||||
emit,
|
||||
})
|
||||
: await runProductionLocalApplicationProcess({
|
||||
configFilePath: command.configFilePath,
|
||||
signals: nodeSignals,
|
||||
emit,
|
||||
});
|
||||
if (stopResult !== 'stopped') process.exitCode = 1;
|
||||
} catch (error) {
|
||||
process.stderr.write(`${JSON.stringify(failureFact(error))}\n`);
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import { inspectLocalSqliteReadinessPath } from '@qinglong/local-sqlite/readiness-inspection';
|
||||
|
||||
import type {
|
||||
LocalApplicationProcessEvent,
|
||||
LocalApplicationProcessSignal,
|
||||
ProductionLocalApplicationProcessOptions,
|
||||
} from './processApplication';
|
||||
import { verifyLocalApplicationCutoverCommitment } from './cutoverCommitment';
|
||||
import { verifyLocalApplicationLegacyDataCommitment } from './legacyDataApplicationCommitment';
|
||||
import { loadLocalApplicationProcessConfig } from './processConfig';
|
||||
import { recordLocalApplicationShutdownReceipt } from './shutdownReceipt';
|
||||
import { recordLocalApplicationStartupReceipt } from './startupReceipt';
|
||||
|
||||
export type ProductionLocalApplicationCutoverProbeOptions = Pick<
|
||||
ProductionLocalApplicationProcessOptions,
|
||||
'configFilePath' | 'emit' | 'signals'
|
||||
>;
|
||||
|
||||
export class LocalApplicationCutoverProbeError extends Error {
|
||||
readonly code = 'QL3_LOCAL_APPLICATION_CUTOVER_PROBE_INVALID';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Local application cutover probe is invalid: ${message}`);
|
||||
this.name = 'LocalApplicationCutoverProbeError';
|
||||
}
|
||||
}
|
||||
|
||||
function event(
|
||||
config: Readonly<{
|
||||
instanceId: string;
|
||||
profile: 'edge' | 'standalone';
|
||||
}>,
|
||||
values: Omit<
|
||||
LocalApplicationProcessEvent,
|
||||
'component' | 'instanceId' | 'profile' | 'schemaVersion'
|
||||
>,
|
||||
): Readonly<LocalApplicationProcessEvent> {
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-local-application',
|
||||
instanceId: config.instanceId,
|
||||
profile: config.profile,
|
||||
...values,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Proves that one adopted target is readable by this exact application image
|
||||
* while recovery, scheduling, execution and product admission remain frozen.
|
||||
* The probe owns no write-capable database connection.
|
||||
*/
|
||||
export async function runProductionLocalApplicationCutoverProbe(
|
||||
options: ProductionLocalApplicationCutoverProbeOptions,
|
||||
): Promise<'stopped'> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
typeof options.configFilePath !== 'string' ||
|
||||
typeof options.emit !== 'function' ||
|
||||
typeof options.signals?.subscribe !== 'function'
|
||||
) {
|
||||
throw new TypeError('Local application cutover probe options are invalid');
|
||||
}
|
||||
const config = loadLocalApplicationProcessConfig(options.configFilePath);
|
||||
if (config.storage.mode !== 'adopted') {
|
||||
throw new LocalApplicationCutoverProbeError(
|
||||
'only adopted storage can be probed',
|
||||
);
|
||||
}
|
||||
verifyLocalApplicationLegacyDataCommitment(config);
|
||||
verifyLocalApplicationCutoverCommitment(config);
|
||||
|
||||
let resolveSignal:
|
||||
| ((signal: LocalApplicationProcessSignal) => void)
|
||||
| undefined;
|
||||
const requestedSignal = new Promise<LocalApplicationProcessSignal>(
|
||||
(resolve) => {
|
||||
resolveSignal = resolve;
|
||||
},
|
||||
);
|
||||
let acceptedSignal = false;
|
||||
const unsubscribe = options.signals.subscribe((signal) => {
|
||||
if (acceptedSignal) return;
|
||||
acceptedSignal = true;
|
||||
resolveSignal?.(signal);
|
||||
});
|
||||
const keepAlive = setInterval(() => undefined, 2_147_483_647);
|
||||
|
||||
try {
|
||||
await new Promise<void>((resolve) => {
|
||||
setImmediate(resolve);
|
||||
});
|
||||
await inspectLocalSqliteReadinessPath({
|
||||
databasePath: config.storage.targetPath,
|
||||
profile: config.profile,
|
||||
...(config.storage.busyTimeoutMs === undefined
|
||||
? {}
|
||||
: { busyTimeoutMs: config.storage.busyTimeoutMs }),
|
||||
});
|
||||
await options.emit(
|
||||
event(config, {
|
||||
level: 'info',
|
||||
event: 'cutover_probe_storage_ready',
|
||||
}),
|
||||
);
|
||||
const startupReceipt = recordLocalApplicationStartupReceipt({
|
||||
configFilePath: options.configFilePath,
|
||||
instanceId: config.instanceId,
|
||||
profile: config.profile,
|
||||
aiStatus: 'deployment_excluded',
|
||||
});
|
||||
if (startupReceipt === undefined) {
|
||||
throw new LocalApplicationCutoverProbeError(
|
||||
'a Linux startup receipt is required',
|
||||
);
|
||||
}
|
||||
await options.emit(
|
||||
event(config, {
|
||||
level: 'info',
|
||||
event: 'cutover_probe_active',
|
||||
aiStatus: 'deployment_excluded',
|
||||
}),
|
||||
);
|
||||
const signal = await requestedSignal;
|
||||
await options.emit(
|
||||
event(config, {
|
||||
level: 'info',
|
||||
event: 'shutdown_requested',
|
||||
signal,
|
||||
}),
|
||||
);
|
||||
recordLocalApplicationShutdownReceipt({
|
||||
configFilePath: options.configFilePath,
|
||||
instanceId: config.instanceId,
|
||||
profile: config.profile,
|
||||
signal,
|
||||
startupReceiptDigest: startupReceipt.sha256,
|
||||
});
|
||||
await options.emit(
|
||||
event(config, {
|
||||
level: 'info',
|
||||
event: 'cutover_probe_stopped',
|
||||
stopResult: 'stopped',
|
||||
}),
|
||||
);
|
||||
return 'stopped';
|
||||
} finally {
|
||||
clearInterval(keepAlive);
|
||||
unsubscribe();
|
||||
resolveSignal = undefined;
|
||||
}
|
||||
}
|
||||
@@ -574,7 +574,7 @@ test('CLI exposes bounded usage and redacted configuration failures', (t) => {
|
||||
assert.equal(help.status, 0, help.stderr);
|
||||
assert.equal(
|
||||
help.stdout,
|
||||
'Usage: ql3-local-application --config /absolute/private-config.json\n',
|
||||
'Usage: ql3-local-application [--cutover-probe] --config /absolute/private-config.json\n',
|
||||
);
|
||||
|
||||
const usage = spawnSync(process.execPath, [cli], { encoding: 'utf8' });
|
||||
@@ -744,3 +744,71 @@ test('CLI boots the real headless runtime and releases it on SIGTERM', async (t)
|
||||
.run(2, 'echo released');
|
||||
writer.close();
|
||||
});
|
||||
|
||||
test(
|
||||
'CLI cutover probe proves adopted readiness without changing SQLite',
|
||||
{ skip: process.platform !== 'linux' },
|
||||
async (t) => {
|
||||
const { configFilePath, value } = await prepareCliFixture(t);
|
||||
const before = crypto
|
||||
.createHash('sha256')
|
||||
.update(fs.readFileSync(value.storage.targetPath))
|
||||
.digest('hex');
|
||||
const cli = path.resolve(__dirname, '../dist/cli.js');
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[cli, '--cutover-probe', '--config', configFilePath],
|
||||
{ stdio: ['ignore', 'pipe', 'pipe'] },
|
||||
);
|
||||
const events = [];
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let signalled = false;
|
||||
child.stderr.setEncoding('utf8');
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.stdout.setEncoding('utf8');
|
||||
child.stdout.on('data', (chunk) => {
|
||||
stdout += chunk;
|
||||
while (stdout.includes('\n')) {
|
||||
const index = stdout.indexOf('\n');
|
||||
const line = stdout.slice(0, index);
|
||||
stdout = stdout.slice(index + 1);
|
||||
if (!line) continue;
|
||||
const record = JSON.parse(line);
|
||||
events.push(record);
|
||||
if (record.event === 'cutover_probe_active' && !signalled) {
|
||||
signalled = true;
|
||||
child.kill('SIGTERM');
|
||||
}
|
||||
}
|
||||
});
|
||||
const timeout = setTimeout(() => child.kill('SIGKILL'), 15_000);
|
||||
timeout.unref();
|
||||
const [code, signal] = await new Promise((resolve) => {
|
||||
child.once('exit', (...args) => resolve(args));
|
||||
});
|
||||
clearTimeout(timeout);
|
||||
|
||||
assert.equal(code, 0, JSON.stringify({ stderr, signal, events }));
|
||||
assert.equal(signal, null);
|
||||
assert.equal(signalled, true);
|
||||
assert.equal(
|
||||
events.some(({ event }) => event === 'cutover_probe_storage_ready'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
events.some(({ event }) => event === 'cutover_probe_stopped'),
|
||||
true,
|
||||
);
|
||||
const after = crypto
|
||||
.createHash('sha256')
|
||||
.update(fs.readFileSync(value.storage.targetPath))
|
||||
.digest('hex');
|
||||
assert.equal(after, before);
|
||||
for (const suffix of ['-wal', '-shm', '-journal']) {
|
||||
assert.equal(fs.existsSync(`${value.storage.targetPath}${suffix}`), false);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@@ -567,6 +567,7 @@ export function parseTargetContainerEvidence(
|
||||
container.Image !== command.request.targetImage.imageId ||
|
||||
JSON.stringify(config.Cmd) !==
|
||||
JSON.stringify([
|
||||
'--cutover-probe',
|
||||
'--config',
|
||||
command.request.expectedTargetApplicationConfigPath,
|
||||
]) ||
|
||||
|
||||
@@ -104,7 +104,13 @@ function targetInspection(state, options = {}) {
|
||||
},
|
||||
Config: {
|
||||
Image: state.targetImage,
|
||||
Cmd: ['--config', state.targetApplicationConfigPath],
|
||||
Cmd: [
|
||||
...(options.normalApplicationCommand === true
|
||||
? []
|
||||
: ['--cutover-probe']),
|
||||
'--config',
|
||||
state.targetApplicationConfigPath,
|
||||
],
|
||||
},
|
||||
HostConfig: {
|
||||
RestartPolicy: { Name: 'no' },
|
||||
@@ -612,6 +618,27 @@ test('requires the target container to keep the Legacy source read-only', async
|
||||
assert.equal(request.evidence.reason, 'target_preflight_unproved');
|
||||
});
|
||||
|
||||
test('requires cutover target-start to use frozen-admission probe mode', async (t) => {
|
||||
const state = fixture(t);
|
||||
const controller = harness(state, { normalApplicationCommand: true });
|
||||
const result = await runLocalDeploymentDockerTarget(
|
||||
command(state),
|
||||
controller,
|
||||
);
|
||||
assert.equal(result.state, 'manual_required');
|
||||
assert.equal(
|
||||
controller.calls.filter((args) => args[1] === 'start').length,
|
||||
0,
|
||||
);
|
||||
const request = JSON.parse(
|
||||
fs.readFileSync(
|
||||
path.join(state.journal, '0003-target-start-decision.json'),
|
||||
'utf8',
|
||||
),
|
||||
);
|
||||
assert.equal(request.evidence.reason, 'target_preflight_unproved');
|
||||
});
|
||||
|
||||
test('starts an offline Trial Kit image only when its local reference and content ID both match', async (t) => {
|
||||
const state = fixture(t);
|
||||
state.targetImageAuthority = 'local-image-id';
|
||||
|
||||
Reference in New Issue
Block a user