mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-23 03:18:09 +08:00
feat(ql3): add frozen-admission cutover probe
This commit is contained in:
@@ -6,9 +6,10 @@ import {
|
|||||||
type LocalApplicationProcessSignal,
|
type LocalApplicationProcessSignal,
|
||||||
type LocalApplicationProcessSignalSource,
|
type LocalApplicationProcessSignalSource,
|
||||||
} from './production-process/processApplication';
|
} from './production-process/processApplication';
|
||||||
|
import { runProductionLocalApplicationCutoverProbe } from './production-process/cutoverProbeProcess';
|
||||||
|
|
||||||
const USAGE =
|
const USAGE =
|
||||||
'Usage: ql3-local-application --config /absolute/private-config.json';
|
'Usage: ql3-local-application [--cutover-probe] --config /absolute/private-config.json';
|
||||||
|
|
||||||
const nodeSignals: LocalApplicationProcessSignalSource = Object.freeze({
|
const nodeSignals: LocalApplicationProcessSignalSource = Object.freeze({
|
||||||
subscribe(
|
subscribe(
|
||||||
@@ -53,9 +54,28 @@ function failureFact(error: unknown): Readonly<Record<string, unknown>> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function configFileArgument(argv: readonly string[]): string | null {
|
function configFileArgument(argv: readonly string[]): Readonly<{
|
||||||
if (argv.length !== 2 || argv[0] !== '--config' || !argv[1]) return null;
|
configFilePath: string;
|
||||||
return argv[1];
|
mode: 'application' | 'cutover_probe';
|
||||||
|
}> | null {
|
||||||
|
if (argv.length === 2 && argv[0] === '--config' && argv[1]) {
|
||||||
|
return Object.freeze({
|
||||||
|
configFilePath: argv[1],
|
||||||
|
mode: 'application' as const,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
argv.length === 3 &&
|
||||||
|
argv[0] === '--cutover-probe' &&
|
||||||
|
argv[1] === '--config' &&
|
||||||
|
argv[2]
|
||||||
|
) {
|
||||||
|
return Object.freeze({
|
||||||
|
configFilePath: argv[2],
|
||||||
|
mode: 'cutover_probe' as const,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function main(argv: readonly string[]): Promise<void> {
|
async function main(argv: readonly string[]): Promise<void> {
|
||||||
@@ -63,8 +83,8 @@ async function main(argv: readonly string[]): Promise<void> {
|
|||||||
process.stdout.write(`${USAGE}\n`);
|
process.stdout.write(`${USAGE}\n`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const configFilePath = configFileArgument(argv);
|
const command = configFileArgument(argv);
|
||||||
if (configFilePath === null) {
|
if (command === null) {
|
||||||
process.stderr.write(
|
process.stderr.write(
|
||||||
`${JSON.stringify({
|
`${JSON.stringify({
|
||||||
code: 'QL3_LOCAL_APPLICATION_CLI_USAGE_INVALID',
|
code: 'QL3_LOCAL_APPLICATION_CLI_USAGE_INVALID',
|
||||||
@@ -75,8 +95,15 @@ async function main(argv: readonly string[]): Promise<void> {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const stopResult = await runProductionLocalApplicationProcess({
|
const stopResult =
|
||||||
configFilePath,
|
command.mode === 'cutover_probe'
|
||||||
|
? await runProductionLocalApplicationCutoverProbe({
|
||||||
|
configFilePath: command.configFilePath,
|
||||||
|
signals: nodeSignals,
|
||||||
|
emit,
|
||||||
|
})
|
||||||
|
: await runProductionLocalApplicationProcess({
|
||||||
|
configFilePath: command.configFilePath,
|
||||||
signals: nodeSignals,
|
signals: nodeSignals,
|
||||||
emit,
|
emit,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,153 @@
|
|||||||
|
import { inspectLocalSqliteReadinessPath } from '@qinglong/local-sqlite/readiness-inspection';
|
||||||
|
|
||||||
|
import type {
|
||||||
|
LocalApplicationProcessEvent,
|
||||||
|
LocalApplicationProcessSignal,
|
||||||
|
ProductionLocalApplicationProcessOptions,
|
||||||
|
} from './processApplication';
|
||||||
|
import { verifyLocalApplicationCutoverCommitment } from './cutoverCommitment';
|
||||||
|
import { verifyLocalApplicationLegacyDataCommitment } from './legacyDataApplicationCommitment';
|
||||||
|
import { loadLocalApplicationProcessConfig } from './processConfig';
|
||||||
|
import { recordLocalApplicationShutdownReceipt } from './shutdownReceipt';
|
||||||
|
import { recordLocalApplicationStartupReceipt } from './startupReceipt';
|
||||||
|
|
||||||
|
export type ProductionLocalApplicationCutoverProbeOptions = Pick<
|
||||||
|
ProductionLocalApplicationProcessOptions,
|
||||||
|
'configFilePath' | 'emit' | 'signals'
|
||||||
|
>;
|
||||||
|
|
||||||
|
export class LocalApplicationCutoverProbeError extends Error {
|
||||||
|
readonly code = 'QL3_LOCAL_APPLICATION_CUTOVER_PROBE_INVALID';
|
||||||
|
|
||||||
|
constructor(message: string) {
|
||||||
|
super(`Local application cutover probe is invalid: ${message}`);
|
||||||
|
this.name = 'LocalApplicationCutoverProbeError';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function event(
|
||||||
|
config: Readonly<{
|
||||||
|
instanceId: string;
|
||||||
|
profile: 'edge' | 'standalone';
|
||||||
|
}>,
|
||||||
|
values: Omit<
|
||||||
|
LocalApplicationProcessEvent,
|
||||||
|
'component' | 'instanceId' | 'profile' | 'schemaVersion'
|
||||||
|
>,
|
||||||
|
): Readonly<LocalApplicationProcessEvent> {
|
||||||
|
return Object.freeze({
|
||||||
|
schemaVersion: 1,
|
||||||
|
component: 'qinglong3-local-application',
|
||||||
|
instanceId: config.instanceId,
|
||||||
|
profile: config.profile,
|
||||||
|
...values,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Proves that one adopted target is readable by this exact application image
|
||||||
|
* while recovery, scheduling, execution and product admission remain frozen.
|
||||||
|
* The probe owns no write-capable database connection.
|
||||||
|
*/
|
||||||
|
export async function runProductionLocalApplicationCutoverProbe(
|
||||||
|
options: ProductionLocalApplicationCutoverProbeOptions,
|
||||||
|
): Promise<'stopped'> {
|
||||||
|
if (
|
||||||
|
!options ||
|
||||||
|
typeof options !== 'object' ||
|
||||||
|
Array.isArray(options) ||
|
||||||
|
typeof options.configFilePath !== 'string' ||
|
||||||
|
typeof options.emit !== 'function' ||
|
||||||
|
typeof options.signals?.subscribe !== 'function'
|
||||||
|
) {
|
||||||
|
throw new TypeError('Local application cutover probe options are invalid');
|
||||||
|
}
|
||||||
|
const config = loadLocalApplicationProcessConfig(options.configFilePath);
|
||||||
|
if (config.storage.mode !== 'adopted') {
|
||||||
|
throw new LocalApplicationCutoverProbeError(
|
||||||
|
'only adopted storage can be probed',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
verifyLocalApplicationLegacyDataCommitment(config);
|
||||||
|
verifyLocalApplicationCutoverCommitment(config);
|
||||||
|
|
||||||
|
let resolveSignal:
|
||||||
|
| ((signal: LocalApplicationProcessSignal) => void)
|
||||||
|
| undefined;
|
||||||
|
const requestedSignal = new Promise<LocalApplicationProcessSignal>(
|
||||||
|
(resolve) => {
|
||||||
|
resolveSignal = resolve;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let acceptedSignal = false;
|
||||||
|
const unsubscribe = options.signals.subscribe((signal) => {
|
||||||
|
if (acceptedSignal) return;
|
||||||
|
acceptedSignal = true;
|
||||||
|
resolveSignal?.(signal);
|
||||||
|
});
|
||||||
|
const keepAlive = setInterval(() => undefined, 2_147_483_647);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
setImmediate(resolve);
|
||||||
|
});
|
||||||
|
await inspectLocalSqliteReadinessPath({
|
||||||
|
databasePath: config.storage.targetPath,
|
||||||
|
profile: config.profile,
|
||||||
|
...(config.storage.busyTimeoutMs === undefined
|
||||||
|
? {}
|
||||||
|
: { busyTimeoutMs: config.storage.busyTimeoutMs }),
|
||||||
|
});
|
||||||
|
await options.emit(
|
||||||
|
event(config, {
|
||||||
|
level: 'info',
|
||||||
|
event: 'cutover_probe_storage_ready',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const startupReceipt = recordLocalApplicationStartupReceipt({
|
||||||
|
configFilePath: options.configFilePath,
|
||||||
|
instanceId: config.instanceId,
|
||||||
|
profile: config.profile,
|
||||||
|
aiStatus: 'deployment_excluded',
|
||||||
|
});
|
||||||
|
if (startupReceipt === undefined) {
|
||||||
|
throw new LocalApplicationCutoverProbeError(
|
||||||
|
'a Linux startup receipt is required',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await options.emit(
|
||||||
|
event(config, {
|
||||||
|
level: 'info',
|
||||||
|
event: 'cutover_probe_active',
|
||||||
|
aiStatus: 'deployment_excluded',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const signal = await requestedSignal;
|
||||||
|
await options.emit(
|
||||||
|
event(config, {
|
||||||
|
level: 'info',
|
||||||
|
event: 'shutdown_requested',
|
||||||
|
signal,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
recordLocalApplicationShutdownReceipt({
|
||||||
|
configFilePath: options.configFilePath,
|
||||||
|
instanceId: config.instanceId,
|
||||||
|
profile: config.profile,
|
||||||
|
signal,
|
||||||
|
startupReceiptDigest: startupReceipt.sha256,
|
||||||
|
});
|
||||||
|
await options.emit(
|
||||||
|
event(config, {
|
||||||
|
level: 'info',
|
||||||
|
event: 'cutover_probe_stopped',
|
||||||
|
stopResult: 'stopped',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return 'stopped';
|
||||||
|
} finally {
|
||||||
|
clearInterval(keepAlive);
|
||||||
|
unsubscribe();
|
||||||
|
resolveSignal = undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -574,7 +574,7 @@ test('CLI exposes bounded usage and redacted configuration failures', (t) => {
|
|||||||
assert.equal(help.status, 0, help.stderr);
|
assert.equal(help.status, 0, help.stderr);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
help.stdout,
|
help.stdout,
|
||||||
'Usage: ql3-local-application --config /absolute/private-config.json\n',
|
'Usage: ql3-local-application [--cutover-probe] --config /absolute/private-config.json\n',
|
||||||
);
|
);
|
||||||
|
|
||||||
const usage = spawnSync(process.execPath, [cli], { encoding: 'utf8' });
|
const usage = spawnSync(process.execPath, [cli], { encoding: 'utf8' });
|
||||||
@@ -744,3 +744,71 @@ test('CLI boots the real headless runtime and releases it on SIGTERM', async (t)
|
|||||||
.run(2, 'echo released');
|
.run(2, 'echo released');
|
||||||
writer.close();
|
writer.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test(
|
||||||
|
'CLI cutover probe proves adopted readiness without changing SQLite',
|
||||||
|
{ skip: process.platform !== 'linux' },
|
||||||
|
async (t) => {
|
||||||
|
const { configFilePath, value } = await prepareCliFixture(t);
|
||||||
|
const before = crypto
|
||||||
|
.createHash('sha256')
|
||||||
|
.update(fs.readFileSync(value.storage.targetPath))
|
||||||
|
.digest('hex');
|
||||||
|
const cli = path.resolve(__dirname, '../dist/cli.js');
|
||||||
|
const child = spawn(
|
||||||
|
process.execPath,
|
||||||
|
[cli, '--cutover-probe', '--config', configFilePath],
|
||||||
|
{ stdio: ['ignore', 'pipe', 'pipe'] },
|
||||||
|
);
|
||||||
|
const events = [];
|
||||||
|
let stdout = '';
|
||||||
|
let stderr = '';
|
||||||
|
let signalled = false;
|
||||||
|
child.stderr.setEncoding('utf8');
|
||||||
|
child.stderr.on('data', (chunk) => {
|
||||||
|
stderr += chunk;
|
||||||
|
});
|
||||||
|
child.stdout.setEncoding('utf8');
|
||||||
|
child.stdout.on('data', (chunk) => {
|
||||||
|
stdout += chunk;
|
||||||
|
while (stdout.includes('\n')) {
|
||||||
|
const index = stdout.indexOf('\n');
|
||||||
|
const line = stdout.slice(0, index);
|
||||||
|
stdout = stdout.slice(index + 1);
|
||||||
|
if (!line) continue;
|
||||||
|
const record = JSON.parse(line);
|
||||||
|
events.push(record);
|
||||||
|
if (record.event === 'cutover_probe_active' && !signalled) {
|
||||||
|
signalled = true;
|
||||||
|
child.kill('SIGTERM');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const timeout = setTimeout(() => child.kill('SIGKILL'), 15_000);
|
||||||
|
timeout.unref();
|
||||||
|
const [code, signal] = await new Promise((resolve) => {
|
||||||
|
child.once('exit', (...args) => resolve(args));
|
||||||
|
});
|
||||||
|
clearTimeout(timeout);
|
||||||
|
|
||||||
|
assert.equal(code, 0, JSON.stringify({ stderr, signal, events }));
|
||||||
|
assert.equal(signal, null);
|
||||||
|
assert.equal(signalled, true);
|
||||||
|
assert.equal(
|
||||||
|
events.some(({ event }) => event === 'cutover_probe_storage_ready'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
events.some(({ event }) => event === 'cutover_probe_stopped'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
const after = crypto
|
||||||
|
.createHash('sha256')
|
||||||
|
.update(fs.readFileSync(value.storage.targetPath))
|
||||||
|
.digest('hex');
|
||||||
|
assert.equal(after, before);
|
||||||
|
for (const suffix of ['-wal', '-shm', '-journal']) {
|
||||||
|
assert.equal(fs.existsSync(`${value.storage.targetPath}${suffix}`), false);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|||||||
@@ -567,6 +567,7 @@ export function parseTargetContainerEvidence(
|
|||||||
container.Image !== command.request.targetImage.imageId ||
|
container.Image !== command.request.targetImage.imageId ||
|
||||||
JSON.stringify(config.Cmd) !==
|
JSON.stringify(config.Cmd) !==
|
||||||
JSON.stringify([
|
JSON.stringify([
|
||||||
|
'--cutover-probe',
|
||||||
'--config',
|
'--config',
|
||||||
command.request.expectedTargetApplicationConfigPath,
|
command.request.expectedTargetApplicationConfigPath,
|
||||||
]) ||
|
]) ||
|
||||||
|
|||||||
@@ -104,7 +104,13 @@ function targetInspection(state, options = {}) {
|
|||||||
},
|
},
|
||||||
Config: {
|
Config: {
|
||||||
Image: state.targetImage,
|
Image: state.targetImage,
|
||||||
Cmd: ['--config', state.targetApplicationConfigPath],
|
Cmd: [
|
||||||
|
...(options.normalApplicationCommand === true
|
||||||
|
? []
|
||||||
|
: ['--cutover-probe']),
|
||||||
|
'--config',
|
||||||
|
state.targetApplicationConfigPath,
|
||||||
|
],
|
||||||
},
|
},
|
||||||
HostConfig: {
|
HostConfig: {
|
||||||
RestartPolicy: { Name: 'no' },
|
RestartPolicy: { Name: 'no' },
|
||||||
@@ -612,6 +618,27 @@ test('requires the target container to keep the Legacy source read-only', async
|
|||||||
assert.equal(request.evidence.reason, 'target_preflight_unproved');
|
assert.equal(request.evidence.reason, 'target_preflight_unproved');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('requires cutover target-start to use frozen-admission probe mode', async (t) => {
|
||||||
|
const state = fixture(t);
|
||||||
|
const controller = harness(state, { normalApplicationCommand: true });
|
||||||
|
const result = await runLocalDeploymentDockerTarget(
|
||||||
|
command(state),
|
||||||
|
controller,
|
||||||
|
);
|
||||||
|
assert.equal(result.state, 'manual_required');
|
||||||
|
assert.equal(
|
||||||
|
controller.calls.filter((args) => args[1] === 'start').length,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
const request = JSON.parse(
|
||||||
|
fs.readFileSync(
|
||||||
|
path.join(state.journal, '0003-target-start-decision.json'),
|
||||||
|
'utf8',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert.equal(request.evidence.reason, 'target_preflight_unproved');
|
||||||
|
});
|
||||||
|
|
||||||
test('starts an offline Trial Kit image only when its local reference and content ID both match', async (t) => {
|
test('starts an offline Trial Kit image only when its local reference and content ID both match', async (t) => {
|
||||||
const state = fixture(t);
|
const state = fixture(t);
|
||||||
state.targetImageAuthority = 'local-image-id';
|
state.targetImageAuthority = 'local-image-id';
|
||||||
|
|||||||
@@ -257,7 +257,7 @@ target_id=$(docker create --name "$target_name" --restart no \
|
|||||||
--tmpfs /tmp:rw,nosuid,nodev,noexec,size=16m \
|
--tmpfs /tmp:rw,nosuid,nodev,noexec,size=16m \
|
||||||
--mount "type=bind,src=$rehearsal_root,dst=$rehearsal_root" \
|
--mount "type=bind,src=$rehearsal_root,dst=$rehearsal_root" \
|
||||||
--mount "type=bind,src=$legacy_root,dst=$legacy_root,readonly" \
|
--mount "type=bind,src=$legacy_root,dst=$legacy_root,readonly" \
|
||||||
"$APPLICATION_IMAGE" --config "$rehearsal_root/local-application.json")
|
"$APPLICATION_IMAGE" --cutover-probe --config "$rehearsal_root/local-application.json")
|
||||||
[ "${#target_id}" -eq 64 ] || fail 'target container ID is invalid'
|
[ "${#target_id}" -eq 64 ] || fail 'target container ID is invalid'
|
||||||
case "$target_id" in *[!0-9a-f]*) fail 'target container ID is invalid' ;; esac
|
case "$target_id" in *[!0-9a-f]*) fail 'target container ID is invalid' ;; esac
|
||||||
|
|
||||||
|
|||||||
@@ -272,6 +272,10 @@ test('materializes and offline-audits one closed two-image trial kit', (t) => {
|
|||||||
cutoverRehearsalContents,
|
cutoverRehearsalContents,
|
||||||
/QingLong Local Alpha target-stop result:/,
|
/QingLong Local Alpha target-stop result:/,
|
||||||
);
|
);
|
||||||
|
assert.match(
|
||||||
|
cutoverRehearsalContents,
|
||||||
|
/\$APPLICATION_IMAGE" --cutover-probe --config/,
|
||||||
|
);
|
||||||
assert.match(
|
assert.match(
|
||||||
cutoverRehearsalContents,
|
cutoverRehearsalContents,
|
||||||
/QingLong Local Alpha target-stop evidence:/,
|
/QingLong Local Alpha target-stop evidence:/,
|
||||||
|
|||||||
Reference in New Issue
Block a user