feat(ql3): prove legacy rollback readiness

This commit is contained in:
whyour
2026-08-20 18:24:52 +08:00
parent 2cd6ea1d55
commit 3c02ae2020
15 changed files with 1491 additions and 11 deletions
@@ -26,6 +26,7 @@ export type LocalCutoverInstanceHeadState =
| 'rollback_prepared'
| 'legacy_restart_requested'
| 'legacy_running'
| 'legacy_ready'
| 'manual_required'
| 'resolution_authorized';
@@ -156,6 +157,7 @@ function parseHead(value: unknown): Readonly<LocalCutoverInstanceHead> {
head.state !== 'rollback_prepared' &&
head.state !== 'legacy_restart_requested' &&
head.state !== 'legacy_running' &&
head.state !== 'legacy_ready' &&
head.state !== 'manual_required' &&
head.state !== 'resolution_authorized') ||
!Number.isSafeInteger(head.generation) ||
@@ -321,6 +323,7 @@ export function advanceLocalCutoverInstanceHead(
| 'rollback_prepared'
| 'legacy_restart_requested'
| 'legacy_running'
| 'legacy_ready'
| 'manual_required',
generation: number,
sourceRecordDigest: string,
@@ -356,7 +359,8 @@ export function advanceLocalCutoverInstanceHead(
current.generation === generation &&
(current.state === 'rollback_prepared' ||
current.state === 'legacy_restart_requested' ||
current.state === 'legacy_running')
current.state === 'legacy_running' ||
current.state === 'legacy_ready')
) {
return current;
}
@@ -374,6 +378,7 @@ export function advanceLocalCutoverInstanceHead(
current.state === 'rollback_prepared') ||
(state === 'legacy_running' &&
current.state === 'legacy_restart_requested') ||
(state === 'legacy_ready' && current.state === 'legacy_running') ||
(state === 'manual_required' &&
(current.state === 'legacy_stopped' ||
current.state === 'target_active' ||
@@ -0,0 +1,198 @@
import path from 'node:path';
import {
currentIdentity,
LocalDeploymentConfigurationError,
type LocalDeploymentProfile,
} from '../../foundation/contract';
const MAX_PATH_BYTES = 4_096;
const SAFE_PATH_PATTERN = /^\/[A-Za-z0-9._/@-]+$/;
const INSTANCE_ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,127}$/;
const CUTOVER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
const LEGACY_VERSION_PATTERN =
/^2\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/;
export interface LocalDeploymentLegacyReadinessCommand {
readonly schemaVersion: 1;
readonly operation: 'local.deployment.cutover.legacy-readiness-probe';
readonly options: Readonly<{
deploymentRoot: string;
allowRootService: boolean;
}>;
readonly request: Readonly<{
cutoverId: string;
profile: LocalDeploymentProfile;
instanceId: string;
generation: number;
expectedActivationDigest: string;
expectedInstanceHeadDigest: string;
expectedLegacyRunningRecordDigest: string;
legacyHttpPort: number;
expectedLegacyVersion: string;
requestedAtMs: number;
}>;
}
function object(value: unknown, label: string): Record<string, unknown> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
(Object.getPrototypeOf(value) !== Object.prototype &&
Object.getPrototypeOf(value) !== null)
) {
throw new LocalDeploymentConfigurationError(`${label} must be an object`);
}
return value as Record<string, unknown>;
}
function exact(
value: Record<string, unknown>,
keys: readonly string[],
label: string,
): void {
const actual = Object.keys(value).sort();
const expected = [...keys].sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index])
) {
throw new LocalDeploymentConfigurationError(`${label} shape is invalid`);
}
}
function safeAbsolutePath(value: unknown, label: string): string {
if (
typeof value !== 'string' ||
!path.isAbsolute(value) ||
path.normalize(value) !== value ||
path.parse(value).root === value ||
value.includes('\0') ||
value.includes('//') ||
!SAFE_PATH_PATTERN.test(value) ||
Buffer.byteLength(value, 'utf8') > MAX_PATH_BYTES
) {
throw new LocalDeploymentConfigurationError(
`${label} must be a supervisor-safe normalized absolute non-root path`,
);
}
return value;
}
function integer(
value: unknown,
minimum: number,
maximum: number,
label: string,
): number {
if (
!Number.isSafeInteger(value) ||
(value as number) < minimum ||
(value as number) > maximum
) {
throw new LocalDeploymentConfigurationError(`${label} is invalid`);
}
return value as number;
}
export function normalizeLocalDeploymentLegacyReadinessCommand(
value: unknown,
): Readonly<LocalDeploymentLegacyReadinessCommand> {
const command = object(value, 'command');
exact(
command,
['operation', 'options', 'request', 'schemaVersion'],
'command',
);
if (
command.schemaVersion !== 1 ||
command.operation !== 'local.deployment.cutover.legacy-readiness-probe'
) {
throw new LocalDeploymentConfigurationError(
'legacy readiness schemaVersion or operation is invalid',
);
}
const identity = currentIdentity();
const options = object(command.options, 'options');
exact(options, ['allowRootService', 'deploymentRoot'], 'options');
if (
typeof options.allowRootService !== 'boolean' ||
(identity.uid === 0) !== options.allowRootService
) {
throw new LocalDeploymentConfigurationError(
'allowRootService does not match the current identity',
);
}
const request = object(command.request, 'request');
exact(
request,
[
'cutoverId',
'expectedActivationDigest',
'expectedInstanceHeadDigest',
'expectedLegacyRunningRecordDigest',
'expectedLegacyVersion',
'generation',
'instanceId',
'legacyHttpPort',
'profile',
'requestedAtMs',
],
'request',
);
if (
typeof request.cutoverId !== 'string' ||
!CUTOVER_ID_PATTERN.test(request.cutoverId) ||
(request.profile !== 'edge' && request.profile !== 'standalone') ||
typeof request.instanceId !== 'string' ||
!INSTANCE_ID_PATTERN.test(request.instanceId) ||
typeof request.expectedActivationDigest !== 'string' ||
!DIGEST_PATTERN.test(request.expectedActivationDigest) ||
typeof request.expectedInstanceHeadDigest !== 'string' ||
!DIGEST_PATTERN.test(request.expectedInstanceHeadDigest) ||
typeof request.expectedLegacyRunningRecordDigest !== 'string' ||
!DIGEST_PATTERN.test(request.expectedLegacyRunningRecordDigest) ||
typeof request.expectedLegacyVersion !== 'string' ||
!LEGACY_VERSION_PATTERN.test(request.expectedLegacyVersion)
) {
throw new LocalDeploymentConfigurationError(
'legacy readiness request identity is invalid',
);
}
return Object.freeze({
schemaVersion: 1 as const,
operation: 'local.deployment.cutover.legacy-readiness-probe' as const,
options: Object.freeze({
deploymentRoot: safeAbsolutePath(
options.deploymentRoot,
'deploymentRoot',
),
allowRootService: options.allowRootService,
}),
request: Object.freeze({
cutoverId: request.cutoverId,
profile: request.profile,
instanceId: request.instanceId,
generation: integer(request.generation, 1, 15, 'generation'),
expectedActivationDigest: request.expectedActivationDigest,
expectedInstanceHeadDigest: request.expectedInstanceHeadDigest,
expectedLegacyRunningRecordDigest:
request.expectedLegacyRunningRecordDigest,
legacyHttpPort: integer(
request.legacyHttpPort,
1,
65_535,
'legacyHttpPort',
),
expectedLegacyVersion: request.expectedLegacyVersion,
requestedAtMs: integer(
request.requestedAtMs,
0,
Number.MAX_SAFE_INTEGER,
'requestedAtMs',
),
}),
});
}
@@ -0,0 +1,538 @@
import fs from 'node:fs';
import http from 'node:http';
import path from 'node:path';
import { readPrivateLocalCommandFile } from '@qinglong/local-command-file';
import {
currentIdentity,
LocalDeploymentConfigurationError,
} from '../../foundation/contract';
import {
preflightPublishedFile,
publishExactFile,
} from '../../foundation/files';
import {
advanceLocalCutoverInstanceHead,
localCutoverInstanceDirectory,
readLocalCutoverInstanceHead,
type LocalCutoverInstanceHead,
} from '../instanceLineage';
import { cutoverDigest } from '../targetEvidence';
import {
normalizeLocalDeploymentLegacyReadinessCommand,
type LocalDeploymentLegacyReadinessCommand,
} from './contract';
const RECEIPT_SCHEMA = 'qinglong3-local-legacy-readiness-receipt';
const LOOPBACK_HOST = '127.0.0.1';
const SYSTEM_PATH = '/api/system';
const MAX_RESPONSE_BYTES = 32 * 1024;
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
export type LocalLegacyReadinessReason =
| 'unavailable'
| 'http_rejected'
| 'response_too_large'
| 'response_invalid'
| 'not_initialized'
| 'version_mismatch';
export type LocalLegacyReadinessObservation =
| Readonly<{
ready: true;
initialized: true;
version: string;
}>
| Readonly<{
ready: false;
reason: LocalLegacyReadinessReason;
}>;
export interface LocalLegacyReadinessProbeInput {
readonly host: typeof LOOPBACK_HOST;
readonly port: number;
readonly path: typeof SYSTEM_PATH;
readonly timeoutMs: number;
readonly maxResponseBytes: number;
}
export interface LocalDeploymentLegacyReadinessDependencies {
readonly probe?: (
input: Readonly<LocalLegacyReadinessProbeInput>,
) => Promise<Readonly<LocalLegacyReadinessObservation>>;
readonly now?: () => number;
readonly wait?: (milliseconds: number) => Promise<void>;
}
interface LocalLegacyReadinessReceipt {
readonly schema: typeof RECEIPT_SCHEMA;
readonly schemaVersion: 1;
readonly state: 'legacy_ready';
readonly cutoverId: string;
readonly profile: 'edge' | 'standalone';
readonly instanceId: string;
readonly generation: number;
readonly activationDigest: string;
readonly previousHeadDigest: string;
readonly legacyRunningRecordDigest: string;
readonly endpoint: Readonly<{
host: typeof LOOPBACK_HOST;
port: number;
path: typeof SYSTEM_PATH;
}>;
readonly expectedVersion: string;
readonly observedVersion: string;
readonly initialized: true;
readonly attempts: number;
readonly observedAtMs: number;
readonly receiptDigest: string;
}
export type LocalDeploymentLegacyReadinessResult =
| Readonly<{
schemaVersion: 1;
operation: LocalDeploymentLegacyReadinessCommand['operation'];
status: 'prepared' | 'existing';
state: 'legacy_ready';
cutoverId: string;
generation: number;
attempts: number;
receiptDigest: string;
instanceHeadDigest: string;
}>
| Readonly<{
schemaVersion: 1;
operation: LocalDeploymentLegacyReadinessCommand['operation'];
status: 'not_ready';
state: 'legacy_running';
reason: LocalLegacyReadinessReason;
cutoverId: string;
generation: number;
attempts: number;
instanceHeadDigest: string;
}>;
function configurationError(message: string, cause?: unknown): never {
throw new LocalDeploymentConfigurationError(message, { cause });
}
function object(value: unknown, label: string): Record<string, unknown> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
(Object.getPrototypeOf(value) !== Object.prototype &&
Object.getPrototypeOf(value) !== null)
) {
configurationError(`${label} must be an object`);
}
return value as Record<string, unknown>;
}
function exact(
value: Record<string, unknown>,
keys: readonly string[],
label: string,
): void {
const actual = Object.keys(value).sort();
const expected = [...keys].sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index])
) {
configurationError(`${label} shape is invalid`);
}
}
function rejected(reason: LocalLegacyReadinessReason) {
return Object.freeze({ ready: false as const, reason });
}
export function probeLegacySystemEndpoint(
input: Readonly<LocalLegacyReadinessProbeInput>,
): Promise<Readonly<LocalLegacyReadinessObservation>> {
return new Promise((resolve) => {
let settled = false;
const finish = (observation: Readonly<LocalLegacyReadinessObservation>) => {
if (settled) return;
settled = true;
resolve(observation);
};
const request = http.request(
{
agent: false,
host: input.host,
port: input.port,
path: input.path,
method: 'GET',
headers: Object.freeze({
accept: 'application/json',
connection: 'close',
}),
},
(response) => {
if (response.statusCode !== 200) {
response.resume();
finish(rejected('http_rejected'));
return;
}
const chunks: Buffer[] = [];
let size = 0;
response.on('data', (chunk: Buffer | string) => {
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
size += bytes.byteLength;
if (size > input.maxResponseBytes) {
finish(rejected('response_too_large'));
response.destroy();
return;
}
chunks.push(bytes);
});
response.on('end', () => {
if (settled) return;
try {
const envelope = object(
JSON.parse(Buffer.concat(chunks).toString('utf8')),
'legacy system response',
);
const data = object(envelope.data, 'legacy system response data');
if (envelope.code !== 200 || typeof data.version !== 'string') {
finish(rejected('response_invalid'));
return;
}
if (data.isInitialized !== true) {
finish(rejected('not_initialized'));
return;
}
finish(
Object.freeze({
ready: true as const,
initialized: true as const,
version: data.version,
}),
);
} catch {
finish(rejected('response_invalid'));
}
});
response.on('error', () => finish(rejected('unavailable')));
},
);
request.setTimeout(input.timeoutMs, () => request.destroy());
request.on('error', () => finish(rejected('unavailable')));
request.end();
});
}
function receiptPath(
command: Readonly<LocalDeploymentLegacyReadinessCommand>,
): string {
return path.join(
localCutoverInstanceDirectory(
command.options.deploymentRoot,
command.request.instanceId,
),
`legacy-readiness-g${command.request.generation}.json`,
);
}
function receiptContents(
receipt: Readonly<LocalLegacyReadinessReceipt>,
): string {
return `${JSON.stringify(receipt, null, 2)}\n`;
}
function parseReceipt(value: unknown): Readonly<LocalLegacyReadinessReceipt> {
const receipt = object(value, 'legacy readiness receipt');
exact(
receipt,
[
'activationDigest',
'attempts',
'cutoverId',
'endpoint',
'expectedVersion',
'generation',
'initialized',
'instanceId',
'legacyRunningRecordDigest',
'observedAtMs',
'observedVersion',
'previousHeadDigest',
'profile',
'receiptDigest',
'schema',
'schemaVersion',
'state',
],
'legacy readiness receipt',
);
const endpoint = object(receipt.endpoint, 'legacy readiness endpoint');
exact(endpoint, ['host', 'path', 'port'], 'legacy readiness endpoint');
const { receiptDigest, ...payload } = receipt;
if (
receipt.schema !== RECEIPT_SCHEMA ||
receipt.schemaVersion !== 1 ||
receipt.state !== 'legacy_ready' ||
typeof receipt.cutoverId !== 'string' ||
(receipt.profile !== 'edge' && receipt.profile !== 'standalone') ||
typeof receipt.instanceId !== 'string' ||
!Number.isSafeInteger(receipt.generation) ||
(receipt.generation as number) < 1 ||
typeof receipt.activationDigest !== 'string' ||
!DIGEST_PATTERN.test(receipt.activationDigest) ||
typeof receipt.previousHeadDigest !== 'string' ||
!DIGEST_PATTERN.test(receipt.previousHeadDigest) ||
typeof receipt.legacyRunningRecordDigest !== 'string' ||
!DIGEST_PATTERN.test(receipt.legacyRunningRecordDigest) ||
endpoint.host !== LOOPBACK_HOST ||
endpoint.path !== SYSTEM_PATH ||
!Number.isSafeInteger(endpoint.port) ||
(endpoint.port as number) < 1 ||
(endpoint.port as number) > 65_535 ||
typeof receipt.expectedVersion !== 'string' ||
typeof receipt.observedVersion !== 'string' ||
receipt.initialized !== true ||
!Number.isSafeInteger(receipt.attempts) ||
(receipt.attempts as number) < 1 ||
!Number.isSafeInteger(receipt.observedAtMs) ||
(receipt.observedAtMs as number) < 0 ||
typeof receiptDigest !== 'string' ||
!DIGEST_PATTERN.test(receiptDigest) ||
cutoverDigest(payload) !== receiptDigest
) {
configurationError('legacy readiness receipt drifted');
}
return receipt as unknown as Readonly<LocalLegacyReadinessReceipt>;
}
function verifyReceipt(
command: Readonly<LocalDeploymentLegacyReadinessCommand>,
receipt: Readonly<LocalLegacyReadinessReceipt>,
): void {
if (
receipt.cutoverId !== command.request.cutoverId ||
receipt.profile !== command.request.profile ||
receipt.instanceId !== command.request.instanceId ||
receipt.generation !== command.request.generation ||
receipt.activationDigest !== command.request.expectedActivationDigest ||
receipt.previousHeadDigest !== command.request.expectedInstanceHeadDigest ||
receipt.legacyRunningRecordDigest !==
command.request.expectedLegacyRunningRecordDigest ||
receipt.endpoint.port !== command.request.legacyHttpPort ||
receipt.expectedVersion !== command.request.expectedLegacyVersion ||
receipt.observedVersion !== command.request.expectedLegacyVersion
) {
configurationError('legacy readiness receipt does not match the command');
}
}
function verifyHead(
command: Readonly<LocalDeploymentLegacyReadinessCommand>,
head: Readonly<LocalCutoverInstanceHead>,
): void {
const replay = head.state === 'legacy_ready';
if (
head.profile !== command.request.profile ||
head.cutoverId !== command.request.cutoverId ||
head.instanceId !== command.request.instanceId ||
head.generation !== command.request.generation ||
head.activationDigest !== command.request.expectedActivationDigest ||
(replay
? head.previousHeadDigest !== command.request.expectedInstanceHeadDigest
: head.headDigest !== command.request.expectedInstanceHeadDigest) ||
(!replay &&
head.sourceRecordDigest !==
command.request.expectedLegacyRunningRecordDigest) ||
(head.state !== 'legacy_running' && head.state !== 'legacy_ready')
) {
configurationError(
'legacy readiness command lost the instance head compare-and-swap',
);
}
}
function successfulResult(
command: Readonly<LocalDeploymentLegacyReadinessCommand>,
status: 'prepared' | 'existing',
receipt: Readonly<LocalLegacyReadinessReceipt>,
head: Readonly<LocalCutoverInstanceHead>,
): Readonly<LocalDeploymentLegacyReadinessResult> {
return Object.freeze({
schemaVersion: 1 as const,
operation: command.operation,
status,
state: 'legacy_ready' as const,
cutoverId: command.request.cutoverId,
generation: command.request.generation,
attempts: receipt.attempts,
receiptDigest: receipt.receiptDigest,
instanceHeadDigest: head.headDigest,
});
}
function completeExistingReceipt(
command: Readonly<LocalDeploymentLegacyReadinessCommand>,
uid: number,
current: Readonly<LocalCutoverInstanceHead>,
): Readonly<LocalDeploymentLegacyReadinessResult> | undefined {
const filePath = receiptPath(command);
if (!fs.existsSync(filePath)) return undefined;
const receipt = parseReceipt(readPrivateLocalCommandFile(filePath));
verifyReceipt(command, receipt);
const head =
current.state === 'legacy_ready'
? current
: advanceLocalCutoverInstanceHead(
command,
uid,
'legacy_ready',
command.request.generation,
receipt.receiptDigest,
);
if (head.sourceRecordDigest !== receipt.receiptDigest) {
configurationError('legacy-ready instance head is not receipt-bound');
}
return successfulResult(command, 'existing', receipt, head);
}
function policy(profile: 'edge' | 'standalone') {
return profile === 'edge'
? Object.freeze({
totalTimeoutMs: 30_000,
requestTimeoutMs: 2_000,
pollIntervalMs: 500,
maximumAttempts: 60,
})
: Object.freeze({
totalTimeoutMs: 60_000,
requestTimeoutMs: 2_000,
pollIntervalMs: 500,
maximumAttempts: 120,
});
}
export async function proveLocalDeploymentLegacyReadiness(
input: unknown,
dependencies: Readonly<LocalDeploymentLegacyReadinessDependencies> = {},
): Promise<Readonly<LocalDeploymentLegacyReadinessResult>> {
const command = normalizeLocalDeploymentLegacyReadinessCommand(input);
const identity = currentIdentity();
const current = readLocalCutoverInstanceHead(
command.options.deploymentRoot,
command.request.instanceId,
identity.uid,
);
verifyHead(command, current);
const existing = completeExistingReceipt(command, identity.uid, current);
if (existing) return existing;
if (current.state !== 'legacy_running') {
configurationError('legacy-ready instance is missing its receipt');
}
const observe = dependencies.probe ?? probeLegacySystemEndpoint;
const now = dependencies.now ?? Date.now;
const wait =
dependencies.wait ??
((milliseconds: number) =>
new Promise<void>((resolve) => setTimeout(resolve, milliseconds)));
const limits = policy(command.request.profile);
const deadline = now() + limits.totalTimeoutMs;
let attempts = 0;
let lastReason: LocalLegacyReadinessReason = 'unavailable';
while (attempts < limits.maximumAttempts && now() <= deadline) {
attempts += 1;
const remaining = Math.max(1, deadline - now());
const observation = await observe(
Object.freeze({
host: LOOPBACK_HOST,
port: command.request.legacyHttpPort,
path: SYSTEM_PATH,
timeoutMs: Math.min(limits.requestTimeoutMs, remaining),
maxResponseBytes: MAX_RESPONSE_BYTES,
}),
);
if (observation.ready === true) {
if (observation.version !== command.request.expectedLegacyVersion) {
lastReason = 'version_mismatch';
break;
}
const payload = Object.freeze({
schema: RECEIPT_SCHEMA,
schemaVersion: 1 as const,
state: 'legacy_ready' as const,
cutoverId: command.request.cutoverId,
profile: command.request.profile,
instanceId: command.request.instanceId,
generation: command.request.generation,
activationDigest: command.request.expectedActivationDigest,
previousHeadDigest: command.request.expectedInstanceHeadDigest,
legacyRunningRecordDigest:
command.request.expectedLegacyRunningRecordDigest,
endpoint: Object.freeze({
host: LOOPBACK_HOST,
port: command.request.legacyHttpPort,
path: SYSTEM_PATH,
}),
expectedVersion: command.request.expectedLegacyVersion,
observedVersion: observation.version,
initialized: true as const,
attempts,
observedAtMs: now(),
});
const receipt = Object.freeze({
...payload,
receiptDigest: cutoverDigest(payload),
});
const filePath = receiptPath(command);
const contents = receiptContents(receipt);
preflightPublishedFile(
filePath,
contents,
0o600,
identity.uid,
'legacy readiness receipt',
);
const status = publishExactFile(
filePath,
contents,
0o600,
identity.uid,
'legacy readiness receipt',
);
const head = advanceLocalCutoverInstanceHead(
command,
identity.uid,
'legacy_ready',
command.request.generation,
receipt.receiptDigest,
);
return successfulResult(command, status, receipt, head);
}
lastReason = observation.reason;
if (attempts >= limits.maximumAttempts || now() >= deadline) break;
await wait(Math.min(limits.pollIntervalMs, deadline - now()));
}
return Object.freeze({
schemaVersion: 1 as const,
operation: command.operation,
status: 'not_ready' as const,
state: 'legacy_running' as const,
reason: lastReason,
cutoverId: command.request.cutoverId,
generation: command.request.generation,
attempts,
instanceHeadDigest: current.headDigest,
});
}
export function proveLocalDeploymentLegacyReadinessCommandFile(
filePath: string,
dependencies: Readonly<LocalDeploymentLegacyReadinessDependencies> = {},
): Promise<Readonly<LocalDeploymentLegacyReadinessResult>> {
return proveLocalDeploymentLegacyReadiness(
readPrivateLocalCommandFile(filePath),
dependencies,
);
}
@@ -69,6 +69,10 @@ import {
runLocalDeploymentLegacyRollback,
runLocalDeploymentLegacyRollbackCommandFile,
} from './cutover/legacyRollback';
import {
proveLocalDeploymentLegacyReadiness,
proveLocalDeploymentLegacyReadinessCommandFile,
} from './cutover/legacy-readiness/probe';
import {
consumeLocalServiceManagerOutcome,
consumeLocalServiceManagerOutcomeCommandFile,
@@ -148,6 +152,17 @@ export {
type LocalDeploymentLegacyRollbackResult,
} from './cutover/legacyRollbackContract';
export { type LocalDeploymentLegacyRollbackDependencies } from './cutover/legacyRollback';
export {
normalizeLocalDeploymentLegacyReadinessCommand,
type LocalDeploymentLegacyReadinessCommand,
} from './cutover/legacy-readiness/contract';
export {
type LocalDeploymentLegacyReadinessDependencies,
type LocalDeploymentLegacyReadinessResult,
type LocalLegacyReadinessObservation,
type LocalLegacyReadinessProbeInput,
type LocalLegacyReadinessReason,
} from './cutover/legacy-readiness/probe';
export {
EMPTY_RESOLUTION_DIGEST,
normalizeLocalDeploymentCutoverManualCommand,
@@ -177,6 +192,8 @@ export {
runLocalDeploymentCutoverManualCommandFile,
runLocalDeploymentLegacyRollback,
runLocalDeploymentLegacyRollbackCommandFile,
proveLocalDeploymentLegacyReadiness,
proveLocalDeploymentLegacyReadinessCommandFile,
runLocalDeploymentDockerTarget,
runLocalDeploymentDockerTargetCommandFile,
switchLocalDeploymentComposeRevision,
@@ -14,6 +14,7 @@ import {
prepareLocalServiceManagerIntentCommandFile,
prepareLocalServiceManagerLegacyRollbackCommandFile,
prepareLocalDeploymentCommandFile,
proveLocalDeploymentLegacyReadinessCommandFile,
restoreLocalDeploymentComposeCommitCommandFile,
restoreLocalDeploymentComposePrepareCommandFile,
runLocalDeploymentCutoverManualCommandFile,
@@ -25,7 +26,7 @@ import {
} from './localDeployment';
const USAGE =
'Usage: ql3-local-deploy <prepare|status|service-intent-prepare|service-outcome-consume|service-cutover-consume|service-legacy-rollback-prepare|service-legacy-rollback-authorize|service-legacy-rollback-consume|cutover-legacy-stop|cutover-target-start|cutover-target-restart|cutover-target-stop|cutover-legacy-rollback-prepare|cutover-legacy-rollback-commit|cutover-manual-diagnose|cutover-manual-resolution-prepare|cutover-manual-resolution-commit|compose-revision|compose-preflight|compose-apply|compose-restore-prepare|compose-restore-commit|compose-evidence-collect-prepare|compose-evidence-collect-commit> --command-file /absolute/private-command.json';
'Usage: ql3-local-deploy <prepare|status|service-intent-prepare|service-outcome-consume|service-cutover-consume|service-legacy-rollback-prepare|service-legacy-rollback-authorize|service-legacy-rollback-consume|cutover-legacy-stop|cutover-target-start|cutover-target-restart|cutover-target-stop|cutover-legacy-rollback-prepare|cutover-legacy-rollback-commit|cutover-legacy-readiness-probe|cutover-manual-diagnose|cutover-manual-resolution-prepare|cutover-manual-resolution-commit|compose-revision|compose-preflight|compose-apply|compose-restore-prepare|compose-restore-commit|compose-evidence-collect-prepare|compose-evidence-collect-commit> --command-file /absolute/private-command.json';
async function main(argv: readonly string[]): Promise<void> {
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
@@ -48,6 +49,7 @@ async function main(argv: readonly string[]): Promise<void> {
argv[0] !== 'cutover-target-stop' &&
argv[0] !== 'cutover-legacy-rollback-prepare' &&
argv[0] !== 'cutover-legacy-rollback-commit' &&
argv[0] !== 'cutover-legacy-readiness-probe' &&
argv[0] !== 'cutover-manual-diagnose' &&
argv[0] !== 'cutover-manual-resolution-prepare' &&
argv[0] !== 'cutover-manual-resolution-commit' &&
@@ -101,6 +103,8 @@ async function main(argv: readonly string[]): Promise<void> {
? 'local.deployment.cutover.legacy-rollback-prepare'
: 'local.deployment.cutover.legacy-rollback-commit',
)
: argv[0] === 'cutover-legacy-readiness-probe'
? proveLocalDeploymentLegacyReadinessCommandFile(argv[2]!)
: argv[0] === 'cutover-manual-diagnose' ||
argv[0] === 'cutover-manual-resolution-prepare' ||
argv[0] === 'cutover-manual-resolution-commit'
@@ -0,0 +1,321 @@
const assert = require('node:assert/strict');
const fs = require('node:fs');
const http = require('node:http');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const {
normalizeLocalDeploymentLegacyReadinessCommand,
proveLocalDeploymentLegacyReadiness,
} = require('../dist/deployment/localDeployment.js');
const {
probeLegacySystemEndpoint,
} = require('../dist/deployment/cutover/legacy-readiness/probe.js');
const {
advanceLocalCutoverInstanceHead,
claimLocalCutoverInstance,
readLocalCutoverInstanceHead,
} = require('../dist/deployment/cutover/instanceLineage.js');
function rootAcknowledgement() {
return typeof process.getuid === 'function' && process.getuid() === 0;
}
function fixture(t, profile = 'edge', instanceId = `${profile}-legacy-1`) {
const deploymentRoot = fs.realpathSync(
fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-legacy-readiness-')),
);
fs.chmodSync(deploymentRoot, 0o700);
fs.mkdirSync(path.join(deploymentRoot, 'service'), { mode: 0o700 });
t.after(() => fs.rmSync(deploymentRoot, { recursive: true, force: true }));
const identity = {
options: { deploymentRoot },
request: {
cutoverId: `cutover-${instanceId}`,
profile,
instanceId,
expectedActivationDigest: 'a'.repeat(64),
requestedAtMs: 1_787_200_000_000,
},
};
const uid = process.getuid();
claimLocalCutoverInstance(identity, uid, '0'.repeat(64));
const transitions = [
['legacy_stopped', 0, '1'.repeat(64)],
['target_active', 1, '2'.repeat(64)],
['target_stopped', 1, '3'.repeat(64)],
['rollback_prepared', 1, '4'.repeat(64)],
['legacy_restart_requested', 1, '5'.repeat(64)],
['legacy_running', 1, '6'.repeat(64)],
];
let head;
for (const [state, generation, sourceDigest] of transitions) {
head = advanceLocalCutoverInstanceHead(
identity,
uid,
state,
generation,
sourceDigest,
);
}
const command = {
schemaVersion: 1,
operation: 'local.deployment.cutover.legacy-readiness-probe',
options: {
deploymentRoot,
allowRootService: rootAcknowledgement(),
},
request: {
cutoverId: identity.request.cutoverId,
profile,
instanceId,
generation: 1,
expectedActivationDigest: identity.request.expectedActivationDigest,
expectedInstanceHeadDigest: head.headDigest,
expectedLegacyRunningRecordDigest: '6'.repeat(64),
legacyHttpPort: 5700,
expectedLegacyVersion: '2.21.0',
requestedAtMs: 1_787_200_030_000,
},
};
return { command, deploymentRoot, head, uid };
}
test('normalizes a closed legacy readiness command', (t) => {
const state = fixture(t);
const normalized = normalizeLocalDeploymentLegacyReadinessCommand(
state.command,
);
assert.equal(normalized.request.legacyHttpPort, 5700);
assert.equal(normalized.request.expectedLegacyVersion, '2.21.0');
assert.throws(
() =>
normalizeLocalDeploymentLegacyReadinessCommand({
...state.command,
request: { ...state.command.request, endpoint: 'http://example.com' },
}),
/request shape is invalid/,
);
assert.throws(
() =>
normalizeLocalDeploymentLegacyReadinessCommand({
...state.command,
request: { ...state.command.request, expectedLegacyVersion: '3.0.0' },
}),
/request identity is invalid/,
);
assert.throws(
() =>
normalizeLocalDeploymentLegacyReadinessCommand({
...state.command,
request: { ...state.command.request, legacyHttpPort: 65_536 },
}),
/legacyHttpPort is invalid/,
);
});
test('persists a legacy-ready receipt and replays without network authority', async (t) => {
const state = fixture(t);
let attempts = 0;
let clock = state.command.request.requestedAtMs;
const result = await proveLocalDeploymentLegacyReadiness(state.command, {
now: () => clock,
wait: async (milliseconds) => {
clock += milliseconds;
},
async probe(input) {
attempts += 1;
assert.deepEqual(
{ host: input.host, port: input.port, path: input.path },
{ host: '127.0.0.1', port: 5700, path: '/api/system' },
);
return attempts < 3
? { ready: false, reason: 'not_initialized' }
: { ready: true, initialized: true, version: '2.21.0' };
},
});
assert.equal(result.status, 'prepared');
assert.equal(result.state, 'legacy_ready');
assert.equal(result.attempts, 3);
assert.match(result.receiptDigest, /^[0-9a-f]{64}$/);
const receiptPath = path.join(
state.deploymentRoot,
'service',
'cutover-instances',
state.command.request.instanceId,
'legacy-readiness-g1.json',
);
const receipt = JSON.parse(fs.readFileSync(receiptPath, 'utf8'));
assert.equal(receipt.state, 'legacy_ready');
assert.equal(receipt.endpoint.host, '127.0.0.1');
assert.equal(receipt.endpoint.path, '/api/system');
assert.equal(receipt.observedVersion, '2.21.0');
assert.equal(fs.statSync(receiptPath).mode & 0o777, 0o600);
const head = readLocalCutoverInstanceHead(
state.deploymentRoot,
state.command.request.instanceId,
state.uid,
);
assert.equal(head.state, 'legacy_ready');
assert.equal(head.previousHeadDigest, state.head.headDigest);
assert.equal(head.sourceRecordDigest, result.receiptDigest);
const replay = await proveLocalDeploymentLegacyReadiness(state.command, {
async probe() {
throw new Error('exact replay must not open loopback HTTP authority');
},
});
assert.equal(replay.status, 'existing');
assert.equal(replay.receiptDigest, result.receiptDigest);
assert.equal(replay.instanceHeadDigest, result.instanceHeadDigest);
});
test('bounds an unavailable edge probe without mutating lineage', async (t) => {
const state = fixture(t);
let clock = state.command.request.requestedAtMs;
let calls = 0;
const result = await proveLocalDeploymentLegacyReadiness(state.command, {
now: () => clock,
wait: async (milliseconds) => {
clock += milliseconds;
},
async probe() {
calls += 1;
return { ready: false, reason: 'unavailable' };
},
});
assert.equal(result.status, 'not_ready');
assert.equal(result.state, 'legacy_running');
assert.equal(result.reason, 'unavailable');
assert.equal(result.attempts, 60);
assert.equal(calls, 60);
assert.equal(
readLocalCutoverInstanceHead(
state.deploymentRoot,
state.command.request.instanceId,
state.uid,
).headDigest,
state.head.headDigest,
);
assert.equal(
fs.existsSync(
path.join(
state.deploymentRoot,
'service',
'cutover-instances',
state.command.request.instanceId,
'legacy-readiness-g1.json',
),
),
false,
);
});
test('fails a version mismatch and stale head before durable mutation', async (t) => {
const state = fixture(t);
const mismatch = await proveLocalDeploymentLegacyReadiness(state.command, {
async probe() {
return { ready: true, initialized: true, version: '2.20.0' };
},
});
assert.equal(mismatch.status, 'not_ready');
assert.equal(mismatch.reason, 'version_mismatch');
assert.equal(mismatch.attempts, 1);
let calls = 0;
await assert.rejects(
proveLocalDeploymentLegacyReadiness(
{
...state.command,
request: {
...state.command.request,
expectedInstanceHeadDigest: 'f'.repeat(64),
},
},
{
async probe() {
calls += 1;
return { ready: true, initialized: true, version: '2.21.0' };
},
},
),
/lost the instance head compare-and-swap/,
);
assert.equal(calls, 0);
});
test('probes only the fixed loopback system endpoint with bounded parsing', async (t) => {
const paths = [];
const server = http.createServer((request, response) => {
paths.push(request.url);
response.setHeader('content-type', 'application/json');
response.end(
JSON.stringify({
code: 200,
data: { isInitialized: true, version: '2.21.0' },
}),
);
});
await new Promise((resolve, reject) => {
server.once('error', reject);
server.listen(0, '127.0.0.1', resolve);
});
t.after(() => new Promise((resolve) => server.close(resolve)));
const address = server.address();
const result = await probeLegacySystemEndpoint({
host: '127.0.0.1',
port: address.port,
path: '/api/system',
timeoutMs: 2_000,
maxResponseBytes: 32 * 1024,
});
assert.deepEqual(result, {
ready: true,
initialized: true,
version: '2.21.0',
});
assert.deepEqual(paths, ['/api/system']);
});
test('rejects oversized and redirect responses without following them', async (t) => {
let redirected = false;
const server = http.createServer((request, response) => {
if (request.url === '/redirected') {
redirected = true;
response.end('unexpected');
return;
}
if (request.url === '/large') {
response.end('x'.repeat(2_048));
return;
}
response.writeHead(302, { location: '/redirected' });
response.end();
});
await new Promise((resolve, reject) => {
server.once('error', reject);
server.listen(0, '127.0.0.1', resolve);
});
t.after(() => new Promise((resolve) => server.close(resolve)));
const address = server.address();
const redirect = await probeLegacySystemEndpoint({
host: '127.0.0.1',
port: address.port,
path: '/api/system',
timeoutMs: 2_000,
maxResponseBytes: 1_024,
});
assert.deepEqual(redirect, { ready: false, reason: 'http_rejected' });
assert.equal(redirected, false);
const large = await probeLegacySystemEndpoint({
host: '127.0.0.1',
port: address.port,
path: '/large',
timeoutMs: 2_000,
maxResponseBytes: 1_024,
});
assert.deepEqual(large, {
ready: false,
reason: 'response_too_large',
});
});