feat(ql3): persist manual primary activation

This commit is contained in:
whyour
2026-08-19 05:13:58 +08:00
parent 9981f47851
commit 36035ac43e
15 changed files with 1191 additions and 43 deletions
@@ -0,0 +1,261 @@
import { randomBytes } from 'crypto';
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import {
createManualPrimaryRuntimeReceipt,
MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE,
MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES,
parseManualPrimaryRuntimeReceipt,
transitionManualPrimaryRuntimeReceipt,
type ManualPrimaryRuntimeProcessIdentity,
type ManualPrimaryRuntimeReceipt,
type ManualPrimaryRuntimeReceiptState,
} from '../../domain/manualPrimaryRuntimeReceipt';
import type { ManualPrimaryRuntimeReceiptLifecycle } from '../../ports/manualPrimaryRuntimeReceipt';
import type { RuntimeRolloutLoadAudit } from '../../ports/runtimeRolloutLoader';
import {
LinuxProcProcessIdentityProvider,
type LocalProcessIdentityProvider,
} from '../local-process/localProcessIdentity';
export interface ManualPrimaryRuntimeReceiptStoreOptions {
clock?: { now(): number };
identityProvider?: LocalProcessIdentityProvider;
platform?: NodeJS.Platform;
pid?: number;
randomId?: () => string;
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as { code?: string }).code === code
);
}
export class ManualPrimaryRuntimeReceiptConflictError extends Error {
constructor(message: string) {
super(message);
this.name = 'ManualPrimaryRuntimeReceiptConflictError';
}
}
export class ManualPrimaryRuntimeReceiptStore
implements ManualPrimaryRuntimeReceiptLifecycle
{
private readonly target: string;
private readonly clock: { now(): number };
private readonly identityProvider: LocalProcessIdentityProvider;
private readonly platform: NodeJS.Platform;
private readonly pid: number;
private readonly randomId: () => string;
private current?: ManualPrimaryRuntimeReceipt;
constructor(
private readonly root: string,
private readonly profile: 'edge' | 'standalone',
options: ManualPrimaryRuntimeReceiptStoreOptions = {},
) {
if (!path.isAbsolute(root)) {
throw new TypeError(
'Manual Primary runtime receipt root must be absolute',
);
}
this.target = path.join(root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE);
this.clock = options.clock ?? { now: Date.now };
this.identityProvider =
options.identityProvider ?? new LinuxProcProcessIdentityProvider();
this.platform = options.platform ?? process.platform;
this.pid = options.pid ?? process.pid;
this.randomId = options.randomId ?? (() => randomBytes(16).toString('hex'));
}
async activated(audit: RuntimeRolloutLoadAudit): Promise<void> {
if (
typeof audit.revision !== 'string' ||
typeof audit.sourceSha256 !== 'string'
) {
throw new TypeError(
'Accepted rollout audit lacks durable revision or source digest',
);
}
await this.assertRoot();
const processIdentity = await this.captureProcessIdentity();
const existing = await this.read();
await this.assertReplaceable(existing);
const receipt = createManualPrimaryRuntimeReceipt({
activationId: this.randomId(),
profile: this.profile,
revision: audit.revision,
rolloutSourceSha256: audit.sourceSha256,
activatedAtMs: this.now(),
process: processIdentity,
});
await this.write(receipt);
this.current = receipt;
}
async stopping(): Promise<void> {
await this.transition('stopping');
}
async stopped(): Promise<void> {
await this.transition('stopped');
}
async failed(): Promise<void> {
if (this.current === undefined) return;
await this.transition('failed');
}
private async transition(
state: Exclude<ManualPrimaryRuntimeReceiptState, 'active'>,
): Promise<void> {
if (this.current === undefined) {
throw new ManualPrimaryRuntimeReceiptConflictError(
'Manual Primary runtime receipt was not activated by this process',
);
}
const observed = await this.read();
if (
observed === undefined ||
observed.activationId !== this.current.activationId ||
observed.receiptSha256 !== this.current.receiptSha256
) {
throw new ManualPrimaryRuntimeReceiptConflictError(
'Manual Primary runtime receipt ownership changed',
);
}
const next = transitionManualPrimaryRuntimeReceipt(
this.current,
state,
Math.max(this.current.updatedAtMs, this.now()),
);
await this.write(next);
this.current = next;
}
private now(): number {
const value = this.clock.now();
if (!Number.isSafeInteger(value) || value < 0) {
throw new TypeError('Manual Primary runtime receipt clock is invalid');
}
return value;
}
private async captureProcessIdentity(): Promise<ManualPrimaryRuntimeProcessIdentity> {
const identity = await this.identityProvider.capture(this.pid);
if (identity !== null) {
return { kind: 'linux-proc', ...identity };
}
if (this.platform === 'linux') {
throw new Error('Linux process identity is unavailable');
}
return { kind: 'portable', platform: this.platform, pid: this.pid };
}
private async assertReplaceable(
existing: ManualPrimaryRuntimeReceipt | undefined,
): Promise<void> {
if (
existing === undefined ||
existing.state === 'stopped' ||
existing.state === 'failed'
) {
return;
}
if (existing.process.kind !== 'linux-proc') return;
const inspection = await this.identityProvider.inspect(existing.process);
if (inspection.status === 'running') {
throw new ManualPrimaryRuntimeReceiptConflictError(
'Another Manual Primary runtime receipt is still live',
);
}
if (!['exited', 'identity_mismatch'].includes(inspection.status)) {
throw new ManualPrimaryRuntimeReceiptConflictError(
'Previous Manual Primary runtime identity cannot be disproved',
);
}
}
private async assertRoot(): Promise<void> {
const stat = await fs.lstat(this.root);
if (
!stat.isDirectory() ||
stat.isSymbolicLink() ||
(stat.mode & 0o022) !== 0 ||
(typeof process.getuid === 'function' && stat.uid !== process.getuid())
) {
throw new TypeError('Manual Primary runtime receipt root is unsafe');
}
}
private async read(): Promise<ManualPrimaryRuntimeReceipt | undefined> {
let handle: fs.FileHandle;
try {
handle = await fs.open(
this.target,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
throw error;
}
try {
const stat = await handle.stat();
if (
!stat.isFile() ||
(stat.mode & 0o077) !== 0 ||
(typeof process.getuid === 'function' &&
stat.uid !== process.getuid()) ||
stat.size < 2 ||
stat.size > MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES
) {
throw new TypeError('Manual Primary runtime receipt file is unsafe');
}
return parseManualPrimaryRuntimeReceipt(
JSON.parse((await handle.readFile()).toString('utf8')),
);
} finally {
await handle.close();
}
}
private async write(receipt: ManualPrimaryRuntimeReceipt): Promise<void> {
const temporary = path.join(
this.root,
`.ql3-runtime-receipt-${this.pid}-${randomBytes(8).toString('hex')}.tmp`,
);
let handle: fs.FileHandle | undefined;
try {
handle = await fs.open(temporary, 'wx', 0o600);
await handle.writeFile(`${JSON.stringify(receipt)}\n`, 'utf8');
await handle.sync();
await handle.close();
handle = undefined;
await fs.rename(temporary, this.target);
await this.bestEffortSyncDirectory();
} finally {
await handle?.close().catch(() => undefined);
await fs.unlink(temporary).catch((error) => {
if (!isCode(error, 'ENOENT')) throw error;
});
}
}
private async bestEffortSyncDirectory(): Promise<void> {
try {
const handle = await fs.open(this.root, constants.O_RDONLY);
try {
await handle.sync();
} finally {
await handle.close();
}
} catch {
// The atomic state remains valid on filesystems without directory fsync.
}
}
}
@@ -10,6 +10,7 @@ import { installManualPrimaryExecutionRouter } from '../../compatibility/manualP
import type { RuntimeRolloutPolicy } from '../../domain/runtimeRollout';
import { parseDeploymentProfile } from '../../domain/deploymentProfile';
import type { RuntimeRolloutLoadResult } from '../../ports/runtimeRolloutLoader';
import type { ManualPrimaryRuntimeReceiptLifecycle } from '../../ports/manualPrimaryRuntimeReceipt';
import { loadRuntimeRolloutManifest } from '../fs/runtimeRolloutManifestLoader';
import type { DefaultManualPrimaryActivationOptions } from './defaultManualPrimaryActivation';
@@ -28,6 +29,7 @@ export interface BootstrapDefaultManualPrimaryRuntimeOptions
loadStack?: () => Promise<DefaultManualPrimaryStackModule>;
install?: typeof installManualPrimaryExecutionRouter;
audit?: (record: ManualPrimaryActivationAudit) => void | Promise<void>;
receipt?: ManualPrimaryRuntimeReceiptLifecycle;
}
/**
@@ -52,6 +54,7 @@ export async function bootstrapDefaultManualPrimaryRuntime(
Logger.info(`[runtime-activation] ${JSON.stringify(record)}`);
});
let stackModule: DefaultManualPrimaryStackModule | undefined;
let receipt: ManualPrimaryRuntimeReceiptLifecycle | undefined;
let selectedProfile: 'edge' | 'standalone' | undefined;
if (selected) {
try {
@@ -71,9 +74,23 @@ export async function bootstrapDefaultManualPrimaryRuntime(
);
}
selectedProfile = profile;
stackModule = await (
options.loadStack ?? (() => import('./defaultManualPrimaryActivation'))
)();
const [loadedStack, loadedReceipt] = await Promise.all([
(
options.loadStack ??
(() => import('./defaultManualPrimaryActivation'))
)(),
options.receipt === undefined
? import('../fs/manualPrimaryRuntimeReceiptStore').then(
({ ManualPrimaryRuntimeReceiptStore }) =>
new ManualPrimaryRuntimeReceiptStore(
config.configPath,
selectedProfile!,
),
)
: Promise.resolve(options.receipt),
]);
stackModule = loadedStack;
receipt = loadedReceipt;
} catch (error) {
try {
await audit({ ...load.audit, activation: 'failed' });
@@ -109,5 +126,6 @@ export async function bootstrapDefaultManualPrimaryRuntime(
},
install: options.install ?? installManualPrimaryExecutionRouter,
audit,
receipt,
});
}
@@ -4,6 +4,7 @@ import type {
RuntimeRolloutLoadAudit,
RuntimeRolloutLoadResult,
} from '../ports/runtimeRolloutLoader';
import type { ManualPrimaryRuntimeReceiptLifecycle } from '../ports/manualPrimaryRuntimeReceipt';
import type { PrimaryCancellationStopResult } from './primaryCancellationLifecycle';
import type { PrimaryCompletionReceiptStopResult } from './primaryCompletionReceiptLifecycle';
import type { PrimaryRunStartupSummary } from './primaryRunStartupSupervisor';
@@ -46,6 +47,7 @@ export interface ManualPrimaryRuntimeActivationOptions {
create(policy: RuntimeRolloutPolicy): ManualPrimaryActivationStack;
install(router: ManualPrimaryExecutionRouter): () => void;
audit(record: ManualPrimaryActivationAudit): void | Promise<void>;
receipt?: ManualPrimaryRuntimeReceiptLifecycle;
}
export interface ManualPrimaryRuntimeActivationResult {
@@ -155,36 +157,65 @@ export async function activateManualPrimaryRuntime(
throw new Error('Primary cancellation lifecycle did not start');
}
dispose = options.install(stack.router);
await options.receipt?.activated(load.audit);
await options.audit({
...load.audit,
activation: 'activated',
recovery: recoveryAudit(recovery),
});
let stopped = false;
let stopPromise: Promise<PrimaryCancellationStopResult> | undefined;
return {
load,
active: true,
recovery,
async stop() {
if (stopped) return 'drained';
stopped = true;
dispose?.();
const result = await stopLifecycles(stack!, {
completion: completionStarted,
timeout: timeoutStarted,
cancellation: cancellationStarted,
});
try {
await options.audit({
...load.audit,
activation: 'stopped',
recovery: recoveryAudit(recovery),
});
} catch {
// Cleanup must not be reversed by a diagnostic failure.
}
return result;
stopPromise ??= (async () => {
let receiptError: unknown;
try {
await options.receipt?.stopping();
} catch (error) {
receiptError = error;
}
dispose?.();
let result: PrimaryCancellationStopResult = 'drained';
let lifecycleError: unknown;
try {
result = await stopLifecycles(stack!, {
completion: completionStarted,
timeout: timeoutStarted,
cancellation: cancellationStarted,
});
} catch (error) {
lifecycleError = error;
}
if (receiptError === undefined && lifecycleError === undefined) {
try {
await options.receipt?.stopped();
} catch (error) {
receiptError = error;
}
}
if (receiptError !== undefined || lifecycleError !== undefined) {
try {
await options.receipt?.failed();
} catch {
// Preserve the first stop failure; a non-active receipt is best effort.
}
throw receiptError ?? lifecycleError;
}
try {
await options.audit({
...load.audit,
activation: 'stopped',
recovery: recoveryAudit(recovery),
});
} catch {
// Cleanup must not be reversed by a diagnostic failure.
}
return result;
})();
return stopPromise;
},
};
} catch (error) {
@@ -200,6 +231,11 @@ export async function activateManualPrimaryRuntime(
// Preserve the activation error after best-effort cleanup.
}
}
try {
await options.receipt?.failed();
} catch {
// Preserve the activation failure; liveness checks still reject stale state.
}
try {
await options.audit({ ...load.audit, activation: 'failed' });
} catch {
@@ -0,0 +1,268 @@
import { createHash } from 'crypto';
export const MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA =
'qinglong/manual-primary-runtime-receipt@v1';
export const MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE =
'qinglong3-manual-primary-runtime.json';
export const MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES = 8 * 1024;
export type ManualPrimaryRuntimeReceiptState =
| 'active'
| 'stopping'
| 'stopped'
| 'failed';
export type ManualPrimaryRuntimeProcessIdentity =
| {
kind: 'linux-proc';
platform: 'linux';
pid: number;
processGroupId: number;
bootId: string;
startTimeTicks: string;
}
| {
kind: 'portable';
platform: string;
pid: number;
};
export interface ManualPrimaryRuntimeReceipt {
schema: typeof MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA;
schemaVersion: 1;
activationId: string;
profile: 'edge' | 'standalone';
revision: string;
rolloutSourceSha256: string;
activatedAtMs: number;
updatedAtMs: number;
state: ManualPrimaryRuntimeReceiptState;
process: ManualPrimaryRuntimeProcessIdentity;
receiptSha256: string;
}
type ReceiptProjection = Omit<ManualPrimaryRuntimeReceipt, 'receiptSha256'>;
const SHA256_PATTERN = /^[a-f0-9]{64}$/u;
const ACTIVATION_ID_PATTERN = /^[a-f0-9]{32}$/u;
const BOOT_ID_PATTERN = /^[A-Za-z0-9-]{1,64}$/u;
const START_TICKS_PATTERN = /^\d{1,32}$/u;
const PLATFORM_PATTERN = /^[a-z0-9_-]{1,32}$/u;
function record(value: unknown, name: string): Record<string, unknown> {
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
throw new TypeError(`${name} must be an object`);
}
return value as Record<string, unknown>;
}
function exactKeys(
value: Record<string, unknown>,
keys: readonly string[],
name: string,
): void {
const actual = Object.keys(value).sort();
const expected = [...keys].sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index])
) {
throw new TypeError(`${name} has an invalid shape`);
}
}
function safePositiveInteger(value: unknown, name: string): number {
if (!Number.isSafeInteger(value) || (value as number) < 1) {
throw new TypeError(`${name} must be a positive safe integer`);
}
return value as number;
}
function safeTimestamp(value: unknown, name: string): number {
if (!Number.isSafeInteger(value) || (value as number) < 0) {
throw new TypeError(`${name} must be a non-negative safe integer`);
}
return value as number;
}
function parseProcessIdentity(
value: unknown,
): ManualPrimaryRuntimeProcessIdentity {
const identity = record(value, 'receipt.process');
if (identity.kind === 'linux-proc') {
exactKeys(
identity,
['kind', 'platform', 'pid', 'processGroupId', 'bootId', 'startTimeTicks'],
'receipt.process',
);
if (
identity.platform !== 'linux' ||
typeof identity.bootId !== 'string' ||
!BOOT_ID_PATTERN.test(identity.bootId) ||
typeof identity.startTimeTicks !== 'string' ||
!START_TICKS_PATTERN.test(identity.startTimeTicks) ||
BigInt(identity.startTimeTicks) < BigInt(1)
) {
throw new TypeError('receipt.process Linux identity is invalid');
}
return {
kind: 'linux-proc',
platform: 'linux',
pid: safePositiveInteger(identity.pid, 'receipt.process.pid'),
processGroupId: safePositiveInteger(
identity.processGroupId,
'receipt.process.processGroupId',
),
bootId: identity.bootId,
startTimeTicks: identity.startTimeTicks,
};
}
exactKeys(identity, ['kind', 'platform', 'pid'], 'receipt.process');
if (
identity.kind !== 'portable' ||
typeof identity.platform !== 'string' ||
!PLATFORM_PATTERN.test(identity.platform)
) {
throw new TypeError('receipt.process portable identity is invalid');
}
return {
kind: 'portable',
platform: identity.platform,
pid: safePositiveInteger(identity.pid, 'receipt.process.pid'),
};
}
function canonical(value: unknown): unknown {
if (Array.isArray(value)) return value.map(canonical);
if (value !== null && typeof value === 'object') {
return Object.fromEntries(
Object.entries(value as Record<string, unknown>)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, nested]) => [key, canonical(nested)]),
);
}
return value;
}
function digest(projection: ReceiptProjection): string {
return createHash('sha256')
.update('qinglong.manual-primary-runtime-receipt.v1\0', 'utf8')
.update(JSON.stringify(canonical(projection)), 'utf8')
.digest('hex');
}
export function createManualPrimaryRuntimeReceipt(input: {
activationId: string;
profile: 'edge' | 'standalone';
revision: string;
rolloutSourceSha256: string;
activatedAtMs: number;
process: ManualPrimaryRuntimeProcessIdentity;
}): ManualPrimaryRuntimeReceipt {
const projection: ReceiptProjection = {
schema: MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA,
schemaVersion: 1,
activationId: input.activationId,
profile: input.profile,
revision: input.revision,
rolloutSourceSha256: input.rolloutSourceSha256,
activatedAtMs: input.activatedAtMs,
updatedAtMs: input.activatedAtMs,
state: 'active',
process: input.process,
};
return parseManualPrimaryRuntimeReceipt({
...projection,
receiptSha256: digest(projection),
});
}
export function transitionManualPrimaryRuntimeReceipt(
receipt: ManualPrimaryRuntimeReceipt,
state: Exclude<ManualPrimaryRuntimeReceiptState, 'active'>,
updatedAtMs: number,
): ManualPrimaryRuntimeReceipt {
const current = parseManualPrimaryRuntimeReceipt(receipt);
const projection: ReceiptProjection = {
schema: current.schema,
schemaVersion: 1,
activationId: current.activationId,
profile: current.profile,
revision: current.revision,
rolloutSourceSha256: current.rolloutSourceSha256,
activatedAtMs: current.activatedAtMs,
updatedAtMs,
state,
process: current.process,
};
return parseManualPrimaryRuntimeReceipt({
...projection,
receiptSha256: digest(projection),
});
}
export function parseManualPrimaryRuntimeReceipt(
value: unknown,
): ManualPrimaryRuntimeReceipt {
const receipt = record(value, 'receipt');
exactKeys(
receipt,
[
'schema',
'schemaVersion',
'activationId',
'profile',
'revision',
'rolloutSourceSha256',
'activatedAtMs',
'updatedAtMs',
'state',
'process',
'receiptSha256',
],
'receipt',
);
const activatedAtMs = safeTimestamp(
receipt.activatedAtMs,
'receipt.activatedAtMs',
);
const updatedAtMs = safeTimestamp(receipt.updatedAtMs, 'receipt.updatedAtMs');
if (
receipt.schema !== MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA ||
receipt.schemaVersion !== 1 ||
typeof receipt.activationId !== 'string' ||
!ACTIVATION_ID_PATTERN.test(receipt.activationId) ||
(receipt.profile !== 'edge' && receipt.profile !== 'standalone') ||
typeof receipt.revision !== 'string' ||
receipt.revision.length < 1 ||
receipt.revision.length > 128 ||
/[\u0000-\u001f\u007f]/u.test(receipt.revision) ||
typeof receipt.rolloutSourceSha256 !== 'string' ||
!SHA256_PATTERN.test(receipt.rolloutSourceSha256) ||
!['active', 'stopping', 'stopped', 'failed'].includes(
receipt.state as string,
) ||
typeof receipt.receiptSha256 !== 'string' ||
!SHA256_PATTERN.test(receipt.receiptSha256) ||
updatedAtMs < activatedAtMs ||
(receipt.state === 'active' && updatedAtMs !== activatedAtMs)
) {
throw new TypeError('Manual Primary runtime receipt is invalid');
}
const projection: ReceiptProjection = {
schema: MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA,
schemaVersion: 1,
activationId: receipt.activationId,
profile: receipt.profile,
revision: receipt.revision,
rolloutSourceSha256: receipt.rolloutSourceSha256,
activatedAtMs,
updatedAtMs,
state: receipt.state as ManualPrimaryRuntimeReceiptState,
process: parseProcessIdentity(receipt.process),
};
if (digest(projection) !== receipt.receiptSha256) {
throw new TypeError('Manual Primary runtime receipt digest is invalid');
}
return { ...projection, receiptSha256: receipt.receiptSha256 };
}
@@ -0,0 +1,8 @@
import type { RuntimeRolloutLoadAudit } from './runtimeRolloutLoader';
export interface ManualPrimaryRuntimeReceiptLifecycle {
activated(audit: RuntimeRolloutLoadAudit): Promise<void>;
stopping(): Promise<void>;
stopped(): Promise<void>;
failed(): Promise<void>;
}