mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): persist manual primary activation
This commit is contained in:
whyour
@@ -0,0 +1,261 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { constants } from 'fs';
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import {
|
||||
createManualPrimaryRuntimeReceipt,
|
||||
MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE,
|
||||
MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES,
|
||||
parseManualPrimaryRuntimeReceipt,
|
||||
transitionManualPrimaryRuntimeReceipt,
|
||||
type ManualPrimaryRuntimeProcessIdentity,
|
||||
type ManualPrimaryRuntimeReceipt,
|
||||
type ManualPrimaryRuntimeReceiptState,
|
||||
} from '../../domain/manualPrimaryRuntimeReceipt';
|
||||
import type { ManualPrimaryRuntimeReceiptLifecycle } from '../../ports/manualPrimaryRuntimeReceipt';
|
||||
import type { RuntimeRolloutLoadAudit } from '../../ports/runtimeRolloutLoader';
|
||||
import {
|
||||
LinuxProcProcessIdentityProvider,
|
||||
type LocalProcessIdentityProvider,
|
||||
} from '../local-process/localProcessIdentity';
|
||||
|
||||
export interface ManualPrimaryRuntimeReceiptStoreOptions {
|
||||
clock?: { now(): number };
|
||||
identityProvider?: LocalProcessIdentityProvider;
|
||||
platform?: NodeJS.Platform;
|
||||
pid?: number;
|
||||
randomId?: () => string;
|
||||
}
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
typeof error === 'object' &&
|
||||
error !== null &&
|
||||
'code' in error &&
|
||||
(error as { code?: string }).code === code
|
||||
);
|
||||
}
|
||||
|
||||
export class ManualPrimaryRuntimeReceiptConflictError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = 'ManualPrimaryRuntimeReceiptConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ManualPrimaryRuntimeReceiptStore
|
||||
implements ManualPrimaryRuntimeReceiptLifecycle
|
||||
{
|
||||
private readonly target: string;
|
||||
private readonly clock: { now(): number };
|
||||
private readonly identityProvider: LocalProcessIdentityProvider;
|
||||
private readonly platform: NodeJS.Platform;
|
||||
private readonly pid: number;
|
||||
private readonly randomId: () => string;
|
||||
private current?: ManualPrimaryRuntimeReceipt;
|
||||
|
||||
constructor(
|
||||
private readonly root: string,
|
||||
private readonly profile: 'edge' | 'standalone',
|
||||
options: ManualPrimaryRuntimeReceiptStoreOptions = {},
|
||||
) {
|
||||
if (!path.isAbsolute(root)) {
|
||||
throw new TypeError(
|
||||
'Manual Primary runtime receipt root must be absolute',
|
||||
);
|
||||
}
|
||||
this.target = path.join(root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE);
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.identityProvider =
|
||||
options.identityProvider ?? new LinuxProcProcessIdentityProvider();
|
||||
this.platform = options.platform ?? process.platform;
|
||||
this.pid = options.pid ?? process.pid;
|
||||
this.randomId = options.randomId ?? (() => randomBytes(16).toString('hex'));
|
||||
}
|
||||
|
||||
async activated(audit: RuntimeRolloutLoadAudit): Promise<void> {
|
||||
if (
|
||||
typeof audit.revision !== 'string' ||
|
||||
typeof audit.sourceSha256 !== 'string'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Accepted rollout audit lacks durable revision or source digest',
|
||||
);
|
||||
}
|
||||
await this.assertRoot();
|
||||
const processIdentity = await this.captureProcessIdentity();
|
||||
const existing = await this.read();
|
||||
await this.assertReplaceable(existing);
|
||||
const receipt = createManualPrimaryRuntimeReceipt({
|
||||
activationId: this.randomId(),
|
||||
profile: this.profile,
|
||||
revision: audit.revision,
|
||||
rolloutSourceSha256: audit.sourceSha256,
|
||||
activatedAtMs: this.now(),
|
||||
process: processIdentity,
|
||||
});
|
||||
await this.write(receipt);
|
||||
this.current = receipt;
|
||||
}
|
||||
|
||||
async stopping(): Promise<void> {
|
||||
await this.transition('stopping');
|
||||
}
|
||||
|
||||
async stopped(): Promise<void> {
|
||||
await this.transition('stopped');
|
||||
}
|
||||
|
||||
async failed(): Promise<void> {
|
||||
if (this.current === undefined) return;
|
||||
await this.transition('failed');
|
||||
}
|
||||
|
||||
private async transition(
|
||||
state: Exclude<ManualPrimaryRuntimeReceiptState, 'active'>,
|
||||
): Promise<void> {
|
||||
if (this.current === undefined) {
|
||||
throw new ManualPrimaryRuntimeReceiptConflictError(
|
||||
'Manual Primary runtime receipt was not activated by this process',
|
||||
);
|
||||
}
|
||||
const observed = await this.read();
|
||||
if (
|
||||
observed === undefined ||
|
||||
observed.activationId !== this.current.activationId ||
|
||||
observed.receiptSha256 !== this.current.receiptSha256
|
||||
) {
|
||||
throw new ManualPrimaryRuntimeReceiptConflictError(
|
||||
'Manual Primary runtime receipt ownership changed',
|
||||
);
|
||||
}
|
||||
const next = transitionManualPrimaryRuntimeReceipt(
|
||||
this.current,
|
||||
state,
|
||||
Math.max(this.current.updatedAtMs, this.now()),
|
||||
);
|
||||
await this.write(next);
|
||||
this.current = next;
|
||||
}
|
||||
|
||||
private now(): number {
|
||||
const value = this.clock.now();
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new TypeError('Manual Primary runtime receipt clock is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private async captureProcessIdentity(): Promise<ManualPrimaryRuntimeProcessIdentity> {
|
||||
const identity = await this.identityProvider.capture(this.pid);
|
||||
if (identity !== null) {
|
||||
return { kind: 'linux-proc', ...identity };
|
||||
}
|
||||
if (this.platform === 'linux') {
|
||||
throw new Error('Linux process identity is unavailable');
|
||||
}
|
||||
return { kind: 'portable', platform: this.platform, pid: this.pid };
|
||||
}
|
||||
|
||||
private async assertReplaceable(
|
||||
existing: ManualPrimaryRuntimeReceipt | undefined,
|
||||
): Promise<void> {
|
||||
if (
|
||||
existing === undefined ||
|
||||
existing.state === 'stopped' ||
|
||||
existing.state === 'failed'
|
||||
) {
|
||||
return;
|
||||
}
|
||||
if (existing.process.kind !== 'linux-proc') return;
|
||||
const inspection = await this.identityProvider.inspect(existing.process);
|
||||
if (inspection.status === 'running') {
|
||||
throw new ManualPrimaryRuntimeReceiptConflictError(
|
||||
'Another Manual Primary runtime receipt is still live',
|
||||
);
|
||||
}
|
||||
if (!['exited', 'identity_mismatch'].includes(inspection.status)) {
|
||||
throw new ManualPrimaryRuntimeReceiptConflictError(
|
||||
'Previous Manual Primary runtime identity cannot be disproved',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async assertRoot(): Promise<void> {
|
||||
const stat = await fs.lstat(this.root);
|
||||
if (
|
||||
!stat.isDirectory() ||
|
||||
stat.isSymbolicLink() ||
|
||||
(stat.mode & 0o022) !== 0 ||
|
||||
(typeof process.getuid === 'function' && stat.uid !== process.getuid())
|
||||
) {
|
||||
throw new TypeError('Manual Primary runtime receipt root is unsafe');
|
||||
}
|
||||
}
|
||||
|
||||
private async read(): Promise<ManualPrimaryRuntimeReceipt | undefined> {
|
||||
let handle: fs.FileHandle;
|
||||
try {
|
||||
handle = await fs.open(
|
||||
this.target,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
} catch (error) {
|
||||
if (isCode(error, 'ENOENT')) return undefined;
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
(stat.mode & 0o077) !== 0 ||
|
||||
(typeof process.getuid === 'function' &&
|
||||
stat.uid !== process.getuid()) ||
|
||||
stat.size < 2 ||
|
||||
stat.size > MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES
|
||||
) {
|
||||
throw new TypeError('Manual Primary runtime receipt file is unsafe');
|
||||
}
|
||||
return parseManualPrimaryRuntimeReceipt(
|
||||
JSON.parse((await handle.readFile()).toString('utf8')),
|
||||
);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
private async write(receipt: ManualPrimaryRuntimeReceipt): Promise<void> {
|
||||
const temporary = path.join(
|
||||
this.root,
|
||||
`.ql3-runtime-receipt-${this.pid}-${randomBytes(8).toString('hex')}.tmp`,
|
||||
);
|
||||
let handle: fs.FileHandle | undefined;
|
||||
try {
|
||||
handle = await fs.open(temporary, 'wx', 0o600);
|
||||
await handle.writeFile(`${JSON.stringify(receipt)}\n`, 'utf8');
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
handle = undefined;
|
||||
await fs.rename(temporary, this.target);
|
||||
await this.bestEffortSyncDirectory();
|
||||
} finally {
|
||||
await handle?.close().catch(() => undefined);
|
||||
await fs.unlink(temporary).catch((error) => {
|
||||
if (!isCode(error, 'ENOENT')) throw error;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async bestEffortSyncDirectory(): Promise<void> {
|
||||
try {
|
||||
const handle = await fs.open(this.root, constants.O_RDONLY);
|
||||
try {
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
} catch {
|
||||
// The atomic state remains valid on filesystems without directory fsync.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import { installManualPrimaryExecutionRouter } from '../../compatibility/manualP
|
||||
import type { RuntimeRolloutPolicy } from '../../domain/runtimeRollout';
|
||||
import { parseDeploymentProfile } from '../../domain/deploymentProfile';
|
||||
import type { RuntimeRolloutLoadResult } from '../../ports/runtimeRolloutLoader';
|
||||
import type { ManualPrimaryRuntimeReceiptLifecycle } from '../../ports/manualPrimaryRuntimeReceipt';
|
||||
import { loadRuntimeRolloutManifest } from '../fs/runtimeRolloutManifestLoader';
|
||||
import type { DefaultManualPrimaryActivationOptions } from './defaultManualPrimaryActivation';
|
||||
|
||||
@@ -28,6 +29,7 @@ export interface BootstrapDefaultManualPrimaryRuntimeOptions
|
||||
loadStack?: () => Promise<DefaultManualPrimaryStackModule>;
|
||||
install?: typeof installManualPrimaryExecutionRouter;
|
||||
audit?: (record: ManualPrimaryActivationAudit) => void | Promise<void>;
|
||||
receipt?: ManualPrimaryRuntimeReceiptLifecycle;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -52,6 +54,7 @@ export async function bootstrapDefaultManualPrimaryRuntime(
|
||||
Logger.info(`[runtime-activation] ${JSON.stringify(record)}`);
|
||||
});
|
||||
let stackModule: DefaultManualPrimaryStackModule | undefined;
|
||||
let receipt: ManualPrimaryRuntimeReceiptLifecycle | undefined;
|
||||
let selectedProfile: 'edge' | 'standalone' | undefined;
|
||||
if (selected) {
|
||||
try {
|
||||
@@ -71,9 +74,23 @@ export async function bootstrapDefaultManualPrimaryRuntime(
|
||||
);
|
||||
}
|
||||
selectedProfile = profile;
|
||||
stackModule = await (
|
||||
options.loadStack ?? (() => import('./defaultManualPrimaryActivation'))
|
||||
)();
|
||||
const [loadedStack, loadedReceipt] = await Promise.all([
|
||||
(
|
||||
options.loadStack ??
|
||||
(() => import('./defaultManualPrimaryActivation'))
|
||||
)(),
|
||||
options.receipt === undefined
|
||||
? import('../fs/manualPrimaryRuntimeReceiptStore').then(
|
||||
({ ManualPrimaryRuntimeReceiptStore }) =>
|
||||
new ManualPrimaryRuntimeReceiptStore(
|
||||
config.configPath,
|
||||
selectedProfile!,
|
||||
),
|
||||
)
|
||||
: Promise.resolve(options.receipt),
|
||||
]);
|
||||
stackModule = loadedStack;
|
||||
receipt = loadedReceipt;
|
||||
} catch (error) {
|
||||
try {
|
||||
await audit({ ...load.audit, activation: 'failed' });
|
||||
@@ -109,5 +126,6 @@ export async function bootstrapDefaultManualPrimaryRuntime(
|
||||
},
|
||||
install: options.install ?? installManualPrimaryExecutionRouter,
|
||||
audit,
|
||||
receipt,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import type {
|
||||
RuntimeRolloutLoadAudit,
|
||||
RuntimeRolloutLoadResult,
|
||||
} from '../ports/runtimeRolloutLoader';
|
||||
import type { ManualPrimaryRuntimeReceiptLifecycle } from '../ports/manualPrimaryRuntimeReceipt';
|
||||
import type { PrimaryCancellationStopResult } from './primaryCancellationLifecycle';
|
||||
import type { PrimaryCompletionReceiptStopResult } from './primaryCompletionReceiptLifecycle';
|
||||
import type { PrimaryRunStartupSummary } from './primaryRunStartupSupervisor';
|
||||
@@ -46,6 +47,7 @@ export interface ManualPrimaryRuntimeActivationOptions {
|
||||
create(policy: RuntimeRolloutPolicy): ManualPrimaryActivationStack;
|
||||
install(router: ManualPrimaryExecutionRouter): () => void;
|
||||
audit(record: ManualPrimaryActivationAudit): void | Promise<void>;
|
||||
receipt?: ManualPrimaryRuntimeReceiptLifecycle;
|
||||
}
|
||||
|
||||
export interface ManualPrimaryRuntimeActivationResult {
|
||||
@@ -155,26 +157,53 @@ export async function activateManualPrimaryRuntime(
|
||||
throw new Error('Primary cancellation lifecycle did not start');
|
||||
}
|
||||
dispose = options.install(stack.router);
|
||||
await options.receipt?.activated(load.audit);
|
||||
await options.audit({
|
||||
...load.audit,
|
||||
activation: 'activated',
|
||||
recovery: recoveryAudit(recovery),
|
||||
});
|
||||
|
||||
let stopped = false;
|
||||
let stopPromise: Promise<PrimaryCancellationStopResult> | undefined;
|
||||
return {
|
||||
load,
|
||||
active: true,
|
||||
recovery,
|
||||
async stop() {
|
||||
if (stopped) return 'drained';
|
||||
stopped = true;
|
||||
stopPromise ??= (async () => {
|
||||
let receiptError: unknown;
|
||||
try {
|
||||
await options.receipt?.stopping();
|
||||
} catch (error) {
|
||||
receiptError = error;
|
||||
}
|
||||
dispose?.();
|
||||
const result = await stopLifecycles(stack!, {
|
||||
let result: PrimaryCancellationStopResult = 'drained';
|
||||
let lifecycleError: unknown;
|
||||
try {
|
||||
result = await stopLifecycles(stack!, {
|
||||
completion: completionStarted,
|
||||
timeout: timeoutStarted,
|
||||
cancellation: cancellationStarted,
|
||||
});
|
||||
} catch (error) {
|
||||
lifecycleError = error;
|
||||
}
|
||||
if (receiptError === undefined && lifecycleError === undefined) {
|
||||
try {
|
||||
await options.receipt?.stopped();
|
||||
} catch (error) {
|
||||
receiptError = error;
|
||||
}
|
||||
}
|
||||
if (receiptError !== undefined || lifecycleError !== undefined) {
|
||||
try {
|
||||
await options.receipt?.failed();
|
||||
} catch {
|
||||
// Preserve the first stop failure; a non-active receipt is best effort.
|
||||
}
|
||||
throw receiptError ?? lifecycleError;
|
||||
}
|
||||
try {
|
||||
await options.audit({
|
||||
...load.audit,
|
||||
@@ -185,6 +214,8 @@ export async function activateManualPrimaryRuntime(
|
||||
// Cleanup must not be reversed by a diagnostic failure.
|
||||
}
|
||||
return result;
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
@@ -200,6 +231,11 @@ export async function activateManualPrimaryRuntime(
|
||||
// Preserve the activation error after best-effort cleanup.
|
||||
}
|
||||
}
|
||||
try {
|
||||
await options.receipt?.failed();
|
||||
} catch {
|
||||
// Preserve the activation failure; liveness checks still reject stale state.
|
||||
}
|
||||
try {
|
||||
await options.audit({ ...load.audit, activation: 'failed' });
|
||||
} catch {
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
export const MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA =
|
||||
'qinglong/manual-primary-runtime-receipt@v1';
|
||||
export const MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE =
|
||||
'qinglong3-manual-primary-runtime.json';
|
||||
export const MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES = 8 * 1024;
|
||||
|
||||
export type ManualPrimaryRuntimeReceiptState =
|
||||
| 'active'
|
||||
| 'stopping'
|
||||
| 'stopped'
|
||||
| 'failed';
|
||||
|
||||
export type ManualPrimaryRuntimeProcessIdentity =
|
||||
| {
|
||||
kind: 'linux-proc';
|
||||
platform: 'linux';
|
||||
pid: number;
|
||||
processGroupId: number;
|
||||
bootId: string;
|
||||
startTimeTicks: string;
|
||||
}
|
||||
| {
|
||||
kind: 'portable';
|
||||
platform: string;
|
||||
pid: number;
|
||||
};
|
||||
|
||||
export interface ManualPrimaryRuntimeReceipt {
|
||||
schema: typeof MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA;
|
||||
schemaVersion: 1;
|
||||
activationId: string;
|
||||
profile: 'edge' | 'standalone';
|
||||
revision: string;
|
||||
rolloutSourceSha256: string;
|
||||
activatedAtMs: number;
|
||||
updatedAtMs: number;
|
||||
state: ManualPrimaryRuntimeReceiptState;
|
||||
process: ManualPrimaryRuntimeProcessIdentity;
|
||||
receiptSha256: string;
|
||||
}
|
||||
|
||||
type ReceiptProjection = Omit<ManualPrimaryRuntimeReceipt, 'receiptSha256'>;
|
||||
|
||||
const SHA256_PATTERN = /^[a-f0-9]{64}$/u;
|
||||
const ACTIVATION_ID_PATTERN = /^[a-f0-9]{32}$/u;
|
||||
const BOOT_ID_PATTERN = /^[A-Za-z0-9-]{1,64}$/u;
|
||||
const START_TICKS_PATTERN = /^\d{1,32}$/u;
|
||||
const PLATFORM_PATTERN = /^[a-z0-9_-]{1,32}$/u;
|
||||
|
||||
function record(value: unknown, name: string): Record<string, unknown> {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError(`${name} must be an object`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function exactKeys(
|
||||
value: Record<string, unknown>,
|
||||
keys: readonly string[],
|
||||
name: string,
|
||||
): void {
|
||||
const actual = Object.keys(value).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) {
|
||||
throw new TypeError(`${name} has an invalid shape`);
|
||||
}
|
||||
}
|
||||
|
||||
function safePositiveInteger(value: unknown, name: string): number {
|
||||
if (!Number.isSafeInteger(value) || (value as number) < 1) {
|
||||
throw new TypeError(`${name} must be a positive safe integer`);
|
||||
}
|
||||
return value as number;
|
||||
}
|
||||
|
||||
function safeTimestamp(value: unknown, name: string): number {
|
||||
if (!Number.isSafeInteger(value) || (value as number) < 0) {
|
||||
throw new TypeError(`${name} must be a non-negative safe integer`);
|
||||
}
|
||||
return value as number;
|
||||
}
|
||||
|
||||
function parseProcessIdentity(
|
||||
value: unknown,
|
||||
): ManualPrimaryRuntimeProcessIdentity {
|
||||
const identity = record(value, 'receipt.process');
|
||||
if (identity.kind === 'linux-proc') {
|
||||
exactKeys(
|
||||
identity,
|
||||
['kind', 'platform', 'pid', 'processGroupId', 'bootId', 'startTimeTicks'],
|
||||
'receipt.process',
|
||||
);
|
||||
if (
|
||||
identity.platform !== 'linux' ||
|
||||
typeof identity.bootId !== 'string' ||
|
||||
!BOOT_ID_PATTERN.test(identity.bootId) ||
|
||||
typeof identity.startTimeTicks !== 'string' ||
|
||||
!START_TICKS_PATTERN.test(identity.startTimeTicks) ||
|
||||
BigInt(identity.startTimeTicks) < BigInt(1)
|
||||
) {
|
||||
throw new TypeError('receipt.process Linux identity is invalid');
|
||||
}
|
||||
return {
|
||||
kind: 'linux-proc',
|
||||
platform: 'linux',
|
||||
pid: safePositiveInteger(identity.pid, 'receipt.process.pid'),
|
||||
processGroupId: safePositiveInteger(
|
||||
identity.processGroupId,
|
||||
'receipt.process.processGroupId',
|
||||
),
|
||||
bootId: identity.bootId,
|
||||
startTimeTicks: identity.startTimeTicks,
|
||||
};
|
||||
}
|
||||
exactKeys(identity, ['kind', 'platform', 'pid'], 'receipt.process');
|
||||
if (
|
||||
identity.kind !== 'portable' ||
|
||||
typeof identity.platform !== 'string' ||
|
||||
!PLATFORM_PATTERN.test(identity.platform)
|
||||
) {
|
||||
throw new TypeError('receipt.process portable identity is invalid');
|
||||
}
|
||||
return {
|
||||
kind: 'portable',
|
||||
platform: identity.platform,
|
||||
pid: safePositiveInteger(identity.pid, 'receipt.process.pid'),
|
||||
};
|
||||
}
|
||||
|
||||
function canonical(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(canonical);
|
||||
if (value !== null && typeof value === 'object') {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value as Record<string, unknown>)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([key, nested]) => [key, canonical(nested)]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function digest(projection: ReceiptProjection): string {
|
||||
return createHash('sha256')
|
||||
.update('qinglong.manual-primary-runtime-receipt.v1\0', 'utf8')
|
||||
.update(JSON.stringify(canonical(projection)), 'utf8')
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
export function createManualPrimaryRuntimeReceipt(input: {
|
||||
activationId: string;
|
||||
profile: 'edge' | 'standalone';
|
||||
revision: string;
|
||||
rolloutSourceSha256: string;
|
||||
activatedAtMs: number;
|
||||
process: ManualPrimaryRuntimeProcessIdentity;
|
||||
}): ManualPrimaryRuntimeReceipt {
|
||||
const projection: ReceiptProjection = {
|
||||
schema: MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA,
|
||||
schemaVersion: 1,
|
||||
activationId: input.activationId,
|
||||
profile: input.profile,
|
||||
revision: input.revision,
|
||||
rolloutSourceSha256: input.rolloutSourceSha256,
|
||||
activatedAtMs: input.activatedAtMs,
|
||||
updatedAtMs: input.activatedAtMs,
|
||||
state: 'active',
|
||||
process: input.process,
|
||||
};
|
||||
return parseManualPrimaryRuntimeReceipt({
|
||||
...projection,
|
||||
receiptSha256: digest(projection),
|
||||
});
|
||||
}
|
||||
|
||||
export function transitionManualPrimaryRuntimeReceipt(
|
||||
receipt: ManualPrimaryRuntimeReceipt,
|
||||
state: Exclude<ManualPrimaryRuntimeReceiptState, 'active'>,
|
||||
updatedAtMs: number,
|
||||
): ManualPrimaryRuntimeReceipt {
|
||||
const current = parseManualPrimaryRuntimeReceipt(receipt);
|
||||
const projection: ReceiptProjection = {
|
||||
schema: current.schema,
|
||||
schemaVersion: 1,
|
||||
activationId: current.activationId,
|
||||
profile: current.profile,
|
||||
revision: current.revision,
|
||||
rolloutSourceSha256: current.rolloutSourceSha256,
|
||||
activatedAtMs: current.activatedAtMs,
|
||||
updatedAtMs,
|
||||
state,
|
||||
process: current.process,
|
||||
};
|
||||
return parseManualPrimaryRuntimeReceipt({
|
||||
...projection,
|
||||
receiptSha256: digest(projection),
|
||||
});
|
||||
}
|
||||
|
||||
export function parseManualPrimaryRuntimeReceipt(
|
||||
value: unknown,
|
||||
): ManualPrimaryRuntimeReceipt {
|
||||
const receipt = record(value, 'receipt');
|
||||
exactKeys(
|
||||
receipt,
|
||||
[
|
||||
'schema',
|
||||
'schemaVersion',
|
||||
'activationId',
|
||||
'profile',
|
||||
'revision',
|
||||
'rolloutSourceSha256',
|
||||
'activatedAtMs',
|
||||
'updatedAtMs',
|
||||
'state',
|
||||
'process',
|
||||
'receiptSha256',
|
||||
],
|
||||
'receipt',
|
||||
);
|
||||
const activatedAtMs = safeTimestamp(
|
||||
receipt.activatedAtMs,
|
||||
'receipt.activatedAtMs',
|
||||
);
|
||||
const updatedAtMs = safeTimestamp(receipt.updatedAtMs, 'receipt.updatedAtMs');
|
||||
if (
|
||||
receipt.schema !== MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA ||
|
||||
receipt.schemaVersion !== 1 ||
|
||||
typeof receipt.activationId !== 'string' ||
|
||||
!ACTIVATION_ID_PATTERN.test(receipt.activationId) ||
|
||||
(receipt.profile !== 'edge' && receipt.profile !== 'standalone') ||
|
||||
typeof receipt.revision !== 'string' ||
|
||||
receipt.revision.length < 1 ||
|
||||
receipt.revision.length > 128 ||
|
||||
/[\u0000-\u001f\u007f]/u.test(receipt.revision) ||
|
||||
typeof receipt.rolloutSourceSha256 !== 'string' ||
|
||||
!SHA256_PATTERN.test(receipt.rolloutSourceSha256) ||
|
||||
!['active', 'stopping', 'stopped', 'failed'].includes(
|
||||
receipt.state as string,
|
||||
) ||
|
||||
typeof receipt.receiptSha256 !== 'string' ||
|
||||
!SHA256_PATTERN.test(receipt.receiptSha256) ||
|
||||
updatedAtMs < activatedAtMs ||
|
||||
(receipt.state === 'active' && updatedAtMs !== activatedAtMs)
|
||||
) {
|
||||
throw new TypeError('Manual Primary runtime receipt is invalid');
|
||||
}
|
||||
const projection: ReceiptProjection = {
|
||||
schema: MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA,
|
||||
schemaVersion: 1,
|
||||
activationId: receipt.activationId,
|
||||
profile: receipt.profile,
|
||||
revision: receipt.revision,
|
||||
rolloutSourceSha256: receipt.rolloutSourceSha256,
|
||||
activatedAtMs,
|
||||
updatedAtMs,
|
||||
state: receipt.state as ManualPrimaryRuntimeReceiptState,
|
||||
process: parseProcessIdentity(receipt.process),
|
||||
};
|
||||
if (digest(projection) !== receipt.receiptSha256) {
|
||||
throw new TypeError('Manual Primary runtime receipt digest is invalid');
|
||||
}
|
||||
return { ...projection, receiptSha256: receipt.receiptSha256 };
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import type { RuntimeRolloutLoadAudit } from './runtimeRolloutLoader';
|
||||
|
||||
export interface ManualPrimaryRuntimeReceiptLifecycle {
|
||||
activated(audit: RuntimeRolloutLoadAudit): Promise<void>;
|
||||
stopping(): Promise<void>;
|
||||
stopped(): Promise<void>;
|
||||
failed(): Promise<void>;
|
||||
}
|
||||
@@ -11,6 +11,18 @@
|
||||
|
||||
最新增量证据（2026-08-19）：
|
||||
|
||||
- D-362/ADR-0455（已接受；首次真实目标实例执行待运维）：manual Primary 默认 bootstrap 在 router 安装后、`activated` 审计前原子发布
|
||||
`qinglong/manual-primary-runtime-receipt@v1` 当前状态，并在清理前后推进 `active → stopping → stopped`；激活或停止失败收敛为 `failed`。receipt 固定为 config root
|
||||
内一个 `0600`、8 KiB 上限、有 domain-separated 自摘要的 observed-state projection，绑定 Profile、manifest revision 与原始 SHA-256，但不成为第二 rollout authority。
|
||||
Linux 以 boot ID、PID、process-group 与 start ticks 联合复验，已有 live identity 时拒绝替换，旧 identity 确定退出后才允许新 generation 接管；非 Linux portable
|
||||
receipt 可用于开发启动，但独立 auditor 必须返回 unsupported，不能宣称 current。canary auditor 新增 `--require=active`，同时复核 plan/qualification/selection/live manifest、
|
||||
receipt binding 与进程 current；`off|rolled-back` 也会拒绝仍存活的旧 Primary。由此明确了无 watcher 约束：审批过期/rollback 使下一次 bootstrap 为 off，但不会自动卸载已运行
|
||||
router，operator 必须停止/重启后再闭合审计。默认关闭路径零新增 I/O/timer/watcher/连接，不新增 package、依赖、schema、migration 或部署对象。
|
||||
阶段门已重跑：聚焦 `25/25`、`build:back`、完整 backend `1,487 pass / 0 fail / 2 conditional skip`、18-package clean build/test、四项架构审计与 `14/14`
|
||||
artifact audit 全部通过；14 档字节与 D-361 一致。隔离 frozen-dependency Linux arm64 Docker 的 128 MiB router/256 MiB Edge release peak 分别为
|
||||
`87,339,008 / 145,506,304` bytes，`memory.events max/oom/oom_kill` 增量均为 0。D-362 不改 PostgreSQL schema/migration、依赖树或 Kubernetes 拓扑，故不重跑
|
||||
PostgreSQL HA；相邻 D-359 的 `142/142` 与 timeline `1→2` 仅作为既有证据。
|
||||
|
||||
- D-361/ADR-0454（已接受；首次真实用户目标实例执行待运维）：把 D-360 的 manual Primary evidence/gate 收敛为一次显式、可重放且不新增常驻组件的目标实例仪式。
|
||||
`prepare → observe → resource → qualify → approve → status/audit → rollback` 固定为七个 operator-driven 阶段；Edge 精确 8 条，Standalone 由维护者在
|
||||
32–128 条内选择精确目标。prepare 绑定当时 absent/disabled rollout 基线并只输出 `QL_DEPLOYMENT_PROFILE`、`QL3_SHADOW_ORIGINS=manual` 和唯一
|
||||
@@ -8332,10 +8344,10 @@ Primary 不使用普通环境变量或宽泛全局开关启用。孵化配置面
|
||||
- edge 不启动文件 watcher。配置只在显式 bootstrap/reload 时读取，禁用时不创建 router、timer、连接或后台任务。
|
||||
- ADR-0454 的 `prepare` 先以不可变 plan 绑定 Profile、精确 admission 目标和当时 absent/disabled rollout SHA-256；`qualify` 再绑定 capture/terminal/resource 的原始文件摘要、
|
||||
canonical 摘要和 Primary gate。只有显式 `approve` 能在复验 live 基线后写入本节 schema v2 manifest。
|
||||
- `approve` 的 durable 结果是 `primary_selected`，不是 `primary_active`。operator 必须重启目标 worker，并从现有 bootstrap 的同 revision `selected → reconciled → activated`
|
||||
结构化审计证明实际运行态；一次性独立 auditor 刻意不输出 active 结论。
|
||||
- `approve` 的 durable 结果是 `primary_selected`，不是 `primary_active`。operator 必须重启目标 worker；ADR-0455 固定 bootstrap 的同 revision
|
||||
`selected → reconciled → router installed → durable active receipt → activated`，独立 auditor 只有在 receipt binding 与 Linux 联合进程身份均成立时才通过 `--require=active`。
|
||||
- `rollback` 在 live manifest 替换前先写绑定 enabled/disabled 摘要的 intent，替换后再写绑定 intent 摘要的 completion；同参数重放覆盖两个 response-loss 窗口。审批过期按 off
|
||||
解释但不会自动改写文件，仍需显式 rollback 和重启。
|
||||
解释但不会自动改写文件，也不会让既有无 watcher 进程自动卸载 router，仍需显式 rollback 和停止/重启；`off|rolled-back` 审计拒绝仍 current 的 runtime receipt。
|
||||
|
||||
实验 manifest 结构固定为：
|
||||
|
||||
@@ -8372,10 +8384,11 @@ Primary 不使用普通环境变量或宽泛全局开关启用。孵化配置面
|
||||
当前 `next` 已在 HTTP worker 接入轻量 manifest bootstrap。文件缺失、disabled、rejected、manual 非 primary、v1 enabled 或 Primary bundle
|
||||
缺失/篡改/不可重放时不会加载完整 Runtime stack、创建 router 或启动 timer；只有 schema v2、五项 capability gate 完整，且 bundle 经 loader 独立重算为 eligible
|
||||
的 manual primary 配置才惰性加载真实组件。激活顺序固定为：重放 evidence→复验实际 Profile→记录 selected 审计→完整有界 startup reconciliation→记录
|
||||
reconciled 审计→启动 completion/timeout/cancel lifecycle→安装 router→记录 activated；任何一步失败都会撤销 router，并按 producer → consumer 顺序停止
|
||||
lifecycle。HTTP shutdown 与监听失败也会执行有界清理。`QL_DEPLOYMENT_PROFILE` 未配置时为 standalone，非法值、receipt Profile 不一致或在
|
||||
reconciled 审计→启动 completion/timeout/cancel lifecycle→安装 router→原子发布 `active` receipt→记录 activated；任何一步失败都会撤销 router，并按 producer → consumer 顺序停止
|
||||
lifecycle。HTTP shutdown 与监听失败先写 `stopping`、执行有界清理，再写 `stopped`；异常路径尝试写 `failed`。Linux receipt 绑定 boot/PID/process-group/start ticks，非 Linux
|
||||
portable receipt 不获得 current 结论。`QL_DEPLOYMENT_PROFILE` 未配置时为 standalone，非法值、Primary gate Profile 不一致或在
|
||||
cluster-control/worker 中误装本机 SQLite Primary 时 fail closed。目标实例仪式、部署配置写入、只读选择状态和操作回滚工具已由 ADR-0454 固化；首次真实 Edge/Standalone
|
||||
完整执行、bootstrap activated durable receipt、固定物理 edge/flash/断电 Gate 完成后仍需单独评审。
|
||||
完整执行、固定物理 edge/flash/断电与多写者配置 authority Gate 完成后仍需单独评审。
|
||||
|
||||
## 26. 交付阶段
|
||||
|
||||
@@ -9201,7 +9214,7 @@ flowchart LR
|
||||
| PR-2 Run 状态机 | Incubating | 纯转换表、终态/时间/错误/执行器元数据规则、Run version 与 event sequence CAS、事务性 RunCommandService、回滚测试 | 重复 Worker callback/fencing、并发数据库压力测试、Primary 执行链接入 |
|
||||
| PR-3 Executor 端口 | Incubating | ADR-0003、ExecutionSpec/Context/Handle/Result、Executor port、LocalProcessExecutor、进程组取消/超时升级、流式背压、Legacy Cron spec builder、真实进程 contract tests、可复现 edge 基准入口 | 固定 edge/多架构设备基线、Legacy builder 与 makeCommand 差异审计、Primary 生产流量接入 |
|
||||
| PR-4 Shadow Run | Incubating | origin 三态策略；默认关闭的 `QL3_SHADOW_ORIGINS`；manual、scheduled_node、boot、subscription、system 与 script 现有 ChildProcess 旁路观察；system crond 显式 origin marker、Shell execution ID、finish-only 准入、确定性 Run/Attempt 与 exact replay；`@once` 保持 manual、gRPC transport 不冒充 origin 的准入裁决；每个 worker 懒加载；Run/Attempt/Event 影子生命周期；稳定且不复制 caller 原文的 task identity/revision 与有界日志引用；同 worker 有界注册表和跨 worker 持久化候选关联；stop all/stop instance、Shell callback、乱序/迟到/歧义处理；监听前一次性、Profile-aware 的 keyset Startup Reconciler，终态证据补齐、lost/abandoned/pending 分流与 terminal Attempt response-loss 修复；origin-bounded 且逐级守恒的版本化 startup difference report、固定字段 metric batch 与一次性 collector；显式、只读、闭合窗口且 Profile-bounded 的 Shadow→Legacy 终态差异审计；128/256 MiB Linux arm64 资源门、SQLite 零增长与 Shadow enabled→off 进程重启回滚；process-epoch Legacy admission/capture/failure/pending 守恒；clean-shutdown `0600` no-replace capture+startup exporter；manual Edge 8/Standalone 32–128 canary；capture/terminal/resource 自包含 Primary bundle；rollout v2 loader 重算 source digest 与 eligibility；不可变 prepare/observe/resource/qualify 目标实例仪式、独立只读 audit；失败开放和契约测试 | 首次真实目标实例完整 canary 与 bootstrap activated 记录、其他 origin 独立 capture/Primary gate、固定物理 edge/flash/断电证据 |
|
||||
| PR-5 Primary LocalExecutor | Incubating（默认不激活，仅 manifest-gated manual） | runtime-owned Run 创建器；持久化先于 spawn；Run/Attempt 完整成功、失败、取消、超时与 lost 闭环；Executor handle 身份校验；spawn 后激活写失败的 stop+lost 补偿；completion rejection 安全收敛；独立 Primary 幂等查询与唯一索引竞态裁决；durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询；最多 64 条一页的 cross-worker cancellation source；独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent；最多 64 页的单周期 cancel supervisor；显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner；Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller；完整有界分页且 fail-closed 的 startup Reconcile supervisor；RunningInstance nullable `run_id/attempt_id` 关联；Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance，失败整体回滚；有界且防穿越的 legacy log output ref；manual owner seam、真实本机装配、单 spawn/fail-closed；严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计；HTTP worker 已接轻量 lazy bootstrap，accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router 顺序激活，失败撤销，监听失败和 shutdown 有界停止；Primary timeout 在 spawn 前持久化绝对 deadline，有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch；代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限，cluster-control/worker 拒绝误装本机 SQLite Primary；统一 CompletionService 原子提交 Attempt/Run/双 Event，spawn 前保存 callback token hash、终态推进 sequence，实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window；manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理；Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace；`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描，使终态残留继续可发现；确定无效的已知 Attempt receipt 先持久化隔离状态，再进入确定性私有分片 quarantine；终态 missing 与 quarantine 按 edge/standalone retention 有界清理；非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI；扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数；ENOSPC 与 launcher receipt 存储失败有代码门禁；显式最长 24 小时 approve 写入、`primary_selected` 只读状态、selection receipt、approval-expiry off 与 intent/completion crash-replay rollback | 首次真实目标实例激活/回滚及 durable runtime activation receipt；PostgreSQL CancellationDispatch adapter；cluster-control 生产启动拓扑；固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练 |
|
||||
| PR-5 Primary LocalExecutor | Incubating（默认不激活，仅 manifest-gated manual） | runtime-owned Run 创建器；持久化先于 spawn；Run/Attempt 完整成功、失败、取消、超时与 lost 闭环；Executor handle 身份校验；spawn 后激活写失败的 stop+lost 补偿；completion rejection 安全收敛；独立 Primary 幂等查询与唯一索引竞态裁决；durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询；最多 64 条一页的 cross-worker cancellation source；独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent；最多 64 页的单周期 cancel supervisor；显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner；Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller；完整有界分页且 fail-closed 的 startup Reconcile supervisor；RunningInstance nullable `run_id/attempt_id` 关联；Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance，失败整体回滚；有界且防穿越的 legacy log output ref；manual owner seam、真实本机装配、单 spawn/fail-closed；严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计；HTTP worker 已接轻量 lazy bootstrap，accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router→durable active receipt 顺序激活，失败撤销，监听失败和 shutdown 以 stopping→有界清理→stopped/failed 失效；receipt 固定为单文件 observed-state projection，Linux 以 boot/PID/process-group/start ticks 复验，独立 auditor 支持 active 且 off/rolled-back 拒绝 live runtime；Primary timeout 在 spawn 前持久化绝对 deadline，有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch；代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限，cluster-control/worker 拒绝误装本机 SQLite Primary；统一 CompletionService 原子提交 Attempt/Run/双 Event，spawn 前保存 callback token hash、终态推进 sequence，实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window；manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理；Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace；`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描，使终态残留继续可发现；确定无效的已知 Attempt receipt 先持久化隔离状态，再进入确定性私有分片 quarantine；终态 missing 与 quarantine 按 edge/standalone retention 有界清理；非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI；扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数；ENOSPC 与 launcher receipt 存储失败有代码门禁；显式最长 24 小时 approve 写入、`primary_selected` 只读状态、selection receipt、approval-expiry off 与 intent/completion crash-replay rollback | 首次真实目标实例完整激活/回滚仪式；PostgreSQL CancellationDispatch adapter；cluster-control 生产启动拓扑；固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练；共享 config 多写者 authority |
|
||||
| PR-7 Worker Session、Run Lease 与启动协议基础 | Incubating（默认关闭，独立入口显式 opt-in） | ADR-0012/0013/0014/0021/0057–0061/0108–0121/0231–0239/0377；有界 capability/Placement/Dispatcher；SQLite 协议孵化与 PostgreSQL v9 Session/Run Lease/credential/attestation authority；immutable revision Placement、数据库时钟 keyset candidate、认证 Worker Pull、digest-only offer recovery；versioned capability-free ExecutionSpec response、stable claim 跨重启退避、单 owner 原子 inbox 准入与 TLS 1.3 mTLS/`ql3w` HTTPS client；同一 package journal 上 revision-fenced starting/spawn/started/running/completion 状态、callback digest、tagged no-spawn 与 ambiguous recovery；PostgreSQL starting/running/start-failure/completion 数据库权威事务、精确重放与 cancellation/timeout 优先终态；batch Secret delivery 在 Attempt advisory lock 下复验 Session/Lease/revision 完整围栏并复用单 Agent，Secret-before-Artifact materializer 将同一 log ID 交给 Executor/journal/running ACK；offer-scoped `wlog-*` 私有文件 spool、Edge/Node 容量策略、append/quota/path 防护、barrier 后 output ownership、受审 POSIX Executor、truncation fact、固定内存流式 source、认证 Artifact stream、共享 immutable store port、S3-compatible SSE/checksum/条件 promotion adapter、upload-before-completion 协调，以及 Local/Cluster 同构、Profile-aware、ETag-fenced range read；用户取消 run.stop mutation 以数据库时间写 intent/Event 并在事务内复验 Project/RoleBinding fence；非执行取消 convergence lifecycle、运行期 expiry 与安全 lost retry 已接入 cluster-control 单一全局 cadence；完整 generation/version/token/Attempt fencing；独立最小权限 Worker ingress、CA/CRL 与连接 generation 热重载；offer journal、spawn barrier、receipt-first recovery；独立 `@qinglong/worker-runtime` 的本地 P-256 CSR、key/chain/trust 验证、generation + active pointer 安装和持久退避；默认关闭的 production process 已装配具体 execution graph、完整 Session heartbeat/drain/offline、direct-file bootstrap、单 Agent/单 cadence、startup reconciliation、证书 maintenance、transport fail-close/recovery 与 Edge/Node 有界预算；真实 PostgreSQL 18 + Linux Node 合约已覆盖 Run completion、credential 和 CA 双轮换且保持同一 Session；真实 K3s 合约已覆盖 TLS/credential Secret 分权、双对象 CAS、Recreate 顺序、identity generation 与单节点 PVC recovery；所有能力默认不可达且受 edge/cluster import audit 约束 | 具体 cert-manager/Vault/SPIFFE/离线 CA adapter 与模板、ingress reload controller、生产 RBAC、证书到期告警和 `ql3w` credential recovery 产品面；具体 KMS/Vault Secret provider、对象存储 credential/temporary lifecycle 与 retention/tombstone；Worker 管理 API；真实 Kubernetes 多节点 CSI/node-loss/production 360 秒 drain 与固定 edge 文件系统 suspend/时钟/断电、x64/arm64 资源门禁 |
|
||||
| PR-8 Project/Policy/Approval Core | Incubating（默认拒绝、无生产业务执行入口） | ADR-0028；统一六类 ActorRef 与 exact-shape 校验；`0017` ownerless default Project 和 append-only versioned RoleBinding；owner/admin/operator/viewer 固定矩阵；Project 内 mutation 幂等、expected-version CAS、双 SQLite 连接竞争门禁；archived read-only、revocation、存储损坏 fail-closed；Agent 写/Secret/Tool `require_approval`；ADR-0047 把六类 subject、role/permission matrix 与 fence 抽到 runtime-core，`pg-0004-project-policy`/capability v3 建立 ownerless PostgreSQL baseline、严格 role/state CHECK、append-only runtime 权限、SERIALIZABLE Project lock、mutation replay、双连接单 winner 和 cluster admission authorizer；ADR-0049/`pg-0005` capability v4 建立 stable IdentitySubject、append-only digest-only API credential、真实 cluster bearer authenticator、write-only durable security audit 与最小权限 runtime role，且已验证 HTTP→credential→Policy→audit→handler 纵向链路；ADR-0051 建立 `/api/v3` 认证前 peer/global 双预算、transport-peer-only、无 timer 且有界内存的 overload shield；ADR-0027 Artifact authorizer adapter；ADR-0029 `AuthenticatedPrincipal` contract、`0018` digest-only versioned challenge、CSPRNG/TTL、同事务消费 challenge + 写首 owner、精确重放与双连接竞争/崩溃回滚门禁；ADR-0030 `0019` stable identity/binding、legacy HS384 + current-session membership、logout/platform/revoke/disable、single-factor 与损坏 fail-closed 门禁；ADR-0031 `0020` digest-bound ApprovalRequest、User-only decision、Project/Role version fence、精确 expiry/重放/并发裁决及同事务 immutable dispatch；ADR-0032 `0021` execution backfill、三表原子 consume、稳定 due keyset、claim/renew/start/result fencing、pre-start takeover/post-start recovery-required、attempt budget、handler inspect/digest barrier 和 bounded dispatcher；ADR-0033/`0022` control/resolution backfill、start/renew/completion 原子联动、稳定 recovery keyset、双 resolver claim/takeover、finding/result 精确重放、自动/人工终结、迟到 completion 单 winner 和 evidence-only bounded reconciler；ADR-0034/`0023` 首个 `run.create` canonical plan、Run/Attempt/Event/receipt 同事务、幂等 collision fail-closed、renew/终态 fence、真实 SQLite handler 与 automatic evidence provider；ADR-0035/`0024` 独立 `approval.recover` 矩阵、稳定 User + 五分钟强认证、Project/RoleBinding fence、human resolution + authorization fact 原子提交、撤权竞态与回滚门禁；ADR-0036 recovery-first 单 timer lifecycle、edge/standalone 独立 cadence/页预算、跨周期 cursor、非重叠与有界 stop；ADR-0074 以新的 Node 24 SQLite v5 ownerless Project/RoleBinding/audit authority 和独立 local-secret-admin 提供强 Principal、`secret.manage`、撤权 fence、envelope+allowed audit 原子提交及不回显语义；ADR-0086 以可信 POSIX console 和 staged delivery 完成本机首 Owner 产品 ceremony | fresh database/pepper setup 与安全迁移向导；`shareStore`/Express 到 authentication core 的 production migration；credential rotation/revocation API、mTLS/Worker enrollment、恢复码；Project/Role/Approval/Secret 管理 CLI/API/UI、audit retention/query/export/alert、preview Artifact/digest/immutable plan builder、真实 MFA/hardware adapter、人工 recovery API/UI/独立 rate limit 与审计事件、handler/provider registry、lifecycle startup/shutdown/指标/admission gate；PostgreSQL action/receipt/provider/recovery-authorization 与 OPA adapter、缓存 version 失效；Tool/Package/Secret/Shell 各自的 handler/evidence contract；Secret/Run/Tool/Workflow waiting_approval 全入口装配；完整回滚演练 |
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
- 关联 RFC：QL-RFC-0001 D-361、PR-4、PR-5
|
||||
- 关联 ADR：ADR-0002、ADR-0449、ADR-0450、ADR-0451、ADR-0453
|
||||
- Amends：ADR-0453 的首次目标实例 canary 操作缺口
|
||||
- Amended by：ADR-0455 的 Profile-bounded durable runtime activation receipt
|
||||
|
||||
## 上下文
|
||||
|
||||
@@ -36,13 +37,12 @@ Edge 可能是 128 MiB 路由设备，不能为 canary 新增 daemon、watcher
|
||||
7. `approve` 是唯一写 live manifest 的操作。它要求显式 `approvedBy`，审批最短一分钟、最长 24 小时；先复核 plan 时的 absent/disabled 基线，disabled 基线先
|
||||
no-replace 归档，再以同目录 fsync 临时文件和最终摘要复核发布 schema v2 manifest。结果状态固定为 `activation_approved`/`primary_selected`，并明确
|
||||
`requiresRestart=true`、`runtimeActivationObserved=false`；配置选择不得冒充 HTTP worker 已完成 startup reconciliation 和 router activation。
|
||||
8. 应用重启后，实际运行态仍由现有 bootstrap 的 `selected → reconciled → activated` audit 证明。独立 `audit:manual-primary-canary:ql3` 只读复核 plan、source、gate、
|
||||
qualification、selection receipt 与 live manifest，可要求 `prepared|qualified|selected|off|rolled-back`；`off` 只证明 loader 当前关闭，`rolled-back` 还要求本 session 的
|
||||
intent/completion 摘要链完整。auditor 刻意不提供 `primary_active` 结论。
|
||||
8. 应用重启后，实际运行态由 bootstrap 的 `selected → reconciled → durable active receipt → activated` 证明。ADR-0455 已让独立 auditor 复核 receipt 与 Linux
|
||||
boot/PID/start-time 联合身份，并新增 `active` 要求；`off|rolled-back` 同时要求不存在仍 current 的 runtime。非 Linux portable receipt 不能获得 current 结论。
|
||||
9. `rollback` 只接受四个固定 reason 与有界 operator。它先 no-replace 发布 intent，绑定当前 enabled manifest SHA-256 和目标 disabled SHA-256，再复核当前摘要并原子替换
|
||||
live manifest，最后发布 completion；intent 或 completion response loss 可用相同参数重放。disabled 已有效但不同于本 session 的目标时拒绝覆盖。
|
||||
10. 审批过期后，既有 loader 必须立即 fail-closed 为 off；状态/auditor 报 `approvalExpired=true` 和 `rolloutMode=off`。operator 仍应执行显式 rollback，把磁盘事实
|
||||
收敛为 schema v2 disabled manifest，然后重启应用；过期不能被当作自动续期或自动删除 authority。
|
||||
10. 审批过期后，loader 对下一次 bootstrap 必须 fail-closed 为 off；状态/auditor 报 `approvalExpired=true` 和 `rolloutMode=off`。无 watcher 的既有进程不会仅因磁盘
|
||||
审批过期而自动卸载 router，故 `off` 审计会在 current receipt 仍存活时失败。operator 必须显式 rollback 并停止/重启应用；过期不能被当作自动续期、自动删除或已停止证明。
|
||||
|
||||
## 被拒绝的替代方案
|
||||
|
||||
@@ -87,4 +87,4 @@ Edge 可能是 128 MiB 路由设备，不能为 canary 新增 daemon、watcher
|
||||
|
||||
维护者仍需在一台真实目标 Edge 或 Standalone 实例执行 [Manual Primary Canary 操作手册](../operations/ql3-manual-primary-canary.md)，并保留 capture、terminal、resource、
|
||||
gate、qualification、selection、bootstrap activated audit 与 rollback completion。其他 origin 必须建立自己的 admission authority 与 gate；固定物理路由/flash/断电、
|
||||
运行态 durable activation receipt 和多写者 config authority 仍是独立后续工作。
|
||||
多写者 config authority 仍是独立后续工作；运行态 durable activation receipt 已由 ADR-0455 补齐，但首次真实目标实例执行仍待运维。
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
# ADR-0455：Profile-bounded Manual Primary 运行态激活凭据
|
||||
|
||||
- 状态：Accepted（实现完成；首次真实目标实例执行仍待运维）
|
||||
- 日期：2026-08-19
|
||||
- 关联 RFC：QL-RFC-0001 D-362、PR-4、PR-5
|
||||
- 关联 ADR：ADR-0002、ADR-0453、ADR-0454
|
||||
- Amends：ADR-0454 的运行态 durable activation receipt 缺口
|
||||
|
||||
## 上下文
|
||||
|
||||
ADR-0454 把 `primary_selected` 与运行态 `activated` 明确分开，但后者仍只存在于进程日志。日志可能没有持久 sink、可能被轮转，也无法在进程退出后区分“曾经激活”与“当前实例仍在运行”。仅记录 PID 又会受到 PID 重用、宿主重启和容器 namespace 变化影响。
|
||||
|
||||
Edge 可能只有 128 MiB 内存与低耐久 flash，不能为运行态证明增加 watcher、心跳 timer、数据库表、遥测 sidecar 或无限增长事件日志。Standalone 即使运行在集群节点上，也仍是本机 Profile，不能把宿主形态当成 cluster-control authority。
|
||||
|
||||
## 决策
|
||||
|
||||
1. 默认 HTTP worker 仅在 accepted schema v2 manifest 精确选择 `manual=primary` 且 Profile 为 `edge|standalone` 时，惰性加载凭据 adapter。disabled、rejected 和非 Primary 路径不 import adapter、不读写凭据。
|
||||
2. 当前状态固定为 config root 下唯一的 `qinglong3-manual-primary-runtime.json`，schema 为 `qinglong/manual-primary-runtime-receipt@v1`。它是 observed-state projection，不是 rollout authority；`qinglong3-rollout.json` 仍是唯一期望配置。
|
||||
3. 凭据只包含随机 activation ID、Profile、manifest revision、manifest 原始 SHA-256、激活/更新时间、`active|stopping|stopped|failed` 状态、进程身份和 domain-separated 自摘要。文件不包含 Task、Run、Cron、命令、路径、用户、日志、Secret 或错误正文。
|
||||
4. 文件必须位于当前 UID 拥有、非 symlink、group/world 不可写的真实 config root，以 `0600` 临时 inode、file fsync、原子 rename 和 best-effort directory fsync 发布。大小硬上限为 8 KiB；读取使用 `O_NOFOLLOW` 并复验 owner、mode、普通文件、exact shape 与自摘要。
|
||||
5. Linux 当前性使用 `/proc` 的 boot ID、PID、process-group ID 和 start-time ticks 联合身份，避免 PID 重用。已有 `active|stopping` 凭据只有在该联合身份确定 `exited|identity_mismatch` 后才能由下一实例替换；仍为 `running` 或无法反证时 fail closed。非 Linux 只写 portable PID，允许开发启动，但独立 auditor 必须返回 `unsupported`，不得宣称 current。
|
||||
6. 激活顺序固定为 reconciliation → 三个 lifecycle → router install → durable `active` → structured `activated` audit。写 `active` 失败时必须立即撤销 router、停止已启动 lifecycle，并保持原激活错误；因此日志中的 `activated` 必然晚于 durable receipt。
|
||||
7. 干净停止先写 `stopping`，再撤销 router 并按 timeout → cancellation → completion 停止 lifecycle，最后写 `stopped`。任一停止或凭据转换失败时尝试写 `failed`，停止调用仍返回第一个错误；`stopping|stopped|failed` 都不能被审计为 active。重复 stop 共用同一 Promise，不重复释放资源或写状态。
|
||||
8. 独立 canary auditor 新增 `--require=active`：必须同时复核 plan、qualification、selection、live enabled manifest、receipt binding 和 Linux 联合进程身份。报告分开输出 `runtimeActivationObserved`、`runtimeActivationCurrent`、receipt state 与 process state，不输出 PID、boot ID 或 activation ID。
|
||||
9. `--require=off|rolled-back` 现在还要求 `runtimeActivationCurrent=false`。manifest 已回滚或审批已过期但旧 worker 尚未停止时，loader 对下一次 bootstrap 的决策虽为 off，当前内存 router 仍可能继续拥有 manual；auditor 必须拒绝把这种状态表述为已关闭。无 watcher 的部署必须显式停止/重启后再完成 off/rollback 审计。
|
||||
10. 本凭据不是跨 UID、共享卷或多写者共识锁。同一 UID 可改写 config root，启动前的 read/inspect/write 也不替代部署系统的单实例约束；共享 config、多主机签名或强互斥需要独立 authority，不能由本地 receipt 推断。
|
||||
|
||||
## 被拒绝的替代方案
|
||||
|
||||
### 用 enabled manifest 表示 active
|
||||
|
||||
拒绝。manifest 可能尚未被当前 worker 读取，reconciliation 可能失败，审批可能过期，回滚后旧进程也可能尚未重启。
|
||||
|
||||
### 每秒刷新心跳文件
|
||||
|
||||
拒绝。它增加永久 timer、写放大和 flash 磨损，仍需处理 suspend、时钟跳变与调度延迟。进程联合身份提供按需审计，不需要周期写盘。
|
||||
|
||||
### 每次启动追加一份不可变 receipt
|
||||
|
||||
拒绝。无限历史会在路由设备上持续增长，而且仍需另一个 current pointer。单个有摘要的 observed-state projection 足以表达本切片状态；不可变 selection 与 rollback 链继续由 ADR-0454 保存。
|
||||
|
||||
### 非 Linux 使用 `kill(pid, 0)` 宣称 current
|
||||
|
||||
拒绝。它不能防 PID 重用，也不能绑定宿主 boot。portable receipt 只证明 bootstrap 曾写入，不提供 current 结论。
|
||||
|
||||
## 资源、安全与部署影响
|
||||
|
||||
- 默认关闭路径零新增 I/O、timer、watcher、listener、连接和数据库操作。
|
||||
- 激活写一次，干净停止最多再写两次；常驻内存只有一个小 receipt 对象，不随任务数增长。
|
||||
- 不新增 workspace package、生产依赖、migration、PostgreSQL/Kubernetes 对象或部署端口。
|
||||
- receipt 是 owner-private 运维证据，不应发布到公开 artifact；auditor stdout 只输出低敏状态。
|
||||
- approval expiry 不会在已运行进程内自动卸载 router。该限制是无 watcher 决策的直接结果，必须通过部署系统的显式 restart/stop 收敛。
|
||||
|
||||
## 验证
|
||||
|
||||
- 聚焦 `25/25` 覆盖 active→stopping→stopped、摘要篡改、live identity 冲突、stale generation 接管、portable identity 不冒充 Linux current、receipt 写失败撤销 ownership、停止顺序、canary `--require=active` 以及 live runtime 阻断 rolled-back。
|
||||
- `build:back` 与完整 backend 通过：`1,487 pass / 0 fail / 2 conditional skip`。首次沙箱执行仅有 Vault loopback 因 `listen EPERM` 失败；允许 loopback 后原命令完整重跑为上述结果。
|
||||
- 18-package clean build/test 退出 0。首次沙箱执行仅有 worker-runtime 三个 TLS/mTLS loopback contract 因相同 `listen EPERM` 失败；允许 loopback 后完整 clean 原命令重跑通过。
|
||||
- Edge import、cluster dependency、package boundary、service-manager bridge 四项架构审计零 finding；package boundary 仍为精确 18，`singleSourcePackages=[]`、`shallowSourcePackages=[]`。
|
||||
- `14/14` Local Profile artifact audit 全部通过且字节与 D-361 基线一致：基础 Edge/Standalone `2589998/2590076`、adopted `2809293/2809416`、application `3632877/3632997`、application-api `3800430/3800574`、AI `3069251/3069341`、application+AI `4493151/4493283`、MCP `7315930/7316038`。
|
||||
- 隔离 frozen-dependency Linux arm64 Node 24 Docker 门通过：128 MiB router stress peak `87,339,008` bytes，256 MiB Edge release peak `145,506,304` bytes，`memory.events max/oom/oom_kill` 增量均为 0；所有 workload 通过。该证据不是物理路由或 flash/断电证明。
|
||||
- 本 ADR 不修改依赖树、PostgreSQL schema/migration 或 Kubernetes 拓扑，因此 PostgreSQL HA 不因本切片重复执行。
|
||||
|
||||
## 后续
|
||||
|
||||
仍需在真实 Edge/Standalone 目标实例执行 prepare→真实 manual cohort→qualify→approve→restart→`--require=active`→rollback→restart→`--require=rolled-back` 完整仪式。物理 flash/断电、PID namespace、同 UID 恶意并发、共享配置目录和签名式多写者 authority 继续作为独立 Gate。
|
||||
@@ -458,6 +458,7 @@
|
||||
| [ADR-0452](./ADR-0452-atomic-flattened-backend-build-publication.md) | 原子且扁平兼容的 Backend 构建发布 | Accepted |
|
||||
| [ADR-0453](./ADR-0453-origin-scoped-legacy-shadow-capture-authority-and-primary-gate.md) | Origin-scoped Legacy Shadow 捕获权威与 Primary 门禁 | Accepted（首次真实目标实例 manual canary 待执行） |
|
||||
| [ADR-0454](./ADR-0454-target-instance-manual-primary-canary-ceremony.md) | 目标实例 Manual Primary Canary 与显式回滚仪式 | Accepted（首次真实用户目标实例执行待运维） |
|
||||
| [ADR-0455](./ADR-0455-profile-bounded-manual-primary-runtime-activation-receipt.md) | Profile-bounded Manual Primary 运行态激活凭据 | Accepted（首次真实目标实例执行待运维） |
|
||||
|
||||
## 规则
|
||||
|
||||
|
||||
@@ -117,9 +117,20 @@ pnpm audit:manual-primary-canary:ql3 -- \
|
||||
```
|
||||
|
||||
正确状态是 `activation_approved`/`rolloutMode=primary_selected`，不是 `primary_active`；`requiresRestart=true` 且 `runtimeActivationObserved=false`。随后重启应用，并在结构化启动审计中依次确认
|
||||
同一 revision 的 `selected`、`reconciled`、`activated`。缺少任一项都不能宣称运行态 Primary 已激活。
|
||||
同一 revision 的 `selected`、`reconciled`、`activated`，再执行：
|
||||
|
||||
审批过期后 loader 自动 fail-closed 为 off，不会续期。不要修改原 manifest 时间；创建新 session 重新采样。
|
||||
```sh
|
||||
pnpm audit:manual-primary-canary:ql3 -- \
|
||||
--root=/ql/data/config \
|
||||
--session=edge-20260819-a \
|
||||
--require=active
|
||||
```
|
||||
|
||||
只有 `runtimeActivationObserved=true`、`runtimeActivationCurrent=true`、`runtimeReceiptState=active` 和 `runtimeProcessState=running` 同时成立，才能宣称当前 Linux worker 已激活。
|
||||
receipt 文件固定为 owner-private `qinglong3-manual-primary-runtime.json`，不要复制到公开 artifact。非 Linux portable receipt 只能证明写入发生过，不能通过 `active` 门。
|
||||
|
||||
审批过期后 loader 对下一次 bootstrap 自动 fail-closed 为 off，不会续期；已经运行的 worker 没有 watcher，不会仅凭磁盘过期自动卸载 router。必须停止/重启并通过
|
||||
`--require=off`，不要修改原 manifest 时间；需要重新启用时创建新 session 重新采样。
|
||||
|
||||
## 7. 回滚并重启
|
||||
|
||||
@@ -141,5 +152,6 @@ pnpm audit:manual-primary-canary:ql3 -- \
|
||||
|
||||
支持的 reason 只有 `operator_request`、`runtime_failure`、`gate_rejected`、`approval_expired`。rollback 先发布 intent，再摘要复核并原子替换 live manifest，最后发布 completion；响应丢失时使用完全相同的参数重跑。
|
||||
|
||||
`rolled-back` 比普通 `off` 更严格：后者在初始 disabled 或审批过期时也成立，前者还要求本 session 的 intent/completion 摘要链完整。完成后重启应用，确认 rollout loader 返回
|
||||
disabled/off，Legacy manual 执行继续可用且 Shadow/Primary 不再接管。保留整个 session 的 `0600` 文件和启动审计用于发布复核，不要覆盖或编辑。
|
||||
`rolled-back` 比普通 `off` 更严格：后者在初始 disabled 或审批过期时也成立，前者还要求本 session 的 intent/completion 摘要链完整；两者现在都拒绝仍存活的 current
|
||||
Primary receipt。rollback 写盘后先停止/重启应用，再运行上面的 `--require=rolled-back`，确认 loader 返回 disabled/off、receipt 为 stopped/failed 或旧 Linux identity 已退出，
|
||||
Legacy manual 执行继续可用且 Primary 不再接管。保留整个 session 的 `0600` 文件和启动审计用于发布复核，不要覆盖或编辑。
|
||||
|
||||
@@ -18,6 +18,11 @@ const {
|
||||
const {
|
||||
parseRuntimeRolloutManifest,
|
||||
} = require('../back/runtime/domain/runtimeRolloutManifest');
|
||||
const {
|
||||
MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE,
|
||||
MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES,
|
||||
parseManualPrimaryRuntimeReceipt,
|
||||
} = require('../back/runtime/domain/manualPrimaryRuntimeReceipt');
|
||||
const {
|
||||
readPrivateJson,
|
||||
serialized,
|
||||
@@ -27,6 +32,7 @@ const REQUIREMENTS = new Set([
|
||||
'prepared',
|
||||
'qualified',
|
||||
'selected',
|
||||
'active',
|
||||
'off',
|
||||
'rolled-back',
|
||||
]);
|
||||
@@ -85,7 +91,44 @@ function regularFile(target) {
|
||||
}
|
||||
}
|
||||
|
||||
function run(options) {
|
||||
function inspectRuntimeProcess(identity) {
|
||||
if (identity.kind !== 'linux-proc' || process.platform !== 'linux') {
|
||||
return 'unsupported';
|
||||
}
|
||||
let bootId;
|
||||
try {
|
||||
bootId = fs.readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim();
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return 'unsupported';
|
||||
throw error;
|
||||
}
|
||||
if (bootId !== identity.bootId) return 'identity_mismatch';
|
||||
let stat;
|
||||
try {
|
||||
stat = fs.readFileSync(`/proc/${identity.pid}/stat`, 'utf8');
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT' || error?.code === 'ESRCH') return 'exited';
|
||||
throw error;
|
||||
}
|
||||
const close = stat.lastIndexOf(')');
|
||||
const fields =
|
||||
close < 1
|
||||
? []
|
||||
: stat
|
||||
.slice(close + 1)
|
||||
.trim()
|
||||
.split(/\s+/u);
|
||||
if (
|
||||
fields.length < 20 ||
|
||||
Number(fields[2]) !== identity.processGroupId ||
|
||||
fields[19] !== identity.startTimeTicks
|
||||
) {
|
||||
return 'identity_mismatch';
|
||||
}
|
||||
return ['Z', 'X', 'x'].includes(fields[0]) ? 'exited' : 'running';
|
||||
}
|
||||
|
||||
function run(options, dependencies = {}) {
|
||||
const rootStat = fs.lstatSync(options.root);
|
||||
if (
|
||||
!rootStat.isDirectory() ||
|
||||
@@ -274,12 +317,55 @@ function run(options) {
|
||||
}
|
||||
rolloutMode = 'off';
|
||||
}
|
||||
let runtimeActivationObserved = false;
|
||||
let runtimeActivationCurrent = false;
|
||||
let runtimeReceiptState = 'missing';
|
||||
let runtimeProcessState = 'missing';
|
||||
const runtimeReceiptPath = path.join(
|
||||
options.root,
|
||||
MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE,
|
||||
);
|
||||
if (regularFile(runtimeReceiptPath)) {
|
||||
const receipt = parseManualPrimaryRuntimeReceipt(
|
||||
readPrivateJson(
|
||||
runtimeReceiptPath,
|
||||
MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES,
|
||||
).value,
|
||||
);
|
||||
runtimeReceiptState = receipt.state;
|
||||
const selectionPath = resolve('selection');
|
||||
if (regularFile(selectionPath)) {
|
||||
const selection = readPrivateJson(selectionPath, 64 * 1024).value;
|
||||
runtimeActivationObserved =
|
||||
receipt.revision === `manual-primary-${plan.sessionId}` &&
|
||||
receipt.profile === plan.profile &&
|
||||
selection?.schema === 'qinglong/manual-primary-canary-selection@v1' &&
|
||||
selection.sessionId === plan.sessionId &&
|
||||
selection.profile === plan.profile &&
|
||||
selection.manifestSha256 === receipt.rolloutSourceSha256;
|
||||
}
|
||||
if (receipt.state === 'active') {
|
||||
runtimeProcessState = (
|
||||
dependencies.inspectRuntimeProcess ?? inspectRuntimeProcess
|
||||
)(receipt.process);
|
||||
runtimeActivationCurrent = runtimeProcessState === 'running';
|
||||
}
|
||||
}
|
||||
const compatible =
|
||||
options.require === 'prepared' ||
|
||||
(options.require === 'qualified' && eligible) ||
|
||||
(options.require === 'selected' && rolloutMode === 'primary_selected') ||
|
||||
(options.require === 'off' && rolloutMode === 'off') ||
|
||||
(options.require === 'rolled-back' && rolloutMode === 'off' && rolledBack);
|
||||
(options.require === 'active' &&
|
||||
rolloutMode === 'primary_selected' &&
|
||||
runtimeActivationObserved &&
|
||||
runtimeActivationCurrent) ||
|
||||
(options.require === 'off' &&
|
||||
rolloutMode === 'off' &&
|
||||
!runtimeActivationCurrent) ||
|
||||
(options.require === 'rolled-back' &&
|
||||
rolloutMode === 'off' &&
|
||||
rolledBack &&
|
||||
!runtimeActivationCurrent);
|
||||
const report = {
|
||||
schema: 'qinglong/manual-primary-canary-audit@v1',
|
||||
schemaVersion: 1,
|
||||
@@ -289,7 +375,10 @@ function run(options) {
|
||||
eligible,
|
||||
rolloutMode,
|
||||
approvalExpired,
|
||||
runtimeActivationObserved: false,
|
||||
runtimeActivationObserved,
|
||||
runtimeActivationCurrent,
|
||||
runtimeReceiptState,
|
||||
runtimeProcessState,
|
||||
rolledBack,
|
||||
planSha256: planRead.sha256,
|
||||
...(rolloutSha256 === undefined ? {} : { rolloutSha256 }),
|
||||
|
||||
@@ -24,6 +24,8 @@ function loadResult(status, mode = 'off') {
|
||||
evaluatedAtMs: NOW,
|
||||
sourcePath: '/data/config/qinglong3-rollout.json',
|
||||
status,
|
||||
revision: 'manual-primary-test',
|
||||
sourceSha256: 'a'.repeat(64),
|
||||
},
|
||||
...(status === 'accepted'
|
||||
? {
|
||||
@@ -150,6 +152,20 @@ test('accepted bootstrap lazily loads the stack and delegates activation', async
|
||||
calls.push('install');
|
||||
return () => calls.push('dispose');
|
||||
},
|
||||
receipt: {
|
||||
async activated() {
|
||||
calls.push('receipt:active');
|
||||
},
|
||||
async stopping() {
|
||||
calls.push('receipt:stopping');
|
||||
},
|
||||
async stopped() {
|
||||
calls.push('receipt:stopped');
|
||||
},
|
||||
async failed() {
|
||||
calls.push('receipt:failed');
|
||||
},
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(`audit:${record.activation}`);
|
||||
},
|
||||
@@ -168,11 +184,14 @@ test('accepted bootstrap lazily loads the stack and delegates activation', async
|
||||
'start-timeout',
|
||||
'start-cancellation',
|
||||
'install',
|
||||
'receipt:active',
|
||||
'audit:activated',
|
||||
'receipt:stopping',
|
||||
'dispose',
|
||||
'stop-timeout',
|
||||
'stop-cancellation',
|
||||
'stop-completion',
|
||||
'receipt:stopped',
|
||||
'audit:stopped',
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@ function loadResult(status, mode = 'off') {
|
||||
sourcePath: '/data/config/qinglong3-rollout.json',
|
||||
status,
|
||||
revision: 'canary-1',
|
||||
sourceSha256: 'a'.repeat(64),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -171,6 +172,138 @@ test('activation reconciles before starting lifecycle and installing ownership',
|
||||
]);
|
||||
});
|
||||
|
||||
test('activation publishes durable state around ownership and lifecycle shutdown', async () => {
|
||||
const calls = [];
|
||||
const result = await activateManualPrimaryRuntime({
|
||||
load: async () => loadResult('accepted', 'primary'),
|
||||
create() {
|
||||
return {
|
||||
router: router(),
|
||||
...completionLifecycle(calls),
|
||||
async reconcile() {
|
||||
return cleanRecovery();
|
||||
},
|
||||
startTimeout() {
|
||||
calls.push('start-timeout');
|
||||
return true;
|
||||
},
|
||||
async stopTimeout() {
|
||||
calls.push('stop-timeout');
|
||||
return 'drained';
|
||||
},
|
||||
startCancellation() {
|
||||
calls.push('start-cancellation');
|
||||
return true;
|
||||
},
|
||||
async stopCancellation() {
|
||||
calls.push('stop-cancellation');
|
||||
return 'drained';
|
||||
},
|
||||
};
|
||||
},
|
||||
install() {
|
||||
calls.push('install');
|
||||
return () => calls.push('dispose');
|
||||
},
|
||||
receipt: {
|
||||
async activated() {
|
||||
calls.push('receipt:active');
|
||||
},
|
||||
async stopping() {
|
||||
calls.push('receipt:stopping');
|
||||
},
|
||||
async stopped() {
|
||||
calls.push('receipt:stopped');
|
||||
},
|
||||
async failed() {
|
||||
calls.push('receipt:failed');
|
||||
},
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(`audit:${record.activation}`);
|
||||
},
|
||||
});
|
||||
|
||||
assert.deepEqual(calls.slice(-4), [
|
||||
'start-cancellation',
|
||||
'install',
|
||||
'receipt:active',
|
||||
'audit:activated',
|
||||
]);
|
||||
await result.stop();
|
||||
assert.deepEqual(calls.slice(-7), [
|
||||
'receipt:stopping',
|
||||
'dispose',
|
||||
'stop-timeout',
|
||||
'stop-cancellation',
|
||||
'stop-completion',
|
||||
'receipt:stopped',
|
||||
'audit:stopped',
|
||||
]);
|
||||
});
|
||||
|
||||
test('activation rolls ownership back when durable receipt publication fails', async () => {
|
||||
const calls = [];
|
||||
await assert.rejects(
|
||||
activateManualPrimaryRuntime({
|
||||
load: async () => loadResult('accepted', 'primary'),
|
||||
create() {
|
||||
return {
|
||||
router: router(),
|
||||
...completionLifecycle(calls),
|
||||
async reconcile() {
|
||||
return cleanRecovery();
|
||||
},
|
||||
startTimeout() {
|
||||
calls.push('start-timeout');
|
||||
return true;
|
||||
},
|
||||
async stopTimeout() {
|
||||
calls.push('stop-timeout');
|
||||
return 'drained';
|
||||
},
|
||||
startCancellation() {
|
||||
calls.push('start-cancellation');
|
||||
return true;
|
||||
},
|
||||
async stopCancellation() {
|
||||
calls.push('stop-cancellation');
|
||||
return 'drained';
|
||||
},
|
||||
};
|
||||
},
|
||||
install() {
|
||||
calls.push('install');
|
||||
return () => calls.push('dispose');
|
||||
},
|
||||
receipt: {
|
||||
async activated() {
|
||||
calls.push('receipt:active');
|
||||
throw new Error('receipt unavailable');
|
||||
},
|
||||
async stopping() {},
|
||||
async stopped() {},
|
||||
async failed() {
|
||||
calls.push('receipt:failed');
|
||||
},
|
||||
},
|
||||
audit(record) {
|
||||
calls.push(`audit:${record.activation}`);
|
||||
},
|
||||
}),
|
||||
/receipt unavailable/,
|
||||
);
|
||||
assert.deepEqual(calls.slice(-7), [
|
||||
'receipt:active',
|
||||
'dispose',
|
||||
'stop-timeout',
|
||||
'stop-cancellation',
|
||||
'stop-completion',
|
||||
'receipt:failed',
|
||||
'audit:failed',
|
||||
]);
|
||||
});
|
||||
|
||||
test('activation rejects unresolved recovery before starting or installing', async () => {
|
||||
const calls = [];
|
||||
await assert.rejects(
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE,
|
||||
parseManualPrimaryRuntimeReceipt,
|
||||
} = require('../../back/runtime/domain/manualPrimaryRuntimeReceipt');
|
||||
const {
|
||||
ManualPrimaryRuntimeReceiptConflictError,
|
||||
ManualPrimaryRuntimeReceiptStore,
|
||||
} = require('../../back/runtime/adapters/fs/manualPrimaryRuntimeReceiptStore');
|
||||
|
||||
const IDENTITY = {
|
||||
platform: 'linux',
|
||||
bootId: '11111111-2222-3333-4444-555555555555',
|
||||
pid: 321,
|
||||
processGroupId: 320,
|
||||
startTimeTicks: '123456',
|
||||
};
|
||||
|
||||
function audit() {
|
||||
return {
|
||||
event: 'runtime.rollout_config_evaluated',
|
||||
evaluatedAtMs: 1_000,
|
||||
sourcePath: '/data/config/qinglong3-rollout.json',
|
||||
sourceSha256: 'a'.repeat(64),
|
||||
revision: 'manual-primary-edge-live-1',
|
||||
status: 'accepted',
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(t, inspection = 'exited') {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-runtime-receipt-'));
|
||||
fs.chmodSync(root, 0o700);
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
let now = 10_000;
|
||||
const options = {
|
||||
clock: { now: () => now++ },
|
||||
platform: 'linux',
|
||||
pid: IDENTITY.pid,
|
||||
randomId: () => '1'.repeat(32),
|
||||
identityProvider: {
|
||||
async capture() {
|
||||
return IDENTITY;
|
||||
},
|
||||
async inspect() {
|
||||
return { status: inspection };
|
||||
},
|
||||
},
|
||||
};
|
||||
return {
|
||||
root,
|
||||
store: new ManualPrimaryRuntimeReceiptStore(root, 'edge', options),
|
||||
options,
|
||||
};
|
||||
}
|
||||
|
||||
test('publishes one private current receipt and transitions it around shutdown', async (t) => {
|
||||
const { root, store } = fixture(t);
|
||||
const target = path.join(root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE);
|
||||
|
||||
await store.activated(audit());
|
||||
let receipt = parseManualPrimaryRuntimeReceipt(
|
||||
JSON.parse(fs.readFileSync(target, 'utf8')),
|
||||
);
|
||||
assert.equal(receipt.state, 'active');
|
||||
assert.equal(receipt.process.kind, 'linux-proc');
|
||||
assert.equal(fs.statSync(target).mode & 0o777, 0o600);
|
||||
|
||||
await store.stopping();
|
||||
receipt = parseManualPrimaryRuntimeReceipt(
|
||||
JSON.parse(fs.readFileSync(target, 'utf8')),
|
||||
);
|
||||
assert.equal(receipt.state, 'stopping');
|
||||
|
||||
await store.stopped();
|
||||
receipt = parseManualPrimaryRuntimeReceipt(
|
||||
JSON.parse(fs.readFileSync(target, 'utf8')),
|
||||
);
|
||||
assert.equal(receipt.state, 'stopped');
|
||||
assert.equal(receipt.activationId, '1'.repeat(32));
|
||||
});
|
||||
|
||||
test('refuses to replace a receipt whose exact Linux process is still live', async (t) => {
|
||||
const first = fixture(t);
|
||||
await first.store.activated(audit());
|
||||
const second = new ManualPrimaryRuntimeReceiptStore(first.root, 'edge', {
|
||||
...first.options,
|
||||
randomId: () => '2'.repeat(32),
|
||||
identityProvider: {
|
||||
...first.options.identityProvider,
|
||||
async inspect() {
|
||||
return { status: 'running' };
|
||||
},
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
second.activated(audit()),
|
||||
ManualPrimaryRuntimeReceiptConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test('replaces a stale process generation and rejects receipt tampering', async (t) => {
|
||||
const first = fixture(t);
|
||||
await first.store.activated(audit());
|
||||
const second = new ManualPrimaryRuntimeReceiptStore(first.root, 'edge', {
|
||||
...first.options,
|
||||
randomId: () => '2'.repeat(32),
|
||||
});
|
||||
await second.activated(audit());
|
||||
const target = path.join(first.root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE);
|
||||
const receipt = JSON.parse(fs.readFileSync(target, 'utf8'));
|
||||
assert.equal(receipt.activationId, '2'.repeat(32));
|
||||
receipt.state = 'stopped';
|
||||
assert.throws(
|
||||
() => parseManualPrimaryRuntimeReceipt(receipt),
|
||||
/digest is invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('portable receipts remain observable but cannot claim Linux liveness', async (t) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-runtime-receipt-'));
|
||||
fs.chmodSync(root, 0o700);
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
const store = new ManualPrimaryRuntimeReceiptStore(root, 'standalone', {
|
||||
clock: { now: () => 20_000 },
|
||||
platform: 'darwin',
|
||||
pid: 432,
|
||||
randomId: () => '3'.repeat(32),
|
||||
identityProvider: {
|
||||
async capture() {
|
||||
return null;
|
||||
},
|
||||
async inspect() {
|
||||
return { status: 'unsupported' };
|
||||
},
|
||||
},
|
||||
});
|
||||
await store.activated(audit());
|
||||
const receipt = parseManualPrimaryRuntimeReceipt(
|
||||
JSON.parse(
|
||||
fs.readFileSync(
|
||||
path.join(root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE),
|
||||
'utf8',
|
||||
),
|
||||
),
|
||||
);
|
||||
assert.deepEqual(receipt.process, {
|
||||
kind: 'portable',
|
||||
platform: 'darwin',
|
||||
pid: 432,
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,10 @@ const { afterEach, test } = require('node:test');
|
||||
const {
|
||||
manualPrimaryCanaryFileSet,
|
||||
} = require('../../back/runtime/domain/manualPrimaryCanaryCeremony');
|
||||
const {
|
||||
createManualPrimaryRuntimeReceipt,
|
||||
MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE,
|
||||
} = require('../../back/runtime/domain/manualPrimaryRuntimeReceipt');
|
||||
const {
|
||||
parseArguments,
|
||||
readPrivateJson,
|
||||
@@ -273,6 +277,35 @@ test('qualifies, explicitly approves, audits and rolls back one target session',
|
||||
.runtimeActivationObserved,
|
||||
false,
|
||||
);
|
||||
const rolloutSha256 = readPrivateJson(
|
||||
path.join(root, files.rollout),
|
||||
64 * 1024,
|
||||
).sha256;
|
||||
writeJson(
|
||||
path.join(root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE),
|
||||
createManualPrimaryRuntimeReceipt({
|
||||
activationId: '1'.repeat(32),
|
||||
profile: 'edge',
|
||||
revision: `manual-primary-${SESSION}`,
|
||||
rolloutSourceSha256: rolloutSha256,
|
||||
activatedAtMs: activatedAt + 1,
|
||||
process: {
|
||||
kind: 'linux-proc',
|
||||
platform: 'linux',
|
||||
pid: 999_999,
|
||||
processGroupId: 999_999,
|
||||
bootId: '11111111-2222-3333-4444-555555555555',
|
||||
startTimeTicks: '123456',
|
||||
},
|
||||
}),
|
||||
);
|
||||
const active = audit(
|
||||
{ root, sessionId: SESSION, require: 'active' },
|
||||
{ inspectRuntimeProcess: () => 'running' },
|
||||
);
|
||||
assert.equal(active.runtimeActivationObserved, true);
|
||||
assert.equal(active.runtimeActivationCurrent, true);
|
||||
assert.equal(active.runtimeReceiptState, 'active');
|
||||
assert.equal(
|
||||
run(
|
||||
{ mode: 'status', root, sessionId: SESSION },
|
||||
@@ -323,6 +356,40 @@ test('qualifies, explicitly approves, audits and rolls back one target session',
|
||||
).publication,
|
||||
'existing',
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
audit(
|
||||
{ root, sessionId: SESSION, require: 'rolled-back' },
|
||||
{ inspectRuntimeProcess: () => 'running' },
|
||||
),
|
||||
/not satisfied/,
|
||||
);
|
||||
writeJson(
|
||||
path.join(root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE),
|
||||
createManualPrimaryRuntimeReceipt({
|
||||
activationId: '2'.repeat(32),
|
||||
profile: 'edge',
|
||||
revision: 'manual-primary-another-session',
|
||||
rolloutSourceSha256: 'b'.repeat(64),
|
||||
activatedAtMs: activatedAt + 3,
|
||||
process: {
|
||||
kind: 'linux-proc',
|
||||
platform: 'linux',
|
||||
pid: 999_998,
|
||||
processGroupId: 999_998,
|
||||
bootId: '11111111-2222-3333-4444-555555555555',
|
||||
startTimeTicks: '123457',
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
audit(
|
||||
{ root, sessionId: SESSION, require: 'rolled-back' },
|
||||
{ inspectRuntimeProcess: () => 'running' },
|
||||
),
|
||||
/not satisfied/,
|
||||
);
|
||||
assert.equal(
|
||||
audit({ root, sessionId: SESSION, require: 'rolled-back' }).rolloutMode,
|
||||
'off',
|
||||
|
||||
Reference in New Issue
Block a user