From 36035ac43e1d18707298b66a817153f242e20cc3 Mon Sep 17 00:00:00 2001 From: whyour Date: Wed, 19 Aug 2026 05:13:58 +0800 Subject: [PATCH] feat(ql3): persist manual primary activation --- .../fs/manualPrimaryRuntimeReceiptStore.ts | 261 +++++++++++++++++ .../bootstrapDefaultManualPrimaryRuntime.ts | 24 +- .../manualPrimaryRuntimeActivation.ts | 74 +++-- .../domain/manualPrimaryRuntimeReceipt.ts | 268 ++++++++++++++++++ .../ports/manualPrimaryRuntimeReceipt.ts | 8 + docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 27 +- ...instance-manual-primary-canary-ceremony.md | 12 +- ...nual-primary-runtime-activation-receipt.md | 66 +++++ docs/adr/README.md | 1 + docs/operations/ql3-manual-primary-canary.md | 20 +- scripts/ql3-manual-primary-canary-audit.cjs | 97 ++++++- ...tstrapDefaultManualPrimaryRuntime.test.cjs | 19 ++ .../manualPrimaryRuntimeActivation.test.cjs | 133 +++++++++ .../back/manualPrimaryRuntimeReceipt.test.cjs | 157 ++++++++++ test/back/ql3ManualPrimaryCanary.test.cjs | 67 +++++ 15 files changed, 1191 insertions(+), 43 deletions(-) create mode 100644 back/runtime/adapters/fs/manualPrimaryRuntimeReceiptStore.ts create mode 100644 back/runtime/domain/manualPrimaryRuntimeReceipt.ts create mode 100644 back/runtime/ports/manualPrimaryRuntimeReceipt.ts create mode 100644 docs/adr/ADR-0455-profile-bounded-manual-primary-runtime-activation-receipt.md create mode 100644 test/back/manualPrimaryRuntimeReceipt.test.cjs diff --git a/back/runtime/adapters/fs/manualPrimaryRuntimeReceiptStore.ts b/back/runtime/adapters/fs/manualPrimaryRuntimeReceiptStore.ts new file mode 100644 index 00000000..476feddc --- /dev/null +++ b/back/runtime/adapters/fs/manualPrimaryRuntimeReceiptStore.ts @@ -0,0 +1,261 @@ +import { randomBytes } from 'crypto'; +import { constants } from 'fs'; +import fs from 'fs/promises'; +import path from 'path'; +import { + createManualPrimaryRuntimeReceipt, + MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE, + MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES, + parseManualPrimaryRuntimeReceipt, + transitionManualPrimaryRuntimeReceipt, + type ManualPrimaryRuntimeProcessIdentity, + type ManualPrimaryRuntimeReceipt, + type ManualPrimaryRuntimeReceiptState, +} from '../../domain/manualPrimaryRuntimeReceipt'; +import type { ManualPrimaryRuntimeReceiptLifecycle } from '../../ports/manualPrimaryRuntimeReceipt'; +import type { RuntimeRolloutLoadAudit } from '../../ports/runtimeRolloutLoader'; +import { + LinuxProcProcessIdentityProvider, + type LocalProcessIdentityProvider, +} from '../local-process/localProcessIdentity'; + +export interface ManualPrimaryRuntimeReceiptStoreOptions { + clock?: { now(): number }; + identityProvider?: LocalProcessIdentityProvider; + platform?: NodeJS.Platform; + pid?: number; + randomId?: () => string; +} + +function isCode(error: unknown, code: string): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error as { code?: string }).code === code + ); +} + +export class ManualPrimaryRuntimeReceiptConflictError extends Error { + constructor(message: string) { + super(message); + this.name = 'ManualPrimaryRuntimeReceiptConflictError'; + } +} + +export class ManualPrimaryRuntimeReceiptStore + implements ManualPrimaryRuntimeReceiptLifecycle +{ + private readonly target: string; + private readonly clock: { now(): number }; + private readonly identityProvider: LocalProcessIdentityProvider; + private readonly platform: NodeJS.Platform; + private readonly pid: number; + private readonly randomId: () => string; + private current?: ManualPrimaryRuntimeReceipt; + + constructor( + private readonly root: string, + private readonly profile: 'edge' | 'standalone', + options: ManualPrimaryRuntimeReceiptStoreOptions = {}, + ) { + if (!path.isAbsolute(root)) { + throw new TypeError( + 'Manual Primary runtime receipt root must be absolute', + ); + } + this.target = path.join(root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE); + this.clock = options.clock ?? { now: Date.now }; + this.identityProvider = + options.identityProvider ?? new LinuxProcProcessIdentityProvider(); + this.platform = options.platform ?? process.platform; + this.pid = options.pid ?? process.pid; + this.randomId = options.randomId ?? (() => randomBytes(16).toString('hex')); + } + + async activated(audit: RuntimeRolloutLoadAudit): Promise { + if ( + typeof audit.revision !== 'string' || + typeof audit.sourceSha256 !== 'string' + ) { + throw new TypeError( + 'Accepted rollout audit lacks durable revision or source digest', + ); + } + await this.assertRoot(); + const processIdentity = await this.captureProcessIdentity(); + const existing = await this.read(); + await this.assertReplaceable(existing); + const receipt = createManualPrimaryRuntimeReceipt({ + activationId: this.randomId(), + profile: this.profile, + revision: audit.revision, + rolloutSourceSha256: audit.sourceSha256, + activatedAtMs: this.now(), + process: processIdentity, + }); + await this.write(receipt); + this.current = receipt; + } + + async stopping(): Promise { + await this.transition('stopping'); + } + + async stopped(): Promise { + await this.transition('stopped'); + } + + async failed(): Promise { + if (this.current === undefined) return; + await this.transition('failed'); + } + + private async transition( + state: Exclude, + ): Promise { + if (this.current === undefined) { + throw new ManualPrimaryRuntimeReceiptConflictError( + 'Manual Primary runtime receipt was not activated by this process', + ); + } + const observed = await this.read(); + if ( + observed === undefined || + observed.activationId !== this.current.activationId || + observed.receiptSha256 !== this.current.receiptSha256 + ) { + throw new ManualPrimaryRuntimeReceiptConflictError( + 'Manual Primary runtime receipt ownership changed', + ); + } + const next = transitionManualPrimaryRuntimeReceipt( + this.current, + state, + Math.max(this.current.updatedAtMs, this.now()), + ); + await this.write(next); + this.current = next; + } + + private now(): number { + const value = this.clock.now(); + if (!Number.isSafeInteger(value) || value < 0) { + throw new TypeError('Manual Primary runtime receipt clock is invalid'); + } + return value; + } + + private async captureProcessIdentity(): Promise { + const identity = await this.identityProvider.capture(this.pid); + if (identity !== null) { + return { kind: 'linux-proc', ...identity }; + } + if (this.platform === 'linux') { + throw new Error('Linux process identity is unavailable'); + } + return { kind: 'portable', platform: this.platform, pid: this.pid }; + } + + private async assertReplaceable( + existing: ManualPrimaryRuntimeReceipt | undefined, + ): Promise { + if ( + existing === undefined || + existing.state === 'stopped' || + existing.state === 'failed' + ) { + return; + } + if (existing.process.kind !== 'linux-proc') return; + const inspection = await this.identityProvider.inspect(existing.process); + if (inspection.status === 'running') { + throw new ManualPrimaryRuntimeReceiptConflictError( + 'Another Manual Primary runtime receipt is still live', + ); + } + if (!['exited', 'identity_mismatch'].includes(inspection.status)) { + throw new ManualPrimaryRuntimeReceiptConflictError( + 'Previous Manual Primary runtime identity cannot be disproved', + ); + } + } + + private async assertRoot(): Promise { + const stat = await fs.lstat(this.root); + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + (stat.mode & 0o022) !== 0 || + (typeof process.getuid === 'function' && stat.uid !== process.getuid()) + ) { + throw new TypeError('Manual Primary runtime receipt root is unsafe'); + } + } + + private async read(): Promise { + let handle: fs.FileHandle; + try { + handle = await fs.open( + this.target, + constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0), + ); + } catch (error) { + if (isCode(error, 'ENOENT')) return undefined; + throw error; + } + try { + const stat = await handle.stat(); + if ( + !stat.isFile() || + (stat.mode & 0o077) !== 0 || + (typeof process.getuid === 'function' && + stat.uid !== process.getuid()) || + stat.size < 2 || + stat.size > MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES + ) { + throw new TypeError('Manual Primary runtime receipt file is unsafe'); + } + return parseManualPrimaryRuntimeReceipt( + JSON.parse((await handle.readFile()).toString('utf8')), + ); + } finally { + await handle.close(); + } + } + + private async write(receipt: ManualPrimaryRuntimeReceipt): Promise { + const temporary = path.join( + this.root, + `.ql3-runtime-receipt-${this.pid}-${randomBytes(8).toString('hex')}.tmp`, + ); + let handle: fs.FileHandle | undefined; + try { + handle = await fs.open(temporary, 'wx', 0o600); + await handle.writeFile(`${JSON.stringify(receipt)}\n`, 'utf8'); + await handle.sync(); + await handle.close(); + handle = undefined; + await fs.rename(temporary, this.target); + await this.bestEffortSyncDirectory(); + } finally { + await handle?.close().catch(() => undefined); + await fs.unlink(temporary).catch((error) => { + if (!isCode(error, 'ENOENT')) throw error; + }); + } + } + + private async bestEffortSyncDirectory(): Promise { + try { + const handle = await fs.open(this.root, constants.O_RDONLY); + try { + await handle.sync(); + } finally { + await handle.close(); + } + } catch { + // The atomic state remains valid on filesystems without directory fsync. + } + } +} diff --git a/back/runtime/adapters/legacy/bootstrapDefaultManualPrimaryRuntime.ts b/back/runtime/adapters/legacy/bootstrapDefaultManualPrimaryRuntime.ts index 0c17fb95..c8be4be7 100644 --- a/back/runtime/adapters/legacy/bootstrapDefaultManualPrimaryRuntime.ts +++ b/back/runtime/adapters/legacy/bootstrapDefaultManualPrimaryRuntime.ts @@ -10,6 +10,7 @@ import { installManualPrimaryExecutionRouter } from '../../compatibility/manualP import type { RuntimeRolloutPolicy } from '../../domain/runtimeRollout'; import { parseDeploymentProfile } from '../../domain/deploymentProfile'; import type { RuntimeRolloutLoadResult } from '../../ports/runtimeRolloutLoader'; +import type { ManualPrimaryRuntimeReceiptLifecycle } from '../../ports/manualPrimaryRuntimeReceipt'; import { loadRuntimeRolloutManifest } from '../fs/runtimeRolloutManifestLoader'; import type { DefaultManualPrimaryActivationOptions } from './defaultManualPrimaryActivation'; @@ -28,6 +29,7 @@ export interface BootstrapDefaultManualPrimaryRuntimeOptions loadStack?: () => Promise; install?: typeof installManualPrimaryExecutionRouter; audit?: (record: ManualPrimaryActivationAudit) => void | Promise; + receipt?: ManualPrimaryRuntimeReceiptLifecycle; } /** @@ -52,6 +54,7 @@ export async function bootstrapDefaultManualPrimaryRuntime( Logger.info(`[runtime-activation] ${JSON.stringify(record)}`); }); let stackModule: DefaultManualPrimaryStackModule | undefined; + let receipt: ManualPrimaryRuntimeReceiptLifecycle | undefined; let selectedProfile: 'edge' | 'standalone' | undefined; if (selected) { try { @@ -71,9 +74,23 @@ export async function bootstrapDefaultManualPrimaryRuntime( ); } selectedProfile = profile; - stackModule = await ( - options.loadStack ?? (() => import('./defaultManualPrimaryActivation')) - )(); + const [loadedStack, loadedReceipt] = await Promise.all([ + ( + options.loadStack ?? + (() => import('./defaultManualPrimaryActivation')) + )(), + options.receipt === undefined + ? import('../fs/manualPrimaryRuntimeReceiptStore').then( + ({ ManualPrimaryRuntimeReceiptStore }) => + new ManualPrimaryRuntimeReceiptStore( + config.configPath, + selectedProfile!, + ), + ) + : Promise.resolve(options.receipt), + ]); + stackModule = loadedStack; + receipt = loadedReceipt; } catch (error) { try { await audit({ ...load.audit, activation: 'failed' }); @@ -109,5 +126,6 @@ export async function bootstrapDefaultManualPrimaryRuntime( }, install: options.install ?? installManualPrimaryExecutionRouter, audit, + receipt, }); } diff --git a/back/runtime/application/manualPrimaryRuntimeActivation.ts b/back/runtime/application/manualPrimaryRuntimeActivation.ts index 4e775298..5d955ee8 100644 --- a/back/runtime/application/manualPrimaryRuntimeActivation.ts +++ b/back/runtime/application/manualPrimaryRuntimeActivation.ts @@ -4,6 +4,7 @@ import type { RuntimeRolloutLoadAudit, RuntimeRolloutLoadResult, } from '../ports/runtimeRolloutLoader'; +import type { ManualPrimaryRuntimeReceiptLifecycle } from '../ports/manualPrimaryRuntimeReceipt'; import type { PrimaryCancellationStopResult } from './primaryCancellationLifecycle'; import type { PrimaryCompletionReceiptStopResult } from './primaryCompletionReceiptLifecycle'; import type { PrimaryRunStartupSummary } from './primaryRunStartupSupervisor'; @@ -46,6 +47,7 @@ export interface ManualPrimaryRuntimeActivationOptions { create(policy: RuntimeRolloutPolicy): ManualPrimaryActivationStack; install(router: ManualPrimaryExecutionRouter): () => void; audit(record: ManualPrimaryActivationAudit): void | Promise; + receipt?: ManualPrimaryRuntimeReceiptLifecycle; } export interface ManualPrimaryRuntimeActivationResult { @@ -155,36 +157,65 @@ export async function activateManualPrimaryRuntime( throw new Error('Primary cancellation lifecycle did not start'); } dispose = options.install(stack.router); + await options.receipt?.activated(load.audit); await options.audit({ ...load.audit, activation: 'activated', recovery: recoveryAudit(recovery), }); - let stopped = false; + let stopPromise: Promise | undefined; return { load, active: true, recovery, async stop() { - if (stopped) return 'drained'; - stopped = true; - dispose?.(); - const result = await stopLifecycles(stack!, { - completion: completionStarted, - timeout: timeoutStarted, - cancellation: cancellationStarted, - }); - try { - await options.audit({ - ...load.audit, - activation: 'stopped', - recovery: recoveryAudit(recovery), - }); - } catch { - // Cleanup must not be reversed by a diagnostic failure. - } - return result; + stopPromise ??= (async () => { + let receiptError: unknown; + try { + await options.receipt?.stopping(); + } catch (error) { + receiptError = error; + } + dispose?.(); + let result: PrimaryCancellationStopResult = 'drained'; + let lifecycleError: unknown; + try { + result = await stopLifecycles(stack!, { + completion: completionStarted, + timeout: timeoutStarted, + cancellation: cancellationStarted, + }); + } catch (error) { + lifecycleError = error; + } + if (receiptError === undefined && lifecycleError === undefined) { + try { + await options.receipt?.stopped(); + } catch (error) { + receiptError = error; + } + } + if (receiptError !== undefined || lifecycleError !== undefined) { + try { + await options.receipt?.failed(); + } catch { + // Preserve the first stop failure; a non-active receipt is best effort. + } + throw receiptError ?? lifecycleError; + } + try { + await options.audit({ + ...load.audit, + activation: 'stopped', + recovery: recoveryAudit(recovery), + }); + } catch { + // Cleanup must not be reversed by a diagnostic failure. + } + return result; + })(); + return stopPromise; }, }; } catch (error) { @@ -200,6 +231,11 @@ export async function activateManualPrimaryRuntime( // Preserve the activation error after best-effort cleanup. } } + try { + await options.receipt?.failed(); + } catch { + // Preserve the activation failure; liveness checks still reject stale state. + } try { await options.audit({ ...load.audit, activation: 'failed' }); } catch { diff --git a/back/runtime/domain/manualPrimaryRuntimeReceipt.ts b/back/runtime/domain/manualPrimaryRuntimeReceipt.ts new file mode 100644 index 00000000..efa0ab0b --- /dev/null +++ b/back/runtime/domain/manualPrimaryRuntimeReceipt.ts @@ -0,0 +1,268 @@ +import { createHash } from 'crypto'; + +export const MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA = + 'qinglong/manual-primary-runtime-receipt@v1'; +export const MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE = + 'qinglong3-manual-primary-runtime.json'; +export const MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES = 8 * 1024; + +export type ManualPrimaryRuntimeReceiptState = + | 'active' + | 'stopping' + | 'stopped' + | 'failed'; + +export type ManualPrimaryRuntimeProcessIdentity = + | { + kind: 'linux-proc'; + platform: 'linux'; + pid: number; + processGroupId: number; + bootId: string; + startTimeTicks: string; + } + | { + kind: 'portable'; + platform: string; + pid: number; + }; + +export interface ManualPrimaryRuntimeReceipt { + schema: typeof MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA; + schemaVersion: 1; + activationId: string; + profile: 'edge' | 'standalone'; + revision: string; + rolloutSourceSha256: string; + activatedAtMs: number; + updatedAtMs: number; + state: ManualPrimaryRuntimeReceiptState; + process: ManualPrimaryRuntimeProcessIdentity; + receiptSha256: string; +} + +type ReceiptProjection = Omit; + +const SHA256_PATTERN = /^[a-f0-9]{64}$/u; +const ACTIVATION_ID_PATTERN = /^[a-f0-9]{32}$/u; +const BOOT_ID_PATTERN = /^[A-Za-z0-9-]{1,64}$/u; +const START_TICKS_PATTERN = /^\d{1,32}$/u; +const PLATFORM_PATTERN = /^[a-z0-9_-]{1,32}$/u; + +function record(value: unknown, name: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new TypeError(`${name} must be an object`); + } + return value as Record; +} + +function exactKeys( + value: Record, + keys: readonly string[], + name: string, +): void { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if ( + actual.length !== expected.length || + actual.some((key, index) => key !== expected[index]) + ) { + throw new TypeError(`${name} has an invalid shape`); + } +} + +function safePositiveInteger(value: unknown, name: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 1) { + throw new TypeError(`${name} must be a positive safe integer`); + } + return value as number; +} + +function safeTimestamp(value: unknown, name: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new TypeError(`${name} must be a non-negative safe integer`); + } + return value as number; +} + +function parseProcessIdentity( + value: unknown, +): ManualPrimaryRuntimeProcessIdentity { + const identity = record(value, 'receipt.process'); + if (identity.kind === 'linux-proc') { + exactKeys( + identity, + ['kind', 'platform', 'pid', 'processGroupId', 'bootId', 'startTimeTicks'], + 'receipt.process', + ); + if ( + identity.platform !== 'linux' || + typeof identity.bootId !== 'string' || + !BOOT_ID_PATTERN.test(identity.bootId) || + typeof identity.startTimeTicks !== 'string' || + !START_TICKS_PATTERN.test(identity.startTimeTicks) || + BigInt(identity.startTimeTicks) < BigInt(1) + ) { + throw new TypeError('receipt.process Linux identity is invalid'); + } + return { + kind: 'linux-proc', + platform: 'linux', + pid: safePositiveInteger(identity.pid, 'receipt.process.pid'), + processGroupId: safePositiveInteger( + identity.processGroupId, + 'receipt.process.processGroupId', + ), + bootId: identity.bootId, + startTimeTicks: identity.startTimeTicks, + }; + } + exactKeys(identity, ['kind', 'platform', 'pid'], 'receipt.process'); + if ( + identity.kind !== 'portable' || + typeof identity.platform !== 'string' || + !PLATFORM_PATTERN.test(identity.platform) + ) { + throw new TypeError('receipt.process portable identity is invalid'); + } + return { + kind: 'portable', + platform: identity.platform, + pid: safePositiveInteger(identity.pid, 'receipt.process.pid'), + }; +} + +function canonical(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonical); + if (value !== null && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, nested]) => [key, canonical(nested)]), + ); + } + return value; +} + +function digest(projection: ReceiptProjection): string { + return createHash('sha256') + .update('qinglong.manual-primary-runtime-receipt.v1\0', 'utf8') + .update(JSON.stringify(canonical(projection)), 'utf8') + .digest('hex'); +} + +export function createManualPrimaryRuntimeReceipt(input: { + activationId: string; + profile: 'edge' | 'standalone'; + revision: string; + rolloutSourceSha256: string; + activatedAtMs: number; + process: ManualPrimaryRuntimeProcessIdentity; +}): ManualPrimaryRuntimeReceipt { + const projection: ReceiptProjection = { + schema: MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA, + schemaVersion: 1, + activationId: input.activationId, + profile: input.profile, + revision: input.revision, + rolloutSourceSha256: input.rolloutSourceSha256, + activatedAtMs: input.activatedAtMs, + updatedAtMs: input.activatedAtMs, + state: 'active', + process: input.process, + }; + return parseManualPrimaryRuntimeReceipt({ + ...projection, + receiptSha256: digest(projection), + }); +} + +export function transitionManualPrimaryRuntimeReceipt( + receipt: ManualPrimaryRuntimeReceipt, + state: Exclude, + updatedAtMs: number, +): ManualPrimaryRuntimeReceipt { + const current = parseManualPrimaryRuntimeReceipt(receipt); + const projection: ReceiptProjection = { + schema: current.schema, + schemaVersion: 1, + activationId: current.activationId, + profile: current.profile, + revision: current.revision, + rolloutSourceSha256: current.rolloutSourceSha256, + activatedAtMs: current.activatedAtMs, + updatedAtMs, + state, + process: current.process, + }; + return parseManualPrimaryRuntimeReceipt({ + ...projection, + receiptSha256: digest(projection), + }); +} + +export function parseManualPrimaryRuntimeReceipt( + value: unknown, +): ManualPrimaryRuntimeReceipt { + const receipt = record(value, 'receipt'); + exactKeys( + receipt, + [ + 'schema', + 'schemaVersion', + 'activationId', + 'profile', + 'revision', + 'rolloutSourceSha256', + 'activatedAtMs', + 'updatedAtMs', + 'state', + 'process', + 'receiptSha256', + ], + 'receipt', + ); + const activatedAtMs = safeTimestamp( + receipt.activatedAtMs, + 'receipt.activatedAtMs', + ); + const updatedAtMs = safeTimestamp(receipt.updatedAtMs, 'receipt.updatedAtMs'); + if ( + receipt.schema !== MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA || + receipt.schemaVersion !== 1 || + typeof receipt.activationId !== 'string' || + !ACTIVATION_ID_PATTERN.test(receipt.activationId) || + (receipt.profile !== 'edge' && receipt.profile !== 'standalone') || + typeof receipt.revision !== 'string' || + receipt.revision.length < 1 || + receipt.revision.length > 128 || + /[\u0000-\u001f\u007f]/u.test(receipt.revision) || + typeof receipt.rolloutSourceSha256 !== 'string' || + !SHA256_PATTERN.test(receipt.rolloutSourceSha256) || + !['active', 'stopping', 'stopped', 'failed'].includes( + receipt.state as string, + ) || + typeof receipt.receiptSha256 !== 'string' || + !SHA256_PATTERN.test(receipt.receiptSha256) || + updatedAtMs < activatedAtMs || + (receipt.state === 'active' && updatedAtMs !== activatedAtMs) + ) { + throw new TypeError('Manual Primary runtime receipt is invalid'); + } + const projection: ReceiptProjection = { + schema: MANUAL_PRIMARY_RUNTIME_RECEIPT_SCHEMA, + schemaVersion: 1, + activationId: receipt.activationId, + profile: receipt.profile, + revision: receipt.revision, + rolloutSourceSha256: receipt.rolloutSourceSha256, + activatedAtMs, + updatedAtMs, + state: receipt.state as ManualPrimaryRuntimeReceiptState, + process: parseProcessIdentity(receipt.process), + }; + if (digest(projection) !== receipt.receiptSha256) { + throw new TypeError('Manual Primary runtime receipt digest is invalid'); + } + return { ...projection, receiptSha256: receipt.receiptSha256 }; +} diff --git a/back/runtime/ports/manualPrimaryRuntimeReceipt.ts b/back/runtime/ports/manualPrimaryRuntimeReceipt.ts new file mode 100644 index 00000000..d40c4b81 --- /dev/null +++ b/back/runtime/ports/manualPrimaryRuntimeReceipt.ts @@ -0,0 +1,8 @@ +import type { RuntimeRolloutLoadAudit } from './runtimeRolloutLoader'; + +export interface ManualPrimaryRuntimeReceiptLifecycle { + activated(audit: RuntimeRolloutLoadAudit): Promise; + stopping(): Promise; + stopped(): Promise; + failed(): Promise; +} diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 33a96e46..81ea31c4 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,18 @@ 最新增量证据（2026-08-19）： +- D-362/ADR-0455（已接受；首次真实目标实例执行待运维）：manual Primary 默认 bootstrap 在 router 安装后、`activated` 审计前原子发布 + `qinglong/manual-primary-runtime-receipt@v1` 当前状态，并在清理前后推进 `active → stopping → stopped`；激活或停止失败收敛为 `failed`。receipt 固定为 config root + 内一个 `0600`、8 KiB 上限、有 domain-separated 自摘要的 observed-state projection，绑定 Profile、manifest revision 与原始 SHA-256，但不成为第二 rollout authority。 + Linux 以 boot ID、PID、process-group 与 start ticks 联合复验，已有 live identity 时拒绝替换，旧 identity 确定退出后才允许新 generation 接管；非 Linux portable + receipt 可用于开发启动，但独立 auditor 必须返回 unsupported，不能宣称 current。canary auditor 新增 `--require=active`，同时复核 plan/qualification/selection/live manifest、 + receipt binding 与进程 current；`off|rolled-back` 也会拒绝仍存活的旧 Primary。由此明确了无 watcher 约束：审批过期/rollback 使下一次 bootstrap 为 off，但不会自动卸载已运行 + router，operator 必须停止/重启后再闭合审计。默认关闭路径零新增 I/O/timer/watcher/连接，不新增 package、依赖、schema、migration 或部署对象。 + 阶段门已重跑：聚焦 `25/25`、`build:back`、完整 backend `1,487 pass / 0 fail / 2 conditional skip`、18-package clean build/test、四项架构审计与 `14/14` + artifact audit 全部通过；14 档字节与 D-361 一致。隔离 frozen-dependency Linux arm64 Docker 的 128 MiB router/256 MiB Edge release peak 分别为 + `87,339,008 / 145,506,304` bytes，`memory.events max/oom/oom_kill` 增量均为 0。D-362 不改 PostgreSQL schema/migration、依赖树或 Kubernetes 拓扑，故不重跑 + PostgreSQL HA；相邻 D-359 的 `142/142` 与 timeline `1→2` 仅作为既有证据。 + - D-361/ADR-0454（已接受；首次真实用户目标实例执行待运维）：把 D-360 的 manual Primary evidence/gate 收敛为一次显式、可重放且不新增常驻组件的目标实例仪式。 `prepare → observe → resource → qualify → approve → status/audit → rollback` 固定为七个 operator-driven 阶段；Edge 精确 8 条，Standalone 由维护者在 32–128 条内选择精确目标。prepare 绑定当时 absent/disabled rollout 基线并只输出 `QL_DEPLOYMENT_PROFILE`、`QL3_SHADOW_ORIGINS=manual` 和唯一 @@ -8332,10 +8344,10 @@ Primary 不使用普通环境变量或宽泛全局开关启用。孵化配置面 - edge 不启动文件 watcher。配置只在显式 bootstrap/reload 时读取，禁用时不创建 router、timer、连接或后台任务。 - ADR-0454 的 `prepare` 先以不可变 plan 绑定 Profile、精确 admission 目标和当时 absent/disabled rollout SHA-256；`qualify` 再绑定 capture/terminal/resource 的原始文件摘要、 canonical 摘要和 Primary gate。只有显式 `approve` 能在复验 live 基线后写入本节 schema v2 manifest。 -- `approve` 的 durable 结果是 `primary_selected`，不是 `primary_active`。operator 必须重启目标 worker，并从现有 bootstrap 的同 revision `selected → reconciled → activated` - 结构化审计证明实际运行态；一次性独立 auditor 刻意不输出 active 结论。 +- `approve` 的 durable 结果是 `primary_selected`，不是 `primary_active`。operator 必须重启目标 worker；ADR-0455 固定 bootstrap 的同 revision + `selected → reconciled → router installed → durable active receipt → activated`，独立 auditor 只有在 receipt binding 与 Linux 联合进程身份均成立时才通过 `--require=active`。 - `rollback` 在 live manifest 替换前先写绑定 enabled/disabled 摘要的 intent，替换后再写绑定 intent 摘要的 completion；同参数重放覆盖两个 response-loss 窗口。审批过期按 off - 解释但不会自动改写文件，仍需显式 rollback 和重启。 + 解释但不会自动改写文件，也不会让既有无 watcher 进程自动卸载 router，仍需显式 rollback 和停止/重启；`off|rolled-back` 审计拒绝仍 current 的 runtime receipt。 实验 manifest 结构固定为： @@ -8372,10 +8384,11 @@ Primary 不使用普通环境变量或宽泛全局开关启用。孵化配置面 当前 `next` 已在 HTTP worker 接入轻量 manifest bootstrap。文件缺失、disabled、rejected、manual 非 primary、v1 enabled 或 Primary bundle 缺失/篡改/不可重放时不会加载完整 Runtime stack、创建 router 或启动 timer；只有 schema v2、五项 capability gate 完整，且 bundle 经 loader 独立重算为 eligible 的 manual primary 配置才惰性加载真实组件。激活顺序固定为：重放 evidence→复验实际 Profile→记录 selected 审计→完整有界 startup reconciliation→记录 -reconciled 审计→启动 completion/timeout/cancel lifecycle→安装 router→记录 activated；任何一步失败都会撤销 router，并按 producer → consumer 顺序停止 -lifecycle。HTTP shutdown 与监听失败也会执行有界清理。`QL_DEPLOYMENT_PROFILE` 未配置时为 standalone，非法值、receipt Profile 不一致或在 +reconciled 审计→启动 completion/timeout/cancel lifecycle→安装 router→原子发布 `active` receipt→记录 activated；任何一步失败都会撤销 router，并按 producer → consumer 顺序停止 +lifecycle。HTTP shutdown 与监听失败先写 `stopping`、执行有界清理，再写 `stopped`；异常路径尝试写 `failed`。Linux receipt 绑定 boot/PID/process-group/start ticks，非 Linux +portable receipt 不获得 current 结论。`QL_DEPLOYMENT_PROFILE` 未配置时为 standalone，非法值、Primary gate Profile 不一致或在 cluster-control/worker 中误装本机 SQLite Primary 时 fail closed。目标实例仪式、部署配置写入、只读选择状态和操作回滚工具已由 ADR-0454 固化；首次真实 Edge/Standalone -完整执行、bootstrap activated durable receipt、固定物理 edge/flash/断电 Gate 完成后仍需单独评审。 +完整执行、固定物理 edge/flash/断电与多写者配置 authority Gate 完成后仍需单独评审。 ## 26. 交付阶段 @@ -9201,7 +9214,7 @@ flowchart LR | PR-2 Run 状态机 | Incubating | 纯转换表、终态/时间/错误/执行器元数据规则、Run version 与 event sequence CAS、事务性 RunCommandService、回滚测试 | 重复 Worker callback/fencing、并发数据库压力测试、Primary 执行链接入 | | PR-3 Executor 端口 | Incubating | ADR-0003、ExecutionSpec/Context/Handle/Result、Executor port、LocalProcessExecutor、进程组取消/超时升级、流式背压、Legacy Cron spec builder、真实进程 contract tests、可复现 edge 基准入口 | 固定 edge/多架构设备基线、Legacy builder 与 makeCommand 差异审计、Primary 生产流量接入 | | PR-4 Shadow Run | Incubating | origin 三态策略；默认关闭的 `QL3_SHADOW_ORIGINS`；manual、scheduled_node、boot、subscription、system 与 script 现有 ChildProcess 旁路观察；system crond 显式 origin marker、Shell execution ID、finish-only 准入、确定性 Run/Attempt 与 exact replay；`@once` 保持 manual、gRPC transport 不冒充 origin 的准入裁决；每个 worker 懒加载；Run/Attempt/Event 影子生命周期；稳定且不复制 caller 原文的 task identity/revision 与有界日志引用；同 worker 有界注册表和跨 worker 持久化候选关联；stop all/stop instance、Shell callback、乱序/迟到/歧义处理；监听前一次性、Profile-aware 的 keyset Startup Reconciler，终态证据补齐、lost/abandoned/pending 分流与 terminal Attempt response-loss 修复；origin-bounded 且逐级守恒的版本化 startup difference report、固定字段 metric batch 与一次性 collector；显式、只读、闭合窗口且 Profile-bounded 的 Shadow→Legacy 终态差异审计；128/256 MiB Linux arm64 资源门、SQLite 零增长与 Shadow enabled→off 进程重启回滚；process-epoch Legacy admission/capture/failure/pending 守恒；clean-shutdown `0600` no-replace capture+startup exporter；manual Edge 8/Standalone 32–128 canary；capture/terminal/resource 自包含 Primary bundle；rollout v2 loader 重算 source digest 与 eligibility；不可变 prepare/observe/resource/qualify 目标实例仪式、独立只读 audit；失败开放和契约测试 | 首次真实目标实例完整 canary 与 bootstrap activated 记录、其他 origin 独立 capture/Primary gate、固定物理 edge/flash/断电证据 | -| PR-5 Primary LocalExecutor | Incubating（默认不激活，仅 manifest-gated manual） | runtime-owned Run 创建器；持久化先于 spawn；Run/Attempt 完整成功、失败、取消、超时与 lost 闭环；Executor handle 身份校验；spawn 后激活写失败的 stop+lost 补偿；completion rejection 安全收敛；独立 Primary 幂等查询与唯一索引竞态裁决；durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询；最多 64 条一页的 cross-worker cancellation source；独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent；最多 64 页的单周期 cancel supervisor；显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner；Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller；完整有界分页且 fail-closed 的 startup Reconcile supervisor；RunningInstance nullable `run_id/attempt_id` 关联；Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance，失败整体回滚；有界且防穿越的 legacy log output ref；manual owner seam、真实本机装配、单 spawn/fail-closed；严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计；HTTP worker 已接轻量 lazy bootstrap，accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router 顺序激活，失败撤销，监听失败和 shutdown 有界停止；Primary timeout 在 spawn 前持久化绝对 deadline，有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch；代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限，cluster-control/worker 拒绝误装本机 SQLite Primary；统一 CompletionService 原子提交 Attempt/Run/双 Event，spawn 前保存 callback token hash、终态推进 sequence，实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window；manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理；Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace；`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描，使终态残留继续可发现；确定无效的已知 Attempt receipt 先持久化隔离状态，再进入确定性私有分片 quarantine；终态 missing 与 quarantine 按 edge/standalone retention 有界清理；非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI；扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数；ENOSPC 与 launcher receipt 存储失败有代码门禁；显式最长 24 小时 approve 写入、`primary_selected` 只读状态、selection receipt、approval-expiry off 与 intent/completion crash-replay rollback | 首次真实目标实例激活/回滚及 durable runtime activation receipt；PostgreSQL CancellationDispatch adapter；cluster-control 生产启动拓扑；固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练 | +| PR-5 Primary LocalExecutor | Incubating（默认不激活，仅 manifest-gated manual） | runtime-owned Run 创建器；持久化先于 spawn；Run/Attempt 完整成功、失败、取消、超时与 lost 闭环；Executor handle 身份校验；spawn 后激活写失败的 stop+lost 补偿；completion rejection 安全收敛；独立 Primary 幂等查询与唯一索引竞态裁决；durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询；最多 64 条一页的 cross-worker cancellation source；独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent；最多 64 页的单周期 cancel supervisor；显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner；Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller；完整有界分页且 fail-closed 的 startup Reconcile supervisor；RunningInstance nullable `run_id/attempt_id` 关联；Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance，失败整体回滚；有界且防穿越的 legacy log output ref；manual owner seam、真实本机装配、单 spawn/fail-closed；严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计；HTTP worker 已接轻量 lazy bootstrap，accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router→durable active receipt 顺序激活，失败撤销，监听失败和 shutdown 以 stopping→有界清理→stopped/failed 失效；receipt 固定为单文件 observed-state projection，Linux 以 boot/PID/process-group/start ticks 复验，独立 auditor 支持 active 且 off/rolled-back 拒绝 live runtime；Primary timeout 在 spawn 前持久化绝对 deadline，有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch；代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限，cluster-control/worker 拒绝误装本机 SQLite Primary；统一 CompletionService 原子提交 Attempt/Run/双 Event，spawn 前保存 callback token hash、终态推进 sequence，实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window；manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理；Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace；`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描，使终态残留继续可发现；确定无效的已知 Attempt receipt 先持久化隔离状态，再进入确定性私有分片 quarantine；终态 missing 与 quarantine 按 edge/standalone retention 有界清理；非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI；扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数；ENOSPC 与 launcher receipt 存储失败有代码门禁；显式最长 24 小时 approve 写入、`primary_selected` 只读状态、selection receipt、approval-expiry off 与 intent/completion crash-replay rollback | 首次真实目标实例完整激活/回滚仪式；PostgreSQL CancellationDispatch adapter；cluster-control 生产启动拓扑；固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练；共享 config 多写者 authority | | PR-7 Worker Session、Run Lease 与启动协议基础 | Incubating（默认关闭，独立入口显式 opt-in） | ADR-0012/0013/0014/0021/0057–0061/0108–0121/0231–0239/0377；有界 capability/Placement/Dispatcher；SQLite 协议孵化与 PostgreSQL v9 Session/Run Lease/credential/attestation authority；immutable revision Placement、数据库时钟 keyset candidate、认证 Worker Pull、digest-only offer recovery；versioned capability-free ExecutionSpec response、stable claim 跨重启退避、单 owner 原子 inbox 准入与 TLS 1.3 mTLS/`ql3w` HTTPS client；同一 package journal 上 revision-fenced starting/spawn/started/running/completion 状态、callback digest、tagged no-spawn 与 ambiguous recovery；PostgreSQL starting/running/start-failure/completion 数据库权威事务、精确重放与 cancellation/timeout 优先终态；batch Secret delivery 在 Attempt advisory lock 下复验 Session/Lease/revision 完整围栏并复用单 Agent，Secret-before-Artifact materializer 将同一 log ID 交给 Executor/journal/running ACK；offer-scoped `wlog-*` 私有文件 spool、Edge/Node 容量策略、append/quota/path 防护、barrier 后 output ownership、受审 POSIX Executor、truncation fact、固定内存流式 source、认证 Artifact stream、共享 immutable store port、S3-compatible SSE/checksum/条件 promotion adapter、upload-before-completion 协调，以及 Local/Cluster 同构、Profile-aware、ETag-fenced range read；用户取消 run.stop mutation 以数据库时间写 intent/Event 并在事务内复验 Project/RoleBinding fence；非执行取消 convergence lifecycle、运行期 expiry 与安全 lost retry 已接入 cluster-control 单一全局 cadence；完整 generation/version/token/Attempt fencing；独立最小权限 Worker ingress、CA/CRL 与连接 generation 热重载；offer journal、spawn barrier、receipt-first recovery；独立 `@qinglong/worker-runtime` 的本地 P-256 CSR、key/chain/trust 验证、generation + active pointer 安装和持久退避；默认关闭的 production process 已装配具体 execution graph、完整 Session heartbeat/drain/offline、direct-file bootstrap、单 Agent/单 cadence、startup reconciliation、证书 maintenance、transport fail-close/recovery 与 Edge/Node 有界预算；真实 PostgreSQL 18 + Linux Node 合约已覆盖 Run completion、credential 和 CA 双轮换且保持同一 Session；真实 K3s 合约已覆盖 TLS/credential Secret 分权、双对象 CAS、Recreate 顺序、identity generation 与单节点 PVC recovery；所有能力默认不可达且受 edge/cluster import audit 约束 | 具体 cert-manager/Vault/SPIFFE/离线 CA adapter 与模板、ingress reload controller、生产 RBAC、证书到期告警和 `ql3w` credential recovery 产品面；具体 KMS/Vault Secret provider、对象存储 credential/temporary lifecycle 与 retention/tombstone；Worker 管理 API；真实 Kubernetes 多节点 CSI/node-loss/production 360 秒 drain 与固定 edge 文件系统 suspend/时钟/断电、x64/arm64 资源门禁 | | PR-8 Project/Policy/Approval Core | Incubating（默认拒绝、无生产业务执行入口） | ADR-0028；统一六类 ActorRef 与 exact-shape 校验；`0017` ownerless default Project 和 append-only versioned RoleBinding；owner/admin/operator/viewer 固定矩阵；Project 内 mutation 幂等、expected-version CAS、双 SQLite 连接竞争门禁；archived read-only、revocation、存储损坏 fail-closed；Agent 写/Secret/Tool `require_approval`；ADR-0047 把六类 subject、role/permission matrix 与 fence 抽到 runtime-core，`pg-0004-project-policy`/capability v3 建立 ownerless PostgreSQL baseline、严格 role/state CHECK、append-only runtime 权限、SERIALIZABLE Project lock、mutation replay、双连接单 winner 和 cluster admission authorizer；ADR-0049/`pg-0005` capability v4 建立 stable IdentitySubject、append-only digest-only API credential、真实 cluster bearer authenticator、write-only durable security audit 与最小权限 runtime role，且已验证 HTTP→credential→Policy→audit→handler 纵向链路；ADR-0051 建立 `/api/v3` 认证前 peer/global 双预算、transport-peer-only、无 timer 且有界内存的 overload shield；ADR-0027 Artifact authorizer adapter；ADR-0029 `AuthenticatedPrincipal` contract、`0018` digest-only versioned challenge、CSPRNG/TTL、同事务消费 challenge + 写首 owner、精确重放与双连接竞争/崩溃回滚门禁；ADR-0030 `0019` stable identity/binding、legacy HS384 + current-session membership、logout/platform/revoke/disable、single-factor 与损坏 fail-closed 门禁；ADR-0031 `0020` digest-bound ApprovalRequest、User-only decision、Project/Role version fence、精确 expiry/重放/并发裁决及同事务 immutable dispatch；ADR-0032 `0021` execution backfill、三表原子 consume、稳定 due keyset、claim/renew/start/result fencing、pre-start takeover/post-start recovery-required、attempt budget、handler inspect/digest barrier 和 bounded dispatcher；ADR-0033/`0022` control/resolution backfill、start/renew/completion 原子联动、稳定 recovery keyset、双 resolver claim/takeover、finding/result 精确重放、自动/人工终结、迟到 completion 单 winner 和 evidence-only bounded reconciler；ADR-0034/`0023` 首个 `run.create` canonical plan、Run/Attempt/Event/receipt 同事务、幂等 collision fail-closed、renew/终态 fence、真实 SQLite handler 与 automatic evidence provider；ADR-0035/`0024` 独立 `approval.recover` 矩阵、稳定 User + 五分钟强认证、Project/RoleBinding fence、human resolution + authorization fact 原子提交、撤权竞态与回滚门禁；ADR-0036 recovery-first 单 timer lifecycle、edge/standalone 独立 cadence/页预算、跨周期 cursor、非重叠与有界 stop；ADR-0074 以新的 Node 24 SQLite v5 ownerless Project/RoleBinding/audit authority 和独立 local-secret-admin 提供强 Principal、`secret.manage`、撤权 fence、envelope+allowed audit 原子提交及不回显语义；ADR-0086 以可信 POSIX console 和 staged delivery 完成本机首 Owner 产品 ceremony | fresh database/pepper setup 与安全迁移向导；`shareStore`/Express 到 authentication core 的 production migration；credential rotation/revocation API、mTLS/Worker enrollment、恢复码；Project/Role/Approval/Secret 管理 CLI/API/UI、audit retention/query/export/alert、preview Artifact/digest/immutable plan builder、真实 MFA/hardware adapter、人工 recovery API/UI/独立 rate limit 与审计事件、handler/provider registry、lifecycle startup/shutdown/指标/admission gate；PostgreSQL action/receipt/provider/recovery-authorization 与 OPA adapter、缓存 version 失效；Tool/Package/Secret/Shell 各自的 handler/evidence contract；Secret/Run/Tool/Workflow waiting_approval 全入口装配；完整回滚演练 | diff --git a/docs/adr/ADR-0454-target-instance-manual-primary-canary-ceremony.md b/docs/adr/ADR-0454-target-instance-manual-primary-canary-ceremony.md index 5a0daafd..f205b254 100644 --- a/docs/adr/ADR-0454-target-instance-manual-primary-canary-ceremony.md +++ b/docs/adr/ADR-0454-target-instance-manual-primary-canary-ceremony.md @@ -5,6 +5,7 @@ - 关联 RFC：QL-RFC-0001 D-361、PR-4、PR-5 - 关联 ADR：ADR-0002、ADR-0449、ADR-0450、ADR-0451、ADR-0453 - Amends：ADR-0453 的首次目标实例 canary 操作缺口 +- Amended by：ADR-0455 的 Profile-bounded durable runtime activation receipt ## 上下文 @@ -36,13 +37,12 @@ Edge 可能是 128 MiB 路由设备，不能为 canary 新增 daemon、watcher 7. `approve` 是唯一写 live manifest 的操作。它要求显式 `approvedBy`，审批最短一分钟、最长 24 小时；先复核 plan 时的 absent/disabled 基线，disabled 基线先 no-replace 归档，再以同目录 fsync 临时文件和最终摘要复核发布 schema v2 manifest。结果状态固定为 `activation_approved`/`primary_selected`，并明确 `requiresRestart=true`、`runtimeActivationObserved=false`；配置选择不得冒充 HTTP worker 已完成 startup reconciliation 和 router activation。 -8. 应用重启后，实际运行态仍由现有 bootstrap 的 `selected → reconciled → activated` audit 证明。独立 `audit:manual-primary-canary:ql3` 只读复核 plan、source、gate、 - qualification、selection receipt 与 live manifest，可要求 `prepared|qualified|selected|off|rolled-back`；`off` 只证明 loader 当前关闭，`rolled-back` 还要求本 session 的 - intent/completion 摘要链完整。auditor 刻意不提供 `primary_active` 结论。 +8. 应用重启后，实际运行态由 bootstrap 的 `selected → reconciled → durable active receipt → activated` 证明。ADR-0455 已让独立 auditor 复核 receipt 与 Linux + boot/PID/start-time 联合身份，并新增 `active` 要求；`off|rolled-back` 同时要求不存在仍 current 的 runtime。非 Linux portable receipt 不能获得 current 结论。 9. `rollback` 只接受四个固定 reason 与有界 operator。它先 no-replace 发布 intent，绑定当前 enabled manifest SHA-256 和目标 disabled SHA-256，再复核当前摘要并原子替换 live manifest，最后发布 completion；intent 或 completion response loss 可用相同参数重放。disabled 已有效但不同于本 session 的目标时拒绝覆盖。 -10. 审批过期后，既有 loader 必须立即 fail-closed 为 off；状态/auditor 报 `approvalExpired=true` 和 `rolloutMode=off`。operator 仍应执行显式 rollback，把磁盘事实 - 收敛为 schema v2 disabled manifest，然后重启应用；过期不能被当作自动续期或自动删除 authority。 +10. 审批过期后，loader 对下一次 bootstrap 必须 fail-closed 为 off；状态/auditor 报 `approvalExpired=true` 和 `rolloutMode=off`。无 watcher 的既有进程不会仅因磁盘 + 审批过期而自动卸载 router，故 `off` 审计会在 current receipt 仍存活时失败。operator 必须显式 rollback 并停止/重启应用；过期不能被当作自动续期、自动删除或已停止证明。 ## 被拒绝的替代方案 @@ -87,4 +87,4 @@ Edge 可能是 128 MiB 路由设备，不能为 canary 新增 daemon、watcher 维护者仍需在一台真实目标 Edge 或 Standalone 实例执行 [Manual Primary Canary 操作手册](../operations/ql3-manual-primary-canary.md)，并保留 capture、terminal、resource、 gate、qualification、selection、bootstrap activated audit 与 rollback completion。其他 origin 必须建立自己的 admission authority 与 gate；固定物理路由/flash/断电、 -运行态 durable activation receipt 和多写者 config authority 仍是独立后续工作。 +多写者 config authority 仍是独立后续工作；运行态 durable activation receipt 已由 ADR-0455 补齐，但首次真实目标实例执行仍待运维。 diff --git a/docs/adr/ADR-0455-profile-bounded-manual-primary-runtime-activation-receipt.md b/docs/adr/ADR-0455-profile-bounded-manual-primary-runtime-activation-receipt.md new file mode 100644 index 00000000..1fd4162b --- /dev/null +++ b/docs/adr/ADR-0455-profile-bounded-manual-primary-runtime-activation-receipt.md @@ -0,0 +1,66 @@ +# ADR-0455：Profile-bounded Manual Primary 运行态激活凭据 + +- 状态：Accepted（实现完成；首次真实目标实例执行仍待运维） +- 日期：2026-08-19 +- 关联 RFC：QL-RFC-0001 D-362、PR-4、PR-5 +- 关联 ADR：ADR-0002、ADR-0453、ADR-0454 +- Amends：ADR-0454 的运行态 durable activation receipt 缺口 + +## 上下文 + +ADR-0454 把 `primary_selected` 与运行态 `activated` 明确分开，但后者仍只存在于进程日志。日志可能没有持久 sink、可能被轮转，也无法在进程退出后区分“曾经激活”与“当前实例仍在运行”。仅记录 PID 又会受到 PID 重用、宿主重启和容器 namespace 变化影响。 + +Edge 可能只有 128 MiB 内存与低耐久 flash，不能为运行态证明增加 watcher、心跳 timer、数据库表、遥测 sidecar 或无限增长事件日志。Standalone 即使运行在集群节点上，也仍是本机 Profile，不能把宿主形态当成 cluster-control authority。 + +## 决策 + +1. 默认 HTTP worker 仅在 accepted schema v2 manifest 精确选择 `manual=primary` 且 Profile 为 `edge|standalone` 时，惰性加载凭据 adapter。disabled、rejected 和非 Primary 路径不 import adapter、不读写凭据。 +2. 当前状态固定为 config root 下唯一的 `qinglong3-manual-primary-runtime.json`，schema 为 `qinglong/manual-primary-runtime-receipt@v1`。它是 observed-state projection，不是 rollout authority；`qinglong3-rollout.json` 仍是唯一期望配置。 +3. 凭据只包含随机 activation ID、Profile、manifest revision、manifest 原始 SHA-256、激活/更新时间、`active|stopping|stopped|failed` 状态、进程身份和 domain-separated 自摘要。文件不包含 Task、Run、Cron、命令、路径、用户、日志、Secret 或错误正文。 +4. 文件必须位于当前 UID 拥有、非 symlink、group/world 不可写的真实 config root，以 `0600` 临时 inode、file fsync、原子 rename 和 best-effort directory fsync 发布。大小硬上限为 8 KiB；读取使用 `O_NOFOLLOW` 并复验 owner、mode、普通文件、exact shape 与自摘要。 +5. Linux 当前性使用 `/proc` 的 boot ID、PID、process-group ID 和 start-time ticks 联合身份，避免 PID 重用。已有 `active|stopping` 凭据只有在该联合身份确定 `exited|identity_mismatch` 后才能由下一实例替换；仍为 `running` 或无法反证时 fail closed。非 Linux 只写 portable PID，允许开发启动，但独立 auditor 必须返回 `unsupported`，不得宣称 current。 +6. 激活顺序固定为 reconciliation → 三个 lifecycle → router install → durable `active` → structured `activated` audit。写 `active` 失败时必须立即撤销 router、停止已启动 lifecycle，并保持原激活错误；因此日志中的 `activated` 必然晚于 durable receipt。 +7. 干净停止先写 `stopping`，再撤销 router 并按 timeout → cancellation → completion 停止 lifecycle，最后写 `stopped`。任一停止或凭据转换失败时尝试写 `failed`，停止调用仍返回第一个错误；`stopping|stopped|failed` 都不能被审计为 active。重复 stop 共用同一 Promise，不重复释放资源或写状态。 +8. 独立 canary auditor 新增 `--require=active`：必须同时复核 plan、qualification、selection、live enabled manifest、receipt binding 和 Linux 联合进程身份。报告分开输出 `runtimeActivationObserved`、`runtimeActivationCurrent`、receipt state 与 process state，不输出 PID、boot ID 或 activation ID。 +9. `--require=off|rolled-back` 现在还要求 `runtimeActivationCurrent=false`。manifest 已回滚或审批已过期但旧 worker 尚未停止时，loader 对下一次 bootstrap 的决策虽为 off，当前内存 router 仍可能继续拥有 manual；auditor 必须拒绝把这种状态表述为已关闭。无 watcher 的部署必须显式停止/重启后再完成 off/rollback 审计。 +10. 本凭据不是跨 UID、共享卷或多写者共识锁。同一 UID 可改写 config root，启动前的 read/inspect/write 也不替代部署系统的单实例约束；共享 config、多主机签名或强互斥需要独立 authority，不能由本地 receipt 推断。 + +## 被拒绝的替代方案 + +### 用 enabled manifest 表示 active + +拒绝。manifest 可能尚未被当前 worker 读取，reconciliation 可能失败，审批可能过期，回滚后旧进程也可能尚未重启。 + +### 每秒刷新心跳文件 + +拒绝。它增加永久 timer、写放大和 flash 磨损，仍需处理 suspend、时钟跳变与调度延迟。进程联合身份提供按需审计，不需要周期写盘。 + +### 每次启动追加一份不可变 receipt + +拒绝。无限历史会在路由设备上持续增长，而且仍需另一个 current pointer。单个有摘要的 observed-state projection 足以表达本切片状态；不可变 selection 与 rollback 链继续由 ADR-0454 保存。 + +### 非 Linux 使用 `kill(pid, 0)` 宣称 current + +拒绝。它不能防 PID 重用，也不能绑定宿主 boot。portable receipt 只证明 bootstrap 曾写入，不提供 current 结论。 + +## 资源、安全与部署影响 + +- 默认关闭路径零新增 I/O、timer、watcher、listener、连接和数据库操作。 +- 激活写一次，干净停止最多再写两次；常驻内存只有一个小 receipt 对象，不随任务数增长。 +- 不新增 workspace package、生产依赖、migration、PostgreSQL/Kubernetes 对象或部署端口。 +- receipt 是 owner-private 运维证据，不应发布到公开 artifact；auditor stdout 只输出低敏状态。 +- approval expiry 不会在已运行进程内自动卸载 router。该限制是无 watcher 决策的直接结果，必须通过部署系统的显式 restart/stop 收敛。 + +## 验证 + +- 聚焦 `25/25` 覆盖 active→stopping→stopped、摘要篡改、live identity 冲突、stale generation 接管、portable identity 不冒充 Linux current、receipt 写失败撤销 ownership、停止顺序、canary `--require=active` 以及 live runtime 阻断 rolled-back。 +- `build:back` 与完整 backend 通过：`1,487 pass / 0 fail / 2 conditional skip`。首次沙箱执行仅有 Vault loopback 因 `listen EPERM` 失败；允许 loopback 后原命令完整重跑为上述结果。 +- 18-package clean build/test 退出 0。首次沙箱执行仅有 worker-runtime 三个 TLS/mTLS loopback contract 因相同 `listen EPERM` 失败；允许 loopback 后完整 clean 原命令重跑通过。 +- Edge import、cluster dependency、package boundary、service-manager bridge 四项架构审计零 finding；package boundary 仍为精确 18，`singleSourcePackages=[]`、`shallowSourcePackages=[]`。 +- `14/14` Local Profile artifact audit 全部通过且字节与 D-361 基线一致：基础 Edge/Standalone `2589998/2590076`、adopted `2809293/2809416`、application `3632877/3632997`、application-api `3800430/3800574`、AI `3069251/3069341`、application+AI `4493151/4493283`、MCP `7315930/7316038`。 +- 隔离 frozen-dependency Linux arm64 Node 24 Docker 门通过：128 MiB router stress peak `87,339,008` bytes，256 MiB Edge release peak `145,506,304` bytes，`memory.events max/oom/oom_kill` 增量均为 0；所有 workload 通过。该证据不是物理路由或 flash/断电证明。 +- 本 ADR 不修改依赖树、PostgreSQL schema/migration 或 Kubernetes 拓扑，因此 PostgreSQL HA 不因本切片重复执行。 + +## 后续 + +仍需在真实 Edge/Standalone 目标实例执行 prepare→真实 manual cohort→qualify→approve→restart→`--require=active`→rollback→restart→`--require=rolled-back` 完整仪式。物理 flash/断电、PID namespace、同 UID 恶意并发、共享配置目录和签名式多写者 authority 继续作为独立 Gate。 diff --git a/docs/adr/README.md b/docs/adr/README.md index 7a0abda8..de7d1cf7 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -458,6 +458,7 @@ | [ADR-0452](./ADR-0452-atomic-flattened-backend-build-publication.md) | 原子且扁平兼容的 Backend 构建发布 | Accepted | | [ADR-0453](./ADR-0453-origin-scoped-legacy-shadow-capture-authority-and-primary-gate.md) | Origin-scoped Legacy Shadow 捕获权威与 Primary 门禁 | Accepted（首次真实目标实例 manual canary 待执行） | | [ADR-0454](./ADR-0454-target-instance-manual-primary-canary-ceremony.md) | 目标实例 Manual Primary Canary 与显式回滚仪式 | Accepted（首次真实用户目标实例执行待运维） | +| [ADR-0455](./ADR-0455-profile-bounded-manual-primary-runtime-activation-receipt.md) | Profile-bounded Manual Primary 运行态激活凭据 | Accepted（首次真实目标实例执行待运维） | ## 规则 diff --git a/docs/operations/ql3-manual-primary-canary.md b/docs/operations/ql3-manual-primary-canary.md index 0d9c0d81..5908b3ca 100644 --- a/docs/operations/ql3-manual-primary-canary.md +++ b/docs/operations/ql3-manual-primary-canary.md @@ -117,9 +117,20 @@ pnpm audit:manual-primary-canary:ql3 -- \ ``` 正确状态是 `activation_approved`/`rolloutMode=primary_selected`，不是 `primary_active`；`requiresRestart=true` 且 `runtimeActivationObserved=false`。随后重启应用，并在结构化启动审计中依次确认 -同一 revision 的 `selected`、`reconciled`、`activated`。缺少任一项都不能宣称运行态 Primary 已激活。 +同一 revision 的 `selected`、`reconciled`、`activated`，再执行： -审批过期后 loader 自动 fail-closed 为 off，不会续期。不要修改原 manifest 时间；创建新 session 重新采样。 +```sh +pnpm audit:manual-primary-canary:ql3 -- \ + --root=/ql/data/config \ + --session=edge-20260819-a \ + --require=active +``` + +只有 `runtimeActivationObserved=true`、`runtimeActivationCurrent=true`、`runtimeReceiptState=active` 和 `runtimeProcessState=running` 同时成立，才能宣称当前 Linux worker 已激活。 +receipt 文件固定为 owner-private `qinglong3-manual-primary-runtime.json`，不要复制到公开 artifact。非 Linux portable receipt 只能证明写入发生过，不能通过 `active` 门。 + +审批过期后 loader 对下一次 bootstrap 自动 fail-closed 为 off，不会续期；已经运行的 worker 没有 watcher，不会仅凭磁盘过期自动卸载 router。必须停止/重启并通过 +`--require=off`，不要修改原 manifest 时间；需要重新启用时创建新 session 重新采样。 ## 7. 回滚并重启 @@ -141,5 +152,6 @@ pnpm audit:manual-primary-canary:ql3 -- \ 支持的 reason 只有 `operator_request`、`runtime_failure`、`gate_rejected`、`approval_expired`。rollback 先发布 intent，再摘要复核并原子替换 live manifest，最后发布 completion；响应丢失时使用完全相同的参数重跑。 -`rolled-back` 比普通 `off` 更严格：后者在初始 disabled 或审批过期时也成立，前者还要求本 session 的 intent/completion 摘要链完整。完成后重启应用，确认 rollout loader 返回 -disabled/off，Legacy manual 执行继续可用且 Shadow/Primary 不再接管。保留整个 session 的 `0600` 文件和启动审计用于发布复核，不要覆盖或编辑。 +`rolled-back` 比普通 `off` 更严格：后者在初始 disabled 或审批过期时也成立，前者还要求本 session 的 intent/completion 摘要链完整；两者现在都拒绝仍存活的 current +Primary receipt。rollback 写盘后先停止/重启应用，再运行上面的 `--require=rolled-back`，确认 loader 返回 disabled/off、receipt 为 stopped/failed 或旧 Linux identity 已退出， +Legacy manual 执行继续可用且 Primary 不再接管。保留整个 session 的 `0600` 文件和启动审计用于发布复核，不要覆盖或编辑。 diff --git a/scripts/ql3-manual-primary-canary-audit.cjs b/scripts/ql3-manual-primary-canary-audit.cjs index 269bbd12..f61a1f20 100644 --- a/scripts/ql3-manual-primary-canary-audit.cjs +++ b/scripts/ql3-manual-primary-canary-audit.cjs @@ -18,6 +18,11 @@ const { const { parseRuntimeRolloutManifest, } = require('../back/runtime/domain/runtimeRolloutManifest'); +const { + MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE, + MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES, + parseManualPrimaryRuntimeReceipt, +} = require('../back/runtime/domain/manualPrimaryRuntimeReceipt'); const { readPrivateJson, serialized, @@ -27,6 +32,7 @@ const REQUIREMENTS = new Set([ 'prepared', 'qualified', 'selected', + 'active', 'off', 'rolled-back', ]); @@ -85,7 +91,44 @@ function regularFile(target) { } } -function run(options) { +function inspectRuntimeProcess(identity) { + if (identity.kind !== 'linux-proc' || process.platform !== 'linux') { + return 'unsupported'; + } + let bootId; + try { + bootId = fs.readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim(); + } catch (error) { + if (error?.code === 'ENOENT') return 'unsupported'; + throw error; + } + if (bootId !== identity.bootId) return 'identity_mismatch'; + let stat; + try { + stat = fs.readFileSync(`/proc/${identity.pid}/stat`, 'utf8'); + } catch (error) { + if (error?.code === 'ENOENT' || error?.code === 'ESRCH') return 'exited'; + throw error; + } + const close = stat.lastIndexOf(')'); + const fields = + close < 1 + ? [] + : stat + .slice(close + 1) + .trim() + .split(/\s+/u); + if ( + fields.length < 20 || + Number(fields[2]) !== identity.processGroupId || + fields[19] !== identity.startTimeTicks + ) { + return 'identity_mismatch'; + } + return ['Z', 'X', 'x'].includes(fields[0]) ? 'exited' : 'running'; +} + +function run(options, dependencies = {}) { const rootStat = fs.lstatSync(options.root); if ( !rootStat.isDirectory() || @@ -274,12 +317,55 @@ function run(options) { } rolloutMode = 'off'; } + let runtimeActivationObserved = false; + let runtimeActivationCurrent = false; + let runtimeReceiptState = 'missing'; + let runtimeProcessState = 'missing'; + const runtimeReceiptPath = path.join( + options.root, + MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE, + ); + if (regularFile(runtimeReceiptPath)) { + const receipt = parseManualPrimaryRuntimeReceipt( + readPrivateJson( + runtimeReceiptPath, + MAX_MANUAL_PRIMARY_RUNTIME_RECEIPT_BYTES, + ).value, + ); + runtimeReceiptState = receipt.state; + const selectionPath = resolve('selection'); + if (regularFile(selectionPath)) { + const selection = readPrivateJson(selectionPath, 64 * 1024).value; + runtimeActivationObserved = + receipt.revision === `manual-primary-${plan.sessionId}` && + receipt.profile === plan.profile && + selection?.schema === 'qinglong/manual-primary-canary-selection@v1' && + selection.sessionId === plan.sessionId && + selection.profile === plan.profile && + selection.manifestSha256 === receipt.rolloutSourceSha256; + } + if (receipt.state === 'active') { + runtimeProcessState = ( + dependencies.inspectRuntimeProcess ?? inspectRuntimeProcess + )(receipt.process); + runtimeActivationCurrent = runtimeProcessState === 'running'; + } + } const compatible = options.require === 'prepared' || (options.require === 'qualified' && eligible) || (options.require === 'selected' && rolloutMode === 'primary_selected') || - (options.require === 'off' && rolloutMode === 'off') || - (options.require === 'rolled-back' && rolloutMode === 'off' && rolledBack); + (options.require === 'active' && + rolloutMode === 'primary_selected' && + runtimeActivationObserved && + runtimeActivationCurrent) || + (options.require === 'off' && + rolloutMode === 'off' && + !runtimeActivationCurrent) || + (options.require === 'rolled-back' && + rolloutMode === 'off' && + rolledBack && + !runtimeActivationCurrent); const report = { schema: 'qinglong/manual-primary-canary-audit@v1', schemaVersion: 1, @@ -289,7 +375,10 @@ function run(options) { eligible, rolloutMode, approvalExpired, - runtimeActivationObserved: false, + runtimeActivationObserved, + runtimeActivationCurrent, + runtimeReceiptState, + runtimeProcessState, rolledBack, planSha256: planRead.sha256, ...(rolloutSha256 === undefined ? {} : { rolloutSha256 }), diff --git a/test/back/bootstrapDefaultManualPrimaryRuntime.test.cjs b/test/back/bootstrapDefaultManualPrimaryRuntime.test.cjs index 0b6610f5..ec434fbf 100644 --- a/test/back/bootstrapDefaultManualPrimaryRuntime.test.cjs +++ b/test/back/bootstrapDefaultManualPrimaryRuntime.test.cjs @@ -24,6 +24,8 @@ function loadResult(status, mode = 'off') { evaluatedAtMs: NOW, sourcePath: '/data/config/qinglong3-rollout.json', status, + revision: 'manual-primary-test', + sourceSha256: 'a'.repeat(64), }, ...(status === 'accepted' ? { @@ -150,6 +152,20 @@ test('accepted bootstrap lazily loads the stack and delegates activation', async calls.push('install'); return () => calls.push('dispose'); }, + receipt: { + async activated() { + calls.push('receipt:active'); + }, + async stopping() { + calls.push('receipt:stopping'); + }, + async stopped() { + calls.push('receipt:stopped'); + }, + async failed() { + calls.push('receipt:failed'); + }, + }, audit(record) { calls.push(`audit:${record.activation}`); }, @@ -168,11 +184,14 @@ test('accepted bootstrap lazily loads the stack and delegates activation', async 'start-timeout', 'start-cancellation', 'install', + 'receipt:active', 'audit:activated', + 'receipt:stopping', 'dispose', 'stop-timeout', 'stop-cancellation', 'stop-completion', + 'receipt:stopped', 'audit:stopped', ]); }); diff --git a/test/back/manualPrimaryRuntimeActivation.test.cjs b/test/back/manualPrimaryRuntimeActivation.test.cjs index a21a03f5..2df83e85 100644 --- a/test/back/manualPrimaryRuntimeActivation.test.cjs +++ b/test/back/manualPrimaryRuntimeActivation.test.cjs @@ -25,6 +25,7 @@ function loadResult(status, mode = 'off') { sourcePath: '/data/config/qinglong3-rollout.json', status, revision: 'canary-1', + sourceSha256: 'a'.repeat(64), }, }; } @@ -171,6 +172,138 @@ test('activation reconciles before starting lifecycle and installing ownership', ]); }); +test('activation publishes durable state around ownership and lifecycle shutdown', async () => { + const calls = []; + const result = await activateManualPrimaryRuntime({ + load: async () => loadResult('accepted', 'primary'), + create() { + return { + router: router(), + ...completionLifecycle(calls), + async reconcile() { + return cleanRecovery(); + }, + startTimeout() { + calls.push('start-timeout'); + return true; + }, + async stopTimeout() { + calls.push('stop-timeout'); + return 'drained'; + }, + startCancellation() { + calls.push('start-cancellation'); + return true; + }, + async stopCancellation() { + calls.push('stop-cancellation'); + return 'drained'; + }, + }; + }, + install() { + calls.push('install'); + return () => calls.push('dispose'); + }, + receipt: { + async activated() { + calls.push('receipt:active'); + }, + async stopping() { + calls.push('receipt:stopping'); + }, + async stopped() { + calls.push('receipt:stopped'); + }, + async failed() { + calls.push('receipt:failed'); + }, + }, + audit(record) { + calls.push(`audit:${record.activation}`); + }, + }); + + assert.deepEqual(calls.slice(-4), [ + 'start-cancellation', + 'install', + 'receipt:active', + 'audit:activated', + ]); + await result.stop(); + assert.deepEqual(calls.slice(-7), [ + 'receipt:stopping', + 'dispose', + 'stop-timeout', + 'stop-cancellation', + 'stop-completion', + 'receipt:stopped', + 'audit:stopped', + ]); +}); + +test('activation rolls ownership back when durable receipt publication fails', async () => { + const calls = []; + await assert.rejects( + activateManualPrimaryRuntime({ + load: async () => loadResult('accepted', 'primary'), + create() { + return { + router: router(), + ...completionLifecycle(calls), + async reconcile() { + return cleanRecovery(); + }, + startTimeout() { + calls.push('start-timeout'); + return true; + }, + async stopTimeout() { + calls.push('stop-timeout'); + return 'drained'; + }, + startCancellation() { + calls.push('start-cancellation'); + return true; + }, + async stopCancellation() { + calls.push('stop-cancellation'); + return 'drained'; + }, + }; + }, + install() { + calls.push('install'); + return () => calls.push('dispose'); + }, + receipt: { + async activated() { + calls.push('receipt:active'); + throw new Error('receipt unavailable'); + }, + async stopping() {}, + async stopped() {}, + async failed() { + calls.push('receipt:failed'); + }, + }, + audit(record) { + calls.push(`audit:${record.activation}`); + }, + }), + /receipt unavailable/, + ); + assert.deepEqual(calls.slice(-7), [ + 'receipt:active', + 'dispose', + 'stop-timeout', + 'stop-cancellation', + 'stop-completion', + 'receipt:failed', + 'audit:failed', + ]); +}); + test('activation rejects unresolved recovery before starting or installing', async () => { const calls = []; await assert.rejects( diff --git a/test/back/manualPrimaryRuntimeReceipt.test.cjs b/test/back/manualPrimaryRuntimeReceipt.test.cjs new file mode 100644 index 00000000..205c41ba --- /dev/null +++ b/test/back/manualPrimaryRuntimeReceipt.test.cjs @@ -0,0 +1,157 @@ +require('ts-node/register/transpile-only'); + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { test } = require('node:test'); +const { + MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE, + parseManualPrimaryRuntimeReceipt, +} = require('../../back/runtime/domain/manualPrimaryRuntimeReceipt'); +const { + ManualPrimaryRuntimeReceiptConflictError, + ManualPrimaryRuntimeReceiptStore, +} = require('../../back/runtime/adapters/fs/manualPrimaryRuntimeReceiptStore'); + +const IDENTITY = { + platform: 'linux', + bootId: '11111111-2222-3333-4444-555555555555', + pid: 321, + processGroupId: 320, + startTimeTicks: '123456', +}; + +function audit() { + return { + event: 'runtime.rollout_config_evaluated', + evaluatedAtMs: 1_000, + sourcePath: '/data/config/qinglong3-rollout.json', + sourceSha256: 'a'.repeat(64), + revision: 'manual-primary-edge-live-1', + status: 'accepted', + }; +} + +function fixture(t, inspection = 'exited') { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-runtime-receipt-')); + fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + let now = 10_000; + const options = { + clock: { now: () => now++ }, + platform: 'linux', + pid: IDENTITY.pid, + randomId: () => '1'.repeat(32), + identityProvider: { + async capture() { + return IDENTITY; + }, + async inspect() { + return { status: inspection }; + }, + }, + }; + return { + root, + store: new ManualPrimaryRuntimeReceiptStore(root, 'edge', options), + options, + }; +} + +test('publishes one private current receipt and transitions it around shutdown', async (t) => { + const { root, store } = fixture(t); + const target = path.join(root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE); + + await store.activated(audit()); + let receipt = parseManualPrimaryRuntimeReceipt( + JSON.parse(fs.readFileSync(target, 'utf8')), + ); + assert.equal(receipt.state, 'active'); + assert.equal(receipt.process.kind, 'linux-proc'); + assert.equal(fs.statSync(target).mode & 0o777, 0o600); + + await store.stopping(); + receipt = parseManualPrimaryRuntimeReceipt( + JSON.parse(fs.readFileSync(target, 'utf8')), + ); + assert.equal(receipt.state, 'stopping'); + + await store.stopped(); + receipt = parseManualPrimaryRuntimeReceipt( + JSON.parse(fs.readFileSync(target, 'utf8')), + ); + assert.equal(receipt.state, 'stopped'); + assert.equal(receipt.activationId, '1'.repeat(32)); +}); + +test('refuses to replace a receipt whose exact Linux process is still live', async (t) => { + const first = fixture(t); + await first.store.activated(audit()); + const second = new ManualPrimaryRuntimeReceiptStore(first.root, 'edge', { + ...first.options, + randomId: () => '2'.repeat(32), + identityProvider: { + ...first.options.identityProvider, + async inspect() { + return { status: 'running' }; + }, + }, + }); + await assert.rejects( + second.activated(audit()), + ManualPrimaryRuntimeReceiptConflictError, + ); +}); + +test('replaces a stale process generation and rejects receipt tampering', async (t) => { + const first = fixture(t); + await first.store.activated(audit()); + const second = new ManualPrimaryRuntimeReceiptStore(first.root, 'edge', { + ...first.options, + randomId: () => '2'.repeat(32), + }); + await second.activated(audit()); + const target = path.join(first.root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE); + const receipt = JSON.parse(fs.readFileSync(target, 'utf8')); + assert.equal(receipt.activationId, '2'.repeat(32)); + receipt.state = 'stopped'; + assert.throws( + () => parseManualPrimaryRuntimeReceipt(receipt), + /digest is invalid/, + ); +}); + +test('portable receipts remain observable but cannot claim Linux liveness', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-runtime-receipt-')); + fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const store = new ManualPrimaryRuntimeReceiptStore(root, 'standalone', { + clock: { now: () => 20_000 }, + platform: 'darwin', + pid: 432, + randomId: () => '3'.repeat(32), + identityProvider: { + async capture() { + return null; + }, + async inspect() { + return { status: 'unsupported' }; + }, + }, + }); + await store.activated(audit()); + const receipt = parseManualPrimaryRuntimeReceipt( + JSON.parse( + fs.readFileSync( + path.join(root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE), + 'utf8', + ), + ), + ); + assert.deepEqual(receipt.process, { + kind: 'portable', + platform: 'darwin', + pid: 432, + }); +}); diff --git a/test/back/ql3ManualPrimaryCanary.test.cjs b/test/back/ql3ManualPrimaryCanary.test.cjs index 40a5eae7..395646fc 100644 --- a/test/back/ql3ManualPrimaryCanary.test.cjs +++ b/test/back/ql3ManualPrimaryCanary.test.cjs @@ -8,6 +8,10 @@ const { afterEach, test } = require('node:test'); const { manualPrimaryCanaryFileSet, } = require('../../back/runtime/domain/manualPrimaryCanaryCeremony'); +const { + createManualPrimaryRuntimeReceipt, + MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE, +} = require('../../back/runtime/domain/manualPrimaryRuntimeReceipt'); const { parseArguments, readPrivateJson, @@ -273,6 +277,35 @@ test('qualifies, explicitly approves, audits and rolls back one target session', .runtimeActivationObserved, false, ); + const rolloutSha256 = readPrivateJson( + path.join(root, files.rollout), + 64 * 1024, + ).sha256; + writeJson( + path.join(root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE), + createManualPrimaryRuntimeReceipt({ + activationId: '1'.repeat(32), + profile: 'edge', + revision: `manual-primary-${SESSION}`, + rolloutSourceSha256: rolloutSha256, + activatedAtMs: activatedAt + 1, + process: { + kind: 'linux-proc', + platform: 'linux', + pid: 999_999, + processGroupId: 999_999, + bootId: '11111111-2222-3333-4444-555555555555', + startTimeTicks: '123456', + }, + }), + ); + const active = audit( + { root, sessionId: SESSION, require: 'active' }, + { inspectRuntimeProcess: () => 'running' }, + ); + assert.equal(active.runtimeActivationObserved, true); + assert.equal(active.runtimeActivationCurrent, true); + assert.equal(active.runtimeReceiptState, 'active'); assert.equal( run( { mode: 'status', root, sessionId: SESSION }, @@ -323,6 +356,40 @@ test('qualifies, explicitly approves, audits and rolls back one target session', ).publication, 'existing', ); + assert.throws( + () => + audit( + { root, sessionId: SESSION, require: 'rolled-back' }, + { inspectRuntimeProcess: () => 'running' }, + ), + /not satisfied/, + ); + writeJson( + path.join(root, MANUAL_PRIMARY_RUNTIME_RECEIPT_FILE), + createManualPrimaryRuntimeReceipt({ + activationId: '2'.repeat(32), + profile: 'edge', + revision: 'manual-primary-another-session', + rolloutSourceSha256: 'b'.repeat(64), + activatedAtMs: activatedAt + 3, + process: { + kind: 'linux-proc', + platform: 'linux', + pid: 999_998, + processGroupId: 999_998, + bootId: '11111111-2222-3333-4444-555555555555', + startTimeTicks: '123457', + }, + }), + ); + assert.throws( + () => + audit( + { root, sessionId: SESSION, require: 'rolled-back' }, + { inspectRuntimeProcess: () => 'running' }, + ), + /not satisfied/, + ); assert.equal( audit({ root, sessionId: SESSION, require: 'rolled-back' }).rolloutMode, 'off',