diff --git a/back/runtime/domain/manualPrimaryCanaryCeremony.ts b/back/runtime/domain/manualPrimaryCanaryCeremony.ts new file mode 100644 index 00000000..54479035 --- /dev/null +++ b/back/runtime/domain/manualPrimaryCanaryCeremony.ts @@ -0,0 +1,476 @@ +import { createHash } from 'crypto'; +import { + parseLegacyShadowPrimaryGateReceipt, + type LegacyShadowPrimaryGateReceipt, +} from './legacyShadowPrimaryGate'; +import { + parseRuntimeRolloutManifest, + REQUIRED_RUNTIME_ROLLOUT_GATES, + type DisabledRuntimeRolloutManifest, + type EnabledRuntimeRolloutManifest, +} from './runtimeRolloutManifest'; + +export const MANUAL_PRIMARY_CANARY_PLAN_SCHEMA = + 'qinglong/manual-primary-canary-plan@v1'; +export const MANUAL_PRIMARY_CANARY_QUALIFICATION_SCHEMA = + 'qinglong/manual-primary-canary-qualification@v1'; +export const MANUAL_PRIMARY_CANARY_MINIMUM_SETTLING_AGE_MS = 5 * 60_000; +export const MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS = 24 * 60 * 60 * 1_000; + +const SESSION_PATTERN = /^[a-z0-9](?:[a-z0-9-]{6,62}[a-z0-9])?$/u; +const SHA256_PATTERN = /^[a-f0-9]{64}$/u; + +export type ManualPrimaryCanaryProfile = 'edge' | 'standalone'; +export type ManualPrimaryCanaryCurrentRollout = + | { state: 'absent' } + | { state: 'disabled'; sha256: string }; + +export interface ManualPrimaryCanaryFileSet { + plan: string; + capture: string; + terminal: string; + resource: string; + primaryGate: string; + qualification: string; + selection: string; + previousRollout: string; + rollbackIntent: string; + rollbackComplete: string; + rollout: 'qinglong3-rollout.json'; +} + +export interface ManualPrimaryCanaryPlan { + schema: typeof MANUAL_PRIMARY_CANARY_PLAN_SCHEMA; + schemaVersion: 1; + sessionId: string; + profile: ManualPrimaryCanaryProfile; + origin: 'manual'; + createdAtMs: number; + admissionTarget: number; + minimumSettlingAgeMs: typeof MANUAL_PRIMARY_CANARY_MINIMUM_SETTLING_AGE_MS; + currentRollout: ManualPrimaryCanaryCurrentRollout; + files: ManualPrimaryCanaryFileSet; + activation: { + maxApprovalMs: typeof MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS; + defaultMode: 'off'; + allowLegacyFallbackBeforeStart: false; + }; +} + +export interface ManualPrimaryCanaryQualification { + schema: typeof MANUAL_PRIMARY_CANARY_QUALIFICATION_SCHEMA; + schemaVersion: 1; + sessionId: string; + profile: ManualPrimaryCanaryProfile; + origin: 'manual'; + qualifiedAtMs: number; + assessment: 'eligible'; + planSha256: string; + primaryGateFileSha256: string; + sourceFileSha256: { + capture: string; + terminal: string; + resource: string; + }; + sourceCanonicalSha256: { + capture: string; + terminal: string; + resource: string; + }; + window: { + startInclusiveMs: number; + endExclusiveMs: number; + }; + counts: { + admitted: number; + captured: number; + terminalScanned: number; + terminalMatched: number; + }; +} + +function record(value: unknown, name: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new TypeError(`${name} must be an object`); + } + return value as Record; +} + +function exactKeys( + value: Record, + expected: readonly string[], + name: string, +): void { + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + if ( + actual.length !== wanted.length || + actual.some((key, index) => key !== wanted[index]) + ) { + throw new TypeError(`${name} shape is invalid`); + } +} + +function safeInteger(value: unknown, name: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 0) { + throw new TypeError(`${name} must be a non-negative safe integer`); + } + return Number(value); +} + +function sha256(value: unknown, name: string): string { + if (typeof value !== 'string' || !SHA256_PATTERN.test(value)) { + throw new TypeError(`${name} must be a SHA-256 digest`); + } + return value; +} + +function sessionId(value: unknown): string { + if (typeof value !== 'string' || !SESSION_PATTERN.test(value)) { + throw new TypeError('Manual Primary canary session id is invalid'); + } + return value; +} + +function profile(value: unknown): ManualPrimaryCanaryProfile { + if (value !== 'edge' && value !== 'standalone') { + throw new TypeError('Manual Primary canary profile is invalid'); + } + return value; +} + +function admissionTarget( + selectedProfile: ManualPrimaryCanaryProfile, + value: unknown, +): number { + const target = safeInteger(value, 'admissionTarget'); + if ( + (selectedProfile === 'edge' && target !== 8) || + (selectedProfile === 'standalone' && (target < 32 || target > 128)) + ) { + throw new TypeError('Manual Primary canary admission target is invalid'); + } + return target; +} + +export function manualPrimaryCanaryFileSet( + rawSessionId: string, +): ManualPrimaryCanaryFileSet { + const id = sessionId(rawSessionId); + const prefix = `ql3-primary-canary-${id}`; + return Object.freeze({ + plan: `${prefix}.plan.json`, + capture: `${prefix}.capture.json`, + terminal: `${prefix}.terminal.json`, + resource: `${prefix}.resource.json`, + primaryGate: `${prefix}.primary-gate.json`, + qualification: `${prefix}.qualification.json`, + selection: `${prefix}.selection.json`, + previousRollout: `${prefix}.previous-rollout.json`, + rollbackIntent: `${prefix}.rollback-intent.json`, + rollbackComplete: `${prefix}.rollback-complete.json`, + rollout: 'qinglong3-rollout.json', + }); +} + +export function manualPrimaryCanarySha256(bytes: string | Buffer): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +export function createManualPrimaryCanaryPlan(input: { + sessionId: string; + profile: ManualPrimaryCanaryProfile; + createdAtMs: number; + admissionTarget: number; + currentRollout: ManualPrimaryCanaryCurrentRollout; +}): ManualPrimaryCanaryPlan { + const id = sessionId(input.sessionId); + const selectedProfile = profile(input.profile); + const createdAtMs = safeInteger(input.createdAtMs, 'createdAtMs'); + const target = admissionTarget(selectedProfile, input.admissionTarget); + const currentRollout = input.currentRollout; + if (currentRollout.state === 'disabled') { + sha256(currentRollout.sha256, 'currentRollout.sha256'); + } else if (currentRollout.state !== 'absent') { + throw new TypeError('Current rollout state is invalid'); + } + return Object.freeze({ + schema: MANUAL_PRIMARY_CANARY_PLAN_SCHEMA, + schemaVersion: 1, + sessionId: id, + profile: selectedProfile, + origin: 'manual', + createdAtMs, + admissionTarget: target, + minimumSettlingAgeMs: MANUAL_PRIMARY_CANARY_MINIMUM_SETTLING_AGE_MS, + currentRollout: { ...currentRollout }, + files: manualPrimaryCanaryFileSet(id), + activation: { + maxApprovalMs: MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS, + defaultMode: 'off' as const, + allowLegacyFallbackBeforeStart: false as const, + }, + }); +} + +export function parseManualPrimaryCanaryPlan( + value: unknown, +): ManualPrimaryCanaryPlan { + const plan = record(value, 'plan'); + exactKeys( + plan, + [ + 'schema', + 'schemaVersion', + 'sessionId', + 'profile', + 'origin', + 'createdAtMs', + 'admissionTarget', + 'minimumSettlingAgeMs', + 'currentRollout', + 'files', + 'activation', + ], + 'plan', + ); + if ( + plan.schema !== MANUAL_PRIMARY_CANARY_PLAN_SCHEMA || + plan.schemaVersion !== 1 || + plan.origin !== 'manual' || + plan.minimumSettlingAgeMs !== MANUAL_PRIMARY_CANARY_MINIMUM_SETTLING_AGE_MS + ) { + throw new TypeError('Manual Primary canary plan header is invalid'); + } + const selectedProfile = profile(plan.profile); + const id = sessionId(plan.sessionId); + const current = record(plan.currentRollout, 'plan.currentRollout'); + if (current.state === 'absent') { + exactKeys(current, ['state'], 'plan.currentRollout'); + } else if (current.state === 'disabled') { + exactKeys(current, ['state', 'sha256'], 'plan.currentRollout'); + sha256(current.sha256, 'plan.currentRollout.sha256'); + } else { + throw new TypeError('Manual Primary canary current rollout is invalid'); + } + const files = record(plan.files, 'plan.files'); + const expectedFiles = manualPrimaryCanaryFileSet(id); + exactKeys(files, Object.keys(expectedFiles), 'plan.files'); + for (const [key, expected] of Object.entries(expectedFiles)) { + if (files[key] !== expected) { + throw new TypeError(`plan.files.${key} is invalid`); + } + } + const activation = record(plan.activation, 'plan.activation'); + exactKeys( + activation, + ['maxApprovalMs', 'defaultMode', 'allowLegacyFallbackBeforeStart'], + 'plan.activation', + ); + if ( + activation.maxApprovalMs !== MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS || + activation.defaultMode !== 'off' || + activation.allowLegacyFallbackBeforeStart !== false + ) { + throw new TypeError('Manual Primary canary activation policy is invalid'); + } + return createManualPrimaryCanaryPlan({ + sessionId: id, + profile: selectedProfile, + createdAtMs: safeInteger(plan.createdAtMs, 'plan.createdAtMs'), + admissionTarget: admissionTarget(selectedProfile, plan.admissionTarget), + currentRollout: current as ManualPrimaryCanaryCurrentRollout, + }); +} + +export function createManualPrimaryCanaryQualification(input: { + plan: ManualPrimaryCanaryPlan; + planSha256: string; + primaryGate: LegacyShadowPrimaryGateReceipt; + primaryGateFileSha256: string; + sourceFileSha256: { + capture: string; + terminal: string; + resource: string; + }; + qualifiedAtMs: number; +}): ManualPrimaryCanaryQualification { + const plan = parseManualPrimaryCanaryPlan(input.plan); + const gate = parseLegacyShadowPrimaryGateReceipt(input.primaryGate); + const qualifiedAtMs = safeInteger(input.qualifiedAtMs, 'qualifiedAtMs'); + if ( + gate.assessment !== 'eligible' || + gate.profile !== plan.profile || + gate.origin !== plan.origin || + gate.counts.admitted !== plan.admissionTarget || + gate.counts.captured !== plan.admissionTarget || + gate.counts.terminalScanned !== plan.admissionTarget || + gate.counts.terminalMatched !== plan.admissionTarget || + gate.window.startInclusiveMs < plan.createdAtMs || + gate.window.endExclusiveMs > gate.generatedAtMs || + gate.generatedAtMs > qualifiedAtMs + ) { + throw new TypeError('Manual Primary canary gate does not match the plan'); + } + return Object.freeze({ + schema: MANUAL_PRIMARY_CANARY_QUALIFICATION_SCHEMA, + schemaVersion: 1, + sessionId: plan.sessionId, + profile: plan.profile, + origin: 'manual', + qualifiedAtMs, + assessment: 'eligible', + planSha256: sha256(input.planSha256, 'planSha256'), + primaryGateFileSha256: sha256( + input.primaryGateFileSha256, + 'primaryGateFileSha256', + ), + sourceFileSha256: { + capture: sha256(input.sourceFileSha256.capture, 'capture file digest'), + terminal: sha256(input.sourceFileSha256.terminal, 'terminal file digest'), + resource: sha256(input.sourceFileSha256.resource, 'resource file digest'), + }, + sourceCanonicalSha256: { + capture: gate.evidence.captureSha256, + terminal: gate.evidence.terminalSha256, + resource: gate.evidence.resourceSha256, + }, + window: { ...gate.window }, + counts: { ...gate.counts }, + }); +} + +export function parseManualPrimaryCanaryQualification( + value: unknown, +): ManualPrimaryCanaryQualification { + const qualification = record(value, 'qualification'); + exactKeys( + qualification, + [ + 'schema', + 'schemaVersion', + 'sessionId', + 'profile', + 'origin', + 'qualifiedAtMs', + 'assessment', + 'planSha256', + 'primaryGateFileSha256', + 'sourceFileSha256', + 'sourceCanonicalSha256', + 'window', + 'counts', + ], + 'qualification', + ); + if ( + qualification.schema !== MANUAL_PRIMARY_CANARY_QUALIFICATION_SCHEMA || + qualification.schemaVersion !== 1 || + qualification.origin !== 'manual' || + qualification.assessment !== 'eligible' + ) { + throw new TypeError('Manual Primary canary qualification is invalid'); + } + sessionId(qualification.sessionId); + profile(qualification.profile); + safeInteger(qualification.qualifiedAtMs, 'qualification.qualifiedAtMs'); + sha256(qualification.planSha256, 'qualification.planSha256'); + sha256( + qualification.primaryGateFileSha256, + 'qualification.primaryGateFileSha256', + ); + for (const name of ['sourceFileSha256', 'sourceCanonicalSha256'] as const) { + const digests = record(qualification[name], `qualification.${name}`); + exactKeys(digests, ['capture', 'terminal', 'resource'], name); + for (const key of ['capture', 'terminal', 'resource']) { + sha256(digests[key], `qualification.${name}.${key}`); + } + } + const window = record(qualification.window, 'qualification.window'); + exactKeys(window, ['startInclusiveMs', 'endExclusiveMs'], 'window'); + const start = safeInteger(window.startInclusiveMs, 'window.startInclusiveMs'); + const end = safeInteger(window.endExclusiveMs, 'window.endExclusiveMs'); + if (start >= end) throw new TypeError('Qualification window is invalid'); + const counts = record(qualification.counts, 'qualification.counts'); + exactKeys( + counts, + ['admitted', 'captured', 'terminalScanned', 'terminalMatched'], + 'counts', + ); + const normalizedCounts = Object.fromEntries( + Object.entries(counts).map(([key, count]) => [ + key, + safeInteger(count, `counts.${key}`), + ]), + ) as ManualPrimaryCanaryQualification['counts']; + if (new Set(Object.values(normalizedCounts)).size !== 1) { + throw new TypeError('Qualification counts do not agree'); + } + return qualification as unknown as ManualPrimaryCanaryQualification; +} + +export function createManualPrimaryCanaryEnabledManifest(input: { + plan: ManualPrimaryCanaryPlan; + qualification: ManualPrimaryCanaryQualification; + approvedBy: string; + approvedAtMs: number; + approvalMs: number; +}): EnabledRuntimeRolloutManifest { + const plan = parseManualPrimaryCanaryPlan(input.plan); + const qualification = parseManualPrimaryCanaryQualification( + input.qualification, + ); + const approvedAtMs = safeInteger(input.approvedAtMs, 'approvedAtMs'); + const approvalMs = safeInteger(input.approvalMs, 'approvalMs'); + if ( + qualification.sessionId !== plan.sessionId || + qualification.profile !== plan.profile || + qualification.qualifiedAtMs > approvedAtMs || + approvalMs < 60_000 || + approvalMs > MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS || + typeof input.approvedBy !== 'string' || + input.approvedBy.trim() !== input.approvedBy || + input.approvedBy.length < 3 || + input.approvedBy.length > 128 || + /[\u0000-\u001f\u007f]/u.test(input.approvedBy) + ) { + throw new TypeError('Manual Primary canary approval is invalid'); + } + const manifest: EnabledRuntimeRolloutManifest = { + schemaVersion: 2, + revision: `manual-primary-${plan.sessionId}`, + enabled: true, + approvedBy: input.approvedBy, + approvedAtMs, + expiresAtMs: approvedAtMs + approvalMs, + rollbackPlanRef: plan.files.plan, + primaryGate: { + schema: 'qinglong/legacy-shadow-primary-gate-reference@v1', + origin: 'manual', + receiptFile: plan.files.primaryGate, + receiptSha256: qualification.primaryGateFileSha256, + }, + rollout: { + defaultMode: 'off', + origins: { manual: 'primary' }, + allowLegacyFallbackBeforeStart: false, + }, + gates: Object.fromEntries( + REQUIRED_RUNTIME_ROLLOUT_GATES.map((gate) => [gate, 'passed']), + ) as EnabledRuntimeRolloutManifest['gates'], + }; + parseRuntimeRolloutManifest(manifest, approvedAtMs); + return Object.freeze(manifest); +} + +export function createManualPrimaryCanaryDisabledManifest( + rawSessionId: string, +): DisabledRuntimeRolloutManifest { + const manifest: DisabledRuntimeRolloutManifest = { + schemaVersion: 2, + revision: `manual-primary-${sessionId(rawSessionId)}-rollback`, + enabled: false, + }; + parseRuntimeRolloutManifest(manifest, 0); + return Object.freeze(manifest); +} diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index 8e97dbda..33a96e46 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,22 @@ 最新增量证据(2026-08-19): +- D-361/ADR-0454(已接受;首次真实用户目标实例执行待运维):把 D-360 的 manual Primary evidence/gate 收敛为一次显式、可重放且不新增常驻组件的目标实例仪式。 + `prepare → observe → resource → qualify → approve → status/audit → rollback` 固定为七个 operator-driven 阶段;Edge 精确 8 条,Standalone 由维护者在 + 32–128 条内选择精确目标。prepare 绑定当时 absent/disabled rollout 基线并只输出 `QL_DEPLOYMENT_PROFILE`、`QL3_SHADOW_ORIGINS=manual` 和唯一 + capture basename,不触发任务、重启或自动启用;observe 固定复用五分钟闭合窗口只读审计,resource 固定执行 compiled-backend full rollback。qualification 同时绑定 + plan、三份原文件 SHA-256、canonical evidence SHA-256 与 Primary gate;approve 是唯一写 live manifest 的动作,显式审批最长 24 小时,并在最终写入前复核基线摘要。 + 写入结果只称 `activation_approved`/`primary_selected`,强制返回 `requiresRestart=true`、`runtimeActivationObserved=false`,不能冒充现有 bootstrap 已完成 + `selected → reconciled → activated`。rollback 以 intent→摘要复核→disabled 原子替换→completion 链支持 response-loss 重放;审批过期立即按 off 解释,但仍要求显式 + rollback 收敛磁盘事实。config root/证据要求当前 UID、非 symlink、group/world 不可写及 `0600` no-replace;路由设备不增加 daemon、watcher、timer、连接、package、 + 生产依赖、schema、migration 或部署对象。真实 compiled backend 已完成一次 Edge `prepare → resource`,只证明仪式固定 child/资源回滚入口可执行,不冒充真实产品入口的 + 8 条 manual admission、运行态 activation、物理路由、flash 或断电证据。阶段门已重跑:聚焦 `24/24`、`build:back`、完整 backend + `1,481 pass / 0 fail / 2 conditional skip`、18-package clean build/test、四项可执行架构审计与 `14/14` artifact audit 全部通过;14 档产物字节与 + D-360 完全一致。使用隔离 frozen-lockfile Linux 依赖重新执行 arm64 Docker 门后,router stress 保持 `128 MiB / 0.5 CPU / 0 swap / 64 PID`,cgroup peak + `81,887,232` bytes;Edge release 保持 `256 MiB / 1 CPU / 0 swap / 128 PID`,13 个 workload 全绿、peak `148,840,448` bytes,两档 + `memory.events max/oom/oom_kill` 增量均为 0。D-361 不改 PostgreSQL schema/migration、依赖树或 Kubernetes 拓扑,故不重跑 PostgreSQL HA;相邻 + D-359 的 `142/142` 与 timeline `1→2` 仅作为既有证据。 + - D-360/ADR-0453(已接受;首次真实目标实例 manual canary bundle 待执行):不再用 2.x RunningInstance 推测 Legacy→Shadow 分母,也不再让 Primary manifest 只信任维护者填写的 `"passed"`。默认 Legacy bridge 为每个已启用 origin admission 分配 process-epoch token,固定守恒 `captured + failed + pending = admitted`,失败只分 `fact/observer/initialization/accept`;只有真实 Shadow writer accept 才结算 captured,Legacy @@ -8307,13 +8323,19 @@ Primary 不使用普通环境变量或宽泛全局开关启用。孵化配置面 - 文件缺失、不可读、超过 64 KiB、JSON 损坏、未知字段、过期审批或 gate 不完整时 fail-closed 为 `off`。 - `defaultMode` 必须保持 `off`;当前 manifest 只允许声明 `manual`,不得用一个配置隐式接管 boot、定时、gRPC 或其他来源。 -- 启用记录必须包含有界 revision、审批人、审批起止时间和 rollback plan 引用;审批窗口最长 30 天。 +- 启用记录必须包含有界 revision、审批人、审批起止时间和 rollback plan 引用;通用 loader 上限为 30 天,ADR-0454 的目标实例 manual canary 仪式进一步收紧为最长 24 小时。 - `durableCancellation`、`startupReconciliation`、`atomicLegacyProjection`、`rollbackDrill`、`edgeBudget` 必须全部为 `passed`;从 schema v2 起这些声明不能 代替 Primary evidence bundle。 - enabled manifest 必须绑定同 config 目录的 `qinglong/legacy-shadow-primary-gate@v1` bundle basename 与 SHA-256。loader 必须 no-follow 读取、重算 embedded capture/startup、terminal、resource source digest 和完整 eligibility,并复验 manual origin、实际 Profile 与审批时序;只验证 receipt 自称 eligible 不成立。 - 审计只记录路径、revision、判定、时间和源文件 SHA-256,不记录完整配置内容;接受判定必须在安装 owner router 前可观测,安装后审计失败必须撤销 router。 - edge 不启动文件 watcher。配置只在显式 bootstrap/reload 时读取,禁用时不创建 router、timer、连接或后台任务。 +- ADR-0454 的 `prepare` 先以不可变 plan 绑定 Profile、精确 admission 目标和当时 absent/disabled rollout SHA-256;`qualify` 再绑定 capture/terminal/resource 的原始文件摘要、 + canonical 摘要和 Primary gate。只有显式 `approve` 能在复验 live 基线后写入本节 schema v2 manifest。 +- `approve` 的 durable 结果是 `primary_selected`,不是 `primary_active`。operator 必须重启目标 worker,并从现有 bootstrap 的同 revision `selected → reconciled → activated` + 结构化审计证明实际运行态;一次性独立 auditor 刻意不输出 active 结论。 +- `rollback` 在 live manifest 替换前先写绑定 enabled/disabled 摘要的 intent,替换后再写绑定 intent 摘要的 completion;同参数重放覆盖两个 response-loss 窗口。审批过期按 off + 解释但不会自动改写文件,仍需显式 rollback 和重启。 实验 manifest 结构固定为: @@ -8352,8 +8374,8 @@ Primary 不使用普通环境变量或宽泛全局开关启用。孵化配置面 的 manual primary 配置才惰性加载真实组件。激活顺序固定为:重放 evidence→复验实际 Profile→记录 selected 审计→完整有界 startup reconciliation→记录 reconciled 审计→启动 completion/timeout/cancel lifecycle→安装 router→记录 activated;任何一步失败都会撤销 router,并按 producer → consumer 顺序停止 lifecycle。HTTP shutdown 与监听失败也会执行有界清理。`QL_DEPLOYMENT_PROFILE` 未配置时为 standalone,非法值、receipt Profile 不一致或在 -cluster-control/worker 中误装本机 SQLite Primary 时 fail closed。首次目标实例 manual canary、固定物理 edge Gate、配置写入/用户可见状态和操作回滚演练完成后 -仍需单独评审。 +cluster-control/worker 中误装本机 SQLite Primary 时 fail closed。目标实例仪式、部署配置写入、只读选择状态和操作回滚工具已由 ADR-0454 固化;首次真实 Edge/Standalone +完整执行、bootstrap activated durable receipt、固定物理 edge/flash/断电 Gate 完成后仍需单独评审。 ## 26. 交付阶段 @@ -8995,6 +9017,7 @@ flowchart LR - 旧 Crontab 状态仍为用户可见事实源。 - 增加旧状态与 Shadow Run 对账日志和指标。 - 发现差异时只报告,不改变任务结果。 +- 目标实例 canary 必须由不可变计划固定 Profile 和精确样本数,并通过真实产品 manual 入口执行;仪式不得代替用户发起任务或自动启用 Primary。 #### PR-5:LocalExecutor 切换 @@ -9006,6 +9029,7 @@ flowchart LR - 取消必须先原子写入 `run.cancel_requested`,再调用 Executor.stop;重复取消保持单事件,晚到 success 不覆盖已接受取消。 - 验证 task_before、task_after、work_dir、log_name 和多实例行为。 - Primary completion 只能写安全错误分类,不得把 Executor 原始错误、命令、环境或路径持久化到 error_summary。 +- rollout 配置选择与运行态激活必须分离:短期审批只产生 `primary_selected`,重启后的 reconciliation/lifecycle/router 审计才可证明 activated;回滚必须保留可重放 intent/completion 链。 #### PR-6:只读 v3 Run API @@ -9176,8 +9200,8 @@ flowchart LR | PR-1 Run Schema | Incubating | Run/RunAttempt/RunEvent schema、nullable cancel request 与 Attempt deadline 字段及恢复索引、CancellationDispatch 状态/version/lease/backoff schema、Repository port、临时 Sequelize adapter、统一事件大小/分页上限、跨 adapter RunRepository contract suite(原子事务、回滚、Run/Attempt/RetryPolicy CAS、唯一错误、分页与取消恢复);ADR-0041 的 `pg-0003-run-retry-policy`、capability v2、driver-neutral PostgreSQL Run Repository 与真实 `pg.Pool` 上的共享 Repository/rollback/SQLSTATE contract;ADR-0063/0069/0071/0073/0074/0076 的独立 Node 24 local-sqlite typed schema、十二条 reviewed migration、capability v6、共享 operation authority、readiness/RunRepository/API credential repository/receipt journal/dispatch plan/encrypted Secret envelope/Project Policy/security audit/authorized mutation/stable Identity catalog、Drizzle↔真实 catalog table/column/index/CHECK/FK lockstep、base/adopted/application edge/standalone 产物门禁;ADR-0064 的 legacy baseline/plan digest、Online Backup recovery、side-by-side target migration、staged manifest、双库栅栏 activation、source 生命周期写栅栏、target stable identity 和重启语义;ADR-0065 的独立 cutover authority、外部副作用停机 evidence、append-only journal、start/restart/stop barrier 与 unknown→manual_required 收敛;ADR-0066 的 adopted storage→Run reconciliation→receipt maintenance→domain recovery→lifecycle→admission application gate、严格有界 recovery summary 与 admission-first reverse stop;ADR-0067 的 SQLite 事实驱动 Run 候选源、256 条硬上限、截断失败关闭和唯一 Repository authority;ADR-0068 的 receipt-first Reconciler、callback token/sequence fence、exact local-process identity、Attempt/Run/双 Event 原子终态推进和最终 verifier;ADR-0069 的 local-process 单向包边界、pre-spawn journal、受审 POSIX launcher、immutable receipt、exact identity 和 Profile-aware cleanup lifecycle;ADR-0070 的独立 local-execution、spawn 前后双 transaction CAS、callback digest、exact stop 补偿与 fail-closed starting 保留;ADR-0071 的独立 local-dispatch、不可变 revision/context、Secret-first materializer、Profile Artifact admission、4/64 MiB output hard quota 和窄 application facade;ADR-0073/0074 的 Project-bound SecretRef、AES-256-GCM、外置 keyring 生命周期、双 SQLite authority CAS、application preflight、强 Principal/Policy 和 envelope+audit 原子提交;ADR-0086 的本机 Owner provisioning/challenge/claim/delivery acknowledgement/credential recovery CLI;ADR-0377 的 Local/Cluster 同构、Profile-aware、Project-scoped Artifact range read | fresh database/pepper setup、credential rotation/GC 运维编排与 Secret/Project/Role/Approval 管理 CLI/API/UI、备份/rekey、2.x/target process controller、人工 recovery、target 写后 reconciliation 与完整 cutover/rollback 演练;retry 产品策略、Artifact retention/tombstone stack、具体本机 lifecycle 和 target executable;Linux x64/arm64、PID namespace、断电与固定路由设备门禁;PostgreSQL 16/18 双连接并发与 failover integration;Task revision/context 跨方言 contract/并发压力与引用感知 retention、Keyv 数据迁移 | | PR-2 Run 状态机 | Incubating | 纯转换表、终态/时间/错误/执行器元数据规则、Run version 与 event sequence CAS、事务性 RunCommandService、回滚测试 | 重复 Worker callback/fencing、并发数据库压力测试、Primary 执行链接入 | | PR-3 Executor 端口 | Incubating | ADR-0003、ExecutionSpec/Context/Handle/Result、Executor port、LocalProcessExecutor、进程组取消/超时升级、流式背压、Legacy Cron spec builder、真实进程 contract tests、可复现 edge 基准入口 | 固定 edge/多架构设备基线、Legacy builder 与 makeCommand 差异审计、Primary 生产流量接入 | -| PR-4 Shadow Run | Incubating | origin 三态策略;默认关闭的 `QL3_SHADOW_ORIGINS`;manual、scheduled_node、boot、subscription、system 与 script 现有 ChildProcess 旁路观察;system crond 显式 origin marker、Shell execution ID、finish-only 准入、确定性 Run/Attempt 与 exact replay;`@once` 保持 manual、gRPC transport 不冒充 origin 的准入裁决;每个 worker 懒加载;Run/Attempt/Event 影子生命周期;稳定且不复制 caller 原文的 task identity/revision 与有界日志引用;同 worker 有界注册表和跨 worker 持久化候选关联;stop all/stop instance、Shell callback、乱序/迟到/歧义处理;监听前一次性、Profile-aware 的 keyset Startup Reconciler,终态证据补齐、lost/abandoned/pending 分流与 terminal Attempt response-loss 修复;origin-bounded 且逐级守恒的版本化 startup difference report、固定字段 metric batch 与一次性 collector;显式、只读、闭合窗口且 Profile-bounded 的 Shadow→Legacy 终态差异审计;128/256 MiB Linux arm64 资源门、SQLite 零增长与 Shadow enabled→off 进程重启回滚;process-epoch Legacy admission/capture/failure/pending 守恒;clean-shutdown `0600` no-replace capture+startup exporter;manual Edge 8/Standalone 32–128 canary;capture/terminal/resource 自包含 Primary bundle;rollout v2 loader 重算 source digest 与 eligibility;失败开放和契约测试 | 首次真实目标实例 manual canary bundle、其他 origin 独立 capture/Primary gate、固定物理 edge/flash/断电证据 | -| PR-5 Primary LocalExecutor | Incubating(默认不激活,仅 manifest-gated manual) | runtime-owned Run 创建器;持久化先于 spawn;Run/Attempt 完整成功、失败、取消、超时与 lost 闭环;Executor handle 身份校验;spawn 后激活写失败的 stop+lost 补偿;completion rejection 安全收敛;独立 Primary 幂等查询与唯一索引竞态裁决;durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询;最多 64 条一页的 cross-worker cancellation source;独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent;最多 64 页的单周期 cancel supervisor;显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner;Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller;完整有界分页且 fail-closed 的 startup Reconcile supervisor;RunningInstance nullable `run_id/attempt_id` 关联;Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance,失败整体回滚;有界且防穿越的 legacy log output ref;manual owner seam、真实本机装配、单 spawn/fail-closed;严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计;HTTP worker 已接轻量 lazy bootstrap,accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router 顺序激活,失败撤销,监听失败和 shutdown 有界停止;Primary timeout 在 spawn 前持久化绝对 deadline,有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch;代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限,cluster-control/worker 拒绝误装本机 SQLite Primary;统一 CompletionService 原子提交 Attempt/Run/双 Event,spawn 前保存 callback token hash、终态推进 sequence,实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window;manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理;Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace;`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描,使终态残留继续可发现;确定无效的已知 Attempt receipt 先持久化隔离状态,再进入确定性私有分片 quarantine;终态 missing 与 quarantine 按 edge/standalone retention 有界清理;非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI;扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数;ENOSPC 与 launcher receipt 存储失败有代码门禁 | 部署配置写入/审批入口与用户可见状态;PostgreSQL CancellationDispatch adapter;cluster-control 生产启动拓扑;固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练 | +| PR-4 Shadow Run | Incubating | origin 三态策略;默认关闭的 `QL3_SHADOW_ORIGINS`;manual、scheduled_node、boot、subscription、system 与 script 现有 ChildProcess 旁路观察;system crond 显式 origin marker、Shell execution ID、finish-only 准入、确定性 Run/Attempt 与 exact replay;`@once` 保持 manual、gRPC transport 不冒充 origin 的准入裁决;每个 worker 懒加载;Run/Attempt/Event 影子生命周期;稳定且不复制 caller 原文的 task identity/revision 与有界日志引用;同 worker 有界注册表和跨 worker 持久化候选关联;stop all/stop instance、Shell callback、乱序/迟到/歧义处理;监听前一次性、Profile-aware 的 keyset Startup Reconciler,终态证据补齐、lost/abandoned/pending 分流与 terminal Attempt response-loss 修复;origin-bounded 且逐级守恒的版本化 startup difference report、固定字段 metric batch 与一次性 collector;显式、只读、闭合窗口且 Profile-bounded 的 Shadow→Legacy 终态差异审计;128/256 MiB Linux arm64 资源门、SQLite 零增长与 Shadow enabled→off 进程重启回滚;process-epoch Legacy admission/capture/failure/pending 守恒;clean-shutdown `0600` no-replace capture+startup exporter;manual Edge 8/Standalone 32–128 canary;capture/terminal/resource 自包含 Primary bundle;rollout v2 loader 重算 source digest 与 eligibility;不可变 prepare/observe/resource/qualify 目标实例仪式、独立只读 audit;失败开放和契约测试 | 首次真实目标实例完整 canary 与 bootstrap activated 记录、其他 origin 独立 capture/Primary gate、固定物理 edge/flash/断电证据 | +| PR-5 Primary LocalExecutor | Incubating(默认不激活,仅 manifest-gated manual) | runtime-owned Run 创建器;持久化先于 spawn;Run/Attempt 完整成功、失败、取消、超时与 lost 闭环;Executor handle 身份校验;spawn 后激活写失败的 stop+lost 补偿;completion rejection 安全收敛;独立 Primary 幂等查询与唯一索引竞态裁决;durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询;最多 64 条一页的 cross-worker cancellation source;独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent;最多 64 页的单周期 cancel supervisor;显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner;Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller;完整有界分页且 fail-closed 的 startup Reconcile supervisor;RunningInstance nullable `run_id/attempt_id` 关联;Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance,失败整体回滚;有界且防穿越的 legacy log output ref;manual owner seam、真实本机装配、单 spawn/fail-closed;严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计;HTTP worker 已接轻量 lazy bootstrap,accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router 顺序激活,失败撤销,监听失败和 shutdown 有界停止;Primary timeout 在 spawn 前持久化绝对 deadline,有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch;代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限,cluster-control/worker 拒绝误装本机 SQLite Primary;统一 CompletionService 原子提交 Attempt/Run/双 Event,spawn 前保存 callback token hash、终态推进 sequence,实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window;manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理;Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace;`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描,使终态残留继续可发现;确定无效的已知 Attempt receipt 先持久化隔离状态,再进入确定性私有分片 quarantine;终态 missing 与 quarantine 按 edge/standalone retention 有界清理;非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI;扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数;ENOSPC 与 launcher receipt 存储失败有代码门禁;显式最长 24 小时 approve 写入、`primary_selected` 只读状态、selection receipt、approval-expiry off 与 intent/completion crash-replay rollback | 首次真实目标实例激活/回滚及 durable runtime activation receipt;PostgreSQL CancellationDispatch adapter;cluster-control 生产启动拓扑;固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练 | | PR-7 Worker Session、Run Lease 与启动协议基础 | Incubating(默认关闭,独立入口显式 opt-in) | ADR-0012/0013/0014/0021/0057–0061/0108–0121/0231–0239/0377;有界 capability/Placement/Dispatcher;SQLite 协议孵化与 PostgreSQL v9 Session/Run Lease/credential/attestation authority;immutable revision Placement、数据库时钟 keyset candidate、认证 Worker Pull、digest-only offer recovery;versioned capability-free ExecutionSpec response、stable claim 跨重启退避、单 owner 原子 inbox 准入与 TLS 1.3 mTLS/`ql3w` HTTPS client;同一 package journal 上 revision-fenced starting/spawn/started/running/completion 状态、callback digest、tagged no-spawn 与 ambiguous recovery;PostgreSQL starting/running/start-failure/completion 数据库权威事务、精确重放与 cancellation/timeout 优先终态;batch Secret delivery 在 Attempt advisory lock 下复验 Session/Lease/revision 完整围栏并复用单 Agent,Secret-before-Artifact materializer 将同一 log ID 交给 Executor/journal/running ACK;offer-scoped `wlog-*` 私有文件 spool、Edge/Node 容量策略、append/quota/path 防护、barrier 后 output ownership、受审 POSIX Executor、truncation fact、固定内存流式 source、认证 Artifact stream、共享 immutable store port、S3-compatible SSE/checksum/条件 promotion adapter、upload-before-completion 协调,以及 Local/Cluster 同构、Profile-aware、ETag-fenced range read;用户取消 run.stop mutation 以数据库时间写 intent/Event 并在事务内复验 Project/RoleBinding fence;非执行取消 convergence lifecycle、运行期 expiry 与安全 lost retry 已接入 cluster-control 单一全局 cadence;完整 generation/version/token/Attempt fencing;独立最小权限 Worker ingress、CA/CRL 与连接 generation 热重载;offer journal、spawn barrier、receipt-first recovery;独立 `@qinglong/worker-runtime` 的本地 P-256 CSR、key/chain/trust 验证、generation + active pointer 安装和持久退避;默认关闭的 production process 已装配具体 execution graph、完整 Session heartbeat/drain/offline、direct-file bootstrap、单 Agent/单 cadence、startup reconciliation、证书 maintenance、transport fail-close/recovery 与 Edge/Node 有界预算;真实 PostgreSQL 18 + Linux Node 合约已覆盖 Run completion、credential 和 CA 双轮换且保持同一 Session;真实 K3s 合约已覆盖 TLS/credential Secret 分权、双对象 CAS、Recreate 顺序、identity generation 与单节点 PVC recovery;所有能力默认不可达且受 edge/cluster import audit 约束 | 具体 cert-manager/Vault/SPIFFE/离线 CA adapter 与模板、ingress reload controller、生产 RBAC、证书到期告警和 `ql3w` credential recovery 产品面;具体 KMS/Vault Secret provider、对象存储 credential/temporary lifecycle 与 retention/tombstone;Worker 管理 API;真实 Kubernetes 多节点 CSI/node-loss/production 360 秒 drain 与固定 edge 文件系统 suspend/时钟/断电、x64/arm64 资源门禁 | | PR-8 Project/Policy/Approval Core | Incubating(默认拒绝、无生产业务执行入口) | ADR-0028;统一六类 ActorRef 与 exact-shape 校验;`0017` ownerless default Project 和 append-only versioned RoleBinding;owner/admin/operator/viewer 固定矩阵;Project 内 mutation 幂等、expected-version CAS、双 SQLite 连接竞争门禁;archived read-only、revocation、存储损坏 fail-closed;Agent 写/Secret/Tool `require_approval`;ADR-0047 把六类 subject、role/permission matrix 与 fence 抽到 runtime-core,`pg-0004-project-policy`/capability v3 建立 ownerless PostgreSQL baseline、严格 role/state CHECK、append-only runtime 权限、SERIALIZABLE Project lock、mutation replay、双连接单 winner 和 cluster admission authorizer;ADR-0049/`pg-0005` capability v4 建立 stable IdentitySubject、append-only digest-only API credential、真实 cluster bearer authenticator、write-only durable security audit 与最小权限 runtime role,且已验证 HTTP→credential→Policy→audit→handler 纵向链路;ADR-0051 建立 `/api/v3` 认证前 peer/global 双预算、transport-peer-only、无 timer 且有界内存的 overload shield;ADR-0027 Artifact authorizer adapter;ADR-0029 `AuthenticatedPrincipal` contract、`0018` digest-only versioned challenge、CSPRNG/TTL、同事务消费 challenge + 写首 owner、精确重放与双连接竞争/崩溃回滚门禁;ADR-0030 `0019` stable identity/binding、legacy HS384 + current-session membership、logout/platform/revoke/disable、single-factor 与损坏 fail-closed 门禁;ADR-0031 `0020` digest-bound ApprovalRequest、User-only decision、Project/Role version fence、精确 expiry/重放/并发裁决及同事务 immutable dispatch;ADR-0032 `0021` execution backfill、三表原子 consume、稳定 due keyset、claim/renew/start/result fencing、pre-start takeover/post-start recovery-required、attempt budget、handler inspect/digest barrier 和 bounded dispatcher;ADR-0033/`0022` control/resolution backfill、start/renew/completion 原子联动、稳定 recovery keyset、双 resolver claim/takeover、finding/result 精确重放、自动/人工终结、迟到 completion 单 winner 和 evidence-only bounded reconciler;ADR-0034/`0023` 首个 `run.create` canonical plan、Run/Attempt/Event/receipt 同事务、幂等 collision fail-closed、renew/终态 fence、真实 SQLite handler 与 automatic evidence provider;ADR-0035/`0024` 独立 `approval.recover` 矩阵、稳定 User + 五分钟强认证、Project/RoleBinding fence、human resolution + authorization fact 原子提交、撤权竞态与回滚门禁;ADR-0036 recovery-first 单 timer lifecycle、edge/standalone 独立 cadence/页预算、跨周期 cursor、非重叠与有界 stop;ADR-0074 以新的 Node 24 SQLite v5 ownerless Project/RoleBinding/audit authority 和独立 local-secret-admin 提供强 Principal、`secret.manage`、撤权 fence、envelope+allowed audit 原子提交及不回显语义;ADR-0086 以可信 POSIX console 和 staged delivery 完成本机首 Owner 产品 ceremony | fresh database/pepper setup 与安全迁移向导;`shareStore`/Express 到 authentication core 的 production migration;credential rotation/revocation API、mTLS/Worker enrollment、恢复码;Project/Role/Approval/Secret 管理 CLI/API/UI、audit retention/query/export/alert、preview Artifact/digest/immutable plan builder、真实 MFA/hardware adapter、人工 recovery API/UI/独立 rate limit 与审计事件、handler/provider registry、lifecycle startup/shutdown/指标/admission gate;PostgreSQL action/receipt/provider/recovery-authorization 与 OPA adapter、缓存 version 失效;Tool/Package/Secret/Shell 各自的 handler/evidence contract;Secret/Run/Tool/Workflow waiting_approval 全入口装配;完整回滚演练 | diff --git a/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md b/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md index 4a508259..9c13a4b7 100644 --- a/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md +++ b/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md @@ -170,10 +170,13 @@ owner 在接受触发时写入执行上下文,并贯穿日志、指标和回 - `defaultMode` 固定为 off,当前只允许 `origins.manual`;Primary 不得通过环境变量、通配 origin 或隐式默认值开启。 - enabled manifest 必须记录 revision、approvedBy、approvedAtMs、expiresAtMs 和 rollbackPlanRef;审批窗口最长 30 天。 - `durableCancellation`、`startupReconciliation`、`atomicLegacyProjection`、`rollbackDrill`、`edgeBudget` 必须全部为 passed。 +- ADR-0453 后 enabled manifest 还必须绑定可由 loader 独立重算的 manual capture/terminal/resource Primary gate bundle;ADR-0454 的一次性目标实例仪式进一步绑定 + exact Profile/admission 计划、原始文件摘要、短期审批、selection receipt 与 rollback intent/completion。配置选择只表示 `primary_selected`,不能冒充当前 worker + 已经完成 `selected → reconciled → activated`。 - 审计只包含 source path/hash、revision、时间和稳定判定。接受审计先于 router 安装;安装后审计失败立即调用 disposer,恢复原 owner。 - edge 不启动 watcher;当前 bootstrap 只在 HTTP worker 启动时读取一次,未来显式 reload 必须复用同一校验和审计边界。 -bootstrap 接入不代表 Primary 已默认开放。文件缺失、disabled、rejected 或 `manual` 非 primary 时保持 Legacy,且不加载完整 Runtime stack、不创建 router 或 timer。只有 accepted 且全部 gate 通过的 manifest 才按 startup reconciliation、cancel lifecycle、router 的顺序激活;任一步失败都会撤销 router 并停止 lifecycle。ADR-0007 定义的 completion/log supervisor、固定 edge 基准、部署配置写入与用户可见状态、操作回滚演练仍需再次评审。 +bootstrap 接入不代表 Primary 已默认开放。文件缺失、disabled、rejected 或 `manual` 非 primary 时保持 Legacy,且不加载完整 Runtime stack、不创建 router 或 timer。只有 accepted 且全部 gate 通过的 manifest 才按 startup reconciliation、cancel lifecycle、router 的顺序激活;任一步失败都会撤销 router 并停止 lifecycle。ADR-0454 已提供部署配置写入、只读选择状态与操作回滚仪式,但首次真实目标实例执行、运行态 durable activation receipt、固定 edge 基准和 ADR-0007 的完整实机恢复仍需继续评审。 ## 8. Legacy 到 Run 的身份映射 diff --git a/docs/adr/ADR-0453-origin-scoped-legacy-shadow-capture-authority-and-primary-gate.md b/docs/adr/ADR-0453-origin-scoped-legacy-shadow-capture-authority-and-primary-gate.md index cc91c4d9..5af18e43 100644 --- a/docs/adr/ADR-0453-origin-scoped-legacy-shadow-capture-authority-and-primary-gate.md +++ b/docs/adr/ADR-0453-origin-scoped-legacy-shadow-capture-authority-and-primary-gate.md @@ -95,6 +95,7 @@ fail-closed 还会改变 Legacy 可用性。 ## 后续 -正式启用 manual Primary 前,维护者必须在目标 edge/standalone 实例完成一次真实、干净关闭的 manual canary,等待 settling 后运行 terminal audit,组合与同版本 -resource report 生成 bundle,再由 v2 loader 重放。固定物理路由、flash 写放大、断电、非干净退出和 config-root 签名/备份仍是独立发布证据;其他 origin 必须分别建立 -自己的 admission authority、样本预算和 rollback gate,不能复用 manual receipt。 +ADR-0454 已把目标实例的 prepare、observe、resource、qualify、显式短期 approve、只读 audit 与 crash-replay rollback 固化为一次性状态机,并明确 +`primary_selected` 不等于运行态 activated。正式启用 manual Primary 前,维护者仍必须在目标 edge/standalone 实例实际执行该仪式并保留 bootstrap activated audit; +仓库内 synthetic fixture 或 compiled resource child 不能替代真实产品入口的八/三十二条 admission。固定物理路由、flash 写放大、断电、非干净退出和 config-root +签名/备份仍是独立发布证据;其他 origin 必须分别建立自己的 admission authority、样本预算和 rollback gate,不能复用 manual receipt。 diff --git a/docs/adr/ADR-0454-target-instance-manual-primary-canary-ceremony.md b/docs/adr/ADR-0454-target-instance-manual-primary-canary-ceremony.md new file mode 100644 index 00000000..5a0daafd --- /dev/null +++ b/docs/adr/ADR-0454-target-instance-manual-primary-canary-ceremony.md @@ -0,0 +1,90 @@ +# ADR-0454:目标实例 Manual Primary Canary 与显式回滚仪式 + +- 状态:Accepted(仪式实现完成;首次真实用户目标实例执行仍待运维) +- 日期:2026-08-19 +- 关联 RFC:QL-RFC-0001 D-361、PR-4、PR-5 +- 关联 ADR:ADR-0002、ADR-0449、ADR-0450、ADR-0451、ADR-0453 +- Amends:ADR-0453 的首次目标实例 canary 操作缺口 + +## 上下文 + +ADR-0453 已提供 process-epoch capture authority、clean-shutdown evidence、closed terminal audit、compiled-backend rollback/resource report、Primary gate bundle 与 +rollout manifest v2 loader,但维护者仍需手工拼接多个命令、文件名、时间窗口和摘要。只提供底层 gate 会留下四类运维缺口: + +1. canary 开始前没有不可变计划绑定 Profile、精确样本数和当时的 live rollout 基线; +2. terminal/resource/gate 输出容易经 shell 重定向得到宽权限、部分写入或互相不属于同一 session 的文件; +3. qualification 与写入 live manifest 之间没有显式人工边界,配置选择又容易被误报成“当前进程已经激活”; +4. rollback 没有绑定被替换 manifest 的摘要,也没有 crash-replay intent/completion 链。 + +Edge 可能是 128 MiB 路由设备,不能为 canary 新增 daemon、watcher、遥测栈或第二数据库;Standalone 可能运行在集群节点上,但本仪式仍只裁决本机 +`edge|standalone` Profile,不得借宿主机形态伪装成 `cluster-control|worker` Primary 证据。 + +## 决策 + +1. 新增 `qinglong/manual-primary-canary-plan@v1`。`prepare` 只接受受控 session ID、`edge|standalone`、Edge 精确 8 条或 Standalone 32–128 条中由 + operator 选定的精确 admission 数;它记录当前 `qinglong3-rollout.json` 为 absent 或 disabled+SHA-256。已有 enabled manifest 时拒绝准备。 +2. plan 固定派生全部 basename,不接受调用方提供输出路径。config root 必须是当前 UID 拥有、非 symlink、group/world 不可写的真实目录;所有 artifact 均以 + `0600`、同目录临时 inode、file fsync、hard-link no-replace 与 directory fsync 发布。重复调用只接受逐字节相同文件。 +3. `prepare` 只输出三项部署环境:实际 Profile、`QL3_SHADOW_ORIGINS=manual` 与唯一 capture basename;它不写 enabled manifest、不启动任务、不重启服务。 + operator 必须在隔离窗口中通过既有 Legacy 产品入口精确执行计划数量的 manual admission,并干净关闭同一 worker;任何额外 manual execution 都使样本不相等。 +4. `observe` 只有在 capture qualified、单一 manual origin、计数精确且窗口结束至少五分钟后,才以固定参数启动 Node 24 terminal auditor。数据库必须为非 symlink、 + group/world 不可写的普通文件;auditor 只读打开 SQLite。调用方不能注入 origin、window、settling、child script 或 shell。 +5. `resource` 用固定 `full + require-compiled + 8 samples` 启动 ADR-0451 runner。它使用自身临时 SQLite,不接触生产数据库;低配设备必须在应用停止后运行,不能把 + 与常驻 workload 争用内存后的失败解释为产品回归。 +6. `qualify` 复用 ADR-0453 gate,但额外绑定 plan 文件摘要、三份 source 的原始文件摘要、三份 canonical JSON 摘要、精确 admission target 与 gate 文件摘要。 + 现有 gate、partial qualification 或 response loss 只能通过原文件重放;source 在 qualification 后被替换时,mutating CLI 与独立 auditor 都失败关闭。 +7. `approve` 是唯一写 live manifest 的操作。它要求显式 `approvedBy`,审批最短一分钟、最长 24 小时;先复核 plan 时的 absent/disabled 基线,disabled 基线先 + no-replace 归档,再以同目录 fsync 临时文件和最终摘要复核发布 schema v2 manifest。结果状态固定为 `activation_approved`/`primary_selected`,并明确 + `requiresRestart=true`、`runtimeActivationObserved=false`;配置选择不得冒充 HTTP worker 已完成 startup reconciliation 和 router activation。 +8. 应用重启后,实际运行态仍由现有 bootstrap 的 `selected → reconciled → activated` audit 证明。独立 `audit:manual-primary-canary:ql3` 只读复核 plan、source、gate、 + qualification、selection receipt 与 live manifest,可要求 `prepared|qualified|selected|off|rolled-back`;`off` 只证明 loader 当前关闭,`rolled-back` 还要求本 session 的 + intent/completion 摘要链完整。auditor 刻意不提供 `primary_active` 结论。 +9. `rollback` 只接受四个固定 reason 与有界 operator。它先 no-replace 发布 intent,绑定当前 enabled manifest SHA-256 和目标 disabled SHA-256,再复核当前摘要并原子替换 + live manifest,最后发布 completion;intent 或 completion response loss 可用相同参数重放。disabled 已有效但不同于本 session 的目标时拒绝覆盖。 +10. 审批过期后,既有 loader 必须立即 fail-closed 为 off;状态/auditor 报 `approvalExpired=true` 和 `rolloutMode=off`。operator 仍应执行显式 rollback,把磁盘事实 + 收敛为 schema v2 disabled manifest,然后重启应用;过期不能被当作自动续期或自动删除 authority。 + +## 被拒绝的替代方案 + +### 常驻 watcher 自动检测 capture 并启用 Primary + +拒绝。它把一次性证据变成后台控制面,增加路由设备 timer/文件监控成本,并绕过独立人工审批。 + +### Canary CLI 自动触发八个用户任务 + +拒绝。工具没有用户会话、Task Policy、脚本内容或副作用 authority;synthetic spawn 不能证明真实 Legacy 产品入口的 admission capture。 + +### 用 enabled manifest 存在表示 Primary 已运行 + +拒绝。manifest 只在下次 bootstrap 被读取;进程可能尚未重启、startup reconciliation 可能失败或审批已经过期。selection 与 runtime activation 必须分开陈述。 + +### 原地覆盖证据和 rollout 文件以方便重试 + +拒绝。覆盖会丢失冲突和 crash window。证据采用 no-replace;live rollout 只在摘要复核后原子替换,并由 previous/intent/completion 文件保留恢复链。 + +## 资源、安全与部署影响 + +- 正常 QingLong runtime 不新增 import、timer、watcher、listener、数据库连接、schema、migration、package 或生产依赖。全部动作由 operator 一次性调用。 +- plan/qualification/selection/rollback 报告不包含数据库路径、命令、Task、Run、Cron、PID、日志、用户输出、错误正文或 Secret;stdout 只返回 stage、Profile、样本数和布尔状态。 +- `observe` 只读生产 SQLite;`resource` 只使用 runner 的临时 SQLite。Edge 应在应用停止后运行 resource,Standalone 也不得与同机生产 canary 并发争用。 +- POSIX owner 是本机信任根。摘要复核能防止误覆盖和非协作漂移,但不能防御同一 UID 在最后复核与 rename 之间的恶意并发;共享 config root 或远程 delegation 需要 + 另行引入签名/锁服务,不得把本仪式描述成多写者共识。 +- 本仪式不适用于 `cluster-control|worker`,也不证明物理路由、flash wear、断电、跨主机签名或真实生产任务安全。 + +## 验证 + +- domain/CLI 聚焦矩阵覆盖 Edge/Standalone 样本预算、exact shape、短期审批、prepare 重放、unsafe root、disabled baseline drift、source replacement、partial + qualification、selection receipt response loss、审批过期、rollback intent/completion response loss及独立审计。 +- 真实 compiled backend 已通过 D361 `prepare → resource`:Edge plan 保持 automatic activation false;resource 阶段产生 qualified full rollback evidence, + enabled/off 进程 peak RSS 分别为 `109,953,024 / 102,383,616` bytes,真实 Legacy child 均 exit 0,Shadow Run delta 从 1 收敛为 0。 +- 阶段门已重跑:聚焦 `24/24`、`build:back`、完整 backend `1,481 pass / 0 fail / 2 conditional skip`、18-package clean build/test、四项架构审计与 + `14/14` artifact audit 全部通过;产物字节与 D-360 一致。隔离 frozen-lockfile Linux arm64 Docker 的 128 MiB router/256 MiB Edge release peak 分别为 + `81,887,232 / 148,840,448` bytes,`memory.events max/oom/oom_kill` 增量均为 0。 +- D361 不修改 PostgreSQL schema/migration、依赖树或 Kubernetes 拓扑,因此不重跑 PostgreSQL HA;任何仓库 synthetic fixture、compiled resource child 或 Docker + arm64 结果都不沿用、也不冒充尚未执行的真实目标实例 manual capture/activation。 + +## 后续 + +维护者仍需在一台真实目标 Edge 或 Standalone 实例执行 [Manual Primary Canary 操作手册](../operations/ql3-manual-primary-canary.md),并保留 capture、terminal、resource、 +gate、qualification、selection、bootstrap activated audit 与 rollback completion。其他 origin 必须建立自己的 admission authority 与 gate;固定物理路由/flash/断电、 +运行态 durable activation receipt 和多写者 config authority 仍是独立后续工作。 diff --git a/docs/adr/README.md b/docs/adr/README.md index de04a052..7a0abda8 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -457,6 +457,7 @@ | [ADR-0451](./ADR-0451-profile-bounded-legacy-shadow-resource-and-off-rollback-evidence.md) | 按 Profile 有界的 Legacy Shadow 资源与关闭回滚证据 | Accepted | | [ADR-0452](./ADR-0452-atomic-flattened-backend-build-publication.md) | 原子且扁平兼容的 Backend 构建发布 | Accepted | | [ADR-0453](./ADR-0453-origin-scoped-legacy-shadow-capture-authority-and-primary-gate.md) | Origin-scoped Legacy Shadow 捕获权威与 Primary 门禁 | Accepted(首次真实目标实例 manual canary 待执行) | +| [ADR-0454](./ADR-0454-target-instance-manual-primary-canary-ceremony.md) | 目标实例 Manual Primary Canary 与显式回滚仪式 | Accepted(首次真实用户目标实例执行待运维) | ## 规则 diff --git a/docs/operations/ql3-manual-primary-canary.md b/docs/operations/ql3-manual-primary-canary.md new file mode 100644 index 00000000..0d9c0d81 --- /dev/null +++ b/docs/operations/ql3-manual-primary-canary.md @@ -0,0 +1,145 @@ +# QingLong 3.0 Manual Primary Canary 操作手册 + +本流程只适用于本机 `edge` 或 `standalone` Profile 的 `manual` origin。它不会自动执行用户任务,也不会在 `prepare`、`observe`、`resource` 或 `qualify` 阶段启用 +Primary。`cluster-control` 与 `worker` 不适用。 + +示例使用: + +```text +CONFIG_ROOT=/ql/data/config +DATABASE=/ql/data/db/database.sqlite +SESSION=edge-20260819-a +``` + +把示例中的绝对路径和 session 替换为目标实例的实际值。config root 必须由运行 QingLong 的同一 UID 拥有,不能是 symlink,也不能允许 group/world 写入;数据库必须是 +非 symlink、group/world 不可写的普通文件。所有命令都应由该 UID 执行。 + +## 1. 准备不可变计划 + +Edge 必须精确 8 条;Standalone 可在 32–128 中选择一个精确目标: + +```sh +pnpm canary:manual-primary:ql3 -- \ + --mode=prepare \ + --root=/ql/data/config \ + --session=edge-20260819-a \ + --profile=edge \ + --admissions=8 +``` + +已有 enabled rollout 时命令拒绝执行;已有 disabled rollout 时 plan 绑定其 SHA-256。输出中的 `automaticActivation` 必须为 `false`,并给出三项环境值。将这些值写入目标 +部署配置后重启当前 Shadow worker: + +```text +QL_DEPLOYMENT_PROFILE=edge +QL3_SHADOW_ORIGINS=manual +QL3_SHADOW_CAPTURE_EVIDENCE_FILE=ql3-primary-canary-edge-20260819-a.capture.json +``` + +不要同时配置其他 Shadow origin,不要手工创建 capture 文件。 + +## 2. 执行真实 Legacy manual 样本并干净关闭 + +在隔离维护窗口中,通过现有 QingLong 用户界面/API 的正常 manual 执行入口精确提交计划数量的任务。不要用 canary 工具直接 spawn 脚本;那不会证明产品入口。 + +窗口期间禁止其他 manual execution。任务全部终态后,使用部署系统的正常 shutdown 停止同一 HTTP worker。只有干净 shutdown 才会 no-replace 写 capture evidence;kill -9、断电、 +重复文件名或部分文件都不具备资格。 + +查看当前持久状态不会修改文件: + +```sh +pnpm canary:manual-primary:ql3 -- \ + --mode=status \ + --root=/ql/data/config \ + --session=edge-20260819-a +``` + +## 3. 等待闭合窗口并运行终态审计 + +从 capture window 的 `endExclusiveMs` 起等待至少五分钟,然后运行: + +```sh +pnpm canary:manual-primary:ql3 -- \ + --mode=observe \ + --root=/ql/data/config \ + --session=edge-20260819-a \ + --database=/ql/data/db/database.sqlite +``` + +工具固定使用 `origin=manual`、plan 内的 window 和五分钟 settling,不接受调用方覆盖。SQLite 以只读方式打开;结果必须为 `terminal_observed`、`assessment=matched`,scanned +必须等于计划样本数。 + +## 4. 在应用停止状态运行资源/回滚证据 + +低配路由设备必须保持应用停止,避免 128 MiB 预算内与常驻进程争用。命令使用临时 SQLite、compiled backend、full rollback 和固定 8 个 audit samples,不接触生产数据库: + +```sh +pnpm canary:manual-primary:ql3 -- \ + --mode=resource \ + --root=/ql/data/config \ + --session=edge-20260819-a +``` + +结果必须为 `resource_proven`、`qualified=true`。Standalone 即使部署在集群节点上,也仍是本机证据,不得据此启用 cluster-control/worker Primary。 + +## 5. 生成资格并独立复核 + +```sh +pnpm canary:manual-primary:ql3 -- \ + --mode=qualify \ + --root=/ql/data/config \ + --session=edge-20260819-a + +pnpm audit:manual-primary-canary:ql3 -- \ + --root=/ql/data/config \ + --session=edge-20260819-a \ + --require=qualified +``` + +`qualify` 生成 Primary gate 与 qualification,但输出仍必须为 `automaticActivation=false`。独立 audit 重新计算 source/gate/file digest;`compatible=true` 只证明可以提交人工审批。 + +## 6. 显式短期审批并重启验证 + +只有维护者完成审阅后才执行。示例审批一小时,允许范围为一分钟至 24 小时: + +```sh +pnpm canary:manual-primary:ql3 -- \ + --mode=approve \ + --root=/ql/data/config \ + --session=edge-20260819-a \ + --approved-by=operator:local-owner \ + --approval-ms=3600000 + +pnpm audit:manual-primary-canary:ql3 -- \ + --root=/ql/data/config \ + --session=edge-20260819-a \ + --require=selected +``` + +正确状态是 `activation_approved`/`rolloutMode=primary_selected`,不是 `primary_active`;`requiresRestart=true` 且 `runtimeActivationObserved=false`。随后重启应用,并在结构化启动审计中依次确认 +同一 revision 的 `selected`、`reconciled`、`activated`。缺少任一项都不能宣称运行态 Primary 已激活。 + +审批过期后 loader 自动 fail-closed 为 off,不会续期。不要修改原 manifest 时间;创建新 session 重新采样。 + +## 7. 回滚并重启 + +演练或出现异常时立即执行: + +```sh +pnpm canary:manual-primary:ql3 -- \ + --mode=rollback \ + --root=/ql/data/config \ + --session=edge-20260819-a \ + --operator=operator:local-owner \ + --reason=operator_request + +pnpm audit:manual-primary-canary:ql3 -- \ + --root=/ql/data/config \ + --session=edge-20260819-a \ + --require=rolled-back +``` + +支持的 reason 只有 `operator_request`、`runtime_failure`、`gate_rejected`、`approval_expired`。rollback 先发布 intent,再摘要复核并原子替换 live manifest,最后发布 completion;响应丢失时使用完全相同的参数重跑。 + +`rolled-back` 比普通 `off` 更严格:后者在初始 disabled 或审批过期时也成立,前者还要求本 session 的 intent/completion 摘要链完整。完成后重启应用,确认 rollout loader 返回 +disabled/off,Legacy manual 执行继续可用且 Shadow/Primary 不再接管。保留整个 session 的 `0600` 文件和启动审计用于发布复核,不要覆盖或编辑。 diff --git a/package.json b/package.json index b3006f92..26f9e985 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,8 @@ "audit:receipts:ql3": "node scripts/ql3-receipt-audit.cjs", "audit:legacy-shadow-terminal:ql3": "node scripts/ql3-legacy-shadow-terminal-audit.cjs", "gate:legacy-shadow-primary:ql3": "node scripts/ql3-legacy-shadow-primary-gate.cjs", + "canary:manual-primary:ql3": "node scripts/ql3-manual-primary-canary.cjs", + "audit:manual-primary-canary:ql3": "node scripts/ql3-manual-primary-canary-audit.cjs", "audit:edge-imports:ql3": "node scripts/ql3-edge-import-audit.cjs", "audit:cluster-dependencies:ql3": "node scripts/ql3-cluster-dependency-audit.cjs", "audit:package-boundaries:ql3": "node scripts/ql3-package-boundary-audit.cjs", diff --git a/scripts/ql3-manual-primary-canary-audit.cjs b/scripts/ql3-manual-primary-canary-audit.cjs new file mode 100644 index 00000000..269bbd12 --- /dev/null +++ b/scripts/ql3-manual-primary-canary-audit.cjs @@ -0,0 +1,326 @@ +#!/usr/bin/env node + +require('ts-node/register/transpile-only'); + +const fs = require('node:fs'); +const path = require('node:path'); +const { + legacyShadowPrimaryEvidenceSha256, + parseLegacyShadowPrimaryGateReceipt, +} = require('../back/runtime/domain/legacyShadowPrimaryGate'); +const { + createManualPrimaryCanaryDisabledManifest, + manualPrimaryCanaryFileSet, + manualPrimaryCanarySha256, + parseManualPrimaryCanaryPlan, + parseManualPrimaryCanaryQualification, +} = require('../back/runtime/domain/manualPrimaryCanaryCeremony'); +const { + parseRuntimeRolloutManifest, +} = require('../back/runtime/domain/runtimeRolloutManifest'); +const { + readPrivateJson, + serialized, +} = require('./ql3-manual-primary-canary.cjs'); + +const REQUIREMENTS = new Set([ + 'prepared', + 'qualified', + 'selected', + 'off', + 'rolled-back', +]); +const ROLLBACK_REASONS = new Set([ + 'operator_request', + 'runtime_failure', + 'gate_rejected', + 'approval_expired', +]); + +function hasExactKeys(value, expected) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + return false; + } + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + return ( + actual.length === wanted.length && + actual.every((key, index) => key === wanted[index]) + ); +} + +function parseArguments(argv) { + const options = { require: 'prepared' }; + for (const argument of argv) { + if (argument === '--') continue; + if (argument.startsWith('--root=')) { + const value = argument.slice('--root='.length); + if (!value) throw new TypeError('--root must not be empty'); + options.root = path.resolve(value); + } else if (argument.startsWith('--session=')) { + options.sessionId = argument.slice('--session='.length); + } else if (argument.startsWith('--require=')) { + options.require = argument.slice('--require='.length); + } else { + throw new TypeError(`Unsupported argument: ${argument}`); + } + } + if (!options.root || !path.isAbsolute(options.root)) { + throw new TypeError('--root must be absolute'); + } + if (!options.sessionId) throw new TypeError('--session is required'); + if (!REQUIREMENTS.has(options.require)) { + throw new TypeError('--require is invalid'); + } + return options; +} + +function regularFile(target) { + try { + const stat = fs.lstatSync(target); + return stat.isFile() && !stat.isSymbolicLink(); + } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } +} + +function run(options) { + const rootStat = fs.lstatSync(options.root); + if ( + !rootStat.isDirectory() || + rootStat.isSymbolicLink() || + (rootStat.mode & 0o022) !== 0 || + (typeof process.getuid === 'function' && rootStat.uid !== process.getuid()) + ) { + throw new TypeError('Canary root is unsafe'); + } + const files = manualPrimaryCanaryFileSet(options.sessionId); + const resolve = (name) => path.join(options.root, files[name]); + const planRead = readPrivateJson(resolve('plan'), 64 * 1024); + const plan = parseManualPrimaryCanaryPlan(planRead.value); + if (plan.sessionId !== options.sessionId) { + throw new TypeError('Canary plan session mismatch'); + } + const gatePresent = regularFile(resolve('primaryGate')); + const qualificationPresent = regularFile(resolve('qualification')); + if (gatePresent !== qualificationPresent) { + throw new TypeError('Canary qualification is partial'); + } + let eligible = false; + if (gatePresent) { + const gateRead = readPrivateJson(resolve('primaryGate'), 64 * 1024); + const gate = parseLegacyShadowPrimaryGateReceipt(gateRead.value); + const qualification = parseManualPrimaryCanaryQualification( + readPrivateJson(resolve('qualification'), 64 * 1024).value, + ); + const sources = { + capture: readPrivateJson(resolve('capture')), + terminal: readPrivateJson(resolve('terminal')), + resource: readPrivateJson(resolve('resource')), + }; + if ( + gate.assessment !== 'eligible' || + gate.profile !== plan.profile || + gate.origin !== 'manual' || + gate.counts.admitted !== plan.admissionTarget || + qualification.sessionId !== plan.sessionId || + qualification.profile !== plan.profile || + qualification.planSha256 !== planRead.sha256 || + qualification.primaryGateFileSha256 !== gateRead.sha256 || + qualification.sourceFileSha256.capture !== sources.capture.sha256 || + qualification.sourceFileSha256.terminal !== sources.terminal.sha256 || + qualification.sourceFileSha256.resource !== sources.resource.sha256 || + qualification.sourceCanonicalSha256.capture !== + gate.evidence.captureSha256 || + qualification.sourceCanonicalSha256.terminal !== + gate.evidence.terminalSha256 || + qualification.sourceCanonicalSha256.resource !== + gate.evidence.resourceSha256 || + qualification.window.startInclusiveMs !== gate.window.startInclusiveMs || + qualification.window.endExclusiveMs !== gate.window.endExclusiveMs || + qualification.counts.admitted !== gate.counts.admitted || + qualification.counts.captured !== gate.counts.captured || + qualification.counts.terminalScanned !== gate.counts.terminalScanned || + qualification.counts.terminalMatched !== gate.counts.terminalMatched || + legacyShadowPrimaryEvidenceSha256(sources.capture.value) !== + qualification.sourceCanonicalSha256.capture || + legacyShadowPrimaryEvidenceSha256(sources.terminal.value) !== + qualification.sourceCanonicalSha256.terminal || + legacyShadowPrimaryEvidenceSha256(sources.resource.value) !== + qualification.sourceCanonicalSha256.resource + ) { + throw new TypeError('Canary qualification drifted'); + } + eligible = true; + } + const rolloutPath = resolve('rollout'); + let rolloutMode = 'off'; + let approvalExpired = false; + let rolloutSha256; + if (regularFile(rolloutPath)) { + const rolloutRead = readPrivateJson(rolloutPath, 64 * 1024); + const raw = rolloutRead.value; + const now = Date.now(); + approvalExpired = + raw?.enabled === true && + Number.isSafeInteger(raw.expiresAtMs) && + raw.expiresAtMs <= now; + const evaluatedAt = approvalExpired ? raw.approvedAtMs : now; + const decision = parseRuntimeRolloutManifest(raw, evaluatedAt); + rolloutSha256 = rolloutRead.sha256; + if (decision.manifest.enabled && !approvalExpired) { + if ( + decision.manifest.revision !== `manual-primary-${plan.sessionId}` || + !eligible || + decision.manifest.primaryGate.receiptSha256 !== + readPrivateJson(resolve('primaryGate'), 64 * 1024).sha256 + ) { + throw new TypeError('Active rollout is not bound to the canary'); + } + if (!regularFile(resolve('selection'))) { + throw new TypeError('Selected rollout has no selection receipt'); + } + const selection = readPrivateJson(resolve('selection'), 64 * 1024).value; + if ( + !hasExactKeys(selection, [ + 'schema', + 'schemaVersion', + 'sessionId', + 'profile', + 'selectedAtMs', + 'expiresAtMs', + 'manifestSha256', + 'priorRollout', + ]) || + selection?.schema !== 'qinglong/manual-primary-canary-selection@v1' || + selection.schemaVersion !== 1 || + selection.sessionId !== plan.sessionId || + selection.profile !== plan.profile || + selection.selectedAtMs !== decision.manifest.approvedAtMs || + selection.manifestSha256 !== rolloutRead.sha256 || + selection.expiresAtMs !== decision.manifest.expiresAtMs || + JSON.stringify(selection.priorRollout) !== + JSON.stringify(plan.currentRollout) + ) { + throw new TypeError('Selection receipt drifted'); + } + rolloutMode = 'primary_selected'; + } + } + const rolledBack = regularFile(resolve('rollbackComplete')); + if (rolledBack) { + const expectedDisabled = serialized( + createManualPrimaryCanaryDisabledManifest(plan.sessionId), + ); + if ( + !regularFile(rolloutPath) || + readPrivateJson(rolloutPath, 64 * 1024).sha256 !== + manualPrimaryCanarySha256(expectedDisabled) + ) { + throw new TypeError('Rollback completion is not effective'); + } + const intentRead = readPrivateJson(resolve('rollbackIntent'), 64 * 1024); + const intent = intentRead.value; + const completed = readPrivateJson( + resolve('rollbackComplete'), + 64 * 1024, + ).value; + const disabledSha256 = manualPrimaryCanarySha256(expectedDisabled); + if ( + !hasExactKeys(intent, [ + 'schema', + 'schemaVersion', + 'sessionId', + 'profile', + 'createdAtMs', + 'operator', + 'reason', + 'enabledManifestSha256', + 'disabledManifestSha256', + ]) || + intent?.schema !== 'qinglong/manual-primary-canary-rollback-intent@v1' || + intent.schemaVersion !== 1 || + intent.sessionId !== plan.sessionId || + intent.profile !== plan.profile || + typeof intent.operator !== 'string' || + intent.operator.length < 3 || + intent.operator.length > 128 || + /[\u0000-\u001f\u007f]/u.test(intent.operator) || + !ROLLBACK_REASONS.has(intent.reason) || + !Number.isSafeInteger(intent.createdAtMs) || + !/^[a-f0-9]{64}$/u.test(intent.enabledManifestSha256) || + intent.disabledManifestSha256 !== disabledSha256 || + !hasExactKeys(completed, [ + 'schema', + 'schemaVersion', + 'sessionId', + 'profile', + 'completedAtMs', + 'intentSha256', + 'disabledManifestSha256', + ]) || + completed?.schema !== + 'qinglong/manual-primary-canary-rollback-complete@v1' || + completed.schemaVersion !== 1 || + completed.sessionId !== plan.sessionId || + completed.profile !== plan.profile || + !Number.isSafeInteger(completed.completedAtMs) || + completed.completedAtMs < intent.createdAtMs || + completed.intentSha256 !== intentRead.sha256 || + completed.disabledManifestSha256 !== disabledSha256 + ) { + throw new TypeError('Rollback receipt chain drifted'); + } + rolloutMode = 'off'; + } + const compatible = + options.require === 'prepared' || + (options.require === 'qualified' && eligible) || + (options.require === 'selected' && rolloutMode === 'primary_selected') || + (options.require === 'off' && rolloutMode === 'off') || + (options.require === 'rolled-back' && rolloutMode === 'off' && rolledBack); + const report = { + schema: 'qinglong/manual-primary-canary-audit@v1', + schemaVersion: 1, + sessionId: plan.sessionId, + profile: plan.profile, + admissionTarget: plan.admissionTarget, + eligible, + rolloutMode, + approvalExpired, + runtimeActivationObserved: false, + rolledBack, + planSha256: planRead.sha256, + ...(rolloutSha256 === undefined ? {} : { rolloutSha256 }), + requirement: options.require, + compatible, + }; + if (!compatible) { + const error = new Error( + 'Manual Primary canary requirement is not satisfied', + ); + error.report = report; + throw error; + } + return report; +} + +function main() { + process.stdout.write( + `${JSON.stringify(run(parseArguments(process.argv.slice(2))))}\n`, + ); +} + +module.exports = { parseArguments, run }; + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write( + `${error instanceof Error ? error.message : String(error)}\n`, + ); + process.exitCode = 1; + } +} diff --git a/scripts/ql3-manual-primary-canary.cjs b/scripts/ql3-manual-primary-canary.cjs new file mode 100644 index 00000000..1c3f3707 --- /dev/null +++ b/scripts/ql3-manual-primary-canary.cjs @@ -0,0 +1,1091 @@ +#!/usr/bin/env node + +require('ts-node/register/transpile-only'); + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { + createLegacyShadowPrimaryGateReceipt, + legacyShadowPrimaryEvidenceSha256, + parseLegacyShadowPrimaryGateReceipt, +} = require('../back/runtime/domain/legacyShadowPrimaryGate'); +const { + createManualPrimaryCanaryDisabledManifest, + createManualPrimaryCanaryEnabledManifest, + createManualPrimaryCanaryPlan, + createManualPrimaryCanaryQualification, + manualPrimaryCanarySha256, + parseManualPrimaryCanaryPlan, + parseManualPrimaryCanaryQualification, +} = require('../back/runtime/domain/manualPrimaryCanaryCeremony'); +const { + parseRuntimeRolloutManifest, +} = require('../back/runtime/domain/runtimeRolloutManifest'); + +const MAX_PRIVATE_JSON_BYTES = 1024 * 1024; +const MODES = new Set([ + 'prepare', + 'observe', + 'resource', + 'qualify', + 'approve', + 'status', + 'rollback', +]); +const ROLLBACK_REASONS = new Set([ + 'operator_request', + 'runtime_failure', + 'gate_rejected', + 'approval_expired', +]); + +class ManualPrimaryCanaryError extends Error { + constructor(code, message = code) { + super(message); + this.name = 'ManualPrimaryCanaryError'; + this.code = code; + } +} + +function integer(name, value, minimum, maximum) { + if (!/^\d+$/u.test(value)) { + throw new ManualPrimaryCanaryError( + 'argument_invalid', + `${name} must be an integer`, + ); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new ManualPrimaryCanaryError( + 'argument_invalid', + `${name} must be between ${minimum} and ${maximum}`, + ); + } + return parsed; +} + +function parseArguments(argv) { + const options = {}; + for (const argument of argv) { + if (argument === '--') continue; + if (argument.startsWith('--mode=')) { + options.mode = argument.slice('--mode='.length); + } else if (argument.startsWith('--root=')) { + const value = argument.slice('--root='.length); + if (!value) throw new ManualPrimaryCanaryError('argument_invalid'); + options.root = path.resolve(value); + } else if (argument.startsWith('--session=')) { + options.sessionId = argument.slice('--session='.length); + } else if (argument.startsWith('--profile=')) { + options.profile = argument.slice('--profile='.length); + } else if (argument.startsWith('--admissions=')) { + options.admissions = integer( + '--admissions', + argument.slice('--admissions='.length), + 1, + 128, + ); + } else if (argument.startsWith('--database=')) { + const value = argument.slice('--database='.length); + if (!value) throw new ManualPrimaryCanaryError('argument_invalid'); + options.database = path.resolve(value); + } else if (argument.startsWith('--approved-by=')) { + options.approvedBy = argument.slice('--approved-by='.length); + } else if (argument.startsWith('--approval-ms=')) { + options.approvalMs = integer( + '--approval-ms', + argument.slice('--approval-ms='.length), + 60_000, + 24 * 60 * 60 * 1_000, + ); + } else if (argument.startsWith('--operator=')) { + options.operator = argument.slice('--operator='.length); + } else if (argument.startsWith('--reason=')) { + options.reason = argument.slice('--reason='.length); + } else { + throw new ManualPrimaryCanaryError( + 'argument_invalid', + `Unsupported argument: ${argument}`, + ); + } + } + if (!MODES.has(options.mode)) { + throw new ManualPrimaryCanaryError('argument_invalid', '--mode is invalid'); + } + if (!options.root || !path.isAbsolute(options.root)) { + throw new ManualPrimaryCanaryError( + 'argument_invalid', + '--root must be an absolute path', + ); + } + if (!options.sessionId) { + throw new ManualPrimaryCanaryError( + 'argument_invalid', + '--session is required', + ); + } + if (options.mode === 'prepare') { + if (options.profile !== 'edge' && options.profile !== 'standalone') { + throw new ManualPrimaryCanaryError( + 'argument_invalid', + '--profile must be edge or standalone', + ); + } + if (options.admissions === undefined) { + options.admissions = options.profile === 'edge' ? 8 : 32; + } + } + if (options.mode === 'observe' && !options.database) { + throw new ManualPrimaryCanaryError( + 'argument_invalid', + '--database is required for observe', + ); + } + if (options.mode === 'approve') { + if (!options.approvedBy) { + throw new ManualPrimaryCanaryError( + 'argument_invalid', + '--approved-by is required for approve', + ); + } + options.approvalMs ??= 60 * 60 * 1_000; + } + if (options.mode === 'rollback') { + if ( + !options.operator || + options.operator.trim() !== options.operator || + options.operator.length < 3 || + options.operator.length > 128 || + /[\u0000-\u001f\u007f]/u.test(options.operator) || + !ROLLBACK_REASONS.has(options.reason) + ) { + throw new ManualPrimaryCanaryError( + 'argument_invalid', + '--operator and a supported --reason are required for rollback', + ); + } + } + return options; +} + +function assertRoot(root) { + if (!path.isAbsolute(root)) { + throw new ManualPrimaryCanaryError('root_unsafe'); + } + const stat = fs.lstatSync(root); + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + (typeof process.getuid === 'function' && stat.uid !== process.getuid()) || + (stat.mode & 0o022) !== 0 + ) { + throw new ManualPrimaryCanaryError( + 'root_unsafe', + 'Canary root must be an owner-controlled real directory', + ); + } +} + +function filePath(root, fileName) { + if (path.basename(fileName) !== fileName || fileName.includes('..')) { + throw new ManualPrimaryCanaryError('file_name_invalid'); + } + return path.join(root, fileName); +} + +function readPrivateJson(sourcePath, maximumBytes = MAX_PRIVATE_JSON_BYTES) { + let descriptor; + try { + descriptor = fs.openSync( + sourcePath, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0), + ); + const stat = fs.fstatSync(descriptor); + if ( + !stat.isFile() || + (stat.mode & 0o077) !== 0 || + (typeof process.getuid === 'function' && stat.uid !== process.getuid()) || + stat.size < 2 || + stat.size > maximumBytes + ) { + throw new ManualPrimaryCanaryError('private_file_unsafe'); + } + const bytes = Buffer.alloc(stat.size); + let offset = 0; + while (offset < bytes.length) { + const count = fs.readSync( + descriptor, + bytes, + offset, + bytes.length - offset, + offset, + ); + if (count === 0) + throw new ManualPrimaryCanaryError('private_file_stalled'); + offset += count; + } + return { + bytes, + sha256: manualPrimaryCanarySha256(bytes), + value: JSON.parse(bytes.toString('utf8')), + }; + } catch (error) { + if (error instanceof ManualPrimaryCanaryError) throw error; + throw new ManualPrimaryCanaryError( + 'private_file_unavailable', + error instanceof Error ? error.message : String(error), + ); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + } +} + +function serialized(value) { + return Buffer.from(`${JSON.stringify(value)}\n`, 'utf8'); +} + +function syncDirectory(directory) { + const descriptor = fs.openSync(directory, fs.constants.O_RDONLY); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function publishNoReplace(targetPath, value) { + const bytes = Buffer.isBuffer(value) ? value : serialized(value); + const directory = path.dirname(targetPath); + const temporaryPath = path.join( + directory, + `.ql3-canary-${process.pid}-${crypto.randomBytes(8).toString('hex')}.tmp`, + ); + let descriptor; + try { + descriptor = fs.openSync(temporaryPath, 'wx', 0o600); + fs.writeFileSync(descriptor, bytes); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = undefined; + fs.linkSync(temporaryPath, targetPath); + syncDirectory(directory); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + try { + fs.unlinkSync(temporaryPath); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + } + return { bytes, sha256: manualPrimaryCanarySha256(bytes), state: 'created' }; +} + +function publishOrVerify(targetPath, value) { + const expected = Buffer.isBuffer(value) ? value : serialized(value); + try { + return publishNoReplace(targetPath, expected); + } catch (error) { + if (error?.code !== 'EEXIST') throw error; + const existing = readPrivateJson(targetPath); + if (!existing.bytes.equals(expected)) { + throw new ManualPrimaryCanaryError('publication_conflict'); + } + return { ...existing, state: 'existing' }; + } +} + +function readPlan(root, sessionId) { + const candidate = createManualPrimaryCanaryPlan({ + sessionId, + profile: 'edge', + createdAtMs: 0, + admissionTarget: 8, + currentRollout: { state: 'absent' }, + }); + const planRead = readPrivateJson( + filePath(root, candidate.files.plan), + 64 * 1024, + ); + const plan = parseManualPrimaryCanaryPlan(planRead.value); + if (plan.sessionId !== sessionId) { + throw new ManualPrimaryCanaryError('plan_session_mismatch'); + } + return { plan, read: planRead }; +} + +function currentRollout(root, now) { + const target = filePath(root, 'qinglong3-rollout.json'); + let read; + try { + read = readPrivateJson(target, 64 * 1024); + } catch (error) { + if (error.code === 'private_file_unavailable' && !fs.existsSync(target)) { + return { state: 'absent', target }; + } + throw error; + } + const decision = parseRuntimeRolloutManifest(read.value, now); + return { + state: decision.manifest.enabled ? 'enabled' : 'disabled', + target, + read, + manifest: decision.manifest, + }; +} + +function assertRolloutExpectation(plan, current) { + if (plan.currentRollout.state !== current.state) { + throw new ManualPrimaryCanaryError('rollout_changed'); + } + if ( + current.state === 'disabled' && + plan.currentRollout.sha256 !== current.read.sha256 + ) { + throw new ManualPrimaryCanaryError('rollout_changed'); + } +} + +function replaceRollout(targetPath, expected, nextBytes) { + const directory = path.dirname(targetPath); + const temporaryPath = path.join( + directory, + `.ql3-rollout-${process.pid}-${crypto.randomBytes(8).toString('hex')}.tmp`, + ); + let descriptor; + try { + descriptor = fs.openSync(temporaryPath, 'wx', 0o600); + fs.writeFileSync(descriptor, nextBytes); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = undefined; + const current = fs.existsSync(targetPath) + ? readPrivateJson(targetPath, 64 * 1024) + : undefined; + if ( + (expected === undefined && current !== undefined) || + (expected !== undefined && current?.sha256 !== expected) + ) { + throw new ManualPrimaryCanaryError('rollout_changed'); + } + if (current === undefined) { + fs.linkSync(temporaryPath, targetPath); + fs.unlinkSync(temporaryPath); + } else { + fs.renameSync(temporaryPath, targetPath); + } + syncDirectory(directory); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + try { + fs.unlinkSync(temporaryPath); + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + } +} + +function validateCapture(plan, value, now) { + const evidence = value && typeof value === 'object' ? value : {}; + const capture = evidence.capture ?? {}; + const totals = capture.totals ?? {}; + if ( + evidence.schema !== 'qinglong/legacy-shadow-capture-evidence@v1' || + evidence.profile !== plan.profile || + evidence.qualification?.passed !== true || + capture.profile !== plan.profile || + capture.assessment !== 'captured' || + capture.capturePermille !== 1_000 || + capture.window?.basis !== 'process_local_legacy_admission' || + !Number.isSafeInteger(capture.window?.startInclusiveMs) || + !Number.isSafeInteger(capture.window?.endExclusiveMs) || + capture.window.startInclusiveMs >= capture.window.endExclusiveMs || + capture.configuredOriginCount !== 1 || + capture.byOrigin?.length !== 1 || + capture.byOrigin[0]?.origin !== 'manual' || + capture.byOrigin[0]?.admitted !== plan.admissionTarget || + capture.byOrigin[0]?.captured !== plan.admissionTarget || + capture.byOrigin[0]?.failed !== 0 || + capture.byOrigin[0]?.pending !== 0 || + totals.admitted !== plan.admissionTarget || + totals.captured !== plan.admissionTarget || + totals.failed !== 0 || + totals.pending !== 0 || + capture.window?.startInclusiveMs < plan.createdAtMs || + capture.window?.endExclusiveMs + plan.minimumSettlingAgeMs > now + ) { + throw new ManualPrimaryCanaryError('capture_not_ready'); + } + return capture.window; +} + +function runJsonChild(script, arguments_, timeoutMs, dependencies) { + const result = (dependencies.spawnSync ?? spawnSync)( + process.execPath, + [script, ...arguments_], + { + cwd: dependencies.workspaceRoot, + encoding: 'utf8', + timeout: timeoutMs, + maxBuffer: MAX_PRIVATE_JSON_BYTES, + env: process.env, + }, + ); + if (result.error || result.signal || result.status !== 0) { + throw new ManualPrimaryCanaryError('evidence_child_failed'); + } + if ( + typeof result.stdout !== 'string' || + Buffer.byteLength(result.stdout) > MAX_PRIVATE_JSON_BYTES + ) { + throw new ManualPrimaryCanaryError('evidence_child_output_invalid'); + } + try { + return JSON.parse(result.stdout); + } catch { + throw new ManualPrimaryCanaryError('evidence_child_output_invalid'); + } +} + +function prepare(options, dependencies) { + assertRoot(options.root); + const now = dependencies.clock.now(); + const current = currentRollout(options.root, now); + if (current.state === 'enabled') { + throw new ManualPrimaryCanaryError('primary_already_enabled'); + } + const expectedFiles = createManualPrimaryCanaryPlan({ + sessionId: options.sessionId, + profile: options.profile, + createdAtMs: 0, + admissionTarget: options.admissions, + currentRollout: { state: 'absent' }, + }).files; + const existingPlanPath = filePath(options.root, expectedFiles.plan); + if (existsRegular(existingPlanPath)) { + const existing = parseManualPrimaryCanaryPlan( + readPrivateJson(existingPlanPath, 64 * 1024).value, + ); + if ( + existing.profile !== options.profile || + existing.admissionTarget !== options.admissions + ) { + throw new ManualPrimaryCanaryError('publication_conflict'); + } + assertRolloutExpectation(existing, current); + return { + state: 'prepared', + publication: 'existing', + sessionId: existing.sessionId, + profile: existing.profile, + admissionTarget: existing.admissionTarget, + environment: { + QL_DEPLOYMENT_PROFILE: existing.profile, + QL3_SHADOW_ORIGINS: 'manual', + QL3_SHADOW_CAPTURE_EVIDENCE_FILE: existing.files.capture, + }, + automaticActivation: false, + }; + } + const plan = createManualPrimaryCanaryPlan({ + sessionId: options.sessionId, + profile: options.profile, + createdAtMs: now, + admissionTarget: options.admissions, + currentRollout: + current.state === 'absent' + ? { state: 'absent' } + : { state: 'disabled', sha256: current.read.sha256 }, + }); + const publication = publishOrVerify( + filePath(options.root, plan.files.plan), + plan, + ); + return { + state: 'prepared', + publication: publication.state, + sessionId: plan.sessionId, + profile: plan.profile, + admissionTarget: plan.admissionTarget, + environment: { + QL_DEPLOYMENT_PROFILE: plan.profile, + QL3_SHADOW_ORIGINS: 'manual', + QL3_SHADOW_CAPTURE_EVIDENCE_FILE: plan.files.capture, + }, + automaticActivation: false, + }; +} + +function observe(options, dependencies) { + assertRoot(options.root); + const { plan } = readPlan(options.root, options.sessionId); + const now = dependencies.clock.now(); + const capture = readPrivateJson(filePath(options.root, plan.files.capture)); + const window = validateCapture(plan, capture.value, now); + const terminalPath = filePath(options.root, plan.files.terminal); + if (existsRegular(terminalPath)) { + const terminal = readPrivateJson(terminalPath).value; + if ( + terminal?.schema !== + 'qinglong/legacy-shadow-terminal-difference-report@v1' || + terminal.profile !== plan.profile || + terminal.assessment !== 'matched' || + terminal.window?.startInclusiveMs !== window.startInclusiveMs || + terminal.window?.endExclusiveMs !== window.endExclusiveMs || + terminal.scanned !== plan.admissionTarget + ) { + throw new ManualPrimaryCanaryError('publication_conflict'); + } + return { + state: 'terminal_observed', + publication: 'existing', + sessionId: plan.sessionId, + profile: plan.profile, + assessment: terminal.assessment, + scanned: terminal.scanned, + }; + } + const databaseStat = fs.lstatSync(options.database); + if ( + !databaseStat.isFile() || + databaseStat.isSymbolicLink() || + (typeof process.getuid === 'function' && + databaseStat.uid !== process.getuid()) || + (databaseStat.mode & 0o022) !== 0 + ) { + throw new ManualPrimaryCanaryError('database_unsafe'); + } + const terminal = runJsonChild( + path.join( + dependencies.workspaceRoot, + 'scripts/ql3-legacy-shadow-terminal-audit.cjs', + ), + [ + `--database=${options.database}`, + `--profile=${plan.profile}`, + '--origin=manual', + `--window-start-ms=${window.startInclusiveMs}`, + `--window-end-ms=${window.endExclusiveMs}`, + `--observed-at-ms=${now}`, + `--minimum-settling-age-ms=${plan.minimumSettlingAgeMs}`, + '--json', + '--fail-on-difference', + ], + 30_000, + dependencies, + ); + const publication = publishOrVerify(terminalPath, terminal); + return { + state: 'terminal_observed', + publication: publication.state, + sessionId: plan.sessionId, + profile: plan.profile, + assessment: terminal.assessment, + scanned: terminal.scanned, + }; +} + +function resource(options, dependencies) { + assertRoot(options.root); + const { plan } = readPlan(options.root, options.sessionId); + const resourcePath = filePath(options.root, plan.files.resource); + if (existsRegular(resourcePath)) { + const existing = readPrivateJson(resourcePath).value; + if ( + existing?.fixture !== + 'qinglong/legacy-shadow-resource-rollback-evidence@v1' || + existing.profile !== plan.profile || + existing.workload?.mode !== 'full' || + existing.workload?.runtime !== 'compiled_backend' || + existing.qualification?.passed !== true || + existing.rollback?.performed !== true || + existing.rollback?.legacyContinued !== true || + existing.rollback?.shadowWritesStopped !== true || + existing.rollback?.databaseIntegrity !== 'ok' + ) { + throw new ManualPrimaryCanaryError('publication_conflict'); + } + return { + state: 'resource_proven', + publication: 'existing', + sessionId: plan.sessionId, + profile: plan.profile, + qualified: true, + }; + } + const report = runJsonChild( + path.join( + dependencies.workspaceRoot, + 'scripts/ql3-legacy-shadow-resource-rollback.cjs', + ), + [ + `--profile=${plan.profile}`, + '--mode=full', + '--samples=8', + '--require-compiled', + '--json', + ], + plan.profile === 'edge' ? 120_000 : 180_000, + dependencies, + ); + const publication = publishOrVerify(resourcePath, report); + return { + state: 'resource_proven', + publication: publication.state, + sessionId: plan.sessionId, + profile: plan.profile, + qualified: report.qualification?.passed === true, + }; +} + +function qualify(options, dependencies) { + assertRoot(options.root); + const { plan, read: planRead } = readPlan(options.root, options.sessionId); + const gatePath = filePath(options.root, plan.files.primaryGate); + const qualificationPath = filePath(options.root, plan.files.qualification); + if (existsRegular(gatePath) && existsRegular(qualificationPath)) { + const { qualification } = verifyQualification(options.root, plan, planRead); + return { + state: 'qualified', + publication: 'existing', + sessionId: plan.sessionId, + profile: plan.profile, + admitted: qualification.counts.admitted, + automaticActivation: false, + }; + } + const capture = readPrivateJson(filePath(options.root, plan.files.capture)); + const terminal = readPrivateJson(filePath(options.root, plan.files.terminal)); + const resourceRead = readPrivateJson( + filePath(options.root, plan.files.resource), + ); + const now = dependencies.clock.now(); + validateCapture(plan, capture.value, now); + const gate = existsRegular(gatePath) + ? parseLegacyShadowPrimaryGateReceipt( + readPrivateJson(gatePath, 64 * 1024).value, + ) + : createLegacyShadowPrimaryGateReceipt({ + profile: plan.profile, + generatedAtMs: now, + capture: capture.value, + terminal: terminal.value, + resource: resourceRead.value, + }); + if (gate.assessment !== 'eligible') { + throw new ManualPrimaryCanaryError( + 'primary_gate_ineligible', + gate.violations.join(','), + ); + } + if ( + legacyShadowPrimaryEvidenceSha256(capture.value) !== + gate.evidence.captureSha256 || + legacyShadowPrimaryEvidenceSha256(terminal.value) !== + gate.evidence.terminalSha256 || + legacyShadowPrimaryEvidenceSha256(resourceRead.value) !== + gate.evidence.resourceSha256 + ) { + throw new ManualPrimaryCanaryError('primary_gate_source_drift'); + } + const gatePublication = publishOrVerify(gatePath, gate); + const qualification = createManualPrimaryCanaryQualification({ + plan, + planSha256: planRead.sha256, + primaryGate: gate, + primaryGateFileSha256: gatePublication.sha256, + sourceFileSha256: { + capture: capture.sha256, + terminal: terminal.sha256, + resource: resourceRead.sha256, + }, + qualifiedAtMs: now, + }); + const qualificationPublication = publishOrVerify( + qualificationPath, + qualification, + ); + return { + state: 'qualified', + publication: qualificationPublication.state, + sessionId: plan.sessionId, + profile: plan.profile, + admitted: qualification.counts.admitted, + automaticActivation: false, + }; +} + +function verifyQualification(root, plan, planRead) { + const gateRead = readPrivateJson( + filePath(root, plan.files.primaryGate), + 64 * 1024, + ); + const gate = parseLegacyShadowPrimaryGateReceipt(gateRead.value); + const qualificationRead = readPrivateJson( + filePath(root, plan.files.qualification), + 64 * 1024, + ); + const qualification = parseManualPrimaryCanaryQualification( + qualificationRead.value, + ); + const capture = readPrivateJson(filePath(root, plan.files.capture)); + const terminal = readPrivateJson(filePath(root, plan.files.terminal)); + const resource = readPrivateJson(filePath(root, plan.files.resource)); + if ( + qualification.sessionId !== plan.sessionId || + qualification.profile !== plan.profile || + qualification.planSha256 !== planRead.sha256 || + qualification.primaryGateFileSha256 !== gateRead.sha256 || + qualification.sourceFileSha256.capture !== capture.sha256 || + qualification.sourceFileSha256.terminal !== terminal.sha256 || + qualification.sourceFileSha256.resource !== resource.sha256 || + qualification.sourceCanonicalSha256.capture !== + gate.evidence.captureSha256 || + qualification.sourceCanonicalSha256.terminal !== + gate.evidence.terminalSha256 || + qualification.sourceCanonicalSha256.resource !== + gate.evidence.resourceSha256 || + qualification.window.startInclusiveMs !== gate.window.startInclusiveMs || + qualification.window.endExclusiveMs !== gate.window.endExclusiveMs || + qualification.counts.admitted !== gate.counts.admitted || + qualification.counts.captured !== gate.counts.captured || + qualification.counts.terminalScanned !== gate.counts.terminalScanned || + qualification.counts.terminalMatched !== gate.counts.terminalMatched || + gate.counts.admitted !== plan.admissionTarget || + legacyShadowPrimaryEvidenceSha256(capture.value) !== + qualification.sourceCanonicalSha256.capture || + legacyShadowPrimaryEvidenceSha256(terminal.value) !== + qualification.sourceCanonicalSha256.terminal || + legacyShadowPrimaryEvidenceSha256(resource.value) !== + qualification.sourceCanonicalSha256.resource || + gate.assessment !== 'eligible' + ) { + throw new ManualPrimaryCanaryError('qualification_drift'); + } + return { gate, gateRead, qualification, qualificationRead }; +} + +function approve(options, dependencies) { + assertRoot(options.root); + const { plan, read: planRead } = readPlan(options.root, options.sessionId); + const { qualification } = verifyQualification(options.root, plan, planRead); + const now = dependencies.clock.now(); + const current = currentRollout(options.root, now); + if ( + current.state === 'enabled' && + current.manifest.revision === `manual-primary-${plan.sessionId}` + ) { + if ( + current.manifest.primaryGate.receiptSha256 !== + qualification.primaryGateFileSha256 || + current.manifest.primaryGate.receiptFile !== plan.files.primaryGate || + current.manifest.rollout.origins.manual !== 'primary' || + current.manifest.approvedBy !== options.approvedBy || + current.manifest.expiresAtMs - current.manifest.approvedAtMs !== + options.approvalMs + ) { + throw new ManualPrimaryCanaryError('active_rollout_drift'); + } + const selection = { + schema: 'qinglong/manual-primary-canary-selection@v1', + schemaVersion: 1, + sessionId: plan.sessionId, + profile: plan.profile, + selectedAtMs: current.manifest.approvedAtMs, + expiresAtMs: current.manifest.expiresAtMs, + manifestSha256: current.read.sha256, + priorRollout: plan.currentRollout, + }; + const publication = publishOrVerify( + filePath(options.root, plan.files.selection), + selection, + ); + return { + state: 'activation_approved', + publication: publication.state, + sessionId: plan.sessionId, + expiresAtMs: current.manifest.expiresAtMs, + requiresRestart: true, + }; + } + assertRolloutExpectation(plan, current); + const manifest = createManualPrimaryCanaryEnabledManifest({ + plan, + qualification, + approvedBy: options.approvedBy, + approvedAtMs: now, + approvalMs: options.approvalMs, + }); + const manifestBytes = serialized(manifest); + if (current.state === 'disabled') { + publishOrVerify( + filePath(options.root, plan.files.previousRollout), + current.read.bytes, + ); + } + replaceRollout( + current.target, + current.state === 'disabled' ? current.read.sha256 : undefined, + manifestBytes, + ); + const manifestSha256 = manualPrimaryCanarySha256(manifestBytes); + const selection = { + schema: 'qinglong/manual-primary-canary-selection@v1', + schemaVersion: 1, + sessionId: plan.sessionId, + profile: plan.profile, + selectedAtMs: now, + expiresAtMs: manifest.expiresAtMs, + manifestSha256, + priorRollout: plan.currentRollout, + }; + const selectionPublication = publishOrVerify( + filePath(options.root, plan.files.selection), + selection, + ); + return { + state: 'activation_approved', + publication: selectionPublication.state, + sessionId: plan.sessionId, + expiresAtMs: manifest.expiresAtMs, + requiresRestart: true, + }; +} + +function rollback(options, dependencies) { + assertRoot(options.root); + const { plan } = readPlan(options.root, options.sessionId); + const now = dependencies.clock.now(); + const target = filePath(options.root, plan.files.rollout); + const currentRead = readPrivateJson(target, 64 * 1024); + const raw = currentRead.value; + const evaluationTime = + raw?.enabled === true && Number.isSafeInteger(raw.approvedAtMs) + ? raw.approvedAtMs + : now; + const decision = parseRuntimeRolloutManifest(raw, evaluationTime); + const disabled = createManualPrimaryCanaryDisabledManifest(plan.sessionId); + const disabledBytes = serialized(disabled); + const disabledSha256 = manualPrimaryCanarySha256(disabledBytes); + if (decision.manifest.enabled === false) { + if (currentRead.sha256 !== disabledSha256) { + throw new ManualPrimaryCanaryError('different_disabled_rollout'); + } + } else if ( + decision.manifest.revision !== `manual-primary-${plan.sessionId}` + ) { + throw new ManualPrimaryCanaryError('rollout_session_mismatch'); + } + const intentPath = filePath(options.root, plan.files.rollbackIntent); + let intent; + let intentSha256; + if (existsRegular(intentPath)) { + const intentRead = readPrivateJson(intentPath, 64 * 1024); + intent = intentRead.value; + intentSha256 = intentRead.sha256; + if ( + intent?.schema !== 'qinglong/manual-primary-canary-rollback-intent@v1' || + intent.sessionId !== plan.sessionId || + intent.profile !== plan.profile || + intent.operator !== options.operator || + intent.reason !== options.reason || + intent.disabledManifestSha256 !== disabledSha256 || + (decision.manifest.enabled === true && + intent.enabledManifestSha256 !== currentRead.sha256) + ) { + throw new ManualPrimaryCanaryError('publication_conflict'); + } + } else { + if (decision.manifest.enabled === false) { + throw new ManualPrimaryCanaryError('rollback_intent_missing'); + } + intent = { + schema: 'qinglong/manual-primary-canary-rollback-intent@v1', + schemaVersion: 1, + sessionId: plan.sessionId, + profile: plan.profile, + createdAtMs: now, + operator: options.operator, + reason: options.reason, + enabledManifestSha256: currentRead.sha256, + disabledManifestSha256: disabledSha256, + }; + intentSha256 = publishNoReplace(intentPath, intent).sha256; + } + if (decision.manifest.enabled === true) { + replaceRollout(target, currentRead.sha256, disabledBytes); + } + const completedPath = filePath(options.root, plan.files.rollbackComplete); + let publication; + if (existsRegular(completedPath)) { + const completed = readPrivateJson(completedPath, 64 * 1024).value; + if ( + completed?.schema !== + 'qinglong/manual-primary-canary-rollback-complete@v1' || + completed.sessionId !== plan.sessionId || + completed.profile !== plan.profile || + completed.intentSha256 !== intentSha256 || + completed.disabledManifestSha256 !== disabledSha256 + ) { + throw new ManualPrimaryCanaryError('publication_conflict'); + } + publication = { state: 'existing' }; + } else { + publication = publishNoReplace(completedPath, { + schema: 'qinglong/manual-primary-canary-rollback-complete@v1', + schemaVersion: 1, + sessionId: plan.sessionId, + profile: plan.profile, + completedAtMs: now, + intentSha256, + disabledManifestSha256: disabledSha256, + }); + } + return { + state: 'rolled_back', + publication: publication.state, + sessionId: plan.sessionId, + profile: plan.profile, + }; +} + +function existsRegular(target) { + try { + const stat = fs.lstatSync(target); + return stat.isFile() && !stat.isSymbolicLink(); + } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } +} + +function status(options, dependencies) { + assertRoot(options.root); + const { plan, read: planRead } = readPlan(options.root, options.sessionId); + const present = Object.fromEntries( + [ + 'capture', + 'terminal', + 'resource', + 'primaryGate', + 'qualification', + 'selection', + 'rollbackIntent', + 'rollbackComplete', + ].map((name) => [ + name, + existsRegular(filePath(options.root, plan.files[name])), + ]), + ); + let stage = 'prepared'; + if (present.capture) stage = 'captured'; + if (present.terminal) stage = 'terminal_observed'; + if (present.resource) stage = 'resource_proven'; + if (present.primaryGate || present.qualification) { + if (!present.primaryGate || !present.qualification) { + throw new ManualPrimaryCanaryError('qualification_incomplete'); + } + verifyQualification(options.root, plan, planRead); + stage = 'qualified'; + } + const rolloutTarget = filePath(options.root, plan.files.rollout); + let rolloutMode = 'off'; + let approvalExpired = false; + if (existsRegular(rolloutTarget)) { + const read = readPrivateJson(rolloutTarget, 64 * 1024); + const raw = read.value; + const now = dependencies.clock.now(); + approvalExpired = + raw?.enabled === true && + Number.isSafeInteger(raw.expiresAtMs) && + raw.expiresAtMs <= now; + const evaluatedAt = approvalExpired ? raw.approvedAtMs : now; + const decision = parseRuntimeRolloutManifest(raw, evaluatedAt); + if (decision.manifest.enabled && !approvalExpired) { + if (decision.manifest.revision !== `manual-primary-${plan.sessionId}`) { + throw new ManualPrimaryCanaryError('rollout_session_mismatch'); + } + rolloutMode = 'primary_selected'; + stage = 'activation_approved'; + } else if (approvalExpired) { + stage = 'approval_expired'; + } + } + if (present.rollbackComplete) { + const disabled = createManualPrimaryCanaryDisabledManifest(plan.sessionId); + const read = readPrivateJson(rolloutTarget, 64 * 1024); + if (read.sha256 !== manualPrimaryCanarySha256(serialized(disabled))) { + throw new ManualPrimaryCanaryError('rollback_not_effective'); + } + rolloutMode = 'off'; + stage = 'rolled_back'; + } + return { + state: stage, + sessionId: plan.sessionId, + profile: plan.profile, + admissionTarget: plan.admissionTarget, + rolloutMode, + approvalExpired, + runtimeActivationObserved: false, + artifacts: present, + }; +} + +function run(options, overrides = {}) { + const dependencies = { + clock: overrides.clock ?? { now: () => Date.now() }, + spawnSync: overrides.spawnSync, + workspaceRoot: overrides.workspaceRoot ?? path.resolve(__dirname, '..'), + }; + switch (options.mode) { + case 'prepare': + return prepare(options, dependencies); + case 'observe': + return observe(options, dependencies); + case 'resource': + return resource(options, dependencies); + case 'qualify': + return qualify(options, dependencies); + case 'approve': + return approve(options, dependencies); + case 'rollback': + return rollback(options, dependencies); + case 'status': + return status(options, dependencies); + default: + throw new ManualPrimaryCanaryError('mode_invalid'); + } +} + +function main() { + const result = run(parseArguments(process.argv.slice(2))); + process.stdout.write(`${JSON.stringify(result)}\n`); +} + +module.exports = { + MAX_PRIVATE_JSON_BYTES, + ManualPrimaryCanaryError, + parseArguments, + publishNoReplace, + publishOrVerify, + readPrivateJson, + run, + serialized, +}; + +if (require.main === module) { + try { + main(); + } catch (error) { + const code = + error instanceof ManualPrimaryCanaryError + ? error.code + : error instanceof Error + ? error.name + : 'unknown'; + process.stderr.write(`${code}\n`); + process.exitCode = 1; + } +} diff --git a/test/back/manualPrimaryCanaryCeremony.test.cjs b/test/back/manualPrimaryCanaryCeremony.test.cjs new file mode 100644 index 00000000..952b6583 --- /dev/null +++ b/test/back/manualPrimaryCanaryCeremony.test.cjs @@ -0,0 +1,281 @@ +require('ts-node/register/transpile-only'); + +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const { + createLegacyShadowPrimaryGateReceipt, +} = require('../../back/runtime/domain/legacyShadowPrimaryGate'); +const { + MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS, + createManualPrimaryCanaryDisabledManifest, + createManualPrimaryCanaryEnabledManifest, + createManualPrimaryCanaryPlan, + createManualPrimaryCanaryQualification, + manualPrimaryCanaryFileSet, + parseManualPrimaryCanaryPlan, + parseManualPrimaryCanaryQualification, +} = require('../../back/runtime/domain/manualPrimaryCanaryCeremony'); +const { + parseRuntimeRolloutManifest, +} = require('../../back/runtime/domain/runtimeRolloutManifest'); + +const START = 1_750_400_000_000; +const END = START + 60_000; +const GENERATED = END + 6 * 60_000; +const DIGESTS = { + plan: '1'.repeat(64), + gate: '2'.repeat(64), + capture: '3'.repeat(64), + terminal: '4'.repeat(64), + resource: '5'.repeat(64), +}; + +function captureEvidence(admitted = 8) { + const outcomes = { + completed: 0, + cancelled: 0, + abandoned: 0, + markedLost: 0, + repaired: 0, + pending: 0, + ambiguous: 0, + skipped: 0, + failed: 0, + }; + return { + schema: 'qinglong/legacy-shadow-capture-evidence@v1', + profile: 'edge', + startup: { + schema: 'qinglong/legacy-shadow-startup-difference-report@v1', + profile: 'edge', + assessment: 'converged', + configuredOriginCount: 1, + coverage: { remaining: false }, + outcomes, + byOrigin: [{ origin: 'manual', scanned: 0, ...outcomes }], + }, + capture: { + schema: 'qinglong/legacy-shadow-capture-report@v1', + profile: 'edge', + assessment: 'captured', + epoch: '019f75d2-5555-7555-8555-555555555555', + window: { + basis: 'process_local_legacy_admission', + startInclusiveMs: START, + endExclusiveMs: END, + }, + configuredOriginCount: 1, + totals: { + admitted, + captured: admitted, + failed: 0, + pending: 0, + failures: { fact: 0, observer: 0, initialization: 0, accept: 0 }, + }, + byOrigin: [ + { + origin: 'manual', + admitted, + captured: admitted, + failed: 0, + pending: 0, + failures: { + fact: 0, + observer: 0, + initialization: 0, + accept: 0, + }, + }, + ], + capturePermille: 1_000, + }, + qualification: { + passed: true, + startupConverged: true, + originCoverageExact: true, + captureComplete: true, + }, + }; +} + +function terminal() { + return { + schema: 'qinglong/legacy-shadow-terminal-difference-report@v1', + profile: 'edge', + observedAtMs: GENERATED - 1, + window: { + basis: 'shadow_run_created_at', + startInclusiveMs: START, + endExclusiveMs: END, + minimumSettlingAgeMs: 300_000, + closed: true, + }, + coverage: { + direction: 'shadow_to_legacy', + cohort: 'legacy_owned_shadow_runs', + legacyWithoutShadow: 'not_measured', + }, + scanned: 8, + remaining: false, + evidenceComplete: true, + assessment: 'matched', + counts: { matched: 8 }, + byOrigin: [{ origin: 'manual', scanned: 8, matched: 8 }], + terminalAgreementPermille: 1_000, + fullyComparablePermille: 1_000, + }; +} + +function resource() { + return { + schemaVersion: 1, + fixture: 'qinglong/legacy-shadow-resource-rollback-evidence@v1', + profile: 'edge', + workload: { mode: 'full', runtime: 'compiled_backend' }, + rollback: { + performed: true, + legacyContinued: true, + shadowWritesStopped: true, + databaseIntegrity: 'ok', + }, + qualification: { passed: true, violations: [] }, + }; +} + +function plan() { + return createManualPrimaryCanaryPlan({ + sessionId: 'edge-0001', + profile: 'edge', + createdAtMs: START - 1_000, + admissionTarget: 8, + currentRollout: { state: 'absent' }, + }); +} + +function gate() { + return createLegacyShadowPrimaryGateReceipt({ + profile: 'edge', + generatedAtMs: GENERATED, + capture: captureEvidence(), + terminal: terminal(), + resource: resource(), + }); +} + +function qualification() { + return createManualPrimaryCanaryQualification({ + plan: plan(), + planSha256: DIGESTS.plan, + primaryGate: gate(), + primaryGateFileSha256: DIGESTS.gate, + sourceFileSha256: { + capture: DIGESTS.capture, + terminal: DIGESTS.terminal, + resource: DIGESTS.resource, + }, + qualifiedAtMs: GENERATED + 1, + }); +} + +test('creates an exact profile-bounded canary plan with no automatic activation', () => { + const value = plan(); + + assert.deepEqual(parseManualPrimaryCanaryPlan(value), value); + assert.deepEqual(value.files, manualPrimaryCanaryFileSet('edge-0001')); + assert.equal(value.admissionTarget, 8); + assert.equal(value.activation.defaultMode, 'off'); + assert.equal(value.activation.allowLegacyFallbackBeforeStart, false); + assert.throws( + () => createManualPrimaryCanaryPlan({ ...value, admissionTarget: 9 }), + /admission target/, + ); + assert.throws( + () => parseManualPrimaryCanaryPlan({ ...value, command: 'enable' }), + /shape/, + ); +}); + +test('allows a selected standalone cohort only inside the reviewed range', () => { + const value = createManualPrimaryCanaryPlan({ + sessionId: 'standalone-0032', + profile: 'standalone', + createdAtMs: START, + admissionTarget: 64, + currentRollout: { state: 'disabled', sha256: DIGESTS.plan }, + }); + + assert.equal(parseManualPrimaryCanaryPlan(value).admissionTarget, 64); + for (const admissionTarget of [31, 129]) { + assert.throws( + () => createManualPrimaryCanaryPlan({ ...value, admissionTarget }), + /admission target/, + ); + } +}); + +test('binds the exact plan, source files and independently reproducible gate', () => { + const value = qualification(); + + assert.deepEqual(parseManualPrimaryCanaryQualification(value), value); + assert.equal(value.assessment, 'eligible'); + assert.deepEqual(value.counts, { + admitted: 8, + captured: 8, + terminalScanned: 8, + terminalMatched: 8, + }); + const wrongPlan = { ...plan(), admissionTarget: 7 }; + assert.throws( + () => + createManualPrimaryCanaryQualification({ + plan: wrongPlan, + planSha256: DIGESTS.plan, + primaryGate: gate(), + primaryGateFileSha256: DIGESTS.gate, + sourceFileSha256: { + capture: DIGESTS.capture, + terminal: DIGESTS.terminal, + resource: DIGESTS.resource, + }, + qualifiedAtMs: GENERATED + 1, + }), + /admission target|match the plan/, + ); +}); + +test('creates only a short-lived manual Primary manifest after qualification', () => { + const approvedAtMs = GENERATED + 2; + const manifest = createManualPrimaryCanaryEnabledManifest({ + plan: plan(), + qualification: qualification(), + approvedBy: 'operator:local-owner', + approvedAtMs, + approvalMs: 60 * 60 * 1_000, + }); + + assert.equal( + parseRuntimeRolloutManifest(manifest, approvedAtMs).policy.modeFor( + 'manual', + ), + 'primary', + ); + assert.equal(manifest.rollout.defaultMode, 'off'); + assert.equal(manifest.primaryGate.receiptSha256, DIGESTS.gate); + assert.equal(manifest.rollbackPlanRef, plan().files.plan); + assert.throws( + () => + createManualPrimaryCanaryEnabledManifest({ + plan: plan(), + qualification: qualification(), + approvedBy: 'operator:local-owner', + approvedAtMs, + approvalMs: MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS + 1, + }), + /approval/, + ); + assert.deepEqual(createManualPrimaryCanaryDisabledManifest('edge-0001'), { + schemaVersion: 2, + revision: 'manual-primary-edge-0001-rollback', + enabled: false, + }); +}); diff --git a/test/back/ql3ManualPrimaryCanary.test.cjs b/test/back/ql3ManualPrimaryCanary.test.cjs new file mode 100644 index 00000000..40a5eae7 --- /dev/null +++ b/test/back/ql3ManualPrimaryCanary.test.cjs @@ -0,0 +1,532 @@ +require('ts-node/register/transpile-only'); + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { afterEach, test } = require('node:test'); +const { + manualPrimaryCanaryFileSet, +} = require('../../back/runtime/domain/manualPrimaryCanaryCeremony'); +const { + parseArguments, + readPrivateJson, + run, +} = require('../../scripts/ql3-manual-primary-canary.cjs'); +const { + run: audit, +} = require('../../scripts/ql3-manual-primary-canary-audit.cjs'); + +const NOW = Date.now(); +const START = NOW - 7 * 60_000; +const END = START + 1_000; +const QUALIFIED_AT = END + 6 * 60_000; +const SESSION = 'edge-live-0001'; +const directories = []; + +function directory() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-canary-')); + directories.push(root); + return root; +} + +function writeJson(target, value) { + fs.writeFileSync(target, `${JSON.stringify(value)}\n`, { mode: 0o600 }); +} + +function captureEvidence(admitted = 8) { + const outcomes = { + completed: 0, + cancelled: 0, + abandoned: 0, + markedLost: 0, + repaired: 0, + pending: 0, + ambiguous: 0, + skipped: 0, + failed: 0, + }; + return { + schema: 'qinglong/legacy-shadow-capture-evidence@v1', + profile: 'edge', + startup: { + schema: 'qinglong/legacy-shadow-startup-difference-report@v1', + profile: 'edge', + assessment: 'converged', + configuredOriginCount: 1, + coverage: { remaining: false }, + outcomes, + byOrigin: [{ origin: 'manual', scanned: 0, ...outcomes }], + }, + capture: { + schema: 'qinglong/legacy-shadow-capture-report@v1', + profile: 'edge', + assessment: 'captured', + epoch: '019f75d2-5555-7555-8555-555555555555', + window: { + basis: 'process_local_legacy_admission', + startInclusiveMs: START, + endExclusiveMs: END, + }, + configuredOriginCount: 1, + totals: { + admitted, + captured: admitted, + failed: 0, + pending: 0, + failures: { fact: 0, observer: 0, initialization: 0, accept: 0 }, + }, + byOrigin: [ + { + origin: 'manual', + admitted, + captured: admitted, + failed: 0, + pending: 0, + failures: { + fact: 0, + observer: 0, + initialization: 0, + accept: 0, + }, + }, + ], + capturePermille: 1_000, + }, + qualification: { + passed: true, + startupConverged: true, + originCoverageExact: true, + captureComplete: true, + }, + }; +} + +function terminal() { + return { + schema: 'qinglong/legacy-shadow-terminal-difference-report@v1', + profile: 'edge', + observedAtMs: QUALIFIED_AT - 1, + window: { + basis: 'shadow_run_created_at', + startInclusiveMs: START, + endExclusiveMs: END, + minimumSettlingAgeMs: 300_000, + closed: true, + }, + coverage: { + direction: 'shadow_to_legacy', + cohort: 'legacy_owned_shadow_runs', + legacyWithoutShadow: 'not_measured', + }, + scanned: 8, + remaining: false, + evidenceComplete: true, + assessment: 'matched', + counts: { matched: 8 }, + byOrigin: [{ origin: 'manual', scanned: 8, matched: 8 }], + terminalAgreementPermille: 1_000, + fullyComparablePermille: 1_000, + }; +} + +function resourceEvidence() { + return { + schemaVersion: 1, + fixture: 'qinglong/legacy-shadow-resource-rollback-evidence@v1', + profile: 'edge', + workload: { mode: 'full', runtime: 'compiled_backend' }, + rollback: { + performed: true, + legacyContinued: true, + shadowWritesStopped: true, + databaseIntegrity: 'ok', + }, + qualification: { passed: true, violations: [] }, + }; +} + +function prepare(root, clock = START - 1_000) { + return run( + { + mode: 'prepare', + root, + sessionId: SESSION, + profile: 'edge', + admissions: 8, + }, + { clock: { now: () => clock } }, + ); +} + +function seedSources(root) { + const files = manualPrimaryCanaryFileSet(SESSION); + writeJson(path.join(root, files.capture), captureEvidence()); + writeJson(path.join(root, files.terminal), terminal()); + writeJson(path.join(root, files.resource), resourceEvidence()); + return files; +} + +afterEach(() => { + for (const root of directories.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('prepares an idempotent private plan and exposes only explicit canary environment', () => { + const root = directory(); + const first = prepare(root); + const second = prepare(root, START); + const files = manualPrimaryCanaryFileSet(SESSION); + + assert.equal(first.state, 'prepared'); + assert.equal(first.automaticActivation, false); + assert.deepEqual(first.environment, { + QL_DEPLOYMENT_PROFILE: 'edge', + QL3_SHADOW_ORIGINS: 'manual', + QL3_SHADOW_CAPTURE_EVIDENCE_FILE: files.capture, + }); + assert.equal(second.publication, 'existing'); + assert.equal(fs.statSync(path.join(root, files.plan)).mode & 0o777, 0o600); + assert.equal( + run({ mode: 'status', root, sessionId: SESSION }).state, + 'prepared', + ); +}); + +test('qualifies, explicitly approves, audits and rolls back one target session', () => { + const root = directory(); + prepare(root); + const files = seedSources(root); + + const qualified = run( + { mode: 'qualify', root, sessionId: SESSION }, + { clock: { now: () => QUALIFIED_AT } }, + ); + assert.equal(qualified.state, 'qualified'); + assert.equal(qualified.automaticActivation, false); + fs.unlinkSync(path.join(root, files.qualification)); + assert.equal( + run( + { mode: 'qualify', root, sessionId: SESSION }, + { clock: { now: () => QUALIFIED_AT + 10 } }, + ).state, + 'qualified', + ); + assert.equal( + audit({ root, sessionId: SESSION, require: 'qualified' }).compatible, + true, + ); + + const activatedAt = Math.max(Date.now(), QUALIFIED_AT + 1); + const approved = run( + { + mode: 'approve', + root, + sessionId: SESSION, + approvedBy: 'operator:local-owner', + approvalMs: 60 * 60 * 1_000, + }, + { clock: { now: () => activatedAt } }, + ); + assert.equal(approved.state, 'activation_approved'); + const selection = readPrivateJson( + path.join(root, files.selection), + 64 * 1024, + ).value; + assert.equal(selection.selectedAtMs, activatedAt); + assert.equal(Object.hasOwn(selection, 'activatedAtMs'), false); + fs.unlinkSync(path.join(root, files.selection)); + assert.throws( + () => + run( + { + mode: 'approve', + root, + sessionId: SESSION, + approvedBy: 'operator:different-owner', + approvalMs: 60 * 60 * 1_000, + }, + { clock: { now: () => activatedAt + 5 } }, + ), + (error) => error.code === 'active_rollout_drift', + ); + assert.equal( + run( + { + mode: 'approve', + root, + sessionId: SESSION, + approvedBy: 'operator:local-owner', + approvalMs: 60 * 60 * 1_000, + }, + { clock: { now: () => activatedAt + 10 } }, + ).state, + 'activation_approved', + ); + assert.equal( + audit({ root, sessionId: SESSION, require: 'selected' }).rolloutMode, + 'primary_selected', + ); + assert.equal( + audit({ root, sessionId: SESSION, require: 'selected' }) + .runtimeActivationObserved, + false, + ); + assert.equal( + run( + { mode: 'status', root, sessionId: SESSION }, + { clock: { now: () => activatedAt + 1 } }, + ).state, + 'activation_approved', + ); + assert.throws( + () => audit({ root, sessionId: SESSION, require: 'rolled-back' }), + /not satisfied/, + ); + + const rolledBack = run( + { + mode: 'rollback', + root, + sessionId: SESSION, + operator: 'operator:local-owner', + reason: 'operator_request', + }, + { clock: { now: () => activatedAt + 2 } }, + ); + assert.equal(rolledBack.state, 'rolled_back'); + fs.unlinkSync(path.join(root, files.rollbackComplete)); + assert.equal( + run( + { + mode: 'rollback', + root, + sessionId: SESSION, + operator: 'operator:local-owner', + reason: 'operator_request', + }, + { clock: { now: () => activatedAt + 3 } }, + ).state, + 'rolled_back', + ); + assert.equal( + run( + { + mode: 'rollback', + root, + sessionId: SESSION, + operator: 'operator:local-owner', + reason: 'operator_request', + }, + { clock: { now: () => activatedAt + 4 } }, + ).publication, + 'existing', + ); + assert.equal( + audit({ root, sessionId: SESSION, require: 'rolled-back' }).rolloutMode, + 'off', + ); + assert.equal( + readPrivateJson(path.join(root, files.rollout), 64 * 1024).value.enabled, + false, + ); + const completionPath = path.join(root, files.rollbackComplete); + const completion = readPrivateJson(completionPath, 64 * 1024).value; + completion.intentSha256 = '0'.repeat(64); + writeJson(completionPath, completion); + assert.throws( + () => audit({ root, sessionId: SESSION, require: 'rolled-back' }), + /receipt chain drifted/, + ); +}); + +test('observe and resource use fixed child commands and recover from existing evidence', () => { + const root = directory(); + prepare(root); + const files = manualPrimaryCanaryFileSet(SESSION); + writeJson(path.join(root, files.capture), captureEvidence()); + const database = path.join(root, 'database.sqlite'); + fs.writeFileSync(database, 'sqlite-fixture', { mode: 0o600 }); + const calls = []; + const spawnSync = (_node, arguments_) => { + calls.push(arguments_); + const value = arguments_[0].endsWith('terminal-audit.cjs') + ? terminal() + : resourceEvidence(); + return { + status: 0, + signal: null, + stdout: JSON.stringify(value), + stderr: '', + }; + }; + const dependencies = { + clock: { now: () => QUALIFIED_AT }, + spawnSync, + workspaceRoot: path.resolve(__dirname, '../..'), + }; + + assert.equal( + run({ mode: 'observe', root, sessionId: SESSION, database }, dependencies) + .state, + 'terminal_observed', + ); + assert.equal( + run({ mode: 'resource', root, sessionId: SESSION }, dependencies).state, + 'resource_proven', + ); + assert.equal(calls.length, 2); + assert.ok(calls[0].includes('--origin=manual')); + assert.ok(calls[1].includes('--require-compiled')); + run({ mode: 'observe', root, sessionId: SESSION, database }, dependencies); + run({ mode: 'resource', root, sessionId: SESSION }, dependencies); + assert.equal(calls.length, 2); +}); + +test('fails closed on unsafe roots, incomplete cohorts and rollout drift', () => { + const root = directory(); + const symlink = `${root}-link`; + fs.symlinkSync(root, symlink); + directories.push(symlink); + assert.throws( + () => + run({ + mode: 'prepare', + root: symlink, + sessionId: SESSION, + profile: 'edge', + admissions: 8, + }), + (error) => error.code === 'root_unsafe', + ); + + prepare(root); + const files = seedSources(root); + const capture = captureEvidence(7); + writeJson(path.join(root, 'wrong-capture.json'), capture); + fs.renameSync( + path.join(root, 'wrong-capture.json'), + path.join(root, files.capture), + ); + assert.throws( + () => + run( + { mode: 'qualify', root, sessionId: SESSION }, + { clock: { now: () => QUALIFIED_AT } }, + ), + /capture_not_ready/, + ); + + assert.throws( + () => + parseArguments([ + '--mode=rollback', + `--root=${root}`, + `--session=${SESSION}`, + '--operator=owner', + '--reason=arbitrary-text', + ]), + /supported/, + ); +}); + +test('binds an existing disabled rollout and rejects a changed live baseline', () => { + const root = directory(); + const disabled = { + schemaVersion: 2, + revision: 'operator-disabled-baseline', + enabled: false, + }; + writeJson(path.join(root, 'qinglong3-rollout.json'), disabled); + prepare(root); + const plan = readPrivateJson( + path.join(root, manualPrimaryCanaryFileSet(SESSION).plan), + 64 * 1024, + ).value; + assert.equal(plan.currentRollout.state, 'disabled'); + + writeJson(path.join(root, 'changed.json'), { + schemaVersion: 2, + revision: 'changed-disabled-baseline', + enabled: false, + }); + fs.renameSync( + path.join(root, 'changed.json'), + path.join(root, 'qinglong3-rollout.json'), + ); + assert.throws( + () => prepare(root, START), + (error) => error.code === 'rollout_changed', + ); +}); + +test('reports an expired approval as runtime-off until explicit rollback', () => { + const root = directory(); + prepare(root); + seedSources(root); + run( + { mode: 'qualify', root, sessionId: SESSION }, + { clock: { now: () => QUALIFIED_AT } }, + ); + const activatedAt = Math.max(Date.now(), QUALIFIED_AT + 1); + run( + { + mode: 'approve', + root, + sessionId: SESSION, + approvedBy: 'operator:local-owner', + approvalMs: 60_000, + }, + { clock: { now: () => activatedAt } }, + ); + + const report = run( + { mode: 'status', root, sessionId: SESSION }, + { clock: { now: () => activatedAt + 60_000 } }, + ); + assert.equal(report.state, 'approval_expired'); + assert.equal(report.rolloutMode, 'off'); + assert.equal(report.approvalExpired, true); +}); + +test('independent audit rejects source replacement after qualification', () => { + const root = directory(); + prepare(root); + const files = seedSources(root); + run( + { mode: 'qualify', root, sessionId: SESSION }, + { clock: { now: () => QUALIFIED_AT } }, + ); + const replaced = resourceEvidence(); + replaced.unreviewed = true; + writeJson(path.join(root, 'replacement.json'), replaced); + fs.renameSync( + path.join(root, 'replacement.json'), + path.join(root, files.resource), + ); + + assert.throws( + () => audit({ root, sessionId: SESSION, require: 'qualified' }), + /drifted/, + ); +}); + +test('independent audit rejects qualification canonical digest drift', () => { + const root = directory(); + prepare(root); + const files = seedSources(root); + run( + { mode: 'qualify', root, sessionId: SESSION }, + { clock: { now: () => QUALIFIED_AT } }, + ); + const qualificationPath = path.join(root, files.qualification); + const qualification = readPrivateJson(qualificationPath, 64 * 1024).value; + qualification.sourceCanonicalSha256.capture = '0'.repeat(64); + writeJson(qualificationPath, qualification); + + assert.throws( + () => audit({ root, sessionId: SESSION, require: 'qualified' }), + /drifted/, + ); +});