mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): operationalize manual primary canary
This commit is contained in:
whyour
@@ -0,0 +1,476 @@
|
||||
import { createHash } from 'crypto';
|
||||
import {
|
||||
parseLegacyShadowPrimaryGateReceipt,
|
||||
type LegacyShadowPrimaryGateReceipt,
|
||||
} from './legacyShadowPrimaryGate';
|
||||
import {
|
||||
parseRuntimeRolloutManifest,
|
||||
REQUIRED_RUNTIME_ROLLOUT_GATES,
|
||||
type DisabledRuntimeRolloutManifest,
|
||||
type EnabledRuntimeRolloutManifest,
|
||||
} from './runtimeRolloutManifest';
|
||||
|
||||
export const MANUAL_PRIMARY_CANARY_PLAN_SCHEMA =
|
||||
'qinglong/manual-primary-canary-plan@v1';
|
||||
export const MANUAL_PRIMARY_CANARY_QUALIFICATION_SCHEMA =
|
||||
'qinglong/manual-primary-canary-qualification@v1';
|
||||
export const MANUAL_PRIMARY_CANARY_MINIMUM_SETTLING_AGE_MS = 5 * 60_000;
|
||||
export const MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS = 24 * 60 * 60 * 1_000;
|
||||
|
||||
const SESSION_PATTERN = /^[a-z0-9](?:[a-z0-9-]{6,62}[a-z0-9])?$/u;
|
||||
const SHA256_PATTERN = /^[a-f0-9]{64}$/u;
|
||||
|
||||
export type ManualPrimaryCanaryProfile = 'edge' | 'standalone';
|
||||
export type ManualPrimaryCanaryCurrentRollout =
|
||||
| { state: 'absent' }
|
||||
| { state: 'disabled'; sha256: string };
|
||||
|
||||
export interface ManualPrimaryCanaryFileSet {
|
||||
plan: string;
|
||||
capture: string;
|
||||
terminal: string;
|
||||
resource: string;
|
||||
primaryGate: string;
|
||||
qualification: string;
|
||||
selection: string;
|
||||
previousRollout: string;
|
||||
rollbackIntent: string;
|
||||
rollbackComplete: string;
|
||||
rollout: 'qinglong3-rollout.json';
|
||||
}
|
||||
|
||||
export interface ManualPrimaryCanaryPlan {
|
||||
schema: typeof MANUAL_PRIMARY_CANARY_PLAN_SCHEMA;
|
||||
schemaVersion: 1;
|
||||
sessionId: string;
|
||||
profile: ManualPrimaryCanaryProfile;
|
||||
origin: 'manual';
|
||||
createdAtMs: number;
|
||||
admissionTarget: number;
|
||||
minimumSettlingAgeMs: typeof MANUAL_PRIMARY_CANARY_MINIMUM_SETTLING_AGE_MS;
|
||||
currentRollout: ManualPrimaryCanaryCurrentRollout;
|
||||
files: ManualPrimaryCanaryFileSet;
|
||||
activation: {
|
||||
maxApprovalMs: typeof MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS;
|
||||
defaultMode: 'off';
|
||||
allowLegacyFallbackBeforeStart: false;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ManualPrimaryCanaryQualification {
|
||||
schema: typeof MANUAL_PRIMARY_CANARY_QUALIFICATION_SCHEMA;
|
||||
schemaVersion: 1;
|
||||
sessionId: string;
|
||||
profile: ManualPrimaryCanaryProfile;
|
||||
origin: 'manual';
|
||||
qualifiedAtMs: number;
|
||||
assessment: 'eligible';
|
||||
planSha256: string;
|
||||
primaryGateFileSha256: string;
|
||||
sourceFileSha256: {
|
||||
capture: string;
|
||||
terminal: string;
|
||||
resource: string;
|
||||
};
|
||||
sourceCanonicalSha256: {
|
||||
capture: string;
|
||||
terminal: string;
|
||||
resource: string;
|
||||
};
|
||||
window: {
|
||||
startInclusiveMs: number;
|
||||
endExclusiveMs: number;
|
||||
};
|
||||
counts: {
|
||||
admitted: number;
|
||||
captured: number;
|
||||
terminalScanned: number;
|
||||
terminalMatched: number;
|
||||
};
|
||||
}
|
||||
|
||||
function record(value: unknown, name: string): Record<string, unknown> {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError(`${name} must be an object`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function exactKeys(
|
||||
value: Record<string, unknown>,
|
||||
expected: readonly string[],
|
||||
name: string,
|
||||
): void {
|
||||
const actual = Object.keys(value).sort();
|
||||
const wanted = [...expected].sort();
|
||||
if (
|
||||
actual.length !== wanted.length ||
|
||||
actual.some((key, index) => key !== wanted[index])
|
||||
) {
|
||||
throw new TypeError(`${name} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function safeInteger(value: unknown, name: string): number {
|
||||
if (!Number.isSafeInteger(value) || Number(value) < 0) {
|
||||
throw new TypeError(`${name} must be a non-negative safe integer`);
|
||||
}
|
||||
return Number(value);
|
||||
}
|
||||
|
||||
function sha256(value: unknown, name: string): string {
|
||||
if (typeof value !== 'string' || !SHA256_PATTERN.test(value)) {
|
||||
throw new TypeError(`${name} must be a SHA-256 digest`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function sessionId(value: unknown): string {
|
||||
if (typeof value !== 'string' || !SESSION_PATTERN.test(value)) {
|
||||
throw new TypeError('Manual Primary canary session id is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function profile(value: unknown): ManualPrimaryCanaryProfile {
|
||||
if (value !== 'edge' && value !== 'standalone') {
|
||||
throw new TypeError('Manual Primary canary profile is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function admissionTarget(
|
||||
selectedProfile: ManualPrimaryCanaryProfile,
|
||||
value: unknown,
|
||||
): number {
|
||||
const target = safeInteger(value, 'admissionTarget');
|
||||
if (
|
||||
(selectedProfile === 'edge' && target !== 8) ||
|
||||
(selectedProfile === 'standalone' && (target < 32 || target > 128))
|
||||
) {
|
||||
throw new TypeError('Manual Primary canary admission target is invalid');
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
export function manualPrimaryCanaryFileSet(
|
||||
rawSessionId: string,
|
||||
): ManualPrimaryCanaryFileSet {
|
||||
const id = sessionId(rawSessionId);
|
||||
const prefix = `ql3-primary-canary-${id}`;
|
||||
return Object.freeze({
|
||||
plan: `${prefix}.plan.json`,
|
||||
capture: `${prefix}.capture.json`,
|
||||
terminal: `${prefix}.terminal.json`,
|
||||
resource: `${prefix}.resource.json`,
|
||||
primaryGate: `${prefix}.primary-gate.json`,
|
||||
qualification: `${prefix}.qualification.json`,
|
||||
selection: `${prefix}.selection.json`,
|
||||
previousRollout: `${prefix}.previous-rollout.json`,
|
||||
rollbackIntent: `${prefix}.rollback-intent.json`,
|
||||
rollbackComplete: `${prefix}.rollback-complete.json`,
|
||||
rollout: 'qinglong3-rollout.json',
|
||||
});
|
||||
}
|
||||
|
||||
export function manualPrimaryCanarySha256(bytes: string | Buffer): string {
|
||||
return createHash('sha256').update(bytes).digest('hex');
|
||||
}
|
||||
|
||||
export function createManualPrimaryCanaryPlan(input: {
|
||||
sessionId: string;
|
||||
profile: ManualPrimaryCanaryProfile;
|
||||
createdAtMs: number;
|
||||
admissionTarget: number;
|
||||
currentRollout: ManualPrimaryCanaryCurrentRollout;
|
||||
}): ManualPrimaryCanaryPlan {
|
||||
const id = sessionId(input.sessionId);
|
||||
const selectedProfile = profile(input.profile);
|
||||
const createdAtMs = safeInteger(input.createdAtMs, 'createdAtMs');
|
||||
const target = admissionTarget(selectedProfile, input.admissionTarget);
|
||||
const currentRollout = input.currentRollout;
|
||||
if (currentRollout.state === 'disabled') {
|
||||
sha256(currentRollout.sha256, 'currentRollout.sha256');
|
||||
} else if (currentRollout.state !== 'absent') {
|
||||
throw new TypeError('Current rollout state is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
schema: MANUAL_PRIMARY_CANARY_PLAN_SCHEMA,
|
||||
schemaVersion: 1,
|
||||
sessionId: id,
|
||||
profile: selectedProfile,
|
||||
origin: 'manual',
|
||||
createdAtMs,
|
||||
admissionTarget: target,
|
||||
minimumSettlingAgeMs: MANUAL_PRIMARY_CANARY_MINIMUM_SETTLING_AGE_MS,
|
||||
currentRollout: { ...currentRollout },
|
||||
files: manualPrimaryCanaryFileSet(id),
|
||||
activation: {
|
||||
maxApprovalMs: MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS,
|
||||
defaultMode: 'off' as const,
|
||||
allowLegacyFallbackBeforeStart: false as const,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function parseManualPrimaryCanaryPlan(
|
||||
value: unknown,
|
||||
): ManualPrimaryCanaryPlan {
|
||||
const plan = record(value, 'plan');
|
||||
exactKeys(
|
||||
plan,
|
||||
[
|
||||
'schema',
|
||||
'schemaVersion',
|
||||
'sessionId',
|
||||
'profile',
|
||||
'origin',
|
||||
'createdAtMs',
|
||||
'admissionTarget',
|
||||
'minimumSettlingAgeMs',
|
||||
'currentRollout',
|
||||
'files',
|
||||
'activation',
|
||||
],
|
||||
'plan',
|
||||
);
|
||||
if (
|
||||
plan.schema !== MANUAL_PRIMARY_CANARY_PLAN_SCHEMA ||
|
||||
plan.schemaVersion !== 1 ||
|
||||
plan.origin !== 'manual' ||
|
||||
plan.minimumSettlingAgeMs !== MANUAL_PRIMARY_CANARY_MINIMUM_SETTLING_AGE_MS
|
||||
) {
|
||||
throw new TypeError('Manual Primary canary plan header is invalid');
|
||||
}
|
||||
const selectedProfile = profile(plan.profile);
|
||||
const id = sessionId(plan.sessionId);
|
||||
const current = record(plan.currentRollout, 'plan.currentRollout');
|
||||
if (current.state === 'absent') {
|
||||
exactKeys(current, ['state'], 'plan.currentRollout');
|
||||
} else if (current.state === 'disabled') {
|
||||
exactKeys(current, ['state', 'sha256'], 'plan.currentRollout');
|
||||
sha256(current.sha256, 'plan.currentRollout.sha256');
|
||||
} else {
|
||||
throw new TypeError('Manual Primary canary current rollout is invalid');
|
||||
}
|
||||
const files = record(plan.files, 'plan.files');
|
||||
const expectedFiles = manualPrimaryCanaryFileSet(id);
|
||||
exactKeys(files, Object.keys(expectedFiles), 'plan.files');
|
||||
for (const [key, expected] of Object.entries(expectedFiles)) {
|
||||
if (files[key] !== expected) {
|
||||
throw new TypeError(`plan.files.${key} is invalid`);
|
||||
}
|
||||
}
|
||||
const activation = record(plan.activation, 'plan.activation');
|
||||
exactKeys(
|
||||
activation,
|
||||
['maxApprovalMs', 'defaultMode', 'allowLegacyFallbackBeforeStart'],
|
||||
'plan.activation',
|
||||
);
|
||||
if (
|
||||
activation.maxApprovalMs !== MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS ||
|
||||
activation.defaultMode !== 'off' ||
|
||||
activation.allowLegacyFallbackBeforeStart !== false
|
||||
) {
|
||||
throw new TypeError('Manual Primary canary activation policy is invalid');
|
||||
}
|
||||
return createManualPrimaryCanaryPlan({
|
||||
sessionId: id,
|
||||
profile: selectedProfile,
|
||||
createdAtMs: safeInteger(plan.createdAtMs, 'plan.createdAtMs'),
|
||||
admissionTarget: admissionTarget(selectedProfile, plan.admissionTarget),
|
||||
currentRollout: current as ManualPrimaryCanaryCurrentRollout,
|
||||
});
|
||||
}
|
||||
|
||||
export function createManualPrimaryCanaryQualification(input: {
|
||||
plan: ManualPrimaryCanaryPlan;
|
||||
planSha256: string;
|
||||
primaryGate: LegacyShadowPrimaryGateReceipt;
|
||||
primaryGateFileSha256: string;
|
||||
sourceFileSha256: {
|
||||
capture: string;
|
||||
terminal: string;
|
||||
resource: string;
|
||||
};
|
||||
qualifiedAtMs: number;
|
||||
}): ManualPrimaryCanaryQualification {
|
||||
const plan = parseManualPrimaryCanaryPlan(input.plan);
|
||||
const gate = parseLegacyShadowPrimaryGateReceipt(input.primaryGate);
|
||||
const qualifiedAtMs = safeInteger(input.qualifiedAtMs, 'qualifiedAtMs');
|
||||
if (
|
||||
gate.assessment !== 'eligible' ||
|
||||
gate.profile !== plan.profile ||
|
||||
gate.origin !== plan.origin ||
|
||||
gate.counts.admitted !== plan.admissionTarget ||
|
||||
gate.counts.captured !== plan.admissionTarget ||
|
||||
gate.counts.terminalScanned !== plan.admissionTarget ||
|
||||
gate.counts.terminalMatched !== plan.admissionTarget ||
|
||||
gate.window.startInclusiveMs < plan.createdAtMs ||
|
||||
gate.window.endExclusiveMs > gate.generatedAtMs ||
|
||||
gate.generatedAtMs > qualifiedAtMs
|
||||
) {
|
||||
throw new TypeError('Manual Primary canary gate does not match the plan');
|
||||
}
|
||||
return Object.freeze({
|
||||
schema: MANUAL_PRIMARY_CANARY_QUALIFICATION_SCHEMA,
|
||||
schemaVersion: 1,
|
||||
sessionId: plan.sessionId,
|
||||
profile: plan.profile,
|
||||
origin: 'manual',
|
||||
qualifiedAtMs,
|
||||
assessment: 'eligible',
|
||||
planSha256: sha256(input.planSha256, 'planSha256'),
|
||||
primaryGateFileSha256: sha256(
|
||||
input.primaryGateFileSha256,
|
||||
'primaryGateFileSha256',
|
||||
),
|
||||
sourceFileSha256: {
|
||||
capture: sha256(input.sourceFileSha256.capture, 'capture file digest'),
|
||||
terminal: sha256(input.sourceFileSha256.terminal, 'terminal file digest'),
|
||||
resource: sha256(input.sourceFileSha256.resource, 'resource file digest'),
|
||||
},
|
||||
sourceCanonicalSha256: {
|
||||
capture: gate.evidence.captureSha256,
|
||||
terminal: gate.evidence.terminalSha256,
|
||||
resource: gate.evidence.resourceSha256,
|
||||
},
|
||||
window: { ...gate.window },
|
||||
counts: { ...gate.counts },
|
||||
});
|
||||
}
|
||||
|
||||
export function parseManualPrimaryCanaryQualification(
|
||||
value: unknown,
|
||||
): ManualPrimaryCanaryQualification {
|
||||
const qualification = record(value, 'qualification');
|
||||
exactKeys(
|
||||
qualification,
|
||||
[
|
||||
'schema',
|
||||
'schemaVersion',
|
||||
'sessionId',
|
||||
'profile',
|
||||
'origin',
|
||||
'qualifiedAtMs',
|
||||
'assessment',
|
||||
'planSha256',
|
||||
'primaryGateFileSha256',
|
||||
'sourceFileSha256',
|
||||
'sourceCanonicalSha256',
|
||||
'window',
|
||||
'counts',
|
||||
],
|
||||
'qualification',
|
||||
);
|
||||
if (
|
||||
qualification.schema !== MANUAL_PRIMARY_CANARY_QUALIFICATION_SCHEMA ||
|
||||
qualification.schemaVersion !== 1 ||
|
||||
qualification.origin !== 'manual' ||
|
||||
qualification.assessment !== 'eligible'
|
||||
) {
|
||||
throw new TypeError('Manual Primary canary qualification is invalid');
|
||||
}
|
||||
sessionId(qualification.sessionId);
|
||||
profile(qualification.profile);
|
||||
safeInteger(qualification.qualifiedAtMs, 'qualification.qualifiedAtMs');
|
||||
sha256(qualification.planSha256, 'qualification.planSha256');
|
||||
sha256(
|
||||
qualification.primaryGateFileSha256,
|
||||
'qualification.primaryGateFileSha256',
|
||||
);
|
||||
for (const name of ['sourceFileSha256', 'sourceCanonicalSha256'] as const) {
|
||||
const digests = record(qualification[name], `qualification.${name}`);
|
||||
exactKeys(digests, ['capture', 'terminal', 'resource'], name);
|
||||
for (const key of ['capture', 'terminal', 'resource']) {
|
||||
sha256(digests[key], `qualification.${name}.${key}`);
|
||||
}
|
||||
}
|
||||
const window = record(qualification.window, 'qualification.window');
|
||||
exactKeys(window, ['startInclusiveMs', 'endExclusiveMs'], 'window');
|
||||
const start = safeInteger(window.startInclusiveMs, 'window.startInclusiveMs');
|
||||
const end = safeInteger(window.endExclusiveMs, 'window.endExclusiveMs');
|
||||
if (start >= end) throw new TypeError('Qualification window is invalid');
|
||||
const counts = record(qualification.counts, 'qualification.counts');
|
||||
exactKeys(
|
||||
counts,
|
||||
['admitted', 'captured', 'terminalScanned', 'terminalMatched'],
|
||||
'counts',
|
||||
);
|
||||
const normalizedCounts = Object.fromEntries(
|
||||
Object.entries(counts).map(([key, count]) => [
|
||||
key,
|
||||
safeInteger(count, `counts.${key}`),
|
||||
]),
|
||||
) as ManualPrimaryCanaryQualification['counts'];
|
||||
if (new Set(Object.values(normalizedCounts)).size !== 1) {
|
||||
throw new TypeError('Qualification counts do not agree');
|
||||
}
|
||||
return qualification as unknown as ManualPrimaryCanaryQualification;
|
||||
}
|
||||
|
||||
export function createManualPrimaryCanaryEnabledManifest(input: {
|
||||
plan: ManualPrimaryCanaryPlan;
|
||||
qualification: ManualPrimaryCanaryQualification;
|
||||
approvedBy: string;
|
||||
approvedAtMs: number;
|
||||
approvalMs: number;
|
||||
}): EnabledRuntimeRolloutManifest {
|
||||
const plan = parseManualPrimaryCanaryPlan(input.plan);
|
||||
const qualification = parseManualPrimaryCanaryQualification(
|
||||
input.qualification,
|
||||
);
|
||||
const approvedAtMs = safeInteger(input.approvedAtMs, 'approvedAtMs');
|
||||
const approvalMs = safeInteger(input.approvalMs, 'approvalMs');
|
||||
if (
|
||||
qualification.sessionId !== plan.sessionId ||
|
||||
qualification.profile !== plan.profile ||
|
||||
qualification.qualifiedAtMs > approvedAtMs ||
|
||||
approvalMs < 60_000 ||
|
||||
approvalMs > MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS ||
|
||||
typeof input.approvedBy !== 'string' ||
|
||||
input.approvedBy.trim() !== input.approvedBy ||
|
||||
input.approvedBy.length < 3 ||
|
||||
input.approvedBy.length > 128 ||
|
||||
/[\u0000-\u001f\u007f]/u.test(input.approvedBy)
|
||||
) {
|
||||
throw new TypeError('Manual Primary canary approval is invalid');
|
||||
}
|
||||
const manifest: EnabledRuntimeRolloutManifest = {
|
||||
schemaVersion: 2,
|
||||
revision: `manual-primary-${plan.sessionId}`,
|
||||
enabled: true,
|
||||
approvedBy: input.approvedBy,
|
||||
approvedAtMs,
|
||||
expiresAtMs: approvedAtMs + approvalMs,
|
||||
rollbackPlanRef: plan.files.plan,
|
||||
primaryGate: {
|
||||
schema: 'qinglong/legacy-shadow-primary-gate-reference@v1',
|
||||
origin: 'manual',
|
||||
receiptFile: plan.files.primaryGate,
|
||||
receiptSha256: qualification.primaryGateFileSha256,
|
||||
},
|
||||
rollout: {
|
||||
defaultMode: 'off',
|
||||
origins: { manual: 'primary' },
|
||||
allowLegacyFallbackBeforeStart: false,
|
||||
},
|
||||
gates: Object.fromEntries(
|
||||
REQUIRED_RUNTIME_ROLLOUT_GATES.map((gate) => [gate, 'passed']),
|
||||
) as EnabledRuntimeRolloutManifest['gates'],
|
||||
};
|
||||
parseRuntimeRolloutManifest(manifest, approvedAtMs);
|
||||
return Object.freeze(manifest);
|
||||
}
|
||||
|
||||
export function createManualPrimaryCanaryDisabledManifest(
|
||||
rawSessionId: string,
|
||||
): DisabledRuntimeRolloutManifest {
|
||||
const manifest: DisabledRuntimeRolloutManifest = {
|
||||
schemaVersion: 2,
|
||||
revision: `manual-primary-${sessionId(rawSessionId)}-rollback`,
|
||||
enabled: false,
|
||||
};
|
||||
parseRuntimeRolloutManifest(manifest, 0);
|
||||
return Object.freeze(manifest);
|
||||
}
|
||||
@@ -11,6 +11,22 @@
|
||||
|
||||
最新增量证据（2026-08-19）：
|
||||
|
||||
- D-361/ADR-0454（已接受；首次真实用户目标实例执行待运维）：把 D-360 的 manual Primary evidence/gate 收敛为一次显式、可重放且不新增常驻组件的目标实例仪式。
|
||||
`prepare → observe → resource → qualify → approve → status/audit → rollback` 固定为七个 operator-driven 阶段；Edge 精确 8 条，Standalone 由维护者在
|
||||
32–128 条内选择精确目标。prepare 绑定当时 absent/disabled rollout 基线并只输出 `QL_DEPLOYMENT_PROFILE`、`QL3_SHADOW_ORIGINS=manual` 和唯一
|
||||
capture basename，不触发任务、重启或自动启用；observe 固定复用五分钟闭合窗口只读审计，resource 固定执行 compiled-backend full rollback。qualification 同时绑定
|
||||
plan、三份原文件 SHA-256、canonical evidence SHA-256 与 Primary gate；approve 是唯一写 live manifest 的动作，显式审批最长 24 小时，并在最终写入前复核基线摘要。
|
||||
写入结果只称 `activation_approved`/`primary_selected`，强制返回 `requiresRestart=true`、`runtimeActivationObserved=false`，不能冒充现有 bootstrap 已完成
|
||||
`selected → reconciled → activated`。rollback 以 intent→摘要复核→disabled 原子替换→completion 链支持 response-loss 重放；审批过期立即按 off 解释，但仍要求显式
|
||||
rollback 收敛磁盘事实。config root/证据要求当前 UID、非 symlink、group/world 不可写及 `0600` no-replace；路由设备不增加 daemon、watcher、timer、连接、package、
|
||||
生产依赖、schema、migration 或部署对象。真实 compiled backend 已完成一次 Edge `prepare → resource`，只证明仪式固定 child/资源回滚入口可执行，不冒充真实产品入口的
|
||||
8 条 manual admission、运行态 activation、物理路由、flash 或断电证据。阶段门已重跑：聚焦 `24/24`、`build:back`、完整 backend
|
||||
`1,481 pass / 0 fail / 2 conditional skip`、18-package clean build/test、四项可执行架构审计与 `14/14` artifact audit 全部通过；14 档产物字节与
|
||||
D-360 完全一致。使用隔离 frozen-lockfile Linux 依赖重新执行 arm64 Docker 门后，router stress 保持 `128 MiB / 0.5 CPU / 0 swap / 64 PID`，cgroup peak
|
||||
`81,887,232` bytes；Edge release 保持 `256 MiB / 1 CPU / 0 swap / 128 PID`，13 个 workload 全绿、peak `148,840,448` bytes，两档
|
||||
`memory.events max/oom/oom_kill` 增量均为 0。D-361 不改 PostgreSQL schema/migration、依赖树或 Kubernetes 拓扑，故不重跑 PostgreSQL HA；相邻
|
||||
D-359 的 `142/142` 与 timeline `1→2` 仅作为既有证据。
|
||||
|
||||
- D-360/ADR-0453（已接受；首次真实目标实例 manual canary bundle 待执行）：不再用 2.x RunningInstance 推测 Legacy→Shadow 分母，也不再让 Primary
|
||||
manifest 只信任维护者填写的 `"passed"`。默认 Legacy bridge 为每个已启用 origin admission 分配 process-epoch token，固定守恒
|
||||
`captured + failed + pending = admitted`，失败只分 `fact/observer/initialization/accept`；只有真实 Shadow writer accept 才结算 captured，Legacy
|
||||
@@ -8307,13 +8323,19 @@ Primary 不使用普通环境变量或宽泛全局开关启用。孵化配置面
|
||||
|
||||
- 文件缺失、不可读、超过 64 KiB、JSON 损坏、未知字段、过期审批或 gate 不完整时 fail-closed 为 `off`。
|
||||
- `defaultMode` 必须保持 `off`；当前 manifest 只允许声明 `manual`，不得用一个配置隐式接管 boot、定时、gRPC 或其他来源。
|
||||
- 启用记录必须包含有界 revision、审批人、审批起止时间和 rollback plan 引用；审批窗口最长 30 天。
|
||||
- 启用记录必须包含有界 revision、审批人、审批起止时间和 rollback plan 引用；通用 loader 上限为 30 天，ADR-0454 的目标实例 manual canary 仪式进一步收紧为最长 24 小时。
|
||||
- `durableCancellation`、`startupReconciliation`、`atomicLegacyProjection`、`rollbackDrill`、`edgeBudget` 必须全部为 `passed`；从 schema v2 起这些声明不能
|
||||
代替 Primary evidence bundle。
|
||||
- enabled manifest 必须绑定同 config 目录的 `qinglong/legacy-shadow-primary-gate@v1` bundle basename 与 SHA-256。loader 必须 no-follow 读取、重算 embedded
|
||||
capture/startup、terminal、resource source digest 和完整 eligibility，并复验 manual origin、实际 Profile 与审批时序；只验证 receipt 自称 eligible 不成立。
|
||||
- 审计只记录路径、revision、判定、时间和源文件 SHA-256，不记录完整配置内容；接受判定必须在安装 owner router 前可观测，安装后审计失败必须撤销 router。
|
||||
- edge 不启动文件 watcher。配置只在显式 bootstrap/reload 时读取，禁用时不创建 router、timer、连接或后台任务。
|
||||
- ADR-0454 的 `prepare` 先以不可变 plan 绑定 Profile、精确 admission 目标和当时 absent/disabled rollout SHA-256；`qualify` 再绑定 capture/terminal/resource 的原始文件摘要、
|
||||
canonical 摘要和 Primary gate。只有显式 `approve` 能在复验 live 基线后写入本节 schema v2 manifest。
|
||||
- `approve` 的 durable 结果是 `primary_selected`，不是 `primary_active`。operator 必须重启目标 worker，并从现有 bootstrap 的同 revision `selected → reconciled → activated`
|
||||
结构化审计证明实际运行态；一次性独立 auditor 刻意不输出 active 结论。
|
||||
- `rollback` 在 live manifest 替换前先写绑定 enabled/disabled 摘要的 intent，替换后再写绑定 intent 摘要的 completion；同参数重放覆盖两个 response-loss 窗口。审批过期按 off
|
||||
解释但不会自动改写文件，仍需显式 rollback 和重启。
|
||||
|
||||
实验 manifest 结构固定为：
|
||||
|
||||
@@ -8352,8 +8374,8 @@ Primary 不使用普通环境变量或宽泛全局开关启用。孵化配置面
|
||||
的 manual primary 配置才惰性加载真实组件。激活顺序固定为：重放 evidence→复验实际 Profile→记录 selected 审计→完整有界 startup reconciliation→记录
|
||||
reconciled 审计→启动 completion/timeout/cancel lifecycle→安装 router→记录 activated；任何一步失败都会撤销 router，并按 producer → consumer 顺序停止
|
||||
lifecycle。HTTP shutdown 与监听失败也会执行有界清理。`QL_DEPLOYMENT_PROFILE` 未配置时为 standalone，非法值、receipt Profile 不一致或在
|
||||
cluster-control/worker 中误装本机 SQLite Primary 时 fail closed。首次目标实例 manual canary、固定物理 edge Gate、配置写入/用户可见状态和操作回滚演练完成后
|
||||
仍需单独评审。
|
||||
cluster-control/worker 中误装本机 SQLite Primary 时 fail closed。目标实例仪式、部署配置写入、只读选择状态和操作回滚工具已由 ADR-0454 固化；首次真实 Edge/Standalone
|
||||
完整执行、bootstrap activated durable receipt、固定物理 edge/flash/断电 Gate 完成后仍需单独评审。
|
||||
|
||||
## 26. 交付阶段
|
||||
|
||||
@@ -8995,6 +9017,7 @@ flowchart LR
|
||||
- 旧 Crontab 状态仍为用户可见事实源。
|
||||
- 增加旧状态与 Shadow Run 对账日志和指标。
|
||||
- 发现差异时只报告，不改变任务结果。
|
||||
- 目标实例 canary 必须由不可变计划固定 Profile 和精确样本数，并通过真实产品 manual 入口执行；仪式不得代替用户发起任务或自动启用 Primary。
|
||||
|
||||
#### PR-5：LocalExecutor 切换
|
||||
|
||||
@@ -9006,6 +9029,7 @@ flowchart LR
|
||||
- 取消必须先原子写入 `run.cancel_requested`，再调用 Executor.stop；重复取消保持单事件，晚到 success 不覆盖已接受取消。
|
||||
- 验证 task_before、task_after、work_dir、log_name 和多实例行为。
|
||||
- Primary completion 只能写安全错误分类，不得把 Executor 原始错误、命令、环境或路径持久化到 error_summary。
|
||||
- rollout 配置选择与运行态激活必须分离：短期审批只产生 `primary_selected`，重启后的 reconciliation/lifecycle/router 审计才可证明 activated；回滚必须保留可重放 intent/completion 链。
|
||||
|
||||
#### PR-6：只读 v3 Run API
|
||||
|
||||
@@ -9176,8 +9200,8 @@ flowchart LR
|
||||
| PR-1 Run Schema | Incubating | Run/RunAttempt/RunEvent schema、nullable cancel request 与 Attempt deadline 字段及恢复索引、CancellationDispatch 状态/version/lease/backoff schema、Repository port、临时 Sequelize adapter、统一事件大小/分页上限、跨 adapter RunRepository contract suite（原子事务、回滚、Run/Attempt/RetryPolicy CAS、唯一错误、分页与取消恢复）；ADR-0041 的 `pg-0003-run-retry-policy`、capability v2、driver-neutral PostgreSQL Run Repository 与真实 `pg.Pool` 上的共享 Repository/rollback/SQLSTATE contract；ADR-0063/0069/0071/0073/0074/0076 的独立 Node 24 local-sqlite typed schema、十二条 reviewed migration、capability v6、共享 operation authority、readiness/RunRepository/API credential repository/receipt journal/dispatch plan/encrypted Secret envelope/Project Policy/security audit/authorized mutation/stable Identity catalog、Drizzle↔真实 catalog table/column/index/CHECK/FK lockstep、base/adopted/application edge/standalone 产物门禁；ADR-0064 的 legacy baseline/plan digest、Online Backup recovery、side-by-side target migration、staged manifest、双库栅栏 activation、source 生命周期写栅栏、target stable identity 和重启语义；ADR-0065 的独立 cutover authority、外部副作用停机 evidence、append-only journal、start/restart/stop barrier 与 unknown→manual_required 收敛；ADR-0066 的 adopted storage→Run reconciliation→receipt maintenance→domain recovery→lifecycle→admission application gate、严格有界 recovery summary 与 admission-first reverse stop；ADR-0067 的 SQLite 事实驱动 Run 候选源、256 条硬上限、截断失败关闭和唯一 Repository authority；ADR-0068 的 receipt-first Reconciler、callback token/sequence fence、exact local-process identity、Attempt/Run/双 Event 原子终态推进和最终 verifier；ADR-0069 的 local-process 单向包边界、pre-spawn journal、受审 POSIX launcher、immutable receipt、exact identity 和 Profile-aware cleanup lifecycle；ADR-0070 的独立 local-execution、spawn 前后双 transaction CAS、callback digest、exact stop 补偿与 fail-closed starting 保留；ADR-0071 的独立 local-dispatch、不可变 revision/context、Secret-first materializer、Profile Artifact admission、4/64 MiB output hard quota 和窄 application facade；ADR-0073/0074 的 Project-bound SecretRef、AES-256-GCM、外置 keyring 生命周期、双 SQLite authority CAS、application preflight、强 Principal/Policy 和 envelope+audit 原子提交；ADR-0086 的本机 Owner provisioning/challenge/claim/delivery acknowledgement/credential recovery CLI；ADR-0377 的 Local/Cluster 同构、Profile-aware、Project-scoped Artifact range read | fresh database/pepper setup、credential rotation/GC 运维编排与 Secret/Project/Role/Approval 管理 CLI/API/UI、备份/rekey、2.x/target process controller、人工 recovery、target 写后 reconciliation 与完整 cutover/rollback 演练；retry 产品策略、Artifact retention/tombstone stack、具体本机 lifecycle 和 target executable；Linux x64/arm64、PID namespace、断电与固定路由设备门禁；PostgreSQL 16/18 双连接并发与 failover integration；Task revision/context 跨方言 contract/并发压力与引用感知 retention、Keyv 数据迁移 |
|
||||
| PR-2 Run 状态机 | Incubating | 纯转换表、终态/时间/错误/执行器元数据规则、Run version 与 event sequence CAS、事务性 RunCommandService、回滚测试 | 重复 Worker callback/fencing、并发数据库压力测试、Primary 执行链接入 |
|
||||
| PR-3 Executor 端口 | Incubating | ADR-0003、ExecutionSpec/Context/Handle/Result、Executor port、LocalProcessExecutor、进程组取消/超时升级、流式背压、Legacy Cron spec builder、真实进程 contract tests、可复现 edge 基准入口 | 固定 edge/多架构设备基线、Legacy builder 与 makeCommand 差异审计、Primary 生产流量接入 |
|
||||
| PR-4 Shadow Run | Incubating | origin 三态策略；默认关闭的 `QL3_SHADOW_ORIGINS`；manual、scheduled_node、boot、subscription、system 与 script 现有 ChildProcess 旁路观察；system crond 显式 origin marker、Shell execution ID、finish-only 准入、确定性 Run/Attempt 与 exact replay；`@once` 保持 manual、gRPC transport 不冒充 origin 的准入裁决；每个 worker 懒加载；Run/Attempt/Event 影子生命周期；稳定且不复制 caller 原文的 task identity/revision 与有界日志引用；同 worker 有界注册表和跨 worker 持久化候选关联；stop all/stop instance、Shell callback、乱序/迟到/歧义处理；监听前一次性、Profile-aware 的 keyset Startup Reconciler，终态证据补齐、lost/abandoned/pending 分流与 terminal Attempt response-loss 修复；origin-bounded 且逐级守恒的版本化 startup difference report、固定字段 metric batch 与一次性 collector；显式、只读、闭合窗口且 Profile-bounded 的 Shadow→Legacy 终态差异审计；128/256 MiB Linux arm64 资源门、SQLite 零增长与 Shadow enabled→off 进程重启回滚；process-epoch Legacy admission/capture/failure/pending 守恒；clean-shutdown `0600` no-replace capture+startup exporter；manual Edge 8/Standalone 32–128 canary；capture/terminal/resource 自包含 Primary bundle；rollout v2 loader 重算 source digest 与 eligibility；失败开放和契约测试 | 首次真实目标实例 manual canary bundle、其他 origin 独立 capture/Primary gate、固定物理 edge/flash/断电证据 |
|
||||
| PR-5 Primary LocalExecutor | Incubating（默认不激活，仅 manifest-gated manual） | runtime-owned Run 创建器；持久化先于 spawn；Run/Attempt 完整成功、失败、取消、超时与 lost 闭环；Executor handle 身份校验；spawn 后激活写失败的 stop+lost 补偿；completion rejection 安全收敛；独立 Primary 幂等查询与唯一索引竞态裁决；durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询；最多 64 条一页的 cross-worker cancellation source；独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent；最多 64 页的单周期 cancel supervisor；显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner；Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller；完整有界分页且 fail-closed 的 startup Reconcile supervisor；RunningInstance nullable `run_id/attempt_id` 关联；Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance，失败整体回滚；有界且防穿越的 legacy log output ref；manual owner seam、真实本机装配、单 spawn/fail-closed；严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计；HTTP worker 已接轻量 lazy bootstrap，accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router 顺序激活，失败撤销，监听失败和 shutdown 有界停止；Primary timeout 在 spawn 前持久化绝对 deadline，有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch；代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限，cluster-control/worker 拒绝误装本机 SQLite Primary；统一 CompletionService 原子提交 Attempt/Run/双 Event，spawn 前保存 callback token hash、终态推进 sequence，实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window；manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理；Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace；`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描，使终态残留继续可发现；确定无效的已知 Attempt receipt 先持久化隔离状态，再进入确定性私有分片 quarantine；终态 missing 与 quarantine 按 edge/standalone retention 有界清理；非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI；扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数；ENOSPC 与 launcher receipt 存储失败有代码门禁 | 部署配置写入/审批入口与用户可见状态；PostgreSQL CancellationDispatch adapter；cluster-control 生产启动拓扑；固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练 |
|
||||
| PR-4 Shadow Run | Incubating | origin 三态策略；默认关闭的 `QL3_SHADOW_ORIGINS`；manual、scheduled_node、boot、subscription、system 与 script 现有 ChildProcess 旁路观察；system crond 显式 origin marker、Shell execution ID、finish-only 准入、确定性 Run/Attempt 与 exact replay；`@once` 保持 manual、gRPC transport 不冒充 origin 的准入裁决；每个 worker 懒加载；Run/Attempt/Event 影子生命周期；稳定且不复制 caller 原文的 task identity/revision 与有界日志引用；同 worker 有界注册表和跨 worker 持久化候选关联；stop all/stop instance、Shell callback、乱序/迟到/歧义处理；监听前一次性、Profile-aware 的 keyset Startup Reconciler，终态证据补齐、lost/abandoned/pending 分流与 terminal Attempt response-loss 修复；origin-bounded 且逐级守恒的版本化 startup difference report、固定字段 metric batch 与一次性 collector；显式、只读、闭合窗口且 Profile-bounded 的 Shadow→Legacy 终态差异审计；128/256 MiB Linux arm64 资源门、SQLite 零增长与 Shadow enabled→off 进程重启回滚；process-epoch Legacy admission/capture/failure/pending 守恒；clean-shutdown `0600` no-replace capture+startup exporter；manual Edge 8/Standalone 32–128 canary；capture/terminal/resource 自包含 Primary bundle；rollout v2 loader 重算 source digest 与 eligibility；不可变 prepare/observe/resource/qualify 目标实例仪式、独立只读 audit；失败开放和契约测试 | 首次真实目标实例完整 canary 与 bootstrap activated 记录、其他 origin 独立 capture/Primary gate、固定物理 edge/flash/断电证据 |
|
||||
| PR-5 Primary LocalExecutor | Incubating（默认不激活，仅 manifest-gated manual） | runtime-owned Run 创建器；持久化先于 spawn；Run/Attempt 完整成功、失败、取消、超时与 lost 闭环；Executor handle 身份校验；spawn 后激活写失败的 stop+lost 补偿；completion rejection 安全收敛；独立 Primary 幂等查询与唯一索引竞态裁决；durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询；最多 64 条一页的 cross-worker cancellation source；独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent；最多 64 页的单周期 cancel supervisor；显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner；Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller；完整有界分页且 fail-closed 的 startup Reconcile supervisor；RunningInstance nullable `run_id/attempt_id` 关联；Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance，失败整体回滚；有界且防穿越的 legacy log output ref；manual owner seam、真实本机装配、单 spawn/fail-closed；严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计；HTTP worker 已接轻量 lazy bootstrap，accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router 顺序激活，失败撤销，监听失败和 shutdown 有界停止；Primary timeout 在 spawn 前持久化绝对 deadline，有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch；代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限，cluster-control/worker 拒绝误装本机 SQLite Primary；统一 CompletionService 原子提交 Attempt/Run/双 Event，spawn 前保存 callback token hash、终态推进 sequence，实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window；manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理；Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace；`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描，使终态残留继续可发现；确定无效的已知 Attempt receipt 先持久化隔离状态，再进入确定性私有分片 quarantine；终态 missing 与 quarantine 按 edge/standalone retention 有界清理；非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI；扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数；ENOSPC 与 launcher receipt 存储失败有代码门禁；显式最长 24 小时 approve 写入、`primary_selected` 只读状态、selection receipt、approval-expiry off 与 intent/completion crash-replay rollback | 首次真实目标实例激活/回滚及 durable runtime activation receipt；PostgreSQL CancellationDispatch adapter；cluster-control 生产启动拓扑；固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练 |
|
||||
| PR-7 Worker Session、Run Lease 与启动协议基础 | Incubating（默认关闭，独立入口显式 opt-in） | ADR-0012/0013/0014/0021/0057–0061/0108–0121/0231–0239/0377；有界 capability/Placement/Dispatcher；SQLite 协议孵化与 PostgreSQL v9 Session/Run Lease/credential/attestation authority；immutable revision Placement、数据库时钟 keyset candidate、认证 Worker Pull、digest-only offer recovery；versioned capability-free ExecutionSpec response、stable claim 跨重启退避、单 owner 原子 inbox 准入与 TLS 1.3 mTLS/`ql3w` HTTPS client；同一 package journal 上 revision-fenced starting/spawn/started/running/completion 状态、callback digest、tagged no-spawn 与 ambiguous recovery；PostgreSQL starting/running/start-failure/completion 数据库权威事务、精确重放与 cancellation/timeout 优先终态；batch Secret delivery 在 Attempt advisory lock 下复验 Session/Lease/revision 完整围栏并复用单 Agent，Secret-before-Artifact materializer 将同一 log ID 交给 Executor/journal/running ACK；offer-scoped `wlog-*` 私有文件 spool、Edge/Node 容量策略、append/quota/path 防护、barrier 后 output ownership、受审 POSIX Executor、truncation fact、固定内存流式 source、认证 Artifact stream、共享 immutable store port、S3-compatible SSE/checksum/条件 promotion adapter、upload-before-completion 协调，以及 Local/Cluster 同构、Profile-aware、ETag-fenced range read；用户取消 run.stop mutation 以数据库时间写 intent/Event 并在事务内复验 Project/RoleBinding fence；非执行取消 convergence lifecycle、运行期 expiry 与安全 lost retry 已接入 cluster-control 单一全局 cadence；完整 generation/version/token/Attempt fencing；独立最小权限 Worker ingress、CA/CRL 与连接 generation 热重载；offer journal、spawn barrier、receipt-first recovery；独立 `@qinglong/worker-runtime` 的本地 P-256 CSR、key/chain/trust 验证、generation + active pointer 安装和持久退避；默认关闭的 production process 已装配具体 execution graph、完整 Session heartbeat/drain/offline、direct-file bootstrap、单 Agent/单 cadence、startup reconciliation、证书 maintenance、transport fail-close/recovery 与 Edge/Node 有界预算；真实 PostgreSQL 18 + Linux Node 合约已覆盖 Run completion、credential 和 CA 双轮换且保持同一 Session；真实 K3s 合约已覆盖 TLS/credential Secret 分权、双对象 CAS、Recreate 顺序、identity generation 与单节点 PVC recovery；所有能力默认不可达且受 edge/cluster import audit 约束 | 具体 cert-manager/Vault/SPIFFE/离线 CA adapter 与模板、ingress reload controller、生产 RBAC、证书到期告警和 `ql3w` credential recovery 产品面；具体 KMS/Vault Secret provider、对象存储 credential/temporary lifecycle 与 retention/tombstone；Worker 管理 API；真实 Kubernetes 多节点 CSI/node-loss/production 360 秒 drain 与固定 edge 文件系统 suspend/时钟/断电、x64/arm64 资源门禁 |
|
||||
| PR-8 Project/Policy/Approval Core | Incubating（默认拒绝、无生产业务执行入口） | ADR-0028；统一六类 ActorRef 与 exact-shape 校验；`0017` ownerless default Project 和 append-only versioned RoleBinding；owner/admin/operator/viewer 固定矩阵；Project 内 mutation 幂等、expected-version CAS、双 SQLite 连接竞争门禁；archived read-only、revocation、存储损坏 fail-closed；Agent 写/Secret/Tool `require_approval`；ADR-0047 把六类 subject、role/permission matrix 与 fence 抽到 runtime-core，`pg-0004-project-policy`/capability v3 建立 ownerless PostgreSQL baseline、严格 role/state CHECK、append-only runtime 权限、SERIALIZABLE Project lock、mutation replay、双连接单 winner 和 cluster admission authorizer；ADR-0049/`pg-0005` capability v4 建立 stable IdentitySubject、append-only digest-only API credential、真实 cluster bearer authenticator、write-only durable security audit 与最小权限 runtime role，且已验证 HTTP→credential→Policy→audit→handler 纵向链路；ADR-0051 建立 `/api/v3` 认证前 peer/global 双预算、transport-peer-only、无 timer 且有界内存的 overload shield；ADR-0027 Artifact authorizer adapter；ADR-0029 `AuthenticatedPrincipal` contract、`0018` digest-only versioned challenge、CSPRNG/TTL、同事务消费 challenge + 写首 owner、精确重放与双连接竞争/崩溃回滚门禁；ADR-0030 `0019` stable identity/binding、legacy HS384 + current-session membership、logout/platform/revoke/disable、single-factor 与损坏 fail-closed 门禁；ADR-0031 `0020` digest-bound ApprovalRequest、User-only decision、Project/Role version fence、精确 expiry/重放/并发裁决及同事务 immutable dispatch；ADR-0032 `0021` execution backfill、三表原子 consume、稳定 due keyset、claim/renew/start/result fencing、pre-start takeover/post-start recovery-required、attempt budget、handler inspect/digest barrier 和 bounded dispatcher；ADR-0033/`0022` control/resolution backfill、start/renew/completion 原子联动、稳定 recovery keyset、双 resolver claim/takeover、finding/result 精确重放、自动/人工终结、迟到 completion 单 winner 和 evidence-only bounded reconciler；ADR-0034/`0023` 首个 `run.create` canonical plan、Run/Attempt/Event/receipt 同事务、幂等 collision fail-closed、renew/终态 fence、真实 SQLite handler 与 automatic evidence provider；ADR-0035/`0024` 独立 `approval.recover` 矩阵、稳定 User + 五分钟强认证、Project/RoleBinding fence、human resolution + authorization fact 原子提交、撤权竞态与回滚门禁；ADR-0036 recovery-first 单 timer lifecycle、edge/standalone 独立 cadence/页预算、跨周期 cursor、非重叠与有界 stop；ADR-0074 以新的 Node 24 SQLite v5 ownerless Project/RoleBinding/audit authority 和独立 local-secret-admin 提供强 Principal、`secret.manage`、撤权 fence、envelope+allowed audit 原子提交及不回显语义；ADR-0086 以可信 POSIX console 和 staged delivery 完成本机首 Owner 产品 ceremony | fresh database/pepper setup 与安全迁移向导；`shareStore`/Express 到 authentication core 的 production migration；credential rotation/revocation API、mTLS/Worker enrollment、恢复码；Project/Role/Approval/Secret 管理 CLI/API/UI、audit retention/query/export/alert、preview Artifact/digest/immutable plan builder、真实 MFA/hardware adapter、人工 recovery API/UI/独立 rate limit 与审计事件、handler/provider registry、lifecycle startup/shutdown/指标/admission gate；PostgreSQL action/receipt/provider/recovery-authorization 与 OPA adapter、缓存 version 失效；Tool/Package/Secret/Shell 各自的 handler/evidence contract；Secret/Run/Tool/Workflow waiting_approval 全入口装配；完整回滚演练 |
|
||||
|
||||
|
||||
@@ -170,10 +170,13 @@ owner 在接受触发时写入执行上下文，并贯穿日志、指标和回
|
||||
- `defaultMode` 固定为 off，当前只允许 `origins.manual`；Primary 不得通过环境变量、通配 origin 或隐式默认值开启。
|
||||
- enabled manifest 必须记录 revision、approvedBy、approvedAtMs、expiresAtMs 和 rollbackPlanRef；审批窗口最长 30 天。
|
||||
- `durableCancellation`、`startupReconciliation`、`atomicLegacyProjection`、`rollbackDrill`、`edgeBudget` 必须全部为 passed。
|
||||
- ADR-0453 后 enabled manifest 还必须绑定可由 loader 独立重算的 manual capture/terminal/resource Primary gate bundle；ADR-0454 的一次性目标实例仪式进一步绑定
|
||||
exact Profile/admission 计划、原始文件摘要、短期审批、selection receipt 与 rollback intent/completion。配置选择只表示 `primary_selected`，不能冒充当前 worker
|
||||
已经完成 `selected → reconciled → activated`。
|
||||
- 审计只包含 source path/hash、revision、时间和稳定判定。接受审计先于 router 安装；安装后审计失败立即调用 disposer，恢复原 owner。
|
||||
- edge 不启动 watcher；当前 bootstrap 只在 HTTP worker 启动时读取一次，未来显式 reload 必须复用同一校验和审计边界。
|
||||
|
||||
bootstrap 接入不代表 Primary 已默认开放。文件缺失、disabled、rejected 或 `manual` 非 primary 时保持 Legacy，且不加载完整 Runtime stack、不创建 router 或 timer。只有 accepted 且全部 gate 通过的 manifest 才按 startup reconciliation、cancel lifecycle、router 的顺序激活；任一步失败都会撤销 router 并停止 lifecycle。ADR-0007 定义的 completion/log supervisor、固定 edge 基准、部署配置写入与用户可见状态、操作回滚演练仍需再次评审。
|
||||
bootstrap 接入不代表 Primary 已默认开放。文件缺失、disabled、rejected 或 `manual` 非 primary 时保持 Legacy，且不加载完整 Runtime stack、不创建 router 或 timer。只有 accepted 且全部 gate 通过的 manifest 才按 startup reconciliation、cancel lifecycle、router 的顺序激活；任一步失败都会撤销 router 并停止 lifecycle。ADR-0454 已提供部署配置写入、只读选择状态与操作回滚仪式，但首次真实目标实例执行、运行态 durable activation receipt、固定 edge 基准和 ADR-0007 的完整实机恢复仍需继续评审。
|
||||
|
||||
## 8. Legacy 到 Run 的身份映射
|
||||
|
||||
|
||||
@@ -95,6 +95,7 @@ fail-closed 还会改变 Legacy 可用性。
|
||||
|
||||
## 后续
|
||||
|
||||
正式启用 manual Primary 前，维护者必须在目标 edge/standalone 实例完成一次真实、干净关闭的 manual canary，等待 settling 后运行 terminal audit，组合与同版本
|
||||
resource report 生成 bundle，再由 v2 loader 重放。固定物理路由、flash 写放大、断电、非干净退出和 config-root 签名/备份仍是独立发布证据；其他 origin 必须分别建立
|
||||
自己的 admission authority、样本预算和 rollback gate，不能复用 manual receipt。
|
||||
ADR-0454 已把目标实例的 prepare、observe、resource、qualify、显式短期 approve、只读 audit 与 crash-replay rollback 固化为一次性状态机，并明确
|
||||
`primary_selected` 不等于运行态 activated。正式启用 manual Primary 前，维护者仍必须在目标 edge/standalone 实例实际执行该仪式并保留 bootstrap activated audit；
|
||||
仓库内 synthetic fixture 或 compiled resource child 不能替代真实产品入口的八/三十二条 admission。固定物理路由、flash 写放大、断电、非干净退出和 config-root
|
||||
签名/备份仍是独立发布证据；其他 origin 必须分别建立自己的 admission authority、样本预算和 rollback gate，不能复用 manual receipt。
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
# ADR-0454：目标实例 Manual Primary Canary 与显式回滚仪式
|
||||
|
||||
- 状态：Accepted（仪式实现完成；首次真实用户目标实例执行仍待运维）
|
||||
- 日期：2026-08-19
|
||||
- 关联 RFC：QL-RFC-0001 D-361、PR-4、PR-5
|
||||
- 关联 ADR：ADR-0002、ADR-0449、ADR-0450、ADR-0451、ADR-0453
|
||||
- Amends：ADR-0453 的首次目标实例 canary 操作缺口
|
||||
|
||||
## 上下文
|
||||
|
||||
ADR-0453 已提供 process-epoch capture authority、clean-shutdown evidence、closed terminal audit、compiled-backend rollback/resource report、Primary gate bundle 与
|
||||
rollout manifest v2 loader，但维护者仍需手工拼接多个命令、文件名、时间窗口和摘要。只提供底层 gate 会留下四类运维缺口：
|
||||
|
||||
1. canary 开始前没有不可变计划绑定 Profile、精确样本数和当时的 live rollout 基线；
|
||||
2. terminal/resource/gate 输出容易经 shell 重定向得到宽权限、部分写入或互相不属于同一 session 的文件；
|
||||
3. qualification 与写入 live manifest 之间没有显式人工边界，配置选择又容易被误报成“当前进程已经激活”；
|
||||
4. rollback 没有绑定被替换 manifest 的摘要，也没有 crash-replay intent/completion 链。
|
||||
|
||||
Edge 可能是 128 MiB 路由设备，不能为 canary 新增 daemon、watcher、遥测栈或第二数据库；Standalone 可能运行在集群节点上，但本仪式仍只裁决本机
|
||||
`edge|standalone` Profile，不得借宿主机形态伪装成 `cluster-control|worker` Primary 证据。
|
||||
|
||||
## 决策
|
||||
|
||||
1. 新增 `qinglong/manual-primary-canary-plan@v1`。`prepare` 只接受受控 session ID、`edge|standalone`、Edge 精确 8 条或 Standalone 32–128 条中由
|
||||
operator 选定的精确 admission 数；它记录当前 `qinglong3-rollout.json` 为 absent 或 disabled+SHA-256。已有 enabled manifest 时拒绝准备。
|
||||
2. plan 固定派生全部 basename，不接受调用方提供输出路径。config root 必须是当前 UID 拥有、非 symlink、group/world 不可写的真实目录；所有 artifact 均以
|
||||
`0600`、同目录临时 inode、file fsync、hard-link no-replace 与 directory fsync 发布。重复调用只接受逐字节相同文件。
|
||||
3. `prepare` 只输出三项部署环境：实际 Profile、`QL3_SHADOW_ORIGINS=manual` 与唯一 capture basename；它不写 enabled manifest、不启动任务、不重启服务。
|
||||
operator 必须在隔离窗口中通过既有 Legacy 产品入口精确执行计划数量的 manual admission，并干净关闭同一 worker；任何额外 manual execution 都使样本不相等。
|
||||
4. `observe` 只有在 capture qualified、单一 manual origin、计数精确且窗口结束至少五分钟后，才以固定参数启动 Node 24 terminal auditor。数据库必须为非 symlink、
|
||||
group/world 不可写的普通文件；auditor 只读打开 SQLite。调用方不能注入 origin、window、settling、child script 或 shell。
|
||||
5. `resource` 用固定 `full + require-compiled + 8 samples` 启动 ADR-0451 runner。它使用自身临时 SQLite，不接触生产数据库；低配设备必须在应用停止后运行，不能把
|
||||
与常驻 workload 争用内存后的失败解释为产品回归。
|
||||
6. `qualify` 复用 ADR-0453 gate，但额外绑定 plan 文件摘要、三份 source 的原始文件摘要、三份 canonical JSON 摘要、精确 admission target 与 gate 文件摘要。
|
||||
现有 gate、partial qualification 或 response loss 只能通过原文件重放；source 在 qualification 后被替换时，mutating CLI 与独立 auditor 都失败关闭。
|
||||
7. `approve` 是唯一写 live manifest 的操作。它要求显式 `approvedBy`，审批最短一分钟、最长 24 小时；先复核 plan 时的 absent/disabled 基线，disabled 基线先
|
||||
no-replace 归档，再以同目录 fsync 临时文件和最终摘要复核发布 schema v2 manifest。结果状态固定为 `activation_approved`/`primary_selected`，并明确
|
||||
`requiresRestart=true`、`runtimeActivationObserved=false`；配置选择不得冒充 HTTP worker 已完成 startup reconciliation 和 router activation。
|
||||
8. 应用重启后，实际运行态仍由现有 bootstrap 的 `selected → reconciled → activated` audit 证明。独立 `audit:manual-primary-canary:ql3` 只读复核 plan、source、gate、
|
||||
qualification、selection receipt 与 live manifest，可要求 `prepared|qualified|selected|off|rolled-back`；`off` 只证明 loader 当前关闭，`rolled-back` 还要求本 session 的
|
||||
intent/completion 摘要链完整。auditor 刻意不提供 `primary_active` 结论。
|
||||
9. `rollback` 只接受四个固定 reason 与有界 operator。它先 no-replace 发布 intent，绑定当前 enabled manifest SHA-256 和目标 disabled SHA-256，再复核当前摘要并原子替换
|
||||
live manifest，最后发布 completion；intent 或 completion response loss 可用相同参数重放。disabled 已有效但不同于本 session 的目标时拒绝覆盖。
|
||||
10. 审批过期后，既有 loader 必须立即 fail-closed 为 off；状态/auditor 报 `approvalExpired=true` 和 `rolloutMode=off`。operator 仍应执行显式 rollback，把磁盘事实
|
||||
收敛为 schema v2 disabled manifest，然后重启应用；过期不能被当作自动续期或自动删除 authority。
|
||||
|
||||
## 被拒绝的替代方案
|
||||
|
||||
### 常驻 watcher 自动检测 capture 并启用 Primary
|
||||
|
||||
拒绝。它把一次性证据变成后台控制面，增加路由设备 timer/文件监控成本，并绕过独立人工审批。
|
||||
|
||||
### Canary CLI 自动触发八个用户任务
|
||||
|
||||
拒绝。工具没有用户会话、Task Policy、脚本内容或副作用 authority；synthetic spawn 不能证明真实 Legacy 产品入口的 admission capture。
|
||||
|
||||
### 用 enabled manifest 存在表示 Primary 已运行
|
||||
|
||||
拒绝。manifest 只在下次 bootstrap 被读取；进程可能尚未重启、startup reconciliation 可能失败或审批已经过期。selection 与 runtime activation 必须分开陈述。
|
||||
|
||||
### 原地覆盖证据和 rollout 文件以方便重试
|
||||
|
||||
拒绝。覆盖会丢失冲突和 crash window。证据采用 no-replace；live rollout 只在摘要复核后原子替换，并由 previous/intent/completion 文件保留恢复链。
|
||||
|
||||
## 资源、安全与部署影响
|
||||
|
||||
- 正常 QingLong runtime 不新增 import、timer、watcher、listener、数据库连接、schema、migration、package 或生产依赖。全部动作由 operator 一次性调用。
|
||||
- plan/qualification/selection/rollback 报告不包含数据库路径、命令、Task、Run、Cron、PID、日志、用户输出、错误正文或 Secret；stdout 只返回 stage、Profile、样本数和布尔状态。
|
||||
- `observe` 只读生产 SQLite；`resource` 只使用 runner 的临时 SQLite。Edge 应在应用停止后运行 resource，Standalone 也不得与同机生产 canary 并发争用。
|
||||
- POSIX owner 是本机信任根。摘要复核能防止误覆盖和非协作漂移，但不能防御同一 UID 在最后复核与 rename 之间的恶意并发；共享 config root 或远程 delegation 需要
|
||||
另行引入签名/锁服务，不得把本仪式描述成多写者共识。
|
||||
- 本仪式不适用于 `cluster-control|worker`，也不证明物理路由、flash wear、断电、跨主机签名或真实生产任务安全。
|
||||
|
||||
## 验证
|
||||
|
||||
- domain/CLI 聚焦矩阵覆盖 Edge/Standalone 样本预算、exact shape、短期审批、prepare 重放、unsafe root、disabled baseline drift、source replacement、partial
|
||||
qualification、selection receipt response loss、审批过期、rollback intent/completion response loss及独立审计。
|
||||
- 真实 compiled backend 已通过 D361 `prepare → resource`：Edge plan 保持 automatic activation false；resource 阶段产生 qualified full rollback evidence，
|
||||
enabled/off 进程 peak RSS 分别为 `109,953,024 / 102,383,616` bytes，真实 Legacy child 均 exit 0，Shadow Run delta 从 1 收敛为 0。
|
||||
- 阶段门已重跑：聚焦 `24/24`、`build:back`、完整 backend `1,481 pass / 0 fail / 2 conditional skip`、18-package clean build/test、四项架构审计与
|
||||
`14/14` artifact audit 全部通过；产物字节与 D-360 一致。隔离 frozen-lockfile Linux arm64 Docker 的 128 MiB router/256 MiB Edge release peak 分别为
|
||||
`81,887,232 / 148,840,448` bytes，`memory.events max/oom/oom_kill` 增量均为 0。
|
||||
- D361 不修改 PostgreSQL schema/migration、依赖树或 Kubernetes 拓扑，因此不重跑 PostgreSQL HA；任何仓库 synthetic fixture、compiled resource child 或 Docker
|
||||
arm64 结果都不沿用、也不冒充尚未执行的真实目标实例 manual capture/activation。
|
||||
|
||||
## 后续
|
||||
|
||||
维护者仍需在一台真实目标 Edge 或 Standalone 实例执行 [Manual Primary Canary 操作手册](../operations/ql3-manual-primary-canary.md)，并保留 capture、terminal、resource、
|
||||
gate、qualification、selection、bootstrap activated audit 与 rollback completion。其他 origin 必须建立自己的 admission authority 与 gate；固定物理路由/flash/断电、
|
||||
运行态 durable activation receipt 和多写者 config authority 仍是独立后续工作。
|
||||
@@ -457,6 +457,7 @@
|
||||
| [ADR-0451](./ADR-0451-profile-bounded-legacy-shadow-resource-and-off-rollback-evidence.md) | 按 Profile 有界的 Legacy Shadow 资源与关闭回滚证据 | Accepted |
|
||||
| [ADR-0452](./ADR-0452-atomic-flattened-backend-build-publication.md) | 原子且扁平兼容的 Backend 构建发布 | Accepted |
|
||||
| [ADR-0453](./ADR-0453-origin-scoped-legacy-shadow-capture-authority-and-primary-gate.md) | Origin-scoped Legacy Shadow 捕获权威与 Primary 门禁 | Accepted（首次真实目标实例 manual canary 待执行） |
|
||||
| [ADR-0454](./ADR-0454-target-instance-manual-primary-canary-ceremony.md) | 目标实例 Manual Primary Canary 与显式回滚仪式 | Accepted（首次真实用户目标实例执行待运维） |
|
||||
|
||||
## 规则
|
||||
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
# QingLong 3.0 Manual Primary Canary 操作手册
|
||||
|
||||
本流程只适用于本机 `edge` 或 `standalone` Profile 的 `manual` origin。它不会自动执行用户任务，也不会在 `prepare`、`observe`、`resource` 或 `qualify` 阶段启用
|
||||
Primary。`cluster-control` 与 `worker` 不适用。
|
||||
|
||||
示例使用：
|
||||
|
||||
```text
|
||||
CONFIG_ROOT=/ql/data/config
|
||||
DATABASE=/ql/data/db/database.sqlite
|
||||
SESSION=edge-20260819-a
|
||||
```
|
||||
|
||||
把示例中的绝对路径和 session 替换为目标实例的实际值。config root 必须由运行 QingLong 的同一 UID 拥有，不能是 symlink，也不能允许 group/world 写入；数据库必须是
|
||||
非 symlink、group/world 不可写的普通文件。所有命令都应由该 UID 执行。
|
||||
|
||||
## 1. 准备不可变计划
|
||||
|
||||
Edge 必须精确 8 条；Standalone 可在 32–128 中选择一个精确目标：
|
||||
|
||||
```sh
|
||||
pnpm canary:manual-primary:ql3 -- \
|
||||
--mode=prepare \
|
||||
--root=/ql/data/config \
|
||||
--session=edge-20260819-a \
|
||||
--profile=edge \
|
||||
--admissions=8
|
||||
```
|
||||
|
||||
已有 enabled rollout 时命令拒绝执行；已有 disabled rollout 时 plan 绑定其 SHA-256。输出中的 `automaticActivation` 必须为 `false`，并给出三项环境值。将这些值写入目标
|
||||
部署配置后重启当前 Shadow worker：
|
||||
|
||||
```text
|
||||
QL_DEPLOYMENT_PROFILE=edge
|
||||
QL3_SHADOW_ORIGINS=manual
|
||||
QL3_SHADOW_CAPTURE_EVIDENCE_FILE=ql3-primary-canary-edge-20260819-a.capture.json
|
||||
```
|
||||
|
||||
不要同时配置其他 Shadow origin，不要手工创建 capture 文件。
|
||||
|
||||
## 2. 执行真实 Legacy manual 样本并干净关闭
|
||||
|
||||
在隔离维护窗口中，通过现有 QingLong 用户界面/API 的正常 manual 执行入口精确提交计划数量的任务。不要用 canary 工具直接 spawn 脚本；那不会证明产品入口。
|
||||
|
||||
窗口期间禁止其他 manual execution。任务全部终态后，使用部署系统的正常 shutdown 停止同一 HTTP worker。只有干净 shutdown 才会 no-replace 写 capture evidence；kill -9、断电、
|
||||
重复文件名或部分文件都不具备资格。
|
||||
|
||||
查看当前持久状态不会修改文件：
|
||||
|
||||
```sh
|
||||
pnpm canary:manual-primary:ql3 -- \
|
||||
--mode=status \
|
||||
--root=/ql/data/config \
|
||||
--session=edge-20260819-a
|
||||
```
|
||||
|
||||
## 3. 等待闭合窗口并运行终态审计
|
||||
|
||||
从 capture window 的 `endExclusiveMs` 起等待至少五分钟，然后运行：
|
||||
|
||||
```sh
|
||||
pnpm canary:manual-primary:ql3 -- \
|
||||
--mode=observe \
|
||||
--root=/ql/data/config \
|
||||
--session=edge-20260819-a \
|
||||
--database=/ql/data/db/database.sqlite
|
||||
```
|
||||
|
||||
工具固定使用 `origin=manual`、plan 内的 window 和五分钟 settling，不接受调用方覆盖。SQLite 以只读方式打开；结果必须为 `terminal_observed`、`assessment=matched`，scanned
|
||||
必须等于计划样本数。
|
||||
|
||||
## 4. 在应用停止状态运行资源/回滚证据
|
||||
|
||||
低配路由设备必须保持应用停止，避免 128 MiB 预算内与常驻进程争用。命令使用临时 SQLite、compiled backend、full rollback 和固定 8 个 audit samples，不接触生产数据库：
|
||||
|
||||
```sh
|
||||
pnpm canary:manual-primary:ql3 -- \
|
||||
--mode=resource \
|
||||
--root=/ql/data/config \
|
||||
--session=edge-20260819-a
|
||||
```
|
||||
|
||||
结果必须为 `resource_proven`、`qualified=true`。Standalone 即使部署在集群节点上，也仍是本机证据，不得据此启用 cluster-control/worker Primary。
|
||||
|
||||
## 5. 生成资格并独立复核
|
||||
|
||||
```sh
|
||||
pnpm canary:manual-primary:ql3 -- \
|
||||
--mode=qualify \
|
||||
--root=/ql/data/config \
|
||||
--session=edge-20260819-a
|
||||
|
||||
pnpm audit:manual-primary-canary:ql3 -- \
|
||||
--root=/ql/data/config \
|
||||
--session=edge-20260819-a \
|
||||
--require=qualified
|
||||
```
|
||||
|
||||
`qualify` 生成 Primary gate 与 qualification，但输出仍必须为 `automaticActivation=false`。独立 audit 重新计算 source/gate/file digest；`compatible=true` 只证明可以提交人工审批。
|
||||
|
||||
## 6. 显式短期审批并重启验证
|
||||
|
||||
只有维护者完成审阅后才执行。示例审批一小时，允许范围为一分钟至 24 小时：
|
||||
|
||||
```sh
|
||||
pnpm canary:manual-primary:ql3 -- \
|
||||
--mode=approve \
|
||||
--root=/ql/data/config \
|
||||
--session=edge-20260819-a \
|
||||
--approved-by=operator:local-owner \
|
||||
--approval-ms=3600000
|
||||
|
||||
pnpm audit:manual-primary-canary:ql3 -- \
|
||||
--root=/ql/data/config \
|
||||
--session=edge-20260819-a \
|
||||
--require=selected
|
||||
```
|
||||
|
||||
正确状态是 `activation_approved`/`rolloutMode=primary_selected`，不是 `primary_active`；`requiresRestart=true` 且 `runtimeActivationObserved=false`。随后重启应用，并在结构化启动审计中依次确认
|
||||
同一 revision 的 `selected`、`reconciled`、`activated`。缺少任一项都不能宣称运行态 Primary 已激活。
|
||||
|
||||
审批过期后 loader 自动 fail-closed 为 off，不会续期。不要修改原 manifest 时间；创建新 session 重新采样。
|
||||
|
||||
## 7. 回滚并重启
|
||||
|
||||
演练或出现异常时立即执行：
|
||||
|
||||
```sh
|
||||
pnpm canary:manual-primary:ql3 -- \
|
||||
--mode=rollback \
|
||||
--root=/ql/data/config \
|
||||
--session=edge-20260819-a \
|
||||
--operator=operator:local-owner \
|
||||
--reason=operator_request
|
||||
|
||||
pnpm audit:manual-primary-canary:ql3 -- \
|
||||
--root=/ql/data/config \
|
||||
--session=edge-20260819-a \
|
||||
--require=rolled-back
|
||||
```
|
||||
|
||||
支持的 reason 只有 `operator_request`、`runtime_failure`、`gate_rejected`、`approval_expired`。rollback 先发布 intent，再摘要复核并原子替换 live manifest，最后发布 completion；响应丢失时使用完全相同的参数重跑。
|
||||
|
||||
`rolled-back` 比普通 `off` 更严格：后者在初始 disabled 或审批过期时也成立，前者还要求本 session 的 intent/completion 摘要链完整。完成后重启应用，确认 rollout loader 返回
|
||||
disabled/off，Legacy manual 执行继续可用且 Shadow/Primary 不再接管。保留整个 session 的 `0600` 文件和启动审计用于发布复核，不要覆盖或编辑。
|
||||
@@ -76,6 +76,8 @@
|
||||
"audit:receipts:ql3": "node scripts/ql3-receipt-audit.cjs",
|
||||
"audit:legacy-shadow-terminal:ql3": "node scripts/ql3-legacy-shadow-terminal-audit.cjs",
|
||||
"gate:legacy-shadow-primary:ql3": "node scripts/ql3-legacy-shadow-primary-gate.cjs",
|
||||
"canary:manual-primary:ql3": "node scripts/ql3-manual-primary-canary.cjs",
|
||||
"audit:manual-primary-canary:ql3": "node scripts/ql3-manual-primary-canary-audit.cjs",
|
||||
"audit:edge-imports:ql3": "node scripts/ql3-edge-import-audit.cjs",
|
||||
"audit:cluster-dependencies:ql3": "node scripts/ql3-cluster-dependency-audit.cjs",
|
||||
"audit:package-boundaries:ql3": "node scripts/ql3-package-boundary-audit.cjs",
|
||||
|
||||
@@ -0,0 +1,326 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const {
|
||||
legacyShadowPrimaryEvidenceSha256,
|
||||
parseLegacyShadowPrimaryGateReceipt,
|
||||
} = require('../back/runtime/domain/legacyShadowPrimaryGate');
|
||||
const {
|
||||
createManualPrimaryCanaryDisabledManifest,
|
||||
manualPrimaryCanaryFileSet,
|
||||
manualPrimaryCanarySha256,
|
||||
parseManualPrimaryCanaryPlan,
|
||||
parseManualPrimaryCanaryQualification,
|
||||
} = require('../back/runtime/domain/manualPrimaryCanaryCeremony');
|
||||
const {
|
||||
parseRuntimeRolloutManifest,
|
||||
} = require('../back/runtime/domain/runtimeRolloutManifest');
|
||||
const {
|
||||
readPrivateJson,
|
||||
serialized,
|
||||
} = require('./ql3-manual-primary-canary.cjs');
|
||||
|
||||
const REQUIREMENTS = new Set([
|
||||
'prepared',
|
||||
'qualified',
|
||||
'selected',
|
||||
'off',
|
||||
'rolled-back',
|
||||
]);
|
||||
const ROLLBACK_REASONS = new Set([
|
||||
'operator_request',
|
||||
'runtime_failure',
|
||||
'gate_rejected',
|
||||
'approval_expired',
|
||||
]);
|
||||
|
||||
function hasExactKeys(value, expected) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
||||
return false;
|
||||
}
|
||||
const actual = Object.keys(value).sort();
|
||||
const wanted = [...expected].sort();
|
||||
return (
|
||||
actual.length === wanted.length &&
|
||||
actual.every((key, index) => key === wanted[index])
|
||||
);
|
||||
}
|
||||
|
||||
function parseArguments(argv) {
|
||||
const options = { require: 'prepared' };
|
||||
for (const argument of argv) {
|
||||
if (argument === '--') continue;
|
||||
if (argument.startsWith('--root=')) {
|
||||
const value = argument.slice('--root='.length);
|
||||
if (!value) throw new TypeError('--root must not be empty');
|
||||
options.root = path.resolve(value);
|
||||
} else if (argument.startsWith('--session=')) {
|
||||
options.sessionId = argument.slice('--session='.length);
|
||||
} else if (argument.startsWith('--require=')) {
|
||||
options.require = argument.slice('--require='.length);
|
||||
} else {
|
||||
throw new TypeError(`Unsupported argument: ${argument}`);
|
||||
}
|
||||
}
|
||||
if (!options.root || !path.isAbsolute(options.root)) {
|
||||
throw new TypeError('--root must be absolute');
|
||||
}
|
||||
if (!options.sessionId) throw new TypeError('--session is required');
|
||||
if (!REQUIREMENTS.has(options.require)) {
|
||||
throw new TypeError('--require is invalid');
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function regularFile(target) {
|
||||
try {
|
||||
const stat = fs.lstatSync(target);
|
||||
return stat.isFile() && !stat.isSymbolicLink();
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function run(options) {
|
||||
const rootStat = fs.lstatSync(options.root);
|
||||
if (
|
||||
!rootStat.isDirectory() ||
|
||||
rootStat.isSymbolicLink() ||
|
||||
(rootStat.mode & 0o022) !== 0 ||
|
||||
(typeof process.getuid === 'function' && rootStat.uid !== process.getuid())
|
||||
) {
|
||||
throw new TypeError('Canary root is unsafe');
|
||||
}
|
||||
const files = manualPrimaryCanaryFileSet(options.sessionId);
|
||||
const resolve = (name) => path.join(options.root, files[name]);
|
||||
const planRead = readPrivateJson(resolve('plan'), 64 * 1024);
|
||||
const plan = parseManualPrimaryCanaryPlan(planRead.value);
|
||||
if (plan.sessionId !== options.sessionId) {
|
||||
throw new TypeError('Canary plan session mismatch');
|
||||
}
|
||||
const gatePresent = regularFile(resolve('primaryGate'));
|
||||
const qualificationPresent = regularFile(resolve('qualification'));
|
||||
if (gatePresent !== qualificationPresent) {
|
||||
throw new TypeError('Canary qualification is partial');
|
||||
}
|
||||
let eligible = false;
|
||||
if (gatePresent) {
|
||||
const gateRead = readPrivateJson(resolve('primaryGate'), 64 * 1024);
|
||||
const gate = parseLegacyShadowPrimaryGateReceipt(gateRead.value);
|
||||
const qualification = parseManualPrimaryCanaryQualification(
|
||||
readPrivateJson(resolve('qualification'), 64 * 1024).value,
|
||||
);
|
||||
const sources = {
|
||||
capture: readPrivateJson(resolve('capture')),
|
||||
terminal: readPrivateJson(resolve('terminal')),
|
||||
resource: readPrivateJson(resolve('resource')),
|
||||
};
|
||||
if (
|
||||
gate.assessment !== 'eligible' ||
|
||||
gate.profile !== plan.profile ||
|
||||
gate.origin !== 'manual' ||
|
||||
gate.counts.admitted !== plan.admissionTarget ||
|
||||
qualification.sessionId !== plan.sessionId ||
|
||||
qualification.profile !== plan.profile ||
|
||||
qualification.planSha256 !== planRead.sha256 ||
|
||||
qualification.primaryGateFileSha256 !== gateRead.sha256 ||
|
||||
qualification.sourceFileSha256.capture !== sources.capture.sha256 ||
|
||||
qualification.sourceFileSha256.terminal !== sources.terminal.sha256 ||
|
||||
qualification.sourceFileSha256.resource !== sources.resource.sha256 ||
|
||||
qualification.sourceCanonicalSha256.capture !==
|
||||
gate.evidence.captureSha256 ||
|
||||
qualification.sourceCanonicalSha256.terminal !==
|
||||
gate.evidence.terminalSha256 ||
|
||||
qualification.sourceCanonicalSha256.resource !==
|
||||
gate.evidence.resourceSha256 ||
|
||||
qualification.window.startInclusiveMs !== gate.window.startInclusiveMs ||
|
||||
qualification.window.endExclusiveMs !== gate.window.endExclusiveMs ||
|
||||
qualification.counts.admitted !== gate.counts.admitted ||
|
||||
qualification.counts.captured !== gate.counts.captured ||
|
||||
qualification.counts.terminalScanned !== gate.counts.terminalScanned ||
|
||||
qualification.counts.terminalMatched !== gate.counts.terminalMatched ||
|
||||
legacyShadowPrimaryEvidenceSha256(sources.capture.value) !==
|
||||
qualification.sourceCanonicalSha256.capture ||
|
||||
legacyShadowPrimaryEvidenceSha256(sources.terminal.value) !==
|
||||
qualification.sourceCanonicalSha256.terminal ||
|
||||
legacyShadowPrimaryEvidenceSha256(sources.resource.value) !==
|
||||
qualification.sourceCanonicalSha256.resource
|
||||
) {
|
||||
throw new TypeError('Canary qualification drifted');
|
||||
}
|
||||
eligible = true;
|
||||
}
|
||||
const rolloutPath = resolve('rollout');
|
||||
let rolloutMode = 'off';
|
||||
let approvalExpired = false;
|
||||
let rolloutSha256;
|
||||
if (regularFile(rolloutPath)) {
|
||||
const rolloutRead = readPrivateJson(rolloutPath, 64 * 1024);
|
||||
const raw = rolloutRead.value;
|
||||
const now = Date.now();
|
||||
approvalExpired =
|
||||
raw?.enabled === true &&
|
||||
Number.isSafeInteger(raw.expiresAtMs) &&
|
||||
raw.expiresAtMs <= now;
|
||||
const evaluatedAt = approvalExpired ? raw.approvedAtMs : now;
|
||||
const decision = parseRuntimeRolloutManifest(raw, evaluatedAt);
|
||||
rolloutSha256 = rolloutRead.sha256;
|
||||
if (decision.manifest.enabled && !approvalExpired) {
|
||||
if (
|
||||
decision.manifest.revision !== `manual-primary-${plan.sessionId}` ||
|
||||
!eligible ||
|
||||
decision.manifest.primaryGate.receiptSha256 !==
|
||||
readPrivateJson(resolve('primaryGate'), 64 * 1024).sha256
|
||||
) {
|
||||
throw new TypeError('Active rollout is not bound to the canary');
|
||||
}
|
||||
if (!regularFile(resolve('selection'))) {
|
||||
throw new TypeError('Selected rollout has no selection receipt');
|
||||
}
|
||||
const selection = readPrivateJson(resolve('selection'), 64 * 1024).value;
|
||||
if (
|
||||
!hasExactKeys(selection, [
|
||||
'schema',
|
||||
'schemaVersion',
|
||||
'sessionId',
|
||||
'profile',
|
||||
'selectedAtMs',
|
||||
'expiresAtMs',
|
||||
'manifestSha256',
|
||||
'priorRollout',
|
||||
]) ||
|
||||
selection?.schema !== 'qinglong/manual-primary-canary-selection@v1' ||
|
||||
selection.schemaVersion !== 1 ||
|
||||
selection.sessionId !== plan.sessionId ||
|
||||
selection.profile !== plan.profile ||
|
||||
selection.selectedAtMs !== decision.manifest.approvedAtMs ||
|
||||
selection.manifestSha256 !== rolloutRead.sha256 ||
|
||||
selection.expiresAtMs !== decision.manifest.expiresAtMs ||
|
||||
JSON.stringify(selection.priorRollout) !==
|
||||
JSON.stringify(plan.currentRollout)
|
||||
) {
|
||||
throw new TypeError('Selection receipt drifted');
|
||||
}
|
||||
rolloutMode = 'primary_selected';
|
||||
}
|
||||
}
|
||||
const rolledBack = regularFile(resolve('rollbackComplete'));
|
||||
if (rolledBack) {
|
||||
const expectedDisabled = serialized(
|
||||
createManualPrimaryCanaryDisabledManifest(plan.sessionId),
|
||||
);
|
||||
if (
|
||||
!regularFile(rolloutPath) ||
|
||||
readPrivateJson(rolloutPath, 64 * 1024).sha256 !==
|
||||
manualPrimaryCanarySha256(expectedDisabled)
|
||||
) {
|
||||
throw new TypeError('Rollback completion is not effective');
|
||||
}
|
||||
const intentRead = readPrivateJson(resolve('rollbackIntent'), 64 * 1024);
|
||||
const intent = intentRead.value;
|
||||
const completed = readPrivateJson(
|
||||
resolve('rollbackComplete'),
|
||||
64 * 1024,
|
||||
).value;
|
||||
const disabledSha256 = manualPrimaryCanarySha256(expectedDisabled);
|
||||
if (
|
||||
!hasExactKeys(intent, [
|
||||
'schema',
|
||||
'schemaVersion',
|
||||
'sessionId',
|
||||
'profile',
|
||||
'createdAtMs',
|
||||
'operator',
|
||||
'reason',
|
||||
'enabledManifestSha256',
|
||||
'disabledManifestSha256',
|
||||
]) ||
|
||||
intent?.schema !== 'qinglong/manual-primary-canary-rollback-intent@v1' ||
|
||||
intent.schemaVersion !== 1 ||
|
||||
intent.sessionId !== plan.sessionId ||
|
||||
intent.profile !== plan.profile ||
|
||||
typeof intent.operator !== 'string' ||
|
||||
intent.operator.length < 3 ||
|
||||
intent.operator.length > 128 ||
|
||||
/[\u0000-\u001f\u007f]/u.test(intent.operator) ||
|
||||
!ROLLBACK_REASONS.has(intent.reason) ||
|
||||
!Number.isSafeInteger(intent.createdAtMs) ||
|
||||
!/^[a-f0-9]{64}$/u.test(intent.enabledManifestSha256) ||
|
||||
intent.disabledManifestSha256 !== disabledSha256 ||
|
||||
!hasExactKeys(completed, [
|
||||
'schema',
|
||||
'schemaVersion',
|
||||
'sessionId',
|
||||
'profile',
|
||||
'completedAtMs',
|
||||
'intentSha256',
|
||||
'disabledManifestSha256',
|
||||
]) ||
|
||||
completed?.schema !==
|
||||
'qinglong/manual-primary-canary-rollback-complete@v1' ||
|
||||
completed.schemaVersion !== 1 ||
|
||||
completed.sessionId !== plan.sessionId ||
|
||||
completed.profile !== plan.profile ||
|
||||
!Number.isSafeInteger(completed.completedAtMs) ||
|
||||
completed.completedAtMs < intent.createdAtMs ||
|
||||
completed.intentSha256 !== intentRead.sha256 ||
|
||||
completed.disabledManifestSha256 !== disabledSha256
|
||||
) {
|
||||
throw new TypeError('Rollback receipt chain drifted');
|
||||
}
|
||||
rolloutMode = 'off';
|
||||
}
|
||||
const compatible =
|
||||
options.require === 'prepared' ||
|
||||
(options.require === 'qualified' && eligible) ||
|
||||
(options.require === 'selected' && rolloutMode === 'primary_selected') ||
|
||||
(options.require === 'off' && rolloutMode === 'off') ||
|
||||
(options.require === 'rolled-back' && rolloutMode === 'off' && rolledBack);
|
||||
const report = {
|
||||
schema: 'qinglong/manual-primary-canary-audit@v1',
|
||||
schemaVersion: 1,
|
||||
sessionId: plan.sessionId,
|
||||
profile: plan.profile,
|
||||
admissionTarget: plan.admissionTarget,
|
||||
eligible,
|
||||
rolloutMode,
|
||||
approvalExpired,
|
||||
runtimeActivationObserved: false,
|
||||
rolledBack,
|
||||
planSha256: planRead.sha256,
|
||||
...(rolloutSha256 === undefined ? {} : { rolloutSha256 }),
|
||||
requirement: options.require,
|
||||
compatible,
|
||||
};
|
||||
if (!compatible) {
|
||||
const error = new Error(
|
||||
'Manual Primary canary requirement is not satisfied',
|
||||
);
|
||||
error.report = report;
|
||||
throw error;
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
function main() {
|
||||
process.stdout.write(
|
||||
`${JSON.stringify(run(parseArguments(process.argv.slice(2))))}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = { parseArguments, run };
|
||||
|
||||
if (require.main === module) {
|
||||
try {
|
||||
main();
|
||||
} catch (error) {
|
||||
process.stderr.write(
|
||||
`${error instanceof Error ? error.message : String(error)}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,281 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
createLegacyShadowPrimaryGateReceipt,
|
||||
} = require('../../back/runtime/domain/legacyShadowPrimaryGate');
|
||||
const {
|
||||
MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS,
|
||||
createManualPrimaryCanaryDisabledManifest,
|
||||
createManualPrimaryCanaryEnabledManifest,
|
||||
createManualPrimaryCanaryPlan,
|
||||
createManualPrimaryCanaryQualification,
|
||||
manualPrimaryCanaryFileSet,
|
||||
parseManualPrimaryCanaryPlan,
|
||||
parseManualPrimaryCanaryQualification,
|
||||
} = require('../../back/runtime/domain/manualPrimaryCanaryCeremony');
|
||||
const {
|
||||
parseRuntimeRolloutManifest,
|
||||
} = require('../../back/runtime/domain/runtimeRolloutManifest');
|
||||
|
||||
const START = 1_750_400_000_000;
|
||||
const END = START + 60_000;
|
||||
const GENERATED = END + 6 * 60_000;
|
||||
const DIGESTS = {
|
||||
plan: '1'.repeat(64),
|
||||
gate: '2'.repeat(64),
|
||||
capture: '3'.repeat(64),
|
||||
terminal: '4'.repeat(64),
|
||||
resource: '5'.repeat(64),
|
||||
};
|
||||
|
||||
function captureEvidence(admitted = 8) {
|
||||
const outcomes = {
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 0,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
};
|
||||
return {
|
||||
schema: 'qinglong/legacy-shadow-capture-evidence@v1',
|
||||
profile: 'edge',
|
||||
startup: {
|
||||
schema: 'qinglong/legacy-shadow-startup-difference-report@v1',
|
||||
profile: 'edge',
|
||||
assessment: 'converged',
|
||||
configuredOriginCount: 1,
|
||||
coverage: { remaining: false },
|
||||
outcomes,
|
||||
byOrigin: [{ origin: 'manual', scanned: 0, ...outcomes }],
|
||||
},
|
||||
capture: {
|
||||
schema: 'qinglong/legacy-shadow-capture-report@v1',
|
||||
profile: 'edge',
|
||||
assessment: 'captured',
|
||||
epoch: '019f75d2-5555-7555-8555-555555555555',
|
||||
window: {
|
||||
basis: 'process_local_legacy_admission',
|
||||
startInclusiveMs: START,
|
||||
endExclusiveMs: END,
|
||||
},
|
||||
configuredOriginCount: 1,
|
||||
totals: {
|
||||
admitted,
|
||||
captured: admitted,
|
||||
failed: 0,
|
||||
pending: 0,
|
||||
failures: { fact: 0, observer: 0, initialization: 0, accept: 0 },
|
||||
},
|
||||
byOrigin: [
|
||||
{
|
||||
origin: 'manual',
|
||||
admitted,
|
||||
captured: admitted,
|
||||
failed: 0,
|
||||
pending: 0,
|
||||
failures: {
|
||||
fact: 0,
|
||||
observer: 0,
|
||||
initialization: 0,
|
||||
accept: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
capturePermille: 1_000,
|
||||
},
|
||||
qualification: {
|
||||
passed: true,
|
||||
startupConverged: true,
|
||||
originCoverageExact: true,
|
||||
captureComplete: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function terminal() {
|
||||
return {
|
||||
schema: 'qinglong/legacy-shadow-terminal-difference-report@v1',
|
||||
profile: 'edge',
|
||||
observedAtMs: GENERATED - 1,
|
||||
window: {
|
||||
basis: 'shadow_run_created_at',
|
||||
startInclusiveMs: START,
|
||||
endExclusiveMs: END,
|
||||
minimumSettlingAgeMs: 300_000,
|
||||
closed: true,
|
||||
},
|
||||
coverage: {
|
||||
direction: 'shadow_to_legacy',
|
||||
cohort: 'legacy_owned_shadow_runs',
|
||||
legacyWithoutShadow: 'not_measured',
|
||||
},
|
||||
scanned: 8,
|
||||
remaining: false,
|
||||
evidenceComplete: true,
|
||||
assessment: 'matched',
|
||||
counts: { matched: 8 },
|
||||
byOrigin: [{ origin: 'manual', scanned: 8, matched: 8 }],
|
||||
terminalAgreementPermille: 1_000,
|
||||
fullyComparablePermille: 1_000,
|
||||
};
|
||||
}
|
||||
|
||||
function resource() {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
fixture: 'qinglong/legacy-shadow-resource-rollback-evidence@v1',
|
||||
profile: 'edge',
|
||||
workload: { mode: 'full', runtime: 'compiled_backend' },
|
||||
rollback: {
|
||||
performed: true,
|
||||
legacyContinued: true,
|
||||
shadowWritesStopped: true,
|
||||
databaseIntegrity: 'ok',
|
||||
},
|
||||
qualification: { passed: true, violations: [] },
|
||||
};
|
||||
}
|
||||
|
||||
function plan() {
|
||||
return createManualPrimaryCanaryPlan({
|
||||
sessionId: 'edge-0001',
|
||||
profile: 'edge',
|
||||
createdAtMs: START - 1_000,
|
||||
admissionTarget: 8,
|
||||
currentRollout: { state: 'absent' },
|
||||
});
|
||||
}
|
||||
|
||||
function gate() {
|
||||
return createLegacyShadowPrimaryGateReceipt({
|
||||
profile: 'edge',
|
||||
generatedAtMs: GENERATED,
|
||||
capture: captureEvidence(),
|
||||
terminal: terminal(),
|
||||
resource: resource(),
|
||||
});
|
||||
}
|
||||
|
||||
function qualification() {
|
||||
return createManualPrimaryCanaryQualification({
|
||||
plan: plan(),
|
||||
planSha256: DIGESTS.plan,
|
||||
primaryGate: gate(),
|
||||
primaryGateFileSha256: DIGESTS.gate,
|
||||
sourceFileSha256: {
|
||||
capture: DIGESTS.capture,
|
||||
terminal: DIGESTS.terminal,
|
||||
resource: DIGESTS.resource,
|
||||
},
|
||||
qualifiedAtMs: GENERATED + 1,
|
||||
});
|
||||
}
|
||||
|
||||
test('creates an exact profile-bounded canary plan with no automatic activation', () => {
|
||||
const value = plan();
|
||||
|
||||
assert.deepEqual(parseManualPrimaryCanaryPlan(value), value);
|
||||
assert.deepEqual(value.files, manualPrimaryCanaryFileSet('edge-0001'));
|
||||
assert.equal(value.admissionTarget, 8);
|
||||
assert.equal(value.activation.defaultMode, 'off');
|
||||
assert.equal(value.activation.allowLegacyFallbackBeforeStart, false);
|
||||
assert.throws(
|
||||
() => createManualPrimaryCanaryPlan({ ...value, admissionTarget: 9 }),
|
||||
/admission target/,
|
||||
);
|
||||
assert.throws(
|
||||
() => parseManualPrimaryCanaryPlan({ ...value, command: 'enable' }),
|
||||
/shape/,
|
||||
);
|
||||
});
|
||||
|
||||
test('allows a selected standalone cohort only inside the reviewed range', () => {
|
||||
const value = createManualPrimaryCanaryPlan({
|
||||
sessionId: 'standalone-0032',
|
||||
profile: 'standalone',
|
||||
createdAtMs: START,
|
||||
admissionTarget: 64,
|
||||
currentRollout: { state: 'disabled', sha256: DIGESTS.plan },
|
||||
});
|
||||
|
||||
assert.equal(parseManualPrimaryCanaryPlan(value).admissionTarget, 64);
|
||||
for (const admissionTarget of [31, 129]) {
|
||||
assert.throws(
|
||||
() => createManualPrimaryCanaryPlan({ ...value, admissionTarget }),
|
||||
/admission target/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('binds the exact plan, source files and independently reproducible gate', () => {
|
||||
const value = qualification();
|
||||
|
||||
assert.deepEqual(parseManualPrimaryCanaryQualification(value), value);
|
||||
assert.equal(value.assessment, 'eligible');
|
||||
assert.deepEqual(value.counts, {
|
||||
admitted: 8,
|
||||
captured: 8,
|
||||
terminalScanned: 8,
|
||||
terminalMatched: 8,
|
||||
});
|
||||
const wrongPlan = { ...plan(), admissionTarget: 7 };
|
||||
assert.throws(
|
||||
() =>
|
||||
createManualPrimaryCanaryQualification({
|
||||
plan: wrongPlan,
|
||||
planSha256: DIGESTS.plan,
|
||||
primaryGate: gate(),
|
||||
primaryGateFileSha256: DIGESTS.gate,
|
||||
sourceFileSha256: {
|
||||
capture: DIGESTS.capture,
|
||||
terminal: DIGESTS.terminal,
|
||||
resource: DIGESTS.resource,
|
||||
},
|
||||
qualifiedAtMs: GENERATED + 1,
|
||||
}),
|
||||
/admission target|match the plan/,
|
||||
);
|
||||
});
|
||||
|
||||
test('creates only a short-lived manual Primary manifest after qualification', () => {
|
||||
const approvedAtMs = GENERATED + 2;
|
||||
const manifest = createManualPrimaryCanaryEnabledManifest({
|
||||
plan: plan(),
|
||||
qualification: qualification(),
|
||||
approvedBy: 'operator:local-owner',
|
||||
approvedAtMs,
|
||||
approvalMs: 60 * 60 * 1_000,
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
parseRuntimeRolloutManifest(manifest, approvedAtMs).policy.modeFor(
|
||||
'manual',
|
||||
),
|
||||
'primary',
|
||||
);
|
||||
assert.equal(manifest.rollout.defaultMode, 'off');
|
||||
assert.equal(manifest.primaryGate.receiptSha256, DIGESTS.gate);
|
||||
assert.equal(manifest.rollbackPlanRef, plan().files.plan);
|
||||
assert.throws(
|
||||
() =>
|
||||
createManualPrimaryCanaryEnabledManifest({
|
||||
plan: plan(),
|
||||
qualification: qualification(),
|
||||
approvedBy: 'operator:local-owner',
|
||||
approvedAtMs,
|
||||
approvalMs: MANUAL_PRIMARY_CANARY_MAX_APPROVAL_MS + 1,
|
||||
}),
|
||||
/approval/,
|
||||
);
|
||||
assert.deepEqual(createManualPrimaryCanaryDisabledManifest('edge-0001'), {
|
||||
schemaVersion: 2,
|
||||
revision: 'manual-primary-edge-0001-rollback',
|
||||
enabled: false,
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,532 @@
|
||||
require('ts-node/register/transpile-only');
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { afterEach, test } = require('node:test');
|
||||
const {
|
||||
manualPrimaryCanaryFileSet,
|
||||
} = require('../../back/runtime/domain/manualPrimaryCanaryCeremony');
|
||||
const {
|
||||
parseArguments,
|
||||
readPrivateJson,
|
||||
run,
|
||||
} = require('../../scripts/ql3-manual-primary-canary.cjs');
|
||||
const {
|
||||
run: audit,
|
||||
} = require('../../scripts/ql3-manual-primary-canary-audit.cjs');
|
||||
|
||||
const NOW = Date.now();
|
||||
const START = NOW - 7 * 60_000;
|
||||
const END = START + 1_000;
|
||||
const QUALIFIED_AT = END + 6 * 60_000;
|
||||
const SESSION = 'edge-live-0001';
|
||||
const directories = [];
|
||||
|
||||
function directory() {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql3-canary-'));
|
||||
directories.push(root);
|
||||
return root;
|
||||
}
|
||||
|
||||
function writeJson(target, value) {
|
||||
fs.writeFileSync(target, `${JSON.stringify(value)}\n`, { mode: 0o600 });
|
||||
}
|
||||
|
||||
function captureEvidence(admitted = 8) {
|
||||
const outcomes = {
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 0,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
};
|
||||
return {
|
||||
schema: 'qinglong/legacy-shadow-capture-evidence@v1',
|
||||
profile: 'edge',
|
||||
startup: {
|
||||
schema: 'qinglong/legacy-shadow-startup-difference-report@v1',
|
||||
profile: 'edge',
|
||||
assessment: 'converged',
|
||||
configuredOriginCount: 1,
|
||||
coverage: { remaining: false },
|
||||
outcomes,
|
||||
byOrigin: [{ origin: 'manual', scanned: 0, ...outcomes }],
|
||||
},
|
||||
capture: {
|
||||
schema: 'qinglong/legacy-shadow-capture-report@v1',
|
||||
profile: 'edge',
|
||||
assessment: 'captured',
|
||||
epoch: '019f75d2-5555-7555-8555-555555555555',
|
||||
window: {
|
||||
basis: 'process_local_legacy_admission',
|
||||
startInclusiveMs: START,
|
||||
endExclusiveMs: END,
|
||||
},
|
||||
configuredOriginCount: 1,
|
||||
totals: {
|
||||
admitted,
|
||||
captured: admitted,
|
||||
failed: 0,
|
||||
pending: 0,
|
||||
failures: { fact: 0, observer: 0, initialization: 0, accept: 0 },
|
||||
},
|
||||
byOrigin: [
|
||||
{
|
||||
origin: 'manual',
|
||||
admitted,
|
||||
captured: admitted,
|
||||
failed: 0,
|
||||
pending: 0,
|
||||
failures: {
|
||||
fact: 0,
|
||||
observer: 0,
|
||||
initialization: 0,
|
||||
accept: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
capturePermille: 1_000,
|
||||
},
|
||||
qualification: {
|
||||
passed: true,
|
||||
startupConverged: true,
|
||||
originCoverageExact: true,
|
||||
captureComplete: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function terminal() {
|
||||
return {
|
||||
schema: 'qinglong/legacy-shadow-terminal-difference-report@v1',
|
||||
profile: 'edge',
|
||||
observedAtMs: QUALIFIED_AT - 1,
|
||||
window: {
|
||||
basis: 'shadow_run_created_at',
|
||||
startInclusiveMs: START,
|
||||
endExclusiveMs: END,
|
||||
minimumSettlingAgeMs: 300_000,
|
||||
closed: true,
|
||||
},
|
||||
coverage: {
|
||||
direction: 'shadow_to_legacy',
|
||||
cohort: 'legacy_owned_shadow_runs',
|
||||
legacyWithoutShadow: 'not_measured',
|
||||
},
|
||||
scanned: 8,
|
||||
remaining: false,
|
||||
evidenceComplete: true,
|
||||
assessment: 'matched',
|
||||
counts: { matched: 8 },
|
||||
byOrigin: [{ origin: 'manual', scanned: 8, matched: 8 }],
|
||||
terminalAgreementPermille: 1_000,
|
||||
fullyComparablePermille: 1_000,
|
||||
};
|
||||
}
|
||||
|
||||
function resourceEvidence() {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
fixture: 'qinglong/legacy-shadow-resource-rollback-evidence@v1',
|
||||
profile: 'edge',
|
||||
workload: { mode: 'full', runtime: 'compiled_backend' },
|
||||
rollback: {
|
||||
performed: true,
|
||||
legacyContinued: true,
|
||||
shadowWritesStopped: true,
|
||||
databaseIntegrity: 'ok',
|
||||
},
|
||||
qualification: { passed: true, violations: [] },
|
||||
};
|
||||
}
|
||||
|
||||
function prepare(root, clock = START - 1_000) {
|
||||
return run(
|
||||
{
|
||||
mode: 'prepare',
|
||||
root,
|
||||
sessionId: SESSION,
|
||||
profile: 'edge',
|
||||
admissions: 8,
|
||||
},
|
||||
{ clock: { now: () => clock } },
|
||||
);
|
||||
}
|
||||
|
||||
function seedSources(root) {
|
||||
const files = manualPrimaryCanaryFileSet(SESSION);
|
||||
writeJson(path.join(root, files.capture), captureEvidence());
|
||||
writeJson(path.join(root, files.terminal), terminal());
|
||||
writeJson(path.join(root, files.resource), resourceEvidence());
|
||||
return files;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of directories.splice(0)) {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('prepares an idempotent private plan and exposes only explicit canary environment', () => {
|
||||
const root = directory();
|
||||
const first = prepare(root);
|
||||
const second = prepare(root, START);
|
||||
const files = manualPrimaryCanaryFileSet(SESSION);
|
||||
|
||||
assert.equal(first.state, 'prepared');
|
||||
assert.equal(first.automaticActivation, false);
|
||||
assert.deepEqual(first.environment, {
|
||||
QL_DEPLOYMENT_PROFILE: 'edge',
|
||||
QL3_SHADOW_ORIGINS: 'manual',
|
||||
QL3_SHADOW_CAPTURE_EVIDENCE_FILE: files.capture,
|
||||
});
|
||||
assert.equal(second.publication, 'existing');
|
||||
assert.equal(fs.statSync(path.join(root, files.plan)).mode & 0o777, 0o600);
|
||||
assert.equal(
|
||||
run({ mode: 'status', root, sessionId: SESSION }).state,
|
||||
'prepared',
|
||||
);
|
||||
});
|
||||
|
||||
test('qualifies, explicitly approves, audits and rolls back one target session', () => {
|
||||
const root = directory();
|
||||
prepare(root);
|
||||
const files = seedSources(root);
|
||||
|
||||
const qualified = run(
|
||||
{ mode: 'qualify', root, sessionId: SESSION },
|
||||
{ clock: { now: () => QUALIFIED_AT } },
|
||||
);
|
||||
assert.equal(qualified.state, 'qualified');
|
||||
assert.equal(qualified.automaticActivation, false);
|
||||
fs.unlinkSync(path.join(root, files.qualification));
|
||||
assert.equal(
|
||||
run(
|
||||
{ mode: 'qualify', root, sessionId: SESSION },
|
||||
{ clock: { now: () => QUALIFIED_AT + 10 } },
|
||||
).state,
|
||||
'qualified',
|
||||
);
|
||||
assert.equal(
|
||||
audit({ root, sessionId: SESSION, require: 'qualified' }).compatible,
|
||||
true,
|
||||
);
|
||||
|
||||
const activatedAt = Math.max(Date.now(), QUALIFIED_AT + 1);
|
||||
const approved = run(
|
||||
{
|
||||
mode: 'approve',
|
||||
root,
|
||||
sessionId: SESSION,
|
||||
approvedBy: 'operator:local-owner',
|
||||
approvalMs: 60 * 60 * 1_000,
|
||||
},
|
||||
{ clock: { now: () => activatedAt } },
|
||||
);
|
||||
assert.equal(approved.state, 'activation_approved');
|
||||
const selection = readPrivateJson(
|
||||
path.join(root, files.selection),
|
||||
64 * 1024,
|
||||
).value;
|
||||
assert.equal(selection.selectedAtMs, activatedAt);
|
||||
assert.equal(Object.hasOwn(selection, 'activatedAtMs'), false);
|
||||
fs.unlinkSync(path.join(root, files.selection));
|
||||
assert.throws(
|
||||
() =>
|
||||
run(
|
||||
{
|
||||
mode: 'approve',
|
||||
root,
|
||||
sessionId: SESSION,
|
||||
approvedBy: 'operator:different-owner',
|
||||
approvalMs: 60 * 60 * 1_000,
|
||||
},
|
||||
{ clock: { now: () => activatedAt + 5 } },
|
||||
),
|
||||
(error) => error.code === 'active_rollout_drift',
|
||||
);
|
||||
assert.equal(
|
||||
run(
|
||||
{
|
||||
mode: 'approve',
|
||||
root,
|
||||
sessionId: SESSION,
|
||||
approvedBy: 'operator:local-owner',
|
||||
approvalMs: 60 * 60 * 1_000,
|
||||
},
|
||||
{ clock: { now: () => activatedAt + 10 } },
|
||||
).state,
|
||||
'activation_approved',
|
||||
);
|
||||
assert.equal(
|
||||
audit({ root, sessionId: SESSION, require: 'selected' }).rolloutMode,
|
||||
'primary_selected',
|
||||
);
|
||||
assert.equal(
|
||||
audit({ root, sessionId: SESSION, require: 'selected' })
|
||||
.runtimeActivationObserved,
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
run(
|
||||
{ mode: 'status', root, sessionId: SESSION },
|
||||
{ clock: { now: () => activatedAt + 1 } },
|
||||
).state,
|
||||
'activation_approved',
|
||||
);
|
||||
assert.throws(
|
||||
() => audit({ root, sessionId: SESSION, require: 'rolled-back' }),
|
||||
/not satisfied/,
|
||||
);
|
||||
|
||||
const rolledBack = run(
|
||||
{
|
||||
mode: 'rollback',
|
||||
root,
|
||||
sessionId: SESSION,
|
||||
operator: 'operator:local-owner',
|
||||
reason: 'operator_request',
|
||||
},
|
||||
{ clock: { now: () => activatedAt + 2 } },
|
||||
);
|
||||
assert.equal(rolledBack.state, 'rolled_back');
|
||||
fs.unlinkSync(path.join(root, files.rollbackComplete));
|
||||
assert.equal(
|
||||
run(
|
||||
{
|
||||
mode: 'rollback',
|
||||
root,
|
||||
sessionId: SESSION,
|
||||
operator: 'operator:local-owner',
|
||||
reason: 'operator_request',
|
||||
},
|
||||
{ clock: { now: () => activatedAt + 3 } },
|
||||
).state,
|
||||
'rolled_back',
|
||||
);
|
||||
assert.equal(
|
||||
run(
|
||||
{
|
||||
mode: 'rollback',
|
||||
root,
|
||||
sessionId: SESSION,
|
||||
operator: 'operator:local-owner',
|
||||
reason: 'operator_request',
|
||||
},
|
||||
{ clock: { now: () => activatedAt + 4 } },
|
||||
).publication,
|
||||
'existing',
|
||||
);
|
||||
assert.equal(
|
||||
audit({ root, sessionId: SESSION, require: 'rolled-back' }).rolloutMode,
|
||||
'off',
|
||||
);
|
||||
assert.equal(
|
||||
readPrivateJson(path.join(root, files.rollout), 64 * 1024).value.enabled,
|
||||
false,
|
||||
);
|
||||
const completionPath = path.join(root, files.rollbackComplete);
|
||||
const completion = readPrivateJson(completionPath, 64 * 1024).value;
|
||||
completion.intentSha256 = '0'.repeat(64);
|
||||
writeJson(completionPath, completion);
|
||||
assert.throws(
|
||||
() => audit({ root, sessionId: SESSION, require: 'rolled-back' }),
|
||||
/receipt chain drifted/,
|
||||
);
|
||||
});
|
||||
|
||||
test('observe and resource use fixed child commands and recover from existing evidence', () => {
|
||||
const root = directory();
|
||||
prepare(root);
|
||||
const files = manualPrimaryCanaryFileSet(SESSION);
|
||||
writeJson(path.join(root, files.capture), captureEvidence());
|
||||
const database = path.join(root, 'database.sqlite');
|
||||
fs.writeFileSync(database, 'sqlite-fixture', { mode: 0o600 });
|
||||
const calls = [];
|
||||
const spawnSync = (_node, arguments_) => {
|
||||
calls.push(arguments_);
|
||||
const value = arguments_[0].endsWith('terminal-audit.cjs')
|
||||
? terminal()
|
||||
: resourceEvidence();
|
||||
return {
|
||||
status: 0,
|
||||
signal: null,
|
||||
stdout: JSON.stringify(value),
|
||||
stderr: '',
|
||||
};
|
||||
};
|
||||
const dependencies = {
|
||||
clock: { now: () => QUALIFIED_AT },
|
||||
spawnSync,
|
||||
workspaceRoot: path.resolve(__dirname, '../..'),
|
||||
};
|
||||
|
||||
assert.equal(
|
||||
run({ mode: 'observe', root, sessionId: SESSION, database }, dependencies)
|
||||
.state,
|
||||
'terminal_observed',
|
||||
);
|
||||
assert.equal(
|
||||
run({ mode: 'resource', root, sessionId: SESSION }, dependencies).state,
|
||||
'resource_proven',
|
||||
);
|
||||
assert.equal(calls.length, 2);
|
||||
assert.ok(calls[0].includes('--origin=manual'));
|
||||
assert.ok(calls[1].includes('--require-compiled'));
|
||||
run({ mode: 'observe', root, sessionId: SESSION, database }, dependencies);
|
||||
run({ mode: 'resource', root, sessionId: SESSION }, dependencies);
|
||||
assert.equal(calls.length, 2);
|
||||
});
|
||||
|
||||
test('fails closed on unsafe roots, incomplete cohorts and rollout drift', () => {
|
||||
const root = directory();
|
||||
const symlink = `${root}-link`;
|
||||
fs.symlinkSync(root, symlink);
|
||||
directories.push(symlink);
|
||||
assert.throws(
|
||||
() =>
|
||||
run({
|
||||
mode: 'prepare',
|
||||
root: symlink,
|
||||
sessionId: SESSION,
|
||||
profile: 'edge',
|
||||
admissions: 8,
|
||||
}),
|
||||
(error) => error.code === 'root_unsafe',
|
||||
);
|
||||
|
||||
prepare(root);
|
||||
const files = seedSources(root);
|
||||
const capture = captureEvidence(7);
|
||||
writeJson(path.join(root, 'wrong-capture.json'), capture);
|
||||
fs.renameSync(
|
||||
path.join(root, 'wrong-capture.json'),
|
||||
path.join(root, files.capture),
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
run(
|
||||
{ mode: 'qualify', root, sessionId: SESSION },
|
||||
{ clock: { now: () => QUALIFIED_AT } },
|
||||
),
|
||||
/capture_not_ready/,
|
||||
);
|
||||
|
||||
assert.throws(
|
||||
() =>
|
||||
parseArguments([
|
||||
'--mode=rollback',
|
||||
`--root=${root}`,
|
||||
`--session=${SESSION}`,
|
||||
'--operator=owner',
|
||||
'--reason=arbitrary-text',
|
||||
]),
|
||||
/supported/,
|
||||
);
|
||||
});
|
||||
|
||||
test('binds an existing disabled rollout and rejects a changed live baseline', () => {
|
||||
const root = directory();
|
||||
const disabled = {
|
||||
schemaVersion: 2,
|
||||
revision: 'operator-disabled-baseline',
|
||||
enabled: false,
|
||||
};
|
||||
writeJson(path.join(root, 'qinglong3-rollout.json'), disabled);
|
||||
prepare(root);
|
||||
const plan = readPrivateJson(
|
||||
path.join(root, manualPrimaryCanaryFileSet(SESSION).plan),
|
||||
64 * 1024,
|
||||
).value;
|
||||
assert.equal(plan.currentRollout.state, 'disabled');
|
||||
|
||||
writeJson(path.join(root, 'changed.json'), {
|
||||
schemaVersion: 2,
|
||||
revision: 'changed-disabled-baseline',
|
||||
enabled: false,
|
||||
});
|
||||
fs.renameSync(
|
||||
path.join(root, 'changed.json'),
|
||||
path.join(root, 'qinglong3-rollout.json'),
|
||||
);
|
||||
assert.throws(
|
||||
() => prepare(root, START),
|
||||
(error) => error.code === 'rollout_changed',
|
||||
);
|
||||
});
|
||||
|
||||
test('reports an expired approval as runtime-off until explicit rollback', () => {
|
||||
const root = directory();
|
||||
prepare(root);
|
||||
seedSources(root);
|
||||
run(
|
||||
{ mode: 'qualify', root, sessionId: SESSION },
|
||||
{ clock: { now: () => QUALIFIED_AT } },
|
||||
);
|
||||
const activatedAt = Math.max(Date.now(), QUALIFIED_AT + 1);
|
||||
run(
|
||||
{
|
||||
mode: 'approve',
|
||||
root,
|
||||
sessionId: SESSION,
|
||||
approvedBy: 'operator:local-owner',
|
||||
approvalMs: 60_000,
|
||||
},
|
||||
{ clock: { now: () => activatedAt } },
|
||||
);
|
||||
|
||||
const report = run(
|
||||
{ mode: 'status', root, sessionId: SESSION },
|
||||
{ clock: { now: () => activatedAt + 60_000 } },
|
||||
);
|
||||
assert.equal(report.state, 'approval_expired');
|
||||
assert.equal(report.rolloutMode, 'off');
|
||||
assert.equal(report.approvalExpired, true);
|
||||
});
|
||||
|
||||
test('independent audit rejects source replacement after qualification', () => {
|
||||
const root = directory();
|
||||
prepare(root);
|
||||
const files = seedSources(root);
|
||||
run(
|
||||
{ mode: 'qualify', root, sessionId: SESSION },
|
||||
{ clock: { now: () => QUALIFIED_AT } },
|
||||
);
|
||||
const replaced = resourceEvidence();
|
||||
replaced.unreviewed = true;
|
||||
writeJson(path.join(root, 'replacement.json'), replaced);
|
||||
fs.renameSync(
|
||||
path.join(root, 'replacement.json'),
|
||||
path.join(root, files.resource),
|
||||
);
|
||||
|
||||
assert.throws(
|
||||
() => audit({ root, sessionId: SESSION, require: 'qualified' }),
|
||||
/drifted/,
|
||||
);
|
||||
});
|
||||
|
||||
test('independent audit rejects qualification canonical digest drift', () => {
|
||||
const root = directory();
|
||||
prepare(root);
|
||||
const files = seedSources(root);
|
||||
run(
|
||||
{ mode: 'qualify', root, sessionId: SESSION },
|
||||
{ clock: { now: () => QUALIFIED_AT } },
|
||||
);
|
||||
const qualificationPath = path.join(root, files.qualification);
|
||||
const qualification = readPrivateJson(qualificationPath, 64 * 1024).value;
|
||||
qualification.sourceCanonicalSha256.capture = '0'.repeat(64);
|
||||
writeJson(qualificationPath, qualification);
|
||||
|
||||
assert.throws(
|
||||
() => audit({ root, sessionId: SESSION, require: 'qualified' }),
|
||||
/drifted/,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user