feat(ql3): gate primary on shadow capture evidence

This commit is contained in:
whyour
2026-08-19 03:27:32 +08:00
parent c60a86d1bf
commit be98d6753b
28 changed files with 2227 additions and 30 deletions
@@ -1,4 +1,5 @@
import { createHash } from 'crypto';
import { constants } from 'fs';
import fs from 'fs/promises';
import path from 'path';
import {
@@ -9,6 +10,10 @@ import type {
RuntimeRolloutLoadAudit,
RuntimeRolloutLoadResult,
} from '../../ports/runtimeRolloutLoader';
import {
parseLegacyShadowPrimaryGateReceipt,
type LegacyShadowPrimaryGateReceipt,
} from '../../domain/legacyShadowPrimaryGate';
export type {
RuntimeRolloutLoadAudit,
@@ -17,6 +22,7 @@ export type {
} from '../../ports/runtimeRolloutLoader';
export const MAX_RUNTIME_ROLLOUT_MANIFEST_BYTES = 64 * 1024;
export const MAX_RUNTIME_PRIMARY_GATE_RECEIPT_BYTES = 64 * 1024;
export interface RuntimeRolloutManifestLoaderOptions {
clock?: { now(): number };
@@ -34,6 +40,47 @@ function rejected(
};
}
async function loadPrimaryGateReceipt(
manifestPath: string,
receiptFile: string,
expectedSha256: string,
approvedAtMs: number,
): Promise<LegacyShadowPrimaryGateReceipt> {
const receiptPath = path.join(path.dirname(manifestPath), receiptFile);
let handle;
try {
handle = await fs.open(
receiptPath,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
const stat = await handle.stat();
if (
!stat.isFile() ||
(stat.mode & 0o077) !== 0 ||
stat.size < 2 ||
stat.size > MAX_RUNTIME_PRIMARY_GATE_RECEIPT_BYTES
) {
throw new TypeError('Primary gate receipt file shape is invalid');
}
const bytes = await handle.readFile();
if (createHash('sha256').update(bytes).digest('hex') !== expectedSha256) {
throw new TypeError('Primary gate receipt digest does not match');
}
const receipt = parseLegacyShadowPrimaryGateReceipt(
JSON.parse(bytes.toString('utf8')),
);
if (
receipt.assessment !== 'eligible' ||
receipt.generatedAtMs > approvedAtMs
) {
throw new TypeError('Primary gate receipt is not eligible for approval');
}
return receipt;
} finally {
await handle?.close();
}
}
export async function loadRuntimeRolloutManifest(
sourcePath: string,
options: RuntimeRolloutManifestLoaderOptions = {},
@@ -90,10 +137,29 @@ export async function loadRuntimeRolloutManifest(
try {
const decision = parseRuntimeRolloutManifest(value, evaluatedAtMs);
const status = decision.manifest.enabled ? 'accepted' : 'disabled';
let primaryGateReceipt: LegacyShadowPrimaryGateReceipt | undefined;
if (decision.manifest.enabled) {
try {
primaryGateReceipt = await loadPrimaryGateReceipt(
sourcePath,
decision.manifest.primaryGate.receiptFile,
decision.manifest.primaryGate.receiptSha256,
decision.manifest.approvedAtMs,
);
} catch (error) {
return rejected(
hashedAudit,
(error as NodeJS.ErrnoException).code === 'ENOENT'
? 'PRIMARY_GATE_READ_FAILED'
: 'PRIMARY_GATE_INVALID',
);
}
}
return {
status,
policy: decision.policy,
manifest: decision.manifest,
...(primaryGateReceipt === undefined ? {} : { primaryGateReceipt }),
audit: {
...hashedAudit,
status,
@@ -52,8 +52,25 @@ export async function bootstrapDefaultManualPrimaryRuntime(
Logger.info(`[runtime-activation] ${JSON.stringify(record)}`);
});
let stackModule: DefaultManualPrimaryStackModule | undefined;
let selectedProfile: 'edge' | 'standalone' | undefined;
if (selected) {
try {
const profile =
options.deploymentProfile ??
parseDeploymentProfile(process.env.QL_DEPLOYMENT_PROFILE);
if (profile === 'cluster-control' || profile === 'worker') {
throw new Error('Local Primary requires edge or standalone Profile');
}
if (
load.primaryGateReceipt?.assessment !== 'eligible' ||
load.primaryGateReceipt.origin !== 'manual' ||
load.primaryGateReceipt.profile !== profile
) {
throw new Error(
'Primary gate receipt does not authorize this deployment Profile',
);
}
selectedProfile = profile;
stackModule = await (
options.loadStack ?? (() => import('./defaultManualPrimaryActivation'))
)();
@@ -87,9 +104,7 @@ export async function bootstrapDefaultManualPrimaryRuntime(
}
return stackModule.createDefaultManualPrimaryActivationStack(rollout, {
...activationOptions,
deploymentProfile:
options.deploymentProfile ??
parseDeploymentProfile(process.env.QL_DEPLOYMENT_PROFILE),
deploymentProfile: selectedProfile!,
});
},
install: options.install ?? installManualPrimaryExecutionRouter,
@@ -0,0 +1,254 @@
import fs from 'fs/promises';
import path from 'path';
import config from '../../../config';
import Logger from '../../../loaders/logger';
import {
createLegacyShadowCaptureReport,
type LegacyShadowCaptureReport,
type LegacyShadowCaptureSnapshot,
} from '../../application/legacyShadowCaptureAuthority';
import {
configuredLegacyShadowOrigins,
legacyShadowCaptureSnapshot,
} from '../../compatibility/legacyExecutionBridge';
import {
parseDeploymentProfile,
type DeploymentProfile,
} from '../../domain/deploymentProfile';
import type { ExecutionOrigin } from '../../domain/run';
import type {
LegacyShadowStartupAudit,
LegacyShadowStartupDifferenceReport,
} from './bootstrapLegacyShadowStartupReconciliation';
export const LEGACY_SHADOW_CAPTURE_EVIDENCE_SCHEMA =
'qinglong/legacy-shadow-capture-evidence@v1';
export const LEGACY_SHADOW_CAPTURE_EVIDENCE_FILE_ENV =
'QL3_SHADOW_CAPTURE_EVIDENCE_FILE';
export interface LegacyShadowCaptureEvidence {
schema: typeof LEGACY_SHADOW_CAPTURE_EVIDENCE_SCHEMA;
profile: 'edge' | 'standalone';
startup: LegacyShadowStartupDifferenceReport;
capture: LegacyShadowCaptureReport;
qualification: {
passed: boolean;
startupConverged: boolean;
originCoverageExact: boolean;
captureComplete: boolean;
};
}
export type LegacyShadowCaptureEvidenceAudit =
| { state: 'disabled' }
| { state: 'profile_rejected'; profile: 'cluster-control' | 'worker' }
| { state: 'armed'; profile: 'edge' | 'standalone'; origins: number }
| {
state: 'exported';
profile: 'edge' | 'standalone';
origins: number;
qualified: boolean;
}
| { state: 'failed'; errorType: string };
export interface LegacyShadowCaptureEvidenceHandle {
active: boolean;
close(): Promise<LegacyShadowCaptureEvidenceAudit>;
}
export interface BootstrapLegacyShadowCaptureEvidenceOptions {
startup: LegacyShadowStartupAudit;
origins?: readonly ExecutionOrigin[];
profile?: DeploymentProfile;
outputPath?: string;
snapshot?: (
origins: readonly ExecutionOrigin[],
) => LegacyShadowCaptureSnapshot;
write?: (
outputPath: string,
evidence: LegacyShadowCaptureEvidence,
) => Promise<void>;
audit?: (record: LegacyShadowCaptureEvidenceAudit) => void | Promise<void>;
}
const FILE_NAME_PATTERN =
/^(?=.{1,128}$)(?!\.)(?!.*\.\.)(?:[A-Za-z0-9][A-Za-z0-9._-]*)\.json$/u;
function configuredOutputPath(): string | undefined {
const fileName = process.env[LEGACY_SHADOW_CAPTURE_EVIDENCE_FILE_ENV]?.trim();
if (!fileName) return undefined;
if (
!FILE_NAME_PATTERN.test(fileName) ||
path.basename(fileName) !== fileName
) {
throw new TypeError('Legacy Shadow capture evidence filename is invalid');
}
return path.join(config.configPath, fileName);
}
async function writeEvidence(
outputPath: string,
evidence: LegacyShadowCaptureEvidence,
): Promise<void> {
if (!path.isAbsolute(outputPath)) {
throw new TypeError('Legacy Shadow capture evidence path must be absolute');
}
const handle = await fs.open(outputPath, 'wx', 0o600);
try {
await handle.writeFile(`${JSON.stringify(evidence)}\n`, 'utf8');
await handle.sync();
} finally {
await handle.close();
}
}
function exactOriginCoverage(
origins: readonly ExecutionOrigin[],
startup: LegacyShadowStartupDifferenceReport,
): boolean {
return (
startup.configuredOriginCount === origins.length &&
startup.byOrigin.length === origins.length &&
origins.every((origin, index) => startup.byOrigin[index]?.origin === origin)
);
}
async function emitAudit(
audit: (record: LegacyShadowCaptureEvidenceAudit) => void | Promise<void>,
record: LegacyShadowCaptureEvidenceAudit,
): Promise<void> {
try {
await audit(record);
} catch {
// Evidence diagnostics must not change Legacy startup or shutdown.
}
}
/**
* Arms one process-lifetime capture window after startup reconciliation. It has
* no timer or watcher and writes exactly one owner-private report on a clean
* shutdown when an explicit filename is configured.
*/
export async function bootstrapLegacyShadowCaptureEvidence(
options: BootstrapLegacyShadowCaptureEvidenceOptions,
): Promise<LegacyShadowCaptureEvidenceHandle> {
const audit =
options.audit ??
((record: LegacyShadowCaptureEvidenceAudit) => {
Logger.info(`[ql3-shadow-capture] ${JSON.stringify(record)}`);
});
const origins = [
...new Set(options.origins ?? configuredLegacyShadowOrigins()),
];
let outputPath: string | undefined;
try {
outputPath = options.outputPath ?? configuredOutputPath();
} catch (error) {
const record: LegacyShadowCaptureEvidenceAudit = {
state: 'failed',
errorType: error instanceof Error ? error.name : 'unknown',
};
await emitAudit(audit, record);
return {
active: false,
async close() {
return record;
},
};
}
if (origins.length === 0 || outputPath === undefined) {
const record: LegacyShadowCaptureEvidenceAudit = { state: 'disabled' };
await emitAudit(audit, record);
return {
active: false,
async close() {
return record;
},
};
}
const profile =
options.profile ??
parseDeploymentProfile(process.env.QL_DEPLOYMENT_PROFILE);
if (profile === 'cluster-control' || profile === 'worker') {
const record: LegacyShadowCaptureEvidenceAudit = {
state: 'profile_rejected',
profile,
};
await emitAudit(audit, record);
return {
active: false,
async close() {
return record;
},
};
}
if (
options.startup.state !== 'reconciled' ||
options.startup.report.profile !== profile
) {
const record: LegacyShadowCaptureEvidenceAudit = {
state: 'failed',
errorType: 'LegacyShadowStartupEvidenceUnavailable',
};
await emitAudit(audit, record);
return {
active: false,
async close() {
return record;
},
};
}
const startupReport = options.startup.report;
const snapshot = options.snapshot ?? legacyShadowCaptureSnapshot;
const before = snapshot(origins);
const armed: LegacyShadowCaptureEvidenceAudit = {
state: 'armed',
profile,
origins: origins.length,
};
await emitAudit(audit, armed);
let closed: LegacyShadowCaptureEvidenceAudit | undefined;
return {
active: true,
async close() {
if (closed) return closed;
try {
const capture = createLegacyShadowCaptureReport(
profile,
origins,
before,
snapshot(origins),
);
const startupConverged = startupReport.assessment === 'converged';
const originCoverageExact = exactOriginCoverage(origins, startupReport);
const captureComplete = capture.assessment === 'captured';
const evidence: LegacyShadowCaptureEvidence = {
schema: LEGACY_SHADOW_CAPTURE_EVIDENCE_SCHEMA,
profile,
startup: startupReport,
capture,
qualification: {
passed: startupConverged && originCoverageExact && captureComplete,
startupConverged,
originCoverageExact,
captureComplete,
},
};
await (options.write ?? writeEvidence)(outputPath!, evidence);
closed = {
state: 'exported',
profile,
origins: origins.length,
qualified: evidence.qualification.passed,
};
} catch (error) {
closed = {
state: 'failed',
errorType: error instanceof Error ? error.name : 'unknown',
};
}
await emitAudit(audit, closed);
return closed;
},
};
}
@@ -0,0 +1,296 @@
import { randomUUID } from 'crypto';
import type { ExecutionOrigin } from '../domain/run';
export const LEGACY_SHADOW_CAPTURE_SNAPSHOT_SCHEMA =
'qinglong/legacy-shadow-capture-snapshot@v1';
export const LEGACY_SHADOW_CAPTURE_REPORT_SCHEMA =
'qinglong/legacy-shadow-capture-report@v1';
export type LegacyShadowCaptureFailureStage =
| 'fact'
| 'observer'
| 'initialization'
| 'accept';
export interface LegacyShadowCaptureCounts {
admitted: number;
captured: number;
failed: number;
pending: number;
failures: Record<LegacyShadowCaptureFailureStage, number>;
}
export interface LegacyShadowCaptureOriginSnapshot
extends LegacyShadowCaptureCounts {
origin: ExecutionOrigin;
}
export interface LegacyShadowCaptureSnapshot {
schema: typeof LEGACY_SHADOW_CAPTURE_SNAPSHOT_SCHEMA;
epoch: string;
observedAtMs: number;
byOrigin: readonly LegacyShadowCaptureOriginSnapshot[];
}
export interface LegacyShadowCaptureReport {
schema: typeof LEGACY_SHADOW_CAPTURE_REPORT_SCHEMA;
profile: 'edge' | 'standalone';
assessment: 'captured' | 'empty' | 'incomplete' | 'failures_observed';
epoch: string;
window: {
basis: 'process_local_legacy_admission';
startInclusiveMs: number;
endExclusiveMs: number;
};
configuredOriginCount: number;
totals: LegacyShadowCaptureCounts;
byOrigin: readonly LegacyShadowCaptureOriginSnapshot[];
capturePermille?: number;
}
export interface LegacyShadowCaptureAdmission {
captured(): void;
failed(stage: LegacyShadowCaptureFailureStage): void;
}
const FAILURE_STAGES: readonly LegacyShadowCaptureFailureStage[] = [
'fact',
'observer',
'initialization',
'accept',
];
function emptyCounts(): LegacyShadowCaptureCounts {
return {
admitted: 0,
captured: 0,
failed: 0,
pending: 0,
failures: { fact: 0, observer: 0, initialization: 0, accept: 0 },
};
}
function cloneCounts(
source: LegacyShadowCaptureCounts,
): LegacyShadowCaptureCounts {
return {
admitted: source.admitted,
captured: source.captured,
failed: source.failed,
pending: source.pending,
failures: { ...source.failures },
};
}
function assertCount(value: number, label: string): void {
if (!Number.isSafeInteger(value) || value < 0) {
throw new RangeError(`${label} must be a non-negative safe integer`);
}
}
function assertConserved(
counts: LegacyShadowCaptureCounts,
label: string,
): void {
for (const key of ['admitted', 'captured', 'failed', 'pending'] as const) {
assertCount(counts[key], `${label}.${key}`);
}
for (const stage of FAILURE_STAGES) {
assertCount(counts.failures[stage], `${label}.failures.${stage}`);
}
if (counts.captured + counts.failed + counts.pending !== counts.admitted) {
throw new RangeError(`${label} does not conserve admitted executions`);
}
if (
FAILURE_STAGES.reduce(
(total, stage) => total + counts.failures[stage],
0,
) !== counts.failed
) {
throw new RangeError(`${label} failure stages do not conserve failures`);
}
}
function originMap(
snapshot: LegacyShadowCaptureSnapshot,
): Map<ExecutionOrigin, LegacyShadowCaptureOriginSnapshot> {
if (snapshot.schema !== LEGACY_SHADOW_CAPTURE_SNAPSHOT_SCHEMA) {
throw new TypeError('Legacy Shadow capture snapshot schema is unsupported');
}
if (!/^[0-9a-f-]{36}$/u.test(snapshot.epoch)) {
throw new TypeError('Legacy Shadow capture epoch is invalid');
}
assertCount(snapshot.observedAtMs, 'snapshot.observedAtMs');
const result = new Map<ExecutionOrigin, LegacyShadowCaptureOriginSnapshot>();
for (const entry of snapshot.byOrigin) {
if (result.has(entry.origin)) {
throw new RangeError('Legacy Shadow capture snapshot repeats an origin');
}
assertConserved(entry, `snapshot.${entry.origin}`);
result.set(entry.origin, entry);
}
return result;
}
function subtractCounts(
before: LegacyShadowCaptureCounts,
after: LegacyShadowCaptureCounts,
label: string,
): LegacyShadowCaptureCounts {
const result = emptyCounts();
for (const key of ['admitted', 'captured', 'failed'] as const) {
result[key] = after[key] - before[key];
assertCount(result[key], `${label}.${key}`);
}
if (before.pending !== 0) {
throw new RangeError(`${label} starts with pending capture work`);
}
result.pending = after.pending;
for (const stage of FAILURE_STAGES) {
result.failures[stage] = after.failures[stage] - before.failures[stage];
assertCount(result.failures[stage], `${label}.failures.${stage}`);
}
assertConserved(result, label);
return result;
}
export class LegacyShadowCaptureAuthority {
private readonly epoch: string;
private readonly counts = new Map<
ExecutionOrigin,
LegacyShadowCaptureCounts
>();
constructor(
private readonly clock: { now(): number } = { now: Date.now },
epoch: string = randomUUID(),
) {
if (!/^[0-9a-f-]{36}$/u.test(epoch)) {
throw new TypeError('Legacy Shadow capture epoch is invalid');
}
this.epoch = epoch;
}
admit(origin: ExecutionOrigin): LegacyShadowCaptureAdmission {
const counts = this.counts.get(origin) ?? emptyCounts();
if (!this.counts.has(origin)) this.counts.set(origin, counts);
counts.admitted += 1;
counts.pending += 1;
let settled = false;
return Object.freeze({
captured: () => {
if (settled) return;
settled = true;
counts.pending -= 1;
counts.captured += 1;
},
failed: (stage: LegacyShadowCaptureFailureStage) => {
if (settled) return;
if (!FAILURE_STAGES.includes(stage)) {
throw new TypeError('Legacy Shadow capture failure stage is invalid');
}
settled = true;
counts.pending -= 1;
counts.failed += 1;
counts.failures[stage] += 1;
},
});
}
snapshot(origins: readonly ExecutionOrigin[]): LegacyShadowCaptureSnapshot {
const observedAtMs = this.clock.now();
assertCount(observedAtMs, 'snapshot.observedAtMs');
const configured = [...new Set(origins)];
return Object.freeze({
schema: LEGACY_SHADOW_CAPTURE_SNAPSHOT_SCHEMA,
epoch: this.epoch,
observedAtMs,
byOrigin: configured.map((origin) =>
Object.freeze({
origin,
...cloneCounts(this.counts.get(origin) ?? emptyCounts()),
}),
),
});
}
}
export function createLegacyShadowCaptureReport(
profile: 'edge' | 'standalone',
origins: readonly ExecutionOrigin[],
before: LegacyShadowCaptureSnapshot,
after: LegacyShadowCaptureSnapshot,
): LegacyShadowCaptureReport {
if (profile !== 'edge' && profile !== 'standalone') {
throw new TypeError('Legacy Shadow capture profile is invalid');
}
const configured = [...new Set(origins)];
if (configured.length < 1 || configured.length > 7) {
throw new RangeError('Legacy Shadow capture origin count is invalid');
}
if (before.epoch !== after.epoch) {
throw new RangeError(
'Legacy Shadow capture snapshots cross process epochs',
);
}
if (before.observedAtMs >= after.observedAtMs) {
throw new RangeError('Legacy Shadow capture window must be non-empty');
}
const beforeOrigins = originMap(before);
const afterOrigins = originMap(after);
if (
beforeOrigins.size !== configured.length ||
afterOrigins.size !== configured.length ||
configured.some(
(origin) => !beforeOrigins.has(origin) || !afterOrigins.has(origin),
)
) {
throw new RangeError('Legacy Shadow capture origin coverage is incomplete');
}
const totals = emptyCounts();
const byOrigin = configured.map((origin) => {
const counts = subtractCounts(
beforeOrigins.get(origin)!,
afterOrigins.get(origin)!,
`capture.${origin}`,
);
for (const key of ['admitted', 'captured', 'failed', 'pending'] as const) {
totals[key] += counts[key];
}
for (const stage of FAILURE_STAGES) {
totals.failures[stage] += counts.failures[stage];
}
return Object.freeze({ origin, ...counts });
});
assertConserved(totals, 'capture.totals');
const assessment =
totals.pending > 0
? 'incomplete'
: totals.failed > 0
? 'failures_observed'
: totals.admitted === 0
? 'empty'
: 'captured';
const report: LegacyShadowCaptureReport = {
schema: LEGACY_SHADOW_CAPTURE_REPORT_SCHEMA,
profile,
assessment,
epoch: before.epoch,
window: {
basis: 'process_local_legacy_admission' as const,
startInclusiveMs: before.observedAtMs,
endExclusiveMs: after.observedAtMs,
},
configuredOriginCount: configured.length,
totals,
byOrigin,
...(totals.admitted > 0
? {
capturePermille: Math.floor(
(totals.captured * 1_000) / totals.admitted,
),
}
: {}),
};
return Object.freeze(report);
}
@@ -64,6 +64,7 @@ function classifyError(error: unknown): string {
class SerialLegacyExecutionObservation
implements TrackedLegacyExecutionObservation
{
private readonly capture: Promise<LegacyShadowRunReference | null>;
private chain: Promise<LegacyShadowRunReference | null>;
constructor(
@@ -72,10 +73,15 @@ class SerialLegacyExecutionObservation
accepted: LegacyExecutionAcceptedFact,
private readonly reporter: ShadowObservationReporter,
) {
this.chain = writer.accept(accepted).catch((error) => {
this.capture = writer.accept(accepted).catch((error) => {
this.report('accept', error);
return null;
});
this.chain = this.capture;
}
async captureSettled(): Promise<'captured' | 'failed'> {
return (await this.capture) === null ? 'failed' : 'captured';
}
spawned(fact: LegacyExecutionSpawnedFact): void {
@@ -1,5 +1,10 @@
import Logger from '../../loaders/logger';
import type { LegacyShadowRunCorrelator } from '../application/legacyShadowRunCorrelator';
import {
LegacyShadowCaptureAuthority,
type LegacyShadowCaptureAdmission,
type LegacyShadowCaptureSnapshot,
} from '../application/legacyShadowCaptureAuthority';
import type { ExecutionOrigin } from '../domain/run';
import type {
LegacyExecutionCallbackFact,
@@ -51,6 +56,12 @@ let configuredOrigins: ReadonlySet<ExecutionOrigin> | undefined;
let defaultObserver: Promise<LegacyExecutionObserver> | undefined;
let defaultCorrelator: Promise<LegacyShadowRunCorrelator> | undefined;
const failureCounters = new Map<string, number>();
let captureAuthority: LegacyShadowCaptureAuthority | undefined;
function getCaptureAuthority(): LegacyShadowCaptureAuthority {
captureAuthority ??= new LegacyShadowCaptureAuthority();
return captureAuthority;
}
const localRegistry = new LegacyExecutionRegistry({
onOverflow() {
incrementFailure('registry:capacity_exceeded');
@@ -185,10 +196,27 @@ function getDefaultCorrelator(): Promise<LegacyShadowRunCorrelator> {
function beginFailOpen(
observer: LegacyExecutionObserver,
accepted: LegacyExecutionAcceptedFact,
capture?: LegacyShadowCaptureAdmission,
): LegacyExecutionObservation {
try {
return observer.begin(accepted);
const observation = observer.begin(accepted);
if (capture) {
const settled = observation.captureSettled?.();
if (!settled) {
capture.failed('observer');
} else {
void settled.then(
(outcome) =>
outcome === 'captured'
? capture.captured()
: capture.failed('accept'),
() => capture.failed('accept'),
);
}
}
return observation;
} catch {
capture?.failed('observer');
incrementFailure(`${accepted.origin}:begin:failed`);
try {
Logger.warn(
@@ -204,6 +232,7 @@ function beginFailOpen(
function createAcceptedFactFailOpen(
origin: ExecutionOrigin,
createFact: LegacyExecutionAcceptedFactFactory,
capture?: LegacyShadowCaptureAdmission,
): LegacyExecutionAcceptedFact | null {
try {
const fact = createFact();
@@ -212,6 +241,7 @@ function createAcceptedFactFailOpen(
}
return fact;
} catch {
capture?.failed('fact');
incrementFailure(`${origin}:fact:failed`);
try {
Logger.warn(`[ql3-shadow] fact creation failed origin=${origin}`);
@@ -225,10 +255,14 @@ function createAcceptedFactFailOpen(
function deferredObservation(
observer: Promise<LegacyExecutionObserver>,
accepted: LegacyExecutionAcceptedFact,
capture: LegacyShadowCaptureAdmission,
): LegacyExecutionObservation {
const delegate = observer
.then((value) => beginFailOpen(value, accepted))
.catch(() => NOOP_OBSERVATION);
.then((value) => beginFailOpen(value, accepted, capture))
.catch(() => {
capture.failed('initialization');
return NOOP_OBSERVATION;
});
const enqueue = <T>(
operation: (observation: LegacyExecutionObservation, fact: T) => void,
fact: T,
@@ -266,11 +300,12 @@ export function observeLegacyExecution(
: NOOP_OBSERVATION;
}
if (!readConfiguredOrigins().has(origin)) return undefined;
const fact = createAcceptedFactFailOpen(origin, createFact);
const capture = getCaptureAuthority().admit(origin);
const fact = createAcceptedFactFailOpen(origin, createFact, capture);
return fact
? localRegistry.register(
fact,
deferredObservation(getDefaultObserver(), fact),
deferredObservation(getDefaultObserver(), fact, capture),
)
: NOOP_OBSERVATION;
}
@@ -302,9 +337,10 @@ export function observeLegacyShellExecutionCallback(
: NOOP_OBSERVATION;
} else {
if (!readConfiguredOrigins().has(origin)) return undefined;
const fact = createAcceptedFactFailOpen(origin, createFact);
const capture = getCaptureAuthority().admit(origin);
const fact = createAcceptedFactFailOpen(origin, createFact, capture);
observation = fact
? deferredObservation(getDefaultObserver(), fact)
? deferredObservation(getDefaultObserver(), fact, capture)
: NOOP_OBSERVATION;
}
@@ -415,3 +451,9 @@ export function shadowBridgeFailureSnapshot(): Readonly<
> {
return Object.fromEntries(failureCounters);
}
export function legacyShadowCaptureSnapshot(
origins: readonly ExecutionOrigin[] = configuredLegacyShadowOrigins(),
): LegacyShadowCaptureSnapshot {
return getCaptureAuthority().snapshot(origins);
}
@@ -0,0 +1,418 @@
export const LEGACY_SHADOW_PRIMARY_GATE_SCHEMA =
'qinglong/legacy-shadow-primary-gate@v1';
export type LegacyShadowPrimaryGateViolation =
| 'capture_schema_invalid'
| 'capture_profile_mismatch'
| 'capture_not_qualified'
| 'capture_origin_coverage_invalid'
| 'capture_window_invalid'
| 'capture_sample_budget_invalid'
| 'capture_conservation_invalid'
| 'startup_not_converged'
| 'terminal_schema_invalid'
| 'terminal_profile_mismatch'
| 'terminal_window_mismatch'
| 'terminal_coverage_invalid'
| 'terminal_not_matched'
| 'resource_schema_invalid'
| 'resource_profile_mismatch'
| 'resource_not_qualified'
| 'resource_not_compiled_full_rollback';
export interface LegacyShadowPrimaryGateReceipt {
schema: typeof LEGACY_SHADOW_PRIMARY_GATE_SCHEMA;
schemaVersion: 1;
profile: 'edge' | 'standalone';
origin: 'manual';
generatedAtMs: number;
assessment: 'eligible' | 'ineligible';
window: {
startInclusiveMs: number;
endExclusiveMs: number;
};
counts: {
admitted: number;
captured: number;
terminalScanned: number;
terminalMatched: number;
};
evidence: {
captureSha256: string;
terminalSha256: string;
resourceSha256: string;
};
sources: {
capture: unknown;
terminal: unknown;
resource: unknown;
};
violations: readonly LegacyShadowPrimaryGateViolation[];
}
export interface LegacyShadowPrimaryGateInput {
profile: 'edge' | 'standalone';
generatedAtMs: number;
capture: unknown;
terminal: unknown;
resource: unknown;
}
const SAMPLE_BUDGETS = Object.freeze({
edge: { minimum: 8, maximum: 8 },
standalone: { minimum: 32, maximum: 128 },
});
const SHA256_PATTERN = /^[a-f0-9]{64}$/u;
function object(value: unknown): Record<string, any> | undefined {
return value !== null && typeof value === 'object' && !Array.isArray(value)
? (value as Record<string, any>)
: undefined;
}
function safeCount(value: unknown): value is number {
return Number.isSafeInteger(value) && Number(value) >= 0;
}
function add(
violations: LegacyShadowPrimaryGateViolation[],
violation: LegacyShadowPrimaryGateViolation,
): void {
if (!violations.includes(violation)) violations.push(violation);
}
function canonicalJson(value: unknown): string {
if (value === null) return 'null';
if (typeof value === 'string' || typeof value === 'boolean') {
return JSON.stringify(value);
}
if (typeof value === 'number') {
if (!Number.isFinite(value))
throw new TypeError('Evidence number is invalid');
return JSON.stringify(value);
}
if (Array.isArray(value)) {
return `[${value.map(canonicalJson).join(',')}]`;
}
const record = object(value);
if (!record) throw new TypeError('Evidence contains a non-JSON value');
return `{${Object.keys(record)
.sort()
.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
.join(',')}}`;
}
export function legacyShadowPrimaryEvidenceSha256(value: unknown): string {
return createHash('sha256').update(canonicalJson(value)).digest('hex');
}
export function createLegacyShadowPrimaryGateReceipt(
input: LegacyShadowPrimaryGateInput,
): LegacyShadowPrimaryGateReceipt {
if (input.profile !== 'edge' && input.profile !== 'standalone') {
throw new TypeError('Primary gate profile is invalid');
}
if (!safeCount(input.generatedAtMs)) {
throw new TypeError('Primary gate timestamp is invalid');
}
const sources = {
capture: JSON.parse(canonicalJson(input.capture)),
terminal: JSON.parse(canonicalJson(input.terminal)),
resource: JSON.parse(canonicalJson(input.resource)),
};
const evidence = {
captureSha256: legacyShadowPrimaryEvidenceSha256(sources.capture),
terminalSha256: legacyShadowPrimaryEvidenceSha256(sources.terminal),
resourceSha256: legacyShadowPrimaryEvidenceSha256(sources.resource),
};
const violations: LegacyShadowPrimaryGateViolation[] = [];
const captureEvidence = object(sources.capture);
const capture = object(captureEvidence?.capture);
const startup = object(captureEvidence?.startup);
const captureWindow = object(capture?.window);
const captureTotals = object(capture?.totals);
const captureByOriginValue = capture?.byOrigin;
const captureByOrigin = Array.isArray(captureByOriginValue)
? captureByOriginValue
: [];
const admittedValue = captureTotals?.admitted;
const capturedValue = captureTotals?.captured;
const admitted = safeCount(admittedValue) ? admittedValue : 0;
const captured = safeCount(capturedValue) ? capturedValue : 0;
if (
captureEvidence?.schema !== 'qinglong/legacy-shadow-capture-evidence@v1' ||
capture?.schema !== 'qinglong/legacy-shadow-capture-report@v1'
) {
add(violations, 'capture_schema_invalid');
}
if (
captureEvidence?.profile !== input.profile ||
capture?.profile !== input.profile
) {
add(violations, 'capture_profile_mismatch');
}
if (
object(captureEvidence?.qualification)?.passed !== true ||
capture?.assessment !== 'captured' ||
capture?.capturePermille !== 1_000
) {
add(violations, 'capture_not_qualified');
}
if (
capture?.configuredOriginCount !== 1 ||
captureByOrigin.length !== 1 ||
object(captureByOrigin[0])?.origin !== 'manual' ||
startup?.configuredOriginCount !== 1 ||
!Array.isArray(startup?.byOrigin) ||
startup.byOrigin.length !== 1 ||
object(startup.byOrigin[0])?.origin !== 'manual'
) {
add(violations, 'capture_origin_coverage_invalid');
}
const startInclusiveMs = captureWindow?.startInclusiveMs;
const endExclusiveMs = captureWindow?.endExclusiveMs;
if (
captureWindow?.basis !== 'process_local_legacy_admission' ||
!safeCount(startInclusiveMs) ||
!safeCount(endExclusiveMs) ||
startInclusiveMs >= endExclusiveMs ||
endExclusiveMs > input.generatedAtMs
) {
add(violations, 'capture_window_invalid');
}
const budget = SAMPLE_BUDGETS[input.profile];
if (admitted < budget.minimum || admitted > budget.maximum) {
add(violations, 'capture_sample_budget_invalid');
}
if (
!safeCount(captureTotals?.failed) ||
!safeCount(captureTotals?.pending) ||
captured !== admitted ||
captureTotals?.failed !== 0 ||
captureTotals?.pending !== 0
) {
add(violations, 'capture_conservation_invalid');
}
if (
startup?.schema !== 'qinglong/legacy-shadow-startup-difference-report@v1' ||
startup?.profile !== input.profile ||
startup?.assessment !== 'converged' ||
object(startup?.coverage)?.remaining !== false
) {
add(violations, 'startup_not_converged');
}
const terminal = object(sources.terminal);
const terminalWindow = object(terminal?.window);
const terminalCoverage = object(terminal?.coverage);
const terminalCounts = object(terminal?.counts);
const terminalByOriginValue = terminal?.byOrigin;
const terminalByOrigin = Array.isArray(terminalByOriginValue)
? terminalByOriginValue
: [];
const terminalScannedValue = terminal?.scanned;
const terminalMatchedValue = terminalCounts?.matched;
const terminalScanned = safeCount(terminalScannedValue)
? terminalScannedValue
: 0;
const terminalMatched = safeCount(terminalMatchedValue)
? terminalMatchedValue
: 0;
const terminalObservedAtMs = terminal?.observedAtMs;
if (
terminal?.schema !== 'qinglong/legacy-shadow-terminal-difference-report@v1'
) {
add(violations, 'terminal_schema_invalid');
}
if (terminal?.profile !== input.profile) {
add(violations, 'terminal_profile_mismatch');
}
if (
terminalWindow?.startInclusiveMs !== startInclusiveMs ||
terminalWindow?.endExclusiveMs !== endExclusiveMs ||
terminalWindow?.closed !== true ||
!safeCount(terminalObservedAtMs) ||
terminalObservedAtMs > input.generatedAtMs
) {
add(violations, 'terminal_window_mismatch');
}
if (
terminalCoverage?.direction !== 'shadow_to_legacy' ||
terminalCoverage?.cohort !== 'legacy_owned_shadow_runs' ||
terminalCoverage?.legacyWithoutShadow !== 'not_measured' ||
terminalByOrigin.length !== 1 ||
object(terminalByOrigin[0])?.origin !== 'manual' ||
object(terminalByOrigin[0])?.scanned !== terminalScanned
) {
add(violations, 'terminal_coverage_invalid');
}
if (
terminal?.assessment !== 'matched' ||
terminal?.remaining !== false ||
terminal?.evidenceComplete !== true ||
terminal?.terminalAgreementPermille !== 1_000 ||
terminal?.fullyComparablePermille !== 1_000 ||
terminalScanned !== captured ||
terminalMatched !== captured
) {
add(violations, 'terminal_not_matched');
}
const resource = object(sources.resource);
const workload = object(resource?.workload);
const rollback = object(resource?.rollback);
const qualification = object(resource?.qualification);
if (
resource?.schemaVersion !== 1 ||
resource?.fixture !== 'qinglong/legacy-shadow-resource-rollback-evidence@v1'
) {
add(violations, 'resource_schema_invalid');
}
if (resource?.profile !== input.profile) {
add(violations, 'resource_profile_mismatch');
}
if (
qualification?.passed !== true ||
qualification?.violations?.length !== 0
) {
add(violations, 'resource_not_qualified');
}
if (
workload?.mode !== 'full' ||
workload?.runtime !== 'compiled_backend' ||
rollback?.performed !== true ||
rollback?.legacyContinued !== true ||
rollback?.shadowWritesStopped !== true ||
rollback?.databaseIntegrity !== 'ok'
) {
add(violations, 'resource_not_compiled_full_rollback');
}
return Object.freeze({
schema: LEGACY_SHADOW_PRIMARY_GATE_SCHEMA,
schemaVersion: 1,
profile: input.profile,
origin: 'manual',
generatedAtMs: input.generatedAtMs,
assessment: violations.length === 0 ? 'eligible' : 'ineligible',
window: {
startInclusiveMs: safeCount(startInclusiveMs) ? startInclusiveMs : 0,
endExclusiveMs: safeCount(endExclusiveMs) ? endExclusiveMs : 0,
},
counts: { admitted, captured, terminalScanned, terminalMatched },
evidence,
sources,
violations,
});
}
function assertExactKeys(
value: Record<string, unknown>,
expected: readonly string[],
label: string,
): void {
const keys = Object.keys(value).sort();
const wanted = [...expected].sort();
if (
keys.length !== wanted.length ||
keys.some((key, index) => key !== wanted[index])
) {
throw new TypeError(`${label} shape is invalid`);
}
}
export function parseLegacyShadowPrimaryGateReceipt(
value: unknown,
): LegacyShadowPrimaryGateReceipt {
const receipt = object(value);
if (!receipt) throw new TypeError('Primary gate receipt must be an object');
assertExactKeys(
receipt,
[
'schema',
'schemaVersion',
'profile',
'origin',
'generatedAtMs',
'assessment',
'window',
'counts',
'evidence',
'sources',
'violations',
],
'receipt',
);
const window = object(receipt.window);
const counts = object(receipt.counts);
const evidence = object(receipt.evidence);
const sources = object(receipt.sources);
if (!window || !counts || !evidence || !sources) {
throw new TypeError('Primary gate receipt nested shape is invalid');
}
assertExactKeys(
window,
['startInclusiveMs', 'endExclusiveMs'],
'receipt.window',
);
assertExactKeys(
sources,
['capture', 'terminal', 'resource'],
'receipt.sources',
);
assertExactKeys(
counts,
['admitted', 'captured', 'terminalScanned', 'terminalMatched'],
'receipt.counts',
);
assertExactKeys(
evidence,
['captureSha256', 'terminalSha256', 'resourceSha256'],
'receipt.evidence',
);
if (
receipt.schema !== LEGACY_SHADOW_PRIMARY_GATE_SCHEMA ||
receipt.schemaVersion !== 1 ||
!['edge', 'standalone'].includes(receipt.profile) ||
receipt.origin !== 'manual' ||
!safeCount(receipt.generatedAtMs) ||
!['eligible', 'ineligible'].includes(receipt.assessment) ||
!safeCount(window.startInclusiveMs) ||
!safeCount(window.endExclusiveMs) ||
window.startInclusiveMs >= window.endExclusiveMs ||
!Object.values(counts).every(safeCount) ||
!Object.values(evidence).every(
(digest) => typeof digest === 'string' && SHA256_PATTERN.test(digest),
) ||
!Array.isArray(receipt.violations) ||
receipt.violations.some(
(violation: unknown) =>
typeof violation !== 'string' || violation.length > 64,
) ||
(receipt.assessment === 'eligible' && receipt.violations.length !== 0) ||
(receipt.assessment === 'ineligible' && receipt.violations.length === 0)
) {
throw new TypeError('Primary gate receipt is invalid');
}
const recomputed = createLegacyShadowPrimaryGateReceipt({
profile: receipt.profile as 'edge' | 'standalone',
generatedAtMs: receipt.generatedAtMs,
capture: sources.capture,
terminal: sources.terminal,
resource: sources.resource,
});
for (const field of [
'assessment',
'window',
'counts',
'evidence',
'violations',
] as const) {
if (canonicalJson(receipt[field]) !== canonicalJson(recomputed[field])) {
throw new TypeError(`Primary gate receipt ${field} was not reproduced`);
}
}
return receipt as unknown as LegacyShadowPrimaryGateReceipt;
}
import { createHash } from 'crypto';
+51 -2
View File
@@ -5,7 +5,7 @@ import {
type RuntimeRolloutConfig,
} from './runtimeRollout';
export const RUNTIME_ROLLOUT_MANIFEST_VERSION = 1;
export const RUNTIME_ROLLOUT_MANIFEST_VERSION = 2;
export const MAX_RUNTIME_ROLLOUT_APPROVAL_MS = 30 * 24 * 60 * 60 * 1000;
export const REQUIRED_RUNTIME_ROLLOUT_GATES = [
@@ -33,6 +33,12 @@ export interface EnabledRuntimeRolloutManifest {
approvedAtMs: number;
expiresAtMs: number;
rollbackPlanRef: string;
primaryGate: {
schema: 'qinglong/legacy-shadow-primary-gate-reference@v1';
origin: 'manual';
receiptFile: string;
receiptSha256: string;
};
rollout: RuntimeRolloutConfig;
gates: Record<RuntimeRolloutGate, 'passed'>;
}
@@ -46,12 +52,15 @@ export interface RuntimeRolloutManifestDecision {
policy: RuntimeRolloutPolicy;
}
const MANUAL_ORIGIN: ExecutionOrigin = 'manual';
const MANUAL_ORIGIN = 'manual' as const satisfies ExecutionOrigin;
const ALLOWED_MANUAL_MODES = new Set<CompatibilityMode>([
'off',
'shadow',
'primary',
]);
const SHA256_PATTERN = /^[a-f0-9]{64}$/u;
const RECEIPT_FILE_PATTERN =
/^(?=.{1,128}$)(?!\.)(?!.*\.\.)(?:[A-Za-z0-9][A-Za-z0-9._-]*)\.json$/u;
function asObject(value: unknown, name: string): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
@@ -139,6 +148,7 @@ export function parseRuntimeRolloutManifest(
'approvedAtMs',
'expiresAtMs',
'rollbackPlanRef',
'primaryGate',
'rollout',
'gates',
],
@@ -201,6 +211,39 @@ export function parseRuntimeRolloutManifest(
}
}
const primaryGateObject = asObject(
object.primaryGate,
'manifest.primaryGate',
);
assertExactKeys(
primaryGateObject,
['schema', 'origin', 'receiptFile', 'receiptSha256'],
'manifest.primaryGate',
);
if (
primaryGateObject.schema !==
'qinglong/legacy-shadow-primary-gate-reference@v1' ||
primaryGateObject.origin !== MANUAL_ORIGIN
) {
throw new TypeError('manifest.primaryGate authority is invalid');
}
const receiptFile = boundedString(
primaryGateObject.receiptFile,
'manifest.primaryGate.receiptFile',
128,
);
if (!RECEIPT_FILE_PATTERN.test(receiptFile)) {
throw new TypeError('manifest.primaryGate.receiptFile is invalid');
}
const receiptSha256 = boundedString(
primaryGateObject.receiptSha256,
'manifest.primaryGate.receiptSha256',
64,
);
if (!SHA256_PATTERN.test(receiptSha256)) {
throw new TypeError('manifest.primaryGate.receiptSha256 is invalid');
}
const rollout: RuntimeRolloutConfig = {
defaultMode: 'off',
origins: { manual: origins.manual as CompatibilityMode },
@@ -218,6 +261,12 @@ export function parseRuntimeRolloutManifest(
'manifest.rollbackPlanRef',
512,
),
primaryGate: {
schema: 'qinglong/legacy-shadow-primary-gate-reference@v1',
origin: MANUAL_ORIGIN,
receiptFile,
receiptSha256,
},
rollout,
gates: Object.fromEntries(
REQUIRED_RUNTIME_ROLLOUT_GATES.map((gate) => [gate, 'passed']),
@@ -42,6 +42,7 @@ export interface LegacyExecutionCancelledFact {
}
export interface LegacyExecutionObservation {
captureSettled?(): Promise<'captured' | 'failed'>;
spawned(fact: LegacyExecutionSpawnedFact): void;
running(fact: LegacyExecutionRunningFact): void;
startFailed(fact: LegacyExecutionStartFailedFact): void;
+5 -1
View File
@@ -1,5 +1,6 @@
import type { RuntimeRolloutPolicy } from '../domain/runtimeRollout';
import type { RuntimeRolloutManifest } from '../domain/runtimeRolloutManifest';
import type { LegacyShadowPrimaryGateReceipt } from '../domain/legacyShadowPrimaryGate';
export type RuntimeRolloutLoadStatus =
| 'missing'
@@ -19,7 +20,9 @@ export interface RuntimeRolloutLoadAudit {
| 'FILE_READ_FAILED'
| 'FILE_TOO_LARGE'
| 'INVALID_JSON'
| 'INVALID_MANIFEST';
| 'INVALID_MANIFEST'
| 'PRIMARY_GATE_READ_FAILED'
| 'PRIMARY_GATE_INVALID';
}
export interface RuntimeRolloutLoadResult {
@@ -27,6 +30,7 @@ export interface RuntimeRolloutLoadResult {
policy: RuntimeRolloutPolicy;
audit: RuntimeRolloutLoadAudit;
manifest?: RuntimeRolloutManifest;
primaryGateReceipt?: LegacyShadowPrimaryGateReceipt;
}
export interface RuntimeRolloutLoader {