feat(ql3): establish 3.0 incubation baseline

This commit is contained in:
whyour
2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
@@ -0,0 +1,357 @@
import Logger from '../../loaders/logger';
import type { LegacyShadowRunCorrelator } from '../application/legacyShadowRunCorrelator';
import type { ExecutionOrigin } from '../domain/run';
import type {
LegacyExecutionCallbackFact,
LegacyExecutionCancellationFact,
} from '../ports/legacyExecutionCorrelation';
import type {
LegacyExecutionAcceptedFact,
LegacyExecutionCancelledFact,
LegacyExecutionExitedFact,
LegacyExecutionObservation,
LegacyExecutionObserver,
LegacyExecutionRunningFact,
LegacyExecutionSpawnedFact,
LegacyExecutionStartFailedFact,
} from '../ports/legacyExecutionObserver';
import { createLegacyLogArtifactId } from './legacyTaskRevision';
import { LegacyExecutionRegistry } from './legacyExecutionRegistry';
const SHADOW_ORIGINS_ENV = 'QL3_SHADOW_ORIGINS';
const SUPPORTED_SHADOW_ORIGINS = new Set<ExecutionOrigin>([
'manual',
'scheduled_node',
]);
const NOOP_OBSERVATION: LegacyExecutionObservation = Object.freeze({
spawned() {},
running() {},
startFailed() {},
exited() {},
cancelled() {},
});
interface ObserverOverride {
token: symbol;
observer: LegacyExecutionObserver;
origins: ReadonlySet<ExecutionOrigin>;
}
export type LegacyExecutionAcceptedFactFactory =
() => LegacyExecutionAcceptedFact;
let override: ObserverOverride | undefined;
let configuredOrigins: ReadonlySet<ExecutionOrigin> | undefined;
let defaultObserver: Promise<LegacyExecutionObserver> | undefined;
let defaultCorrelator: Promise<LegacyShadowRunCorrelator> | undefined;
const failureCounters = new Map<string, number>();
const localRegistry = new LegacyExecutionRegistry({
onOverflow() {
incrementFailure('registry:capacity_exceeded');
try {
Logger.warn('[ql3-shadow] local registry capacity exceeded');
} catch {
// Compatibility diagnostics must not affect legacy execution.
}
},
onDispatchFailure() {
incrementFailure('registry:dispatch_failed');
try {
Logger.warn('[ql3-shadow] local observation dispatch failed');
} catch {
// Compatibility diagnostics must not affect legacy execution.
}
},
});
function incrementFailure(key: string): void {
failureCounters.set(key, (failureCounters.get(key) ?? 0) + 1);
}
function readConfiguredOrigins(): ReadonlySet<ExecutionOrigin> {
if (configuredOrigins) return configuredOrigins;
const origins = new Set<ExecutionOrigin>();
const raw = process.env[SHADOW_ORIGINS_ENV]?.trim();
if (raw) {
for (const value of raw.split(',').map((item) => item.trim())) {
if (SUPPORTED_SHADOW_ORIGINS.has(value as ExecutionOrigin)) {
origins.add(value as ExecutionOrigin);
} else if (value) {
incrementFailure('configuration:unsupported_origin');
try {
Logger.warn(
'[ql3-shadow] ignored unsupported origin; this slice supports manual,scheduled_node',
);
} catch {
// Invalid compatibility configuration must not affect legacy paths.
}
}
}
}
configuredOrigins = origins;
return configuredOrigins;
}
async function createDefaultObserver(): Promise<LegacyExecutionObserver> {
const [data, repositoryModule, observerModule, writerModule, rolloutModule] =
await Promise.all([
import('../../data'),
import('../adapters/legacy-sequelize/runRepository'),
import('../application/legacyShadowRunObserver'),
import('../application/legacyShadowRunWriter'),
import('../domain/runtimeRollout'),
]);
const origins = [...readConfiguredOrigins()];
const repository = new repositoryModule.LegacySequelizeRunRepository(
data.sequelize,
);
const writer = new writerModule.LegacyShadowRunWriter(repository);
return new observerModule.LegacyShadowRunObserver(
rolloutModule.shadowOnlyRollout(origins),
writer,
{
failure(failure) {
const key = `${failure.origin}:${failure.operation}:${failure.errorCode}`;
incrementFailure(key);
Logger.warn(
`[ql3-shadow] write failed origin=${failure.origin} operation=${failure.operation} code=${failure.errorCode}`,
);
},
},
);
}
function getDefaultObserver(): Promise<LegacyExecutionObserver> {
defaultObserver ??= createDefaultObserver().catch((error) => {
defaultObserver = undefined;
incrementFailure('initialization:failed');
Logger.warn(
`[ql3-shadow] observer initialization failed type=${
error instanceof Error ? error.name : 'unknown'
}`,
);
throw error;
});
return defaultObserver;
}
async function createDefaultCorrelator(): Promise<LegacyShadowRunCorrelator> {
const [data, repositoryModule, correlatorModule, writerModule] =
await Promise.all([
import('../../data'),
import('../adapters/legacy-sequelize/runRepository'),
import('../application/legacyShadowRunCorrelator'),
import('../application/legacyShadowRunWriter'),
]);
const repository = new repositoryModule.LegacySequelizeRunRepository(
data.sequelize,
);
const writer = new writerModule.LegacyShadowRunWriter(repository);
return new correlatorModule.LegacyShadowRunCorrelator(repository, writer, {
failure(failure) {
const key = `correlation:${failure.operation}:${failure.reason}`;
incrementFailure(key);
Logger.warn(
`[ql3-shadow] correlation skipped operation=${failure.operation} reason=${failure.reason} candidates=${failure.candidateCount}`,
);
},
});
}
function getDefaultCorrelator(): Promise<LegacyShadowRunCorrelator> {
defaultCorrelator ??= createDefaultCorrelator().catch((error) => {
defaultCorrelator = undefined;
incrementFailure('correlation:initialization_failed');
try {
Logger.warn('[ql3-shadow] correlator initialization failed');
} catch {
// Compatibility diagnostics must not affect legacy execution.
}
throw error;
});
return defaultCorrelator;
}
function beginFailOpen(
observer: LegacyExecutionObserver,
accepted: LegacyExecutionAcceptedFact,
): LegacyExecutionObservation {
try {
return observer.begin(accepted);
} catch {
incrementFailure(`${accepted.origin}:begin:failed`);
try {
Logger.warn(
`[ql3-shadow] observer begin failed origin=${accepted.origin}`,
);
} catch {
// Compatibility observation must never become a legacy execution failure.
}
return NOOP_OBSERVATION;
}
}
function createAcceptedFactFailOpen(
origin: ExecutionOrigin,
createFact: LegacyExecutionAcceptedFactFactory,
): LegacyExecutionAcceptedFact | null {
try {
const fact = createFact();
if (fact.origin !== origin) {
throw new TypeError('Legacy execution fact origin does not match flag');
}
return fact;
} catch {
incrementFailure(`${origin}:fact:failed`);
try {
Logger.warn(`[ql3-shadow] fact creation failed origin=${origin}`);
} catch {
// Compatibility observation must never become a legacy execution failure.
}
return null;
}
}
function deferredObservation(
observer: Promise<LegacyExecutionObserver>,
accepted: LegacyExecutionAcceptedFact,
): LegacyExecutionObservation {
const delegate = observer
.then((value) => beginFailOpen(value, accepted))
.catch(() => NOOP_OBSERVATION);
const enqueue = <T>(
operation: (observation: LegacyExecutionObservation, fact: T) => void,
fact: T,
) => {
void delegate.then((observation) => operation(observation, fact));
};
return {
spawned(fact: LegacyExecutionSpawnedFact) {
enqueue((observation, value) => observation.spawned(value), fact);
},
running(fact: LegacyExecutionRunningFact) {
enqueue((observation, value) => observation.running(value), fact);
},
startFailed(fact: LegacyExecutionStartFailedFact) {
enqueue((observation, value) => observation.startFailed(value), fact);
},
exited(fact: LegacyExecutionExitedFact) {
enqueue((observation, value) => observation.exited(value), fact);
},
cancelled(fact: LegacyExecutionCancelledFact) {
enqueue((observation, value) => observation.cancelled(value), fact);
},
};
}
export function observeLegacyExecution(
origin: ExecutionOrigin,
createFact: LegacyExecutionAcceptedFactFactory,
): LegacyExecutionObservation | undefined {
if (override) {
if (!override.origins.has(origin)) return undefined;
const fact = createAcceptedFactFailOpen(origin, createFact);
return fact
? localRegistry.register(fact, beginFailOpen(override.observer, fact))
: NOOP_OBSERVATION;
}
if (!readConfiguredOrigins().has(origin)) return undefined;
const fact = createAcceptedFactFailOpen(origin, createFact);
return fact
? localRegistry.register(
fact,
deferredObservation(getDefaultObserver(), fact),
)
: NOOP_OBSERVATION;
}
export interface LegacyExecutionCancellationInput {
legacyCronId: number;
pid?: number;
logPath?: string;
atMs: number;
scope: 'all' | 'one';
reason: LegacyExecutionCancellationFact['reason'];
}
export interface LegacyExecutionCallbackInput {
legacyCronId: number;
pid?: number;
logPath?: string;
atMs: number;
phase: LegacyExecutionCallbackFact['phase'];
exitCode?: number;
}
export function observeLegacyCancellation(
input: LegacyExecutionCancellationInput,
): void {
const origins = override?.origins ?? readConfiguredOrigins();
if (origins.size === 0) return;
const fact: LegacyExecutionCancellationFact = {
legacyCronId: input.legacyCronId,
atMs: input.atMs,
scope: input.scope,
reason: input.reason,
...(input.pid === undefined ? {} : { pid: input.pid }),
...(input.logPath === undefined
? {}
: { logArtifactId: createLegacyLogArtifactId(input.logPath) }),
};
const localMatches = localRegistry.cancel(fact);
if (override || (input.scope === 'one' && localMatches === 1)) return;
dispatchCorrelation((correlator) => correlator.cancel(fact, [...origins]));
}
export function observeLegacyExecutionCallback(
input: LegacyExecutionCallbackInput,
): void {
const origins = override?.origins ?? readConfiguredOrigins();
if (origins.size === 0) return;
const fact: LegacyExecutionCallbackFact = {
legacyCronId: input.legacyCronId,
atMs: input.atMs,
phase: input.phase,
...(input.pid === undefined ? {} : { pid: input.pid }),
...(input.logPath === undefined
? {}
: { logArtifactId: createLegacyLogArtifactId(input.logPath) }),
...(input.exitCode === undefined ? {} : { exitCode: input.exitCode }),
};
if (localRegistry.callback(fact) === 1 || override) return;
dispatchCorrelation((correlator) => correlator.callback(fact, [...origins]));
}
function dispatchCorrelation(
operation: (correlator: LegacyShadowRunCorrelator) => Promise<unknown>,
): void {
void getDefaultCorrelator()
.then(operation)
.catch(() => {
incrementFailure('correlation:operation_failed');
try {
Logger.warn('[ql3-shadow] correlation operation failed');
} catch {
// Compatibility diagnostics must not affect legacy execution.
}
});
}
export function installLegacyExecutionObserver(
observer: LegacyExecutionObserver,
origins: readonly ExecutionOrigin[],
): () => void {
const token = Symbol('legacy-shadow-observer');
const previous = override;
override = { token, observer, origins: new Set(origins) };
return () => {
if (override?.token === token) override = previous;
};
}
export function shadowBridgeFailureSnapshot(): Readonly<
Record<string, number>
> {
return Object.fromEntries(failureCounters);
}
@@ -0,0 +1,159 @@
import type { LegacyExecutionAcceptedFact } from '../ports/legacyExecutionObserver';
import { selectOneLegacyExecution } from '../domain/legacyExecutionSelection';
import type {
LegacyExecutionCancellationFact,
LegacyExecutionCallbackFact,
LegacyExecutionSelector,
} from '../ports/legacyExecutionCorrelation';
import type { LegacyExecutionObservation } from '../ports/legacyExecutionObserver';
export const DEFAULT_MAX_LOCAL_LEGACY_EXECUTIONS = 256;
interface RegistryEntry {
accepted: LegacyExecutionAcceptedFact;
observation: LegacyExecutionObservation;
pid?: number;
logArtifactId?: string;
}
export interface LegacyExecutionRegistryOptions {
maxEntries?: number;
onOverflow?: () => void;
onDispatchFailure?: () => void;
}
export class LegacyExecutionRegistry {
private readonly entries = new Map<number, Set<RegistryEntry>>();
private readonly maxEntries: number;
private readonly onOverflow: () => void;
private readonly onDispatchFailure: () => void;
private entryCount = 0;
constructor(options: LegacyExecutionRegistryOptions = {}) {
const maxEntries =
options.maxEntries ?? DEFAULT_MAX_LOCAL_LEGACY_EXECUTIONS;
if (!Number.isSafeInteger(maxEntries) || maxEntries < 1) {
throw new TypeError('maxEntries must be a positive safe integer');
}
this.maxEntries = maxEntries;
this.onOverflow = options.onOverflow ?? (() => {});
this.onDispatchFailure = options.onDispatchFailure ?? (() => {});
}
register(
accepted: LegacyExecutionAcceptedFact,
observation: LegacyExecutionObservation,
): LegacyExecutionObservation {
if (accepted.legacyCronId === undefined) return observation;
if (this.entryCount >= this.maxEntries) {
try {
this.onOverflow();
} catch {
// Registry diagnostics must not affect the legacy execution path.
}
return observation;
}
const entry: RegistryEntry = { accepted, observation };
const entries = this.entries.get(accepted.legacyCronId) ?? new Set();
entries.add(entry);
this.entries.set(accepted.legacyCronId, entries);
this.entryCount += 1;
const remove = () => this.remove(accepted.legacyCronId!, entry);
return {
spawned: (fact) => {
if (fact.pid !== undefined) entry.pid = fact.pid;
if (fact.logArtifactId !== undefined) {
entry.logArtifactId = fact.logArtifactId;
}
this.invoke(() => observation.spawned(fact));
},
running: (fact) => this.invoke(() => observation.running(fact)),
startFailed: (fact) => {
this.invoke(() => observation.startFailed(fact));
remove();
},
exited: (fact) => {
this.invoke(() => observation.exited(fact));
remove();
},
cancelled: (fact) => {
this.invoke(() => observation.cancelled(fact));
remove();
},
};
}
cancel(fact: LegacyExecutionCancellationFact): number {
const matches =
fact.scope === 'all'
? [...(this.entries.get(fact.legacyCronId) ?? [])]
: this.selectOne(fact);
for (const entry of matches) {
this.invoke(() =>
entry.observation.cancelled({
atMs: fact.atMs,
reason: fact.reason,
}),
);
this.remove(fact.legacyCronId, entry);
}
return matches.length;
}
callback(fact: LegacyExecutionCallbackFact): number {
const matches = this.selectOne(fact);
if (matches.length !== 1) return 0;
const [entry] = matches;
if (fact.phase === 'running') {
this.invoke(() =>
entry.observation.spawned({
atMs: fact.atMs,
...(fact.pid === undefined ? {} : { pid: fact.pid }),
...(fact.logArtifactId === undefined
? {}
: { logArtifactId: fact.logArtifactId }),
}),
);
this.invoke(() => entry.observation.running({ atMs: fact.atMs }));
} else {
this.invoke(() =>
entry.observation.exited({
atMs: fact.atMs,
exitCode: fact.exitCode ?? 0,
}),
);
this.remove(fact.legacyCronId, entry);
}
return 1;
}
size(): number {
return this.entryCount;
}
private selectOne(selector: LegacyExecutionSelector): RegistryEntry[] {
const candidates = [...(this.entries.get(selector.legacyCronId) ?? [])];
return selectOneLegacyExecution(candidates, selector);
}
private remove(legacyCronId: number, entry: RegistryEntry): void {
const entries = this.entries.get(legacyCronId);
if (!entries?.delete(entry)) return;
this.entryCount -= 1;
if (entries.size === 0) this.entries.delete(legacyCronId);
}
private invoke(operation: () => void): void {
try {
operation();
} catch {
try {
this.onDispatchFailure();
} catch {
// Registry diagnostics must not affect the legacy execution path.
}
}
}
}
@@ -0,0 +1,62 @@
import path from 'path';
export const LEGACY_LOG_OUTPUT_REF_PREFIX = 'legacy-log-v1.';
export const MAX_LEGACY_LOG_PATH_BYTES = 360;
export const MAX_LEGACY_LOG_OUTPUT_REF_BYTES = 512;
function normalizeRelativeLogPath(value: string): string {
if (
!value ||
value.includes('\0') ||
value.includes('\\') ||
path.posix.isAbsolute(value) ||
path.win32.isAbsolute(value) ||
Buffer.byteLength(value, 'utf8') > MAX_LEGACY_LOG_PATH_BYTES
) {
throw new Error('Legacy log path must be a bounded relative POSIX path');
}
const normalized = path.posix.normalize(value);
if (
normalized === '.' ||
normalized === '..' ||
normalized.startsWith('../') ||
normalized.split('/').includes('..')
) {
throw new Error('Legacy log path escapes its configured root');
}
return normalized;
}
export function createLegacyLogOutputRef(logPath: string): string {
const normalized = normalizeRelativeLogPath(logPath);
const outputRef = `${LEGACY_LOG_OUTPUT_REF_PREFIX}${Buffer.from(
normalized,
'utf8',
).toString('base64url')}`;
if (Buffer.byteLength(outputRef, 'utf8') > MAX_LEGACY_LOG_OUTPUT_REF_BYTES) {
throw new Error('Legacy log output reference exceeds its size limit');
}
return outputRef;
}
export function parseLegacyLogOutputRef(outputRef?: string): string | null {
if (
!outputRef ||
!outputRef.startsWith(LEGACY_LOG_OUTPUT_REF_PREFIX) ||
Buffer.byteLength(outputRef, 'utf8') > MAX_LEGACY_LOG_OUTPUT_REF_BYTES
) {
return null;
}
const encoded = outputRef.slice(LEGACY_LOG_OUTPUT_REF_PREFIX.length);
if (!encoded || !/^[A-Za-z0-9_-]+$/.test(encoded)) return null;
try {
const bytes = Buffer.from(encoded, 'base64url');
if (bytes.toString('base64url') !== encoded) return null;
const decoded = bytes.toString('utf8');
if (Buffer.from(decoded, 'utf8').compare(bytes) !== 0) return null;
const normalized = normalizeRelativeLogPath(decoded);
return normalized === decoded ? decoded : null;
} catch {
return null;
}
}
@@ -0,0 +1,48 @@
import { createHash } from 'crypto';
export interface LegacyTaskRevisionInput {
command: string;
schedule?: string;
extraSchedules?: readonly string[];
taskBefore?: string;
taskAfter?: string;
workDirectory?: string;
logName?: string;
environmentRevision?: string;
sourceRevision?: string;
}
export function createLegacyLogArtifactId(logPath: string): string {
return `legacy-log:${createHash('sha256')
.update(logPath)
.digest('hex')
.slice(0, 25)}`;
}
export function createLegacyTaskRevision(
input: LegacyTaskRevisionInput,
): string {
const snapshot = {
schema: 1,
command: input.command,
...(input.schedule === undefined ? {} : { schedule: input.schedule }),
...(input.extraSchedules === undefined
? {}
: { extraSchedules: [...input.extraSchedules] }),
...(input.taskBefore === undefined ? {} : { taskBefore: input.taskBefore }),
...(input.taskAfter === undefined ? {} : { taskAfter: input.taskAfter }),
...(input.workDirectory === undefined
? {}
: { workDirectory: input.workDirectory }),
...(input.logName === undefined ? {} : { logName: input.logName }),
...(input.environmentRevision === undefined
? {}
: { environmentRevision: input.environmentRevision }),
...(input.sourceRevision === undefined
? {}
: { sourceRevision: input.sourceRevision }),
};
return `sha256:${createHash('sha256')
.update(JSON.stringify(snapshot))
.digest('hex')}`;
}
@@ -0,0 +1,127 @@
import type { ExecutionOutcome } from '../domain/execution';
export interface ManualPrimaryCronSnapshot {
id: number;
name?: string;
command: string;
schedule?: string;
extraSchedules: readonly string[];
taskBefore?: string;
taskAfter?: string;
workDirectory?: string;
logName?: string;
}
export interface ManualPrimaryStartInput {
cron: ManualPrimaryCronSnapshot;
acceptedAtMs: number;
}
export interface ManualPrimaryCompletion {
runId: string;
attemptId: string;
outcome: ExecutionOutcome;
exitCode?: number;
}
export interface ManualPrimaryStartedExecution {
runId: string;
attemptId: string;
pid?: number;
logPath: string;
completion: Promise<ManualPrimaryCompletion>;
}
export interface ManualPrimaryStopResult {
matched: number;
failed: number;
}
export interface ManualPrimaryExecutionRouter {
/** Immutable selection for new manual triggers; off/shadow return false. */
ownsNewRuns(): boolean;
start(input: ManualPrimaryStartInput): Promise<ManualPrimaryStartedExecution>;
stopCron(
cronId: number,
requestedAtMs: number,
): Promise<ManualPrimaryStopResult>;
stopAttempt(
attemptId: string,
requestedAtMs: number,
): Promise<ManualPrimaryStopResult>;
}
let triggerRouter: ManualPrimaryExecutionRouter | undefined;
const ownerRouters = new Map<ManualPrimaryExecutionRouter, number>();
/**
* Returns the selected owner object so a concurrent config change cannot switch
* owner between the decision and start calls.
*/
export function selectManualPrimaryExecutionRouter():
| ManualPrimaryExecutionRouter
| undefined {
const selected = triggerRouter;
if (!selected) return undefined;
try {
return selected.ownsNewRuns() ? selected : undefined;
} catch {
// Invalid/unavailable configuration is off before Runtime accepts a Run.
return undefined;
}
}
export async function stopManualPrimaryCron(
cronId: number,
requestedAtMs: number,
): Promise<ManualPrimaryStopResult> {
return stopAcrossOwners((owner) => owner.stopCron(cronId, requestedAtMs));
}
export async function stopManualPrimaryAttempt(
attemptId: string,
requestedAtMs: number,
): Promise<ManualPrimaryStopResult> {
return stopAcrossOwners((owner) =>
owner.stopAttempt(attemptId, requestedAtMs),
);
}
async function stopAcrossOwners(
stop: (
router: ManualPrimaryExecutionRouter,
) => Promise<ManualPrimaryStopResult>,
): Promise<ManualPrimaryStopResult> {
const results = await Promise.allSettled([...ownerRouters.keys()].map(stop));
let matched = 0;
let failed = 0;
for (const result of results) {
if (result.status === 'fulfilled') {
matched += result.value.matched;
failed += result.value.failed;
} else {
// Conservatively protect a possibly-owned execution from PID fallback.
matched += 1;
failed += 1;
}
}
return { matched, failed };
}
/**
* Installs one manifest-gated owner. Default boot has no router unless the
* lightweight HTTP bootstrap accepts an explicit manual Primary manifest.
*/
export function installManualPrimaryExecutionRouter(
next: ManualPrimaryExecutionRouter,
): () => void {
const previous = triggerRouter;
triggerRouter = next;
ownerRouters.set(next, (ownerRouters.get(next) ?? 0) + 1);
return () => {
if (triggerRouter === next) triggerRouter = previous;
const references = ownerRouters.get(next);
if (references === 1) ownerRouters.delete(next);
else if (references !== undefined) ownerRouters.set(next, references - 1);
};
}
@@ -0,0 +1,42 @@
import type { ChildProcess } from 'child_process';
import type { LegacyExecutionObservation } from '../ports/legacyExecutionObserver';
export interface LegacyProcessObservationOptions {
now?: () => number;
logArtifactId?: string;
}
export function observeLegacyChildProcess(
child: ChildProcess,
observation: LegacyExecutionObservation,
options: LegacyProcessObservationOptions = {},
): void {
const now = options.now ?? Date.now;
child.once('spawn', () => {
const atMs = now();
observation.spawned({
atMs,
...(child.pid === undefined ? {} : { pid: child.pid }),
...(child.pid === undefined
? {}
: { executorHandle: `legacy-local:${child.pid}` }),
...(options.logArtifactId === undefined
? {}
: { logArtifactId: options.logArtifactId }),
});
observation.running({ atMs });
});
child.on('error', () => {
observation.startFailed({
atMs: now(),
errorCode: 'LEGACY_PROCESS_ERROR',
});
});
child.once('exit', (exitCode, signal) => {
observation.exited({
atMs: now(),
exitCode,
...(signal === null ? {} : { signal }),
});
});
}