mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): report legacy shadow startup differences
This commit is contained in:
whyour
@@ -6,7 +6,12 @@ import {
|
||||
} from '../../domain/deploymentProfile';
|
||||
import { configuredLegacyShadowOrigins } from '../../compatibility/legacyExecutionBridge';
|
||||
import { createLegacyLogArtifactId } from '../../compatibility/legacyTaskRevision';
|
||||
import type { LegacyShadowStartupSummary } from '../../application/legacyShadowStartupReconciler';
|
||||
import {
|
||||
LEGACY_SHADOW_STARTUP_OUTCOMES,
|
||||
type LegacyShadowStartupOriginSummary,
|
||||
type LegacyShadowStartupOutcomeCounts,
|
||||
type LegacyShadowStartupSummary,
|
||||
} from '../../application/legacyShadowStartupReconciler';
|
||||
|
||||
export interface LegacyShadowStartupBudget {
|
||||
pageSize: number;
|
||||
@@ -18,6 +23,60 @@ export interface LegacyShadowStartupRequest extends LegacyShadowStartupBudget {
|
||||
profile: 'edge' | 'standalone';
|
||||
}
|
||||
|
||||
export const LEGACY_SHADOW_STARTUP_DIFFERENCE_REPORT_SCHEMA =
|
||||
'qinglong/legacy-shadow-startup-difference-report@v1';
|
||||
export const LEGACY_SHADOW_STARTUP_METRIC_BATCH_SCHEMA =
|
||||
'qinglong/legacy-shadow-startup-metric-batch@v1';
|
||||
|
||||
export type LegacyShadowStartupAssessment =
|
||||
| 'converged'
|
||||
| 'waiting_external_callback'
|
||||
| 'incomplete'
|
||||
| 'attention_required';
|
||||
|
||||
export interface LegacyShadowStartupDifferenceReport {
|
||||
schemaVersion: 1;
|
||||
schema: typeof LEGACY_SHADOW_STARTUP_DIFFERENCE_REPORT_SCHEMA;
|
||||
profile: 'edge' | 'standalone';
|
||||
assessment: LegacyShadowStartupAssessment;
|
||||
configuredOriginCount: number;
|
||||
budget: {
|
||||
pageSize: number;
|
||||
maxPages: number;
|
||||
maxCandidates: number;
|
||||
};
|
||||
coverage: {
|
||||
pages: number;
|
||||
scanned: number;
|
||||
stopReason: LegacyShadowStartupSummary['stopReason'];
|
||||
remaining: boolean;
|
||||
resumeAvailable: boolean;
|
||||
};
|
||||
outcomes: LegacyShadowStartupOutcomeCounts;
|
||||
byOrigin: readonly LegacyShadowStartupOriginSummary[];
|
||||
}
|
||||
|
||||
export interface LegacyShadowStartupMetricBatch {
|
||||
schemaVersion: 1;
|
||||
schema: typeof LEGACY_SHADOW_STARTUP_METRIC_BATCH_SCHEMA;
|
||||
dimensions: {
|
||||
profile: 'edge' | 'standalone';
|
||||
assessment: LegacyShadowStartupAssessment;
|
||||
stopReason: LegacyShadowStartupSummary['stopReason'];
|
||||
};
|
||||
values: LegacyShadowStartupOutcomeCounts & {
|
||||
configuredOrigins: number;
|
||||
pageSize: number;
|
||||
maxPages: number;
|
||||
maxCandidates: number;
|
||||
pages: number;
|
||||
scanned: number;
|
||||
remaining: 0 | 1;
|
||||
resumeAvailable: 0 | 1;
|
||||
};
|
||||
byOrigin: readonly LegacyShadowStartupOriginSummary[];
|
||||
}
|
||||
|
||||
export type LegacyShadowStartupAudit =
|
||||
| {
|
||||
state: 'disabled';
|
||||
@@ -30,9 +89,11 @@ export type LegacyShadowStartupAudit =
|
||||
state: 'reconciled' | 'incomplete';
|
||||
profile: 'edge' | 'standalone';
|
||||
origins: number;
|
||||
summary: Omit<LegacyShadowStartupSummary, 'nextCursor'> & {
|
||||
summary: Omit<LegacyShadowStartupSummary, 'nextCursor' | 'byOrigin'> & {
|
||||
resumeAvailable: boolean;
|
||||
};
|
||||
report: LegacyShadowStartupDifferenceReport;
|
||||
metrics: LegacyShadowStartupMetricBatch;
|
||||
}
|
||||
| {
|
||||
state: 'failed';
|
||||
@@ -46,6 +107,7 @@ export interface BootstrapLegacyShadowStartupOptions {
|
||||
request: LegacyShadowStartupRequest,
|
||||
) => Promise<LegacyShadowStartupSummary>;
|
||||
audit?: (record: LegacyShadowStartupAudit) => void | Promise<void>;
|
||||
collect?: (metrics: LegacyShadowStartupMetricBatch) => void | Promise<void>;
|
||||
}
|
||||
|
||||
const BUDGETS: Readonly<
|
||||
@@ -54,17 +116,195 @@ const BUDGETS: Readonly<
|
||||
edge: { pageSize: 8, maxPages: 1 },
|
||||
standalone: { pageSize: 32, maxPages: 4 },
|
||||
};
|
||||
const MAX_CONFIGURED_LEGACY_SHADOW_ORIGINS = 7;
|
||||
|
||||
function auditSummary(summary: LegacyShadowStartupSummary): Omit<
|
||||
LegacyShadowStartupSummary,
|
||||
'nextCursor'
|
||||
'nextCursor' | 'byOrigin'
|
||||
> & {
|
||||
resumeAvailable: boolean;
|
||||
} {
|
||||
const { nextCursor, ...bounded } = summary;
|
||||
const { nextCursor, byOrigin: _byOrigin, ...bounded } = summary;
|
||||
return { ...bounded, resumeAvailable: nextCursor !== undefined };
|
||||
}
|
||||
|
||||
function outcomeCounts(
|
||||
source: LegacyShadowStartupOutcomeCounts,
|
||||
): LegacyShadowStartupOutcomeCounts {
|
||||
return Object.fromEntries(
|
||||
LEGACY_SHADOW_STARTUP_OUTCOMES.map((outcome) => [outcome, source[outcome]]),
|
||||
) as LegacyShadowStartupOutcomeCounts;
|
||||
}
|
||||
|
||||
function outcomeTotal(source: LegacyShadowStartupOutcomeCounts): number {
|
||||
return LEGACY_SHADOW_STARTUP_OUTCOMES.reduce(
|
||||
(total, outcome) => total + source[outcome],
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
function assertCount(value: number, label: string, maximum: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0 || value > maximum) {
|
||||
throw new RangeError(`${label} is outside its startup budget`);
|
||||
}
|
||||
}
|
||||
|
||||
function assessment(
|
||||
summary: LegacyShadowStartupSummary,
|
||||
): LegacyShadowStartupAssessment {
|
||||
if (summary.ambiguous > 0 || summary.skipped > 0 || summary.failed > 0) {
|
||||
return 'attention_required';
|
||||
}
|
||||
if (summary.remaining) return 'incomplete';
|
||||
if (summary.pending > 0) return 'waiting_external_callback';
|
||||
return 'converged';
|
||||
}
|
||||
|
||||
export function createLegacyShadowStartupDifferenceReport(
|
||||
request: LegacyShadowStartupRequest,
|
||||
summary: LegacyShadowStartupSummary,
|
||||
): LegacyShadowStartupDifferenceReport {
|
||||
const expectedBudget = BUDGETS[request.profile];
|
||||
if (
|
||||
request.pageSize !== expectedBudget.pageSize ||
|
||||
request.maxPages !== expectedBudget.maxPages
|
||||
) {
|
||||
throw new RangeError('startup report budget does not match its Profile');
|
||||
}
|
||||
const configuredOrigins = [...new Set(request.origins)];
|
||||
if (
|
||||
configuredOrigins.length < 1 ||
|
||||
configuredOrigins.length > MAX_CONFIGURED_LEGACY_SHADOW_ORIGINS
|
||||
) {
|
||||
throw new RangeError(
|
||||
'configured Shadow origin count is outside its budget',
|
||||
);
|
||||
}
|
||||
const maxCandidates = request.pageSize * request.maxPages;
|
||||
assertCount(summary.pages, 'pages', request.maxPages);
|
||||
if (summary.pages < 1) {
|
||||
throw new RangeError('pages must include the executed startup page');
|
||||
}
|
||||
assertCount(summary.scanned, 'scanned', request.pageSize * summary.pages);
|
||||
for (const outcome of LEGACY_SHADOW_STARTUP_OUTCOMES) {
|
||||
assertCount(summary[outcome], outcome, summary.scanned);
|
||||
}
|
||||
if (outcomeTotal(summary) !== summary.scanned) {
|
||||
throw new RangeError('startup outcomes do not conserve scanned candidates');
|
||||
}
|
||||
if (summary.remaining !== (summary.stopReason !== 'complete')) {
|
||||
throw new RangeError('startup remaining and stop reason disagree');
|
||||
}
|
||||
if (
|
||||
!['complete', 'page_limit', 'cursor_stalled'].includes(summary.stopReason)
|
||||
) {
|
||||
throw new RangeError('startup stop reason is invalid');
|
||||
}
|
||||
if (summary.stopReason === 'page_limit' && summary.nextCursor === undefined) {
|
||||
throw new RangeError('page-limited startup summary has no resume cursor');
|
||||
}
|
||||
if (summary.stopReason !== 'page_limit' && summary.nextCursor !== undefined) {
|
||||
throw new RangeError('resume cursor is present without a page limit');
|
||||
}
|
||||
|
||||
if (summary.byOrigin.length !== configuredOrigins.length) {
|
||||
throw new RangeError('origin outcome coverage is incomplete');
|
||||
}
|
||||
const origins = new Map<ExecutionOrigin, LegacyShadowStartupOriginSummary>();
|
||||
for (const origin of summary.byOrigin) {
|
||||
if (
|
||||
origins.has(origin.origin) ||
|
||||
!configuredOrigins.includes(origin.origin)
|
||||
) {
|
||||
throw new RangeError('origin outcome coverage is invalid');
|
||||
}
|
||||
assertCount(origin.scanned, `${origin.origin}:scanned`, summary.scanned);
|
||||
for (const outcome of LEGACY_SHADOW_STARTUP_OUTCOMES) {
|
||||
assertCount(
|
||||
origin[outcome],
|
||||
`${origin.origin}:${outcome}`,
|
||||
origin.scanned,
|
||||
);
|
||||
}
|
||||
if (outcomeTotal(origin) !== origin.scanned) {
|
||||
throw new RangeError(
|
||||
'origin outcomes do not conserve scanned candidates',
|
||||
);
|
||||
}
|
||||
origins.set(origin.origin, {
|
||||
origin: origin.origin,
|
||||
scanned: origin.scanned,
|
||||
...outcomeCounts(origin),
|
||||
});
|
||||
}
|
||||
const orderedOrigins = configuredOrigins.map(
|
||||
(origin) => origins.get(origin)!,
|
||||
);
|
||||
if (
|
||||
orderedOrigins.reduce((total, origin) => total + origin.scanned, 0) !==
|
||||
summary.scanned ||
|
||||
LEGACY_SHADOW_STARTUP_OUTCOMES.some(
|
||||
(outcome) =>
|
||||
orderedOrigins.reduce((total, origin) => total + origin[outcome], 0) !==
|
||||
summary[outcome],
|
||||
)
|
||||
) {
|
||||
throw new RangeError('origin outcomes do not match aggregate outcomes');
|
||||
}
|
||||
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
schema: LEGACY_SHADOW_STARTUP_DIFFERENCE_REPORT_SCHEMA,
|
||||
profile: request.profile,
|
||||
assessment: assessment(summary),
|
||||
configuredOriginCount: configuredOrigins.length,
|
||||
budget: {
|
||||
pageSize: request.pageSize,
|
||||
maxPages: request.maxPages,
|
||||
maxCandidates,
|
||||
},
|
||||
coverage: {
|
||||
pages: summary.pages,
|
||||
scanned: summary.scanned,
|
||||
stopReason: summary.stopReason,
|
||||
remaining: summary.remaining,
|
||||
resumeAvailable: summary.nextCursor !== undefined,
|
||||
},
|
||||
outcomes: outcomeCounts(summary),
|
||||
byOrigin: orderedOrigins,
|
||||
};
|
||||
}
|
||||
|
||||
export function createLegacyShadowStartupMetricBatch(
|
||||
report: LegacyShadowStartupDifferenceReport,
|
||||
): LegacyShadowStartupMetricBatch {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
schema: LEGACY_SHADOW_STARTUP_METRIC_BATCH_SCHEMA,
|
||||
dimensions: {
|
||||
profile: report.profile,
|
||||
assessment: report.assessment,
|
||||
stopReason: report.coverage.stopReason,
|
||||
},
|
||||
values: {
|
||||
configuredOrigins: report.configuredOriginCount,
|
||||
pageSize: report.budget.pageSize,
|
||||
maxPages: report.budget.maxPages,
|
||||
maxCandidates: report.budget.maxCandidates,
|
||||
pages: report.coverage.pages,
|
||||
scanned: report.coverage.scanned,
|
||||
remaining: report.coverage.remaining ? 1 : 0,
|
||||
resumeAvailable: report.coverage.resumeAvailable ? 1 : 0,
|
||||
...outcomeCounts(report.outcomes),
|
||||
},
|
||||
byOrigin: report.byOrigin.map((origin) => ({
|
||||
origin: origin.origin,
|
||||
scanned: origin.scanned,
|
||||
...outcomeCounts(origin),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
async function executeDefault(
|
||||
request: LegacyShadowStartupRequest,
|
||||
): Promise<LegacyShadowStartupSummary> {
|
||||
@@ -137,22 +377,32 @@ export async function bootstrapLegacyShadowStartupReconciliation(
|
||||
}
|
||||
return record;
|
||||
}
|
||||
const summary = await (options.execute ?? executeDefault)({
|
||||
const request: LegacyShadowStartupRequest = {
|
||||
origins,
|
||||
profile,
|
||||
...BUDGETS[profile],
|
||||
});
|
||||
};
|
||||
const summary = await (options.execute ?? executeDefault)(request);
|
||||
const report = createLegacyShadowStartupDifferenceReport(request, summary);
|
||||
const metrics = createLegacyShadowStartupMetricBatch(report);
|
||||
const record: LegacyShadowStartupAudit = {
|
||||
state: summary.remaining ? 'incomplete' : 'reconciled',
|
||||
state: report.assessment === 'converged' ? 'reconciled' : 'incomplete',
|
||||
profile,
|
||||
origins: origins.length,
|
||||
summary: auditSummary(summary),
|
||||
report,
|
||||
metrics,
|
||||
};
|
||||
try {
|
||||
await audit(record);
|
||||
} catch {
|
||||
// Shadow audit output must not affect Legacy startup.
|
||||
}
|
||||
try {
|
||||
await options.collect?.(metrics);
|
||||
} catch {
|
||||
// Shadow metric collection must not affect Legacy startup.
|
||||
}
|
||||
return record;
|
||||
} catch (error) {
|
||||
const record: LegacyShadowStartupAudit = {
|
||||
|
||||
@@ -23,41 +23,86 @@ export interface LegacyShadowStartupClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface LegacyShadowStartupReconcileSummary {
|
||||
scanned: number;
|
||||
completed: number;
|
||||
cancelled: number;
|
||||
abandoned: number;
|
||||
markedLost: number;
|
||||
repaired: number;
|
||||
pending: number;
|
||||
ambiguous: number;
|
||||
skipped: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
nextCursor?: LegacyShadowStartupCursor;
|
||||
}
|
||||
export const LEGACY_SHADOW_STARTUP_OUTCOMES = Object.freeze([
|
||||
'completed',
|
||||
'cancelled',
|
||||
'abandoned',
|
||||
'markedLost',
|
||||
'repaired',
|
||||
'pending',
|
||||
'ambiguous',
|
||||
'skipped',
|
||||
'failed',
|
||||
] as const);
|
||||
|
||||
export type LegacyShadowStartupOutcome =
|
||||
(typeof LEGACY_SHADOW_STARTUP_OUTCOMES)[number];
|
||||
|
||||
export type LegacyShadowStartupOutcomeCounts = Record<
|
||||
LegacyShadowStartupOutcome,
|
||||
number
|
||||
>;
|
||||
|
||||
export type LegacyShadowStartupOriginSummary =
|
||||
LegacyShadowStartupOutcomeCounts & {
|
||||
origin: RunRecord['executionOrigin'];
|
||||
scanned: number;
|
||||
};
|
||||
|
||||
export type LegacyShadowStartupReconcileSummary =
|
||||
LegacyShadowStartupOutcomeCounts & {
|
||||
scanned: number;
|
||||
byOrigin: readonly LegacyShadowStartupOriginSummary[];
|
||||
truncated: boolean;
|
||||
nextCursor?: LegacyShadowStartupCursor;
|
||||
};
|
||||
|
||||
export type LegacyShadowStartupStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface LegacyShadowStartupSummary
|
||||
extends Omit<
|
||||
LegacyShadowStartupReconcileSummary,
|
||||
'truncated' | 'nextCursor'
|
||||
> {
|
||||
export type LegacyShadowStartupSummary = Omit<
|
||||
LegacyShadowStartupReconcileSummary,
|
||||
'truncated' | 'nextCursor'
|
||||
> & {
|
||||
pages: number;
|
||||
stopReason: LegacyShadowStartupStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: LegacyShadowStartupCursor;
|
||||
}
|
||||
};
|
||||
|
||||
type EvidenceSelection =
|
||||
| { status: 'matched'; evidence: LegacyRunningInstanceEvidence }
|
||||
| { status: 'none' | 'ambiguous' };
|
||||
|
||||
function emptyOutcomeCounts(): LegacyShadowStartupOutcomeCounts {
|
||||
return {
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 0,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function originSummary(
|
||||
origin: RunRecord['executionOrigin'],
|
||||
): LegacyShadowStartupOriginSummary {
|
||||
return { origin, scanned: 0, ...emptyOutcomeCounts() };
|
||||
}
|
||||
|
||||
function incrementOutcome(
|
||||
target: LegacyShadowStartupOutcomeCounts,
|
||||
outcome: LegacyShadowStartupOutcome,
|
||||
): void {
|
||||
target[outcome] += 1;
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: LegacyShadowStartupCursor | undefined,
|
||||
right: LegacyShadowStartupCursor,
|
||||
@@ -168,27 +213,34 @@ export class LegacyShadowStartupReconciler {
|
||||
...(options.cursor === undefined ? {} : { cursor: options.cursor }),
|
||||
...(options.limit === undefined ? {} : { limit: options.limit }),
|
||||
});
|
||||
const originSummaries = new Map(
|
||||
enabledOrigins.map((origin) => [origin, originSummary(origin)]),
|
||||
);
|
||||
const summary: LegacyShadowStartupReconcileSummary = {
|
||||
scanned: page.candidates.length,
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 0,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
...emptyOutcomeCounts(),
|
||||
byOrigin: [...originSummaries.values()],
|
||||
truncated: page.truncated,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
};
|
||||
const origins = new Set(enabledOrigins);
|
||||
for (const candidate of page.candidates) {
|
||||
try {
|
||||
await this.reconcileCandidate(candidate, origins, summary);
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
const perOrigin =
|
||||
originSummaries.get(candidate.origin) ??
|
||||
originSummary(candidate.origin);
|
||||
if (!originSummaries.has(candidate.origin)) {
|
||||
originSummaries.set(candidate.origin, perOrigin);
|
||||
summary.byOrigin = [...originSummaries.values()];
|
||||
}
|
||||
perOrigin.scanned += 1;
|
||||
let outcome: LegacyShadowStartupOutcome;
|
||||
try {
|
||||
outcome = await this.reconcileCandidate(candidate, origins);
|
||||
} catch {
|
||||
outcome = 'failed';
|
||||
}
|
||||
incrementOutcome(summary, outcome);
|
||||
incrementOutcome(perOrigin, outcome);
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
@@ -196,26 +248,23 @@ export class LegacyShadowStartupReconciler {
|
||||
private async reconcileCandidate(
|
||||
candidate: LegacyShadowStartupCandidate,
|
||||
origins: ReadonlySet<RunRecord['executionOrigin']>,
|
||||
summary: LegacyShadowStartupReconcileSummary,
|
||||
): Promise<void> {
|
||||
): Promise<LegacyShadowStartupOutcome> {
|
||||
const run = await this.repository.findRunById(candidate.runId);
|
||||
if (
|
||||
!run ||
|
||||
run.executionOwner !== 'legacy' ||
|
||||
!origins.has(run.executionOrigin) ||
|
||||
run.executionOrigin !== candidate.origin ||
|
||||
isTerminalRunStatus(run.status) ||
|
||||
run.status === 'lost'
|
||||
) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
return 'skipped';
|
||||
}
|
||||
if (candidate.activeAttemptCount === 0) {
|
||||
await this.repairTerminalAttempt(run, summary);
|
||||
return;
|
||||
return this.repairTerminalAttempt(run);
|
||||
}
|
||||
if (candidate.activeAttemptCount !== 1 || !candidate.attempt) {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
return 'ambiguous';
|
||||
}
|
||||
const attempt = await this.repository.findAttemptById(
|
||||
candidate.attempt.attemptId,
|
||||
@@ -225,8 +274,7 @@ export class LegacyShadowStartupReconciler {
|
||||
attempt.runId !== run.id ||
|
||||
isTerminalRunAttemptStatus(attempt.status)
|
||||
) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
return 'skipped';
|
||||
}
|
||||
|
||||
const evidence =
|
||||
@@ -236,13 +284,11 @@ export class LegacyShadowStartupReconciler {
|
||||
legacyCronId: candidate.legacyCronId,
|
||||
});
|
||||
if (evidence.truncated) {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
return 'ambiguous';
|
||||
}
|
||||
const selected = selectEvidence(attempt, evidence.evidence);
|
||||
if (selected.status === 'ambiguous') {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
return 'ambiguous';
|
||||
}
|
||||
if (
|
||||
selected.status === 'matched' &&
|
||||
@@ -254,8 +300,7 @@ export class LegacyShadowStartupReconciler {
|
||||
{ runId: run.id, attemptId: attempt.id },
|
||||
{ atMs, reason: 'reconcile' },
|
||||
);
|
||||
summary.cancelled += 1;
|
||||
return;
|
||||
return 'cancelled';
|
||||
}
|
||||
if (
|
||||
selected.evidence.outcome === 'succeeded' ||
|
||||
@@ -274,35 +319,30 @@ export class LegacyShadowStartupReconciler {
|
||||
: selected.evidence.exitCode,
|
||||
},
|
||||
);
|
||||
summary.completed += 1;
|
||||
return;
|
||||
return 'completed';
|
||||
}
|
||||
}
|
||||
|
||||
if (run.executionOrigin === 'scheduled_system') {
|
||||
summary.pending += 1;
|
||||
return;
|
||||
return 'pending';
|
||||
}
|
||||
if (run.status === 'queued' && attempt.status === 'claimed') {
|
||||
await this.abandon(run, attempt);
|
||||
summary.abandoned += 1;
|
||||
return;
|
||||
return 'abandoned';
|
||||
}
|
||||
if (
|
||||
(run.status === 'dispatching' || run.status === 'running') &&
|
||||
['claimed', 'starting', 'running'].includes(attempt.status)
|
||||
) {
|
||||
await this.markLost(run, attempt);
|
||||
summary.markedLost += 1;
|
||||
return;
|
||||
return 'markedLost';
|
||||
}
|
||||
summary.ambiguous += 1;
|
||||
return 'ambiguous';
|
||||
}
|
||||
|
||||
private async repairTerminalAttempt(
|
||||
run: RunRecord,
|
||||
summary: LegacyShadowStartupReconcileSummary,
|
||||
): Promise<void> {
|
||||
): Promise<'repaired' | 'ambiguous'> {
|
||||
const attempt = await this.repository.findLatestAttemptByRunId(run.id);
|
||||
const target = attempt ? terminalRunTarget(attempt) : undefined;
|
||||
if (
|
||||
@@ -310,8 +350,7 @@ export class LegacyShadowStartupReconciler {
|
||||
target === undefined ||
|
||||
!RUN_TRANSITIONS[run.status].includes(target)
|
||||
) {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
return 'ambiguous';
|
||||
}
|
||||
await this.commands.transitionRun({
|
||||
runId: run.id,
|
||||
@@ -327,7 +366,7 @@ export class LegacyShadowStartupReconciler {
|
||||
actor: { type: 'reconciler' },
|
||||
dedupeKey: `legacy-startup-repair:${attempt.id}:${target}`,
|
||||
});
|
||||
summary.repaired += 1;
|
||||
return 'repaired';
|
||||
}
|
||||
|
||||
private async abandon(
|
||||
@@ -438,18 +477,17 @@ export class LegacyShadowStartupSupervisor {
|
||||
);
|
||||
}
|
||||
|
||||
const originSummaries = new Map(
|
||||
[...new Set(options.origins)].map((origin) => [
|
||||
origin,
|
||||
originSummary(origin),
|
||||
]),
|
||||
);
|
||||
const total: LegacyShadowStartupSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 0,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
...emptyOutcomeCounts(),
|
||||
byOrigin: [...originSummaries.values()],
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
@@ -471,6 +509,19 @@ export class LegacyShadowStartupSupervisor {
|
||||
total.ambiguous += page.ambiguous;
|
||||
total.skipped += page.skipped;
|
||||
total.failed += page.failed;
|
||||
for (const pageOrigin of page.byOrigin) {
|
||||
const aggregate =
|
||||
originSummaries.get(pageOrigin.origin) ??
|
||||
originSummary(pageOrigin.origin);
|
||||
if (!originSummaries.has(pageOrigin.origin)) {
|
||||
originSummaries.set(pageOrigin.origin, aggregate);
|
||||
total.byOrigin = [...originSummaries.values()];
|
||||
}
|
||||
aggregate.scanned += pageOrigin.scanned;
|
||||
for (const outcome of LEGACY_SHADOW_STARTUP_OUTCOMES) {
|
||||
aggregate[outcome] += pageOrigin[outcome];
|
||||
}
|
||||
}
|
||||
if (!page.truncated) return total;
|
||||
if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
|
||||
@@ -11,6 +11,22 @@
|
||||
|
||||
最新增量证据（2026-08-18）：
|
||||
|
||||
- D-357/ADR-0449（已接受）：将 D-356 的进程内 startup summary 收紧为可供后续 origin-scoped gate 使用的版本化差异证据。每个候选只归入
|
||||
completed/cancelled/abandoned/markedLost/repaired/pending/ambiguous/skipped/failed 九个固定 outcome，总量与最多七条已配置 origin
|
||||
matrix 必须逐级守恒；bootstrap 还验证 Profile 页/候选预算、origin exact coverage、remaining/stopReason 与 page-limit cursor，畸形 summary
|
||||
继续失败开放。`qinglong/legacy-shadow-startup-difference-report@v1` 只输出 Profile、预算、覆盖范围、固定计数与
|
||||
converged/waiting_external_callback/incomplete/attention_required 四态 assessment，不含 Run/Cron/Attempt/PID/log/task/user/error
|
||||
原文，也不在未知分母上伪造完整率。`qinglong/legacy-shadow-startup-metric-batch@v1` 以 profile/assessment/stopReason 三个固定维度和固定数值
|
||||
字段提供单次 snapshot；默认结构化 audit 可直接采集，组合方也可注入一次性 collector，collector 失败不影响 Legacy。实现复用同一次扫描，只增加
|
||||
最多七行常数内存，不新增 package、生产依赖、查询、schema/migration、连接、timer/watcher、进程、端口或部署对象；默认 off 与
|
||||
cluster-control/worker 仍为零 Repository/报告/指标。本切片不把 startup snapshot 冒充跨测量窗口终态对账、具体 exporter、资源/回滚证据或
|
||||
Primary gate。阶段门已重跑：聚焦 `21/21`、Legacy/Shadow 串行扩展 `77/77`、`build:back`、完整 backend
|
||||
`1,440 pass / 0 fail / 2 conditional skip`、18-package clean build/test、14/14 static audit 与 14/14 artifact audit 全部通过。
|
||||
edge/standalone 产物字节继续为：base `2,589,998 / 2,590,076`、adopted `2,809,293 / 2,809,416`、application
|
||||
`3,632,877 / 3,632,997`、application-api `3,800,430 / 3,800,574`、AI `3,069,251 / 3,069,341`、
|
||||
application+AI `4,493,151 / 4,493,283`、MCP `7,315,930 / 7,316,038`，与 D-356 完全一致。由于本阶段不改数据库、
|
||||
容器或 Kubernetes 部署面，不重跑物理 PostgreSQL HA/K3s；对象存储恢复/PITR 与 cert-manager mTLS 轮换仍是既有发布最终化现场证据门。
|
||||
|
||||
- D-356/ADR-0448（已接受）：补齐监听前、一次性且 Profile-aware 的 Legacy Shadow Startup Reconciler。它只在至少一个已审 origin 显式
|
||||
Shadow 时，于 Legacy `initData` 归一之后、manual Primary activation/HTTP listen 之前加载；默认 off 与 cluster-control/worker 均保持零
|
||||
Repository/恢复查询/写入。只读 Source 使用 `(created_at_ms, run_id)` keyset，单页硬上限 64、每 Cron RunningInstance 证据硬上限 8，原始
|
||||
@@ -9099,7 +9115,7 @@ flowchart LR
|
||||
| PR-1 Run Schema | Incubating | Run/RunAttempt/RunEvent schema、nullable cancel request 与 Attempt deadline 字段及恢复索引、CancellationDispatch 状态/version/lease/backoff schema、Repository port、临时 Sequelize adapter、统一事件大小/分页上限、跨 adapter RunRepository contract suite（原子事务、回滚、Run/Attempt/RetryPolicy CAS、唯一错误、分页与取消恢复）；ADR-0041 的 `pg-0003-run-retry-policy`、capability v2、driver-neutral PostgreSQL Run Repository 与真实 `pg.Pool` 上的共享 Repository/rollback/SQLSTATE contract；ADR-0063/0069/0071/0073/0074/0076 的独立 Node 24 local-sqlite typed schema、十二条 reviewed migration、capability v6、共享 operation authority、readiness/RunRepository/API credential repository/receipt journal/dispatch plan/encrypted Secret envelope/Project Policy/security audit/authorized mutation/stable Identity catalog、Drizzle↔真实 catalog table/column/index/CHECK/FK lockstep、base/adopted/application edge/standalone 产物门禁；ADR-0064 的 legacy baseline/plan digest、Online Backup recovery、side-by-side target migration、staged manifest、双库栅栏 activation、source 生命周期写栅栏、target stable identity 和重启语义；ADR-0065 的独立 cutover authority、外部副作用停机 evidence、append-only journal、start/restart/stop barrier 与 unknown→manual_required 收敛；ADR-0066 的 adopted storage→Run reconciliation→receipt maintenance→domain recovery→lifecycle→admission application gate、严格有界 recovery summary 与 admission-first reverse stop；ADR-0067 的 SQLite 事实驱动 Run 候选源、256 条硬上限、截断失败关闭和唯一 Repository authority；ADR-0068 的 receipt-first Reconciler、callback token/sequence fence、exact local-process identity、Attempt/Run/双 Event 原子终态推进和最终 verifier；ADR-0069 的 local-process 单向包边界、pre-spawn journal、受审 POSIX launcher、immutable receipt、exact identity 和 Profile-aware cleanup lifecycle；ADR-0070 的独立 local-execution、spawn 前后双 transaction CAS、callback digest、exact stop 补偿与 fail-closed starting 保留；ADR-0071 的独立 local-dispatch、不可变 revision/context、Secret-first materializer、Profile Artifact admission、4/64 MiB output hard quota 和窄 application facade；ADR-0073/0074 的 Project-bound SecretRef、AES-256-GCM、外置 keyring 生命周期、双 SQLite authority CAS、application preflight、强 Principal/Policy 和 envelope+audit 原子提交；ADR-0086 的本机 Owner provisioning/challenge/claim/delivery acknowledgement/credential recovery CLI；ADR-0377 的 Local/Cluster 同构、Profile-aware、Project-scoped Artifact range read | fresh database/pepper setup、credential rotation/GC 运维编排与 Secret/Project/Role/Approval 管理 CLI/API/UI、备份/rekey、2.x/target process controller、人工 recovery、target 写后 reconciliation 与完整 cutover/rollback 演练；retry 产品策略、Artifact retention/tombstone stack、具体本机 lifecycle 和 target executable；Linux x64/arm64、PID namespace、断电与固定路由设备门禁；PostgreSQL 16/18 双连接并发与 failover integration；Task revision/context 跨方言 contract/并发压力与引用感知 retention、Keyv 数据迁移 |
|
||||
| PR-2 Run 状态机 | Incubating | 纯转换表、终态/时间/错误/执行器元数据规则、Run version 与 event sequence CAS、事务性 RunCommandService、回滚测试 | 重复 Worker callback/fencing、并发数据库压力测试、Primary 执行链接入 |
|
||||
| PR-3 Executor 端口 | Incubating | ADR-0003、ExecutionSpec/Context/Handle/Result、Executor port、LocalProcessExecutor、进程组取消/超时升级、流式背压、Legacy Cron spec builder、真实进程 contract tests、可复现 edge 基准入口 | 固定 edge/多架构设备基线、Legacy builder 与 makeCommand 差异审计、Primary 生产流量接入 |
|
||||
| PR-4 Shadow Run | Incubating | origin 三态策略；默认关闭的 `QL3_SHADOW_ORIGINS`；manual、scheduled_node、boot、subscription、system 与 script 现有 ChildProcess 旁路观察；system crond 显式 origin marker、Shell execution ID、finish-only 准入、确定性 Run/Attempt 与 exact replay；`@once` 保持 manual、gRPC transport 不冒充 origin 的准入裁决；每个 worker 懒加载；Run/Attempt/Event 影子生命周期；稳定且不复制 caller 原文的 task identity/revision 与有界日志引用；同 worker 有界注册表和跨 worker 持久化候选关联；stop all/stop instance、Shell callback、乱序/迟到/歧义处理；监听前一次性、Profile-aware 的 keyset Startup Reconciler，终态证据补齐、lost/abandoned/pending 分流与 terminal Attempt response-loss 修复；失败开放和契约测试 | 差异报表、可采集指标、资源压力、回滚演练和 Primary 门禁 |
|
||||
| PR-4 Shadow Run | Incubating | origin 三态策略；默认关闭的 `QL3_SHADOW_ORIGINS`；manual、scheduled_node、boot、subscription、system 与 script 现有 ChildProcess 旁路观察；system crond 显式 origin marker、Shell execution ID、finish-only 准入、确定性 Run/Attempt 与 exact replay；`@once` 保持 manual、gRPC transport 不冒充 origin 的准入裁决；每个 worker 懒加载；Run/Attempt/Event 影子生命周期；稳定且不复制 caller 原文的 task identity/revision 与有界日志引用；同 worker 有界注册表和跨 worker 持久化候选关联；stop all/stop instance、Shell callback、乱序/迟到/歧义处理；监听前一次性、Profile-aware 的 keyset Startup Reconciler，终态证据补齐、lost/abandoned/pending 分流与 terminal Attempt response-loss 修复；origin-bounded 且逐级守恒的版本化 startup difference report、固定字段 metric batch 与一次性 collector；失败开放和契约测试 | 跨测量窗口终态差异查询、具体 exporter、资源压力、回滚演练和 Primary 门禁 |
|
||||
| PR-5 Primary LocalExecutor | Incubating（默认不激活，仅 manifest-gated manual） | runtime-owned Run 创建器；持久化先于 spawn；Run/Attempt 完整成功、失败、取消、超时与 lost 闭环；Executor handle 身份校验；spawn 后激活写失败的 stop+lost 补偿；completion rejection 安全收敛；独立 Primary 幂等查询与唯一索引竞态裁决；durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询；最多 64 条一页的 cross-worker cancellation source；独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent；最多 64 页的单周期 cancel supervisor；显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner；Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller；完整有界分页且 fail-closed 的 startup Reconcile supervisor；RunningInstance nullable `run_id/attempt_id` 关联；Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance，失败整体回滚；有界且防穿越的 legacy log output ref；manual owner seam、真实本机装配、单 spawn/fail-closed；严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计；HTTP worker 已接轻量 lazy bootstrap，accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router 顺序激活，失败撤销，监听失败和 shutdown 有界停止；Primary timeout 在 spawn 前持久化绝对 deadline，有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch；代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限，cluster-control/worker 拒绝误装本机 SQLite Primary；统一 CompletionService 原子提交 Attempt/Run/双 Event，spawn 前保存 callback token hash、终态推进 sequence，实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window；manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理；Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace；`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描，使终态残留继续可发现；确定无效的已知 Attempt receipt 先持久化隔离状态，再进入确定性私有分片 quarantine；终态 missing 与 quarantine 按 edge/standalone retention 有界清理；非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI；扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数；ENOSPC 与 launcher receipt 存储失败有代码门禁 | 部署配置写入/审批入口与用户可见状态；PostgreSQL CancellationDispatch adapter；cluster-control 生产启动拓扑；固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练 |
|
||||
| PR-7 Worker Session、Run Lease 与启动协议基础 | Incubating（默认关闭，独立入口显式 opt-in） | ADR-0012/0013/0014/0021/0057–0061/0108–0121/0231–0239/0377；有界 capability/Placement/Dispatcher；SQLite 协议孵化与 PostgreSQL v9 Session/Run Lease/credential/attestation authority；immutable revision Placement、数据库时钟 keyset candidate、认证 Worker Pull、digest-only offer recovery；versioned capability-free ExecutionSpec response、stable claim 跨重启退避、单 owner 原子 inbox 准入与 TLS 1.3 mTLS/`ql3w` HTTPS client；同一 package journal 上 revision-fenced starting/spawn/started/running/completion 状态、callback digest、tagged no-spawn 与 ambiguous recovery；PostgreSQL starting/running/start-failure/completion 数据库权威事务、精确重放与 cancellation/timeout 优先终态；batch Secret delivery 在 Attempt advisory lock 下复验 Session/Lease/revision 完整围栏并复用单 Agent，Secret-before-Artifact materializer 将同一 log ID 交给 Executor/journal/running ACK；offer-scoped `wlog-*` 私有文件 spool、Edge/Node 容量策略、append/quota/path 防护、barrier 后 output ownership、受审 POSIX Executor、truncation fact、固定内存流式 source、认证 Artifact stream、共享 immutable store port、S3-compatible SSE/checksum/条件 promotion adapter、upload-before-completion 协调，以及 Local/Cluster 同构、Profile-aware、ETag-fenced range read；用户取消 run.stop mutation 以数据库时间写 intent/Event 并在事务内复验 Project/RoleBinding fence；非执行取消 convergence lifecycle、运行期 expiry 与安全 lost retry 已接入 cluster-control 单一全局 cadence；完整 generation/version/token/Attempt fencing；独立最小权限 Worker ingress、CA/CRL 与连接 generation 热重载；offer journal、spawn barrier、receipt-first recovery；独立 `@qinglong/worker-runtime` 的本地 P-256 CSR、key/chain/trust 验证、generation + active pointer 安装和持久退避；默认关闭的 production process 已装配具体 execution graph、完整 Session heartbeat/drain/offline、direct-file bootstrap、单 Agent/单 cadence、startup reconciliation、证书 maintenance、transport fail-close/recovery 与 Edge/Node 有界预算；真实 PostgreSQL 18 + Linux Node 合约已覆盖 Run completion、credential 和 CA 双轮换且保持同一 Session；真实 K3s 合约已覆盖 TLS/credential Secret 分权、双对象 CAS、Recreate 顺序、identity generation 与单节点 PVC recovery；所有能力默认不可达且受 edge/cluster import audit 约束 | 具体 cert-manager/Vault/SPIFFE/离线 CA adapter 与模板、ingress reload controller、生产 RBAC、证书到期告警和 `ql3w` credential recovery 产品面；具体 KMS/Vault Secret provider、对象存储 credential/temporary lifecycle 与 retention/tombstone；Worker 管理 API；真实 Kubernetes 多节点 CSI/node-loss/production 360 秒 drain 与固定 edge 文件系统 suspend/时钟/断电、x64/arm64 资源门禁 |
|
||||
| PR-8 Project/Policy/Approval Core | Incubating（默认拒绝、无生产业务执行入口） | ADR-0028；统一六类 ActorRef 与 exact-shape 校验；`0017` ownerless default Project 和 append-only versioned RoleBinding；owner/admin/operator/viewer 固定矩阵；Project 内 mutation 幂等、expected-version CAS、双 SQLite 连接竞争门禁；archived read-only、revocation、存储损坏 fail-closed；Agent 写/Secret/Tool `require_approval`；ADR-0047 把六类 subject、role/permission matrix 与 fence 抽到 runtime-core，`pg-0004-project-policy`/capability v3 建立 ownerless PostgreSQL baseline、严格 role/state CHECK、append-only runtime 权限、SERIALIZABLE Project lock、mutation replay、双连接单 winner 和 cluster admission authorizer；ADR-0049/`pg-0005` capability v4 建立 stable IdentitySubject、append-only digest-only API credential、真实 cluster bearer authenticator、write-only durable security audit 与最小权限 runtime role，且已验证 HTTP→credential→Policy→audit→handler 纵向链路；ADR-0051 建立 `/api/v3` 认证前 peer/global 双预算、transport-peer-only、无 timer 且有界内存的 overload shield；ADR-0027 Artifact authorizer adapter；ADR-0029 `AuthenticatedPrincipal` contract、`0018` digest-only versioned challenge、CSPRNG/TTL、同事务消费 challenge + 写首 owner、精确重放与双连接竞争/崩溃回滚门禁；ADR-0030 `0019` stable identity/binding、legacy HS384 + current-session membership、logout/platform/revoke/disable、single-factor 与损坏 fail-closed 门禁；ADR-0031 `0020` digest-bound ApprovalRequest、User-only decision、Project/Role version fence、精确 expiry/重放/并发裁决及同事务 immutable dispatch；ADR-0032 `0021` execution backfill、三表原子 consume、稳定 due keyset、claim/renew/start/result fencing、pre-start takeover/post-start recovery-required、attempt budget、handler inspect/digest barrier 和 bounded dispatcher；ADR-0033/`0022` control/resolution backfill、start/renew/completion 原子联动、稳定 recovery keyset、双 resolver claim/takeover、finding/result 精确重放、自动/人工终结、迟到 completion 单 winner 和 evidence-only bounded reconciler；ADR-0034/`0023` 首个 `run.create` canonical plan、Run/Attempt/Event/receipt 同事务、幂等 collision fail-closed、renew/终态 fence、真实 SQLite handler 与 automatic evidence provider；ADR-0035/`0024` 独立 `approval.recover` 矩阵、稳定 User + 五分钟强认证、Project/RoleBinding fence、human resolution + authorization fact 原子提交、撤权竞态与回滚门禁；ADR-0036 recovery-first 单 timer lifecycle、edge/standalone 独立 cadence/页预算、跨周期 cursor、非重叠与有界 stop；ADR-0074 以新的 Node 24 SQLite v5 ownerless Project/RoleBinding/audit authority 和独立 local-secret-admin 提供强 Principal、`secret.manage`、撤权 fence、envelope+allowed audit 原子提交及不回显语义；ADR-0086 以可信 POSIX console 和 staged delivery 完成本机首 Owner 产品 ceremony | fresh database/pepper setup 与安全迁移向导；`shareStore`/Express 到 authentication core 的 production migration；credential rotation/revocation API、mTLS/Worker enrollment、恢复码；Project/Role/Approval/Secret 管理 CLI/API/UI、audit retention/query/export/alert、preview Artifact/digest/immutable plan builder、真实 MFA/hardware adapter、人工 recovery API/UI/独立 rate limit 与审计事件、handler/provider registry、lifecycle startup/shutdown/指标/admission gate；PostgreSQL action/receipt/provider/recovery-authorization 与 OPA adapter、缓存 version 失效；Tool/Package/Secret/Shell 各自的 handler/evidence contract；Secret/Run/Tool/Workflow waiting_approval 全入口装配；完整回滚演练 |
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
- 决策者：QingLong Maintainers
|
||||
- 关联 RFC：[QL-RFC-0001](../QINGLONG_3_0_ARCHITECTURE_RFC.md)
|
||||
- 前置决策：[ADR-0001](./ADR-0001-run-state-and-transaction-boundaries.md)
|
||||
- Amended by：[ADR-0445](./ADR-0445-schedule-service-origin-shadow-run-coverage.md)、[ADR-0446](./ADR-0446-system-crond-stable-shadow-admission.md)、[ADR-0447](./ADR-0447-boot-shadow-and-non-origin-boundaries.md)、[ADR-0448](./ADR-0448-bounded-legacy-shadow-startup-reconciliation.md)
|
||||
- Amended by：[ADR-0445](./ADR-0445-schedule-service-origin-shadow-run-coverage.md)、[ADR-0446](./ADR-0446-system-crond-stable-shadow-admission.md)、[ADR-0447](./ADR-0447-boot-shadow-and-non-origin-boundaries.md)、[ADR-0448](./ADR-0448-bounded-legacy-shadow-startup-reconciliation.md)、[ADR-0449](./ADR-0449-versioned-legacy-shadow-startup-difference-report-and-metrics.md)
|
||||
|
||||
## 1. 决策摘要
|
||||
|
||||
@@ -239,7 +239,7 @@ Shadow Adapter 不得：
|
||||
|
||||
Shadow 转换仍必须遵守 ADR-0001。无法合法映射时追加 compat.transition_mismatch，不能强行覆盖终态。
|
||||
|
||||
当前 Alpha 切片对已审 Node worker origin 直接观察同一 ChildProcess 的 spawn、error 和 exit 事件，因此不依赖 Shell callback 才能形成基本终态。下述两级关联已补充 Shell callback、stop/cancel 和乱序/迟到回调；ADR-0448 又补充了监听前一次性启动恢复。差异对账、可采集指标与 Primary gate 仍不能由进程内观察或启动 summary 替代。
|
||||
当前 Alpha 切片对已审 Node worker origin 直接观察同一 ChildProcess 的 spawn、error 和 exit 事件，因此不依赖 Shell callback 才能形成基本终态。下述两级关联已补充 Shell callback、stop/cancel 和乱序/迟到回调；ADR-0448 又补充了监听前一次性启动恢复，ADR-0449 将其投影为 origin-bounded、版本化的差异报告与固定字段 metric batch。该 startup snapshot 仍不能替代跨测量窗口的历史终态对账与正式 Primary gate。
|
||||
|
||||
### 9.4 `next` Alpha callback 与 stop 关联
|
||||
|
||||
@@ -253,7 +253,7 @@ Shadow 转换仍必须遵守 ADR-0001。无法合法映射时追加 compat.trans
|
||||
6. 取消事实在 Legacy kill 前投递；同 worker 的后续 exit 排在取消之后。跨 worker 使用持久化定位器尽力关联，任何查询或写入失败都不能阻断 kill 或改变 2.x API 响应。
|
||||
7. 乱序 finished 可以从 queued/claimed 补齐 dispatching、starting、running 和终态；重复终态 callback、取消后的迟到成功 callback 不覆盖终态，也不追加重复完成事件。
|
||||
|
||||
这仍不是完整的 Shadow→Primary 门禁：ADR-0448 已提供启动后有界批量扫描、终态证据补齐、lost/abandoned 收敛和两事务 response-loss 修复；当前仍没有公开指标采集器、Shadow/Legacy 差异报表与正式 Primary gate。后续能力不得让 edge 增加常驻 watcher 或无界内存队列。
|
||||
这仍不是完整的 Shadow→Primary 门禁：ADR-0448 已提供启动后有界批量扫描、终态证据补齐、lost/abandoned 收敛和两事务 response-loss 修复；ADR-0449 已增加 startup 差异报表、固定低基数 metric batch 与可注入单次 collector，但尚未完成跨窗口终态差异查询、具体 exporter、资源/回滚演练和正式 Primary gate。后续能力不得让 edge 增加常驻 watcher 或无界内存队列。
|
||||
|
||||
### 9.3 Shadow 写失败
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
- 关联 RFC：QL-RFC-0001 D-02、D-356、PR-4
|
||||
- 关联 ADR：ADR-0001、ADR-0002、ADR-0445、ADR-0446、ADR-0447
|
||||
- Amends：ADR-0002 的 Shadow 启动与恢复边界
|
||||
- Amended by：ADR-0449
|
||||
|
||||
## 上下文
|
||||
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
# ADR-0449：版本化 Legacy Shadow 启动差异报告与指标批次
|
||||
|
||||
- 状态：Accepted
|
||||
- 日期：2026-08-18
|
||||
- 关联 RFC：QL-RFC-0001 D-02、D-357、PR-4
|
||||
- 关联 ADR：ADR-0002、ADR-0448
|
||||
- Amends：ADR-0448 的启动恢复可观测性边界
|
||||
|
||||
## 上下文
|
||||
|
||||
ADR-0448 已在监听前对 active Legacy Shadow Run 执行一次有界恢复，但返回值只有进程内总数。总数无法回答某个 execution origin 是否存在
|
||||
unresolved difference，也没有稳定 schema 可供日志采集器、测试门或后续 Primary admission 使用。直接输出 Run/Cron/Attempt identity 会扩大敏感面；
|
||||
用“完整率百分比”又会在分页未完成、历史分母未知时制造虚假精度。
|
||||
|
||||
本决策只关闭 startup reconciliation 的差异可见性，不宣称已经完成跨测量窗口的 Shadow/Legacy 历史终态对账。后者仍需独立读取终态、时间、
|
||||
exit code 与 Artifact existence，并形成可审计窗口。
|
||||
|
||||
## 决策
|
||||
|
||||
1. 每个 startup candidate 必须且只能归入九个固定 outcome 之一：`completed`、`cancelled`、`abandoned`、`markedLost`、`repaired`、
|
||||
`pending`、`ambiguous`、`skipped`、`failed`。总 outcome 之和必须等于 scanned。
|
||||
2. Reconciler 同时维护 aggregate 与按已配置 origin 的 outcome matrix。origin 只能来自受审 `ExecutionOrigin` 枚举，当前 Shadow 配置最多七项；
|
||||
不允许 Run ID、Cron ID、Attempt ID、PID、log path、task identity、用户名或错误消息进入维度。
|
||||
3. Supervisor 必须跨页合并同一矩阵，不启动第二次扫描。bootstrap 在输出前验证页数、候选数、aggregate conservation、origin conservation、
|
||||
origin exact coverage、`remaining/stopReason` 和 page-limit resume cursor；任一不一致按 `RangeError` 失败开放。
|
||||
4. 差异报告使用 `qinglong/legacy-shadow-startup-difference-report@v1`，只包含 Profile、固定预算、覆盖范围、aggregate outcomes、最多七条
|
||||
origin outcomes 和四态 assessment：
|
||||
- `converged`：扫描完成且没有 pending、ambiguous、skipped、failed；
|
||||
- `waiting_external_callback`：只剩可由 system crond 稳定 callback 收敛的 pending；
|
||||
- `incomplete`：页预算或 cursor stall 导致 remaining；
|
||||
- `attention_required`：存在 ambiguous、skipped 或 failed，优先级高于分页状态。
|
||||
5. 不报告比例。`remaining=true` 时未知尾页不属于可证明分母；即使扫描完成，startup report 也只覆盖当次 active candidate，不冒充历史完整率。
|
||||
6. 指标批次使用 `qinglong/legacy-shadow-startup-metric-batch@v1`。维度固定为 profile、assessment、stopReason；数值字段固定为预算、页数、
|
||||
scanned、remaining/resumeAvailable 和九个 outcome，并携带同一最多七条 origin matrix。字段是单次启动 snapshot，不伪装为进程内累计 counter。
|
||||
7. 默认结构化 startup audit 内同时携带 report 与 metric batch；组合方也可注入单次 `collect` sink。sink 失败不得改变 audit、Legacy 启动或任务结果，
|
||||
不做内存积压、磁盘重试或网络重试。
|
||||
8. `reconciled` audit 只用于 `converged`；其余可验证但未闭合状态均标为 `incomplete`。该状态本阶段仍只报告，不阻止 HTTP listen，正式
|
||||
origin-scoped Primary gate 必须在后续决策中显式消费报告并保持副作用前 fail-closed。
|
||||
|
||||
## 资源与部署影响
|
||||
|
||||
- 不新增 package、生产依赖、数据库查询、schema、migration、表、索引、连接、timer、watcher、线程、进程、端口或 Kubernetes 对象。
|
||||
- 每个 candidate 只增加一次常数计数；矩阵最多七行，edge 仍最多处理 8 个 candidate，standalone 仍最多 128 个。
|
||||
- 默认关闭和 cluster-control/worker 仍不加载 Repository，也不生成本机恢复 report/metrics。
|
||||
- metric collector 是可选调用端口，不拥有 exporter、队列或重试 authority；后续 Prometheus/OTel adapter 必须保持固定维度并单独评审。
|
||||
|
||||
## 被拒绝的替代方案
|
||||
|
||||
### 只把原始 summary 打进日志
|
||||
|
||||
拒绝。它没有 schema、origin 分解、conservation fence 或稳定指标字段，后续 gate 只能猜测日志形状。
|
||||
|
||||
### 输出每条差异的 Run/Cron identity
|
||||
|
||||
拒绝。启动日志和 metric labels 会形成高基数与任务信息泄漏；需要逐条诊断时应由受认证、有界的独立查询产品提供。
|
||||
|
||||
### 用成功数除以 scanned 作为完整率
|
||||
|
||||
拒绝。startup active scan 不是测量窗口，remaining 时分母未知，pending 也不是失败；百分比会把局部 snapshot 冒充 rollout 证据。
|
||||
|
||||
### 在路由设备内常驻 metrics registry/exporter
|
||||
|
||||
拒绝。PR-4 当前只需要稳定采集合同；常驻 exporter、队列、网络重试和生命周期必须由 Profile 部署层独立决定。
|
||||
|
||||
## 验证
|
||||
|
||||
- 真实 SQLite 覆盖 manual lost 与 scheduled_system pending 的双 origin matrix，aggregate 与 origin outcome 精确守恒。
|
||||
- Bootstrap 覆盖 report/metric schema、standalone 128 candidate 预算、waiting/incomplete/attention assessment、cursor identity 脱敏、畸形 summary
|
||||
失败开放和 collector failure 失败开放。
|
||||
- 聚焦测试 `21/21`，Legacy/Shadow 串行扩展 `77/77`；`build:back` 与完整 backend
|
||||
`1,440 pass / 0 fail / 2 conditional skip`；18 个 QL3 package 均完成 clean build/test。
|
||||
- 14/14 static audit 与 14/14 artifact audit 通过；七档 edge/standalone 产物字节与 D-356 完全一致，证明 Legacy report/metrics
|
||||
未穿透 QL3 package 产物边界。
|
||||
- 本切片不改数据库/schema/migration、容器或 Kubernetes 部署面，未重跑物理 PostgreSQL HA/K3s 门；对象存储恢复/PITR 与
|
||||
cert-manager mTLS 轮换继续保留为发布最终化现场证据门。
|
||||
@@ -452,6 +452,7 @@
|
||||
| [ADR-0446](./ADR-0446-system-crond-stable-shadow-admission.md) | System Crond 稳定 Shadow 准入与回调重放 | Accepted |
|
||||
| [ADR-0447](./ADR-0447-boot-shadow-and-non-origin-boundaries.md) | Boot Shadow 准入与 once/gRPC 非 Origin 边界 | Accepted |
|
||||
| [ADR-0448](./ADR-0448-bounded-legacy-shadow-startup-reconciliation.md) | 有界 Legacy Shadow 启动恢复 | Accepted |
|
||||
| [ADR-0449](./ADR-0449-versioned-legacy-shadow-startup-difference-report-and-metrics.md) | 版本化 Legacy Shadow 启动差异报告与指标批次 | Accepted |
|
||||
|
||||
## 规则
|
||||
|
||||
|
||||
@@ -6,10 +6,40 @@ const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
bootstrapLegacyShadowStartupReconciliation,
|
||||
createLegacyShadowStartupDifferenceReport,
|
||||
} = require('../../back/runtime/adapters/legacy/bootstrapLegacyShadowStartupReconciliation');
|
||||
|
||||
function summary(overrides = {}) {
|
||||
const OUTCOMES = [
|
||||
'completed',
|
||||
'cancelled',
|
||||
'abandoned',
|
||||
'markedLost',
|
||||
'repaired',
|
||||
'pending',
|
||||
'ambiguous',
|
||||
'skipped',
|
||||
'failed',
|
||||
];
|
||||
|
||||
function originSummary(origin, overrides = {}) {
|
||||
return {
|
||||
origin,
|
||||
scanned: 0,
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 0,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function summary(overrides = {}, origins = ['manual']) {
|
||||
const value = {
|
||||
pages: 1,
|
||||
scanned: 0,
|
||||
completed: 0,
|
||||
@@ -25,6 +55,8 @@ function summary(overrides = {}) {
|
||||
remaining: false,
|
||||
...overrides,
|
||||
};
|
||||
value.byOrigin ??= origins.map((origin) => originSummary(origin));
|
||||
return value;
|
||||
}
|
||||
|
||||
test('disabled bootstrap remains fully lazy', async () => {
|
||||
@@ -67,7 +99,7 @@ test('applies distinct one-shot edge and standalone budgets', async () => {
|
||||
profile,
|
||||
async execute(request) {
|
||||
requests.push(request);
|
||||
return summary();
|
||||
return summary({}, request.origins);
|
||||
},
|
||||
audit() {},
|
||||
});
|
||||
@@ -87,6 +119,155 @@ test('applies distinct one-shot edge and standalone budgets', async () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('emits one versioned origin-bounded difference report and metric batch', async () => {
|
||||
const batches = [];
|
||||
const result = await bootstrapLegacyShadowStartupReconciliation({
|
||||
origins: ['manual', 'scheduled_system'],
|
||||
profile: 'standalone',
|
||||
async execute(request) {
|
||||
return summary(
|
||||
{
|
||||
scanned: 4,
|
||||
completed: 1,
|
||||
markedLost: 1,
|
||||
repaired: 1,
|
||||
pending: 1,
|
||||
byOrigin: [
|
||||
originSummary('manual', {
|
||||
scanned: 3,
|
||||
completed: 1,
|
||||
markedLost: 1,
|
||||
repaired: 1,
|
||||
}),
|
||||
originSummary('scheduled_system', {
|
||||
scanned: 1,
|
||||
pending: 1,
|
||||
}),
|
||||
],
|
||||
},
|
||||
request.origins,
|
||||
);
|
||||
},
|
||||
audit() {},
|
||||
collect(batch) {
|
||||
batches.push(batch);
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.state, 'incomplete');
|
||||
assert.equal(
|
||||
result.report.schema,
|
||||
'qinglong/legacy-shadow-startup-difference-report@v1',
|
||||
);
|
||||
assert.equal(result.report.assessment, 'waiting_external_callback');
|
||||
assert.equal(result.report.budget.maxCandidates, 128);
|
||||
assert.deepEqual(
|
||||
result.report.byOrigin.map(({ origin, scanned }) => ({ origin, scanned })),
|
||||
[
|
||||
{ origin: 'manual', scanned: 3 },
|
||||
{ origin: 'scheduled_system', scanned: 1 },
|
||||
],
|
||||
);
|
||||
assert.equal(batches.length, 1);
|
||||
assert.equal(
|
||||
batches[0].schema,
|
||||
'qinglong/legacy-shadow-startup-metric-batch@v1',
|
||||
);
|
||||
assert.equal(batches[0].dimensions.assessment, 'waiting_external_callback');
|
||||
assert.equal(batches[0].values.scanned, 4);
|
||||
assert.equal(batches[0].values.pending, 1);
|
||||
assert.deepEqual(
|
||||
Object.keys(result.report.outcomes).sort(),
|
||||
[...OUTCOMES].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
test('turns inconsistent report input into a fail-open low-sensitivity audit', async () => {
|
||||
let collected = false;
|
||||
const result = await bootstrapLegacyShadowStartupReconciliation({
|
||||
origins: ['manual'],
|
||||
profile: 'edge',
|
||||
async execute() {
|
||||
return summary({ scanned: 1 });
|
||||
},
|
||||
audit() {},
|
||||
collect() {
|
||||
collected = true;
|
||||
},
|
||||
});
|
||||
|
||||
assert.deepEqual(result, { state: 'failed', errorType: 'RangeError' });
|
||||
assert.equal(collected, false);
|
||||
});
|
||||
|
||||
test('rejects non-Profile budgets and cursors outside page-limit reports', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
createLegacyShadowStartupDifferenceReport(
|
||||
{
|
||||
origins: ['manual'],
|
||||
profile: 'edge',
|
||||
pageSize: 32,
|
||||
maxPages: 1,
|
||||
},
|
||||
summary(),
|
||||
),
|
||||
RangeError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
createLegacyShadowStartupDifferenceReport(
|
||||
{ origins: ['manual'], profile: 'edge', pageSize: 8, maxPages: 1 },
|
||||
summary({
|
||||
nextCursor: { createdAtMs: 10, runId: 'must-not-escape' },
|
||||
}),
|
||||
),
|
||||
RangeError,
|
||||
);
|
||||
});
|
||||
|
||||
test('classifies ambiguous, skipped, or failed comparisons as attention required', async () => {
|
||||
const result = await bootstrapLegacyShadowStartupReconciliation({
|
||||
origins: ['manual'],
|
||||
profile: 'edge',
|
||||
async execute(request) {
|
||||
return summary(
|
||||
{
|
||||
scanned: 1,
|
||||
ambiguous: 1,
|
||||
byOrigin: [originSummary('manual', { scanned: 1, ambiguous: 1 })],
|
||||
},
|
||||
request.origins,
|
||||
);
|
||||
},
|
||||
audit() {},
|
||||
});
|
||||
|
||||
assert.equal(result.state, 'incomplete');
|
||||
assert.equal(result.report.assessment, 'attention_required');
|
||||
assert.equal(result.metrics.dimensions.assessment, 'attention_required');
|
||||
});
|
||||
|
||||
test('keeps startup fail-open when metric collection fails', async () => {
|
||||
const result = await bootstrapLegacyShadowStartupReconciliation({
|
||||
origins: ['manual'],
|
||||
profile: 'edge',
|
||||
async execute(request) {
|
||||
return summary({}, request.origins);
|
||||
},
|
||||
audit() {},
|
||||
collect() {
|
||||
throw new Error('collector transport secret');
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.state, 'reconciled');
|
||||
assert.equal(
|
||||
JSON.stringify(result).includes('collector transport secret'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails open with a low-sensitivity error type', async () => {
|
||||
const result = await bootstrapLegacyShadowStartupReconciliation({
|
||||
origins: ['manual'],
|
||||
@@ -121,6 +302,9 @@ test('redacts the resume cursor Run identity from startup audit output', async (
|
||||
assert.equal(result.state, 'incomplete');
|
||||
assert.equal(result.summary.resumeAvailable, true);
|
||||
assert.equal('nextCursor' in result.summary, false);
|
||||
assert.equal(result.report.assessment, 'incomplete');
|
||||
assert.equal(result.metrics.values.resumeAvailable, 1);
|
||||
assert.equal('nextCursor' in result.report.coverage, false);
|
||||
assert.equal(records.join('').includes('run-secret-identity'), false);
|
||||
});
|
||||
|
||||
|
||||
@@ -281,6 +281,50 @@ test('keeps system-crond pending without terminal callback evidence', async () =
|
||||
);
|
||||
});
|
||||
|
||||
test('keeps a bounded per-origin outcome matrix for later ownership gates', async () => {
|
||||
const { database, writer, reconciler } = await createStack();
|
||||
await activeShadow(writer, { legacyCronId: 15, pid: 4311 });
|
||||
await activeShadow(writer, {
|
||||
origin: 'scheduled_system',
|
||||
legacyCronId: 16,
|
||||
pid: 4312,
|
||||
});
|
||||
await insertInstance(database, {
|
||||
cron_id: 15,
|
||||
pid: 4311,
|
||||
status: 2,
|
||||
finished_at: null,
|
||||
});
|
||||
await insertInstance(database, {
|
||||
cron_id: 16,
|
||||
pid: 4312,
|
||||
status: 2,
|
||||
finished_at: null,
|
||||
});
|
||||
|
||||
const summary = await reconciler.reconcileBatch({
|
||||
origins: ['manual', 'scheduled_system'],
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
summary.byOrigin.map(({ origin, scanned, markedLost, pending }) => ({
|
||||
origin,
|
||||
scanned,
|
||||
markedLost,
|
||||
pending,
|
||||
})),
|
||||
[
|
||||
{ origin: 'manual', scanned: 1, markedLost: 1, pending: 0 },
|
||||
{
|
||||
origin: 'scheduled_system',
|
||||
scanned: 1,
|
||||
markedLost: 0,
|
||||
pending: 1,
|
||||
},
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('refuses ambiguous duplicate identity evidence instead of guessing a terminal result', async () => {
|
||||
const { database, repository, writer, reconciler } = await createStack();
|
||||
const logPath = 'cron/ambiguous.log';
|
||||
@@ -349,6 +393,21 @@ test('supervisor preserves a stable cursor when its Profile budget is exhausted'
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
byOrigin: [
|
||||
{
|
||||
origin: 'manual',
|
||||
scanned: 1,
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 1,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
},
|
||||
],
|
||||
truncated: true,
|
||||
nextCursor: { createdAtMs: 10, runId: 'run-10' },
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user