From d38f62426256e9f7133c2341cbc2d944a19c3b95 Mon Sep 17 00:00:00 2001 From: whyour Date: Tue, 18 Aug 2026 09:58:16 +0800 Subject: [PATCH] feat(ql3): report legacy shadow startup differences --- ...tstrapLegacyShadowStartupReconciliation.ts | 264 +++++++++++++++++- .../legacyShadowStartupReconciler.ts | 195 ++++++++----- docs/QINGLONG_3_0_ARCHITECTURE_RFC.md | 18 +- ...-crontab-compatibility-and-shadow-write.md | 6 +- ...ed-legacy-shadow-startup-reconciliation.md | 1 + ...w-startup-difference-report-and-metrics.md | 75 +++++ docs/adr/README.md | 1 + ...LegacyShadowStartupReconciliation.test.cjs | 188 ++++++++++++- ...legacyShadowStartupReconciliation.test.cjs | 59 ++++ 9 files changed, 722 insertions(+), 85 deletions(-) create mode 100644 docs/adr/ADR-0449-versioned-legacy-shadow-startup-difference-report-and-metrics.md diff --git a/back/runtime/adapters/legacy/bootstrapLegacyShadowStartupReconciliation.ts b/back/runtime/adapters/legacy/bootstrapLegacyShadowStartupReconciliation.ts index 0488628b..d325f2fa 100644 --- a/back/runtime/adapters/legacy/bootstrapLegacyShadowStartupReconciliation.ts +++ b/back/runtime/adapters/legacy/bootstrapLegacyShadowStartupReconciliation.ts @@ -6,7 +6,12 @@ import { } from '../../domain/deploymentProfile'; import { configuredLegacyShadowOrigins } from '../../compatibility/legacyExecutionBridge'; import { createLegacyLogArtifactId } from '../../compatibility/legacyTaskRevision'; -import type { LegacyShadowStartupSummary } from '../../application/legacyShadowStartupReconciler'; +import { + LEGACY_SHADOW_STARTUP_OUTCOMES, + type LegacyShadowStartupOriginSummary, + type LegacyShadowStartupOutcomeCounts, + type LegacyShadowStartupSummary, +} from '../../application/legacyShadowStartupReconciler'; export interface LegacyShadowStartupBudget { pageSize: number; @@ -18,6 +23,60 @@ export interface LegacyShadowStartupRequest extends LegacyShadowStartupBudget { profile: 'edge' | 'standalone'; } +export const LEGACY_SHADOW_STARTUP_DIFFERENCE_REPORT_SCHEMA = + 'qinglong/legacy-shadow-startup-difference-report@v1'; +export const LEGACY_SHADOW_STARTUP_METRIC_BATCH_SCHEMA = + 'qinglong/legacy-shadow-startup-metric-batch@v1'; + +export type LegacyShadowStartupAssessment = + | 'converged' + | 'waiting_external_callback' + | 'incomplete' + | 'attention_required'; + +export interface LegacyShadowStartupDifferenceReport { + schemaVersion: 1; + schema: typeof LEGACY_SHADOW_STARTUP_DIFFERENCE_REPORT_SCHEMA; + profile: 'edge' | 'standalone'; + assessment: LegacyShadowStartupAssessment; + configuredOriginCount: number; + budget: { + pageSize: number; + maxPages: number; + maxCandidates: number; + }; + coverage: { + pages: number; + scanned: number; + stopReason: LegacyShadowStartupSummary['stopReason']; + remaining: boolean; + resumeAvailable: boolean; + }; + outcomes: LegacyShadowStartupOutcomeCounts; + byOrigin: readonly LegacyShadowStartupOriginSummary[]; +} + +export interface LegacyShadowStartupMetricBatch { + schemaVersion: 1; + schema: typeof LEGACY_SHADOW_STARTUP_METRIC_BATCH_SCHEMA; + dimensions: { + profile: 'edge' | 'standalone'; + assessment: LegacyShadowStartupAssessment; + stopReason: LegacyShadowStartupSummary['stopReason']; + }; + values: LegacyShadowStartupOutcomeCounts & { + configuredOrigins: number; + pageSize: number; + maxPages: number; + maxCandidates: number; + pages: number; + scanned: number; + remaining: 0 | 1; + resumeAvailable: 0 | 1; + }; + byOrigin: readonly LegacyShadowStartupOriginSummary[]; +} + export type LegacyShadowStartupAudit = | { state: 'disabled'; @@ -30,9 +89,11 @@ export type LegacyShadowStartupAudit = state: 'reconciled' | 'incomplete'; profile: 'edge' | 'standalone'; origins: number; - summary: Omit & { + summary: Omit & { resumeAvailable: boolean; }; + report: LegacyShadowStartupDifferenceReport; + metrics: LegacyShadowStartupMetricBatch; } | { state: 'failed'; @@ -46,6 +107,7 @@ export interface BootstrapLegacyShadowStartupOptions { request: LegacyShadowStartupRequest, ) => Promise; audit?: (record: LegacyShadowStartupAudit) => void | Promise; + collect?: (metrics: LegacyShadowStartupMetricBatch) => void | Promise; } const BUDGETS: Readonly< @@ -54,17 +116,195 @@ const BUDGETS: Readonly< edge: { pageSize: 8, maxPages: 1 }, standalone: { pageSize: 32, maxPages: 4 }, }; +const MAX_CONFIGURED_LEGACY_SHADOW_ORIGINS = 7; function auditSummary(summary: LegacyShadowStartupSummary): Omit< LegacyShadowStartupSummary, - 'nextCursor' + 'nextCursor' | 'byOrigin' > & { resumeAvailable: boolean; } { - const { nextCursor, ...bounded } = summary; + const { nextCursor, byOrigin: _byOrigin, ...bounded } = summary; return { ...bounded, resumeAvailable: nextCursor !== undefined }; } +function outcomeCounts( + source: LegacyShadowStartupOutcomeCounts, +): LegacyShadowStartupOutcomeCounts { + return Object.fromEntries( + LEGACY_SHADOW_STARTUP_OUTCOMES.map((outcome) => [outcome, source[outcome]]), + ) as LegacyShadowStartupOutcomeCounts; +} + +function outcomeTotal(source: LegacyShadowStartupOutcomeCounts): number { + return LEGACY_SHADOW_STARTUP_OUTCOMES.reduce( + (total, outcome) => total + source[outcome], + 0, + ); +} + +function assertCount(value: number, label: string, maximum: number): void { + if (!Number.isSafeInteger(value) || value < 0 || value > maximum) { + throw new RangeError(`${label} is outside its startup budget`); + } +} + +function assessment( + summary: LegacyShadowStartupSummary, +): LegacyShadowStartupAssessment { + if (summary.ambiguous > 0 || summary.skipped > 0 || summary.failed > 0) { + return 'attention_required'; + } + if (summary.remaining) return 'incomplete'; + if (summary.pending > 0) return 'waiting_external_callback'; + return 'converged'; +} + +export function createLegacyShadowStartupDifferenceReport( + request: LegacyShadowStartupRequest, + summary: LegacyShadowStartupSummary, +): LegacyShadowStartupDifferenceReport { + const expectedBudget = BUDGETS[request.profile]; + if ( + request.pageSize !== expectedBudget.pageSize || + request.maxPages !== expectedBudget.maxPages + ) { + throw new RangeError('startup report budget does not match its Profile'); + } + const configuredOrigins = [...new Set(request.origins)]; + if ( + configuredOrigins.length < 1 || + configuredOrigins.length > MAX_CONFIGURED_LEGACY_SHADOW_ORIGINS + ) { + throw new RangeError( + 'configured Shadow origin count is outside its budget', + ); + } + const maxCandidates = request.pageSize * request.maxPages; + assertCount(summary.pages, 'pages', request.maxPages); + if (summary.pages < 1) { + throw new RangeError('pages must include the executed startup page'); + } + assertCount(summary.scanned, 'scanned', request.pageSize * summary.pages); + for (const outcome of LEGACY_SHADOW_STARTUP_OUTCOMES) { + assertCount(summary[outcome], outcome, summary.scanned); + } + if (outcomeTotal(summary) !== summary.scanned) { + throw new RangeError('startup outcomes do not conserve scanned candidates'); + } + if (summary.remaining !== (summary.stopReason !== 'complete')) { + throw new RangeError('startup remaining and stop reason disagree'); + } + if ( + !['complete', 'page_limit', 'cursor_stalled'].includes(summary.stopReason) + ) { + throw new RangeError('startup stop reason is invalid'); + } + if (summary.stopReason === 'page_limit' && summary.nextCursor === undefined) { + throw new RangeError('page-limited startup summary has no resume cursor'); + } + if (summary.stopReason !== 'page_limit' && summary.nextCursor !== undefined) { + throw new RangeError('resume cursor is present without a page limit'); + } + + if (summary.byOrigin.length !== configuredOrigins.length) { + throw new RangeError('origin outcome coverage is incomplete'); + } + const origins = new Map(); + for (const origin of summary.byOrigin) { + if ( + origins.has(origin.origin) || + !configuredOrigins.includes(origin.origin) + ) { + throw new RangeError('origin outcome coverage is invalid'); + } + assertCount(origin.scanned, `${origin.origin}:scanned`, summary.scanned); + for (const outcome of LEGACY_SHADOW_STARTUP_OUTCOMES) { + assertCount( + origin[outcome], + `${origin.origin}:${outcome}`, + origin.scanned, + ); + } + if (outcomeTotal(origin) !== origin.scanned) { + throw new RangeError( + 'origin outcomes do not conserve scanned candidates', + ); + } + origins.set(origin.origin, { + origin: origin.origin, + scanned: origin.scanned, + ...outcomeCounts(origin), + }); + } + const orderedOrigins = configuredOrigins.map( + (origin) => origins.get(origin)!, + ); + if ( + orderedOrigins.reduce((total, origin) => total + origin.scanned, 0) !== + summary.scanned || + LEGACY_SHADOW_STARTUP_OUTCOMES.some( + (outcome) => + orderedOrigins.reduce((total, origin) => total + origin[outcome], 0) !== + summary[outcome], + ) + ) { + throw new RangeError('origin outcomes do not match aggregate outcomes'); + } + + return { + schemaVersion: 1, + schema: LEGACY_SHADOW_STARTUP_DIFFERENCE_REPORT_SCHEMA, + profile: request.profile, + assessment: assessment(summary), + configuredOriginCount: configuredOrigins.length, + budget: { + pageSize: request.pageSize, + maxPages: request.maxPages, + maxCandidates, + }, + coverage: { + pages: summary.pages, + scanned: summary.scanned, + stopReason: summary.stopReason, + remaining: summary.remaining, + resumeAvailable: summary.nextCursor !== undefined, + }, + outcomes: outcomeCounts(summary), + byOrigin: orderedOrigins, + }; +} + +export function createLegacyShadowStartupMetricBatch( + report: LegacyShadowStartupDifferenceReport, +): LegacyShadowStartupMetricBatch { + return { + schemaVersion: 1, + schema: LEGACY_SHADOW_STARTUP_METRIC_BATCH_SCHEMA, + dimensions: { + profile: report.profile, + assessment: report.assessment, + stopReason: report.coverage.stopReason, + }, + values: { + configuredOrigins: report.configuredOriginCount, + pageSize: report.budget.pageSize, + maxPages: report.budget.maxPages, + maxCandidates: report.budget.maxCandidates, + pages: report.coverage.pages, + scanned: report.coverage.scanned, + remaining: report.coverage.remaining ? 1 : 0, + resumeAvailable: report.coverage.resumeAvailable ? 1 : 0, + ...outcomeCounts(report.outcomes), + }, + byOrigin: report.byOrigin.map((origin) => ({ + origin: origin.origin, + scanned: origin.scanned, + ...outcomeCounts(origin), + })), + }; +} + async function executeDefault( request: LegacyShadowStartupRequest, ): Promise { @@ -137,22 +377,32 @@ export async function bootstrapLegacyShadowStartupReconciliation( } return record; } - const summary = await (options.execute ?? executeDefault)({ + const request: LegacyShadowStartupRequest = { origins, profile, ...BUDGETS[profile], - }); + }; + const summary = await (options.execute ?? executeDefault)(request); + const report = createLegacyShadowStartupDifferenceReport(request, summary); + const metrics = createLegacyShadowStartupMetricBatch(report); const record: LegacyShadowStartupAudit = { - state: summary.remaining ? 'incomplete' : 'reconciled', + state: report.assessment === 'converged' ? 'reconciled' : 'incomplete', profile, origins: origins.length, summary: auditSummary(summary), + report, + metrics, }; try { await audit(record); } catch { // Shadow audit output must not affect Legacy startup. } + try { + await options.collect?.(metrics); + } catch { + // Shadow metric collection must not affect Legacy startup. + } return record; } catch (error) { const record: LegacyShadowStartupAudit = { diff --git a/back/runtime/application/legacyShadowStartupReconciler.ts b/back/runtime/application/legacyShadowStartupReconciler.ts index 9e608aa9..e6945a48 100644 --- a/back/runtime/application/legacyShadowStartupReconciler.ts +++ b/back/runtime/application/legacyShadowStartupReconciler.ts @@ -23,41 +23,86 @@ export interface LegacyShadowStartupClock { now(): number; } -export interface LegacyShadowStartupReconcileSummary { - scanned: number; - completed: number; - cancelled: number; - abandoned: number; - markedLost: number; - repaired: number; - pending: number; - ambiguous: number; - skipped: number; - failed: number; - truncated: boolean; - nextCursor?: LegacyShadowStartupCursor; -} +export const LEGACY_SHADOW_STARTUP_OUTCOMES = Object.freeze([ + 'completed', + 'cancelled', + 'abandoned', + 'markedLost', + 'repaired', + 'pending', + 'ambiguous', + 'skipped', + 'failed', +] as const); + +export type LegacyShadowStartupOutcome = + (typeof LEGACY_SHADOW_STARTUP_OUTCOMES)[number]; + +export type LegacyShadowStartupOutcomeCounts = Record< + LegacyShadowStartupOutcome, + number +>; + +export type LegacyShadowStartupOriginSummary = + LegacyShadowStartupOutcomeCounts & { + origin: RunRecord['executionOrigin']; + scanned: number; + }; + +export type LegacyShadowStartupReconcileSummary = + LegacyShadowStartupOutcomeCounts & { + scanned: number; + byOrigin: readonly LegacyShadowStartupOriginSummary[]; + truncated: boolean; + nextCursor?: LegacyShadowStartupCursor; + }; export type LegacyShadowStartupStopReason = | 'complete' | 'page_limit' | 'cursor_stalled'; -export interface LegacyShadowStartupSummary - extends Omit< - LegacyShadowStartupReconcileSummary, - 'truncated' | 'nextCursor' - > { +export type LegacyShadowStartupSummary = Omit< + LegacyShadowStartupReconcileSummary, + 'truncated' | 'nextCursor' +> & { pages: number; stopReason: LegacyShadowStartupStopReason; remaining: boolean; nextCursor?: LegacyShadowStartupCursor; -} +}; type EvidenceSelection = | { status: 'matched'; evidence: LegacyRunningInstanceEvidence } | { status: 'none' | 'ambiguous' }; +function emptyOutcomeCounts(): LegacyShadowStartupOutcomeCounts { + return { + completed: 0, + cancelled: 0, + abandoned: 0, + markedLost: 0, + repaired: 0, + pending: 0, + ambiguous: 0, + skipped: 0, + failed: 0, + }; +} + +function originSummary( + origin: RunRecord['executionOrigin'], +): LegacyShadowStartupOriginSummary { + return { origin, scanned: 0, ...emptyOutcomeCounts() }; +} + +function incrementOutcome( + target: LegacyShadowStartupOutcomeCounts, + outcome: LegacyShadowStartupOutcome, +): void { + target[outcome] += 1; +} + function sameCursor( left: LegacyShadowStartupCursor | undefined, right: LegacyShadowStartupCursor, @@ -168,27 +213,34 @@ export class LegacyShadowStartupReconciler { ...(options.cursor === undefined ? {} : { cursor: options.cursor }), ...(options.limit === undefined ? {} : { limit: options.limit }), }); + const originSummaries = new Map( + enabledOrigins.map((origin) => [origin, originSummary(origin)]), + ); const summary: LegacyShadowStartupReconcileSummary = { scanned: page.candidates.length, - completed: 0, - cancelled: 0, - abandoned: 0, - markedLost: 0, - repaired: 0, - pending: 0, - ambiguous: 0, - skipped: 0, - failed: 0, + ...emptyOutcomeCounts(), + byOrigin: [...originSummaries.values()], truncated: page.truncated, ...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }), }; const origins = new Set(enabledOrigins); for (const candidate of page.candidates) { - try { - await this.reconcileCandidate(candidate, origins, summary); - } catch { - summary.failed += 1; + const perOrigin = + originSummaries.get(candidate.origin) ?? + originSummary(candidate.origin); + if (!originSummaries.has(candidate.origin)) { + originSummaries.set(candidate.origin, perOrigin); + summary.byOrigin = [...originSummaries.values()]; } + perOrigin.scanned += 1; + let outcome: LegacyShadowStartupOutcome; + try { + outcome = await this.reconcileCandidate(candidate, origins); + } catch { + outcome = 'failed'; + } + incrementOutcome(summary, outcome); + incrementOutcome(perOrigin, outcome); } return summary; } @@ -196,26 +248,23 @@ export class LegacyShadowStartupReconciler { private async reconcileCandidate( candidate: LegacyShadowStartupCandidate, origins: ReadonlySet, - summary: LegacyShadowStartupReconcileSummary, - ): Promise { + ): Promise { const run = await this.repository.findRunById(candidate.runId); if ( !run || run.executionOwner !== 'legacy' || !origins.has(run.executionOrigin) || + run.executionOrigin !== candidate.origin || isTerminalRunStatus(run.status) || run.status === 'lost' ) { - summary.skipped += 1; - return; + return 'skipped'; } if (candidate.activeAttemptCount === 0) { - await this.repairTerminalAttempt(run, summary); - return; + return this.repairTerminalAttempt(run); } if (candidate.activeAttemptCount !== 1 || !candidate.attempt) { - summary.ambiguous += 1; - return; + return 'ambiguous'; } const attempt = await this.repository.findAttemptById( candidate.attempt.attemptId, @@ -225,8 +274,7 @@ export class LegacyShadowStartupReconciler { attempt.runId !== run.id || isTerminalRunAttemptStatus(attempt.status) ) { - summary.skipped += 1; - return; + return 'skipped'; } const evidence = @@ -236,13 +284,11 @@ export class LegacyShadowStartupReconciler { legacyCronId: candidate.legacyCronId, }); if (evidence.truncated) { - summary.ambiguous += 1; - return; + return 'ambiguous'; } const selected = selectEvidence(attempt, evidence.evidence); if (selected.status === 'ambiguous') { - summary.ambiguous += 1; - return; + return 'ambiguous'; } if ( selected.status === 'matched' && @@ -254,8 +300,7 @@ export class LegacyShadowStartupReconciler { { runId: run.id, attemptId: attempt.id }, { atMs, reason: 'reconcile' }, ); - summary.cancelled += 1; - return; + return 'cancelled'; } if ( selected.evidence.outcome === 'succeeded' || @@ -274,35 +319,30 @@ export class LegacyShadowStartupReconciler { : selected.evidence.exitCode, }, ); - summary.completed += 1; - return; + return 'completed'; } } if (run.executionOrigin === 'scheduled_system') { - summary.pending += 1; - return; + return 'pending'; } if (run.status === 'queued' && attempt.status === 'claimed') { await this.abandon(run, attempt); - summary.abandoned += 1; - return; + return 'abandoned'; } if ( (run.status === 'dispatching' || run.status === 'running') && ['claimed', 'starting', 'running'].includes(attempt.status) ) { await this.markLost(run, attempt); - summary.markedLost += 1; - return; + return 'markedLost'; } - summary.ambiguous += 1; + return 'ambiguous'; } private async repairTerminalAttempt( run: RunRecord, - summary: LegacyShadowStartupReconcileSummary, - ): Promise { + ): Promise<'repaired' | 'ambiguous'> { const attempt = await this.repository.findLatestAttemptByRunId(run.id); const target = attempt ? terminalRunTarget(attempt) : undefined; if ( @@ -310,8 +350,7 @@ export class LegacyShadowStartupReconciler { target === undefined || !RUN_TRANSITIONS[run.status].includes(target) ) { - summary.ambiguous += 1; - return; + return 'ambiguous'; } await this.commands.transitionRun({ runId: run.id, @@ -327,7 +366,7 @@ export class LegacyShadowStartupReconciler { actor: { type: 'reconciler' }, dedupeKey: `legacy-startup-repair:${attempt.id}:${target}`, }); - summary.repaired += 1; + return 'repaired'; } private async abandon( @@ -438,18 +477,17 @@ export class LegacyShadowStartupSupervisor { ); } + const originSummaries = new Map( + [...new Set(options.origins)].map((origin) => [ + origin, + originSummary(origin), + ]), + ); const total: LegacyShadowStartupSummary = { pages: 0, scanned: 0, - completed: 0, - cancelled: 0, - abandoned: 0, - markedLost: 0, - repaired: 0, - pending: 0, - ambiguous: 0, - skipped: 0, - failed: 0, + ...emptyOutcomeCounts(), + byOrigin: [...originSummaries.values()], stopReason: 'complete', remaining: false, }; @@ -471,6 +509,19 @@ export class LegacyShadowStartupSupervisor { total.ambiguous += page.ambiguous; total.skipped += page.skipped; total.failed += page.failed; + for (const pageOrigin of page.byOrigin) { + const aggregate = + originSummaries.get(pageOrigin.origin) ?? + originSummary(pageOrigin.origin); + if (!originSummaries.has(pageOrigin.origin)) { + originSummaries.set(pageOrigin.origin, aggregate); + total.byOrigin = [...originSummaries.values()]; + } + aggregate.scanned += pageOrigin.scanned; + for (const outcome of LEGACY_SHADOW_STARTUP_OUTCOMES) { + aggregate[outcome] += pageOrigin[outcome]; + } + } if (!page.truncated) return total; if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) { total.stopReason = 'cursor_stalled'; diff --git a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md index e8247136..4d8892da 100644 --- a/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md +++ b/docs/QINGLONG_3_0_ARCHITECTURE_RFC.md @@ -11,6 +11,22 @@ 最新增量证据（2026-08-18）： +- D-357/ADR-0449（已接受）：将 D-356 的进程内 startup summary 收紧为可供后续 origin-scoped gate 使用的版本化差异证据。每个候选只归入 + completed/cancelled/abandoned/markedLost/repaired/pending/ambiguous/skipped/failed 九个固定 outcome，总量与最多七条已配置 origin + matrix 必须逐级守恒；bootstrap 还验证 Profile 页/候选预算、origin exact coverage、remaining/stopReason 与 page-limit cursor，畸形 summary + 继续失败开放。`qinglong/legacy-shadow-startup-difference-report@v1` 只输出 Profile、预算、覆盖范围、固定计数与 + converged/waiting_external_callback/incomplete/attention_required 四态 assessment，不含 Run/Cron/Attempt/PID/log/task/user/error + 原文，也不在未知分母上伪造完整率。`qinglong/legacy-shadow-startup-metric-batch@v1` 以 profile/assessment/stopReason 三个固定维度和固定数值 + 字段提供单次 snapshot；默认结构化 audit 可直接采集，组合方也可注入一次性 collector，collector 失败不影响 Legacy。实现复用同一次扫描，只增加 + 最多七行常数内存，不新增 package、生产依赖、查询、schema/migration、连接、timer/watcher、进程、端口或部署对象；默认 off 与 + cluster-control/worker 仍为零 Repository/报告/指标。本切片不把 startup snapshot 冒充跨测量窗口终态对账、具体 exporter、资源/回滚证据或 + Primary gate。阶段门已重跑：聚焦 `21/21`、Legacy/Shadow 串行扩展 `77/77`、`build:back`、完整 backend + `1,440 pass / 0 fail / 2 conditional skip`、18-package clean build/test、14/14 static audit 与 14/14 artifact audit 全部通过。 + edge/standalone 产物字节继续为：base `2,589,998 / 2,590,076`、adopted `2,809,293 / 2,809,416`、application + `3,632,877 / 3,632,997`、application-api `3,800,430 / 3,800,574`、AI `3,069,251 / 3,069,341`、 + application+AI `4,493,151 / 4,493,283`、MCP `7,315,930 / 7,316,038`，与 D-356 完全一致。由于本阶段不改数据库、 + 容器或 Kubernetes 部署面，不重跑物理 PostgreSQL HA/K3s；对象存储恢复/PITR 与 cert-manager mTLS 轮换仍是既有发布最终化现场证据门。 + - D-356/ADR-0448（已接受）：补齐监听前、一次性且 Profile-aware 的 Legacy Shadow Startup Reconciler。它只在至少一个已审 origin 显式 Shadow 时，于 Legacy `initData` 归一之后、manual Primary activation/HTTP listen 之前加载；默认 off 与 cluster-control/worker 均保持零 Repository/恢复查询/写入。只读 Source 使用 `(created_at_ms, run_id)` keyset，单页硬上限 64、每 Cron RunningInstance 证据硬上限 8，原始 @@ -9099,7 +9115,7 @@ flowchart LR | PR-1 Run Schema | Incubating | Run/RunAttempt/RunEvent schema、nullable cancel request 与 Attempt deadline 字段及恢复索引、CancellationDispatch 状态/version/lease/backoff schema、Repository port、临时 Sequelize adapter、统一事件大小/分页上限、跨 adapter RunRepository contract suite（原子事务、回滚、Run/Attempt/RetryPolicy CAS、唯一错误、分页与取消恢复）；ADR-0041 的 `pg-0003-run-retry-policy`、capability v2、driver-neutral PostgreSQL Run Repository 与真实 `pg.Pool` 上的共享 Repository/rollback/SQLSTATE contract；ADR-0063/0069/0071/0073/0074/0076 的独立 Node 24 local-sqlite typed schema、十二条 reviewed migration、capability v6、共享 operation authority、readiness/RunRepository/API credential repository/receipt journal/dispatch plan/encrypted Secret envelope/Project Policy/security audit/authorized mutation/stable Identity catalog、Drizzle↔真实 catalog table/column/index/CHECK/FK lockstep、base/adopted/application edge/standalone 产物门禁；ADR-0064 的 legacy baseline/plan digest、Online Backup recovery、side-by-side target migration、staged manifest、双库栅栏 activation、source 生命周期写栅栏、target stable identity 和重启语义；ADR-0065 的独立 cutover authority、外部副作用停机 evidence、append-only journal、start/restart/stop barrier 与 unknown→manual_required 收敛；ADR-0066 的 adopted storage→Run reconciliation→receipt maintenance→domain recovery→lifecycle→admission application gate、严格有界 recovery summary 与 admission-first reverse stop；ADR-0067 的 SQLite 事实驱动 Run 候选源、256 条硬上限、截断失败关闭和唯一 Repository authority；ADR-0068 的 receipt-first Reconciler、callback token/sequence fence、exact local-process identity、Attempt/Run/双 Event 原子终态推进和最终 verifier；ADR-0069 的 local-process 单向包边界、pre-spawn journal、受审 POSIX launcher、immutable receipt、exact identity 和 Profile-aware cleanup lifecycle；ADR-0070 的独立 local-execution、spawn 前后双 transaction CAS、callback digest、exact stop 补偿与 fail-closed starting 保留；ADR-0071 的独立 local-dispatch、不可变 revision/context、Secret-first materializer、Profile Artifact admission、4/64 MiB output hard quota 和窄 application facade；ADR-0073/0074 的 Project-bound SecretRef、AES-256-GCM、外置 keyring 生命周期、双 SQLite authority CAS、application preflight、强 Principal/Policy 和 envelope+audit 原子提交；ADR-0086 的本机 Owner provisioning/challenge/claim/delivery acknowledgement/credential recovery CLI；ADR-0377 的 Local/Cluster 同构、Profile-aware、Project-scoped Artifact range read | fresh database/pepper setup、credential rotation/GC 运维编排与 Secret/Project/Role/Approval 管理 CLI/API/UI、备份/rekey、2.x/target process controller、人工 recovery、target 写后 reconciliation 与完整 cutover/rollback 演练；retry 产品策略、Artifact retention/tombstone stack、具体本机 lifecycle 和 target executable；Linux x64/arm64、PID namespace、断电与固定路由设备门禁；PostgreSQL 16/18 双连接并发与 failover integration；Task revision/context 跨方言 contract/并发压力与引用感知 retention、Keyv 数据迁移 | | PR-2 Run 状态机 | Incubating | 纯转换表、终态/时间/错误/执行器元数据规则、Run version 与 event sequence CAS、事务性 RunCommandService、回滚测试 | 重复 Worker callback/fencing、并发数据库压力测试、Primary 执行链接入 | | PR-3 Executor 端口 | Incubating | ADR-0003、ExecutionSpec/Context/Handle/Result、Executor port、LocalProcessExecutor、进程组取消/超时升级、流式背压、Legacy Cron spec builder、真实进程 contract tests、可复现 edge 基准入口 | 固定 edge/多架构设备基线、Legacy builder 与 makeCommand 差异审计、Primary 生产流量接入 | -| PR-4 Shadow Run | Incubating | origin 三态策略；默认关闭的 `QL3_SHADOW_ORIGINS`；manual、scheduled_node、boot、subscription、system 与 script 现有 ChildProcess 旁路观察；system crond 显式 origin marker、Shell execution ID、finish-only 准入、确定性 Run/Attempt 与 exact replay；`@once` 保持 manual、gRPC transport 不冒充 origin 的准入裁决；每个 worker 懒加载；Run/Attempt/Event 影子生命周期；稳定且不复制 caller 原文的 task identity/revision 与有界日志引用；同 worker 有界注册表和跨 worker 持久化候选关联；stop all/stop instance、Shell callback、乱序/迟到/歧义处理；监听前一次性、Profile-aware 的 keyset Startup Reconciler，终态证据补齐、lost/abandoned/pending 分流与 terminal Attempt response-loss 修复；失败开放和契约测试 | 差异报表、可采集指标、资源压力、回滚演练和 Primary 门禁 | +| PR-4 Shadow Run | Incubating | origin 三态策略；默认关闭的 `QL3_SHADOW_ORIGINS`；manual、scheduled_node、boot、subscription、system 与 script 现有 ChildProcess 旁路观察；system crond 显式 origin marker、Shell execution ID、finish-only 准入、确定性 Run/Attempt 与 exact replay；`@once` 保持 manual、gRPC transport 不冒充 origin 的准入裁决；每个 worker 懒加载；Run/Attempt/Event 影子生命周期；稳定且不复制 caller 原文的 task identity/revision 与有界日志引用；同 worker 有界注册表和跨 worker 持久化候选关联；stop all/stop instance、Shell callback、乱序/迟到/歧义处理；监听前一次性、Profile-aware 的 keyset Startup Reconciler，终态证据补齐、lost/abandoned/pending 分流与 terminal Attempt response-loss 修复；origin-bounded 且逐级守恒的版本化 startup difference report、固定字段 metric batch 与一次性 collector；失败开放和契约测试 | 跨测量窗口终态差异查询、具体 exporter、资源压力、回滚演练和 Primary 门禁 | | PR-5 Primary LocalExecutor | Incubating（默认不激活，仅 manifest-gated manual） | runtime-owned Run 创建器；持久化先于 spawn；Run/Attempt 完整成功、失败、取消、超时与 lost 闭环；Executor handle 身份校验；spawn 后激活写失败的 stop+lost 补偿；completion rejection 安全收敛；独立 Primary 幂等查询与唯一索引竞态裁决；durable `run.cancel_requested`、stop-before-signal、首次请求幂等、晚到完成裁决与待取消有界恢复查询；最多 64 条一页的 cross-worker cancellation source；独立 CancellationDispatch Repository 原子 claim/result、lease expiry 接管、owner/token/version fencing、指数退避与结果 RunEvent；最多 64 页的单周期 cancel supervisor；显式 start/stop、无重叠、错误隔离、停止等待有上限且 timer unref 的 lifecycle runner；Linux durable handle 的 PID/boot/start ticks/process-group 复验与 TERM/KILL controller；完整有界分页且 fail-closed 的 startup Reconcile supervisor；RunningInstance nullable `run_id/attempt_id` 关联；Primary 专用组合 Repository 在同一 SQLite 事务提交前投影 Crontab/RunningInstance，失败整体回滚；有界且防穿越的 legacy log output ref；manual owner seam、真实本机装配、单 spawn/fail-closed；严格 manual-only rollout manifest loader、短期审批/gate、配置哈希审计；HTTP worker 已接轻量 lazy bootstrap，accepted 后按 receipt-first reconcile→completion receipt lifecycle→timeout intent lifecycle→cancel dispatch lifecycle→router 顺序激活，失败撤销，监听失败和 shutdown 有界停止；Primary timeout 在 spawn 前持久化绝对 deadline，有界 source/requester/supervisor 只提交 timeout 意图并复用 CancellationDispatch；代码级 edge/standalone Profile 为各 lifecycle 提供不同 cadence 与页上限，cluster-control/worker 拒绝误装本机 SQLite Primary；统一 CompletionService 原子提交 Attempt/Run/双 Event，spawn 前保存 callback token hash、终态推进 sequence，实时回调与 receipt consumer 共享入口并覆盖两个清理 crash window；manual Primary 已接入受限 POSIX launcher、`0600` direct-file stdout/stderr、父进程退出后续写、不可覆盖 receipt 生产、回执环境清除、TERM 转发等待及 live transaction 后清理；Startup Reconciler receipt-first 双检查并在确定 exited 后执行 profile 化的单次 50/100 ms publish grace；`0007` 独立 CompletionReceiptJournal 在 spawn 前登记、为升级前 active Attempt 补登记并驱动周期扫描，使终态残留继续可发现；确定无效的已知 Attempt receipt 先持久化隔离状态，再进入确定性私有分片 quarantine；终态 missing 与 quarantine 按 edge/standalone retention 有界清理；非 Journal 文件具备只读优先、固定分片/条目上限、overflow fail-closed、显式同盘隔离的 Node 24 运维 CLI；扫描具备页上限、resume cursor、timer unref、无重叠、有界 stop 和低敏计数；ENOSPC 与 launcher receipt 存储失败有代码门禁 | 部署配置写入/审批入口与用户可见状态；PostgreSQL CancellationDispatch adapter；cluster-control 生产启动拓扑；固定 edge/Linux 多架构与真实磁盘压力基线、完整 2.x API 契约和回滚演练 | | PR-7 Worker Session、Run Lease 与启动协议基础 | Incubating（默认关闭，独立入口显式 opt-in） | ADR-0012/0013/0014/0021/0057–0061/0108–0121/0231–0239/0377；有界 capability/Placement/Dispatcher；SQLite 协议孵化与 PostgreSQL v9 Session/Run Lease/credential/attestation authority；immutable revision Placement、数据库时钟 keyset candidate、认证 Worker Pull、digest-only offer recovery；versioned capability-free ExecutionSpec response、stable claim 跨重启退避、单 owner 原子 inbox 准入与 TLS 1.3 mTLS/`ql3w` HTTPS client；同一 package journal 上 revision-fenced starting/spawn/started/running/completion 状态、callback digest、tagged no-spawn 与 ambiguous recovery；PostgreSQL starting/running/start-failure/completion 数据库权威事务、精确重放与 cancellation/timeout 优先终态；batch Secret delivery 在 Attempt advisory lock 下复验 Session/Lease/revision 完整围栏并复用单 Agent，Secret-before-Artifact materializer 将同一 log ID 交给 Executor/journal/running ACK；offer-scoped `wlog-*` 私有文件 spool、Edge/Node 容量策略、append/quota/path 防护、barrier 后 output ownership、受审 POSIX Executor、truncation fact、固定内存流式 source、认证 Artifact stream、共享 immutable store port、S3-compatible SSE/checksum/条件 promotion adapter、upload-before-completion 协调，以及 Local/Cluster 同构、Profile-aware、ETag-fenced range read；用户取消 run.stop mutation 以数据库时间写 intent/Event 并在事务内复验 Project/RoleBinding fence；非执行取消 convergence lifecycle、运行期 expiry 与安全 lost retry 已接入 cluster-control 单一全局 cadence；完整 generation/version/token/Attempt fencing；独立最小权限 Worker ingress、CA/CRL 与连接 generation 热重载；offer journal、spawn barrier、receipt-first recovery；独立 `@qinglong/worker-runtime` 的本地 P-256 CSR、key/chain/trust 验证、generation + active pointer 安装和持久退避；默认关闭的 production process 已装配具体 execution graph、完整 Session heartbeat/drain/offline、direct-file bootstrap、单 Agent/单 cadence、startup reconciliation、证书 maintenance、transport fail-close/recovery 与 Edge/Node 有界预算；真实 PostgreSQL 18 + Linux Node 合约已覆盖 Run completion、credential 和 CA 双轮换且保持同一 Session；真实 K3s 合约已覆盖 TLS/credential Secret 分权、双对象 CAS、Recreate 顺序、identity generation 与单节点 PVC recovery；所有能力默认不可达且受 edge/cluster import audit 约束 | 具体 cert-manager/Vault/SPIFFE/离线 CA adapter 与模板、ingress reload controller、生产 RBAC、证书到期告警和 `ql3w` credential recovery 产品面；具体 KMS/Vault Secret provider、对象存储 credential/temporary lifecycle 与 retention/tombstone；Worker 管理 API；真实 Kubernetes 多节点 CSI/node-loss/production 360 秒 drain 与固定 edge 文件系统 suspend/时钟/断电、x64/arm64 资源门禁 | | PR-8 Project/Policy/Approval Core | Incubating（默认拒绝、无生产业务执行入口） | ADR-0028；统一六类 ActorRef 与 exact-shape 校验；`0017` ownerless default Project 和 append-only versioned RoleBinding；owner/admin/operator/viewer 固定矩阵；Project 内 mutation 幂等、expected-version CAS、双 SQLite 连接竞争门禁；archived read-only、revocation、存储损坏 fail-closed；Agent 写/Secret/Tool `require_approval`；ADR-0047 把六类 subject、role/permission matrix 与 fence 抽到 runtime-core，`pg-0004-project-policy`/capability v3 建立 ownerless PostgreSQL baseline、严格 role/state CHECK、append-only runtime 权限、SERIALIZABLE Project lock、mutation replay、双连接单 winner 和 cluster admission authorizer；ADR-0049/`pg-0005` capability v4 建立 stable IdentitySubject、append-only digest-only API credential、真实 cluster bearer authenticator、write-only durable security audit 与最小权限 runtime role，且已验证 HTTP→credential→Policy→audit→handler 纵向链路；ADR-0051 建立 `/api/v3` 认证前 peer/global 双预算、transport-peer-only、无 timer 且有界内存的 overload shield；ADR-0027 Artifact authorizer adapter；ADR-0029 `AuthenticatedPrincipal` contract、`0018` digest-only versioned challenge、CSPRNG/TTL、同事务消费 challenge + 写首 owner、精确重放与双连接竞争/崩溃回滚门禁；ADR-0030 `0019` stable identity/binding、legacy HS384 + current-session membership、logout/platform/revoke/disable、single-factor 与损坏 fail-closed 门禁；ADR-0031 `0020` digest-bound ApprovalRequest、User-only decision、Project/Role version fence、精确 expiry/重放/并发裁决及同事务 immutable dispatch；ADR-0032 `0021` execution backfill、三表原子 consume、稳定 due keyset、claim/renew/start/result fencing、pre-start takeover/post-start recovery-required、attempt budget、handler inspect/digest barrier 和 bounded dispatcher；ADR-0033/`0022` control/resolution backfill、start/renew/completion 原子联动、稳定 recovery keyset、双 resolver claim/takeover、finding/result 精确重放、自动/人工终结、迟到 completion 单 winner 和 evidence-only bounded reconciler；ADR-0034/`0023` 首个 `run.create` canonical plan、Run/Attempt/Event/receipt 同事务、幂等 collision fail-closed、renew/终态 fence、真实 SQLite handler 与 automatic evidence provider；ADR-0035/`0024` 独立 `approval.recover` 矩阵、稳定 User + 五分钟强认证、Project/RoleBinding fence、human resolution + authorization fact 原子提交、撤权竞态与回滚门禁；ADR-0036 recovery-first 单 timer lifecycle、edge/standalone 独立 cadence/页预算、跨周期 cursor、非重叠与有界 stop；ADR-0074 以新的 Node 24 SQLite v5 ownerless Project/RoleBinding/audit authority 和独立 local-secret-admin 提供强 Principal、`secret.manage`、撤权 fence、envelope+allowed audit 原子提交及不回显语义；ADR-0086 以可信 POSIX console 和 staged delivery 完成本机首 Owner 产品 ceremony | fresh database/pepper setup 与安全迁移向导；`shareStore`/Express 到 authentication core 的 production migration；credential rotation/revocation API、mTLS/Worker enrollment、恢复码；Project/Role/Approval/Secret 管理 CLI/API/UI、audit retention/query/export/alert、preview Artifact/digest/immutable plan builder、真实 MFA/hardware adapter、人工 recovery API/UI/独立 rate limit 与审计事件、handler/provider registry、lifecycle startup/shutdown/指标/admission gate；PostgreSQL action/receipt/provider/recovery-authorization 与 OPA adapter、缓存 version 失效；Tool/Package/Secret/Shell 各自的 handler/evidence contract；Secret/Run/Tool/Workflow waiting_approval 全入口装配；完整回滚演练 | diff --git a/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md b/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md index 345cd01f..7120ad4c 100644 --- a/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md +++ b/docs/adr/ADR-0002-legacy-crontab-compatibility-and-shadow-write.md @@ -5,7 +5,7 @@ - 决策者：QingLong Maintainers - 关联 RFC：[QL-RFC-0001](../QINGLONG_3_0_ARCHITECTURE_RFC.md) - 前置决策：[ADR-0001](./ADR-0001-run-state-and-transaction-boundaries.md) -- Amended by：[ADR-0445](./ADR-0445-schedule-service-origin-shadow-run-coverage.md)、[ADR-0446](./ADR-0446-system-crond-stable-shadow-admission.md)、[ADR-0447](./ADR-0447-boot-shadow-and-non-origin-boundaries.md)、[ADR-0448](./ADR-0448-bounded-legacy-shadow-startup-reconciliation.md) +- Amended by：[ADR-0445](./ADR-0445-schedule-service-origin-shadow-run-coverage.md)、[ADR-0446](./ADR-0446-system-crond-stable-shadow-admission.md)、[ADR-0447](./ADR-0447-boot-shadow-and-non-origin-boundaries.md)、[ADR-0448](./ADR-0448-bounded-legacy-shadow-startup-reconciliation.md)、[ADR-0449](./ADR-0449-versioned-legacy-shadow-startup-difference-report-and-metrics.md) ## 1. 决策摘要 @@ -239,7 +239,7 @@ Shadow Adapter 不得： Shadow 转换仍必须遵守 ADR-0001。无法合法映射时追加 compat.transition_mismatch，不能强行覆盖终态。 -当前 Alpha 切片对已审 Node worker origin 直接观察同一 ChildProcess 的 spawn、error 和 exit 事件，因此不依赖 Shell callback 才能形成基本终态。下述两级关联已补充 Shell callback、stop/cancel 和乱序/迟到回调；ADR-0448 又补充了监听前一次性启动恢复。差异对账、可采集指标与 Primary gate 仍不能由进程内观察或启动 summary 替代。 +当前 Alpha 切片对已审 Node worker origin 直接观察同一 ChildProcess 的 spawn、error 和 exit 事件，因此不依赖 Shell callback 才能形成基本终态。下述两级关联已补充 Shell callback、stop/cancel 和乱序/迟到回调；ADR-0448 又补充了监听前一次性启动恢复，ADR-0449 将其投影为 origin-bounded、版本化的差异报告与固定字段 metric batch。该 startup snapshot 仍不能替代跨测量窗口的历史终态对账与正式 Primary gate。 ### 9.4 `next` Alpha callback 与 stop 关联 @@ -253,7 +253,7 @@ Shadow 转换仍必须遵守 ADR-0001。无法合法映射时追加 compat.trans 6. 取消事实在 Legacy kill 前投递；同 worker 的后续 exit 排在取消之后。跨 worker 使用持久化定位器尽力关联，任何查询或写入失败都不能阻断 kill 或改变 2.x API 响应。 7. 乱序 finished 可以从 queued/claimed 补齐 dispatching、starting、running 和终态；重复终态 callback、取消后的迟到成功 callback 不覆盖终态，也不追加重复完成事件。 -这仍不是完整的 Shadow→Primary 门禁：ADR-0448 已提供启动后有界批量扫描、终态证据补齐、lost/abandoned 收敛和两事务 response-loss 修复；当前仍没有公开指标采集器、Shadow/Legacy 差异报表与正式 Primary gate。后续能力不得让 edge 增加常驻 watcher 或无界内存队列。 +这仍不是完整的 Shadow→Primary 门禁：ADR-0448 已提供启动后有界批量扫描、终态证据补齐、lost/abandoned 收敛和两事务 response-loss 修复；ADR-0449 已增加 startup 差异报表、固定低基数 metric batch 与可注入单次 collector，但尚未完成跨窗口终态差异查询、具体 exporter、资源/回滚演练和正式 Primary gate。后续能力不得让 edge 增加常驻 watcher 或无界内存队列。 ### 9.3 Shadow 写失败 diff --git a/docs/adr/ADR-0448-bounded-legacy-shadow-startup-reconciliation.md b/docs/adr/ADR-0448-bounded-legacy-shadow-startup-reconciliation.md index bf32b44e..2c9d5cf2 100644 --- a/docs/adr/ADR-0448-bounded-legacy-shadow-startup-reconciliation.md +++ b/docs/adr/ADR-0448-bounded-legacy-shadow-startup-reconciliation.md @@ -5,6 +5,7 @@ - 关联 RFC：QL-RFC-0001 D-02、D-356、PR-4 - 关联 ADR：ADR-0001、ADR-0002、ADR-0445、ADR-0446、ADR-0447 - Amends：ADR-0002 的 Shadow 启动与恢复边界 +- Amended by：ADR-0449 ## 上下文 diff --git a/docs/adr/ADR-0449-versioned-legacy-shadow-startup-difference-report-and-metrics.md b/docs/adr/ADR-0449-versioned-legacy-shadow-startup-difference-report-and-metrics.md new file mode 100644 index 00000000..76dad745 --- /dev/null +++ b/docs/adr/ADR-0449-versioned-legacy-shadow-startup-difference-report-and-metrics.md @@ -0,0 +1,75 @@ +# ADR-0449：版本化 Legacy Shadow 启动差异报告与指标批次 + +- 状态：Accepted +- 日期：2026-08-18 +- 关联 RFC：QL-RFC-0001 D-02、D-357、PR-4 +- 关联 ADR：ADR-0002、ADR-0448 +- Amends：ADR-0448 的启动恢复可观测性边界 + +## 上下文 + +ADR-0448 已在监听前对 active Legacy Shadow Run 执行一次有界恢复，但返回值只有进程内总数。总数无法回答某个 execution origin 是否存在 +unresolved difference，也没有稳定 schema 可供日志采集器、测试门或后续 Primary admission 使用。直接输出 Run/Cron/Attempt identity 会扩大敏感面； +用“完整率百分比”又会在分页未完成、历史分母未知时制造虚假精度。 + +本决策只关闭 startup reconciliation 的差异可见性，不宣称已经完成跨测量窗口的 Shadow/Legacy 历史终态对账。后者仍需独立读取终态、时间、 +exit code 与 Artifact existence，并形成可审计窗口。 + +## 决策 + +1. 每个 startup candidate 必须且只能归入九个固定 outcome 之一：`completed`、`cancelled`、`abandoned`、`markedLost`、`repaired`、 + `pending`、`ambiguous`、`skipped`、`failed`。总 outcome 之和必须等于 scanned。 +2. Reconciler 同时维护 aggregate 与按已配置 origin 的 outcome matrix。origin 只能来自受审 `ExecutionOrigin` 枚举，当前 Shadow 配置最多七项； + 不允许 Run ID、Cron ID、Attempt ID、PID、log path、task identity、用户名或错误消息进入维度。 +3. Supervisor 必须跨页合并同一矩阵，不启动第二次扫描。bootstrap 在输出前验证页数、候选数、aggregate conservation、origin conservation、 + origin exact coverage、`remaining/stopReason` 和 page-limit resume cursor；任一不一致按 `RangeError` 失败开放。 +4. 差异报告使用 `qinglong/legacy-shadow-startup-difference-report@v1`，只包含 Profile、固定预算、覆盖范围、aggregate outcomes、最多七条 + origin outcomes 和四态 assessment： + - `converged`：扫描完成且没有 pending、ambiguous、skipped、failed； + - `waiting_external_callback`：只剩可由 system crond 稳定 callback 收敛的 pending； + - `incomplete`：页预算或 cursor stall 导致 remaining； + - `attention_required`：存在 ambiguous、skipped 或 failed，优先级高于分页状态。 +5. 不报告比例。`remaining=true` 时未知尾页不属于可证明分母；即使扫描完成，startup report 也只覆盖当次 active candidate，不冒充历史完整率。 +6. 指标批次使用 `qinglong/legacy-shadow-startup-metric-batch@v1`。维度固定为 profile、assessment、stopReason；数值字段固定为预算、页数、 + scanned、remaining/resumeAvailable 和九个 outcome，并携带同一最多七条 origin matrix。字段是单次启动 snapshot，不伪装为进程内累计 counter。 +7. 默认结构化 startup audit 内同时携带 report 与 metric batch；组合方也可注入单次 `collect` sink。sink 失败不得改变 audit、Legacy 启动或任务结果， + 不做内存积压、磁盘重试或网络重试。 +8. `reconciled` audit 只用于 `converged`；其余可验证但未闭合状态均标为 `incomplete`。该状态本阶段仍只报告，不阻止 HTTP listen，正式 + origin-scoped Primary gate 必须在后续决策中显式消费报告并保持副作用前 fail-closed。 + +## 资源与部署影响 + +- 不新增 package、生产依赖、数据库查询、schema、migration、表、索引、连接、timer、watcher、线程、进程、端口或 Kubernetes 对象。 +- 每个 candidate 只增加一次常数计数；矩阵最多七行，edge 仍最多处理 8 个 candidate，standalone 仍最多 128 个。 +- 默认关闭和 cluster-control/worker 仍不加载 Repository，也不生成本机恢复 report/metrics。 +- metric collector 是可选调用端口，不拥有 exporter、队列或重试 authority；后续 Prometheus/OTel adapter 必须保持固定维度并单独评审。 + +## 被拒绝的替代方案 + +### 只把原始 summary 打进日志 + +拒绝。它没有 schema、origin 分解、conservation fence 或稳定指标字段，后续 gate 只能猜测日志形状。 + +### 输出每条差异的 Run/Cron identity + +拒绝。启动日志和 metric labels 会形成高基数与任务信息泄漏；需要逐条诊断时应由受认证、有界的独立查询产品提供。 + +### 用成功数除以 scanned 作为完整率 + +拒绝。startup active scan 不是测量窗口，remaining 时分母未知，pending 也不是失败；百分比会把局部 snapshot 冒充 rollout 证据。 + +### 在路由设备内常驻 metrics registry/exporter + +拒绝。PR-4 当前只需要稳定采集合同；常驻 exporter、队列、网络重试和生命周期必须由 Profile 部署层独立决定。 + +## 验证 + +- 真实 SQLite 覆盖 manual lost 与 scheduled_system pending 的双 origin matrix，aggregate 与 origin outcome 精确守恒。 +- Bootstrap 覆盖 report/metric schema、standalone 128 candidate 预算、waiting/incomplete/attention assessment、cursor identity 脱敏、畸形 summary + 失败开放和 collector failure 失败开放。 +- 聚焦测试 `21/21`，Legacy/Shadow 串行扩展 `77/77`；`build:back` 与完整 backend + `1,440 pass / 0 fail / 2 conditional skip`；18 个 QL3 package 均完成 clean build/test。 +- 14/14 static audit 与 14/14 artifact audit 通过；七档 edge/standalone 产物字节与 D-356 完全一致，证明 Legacy report/metrics + 未穿透 QL3 package 产物边界。 +- 本切片不改数据库/schema/migration、容器或 Kubernetes 部署面，未重跑物理 PostgreSQL HA/K3s 门；对象存储恢复/PITR 与 + cert-manager mTLS 轮换继续保留为发布最终化现场证据门。 diff --git a/docs/adr/README.md b/docs/adr/README.md index dfdeabe8..c90ffe64 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -452,6 +452,7 @@ | [ADR-0446](./ADR-0446-system-crond-stable-shadow-admission.md) | System Crond 稳定 Shadow 准入与回调重放 | Accepted | | [ADR-0447](./ADR-0447-boot-shadow-and-non-origin-boundaries.md) | Boot Shadow 准入与 once/gRPC 非 Origin 边界 | Accepted | | [ADR-0448](./ADR-0448-bounded-legacy-shadow-startup-reconciliation.md) | 有界 Legacy Shadow 启动恢复 | Accepted | +| [ADR-0449](./ADR-0449-versioned-legacy-shadow-startup-difference-report-and-metrics.md) | 版本化 Legacy Shadow 启动差异报告与指标批次 | Accepted | ## 规则 diff --git a/test/back/bootstrapLegacyShadowStartupReconciliation.test.cjs b/test/back/bootstrapLegacyShadowStartupReconciliation.test.cjs index 7ae8d849..69c75bbc 100644 --- a/test/back/bootstrapLegacyShadowStartupReconciliation.test.cjs +++ b/test/back/bootstrapLegacyShadowStartupReconciliation.test.cjs @@ -6,10 +6,40 @@ const path = require('node:path'); const { test } = require('node:test'); const { bootstrapLegacyShadowStartupReconciliation, + createLegacyShadowStartupDifferenceReport, } = require('../../back/runtime/adapters/legacy/bootstrapLegacyShadowStartupReconciliation'); -function summary(overrides = {}) { +const OUTCOMES = [ + 'completed', + 'cancelled', + 'abandoned', + 'markedLost', + 'repaired', + 'pending', + 'ambiguous', + 'skipped', + 'failed', +]; + +function originSummary(origin, overrides = {}) { return { + origin, + scanned: 0, + completed: 0, + cancelled: 0, + abandoned: 0, + markedLost: 0, + repaired: 0, + pending: 0, + ambiguous: 0, + skipped: 0, + failed: 0, + ...overrides, + }; +} + +function summary(overrides = {}, origins = ['manual']) { + const value = { pages: 1, scanned: 0, completed: 0, @@ -25,6 +55,8 @@ function summary(overrides = {}) { remaining: false, ...overrides, }; + value.byOrigin ??= origins.map((origin) => originSummary(origin)); + return value; } test('disabled bootstrap remains fully lazy', async () => { @@ -67,7 +99,7 @@ test('applies distinct one-shot edge and standalone budgets', async () => { profile, async execute(request) { requests.push(request); - return summary(); + return summary({}, request.origins); }, audit() {}, }); @@ -87,6 +119,155 @@ test('applies distinct one-shot edge and standalone budgets', async () => { ); }); +test('emits one versioned origin-bounded difference report and metric batch', async () => { + const batches = []; + const result = await bootstrapLegacyShadowStartupReconciliation({ + origins: ['manual', 'scheduled_system'], + profile: 'standalone', + async execute(request) { + return summary( + { + scanned: 4, + completed: 1, + markedLost: 1, + repaired: 1, + pending: 1, + byOrigin: [ + originSummary('manual', { + scanned: 3, + completed: 1, + markedLost: 1, + repaired: 1, + }), + originSummary('scheduled_system', { + scanned: 1, + pending: 1, + }), + ], + }, + request.origins, + ); + }, + audit() {}, + collect(batch) { + batches.push(batch); + }, + }); + + assert.equal(result.state, 'incomplete'); + assert.equal( + result.report.schema, + 'qinglong/legacy-shadow-startup-difference-report@v1', + ); + assert.equal(result.report.assessment, 'waiting_external_callback'); + assert.equal(result.report.budget.maxCandidates, 128); + assert.deepEqual( + result.report.byOrigin.map(({ origin, scanned }) => ({ origin, scanned })), + [ + { origin: 'manual', scanned: 3 }, + { origin: 'scheduled_system', scanned: 1 }, + ], + ); + assert.equal(batches.length, 1); + assert.equal( + batches[0].schema, + 'qinglong/legacy-shadow-startup-metric-batch@v1', + ); + assert.equal(batches[0].dimensions.assessment, 'waiting_external_callback'); + assert.equal(batches[0].values.scanned, 4); + assert.equal(batches[0].values.pending, 1); + assert.deepEqual( + Object.keys(result.report.outcomes).sort(), + [...OUTCOMES].sort(), + ); +}); + +test('turns inconsistent report input into a fail-open low-sensitivity audit', async () => { + let collected = false; + const result = await bootstrapLegacyShadowStartupReconciliation({ + origins: ['manual'], + profile: 'edge', + async execute() { + return summary({ scanned: 1 }); + }, + audit() {}, + collect() { + collected = true; + }, + }); + + assert.deepEqual(result, { state: 'failed', errorType: 'RangeError' }); + assert.equal(collected, false); +}); + +test('rejects non-Profile budgets and cursors outside page-limit reports', () => { + assert.throws( + () => + createLegacyShadowStartupDifferenceReport( + { + origins: ['manual'], + profile: 'edge', + pageSize: 32, + maxPages: 1, + }, + summary(), + ), + RangeError, + ); + assert.throws( + () => + createLegacyShadowStartupDifferenceReport( + { origins: ['manual'], profile: 'edge', pageSize: 8, maxPages: 1 }, + summary({ + nextCursor: { createdAtMs: 10, runId: 'must-not-escape' }, + }), + ), + RangeError, + ); +}); + +test('classifies ambiguous, skipped, or failed comparisons as attention required', async () => { + const result = await bootstrapLegacyShadowStartupReconciliation({ + origins: ['manual'], + profile: 'edge', + async execute(request) { + return summary( + { + scanned: 1, + ambiguous: 1, + byOrigin: [originSummary('manual', { scanned: 1, ambiguous: 1 })], + }, + request.origins, + ); + }, + audit() {}, + }); + + assert.equal(result.state, 'incomplete'); + assert.equal(result.report.assessment, 'attention_required'); + assert.equal(result.metrics.dimensions.assessment, 'attention_required'); +}); + +test('keeps startup fail-open when metric collection fails', async () => { + const result = await bootstrapLegacyShadowStartupReconciliation({ + origins: ['manual'], + profile: 'edge', + async execute(request) { + return summary({}, request.origins); + }, + audit() {}, + collect() { + throw new Error('collector transport secret'); + }, + }); + + assert.equal(result.state, 'reconciled'); + assert.equal( + JSON.stringify(result).includes('collector transport secret'), + false, + ); +}); + test('fails open with a low-sensitivity error type', async () => { const result = await bootstrapLegacyShadowStartupReconciliation({ origins: ['manual'], @@ -121,6 +302,9 @@ test('redacts the resume cursor Run identity from startup audit output', async ( assert.equal(result.state, 'incomplete'); assert.equal(result.summary.resumeAvailable, true); assert.equal('nextCursor' in result.summary, false); + assert.equal(result.report.assessment, 'incomplete'); + assert.equal(result.metrics.values.resumeAvailable, 1); + assert.equal('nextCursor' in result.report.coverage, false); assert.equal(records.join('').includes('run-secret-identity'), false); }); diff --git a/test/back/legacyShadowStartupReconciliation.test.cjs b/test/back/legacyShadowStartupReconciliation.test.cjs index 581226b2..557cdc58 100644 --- a/test/back/legacyShadowStartupReconciliation.test.cjs +++ b/test/back/legacyShadowStartupReconciliation.test.cjs @@ -281,6 +281,50 @@ test('keeps system-crond pending without terminal callback evidence', async () = ); }); +test('keeps a bounded per-origin outcome matrix for later ownership gates', async () => { + const { database, writer, reconciler } = await createStack(); + await activeShadow(writer, { legacyCronId: 15, pid: 4311 }); + await activeShadow(writer, { + origin: 'scheduled_system', + legacyCronId: 16, + pid: 4312, + }); + await insertInstance(database, { + cron_id: 15, + pid: 4311, + status: 2, + finished_at: null, + }); + await insertInstance(database, { + cron_id: 16, + pid: 4312, + status: 2, + finished_at: null, + }); + + const summary = await reconciler.reconcileBatch({ + origins: ['manual', 'scheduled_system'], + }); + + assert.deepEqual( + summary.byOrigin.map(({ origin, scanned, markedLost, pending }) => ({ + origin, + scanned, + markedLost, + pending, + })), + [ + { origin: 'manual', scanned: 1, markedLost: 1, pending: 0 }, + { + origin: 'scheduled_system', + scanned: 1, + markedLost: 0, + pending: 1, + }, + ], + ); +}); + test('refuses ambiguous duplicate identity evidence instead of guessing a terminal result', async () => { const { database, repository, writer, reconciler } = await createStack(); const logPath = 'cron/ambiguous.log'; @@ -349,6 +393,21 @@ test('supervisor preserves a stable cursor when its Profile budget is exhausted' ambiguous: 0, skipped: 0, failed: 0, + byOrigin: [ + { + origin: 'manual', + scanned: 1, + completed: 0, + cancelled: 0, + abandoned: 0, + markedLost: 1, + repaired: 0, + pending: 0, + ambiguous: 0, + skipped: 0, + failed: 0, + }, + ], truncated: true, nextCursor: { createdAtMs: 10, runId: 'run-10' }, };