feat(ql3): report legacy shadow startup differences

back/runtime/adapters/legacy/bootstrapLegacyShadowStartupReconciliation.ts

@@ -6,7 +6,12 @@ import {
 } from '../../domain/deploymentProfile';
 import { configuredLegacyShadowOrigins } from '../../compatibility/legacyExecutionBridge';
 import { createLegacyLogArtifactId } from '../../compatibility/legacyTaskRevision';
-import type { LegacyShadowStartupSummary } from '../../application/legacyShadowStartupReconciler';
+import {
+  LEGACY_SHADOW_STARTUP_OUTCOMES,
+  type LegacyShadowStartupOriginSummary,
+  type LegacyShadowStartupOutcomeCounts,
+  type LegacyShadowStartupSummary,
+} from '../../application/legacyShadowStartupReconciler';
 
 export interface LegacyShadowStartupBudget {
   pageSize: number;
@@ -18,6 +23,60 @@ export interface LegacyShadowStartupRequest extends LegacyShadowStartupBudget {
   profile: 'edge' | 'standalone';
 }
 
+export const LEGACY_SHADOW_STARTUP_DIFFERENCE_REPORT_SCHEMA =
+  'qinglong/legacy-shadow-startup-difference-report@v1';
+export const LEGACY_SHADOW_STARTUP_METRIC_BATCH_SCHEMA =
+  'qinglong/legacy-shadow-startup-metric-batch@v1';
+
+export type LegacyShadowStartupAssessment =
+  | 'converged'
+  | 'waiting_external_callback'
+  | 'incomplete'
+  | 'attention_required';
+
+export interface LegacyShadowStartupDifferenceReport {
+  schemaVersion: 1;
+  schema: typeof LEGACY_SHADOW_STARTUP_DIFFERENCE_REPORT_SCHEMA;
+  profile: 'edge' | 'standalone';
+  assessment: LegacyShadowStartupAssessment;
+  configuredOriginCount: number;
+  budget: {
+    pageSize: number;
+    maxPages: number;
+    maxCandidates: number;
+  };
+  coverage: {
+    pages: number;
+    scanned: number;
+    stopReason: LegacyShadowStartupSummary['stopReason'];
+    remaining: boolean;
+    resumeAvailable: boolean;
+  };
+  outcomes: LegacyShadowStartupOutcomeCounts;
+  byOrigin: readonly LegacyShadowStartupOriginSummary[];
+}
+
+export interface LegacyShadowStartupMetricBatch {
+  schemaVersion: 1;
+  schema: typeof LEGACY_SHADOW_STARTUP_METRIC_BATCH_SCHEMA;
+  dimensions: {
+    profile: 'edge' | 'standalone';
+    assessment: LegacyShadowStartupAssessment;
+    stopReason: LegacyShadowStartupSummary['stopReason'];
+  };
+  values: LegacyShadowStartupOutcomeCounts & {
+    configuredOrigins: number;
+    pageSize: number;
+    maxPages: number;
+    maxCandidates: number;
+    pages: number;
+    scanned: number;
+    remaining: 0 | 1;
+    resumeAvailable: 0 | 1;
+  };
+  byOrigin: readonly LegacyShadowStartupOriginSummary[];
+}
+
 export type LegacyShadowStartupAudit =
   | {
       state: 'disabled';
@@ -30,9 +89,11 @@ export type LegacyShadowStartupAudit =
       state: 'reconciled' | 'incomplete';
       profile: 'edge' | 'standalone';
       origins: number;
-      summary: Omit<LegacyShadowStartupSummary, 'nextCursor'> & {
+      summary: Omit<LegacyShadowStartupSummary, 'nextCursor' | 'byOrigin'> & {
         resumeAvailable: boolean;
       };
+      report: LegacyShadowStartupDifferenceReport;
+      metrics: LegacyShadowStartupMetricBatch;
     }
   | {
       state: 'failed';
@@ -46,6 +107,7 @@ export interface BootstrapLegacyShadowStartupOptions {
     request: LegacyShadowStartupRequest,
   ) => Promise<LegacyShadowStartupSummary>;
   audit?: (record: LegacyShadowStartupAudit) => void | Promise<void>;
+  collect?: (metrics: LegacyShadowStartupMetricBatch) => void | Promise<void>;
 }
 
 const BUDGETS: Readonly<
@@ -54,17 +116,195 @@ const BUDGETS: Readonly<
   edge: { pageSize: 8, maxPages: 1 },
   standalone: { pageSize: 32, maxPages: 4 },
 };
+const MAX_CONFIGURED_LEGACY_SHADOW_ORIGINS = 7;
 
 function auditSummary(summary: LegacyShadowStartupSummary): Omit<
   LegacyShadowStartupSummary,
-  'nextCursor'
+  'nextCursor' | 'byOrigin'
 > & {
   resumeAvailable: boolean;
 } {
-  const { nextCursor, ...bounded } = summary;
+  const { nextCursor, byOrigin: _byOrigin, ...bounded } = summary;
   return { ...bounded, resumeAvailable: nextCursor !== undefined };
 }
 
+function outcomeCounts(
+  source: LegacyShadowStartupOutcomeCounts,
+): LegacyShadowStartupOutcomeCounts {
+  return Object.fromEntries(
+    LEGACY_SHADOW_STARTUP_OUTCOMES.map((outcome) => [outcome, source[outcome]]),
+  ) as LegacyShadowStartupOutcomeCounts;
+}
+
+function outcomeTotal(source: LegacyShadowStartupOutcomeCounts): number {
+  return LEGACY_SHADOW_STARTUP_OUTCOMES.reduce(
+    (total, outcome) => total + source[outcome],
+    0,
+  );
+}
+
+function assertCount(value: number, label: string, maximum: number): void {
+  if (!Number.isSafeInteger(value) || value < 0 || value > maximum) {
+    throw new RangeError(`${label} is outside its startup budget`);
+  }
+}
+
+function assessment(
+  summary: LegacyShadowStartupSummary,
+): LegacyShadowStartupAssessment {
+  if (summary.ambiguous > 0 || summary.skipped > 0 || summary.failed > 0) {
+    return 'attention_required';
+  }
+  if (summary.remaining) return 'incomplete';
+  if (summary.pending > 0) return 'waiting_external_callback';
+  return 'converged';
+}
+
+export function createLegacyShadowStartupDifferenceReport(
+  request: LegacyShadowStartupRequest,
+  summary: LegacyShadowStartupSummary,
+): LegacyShadowStartupDifferenceReport {
+  const expectedBudget = BUDGETS[request.profile];
+  if (
+    request.pageSize !== expectedBudget.pageSize ||
+    request.maxPages !== expectedBudget.maxPages
+  ) {
+    throw new RangeError('startup report budget does not match its Profile');
+  }
+  const configuredOrigins = [...new Set(request.origins)];
+  if (
+    configuredOrigins.length < 1 ||
+    configuredOrigins.length > MAX_CONFIGURED_LEGACY_SHADOW_ORIGINS
+  ) {
+    throw new RangeError(
+      'configured Shadow origin count is outside its budget',
+    );
+  }
+  const maxCandidates = request.pageSize * request.maxPages;
+  assertCount(summary.pages, 'pages', request.maxPages);
+  if (summary.pages < 1) {
+    throw new RangeError('pages must include the executed startup page');
+  }
+  assertCount(summary.scanned, 'scanned', request.pageSize * summary.pages);
+  for (const outcome of LEGACY_SHADOW_STARTUP_OUTCOMES) {
+    assertCount(summary[outcome], outcome, summary.scanned);
+  }
+  if (outcomeTotal(summary) !== summary.scanned) {
+    throw new RangeError('startup outcomes do not conserve scanned candidates');
+  }
+  if (summary.remaining !== (summary.stopReason !== 'complete')) {
+    throw new RangeError('startup remaining and stop reason disagree');
+  }
+  if (
+    !['complete', 'page_limit', 'cursor_stalled'].includes(summary.stopReason)
+  ) {
+    throw new RangeError('startup stop reason is invalid');
+  }
+  if (summary.stopReason === 'page_limit' && summary.nextCursor === undefined) {
+    throw new RangeError('page-limited startup summary has no resume cursor');
+  }
+  if (summary.stopReason !== 'page_limit' && summary.nextCursor !== undefined) {
+    throw new RangeError('resume cursor is present without a page limit');
+  }
+
+  if (summary.byOrigin.length !== configuredOrigins.length) {
+    throw new RangeError('origin outcome coverage is incomplete');
+  }
+  const origins = new Map<ExecutionOrigin, LegacyShadowStartupOriginSummary>();
+  for (const origin of summary.byOrigin) {
+    if (
+      origins.has(origin.origin) ||
+      !configuredOrigins.includes(origin.origin)
+    ) {
+      throw new RangeError('origin outcome coverage is invalid');
+    }
+    assertCount(origin.scanned, `${origin.origin}:scanned`, summary.scanned);
+    for (const outcome of LEGACY_SHADOW_STARTUP_OUTCOMES) {
+      assertCount(
+        origin[outcome],
+        `${origin.origin}:${outcome}`,
+        origin.scanned,
+      );
+    }
+    if (outcomeTotal(origin) !== origin.scanned) {
+      throw new RangeError(
+        'origin outcomes do not conserve scanned candidates',
+      );
+    }
+    origins.set(origin.origin, {
+      origin: origin.origin,
+      scanned: origin.scanned,
+      ...outcomeCounts(origin),
+    });
+  }
+  const orderedOrigins = configuredOrigins.map(
+    (origin) => origins.get(origin)!,
+  );
+  if (
+    orderedOrigins.reduce((total, origin) => total + origin.scanned, 0) !==
+      summary.scanned ||
+    LEGACY_SHADOW_STARTUP_OUTCOMES.some(
+      (outcome) =>
+        orderedOrigins.reduce((total, origin) => total + origin[outcome], 0) !==
+        summary[outcome],
+    )
+  ) {
+    throw new RangeError('origin outcomes do not match aggregate outcomes');
+  }
+
+  return {
+    schemaVersion: 1,
+    schema: LEGACY_SHADOW_STARTUP_DIFFERENCE_REPORT_SCHEMA,
+    profile: request.profile,
+    assessment: assessment(summary),
+    configuredOriginCount: configuredOrigins.length,
+    budget: {
+      pageSize: request.pageSize,
+      maxPages: request.maxPages,
+      maxCandidates,
+    },
+    coverage: {
+      pages: summary.pages,
+      scanned: summary.scanned,
+      stopReason: summary.stopReason,
+      remaining: summary.remaining,
+      resumeAvailable: summary.nextCursor !== undefined,
+    },
+    outcomes: outcomeCounts(summary),
+    byOrigin: orderedOrigins,
+  };
+}
+
+export function createLegacyShadowStartupMetricBatch(
+  report: LegacyShadowStartupDifferenceReport,
+): LegacyShadowStartupMetricBatch {
+  return {
+    schemaVersion: 1,
+    schema: LEGACY_SHADOW_STARTUP_METRIC_BATCH_SCHEMA,
+    dimensions: {
+      profile: report.profile,
+      assessment: report.assessment,
+      stopReason: report.coverage.stopReason,
+    },
+    values: {
+      configuredOrigins: report.configuredOriginCount,
+      pageSize: report.budget.pageSize,
+      maxPages: report.budget.maxPages,
+      maxCandidates: report.budget.maxCandidates,
+      pages: report.coverage.pages,
+      scanned: report.coverage.scanned,
+      remaining: report.coverage.remaining ? 1 : 0,
+      resumeAvailable: report.coverage.resumeAvailable ? 1 : 0,
+      ...outcomeCounts(report.outcomes),
+    },
+    byOrigin: report.byOrigin.map((origin) => ({
+      origin: origin.origin,
+      scanned: origin.scanned,
+      ...outcomeCounts(origin),
+    })),
+  };
+}
+
 async function executeDefault(
   request: LegacyShadowStartupRequest,
 ): Promise<LegacyShadowStartupSummary> {
@@ -137,22 +377,32 @@ export async function bootstrapLegacyShadowStartupReconciliation(
       }
       return record;
     }
-    const summary = await (options.execute ?? executeDefault)({
+    const request: LegacyShadowStartupRequest = {
       origins,
       profile,
       ...BUDGETS[profile],
-    });
+    };
+    const summary = await (options.execute ?? executeDefault)(request);
+    const report = createLegacyShadowStartupDifferenceReport(request, summary);
+    const metrics = createLegacyShadowStartupMetricBatch(report);
     const record: LegacyShadowStartupAudit = {
-      state: summary.remaining ? 'incomplete' : 'reconciled',
+      state: report.assessment === 'converged' ? 'reconciled' : 'incomplete',
       profile,
       origins: origins.length,
       summary: auditSummary(summary),
+      report,
+      metrics,
     };
     try {
       await audit(record);
     } catch {
       // Shadow audit output must not affect Legacy startup.
     }
+    try {
+      await options.collect?.(metrics);
+    } catch {
+      // Shadow metric collection must not affect Legacy startup.
+    }
     return record;
   } catch (error) {
     const record: LegacyShadowStartupAudit = {

back/runtime/application/legacyShadowStartupReconciler.ts

@@ -23,41 +23,86 @@ export interface LegacyShadowStartupClock {
   now(): number;
 }
 
-export interface LegacyShadowStartupReconcileSummary {
-  scanned: number;
-  completed: number;
-  cancelled: number;
-  abandoned: number;
-  markedLost: number;
-  repaired: number;
-  pending: number;
-  ambiguous: number;
-  skipped: number;
-  failed: number;
-  truncated: boolean;
-  nextCursor?: LegacyShadowStartupCursor;
-}
+export const LEGACY_SHADOW_STARTUP_OUTCOMES = Object.freeze([
+  'completed',
+  'cancelled',
+  'abandoned',
+  'markedLost',
+  'repaired',
+  'pending',
+  'ambiguous',
+  'skipped',
+  'failed',
+] as const);
+
+export type LegacyShadowStartupOutcome =
+  (typeof LEGACY_SHADOW_STARTUP_OUTCOMES)[number];
+
+export type LegacyShadowStartupOutcomeCounts = Record<
+  LegacyShadowStartupOutcome,
+  number
+>;
+
+export type LegacyShadowStartupOriginSummary =
+  LegacyShadowStartupOutcomeCounts & {
+    origin: RunRecord['executionOrigin'];
+    scanned: number;
+  };
+
+export type LegacyShadowStartupReconcileSummary =
+  LegacyShadowStartupOutcomeCounts & {
+    scanned: number;
+    byOrigin: readonly LegacyShadowStartupOriginSummary[];
+    truncated: boolean;
+    nextCursor?: LegacyShadowStartupCursor;
+  };
 
 export type LegacyShadowStartupStopReason =
   | 'complete'
   | 'page_limit'
   | 'cursor_stalled';
 
-export interface LegacyShadowStartupSummary
-  extends Omit<
-    LegacyShadowStartupReconcileSummary,
-    'truncated' | 'nextCursor'
-  > {
+export type LegacyShadowStartupSummary = Omit<
+  LegacyShadowStartupReconcileSummary,
+  'truncated' | 'nextCursor'
+> & {
   pages: number;
   stopReason: LegacyShadowStartupStopReason;
   remaining: boolean;
   nextCursor?: LegacyShadowStartupCursor;
-}
+};
 
 type EvidenceSelection =
   | { status: 'matched'; evidence: LegacyRunningInstanceEvidence }
   | { status: 'none' | 'ambiguous' };
 
+function emptyOutcomeCounts(): LegacyShadowStartupOutcomeCounts {
+  return {
+    completed: 0,
+    cancelled: 0,
+    abandoned: 0,
+    markedLost: 0,
+    repaired: 0,
+    pending: 0,
+    ambiguous: 0,
+    skipped: 0,
+    failed: 0,
+  };
+}
+
+function originSummary(
+  origin: RunRecord['executionOrigin'],
+): LegacyShadowStartupOriginSummary {
+  return { origin, scanned: 0, ...emptyOutcomeCounts() };
+}
+
+function incrementOutcome(
+  target: LegacyShadowStartupOutcomeCounts,
+  outcome: LegacyShadowStartupOutcome,
+): void {
+  target[outcome] += 1;
+}
+
 function sameCursor(
   left: LegacyShadowStartupCursor | undefined,
   right: LegacyShadowStartupCursor,
@@ -168,27 +213,34 @@ export class LegacyShadowStartupReconciler {
       ...(options.cursor === undefined ? {} : { cursor: options.cursor }),
       ...(options.limit === undefined ? {} : { limit: options.limit }),
     });
+    const originSummaries = new Map(
+      enabledOrigins.map((origin) => [origin, originSummary(origin)]),
+    );
     const summary: LegacyShadowStartupReconcileSummary = {
       scanned: page.candidates.length,
-      completed: 0,
-      cancelled: 0,
-      abandoned: 0,
-      markedLost: 0,
-      repaired: 0,
-      pending: 0,
-      ambiguous: 0,
-      skipped: 0,
-      failed: 0,
+      ...emptyOutcomeCounts(),
+      byOrigin: [...originSummaries.values()],
       truncated: page.truncated,
       ...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
     };
     const origins = new Set(enabledOrigins);
     for (const candidate of page.candidates) {
-      try {
-        await this.reconcileCandidate(candidate, origins, summary);
-      } catch {
-        summary.failed += 1;
+      const perOrigin =
+        originSummaries.get(candidate.origin) ??
+        originSummary(candidate.origin);
+      if (!originSummaries.has(candidate.origin)) {
+        originSummaries.set(candidate.origin, perOrigin);
+        summary.byOrigin = [...originSummaries.values()];
       }
+      perOrigin.scanned += 1;
+      let outcome: LegacyShadowStartupOutcome;
+      try {
+        outcome = await this.reconcileCandidate(candidate, origins);
+      } catch {
+        outcome = 'failed';
+      }
+      incrementOutcome(summary, outcome);
+      incrementOutcome(perOrigin, outcome);
     }
     return summary;
   }
@@ -196,26 +248,23 @@ export class LegacyShadowStartupReconciler {
   private async reconcileCandidate(
     candidate: LegacyShadowStartupCandidate,
     origins: ReadonlySet<RunRecord['executionOrigin']>,
-    summary: LegacyShadowStartupReconcileSummary,
-  ): Promise<void> {
+  ): Promise<LegacyShadowStartupOutcome> {
     const run = await this.repository.findRunById(candidate.runId);
     if (
       !run ||
       run.executionOwner !== 'legacy' ||
       !origins.has(run.executionOrigin) ||
+      run.executionOrigin !== candidate.origin ||
       isTerminalRunStatus(run.status) ||
       run.status === 'lost'
     ) {
-      summary.skipped += 1;
-      return;
+      return 'skipped';
     }
     if (candidate.activeAttemptCount === 0) {
-      await this.repairTerminalAttempt(run, summary);
-      return;
+      return this.repairTerminalAttempt(run);
     }
     if (candidate.activeAttemptCount !== 1 || !candidate.attempt) {
-      summary.ambiguous += 1;
-      return;
+      return 'ambiguous';
     }
     const attempt = await this.repository.findAttemptById(
       candidate.attempt.attemptId,
@@ -225,8 +274,7 @@ export class LegacyShadowStartupReconciler {
       attempt.runId !== run.id ||
       isTerminalRunAttemptStatus(attempt.status)
     ) {
-      summary.skipped += 1;
-      return;
+      return 'skipped';
     }
 
     const evidence =
@@ -236,13 +284,11 @@ export class LegacyShadowStartupReconciler {
             legacyCronId: candidate.legacyCronId,
           });
     if (evidence.truncated) {
-      summary.ambiguous += 1;
-      return;
+      return 'ambiguous';
     }
     const selected = selectEvidence(attempt, evidence.evidence);
     if (selected.status === 'ambiguous') {
-      summary.ambiguous += 1;
-      return;
+      return 'ambiguous';
     }
     if (
       selected.status === 'matched' &&
@@ -254,8 +300,7 @@ export class LegacyShadowStartupReconciler {
           { runId: run.id, attemptId: attempt.id },
           { atMs, reason: 'reconcile' },
         );
-        summary.cancelled += 1;
-        return;
+        return 'cancelled';
       }
       if (
         selected.evidence.outcome === 'succeeded' ||
@@ -274,35 +319,30 @@ export class LegacyShadowStartupReconciler {
               : selected.evidence.exitCode,
           },
         );
-        summary.completed += 1;
-        return;
+        return 'completed';
       }
     }
 
     if (run.executionOrigin === 'scheduled_system') {
-      summary.pending += 1;
-      return;
+      return 'pending';
     }
     if (run.status === 'queued' && attempt.status === 'claimed') {
       await this.abandon(run, attempt);
-      summary.abandoned += 1;
-      return;
+      return 'abandoned';
     }
     if (
       (run.status === 'dispatching' || run.status === 'running') &&
       ['claimed', 'starting', 'running'].includes(attempt.status)
     ) {
       await this.markLost(run, attempt);
-      summary.markedLost += 1;
-      return;
+      return 'markedLost';
     }
-    summary.ambiguous += 1;
+    return 'ambiguous';
   }
 
   private async repairTerminalAttempt(
     run: RunRecord,
-    summary: LegacyShadowStartupReconcileSummary,
-  ): Promise<void> {
+  ): Promise<'repaired' | 'ambiguous'> {
     const attempt = await this.repository.findLatestAttemptByRunId(run.id);
     const target = attempt ? terminalRunTarget(attempt) : undefined;
     if (
@@ -310,8 +350,7 @@ export class LegacyShadowStartupReconciler {
       target === undefined ||
       !RUN_TRANSITIONS[run.status].includes(target)
     ) {
-      summary.ambiguous += 1;
-      return;
+      return 'ambiguous';
     }
     await this.commands.transitionRun({
       runId: run.id,
@@ -327,7 +366,7 @@ export class LegacyShadowStartupReconciler {
       actor: { type: 'reconciler' },
       dedupeKey: `legacy-startup-repair:${attempt.id}:${target}`,
     });
-    summary.repaired += 1;
+    return 'repaired';
   }
 
   private async abandon(
@@ -438,18 +477,17 @@ export class LegacyShadowStartupSupervisor {
       );
     }
 
+    const originSummaries = new Map(
+      [...new Set(options.origins)].map((origin) => [
+        origin,
+        originSummary(origin),
+      ]),
+    );
     const total: LegacyShadowStartupSummary = {
       pages: 0,
       scanned: 0,
-      completed: 0,
-      cancelled: 0,
-      abandoned: 0,
-      markedLost: 0,
-      repaired: 0,
-      pending: 0,
-      ambiguous: 0,
-      skipped: 0,
-      failed: 0,
+      ...emptyOutcomeCounts(),
+      byOrigin: [...originSummaries.values()],
       stopReason: 'complete',
       remaining: false,
     };
@@ -471,6 +509,19 @@ export class LegacyShadowStartupSupervisor {
       total.ambiguous += page.ambiguous;
       total.skipped += page.skipped;
       total.failed += page.failed;
+      for (const pageOrigin of page.byOrigin) {
+        const aggregate =
+          originSummaries.get(pageOrigin.origin) ??
+          originSummary(pageOrigin.origin);
+        if (!originSummaries.has(pageOrigin.origin)) {
+          originSummaries.set(pageOrigin.origin, aggregate);
+          total.byOrigin = [...originSummaries.values()];
+        }
+        aggregate.scanned += pageOrigin.scanned;
+        for (const outcome of LEGACY_SHADOW_STARTUP_OUTCOMES) {
+          aggregate[outcome] += pageOrigin[outcome];
+        }
+      }
       if (!page.truncated) return total;
       if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
         total.stopReason = 'cursor_stalled';