import type { ExecutionOrigin, RunAttemptStatus, RunStatus, } from '../domain/run'; import { EXECUTION_ORIGINS } from '../domain/run'; import { MAX_LEGACY_SHADOW_TERMINAL_EVIDENCE_PER_PAGE, MAX_LEGACY_SHADOW_TERMINAL_PAGE_SIZE, type LegacyShadowTerminalAttemptEvidence, type LegacyShadowTerminalCandidate, type LegacyShadowTerminalCursor, type LegacyShadowTerminalDifferenceSource, type LegacyShadowTerminalPage, type LegacyTerminalEvidence, type LegacyTerminalOutcome, } from '../ports/legacyShadowTerminalDifference'; export type LegacyShadowTerminalAuditProfile = 'edge' | 'standalone'; export const LEGACY_SHADOW_TERMINAL_CATEGORIES = Object.freeze([ 'matched', 'shadow_not_terminal', 'shadow_attempt_missing', 'shadow_attempt_ambiguous', 'legacy_evidence_missing', 'legacy_evidence_ambiguous', 'status_mismatch', 'field_mismatch', ] as const); export type LegacyShadowTerminalCategory = (typeof LEGACY_SHADOW_TERMINAL_CATEGORIES)[number]; export type LegacyShadowTerminalCategoryCounts = Record< LegacyShadowTerminalCategory, number >; export interface LegacyShadowTerminalDimensionCounts { compared: number; matched: number; mismatched: number; unavailable: number; } export interface LegacyShadowTerminalDimensions { status: LegacyShadowTerminalDimensionCounts; exitCode: LegacyShadowTerminalDimensionCounts; startedAt: LegacyShadowTerminalDimensionCounts; finishedAt: LegacyShadowTerminalDimensionCounts; logArtifact: LegacyShadowTerminalDimensionCounts; } export interface LegacyShadowTerminalOriginSummary extends LegacyShadowTerminalCategoryCounts { origin: ExecutionOrigin; scanned: number; } export type LegacyShadowTerminalAuditAssessment = | 'matched' | 'empty' | 'differences_found' | 'window_open' | 'incomplete'; export type LegacyShadowTerminalAuditStopReason = | 'complete' | 'page_limit' | 'cursor_stalled'; export interface LegacyShadowTerminalDifferenceReport { schema: 'qinglong/legacy-shadow-terminal-difference-report@v1'; profile: LegacyShadowTerminalAuditProfile; observedAtMs: number; window: { basis: 'shadow_run_created_at'; startInclusiveMs: number; endExclusiveMs: number; minimumSettlingAgeMs: number; closed: boolean; }; coverage: { direction: 'shadow_to_legacy'; cohort: 'legacy_owned_shadow_runs'; legacyWithoutShadow: 'not_measured'; }; budget: { pageSize: number; maxPages: number; maxCandidates: number }; pages: number; scanned: number; stopReason: LegacyShadowTerminalAuditStopReason; remaining: boolean; evidenceComplete: boolean; evidenceOverflowPages: number; assessment: LegacyShadowTerminalAuditAssessment; counts: LegacyShadowTerminalCategoryCounts; byOrigin: readonly LegacyShadowTerminalOriginSummary[]; dimensions: LegacyShadowTerminalDimensions; terminalAgreementPermille?: number; fullyComparablePermille?: number; } export interface LegacyShadowTerminalAuditorOptions { profile: LegacyShadowTerminalAuditProfile; projectId?: string; origins: readonly ExecutionOrigin[]; windowStartMs: number; windowEndMs: number; observedAtMs?: number; minimumSettlingAgeMs?: number; correlationToleranceMs?: number; clock?: { now(): number }; } interface Selection { status: 'matched' | 'missing' | 'ambiguous'; evidence?: LegacyTerminalEvidence; } interface Classification { category: LegacyShadowTerminalCategory; fullyComparable: boolean; } const BUDGETS: Readonly< Record< LegacyShadowTerminalAuditProfile, { pageSize: number; maxPages: number } > > = { edge: { pageSize: 8, maxPages: 1 }, standalone: { pageSize: 32, maxPages: 4 }, }; const MAX_CONFIGURED_ORIGINS = 7; const MAX_CORRELATION_TOLERANCE_MS = 60_000; const EXECUTION_ORIGIN_SET = new Set(EXECUTION_ORIGINS); function assertSafeInteger(name: string, value: number): void { if (!Number.isSafeInteger(value) || value < 0) { throw new RangeError(`${name} must be a non-negative safe integer`); } } function emptyCounts(): LegacyShadowTerminalCategoryCounts { return Object.fromEntries( LEGACY_SHADOW_TERMINAL_CATEGORIES.map((category) => [category, 0]), ) as LegacyShadowTerminalCategoryCounts; } function emptyDimension(): LegacyShadowTerminalDimensionCounts { return { compared: 0, matched: 0, mismatched: 0, unavailable: 0 }; } function emptyDimensions(): LegacyShadowTerminalDimensions { return { status: emptyDimension(), exitCode: emptyDimension(), startedAt: emptyDimension(), finishedAt: emptyDimension(), logArtifact: emptyDimension(), }; } function originSummary( origin: ExecutionOrigin, ): LegacyShadowTerminalOriginSummary { return { origin, scanned: 0, ...emptyCounts() }; } function compareDimension( target: LegacyShadowTerminalDimensionCounts, left: unknown, right: unknown, matches: (left: unknown, right: unknown) => boolean = (a, b) => a === b, ): boolean | undefined { if (left === undefined || right === undefined) { target.unavailable += 1; return undefined; } target.compared += 1; if (matches(left, right)) { target.matched += 1; return true; } target.mismatched += 1; return false; } function markUnavailable(dimensions: LegacyShadowTerminalDimensions): void { for (const dimension of Object.values(dimensions)) { dimension.unavailable += 1; } } function expectedOutcome( status: RunStatus | RunAttemptStatus, ): LegacyTerminalOutcome | undefined { if (status === 'succeeded') return 'succeeded'; if (status === 'failed') return 'failed'; if (status === 'cancelled') return 'stopped'; return undefined; } function isTerminalShadowStatus(status: RunStatus | RunAttemptStatus): boolean { return ['succeeded', 'failed', 'cancelled', 'timed_out', 'lost'].includes( status, ); } function selectEvidence( candidate: LegacyShadowTerminalCandidate, attempt: LegacyShadowTerminalAttemptEvidence, evidence: readonly LegacyTerminalEvidence[], toleranceMs: number, ): Selection { const direct = evidence.filter( (item) => item.attemptId === attempt.attemptId || item.runId === candidate.runId, ); if ( direct.some( (item) => (item.attemptId !== undefined && item.attemptId !== attempt.attemptId) || (item.runId !== undefined && item.runId !== candidate.runId), ) ) { return { status: 'ambiguous' }; } if (direct.length === 1) return { status: 'matched', evidence: direct[0] }; if (direct.length > 1) return { status: 'ambiguous' }; if (candidate.legacyCronId === undefined) return { status: 'missing' }; const sameCron = evidence.filter( (item) => item.legacyCronId === candidate.legacyCronId, ); const byLog = attempt.logArtifactId === undefined ? [] : sameCron.filter((item) => item.logArtifactId === attempt.logArtifactId); const byPid = attempt.pid === undefined ? [] : sameCron.filter((item) => item.pid === attempt.pid); if (attempt.logArtifactId !== undefined && attempt.pid !== undefined) { const byPidSet = new Set(byPid); const intersection = byLog.filter((item) => byPidSet.has(item)); if (intersection.length === 1) { return { status: 'matched', evidence: intersection[0] }; } if (intersection.length > 1 || byLog.length > 1 || byPid.length > 1) { return { status: 'ambiguous' }; } if (byLog.length === 1 && byPid.length === 0) { return { status: 'matched', evidence: byLog[0] }; } if (byPid.length === 1 && byLog.length === 0) { return { status: 'matched', evidence: byPid[0] }; } if (byLog.length === 1 && byPid.length === 1) { return { status: 'ambiguous' }; } } else if (byLog.length === 1) { return { status: 'matched', evidence: byLog[0] }; } else if (byLog.length > 1) { return { status: 'ambiguous' }; } else if (byPid.length === 1) { return { status: 'matched', evidence: byPid[0] }; } else if (byPid.length > 1) { return { status: 'ambiguous' }; } const shadowStartedAtMs = attempt.startedAtMs ?? candidate.startedAtMs; if (shadowStartedAtMs === undefined) return { status: 'missing' }; const byStart = sameCron.filter( (item) => Math.abs(item.startedAtMs - shadowStartedAtMs) <= toleranceMs, ); if (byStart.length === 1) { return { status: 'matched', evidence: byStart[0] }; } return { status: byStart.length > 1 ? 'ambiguous' : 'missing' }; } function classify( candidate: LegacyShadowTerminalCandidate, evidence: readonly LegacyTerminalEvidence[], evidenceTruncated: boolean, toleranceMs: number, dimensions: LegacyShadowTerminalDimensions, ): Classification { if ( !isTerminalShadowStatus(candidate.runStatus) || candidate.finishedAtMs === undefined ) { markUnavailable(dimensions); return { category: 'shadow_not_terminal', fullyComparable: false }; } if (candidate.attemptCount === 0 || candidate.attempt === undefined) { markUnavailable(dimensions); return { category: 'shadow_attempt_missing', fullyComparable: false }; } if (candidate.attemptCount !== 1) { markUnavailable(dimensions); return { category: 'shadow_attempt_ambiguous', fullyComparable: false }; } if ( !isTerminalShadowStatus(candidate.attempt.status) || candidate.attempt.finishedAtMs === undefined ) { markUnavailable(dimensions); return { category: 'shadow_not_terminal', fullyComparable: false }; } if (evidenceTruncated) { markUnavailable(dimensions); return { category: 'legacy_evidence_ambiguous', fullyComparable: false }; } const selection = selectEvidence( candidate, candidate.attempt, evidence, toleranceMs, ); if (selection.status === 'ambiguous') { markUnavailable(dimensions); return { category: 'legacy_evidence_ambiguous', fullyComparable: false }; } if (selection.status === 'missing' || selection.evidence === undefined) { markUnavailable(dimensions); return { category: 'legacy_evidence_missing', fullyComparable: false }; } const expectedRunOutcome = expectedOutcome(candidate.runStatus); const expectedAttemptOutcome = expectedOutcome(candidate.attempt.status); let statusMatches: boolean | undefined; if ( expectedRunOutcome === undefined || expectedAttemptOutcome === undefined ) { dimensions.status.unavailable += 1; } else if (expectedRunOutcome !== expectedAttemptOutcome) { dimensions.status.compared += 1; dimensions.status.mismatched += 1; statusMatches = false; } else { statusMatches = compareDimension( dimensions.status, expectedRunOutcome, selection.evidence.outcome, ); } if ( expectedRunOutcome === undefined || expectedAttemptOutcome === undefined || expectedRunOutcome !== expectedAttemptOutcome || statusMatches !== true ) { for (const dimension of [ dimensions.exitCode, dimensions.startedAt, dimensions.finishedAt, dimensions.logArtifact, ]) { dimension.unavailable += 1; } return { category: 'status_mismatch', fullyComparable: false }; } const exitMatches = compareDimension( dimensions.exitCode, candidate.attempt.exitCode, selection.evidence.exitCode, ); const startMatches = compareDimension( dimensions.startedAt, candidate.attempt.startedAtMs ?? candidate.startedAtMs, selection.evidence.startedAtMs, (left, right) => Math.abs(Number(left) - Number(right)) <= toleranceMs, ); const finishMatches = compareDimension( dimensions.finishedAt, candidate.attempt.finishedAtMs ?? candidate.finishedAtMs, selection.evidence.finishedAtMs, (left, right) => Math.abs(Number(left) - Number(right)) <= toleranceMs, ); const logMatches = compareDimension( dimensions.logArtifact, candidate.attempt.logArtifactId, selection.evidence.logArtifactId, ); const compared = [true, exitMatches, startMatches, finishMatches, logMatches]; const hasMismatch = compared.includes(false); return { category: hasMismatch ? 'field_mismatch' : 'matched', fullyComparable: compared.every((value) => value === true), }; } function sameCursor( left: LegacyShadowTerminalCursor | undefined, right: LegacyShadowTerminalCursor, ): boolean { return ( left !== undefined && left.createdAtMs === right.createdAtMs && left.runId === right.runId ); } function validatePage( page: LegacyShadowTerminalPage, cursor: LegacyShadowTerminalCursor | undefined, limit: number, allowedOrigins: ReadonlySet, windowStartMs: number, windowEndMs: number, ): void { if (page.candidates.length > limit) { throw new Error('Legacy Shadow terminal source exceeded candidate limit'); } if ( page.evidence.length > Math.min( MAX_LEGACY_SHADOW_TERMINAL_EVIDENCE_PER_PAGE, page.candidates.length * 8, ) ) { throw new Error('Legacy Shadow terminal source exceeded evidence limit'); } let previous = cursor; const runIds = new Set(); for (const candidate of page.candidates) { if ( candidate.runId.length === 0 || runIds.has(candidate.runId) || !allowedOrigins.has(candidate.origin) || !Number.isSafeInteger(candidate.createdAtMs) || candidate.createdAtMs < windowStartMs || candidate.createdAtMs >= windowEndMs || (previous !== undefined && (candidate.createdAtMs < previous.createdAtMs || (candidate.createdAtMs === previous.createdAtMs && candidate.runId <= previous.runId))) ) { throw new Error('Legacy Shadow terminal source returned an invalid page'); } runIds.add(candidate.runId); previous = { createdAtMs: candidate.createdAtMs, runId: candidate.runId }; } if (page.truncated) { if ( page.candidates.length !== limit || page.nextCursor === undefined || previous === undefined || page.nextCursor.createdAtMs !== previous.createdAtMs || page.nextCursor.runId !== previous.runId ) { throw new Error( 'Legacy Shadow terminal source returned an invalid cursor', ); } } else if (page.nextCursor !== undefined) { throw new Error('Completed Legacy Shadow terminal page returned a cursor'); } } function permille(numerator: number, denominator: number): number { return Math.floor((numerator * 1_000) / denominator); } /** * Explicit, timer-free audit for a closed Shadow cohort. The report deliberately * does not claim Legacy-to-Shadow capture coverage because RunningInstances have * no trustworthy execution-origin field in the 2.x schema. */ export class LegacyShadowTerminalDifferenceAuditor { constructor(private readonly source: LegacyShadowTerminalDifferenceSource) {} async run( options: LegacyShadowTerminalAuditorOptions, ): Promise { if (!['edge', 'standalone'].includes(options.profile)) { throw new RangeError('profile must be edge or standalone'); } const projectId = options.projectId ?? 'default'; if (projectId.length < 1 || projectId.length > 128) { throw new RangeError('projectId length is invalid'); } const origins = [...new Set(options.origins)]; if (origins.length < 1 || origins.length > MAX_CONFIGURED_ORIGINS) { throw new RangeError('origin count is outside its audit budget'); } if (origins.some((origin) => !EXECUTION_ORIGIN_SET.has(origin))) { throw new RangeError('terminal audit origin is invalid'); } assertSafeInteger('windowStartMs', options.windowStartMs); assertSafeInteger('windowEndMs', options.windowEndMs); if (options.windowStartMs >= options.windowEndMs) { throw new RangeError('terminal audit window must be non-empty'); } const observedAtMs = options.observedAtMs ?? options.clock?.now() ?? Date.now(); const minimumSettlingAgeMs = options.minimumSettlingAgeMs ?? 5 * 60_000; const correlationToleranceMs = options.correlationToleranceMs ?? 2_000; assertSafeInteger('observedAtMs', observedAtMs); assertSafeInteger('minimumSettlingAgeMs', minimumSettlingAgeMs); assertSafeInteger('correlationToleranceMs', correlationToleranceMs); if (correlationToleranceMs > MAX_CORRELATION_TOLERANCE_MS) { throw new RangeError('correlationToleranceMs exceeds its hard limit'); } const budget = BUDGETS[options.profile]; if (budget.pageSize > MAX_LEGACY_SHADOW_TERMINAL_PAGE_SIZE) { throw new RangeError('terminal audit Profile exceeds the source limit'); } const counts = emptyCounts(); const dimensions = emptyDimensions(); const originSummaries = new Map( origins.map((origin) => [origin, originSummary(origin)]), ); const allowedOrigins = new Set(origins); let cursor: LegacyShadowTerminalCursor | undefined; let pages = 0; let scanned = 0; let fullyComparable = 0; let evidenceOverflowPages = 0; let remaining = false; let stopReason: LegacyShadowTerminalAuditStopReason = 'complete'; while (pages < budget.maxPages) { const page = await this.source.listCandidates({ projectId, origins, windowStartMs: options.windowStartMs, windowEndMs: options.windowEndMs, observedAtMs, correlationToleranceMs, ...(cursor === undefined ? {} : { cursor }), limit: budget.pageSize, }); pages += 1; validatePage( page, cursor, budget.pageSize, allowedOrigins, options.windowStartMs, options.windowEndMs, ); if (page.evidenceTruncated) evidenceOverflowPages += 1; for (const candidate of page.candidates) { const result = classify( candidate, page.evidence, page.evidenceTruncated, correlationToleranceMs, dimensions, ); counts[result.category] += 1; const perOrigin = originSummaries.get(candidate.origin)!; perOrigin.scanned += 1; perOrigin[result.category] += 1; scanned += 1; if (result.fullyComparable) fullyComparable += 1; } if (!page.truncated) break; if ( page.nextCursor === undefined || sameCursor(cursor, page.nextCursor) ) { remaining = true; stopReason = 'cursor_stalled'; break; } cursor = page.nextCursor; if (pages === budget.maxPages) { remaining = true; stopReason = 'page_limit'; } } const windowClosed = options.windowEndMs <= observedAtMs - minimumSettlingAgeMs; const evidenceComplete = evidenceOverflowPages === 0; const assessment: LegacyShadowTerminalAuditAssessment = !windowClosed ? 'window_open' : remaining || !evidenceComplete ? 'incomplete' : scanned === 0 ? 'empty' : counts.matched === scanned ? 'matched' : 'differences_found'; const ratiosAvailable = windowClosed && !remaining && evidenceComplete && scanned > 0; return { schema: 'qinglong/legacy-shadow-terminal-difference-report@v1', profile: options.profile, observedAtMs, window: { basis: 'shadow_run_created_at', startInclusiveMs: options.windowStartMs, endExclusiveMs: options.windowEndMs, minimumSettlingAgeMs, closed: windowClosed, }, coverage: { direction: 'shadow_to_legacy', cohort: 'legacy_owned_shadow_runs', legacyWithoutShadow: 'not_measured', }, budget: { pageSize: budget.pageSize, maxPages: budget.maxPages, maxCandidates: budget.pageSize * budget.maxPages, }, pages, scanned, stopReason, remaining, evidenceComplete, evidenceOverflowPages, assessment, counts, byOrigin: [...originSummaries.values()], dimensions, ...(ratiosAvailable ? { terminalAgreementPermille: permille(counts.matched, scanned), fullyComparablePermille: permille(fullyComparable, scanned), } : {}), }; } }