mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): audit legacy shadow terminal windows
This commit is contained in:
whyour
@@ -0,0 +1,431 @@
|
||||
import { QueryTypes, type Sequelize } from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { RUNNING_INSTANCE_TABLE } from '../../../migrations/0003-running-instance-run-reference';
|
||||
import {
|
||||
EXECUTION_ORIGINS,
|
||||
RUN_ATTEMPT_STATUSES,
|
||||
RUN_STATUSES,
|
||||
type ExecutionOrigin,
|
||||
type RunAttemptStatus,
|
||||
type RunStatus,
|
||||
} from '../../domain/run';
|
||||
import {
|
||||
MAX_LEGACY_SHADOW_TERMINAL_EVIDENCE_PER_PAGE,
|
||||
MAX_LEGACY_SHADOW_TERMINAL_PAGE_SIZE,
|
||||
type LegacyShadowTerminalCandidate,
|
||||
type LegacyShadowTerminalCursor,
|
||||
type LegacyShadowTerminalDifferenceSource,
|
||||
type LegacyShadowTerminalPage,
|
||||
type LegacyTerminalEvidence,
|
||||
type LegacyTerminalOutcome,
|
||||
} from '../../ports/legacyShadowTerminalDifference';
|
||||
|
||||
interface CandidateRow {
|
||||
runId: string;
|
||||
legacyCronId: number | null;
|
||||
origin: string;
|
||||
runStatus: string;
|
||||
createdAtMs: number | string;
|
||||
startedAtMs: number | string | null;
|
||||
finishedAtMs: number | string | null;
|
||||
attemptCount: number | string;
|
||||
attemptId: string | null;
|
||||
attemptStatus: string | null;
|
||||
attemptPid: number | null;
|
||||
attemptLogArtifactId: string | null;
|
||||
attemptCreatedAtMs: number | string | null;
|
||||
attemptStartedAtMs: number | string | null;
|
||||
attemptFinishedAtMs: number | string | null;
|
||||
attemptExitCode: number | null;
|
||||
}
|
||||
|
||||
interface EvidenceRow {
|
||||
instanceId: number | string;
|
||||
legacyCronId: number | string;
|
||||
runId: string | null;
|
||||
attemptId: string | null;
|
||||
pid: number | null;
|
||||
logPath: string | null;
|
||||
startedAt: number | string;
|
||||
finishedAt: number | string;
|
||||
status: number | string;
|
||||
exitCode: number | null;
|
||||
}
|
||||
|
||||
export type LegacyTerminalLogArtifactIdFactory = (logPath: string) => string;
|
||||
|
||||
const ORIGINS = new Set<string>(EXECUTION_ORIGINS);
|
||||
const RUN_STATUS_SET = new Set<string>(RUN_STATUSES);
|
||||
const ATTEMPT_STATUS_SET = new Set<string>(RUN_ATTEMPT_STATUSES);
|
||||
|
||||
function safeInteger(value: number | string, name: string): number {
|
||||
const parsed = typeof value === 'number' ? value : Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 0) {
|
||||
throw new TypeError(`${name} is invalid`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function optionalPositiveInteger(
|
||||
value: number | null,
|
||||
name: string,
|
||||
): number | undefined {
|
||||
if (value === null) return undefined;
|
||||
const parsed = safeInteger(value, name);
|
||||
if (parsed < 1) throw new TypeError(`${name} is invalid`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function secondsToMilliseconds(value: number | string, name: string): number {
|
||||
const milliseconds = safeInteger(value, name) * 1_000;
|
||||
if (!Number.isSafeInteger(milliseconds)) {
|
||||
throw new TypeError(`${name} is invalid`);
|
||||
}
|
||||
return milliseconds;
|
||||
}
|
||||
|
||||
function assertLimit(limit: number): void {
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_LEGACY_SHADOW_TERMINAL_PAGE_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
`limit must be between 1 and ${MAX_LEGACY_SHADOW_TERMINAL_PAGE_SIZE}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function runStatus(value: string): RunStatus {
|
||||
if (!RUN_STATUS_SET.has(value)) {
|
||||
throw new TypeError('Run status is invalid');
|
||||
}
|
||||
return value as RunStatus;
|
||||
}
|
||||
|
||||
function attemptStatus(value: string): RunAttemptStatus {
|
||||
if (!ATTEMPT_STATUS_SET.has(value)) {
|
||||
throw new TypeError('Run Attempt status is invalid');
|
||||
}
|
||||
return value as RunAttemptStatus;
|
||||
}
|
||||
|
||||
function origin(value: string): ExecutionOrigin {
|
||||
if (!ORIGINS.has(value)) {
|
||||
throw new TypeError('Run execution origin is invalid');
|
||||
}
|
||||
return value as ExecutionOrigin;
|
||||
}
|
||||
|
||||
function terminalOutcome(value: number): LegacyTerminalOutcome {
|
||||
if (value === 1) return 'succeeded';
|
||||
if (value === 2) return 'stopped';
|
||||
if (value === 3) return 'failed';
|
||||
throw new TypeError('RunningInstance terminal status is invalid');
|
||||
}
|
||||
|
||||
function candidateFromRow(row: CandidateRow): LegacyShadowTerminalCandidate {
|
||||
const attemptCount = safeInteger(row.attemptCount, 'attemptCount');
|
||||
if (
|
||||
(attemptCount === 0 && row.attemptId !== null) ||
|
||||
(attemptCount > 0 &&
|
||||
(row.attemptId === null ||
|
||||
row.attemptStatus === null ||
|
||||
row.attemptCreatedAtMs === null))
|
||||
) {
|
||||
throw new TypeError('Run Attempt projection is invalid');
|
||||
}
|
||||
return {
|
||||
runId: row.runId,
|
||||
...(row.legacyCronId === null
|
||||
? {}
|
||||
: {
|
||||
legacyCronId: optionalPositiveInteger(
|
||||
row.legacyCronId,
|
||||
'legacyCronId',
|
||||
),
|
||||
}),
|
||||
origin: origin(row.origin),
|
||||
runStatus: runStatus(row.runStatus),
|
||||
createdAtMs: safeInteger(row.createdAtMs, 'createdAtMs'),
|
||||
...(row.startedAtMs === null
|
||||
? {}
|
||||
: { startedAtMs: safeInteger(row.startedAtMs, 'startedAtMs') }),
|
||||
...(row.finishedAtMs === null
|
||||
? {}
|
||||
: { finishedAtMs: safeInteger(row.finishedAtMs, 'finishedAtMs') }),
|
||||
attemptCount,
|
||||
...(row.attemptId === null
|
||||
? {}
|
||||
: {
|
||||
attempt: {
|
||||
attemptId: row.attemptId,
|
||||
status: attemptStatus(row.attemptStatus!),
|
||||
...(row.attemptPid === null
|
||||
? {}
|
||||
: { pid: optionalPositiveInteger(row.attemptPid, 'attemptPid') }),
|
||||
...(row.attemptLogArtifactId === null
|
||||
? {}
|
||||
: { logArtifactId: row.attemptLogArtifactId }),
|
||||
createdAtMs: safeInteger(
|
||||
row.attemptCreatedAtMs!,
|
||||
'attemptCreatedAtMs',
|
||||
),
|
||||
...(row.attemptStartedAtMs === null
|
||||
? {}
|
||||
: {
|
||||
startedAtMs: safeInteger(
|
||||
row.attemptStartedAtMs,
|
||||
'attemptStartedAtMs',
|
||||
),
|
||||
}),
|
||||
...(row.attemptFinishedAtMs === null
|
||||
? {}
|
||||
: {
|
||||
finishedAtMs: safeInteger(
|
||||
row.attemptFinishedAtMs,
|
||||
'attemptFinishedAtMs',
|
||||
),
|
||||
}),
|
||||
...(row.attemptExitCode === null
|
||||
? {}
|
||||
: {
|
||||
exitCode: safeInteger(row.attemptExitCode, 'attemptExitCode'),
|
||||
}),
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** SQLite/Sequelize adapter for the explicit, read-only local audit. */
|
||||
export class LegacySequelizeShadowTerminalDifferenceSource
|
||||
implements LegacyShadowTerminalDifferenceSource
|
||||
{
|
||||
constructor(
|
||||
private readonly database: Sequelize,
|
||||
private readonly createLogArtifactId: LegacyTerminalLogArtifactIdFactory,
|
||||
) {}
|
||||
|
||||
async listCandidates({
|
||||
projectId,
|
||||
origins,
|
||||
windowStartMs,
|
||||
windowEndMs,
|
||||
observedAtMs,
|
||||
correlationToleranceMs,
|
||||
cursor,
|
||||
limit,
|
||||
}: {
|
||||
projectId: string;
|
||||
origins: readonly ExecutionOrigin[];
|
||||
windowStartMs: number;
|
||||
windowEndMs: number;
|
||||
observedAtMs: number;
|
||||
correlationToleranceMs: number;
|
||||
cursor?: LegacyShadowTerminalCursor;
|
||||
limit: number;
|
||||
}): Promise<LegacyShadowTerminalPage> {
|
||||
assertLimit(limit);
|
||||
if (origins.length === 0) {
|
||||
return {
|
||||
candidates: [],
|
||||
evidence: [],
|
||||
evidenceTruncated: false,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
const rows = await this.database.query<CandidateRow>(
|
||||
`SELECT
|
||||
r.id AS "runId",
|
||||
r.legacy_cron_id AS "legacyCronId",
|
||||
r.execution_origin AS origin,
|
||||
r.status AS "runStatus",
|
||||
r.created_at_ms AS "createdAtMs",
|
||||
r.started_at_ms AS "startedAtMs",
|
||||
r.finished_at_ms AS "finishedAtMs",
|
||||
(SELECT COUNT(*)
|
||||
FROM ${RUN_ATTEMPT_TABLE} counted
|
||||
WHERE counted.run_id = r.id) AS "attemptCount",
|
||||
attempt.id AS "attemptId",
|
||||
attempt.status AS "attemptStatus",
|
||||
attempt.pid AS "attemptPid",
|
||||
attempt.log_artifact_id AS "attemptLogArtifactId",
|
||||
attempt.created_at_ms AS "attemptCreatedAtMs",
|
||||
attempt.started_at_ms AS "attemptStartedAtMs",
|
||||
attempt.finished_at_ms AS "attemptFinishedAtMs",
|
||||
attempt.exit_code AS "attemptExitCode"
|
||||
FROM ${RUN_TABLE} r
|
||||
LEFT JOIN ${RUN_ATTEMPT_TABLE} attempt
|
||||
ON attempt.id = (
|
||||
SELECT latest.id
|
||||
FROM ${RUN_ATTEMPT_TABLE} latest
|
||||
WHERE latest.run_id = r.id
|
||||
ORDER BY latest.attempt DESC, latest.id DESC
|
||||
LIMIT 1
|
||||
)
|
||||
WHERE r.project_id = :projectId
|
||||
AND r.execution_owner = 'legacy'
|
||||
AND r.execution_origin IN (:origins)
|
||||
AND r.created_at_ms >= :windowStartMs
|
||||
AND r.created_at_ms < :windowEndMs
|
||||
${
|
||||
cursor === undefined
|
||||
? ''
|
||||
: `AND (
|
||||
r.created_at_ms > :cursorCreatedAtMs OR
|
||||
(r.created_at_ms = :cursorCreatedAtMs AND r.id > :cursorRunId)
|
||||
)`
|
||||
}
|
||||
ORDER BY r.created_at_ms ASC, r.id ASC
|
||||
LIMIT :fetchLimit`,
|
||||
{
|
||||
replacements: {
|
||||
projectId,
|
||||
origins: [...new Set(origins)],
|
||||
windowStartMs,
|
||||
windowEndMs,
|
||||
...(cursor === undefined
|
||||
? {}
|
||||
: {
|
||||
cursorCreatedAtMs: cursor.createdAtMs,
|
||||
cursorRunId: cursor.runId,
|
||||
}),
|
||||
fetchLimit: limit + 1,
|
||||
},
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
);
|
||||
const truncated = rows.length > limit;
|
||||
const candidates = rows.slice(0, limit).map(candidateFromRow);
|
||||
if (candidates.length === 0) {
|
||||
return {
|
||||
candidates,
|
||||
evidence: [],
|
||||
evidenceTruncated: false,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
const evidence = await this.listEvidence({
|
||||
candidates,
|
||||
windowStartMs,
|
||||
observedAtMs,
|
||||
correlationToleranceMs,
|
||||
});
|
||||
const last = candidates[candidates.length - 1];
|
||||
return {
|
||||
candidates,
|
||||
...evidence,
|
||||
truncated,
|
||||
...(truncated
|
||||
? {
|
||||
nextCursor: {
|
||||
createdAtMs: last.createdAtMs,
|
||||
runId: last.runId,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
private async listEvidence({
|
||||
candidates,
|
||||
windowStartMs,
|
||||
observedAtMs,
|
||||
correlationToleranceMs,
|
||||
}: {
|
||||
candidates: readonly LegacyShadowTerminalCandidate[];
|
||||
windowStartMs: number;
|
||||
observedAtMs: number;
|
||||
correlationToleranceMs: number;
|
||||
}): Promise<{
|
||||
evidence: readonly LegacyTerminalEvidence[];
|
||||
evidenceTruncated: boolean;
|
||||
}> {
|
||||
const runIds = candidates.map((candidate) => candidate.runId);
|
||||
const attemptIds = candidates.flatMap((candidate) =>
|
||||
candidate.attempt ? [candidate.attempt.attemptId] : [],
|
||||
);
|
||||
const legacyCronIds = [
|
||||
...new Set(
|
||||
candidates.flatMap((candidate) =>
|
||||
candidate.legacyCronId === undefined ? [] : [candidate.legacyCronId],
|
||||
),
|
||||
),
|
||||
];
|
||||
const clauses = ['run_id IN (:runIds)'];
|
||||
if (attemptIds.length > 0) clauses.push('attempt_id IN (:attemptIds)');
|
||||
if (legacyCronIds.length > 0) {
|
||||
clauses.push(`(
|
||||
cron_id IN (:legacyCronIds)
|
||||
AND started_at >= :startedAfterSeconds
|
||||
AND started_at <= :startedBeforeSeconds
|
||||
)`);
|
||||
}
|
||||
const evidenceLimit = Math.min(
|
||||
MAX_LEGACY_SHADOW_TERMINAL_EVIDENCE_PER_PAGE,
|
||||
candidates.length * 8,
|
||||
);
|
||||
const rows = await this.database.query<EvidenceRow>(
|
||||
`SELECT
|
||||
id AS "instanceId",
|
||||
cron_id AS "legacyCronId",
|
||||
run_id AS "runId",
|
||||
attempt_id AS "attemptId",
|
||||
pid,
|
||||
log_path AS "logPath",
|
||||
started_at AS "startedAt",
|
||||
finished_at AS "finishedAt",
|
||||
status,
|
||||
exit_code AS "exitCode"
|
||||
FROM ${RUNNING_INSTANCE_TABLE}
|
||||
WHERE status IN (1, 2, 3)
|
||||
AND finished_at IS NOT NULL
|
||||
AND (${clauses.join(' OR ')})
|
||||
ORDER BY started_at ASC, id ASC
|
||||
LIMIT :fetchLimit`,
|
||||
{
|
||||
replacements: {
|
||||
runIds,
|
||||
...(attemptIds.length === 0 ? {} : { attemptIds }),
|
||||
...(legacyCronIds.length === 0
|
||||
? {}
|
||||
: {
|
||||
legacyCronIds,
|
||||
startedAfterSeconds: Math.floor(
|
||||
Math.max(0, windowStartMs - correlationToleranceMs) / 1_000,
|
||||
),
|
||||
startedBeforeSeconds: Math.ceil(observedAtMs / 1_000),
|
||||
}),
|
||||
fetchLimit: evidenceLimit + 1,
|
||||
},
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
);
|
||||
const evidenceTruncated = rows.length > evidenceLimit;
|
||||
return {
|
||||
evidence: rows.slice(0, evidenceLimit).map((row) => ({
|
||||
instanceId: safeInteger(row.instanceId, 'instanceId'),
|
||||
legacyCronId: safeInteger(row.legacyCronId, 'legacyCronId'),
|
||||
...(row.runId === null ? {} : { runId: row.runId }),
|
||||
...(row.attemptId === null ? {} : { attemptId: row.attemptId }),
|
||||
...(row.pid === null
|
||||
? {}
|
||||
: { pid: optionalPositiveInteger(row.pid, 'pid') }),
|
||||
...(row.logPath === null
|
||||
? {}
|
||||
: { logArtifactId: this.createLogArtifactId(row.logPath) }),
|
||||
startedAtMs: secondsToMilliseconds(row.startedAt, 'startedAt'),
|
||||
finishedAtMs: secondsToMilliseconds(row.finishedAt, 'finishedAt'),
|
||||
outcome: terminalOutcome(safeInteger(row.status, 'status')),
|
||||
...(row.exitCode === null
|
||||
? {}
|
||||
: { exitCode: safeInteger(row.exitCode, 'exitCode') }),
|
||||
})),
|
||||
evidenceTruncated,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,641 @@
|
||||
import type {
|
||||
ExecutionOrigin,
|
||||
RunAttemptStatus,
|
||||
RunStatus,
|
||||
} from '../domain/run';
|
||||
import { EXECUTION_ORIGINS } from '../domain/run';
|
||||
import {
|
||||
MAX_LEGACY_SHADOW_TERMINAL_EVIDENCE_PER_PAGE,
|
||||
MAX_LEGACY_SHADOW_TERMINAL_PAGE_SIZE,
|
||||
type LegacyShadowTerminalAttemptEvidence,
|
||||
type LegacyShadowTerminalCandidate,
|
||||
type LegacyShadowTerminalCursor,
|
||||
type LegacyShadowTerminalDifferenceSource,
|
||||
type LegacyShadowTerminalPage,
|
||||
type LegacyTerminalEvidence,
|
||||
type LegacyTerminalOutcome,
|
||||
} from '../ports/legacyShadowTerminalDifference';
|
||||
|
||||
export type LegacyShadowTerminalAuditProfile = 'edge' | 'standalone';
|
||||
|
||||
export const LEGACY_SHADOW_TERMINAL_CATEGORIES = Object.freeze([
|
||||
'matched',
|
||||
'shadow_not_terminal',
|
||||
'shadow_attempt_missing',
|
||||
'shadow_attempt_ambiguous',
|
||||
'legacy_evidence_missing',
|
||||
'legacy_evidence_ambiguous',
|
||||
'status_mismatch',
|
||||
'field_mismatch',
|
||||
] as const);
|
||||
|
||||
export type LegacyShadowTerminalCategory =
|
||||
(typeof LEGACY_SHADOW_TERMINAL_CATEGORIES)[number];
|
||||
|
||||
export type LegacyShadowTerminalCategoryCounts = Record<
|
||||
LegacyShadowTerminalCategory,
|
||||
number
|
||||
>;
|
||||
|
||||
export interface LegacyShadowTerminalDimensionCounts {
|
||||
compared: number;
|
||||
matched: number;
|
||||
mismatched: number;
|
||||
unavailable: number;
|
||||
}
|
||||
|
||||
export interface LegacyShadowTerminalDimensions {
|
||||
status: LegacyShadowTerminalDimensionCounts;
|
||||
exitCode: LegacyShadowTerminalDimensionCounts;
|
||||
startedAt: LegacyShadowTerminalDimensionCounts;
|
||||
finishedAt: LegacyShadowTerminalDimensionCounts;
|
||||
logArtifact: LegacyShadowTerminalDimensionCounts;
|
||||
}
|
||||
|
||||
export interface LegacyShadowTerminalOriginSummary
|
||||
extends LegacyShadowTerminalCategoryCounts {
|
||||
origin: ExecutionOrigin;
|
||||
scanned: number;
|
||||
}
|
||||
|
||||
export type LegacyShadowTerminalAuditAssessment =
|
||||
| 'matched'
|
||||
| 'empty'
|
||||
| 'differences_found'
|
||||
| 'window_open'
|
||||
| 'incomplete';
|
||||
|
||||
export type LegacyShadowTerminalAuditStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface LegacyShadowTerminalDifferenceReport {
|
||||
schema: 'qinglong/legacy-shadow-terminal-difference-report@v1';
|
||||
profile: LegacyShadowTerminalAuditProfile;
|
||||
observedAtMs: number;
|
||||
window: {
|
||||
basis: 'shadow_run_created_at';
|
||||
startInclusiveMs: number;
|
||||
endExclusiveMs: number;
|
||||
minimumSettlingAgeMs: number;
|
||||
closed: boolean;
|
||||
};
|
||||
coverage: {
|
||||
direction: 'shadow_to_legacy';
|
||||
cohort: 'legacy_owned_shadow_runs';
|
||||
legacyWithoutShadow: 'not_measured';
|
||||
};
|
||||
budget: { pageSize: number; maxPages: number; maxCandidates: number };
|
||||
pages: number;
|
||||
scanned: number;
|
||||
stopReason: LegacyShadowTerminalAuditStopReason;
|
||||
remaining: boolean;
|
||||
evidenceComplete: boolean;
|
||||
evidenceOverflowPages: number;
|
||||
assessment: LegacyShadowTerminalAuditAssessment;
|
||||
counts: LegacyShadowTerminalCategoryCounts;
|
||||
byOrigin: readonly LegacyShadowTerminalOriginSummary[];
|
||||
dimensions: LegacyShadowTerminalDimensions;
|
||||
terminalAgreementPermille?: number;
|
||||
fullyComparablePermille?: number;
|
||||
}
|
||||
|
||||
export interface LegacyShadowTerminalAuditorOptions {
|
||||
profile: LegacyShadowTerminalAuditProfile;
|
||||
projectId?: string;
|
||||
origins: readonly ExecutionOrigin[];
|
||||
windowStartMs: number;
|
||||
windowEndMs: number;
|
||||
observedAtMs?: number;
|
||||
minimumSettlingAgeMs?: number;
|
||||
correlationToleranceMs?: number;
|
||||
clock?: { now(): number };
|
||||
}
|
||||
|
||||
interface Selection {
|
||||
status: 'matched' | 'missing' | 'ambiguous';
|
||||
evidence?: LegacyTerminalEvidence;
|
||||
}
|
||||
|
||||
interface Classification {
|
||||
category: LegacyShadowTerminalCategory;
|
||||
fullyComparable: boolean;
|
||||
}
|
||||
|
||||
const BUDGETS: Readonly<
|
||||
Record<
|
||||
LegacyShadowTerminalAuditProfile,
|
||||
{ pageSize: number; maxPages: number }
|
||||
>
|
||||
> = {
|
||||
edge: { pageSize: 8, maxPages: 1 },
|
||||
standalone: { pageSize: 32, maxPages: 4 },
|
||||
};
|
||||
|
||||
const MAX_CONFIGURED_ORIGINS = 7;
|
||||
const MAX_CORRELATION_TOLERANCE_MS = 60_000;
|
||||
const EXECUTION_ORIGIN_SET = new Set<ExecutionOrigin>(EXECUTION_ORIGINS);
|
||||
|
||||
function assertSafeInteger(name: string, value: number): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new RangeError(`${name} must be a non-negative safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function emptyCounts(): LegacyShadowTerminalCategoryCounts {
|
||||
return Object.fromEntries(
|
||||
LEGACY_SHADOW_TERMINAL_CATEGORIES.map((category) => [category, 0]),
|
||||
) as LegacyShadowTerminalCategoryCounts;
|
||||
}
|
||||
|
||||
function emptyDimension(): LegacyShadowTerminalDimensionCounts {
|
||||
return { compared: 0, matched: 0, mismatched: 0, unavailable: 0 };
|
||||
}
|
||||
|
||||
function emptyDimensions(): LegacyShadowTerminalDimensions {
|
||||
return {
|
||||
status: emptyDimension(),
|
||||
exitCode: emptyDimension(),
|
||||
startedAt: emptyDimension(),
|
||||
finishedAt: emptyDimension(),
|
||||
logArtifact: emptyDimension(),
|
||||
};
|
||||
}
|
||||
|
||||
function originSummary(
|
||||
origin: ExecutionOrigin,
|
||||
): LegacyShadowTerminalOriginSummary {
|
||||
return { origin, scanned: 0, ...emptyCounts() };
|
||||
}
|
||||
|
||||
function compareDimension(
|
||||
target: LegacyShadowTerminalDimensionCounts,
|
||||
left: unknown,
|
||||
right: unknown,
|
||||
matches: (left: unknown, right: unknown) => boolean = (a, b) => a === b,
|
||||
): boolean | undefined {
|
||||
if (left === undefined || right === undefined) {
|
||||
target.unavailable += 1;
|
||||
return undefined;
|
||||
}
|
||||
target.compared += 1;
|
||||
if (matches(left, right)) {
|
||||
target.matched += 1;
|
||||
return true;
|
||||
}
|
||||
target.mismatched += 1;
|
||||
return false;
|
||||
}
|
||||
|
||||
function markUnavailable(dimensions: LegacyShadowTerminalDimensions): void {
|
||||
for (const dimension of Object.values(dimensions)) {
|
||||
dimension.unavailable += 1;
|
||||
}
|
||||
}
|
||||
|
||||
function expectedOutcome(
|
||||
status: RunStatus | RunAttemptStatus,
|
||||
): LegacyTerminalOutcome | undefined {
|
||||
if (status === 'succeeded') return 'succeeded';
|
||||
if (status === 'failed') return 'failed';
|
||||
if (status === 'cancelled') return 'stopped';
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isTerminalShadowStatus(status: RunStatus | RunAttemptStatus): boolean {
|
||||
return ['succeeded', 'failed', 'cancelled', 'timed_out', 'lost'].includes(
|
||||
status,
|
||||
);
|
||||
}
|
||||
|
||||
function selectEvidence(
|
||||
candidate: LegacyShadowTerminalCandidate,
|
||||
attempt: LegacyShadowTerminalAttemptEvidence,
|
||||
evidence: readonly LegacyTerminalEvidence[],
|
||||
toleranceMs: number,
|
||||
): Selection {
|
||||
const direct = evidence.filter(
|
||||
(item) =>
|
||||
item.attemptId === attempt.attemptId || item.runId === candidate.runId,
|
||||
);
|
||||
if (
|
||||
direct.some(
|
||||
(item) =>
|
||||
(item.attemptId !== undefined &&
|
||||
item.attemptId !== attempt.attemptId) ||
|
||||
(item.runId !== undefined && item.runId !== candidate.runId),
|
||||
)
|
||||
) {
|
||||
return { status: 'ambiguous' };
|
||||
}
|
||||
if (direct.length === 1) return { status: 'matched', evidence: direct[0] };
|
||||
if (direct.length > 1) return { status: 'ambiguous' };
|
||||
if (candidate.legacyCronId === undefined) return { status: 'missing' };
|
||||
|
||||
const sameCron = evidence.filter(
|
||||
(item) => item.legacyCronId === candidate.legacyCronId,
|
||||
);
|
||||
const byLog =
|
||||
attempt.logArtifactId === undefined
|
||||
? []
|
||||
: sameCron.filter((item) => item.logArtifactId === attempt.logArtifactId);
|
||||
const byPid =
|
||||
attempt.pid === undefined
|
||||
? []
|
||||
: sameCron.filter((item) => item.pid === attempt.pid);
|
||||
|
||||
if (attempt.logArtifactId !== undefined && attempt.pid !== undefined) {
|
||||
const byPidSet = new Set(byPid);
|
||||
const intersection = byLog.filter((item) => byPidSet.has(item));
|
||||
if (intersection.length === 1) {
|
||||
return { status: 'matched', evidence: intersection[0] };
|
||||
}
|
||||
if (intersection.length > 1 || byLog.length > 1 || byPid.length > 1) {
|
||||
return { status: 'ambiguous' };
|
||||
}
|
||||
if (byLog.length === 1 && byPid.length === 0) {
|
||||
return { status: 'matched', evidence: byLog[0] };
|
||||
}
|
||||
if (byPid.length === 1 && byLog.length === 0) {
|
||||
return { status: 'matched', evidence: byPid[0] };
|
||||
}
|
||||
if (byLog.length === 1 && byPid.length === 1) {
|
||||
return { status: 'ambiguous' };
|
||||
}
|
||||
} else if (byLog.length === 1) {
|
||||
return { status: 'matched', evidence: byLog[0] };
|
||||
} else if (byLog.length > 1) {
|
||||
return { status: 'ambiguous' };
|
||||
} else if (byPid.length === 1) {
|
||||
return { status: 'matched', evidence: byPid[0] };
|
||||
} else if (byPid.length > 1) {
|
||||
return { status: 'ambiguous' };
|
||||
}
|
||||
|
||||
const shadowStartedAtMs = attempt.startedAtMs ?? candidate.startedAtMs;
|
||||
if (shadowStartedAtMs === undefined) return { status: 'missing' };
|
||||
const byStart = sameCron.filter(
|
||||
(item) => Math.abs(item.startedAtMs - shadowStartedAtMs) <= toleranceMs,
|
||||
);
|
||||
if (byStart.length === 1) {
|
||||
return { status: 'matched', evidence: byStart[0] };
|
||||
}
|
||||
return { status: byStart.length > 1 ? 'ambiguous' : 'missing' };
|
||||
}
|
||||
|
||||
function classify(
|
||||
candidate: LegacyShadowTerminalCandidate,
|
||||
evidence: readonly LegacyTerminalEvidence[],
|
||||
evidenceTruncated: boolean,
|
||||
toleranceMs: number,
|
||||
dimensions: LegacyShadowTerminalDimensions,
|
||||
): Classification {
|
||||
if (
|
||||
!isTerminalShadowStatus(candidate.runStatus) ||
|
||||
candidate.finishedAtMs === undefined
|
||||
) {
|
||||
markUnavailable(dimensions);
|
||||
return { category: 'shadow_not_terminal', fullyComparable: false };
|
||||
}
|
||||
if (candidate.attemptCount === 0 || candidate.attempt === undefined) {
|
||||
markUnavailable(dimensions);
|
||||
return { category: 'shadow_attempt_missing', fullyComparable: false };
|
||||
}
|
||||
if (candidate.attemptCount !== 1) {
|
||||
markUnavailable(dimensions);
|
||||
return { category: 'shadow_attempt_ambiguous', fullyComparable: false };
|
||||
}
|
||||
if (
|
||||
!isTerminalShadowStatus(candidate.attempt.status) ||
|
||||
candidate.attempt.finishedAtMs === undefined
|
||||
) {
|
||||
markUnavailable(dimensions);
|
||||
return { category: 'shadow_not_terminal', fullyComparable: false };
|
||||
}
|
||||
if (evidenceTruncated) {
|
||||
markUnavailable(dimensions);
|
||||
return { category: 'legacy_evidence_ambiguous', fullyComparable: false };
|
||||
}
|
||||
|
||||
const selection = selectEvidence(
|
||||
candidate,
|
||||
candidate.attempt,
|
||||
evidence,
|
||||
toleranceMs,
|
||||
);
|
||||
if (selection.status === 'ambiguous') {
|
||||
markUnavailable(dimensions);
|
||||
return { category: 'legacy_evidence_ambiguous', fullyComparable: false };
|
||||
}
|
||||
if (selection.status === 'missing' || selection.evidence === undefined) {
|
||||
markUnavailable(dimensions);
|
||||
return { category: 'legacy_evidence_missing', fullyComparable: false };
|
||||
}
|
||||
|
||||
const expectedRunOutcome = expectedOutcome(candidate.runStatus);
|
||||
const expectedAttemptOutcome = expectedOutcome(candidate.attempt.status);
|
||||
let statusMatches: boolean | undefined;
|
||||
if (
|
||||
expectedRunOutcome === undefined ||
|
||||
expectedAttemptOutcome === undefined
|
||||
) {
|
||||
dimensions.status.unavailable += 1;
|
||||
} else if (expectedRunOutcome !== expectedAttemptOutcome) {
|
||||
dimensions.status.compared += 1;
|
||||
dimensions.status.mismatched += 1;
|
||||
statusMatches = false;
|
||||
} else {
|
||||
statusMatches = compareDimension(
|
||||
dimensions.status,
|
||||
expectedRunOutcome,
|
||||
selection.evidence.outcome,
|
||||
);
|
||||
}
|
||||
if (
|
||||
expectedRunOutcome === undefined ||
|
||||
expectedAttemptOutcome === undefined ||
|
||||
expectedRunOutcome !== expectedAttemptOutcome ||
|
||||
statusMatches !== true
|
||||
) {
|
||||
for (const dimension of [
|
||||
dimensions.exitCode,
|
||||
dimensions.startedAt,
|
||||
dimensions.finishedAt,
|
||||
dimensions.logArtifact,
|
||||
]) {
|
||||
dimension.unavailable += 1;
|
||||
}
|
||||
return { category: 'status_mismatch', fullyComparable: false };
|
||||
}
|
||||
|
||||
const exitMatches = compareDimension(
|
||||
dimensions.exitCode,
|
||||
candidate.attempt.exitCode,
|
||||
selection.evidence.exitCode,
|
||||
);
|
||||
const startMatches = compareDimension(
|
||||
dimensions.startedAt,
|
||||
candidate.attempt.startedAtMs ?? candidate.startedAtMs,
|
||||
selection.evidence.startedAtMs,
|
||||
(left, right) => Math.abs(Number(left) - Number(right)) <= toleranceMs,
|
||||
);
|
||||
const finishMatches = compareDimension(
|
||||
dimensions.finishedAt,
|
||||
candidate.attempt.finishedAtMs ?? candidate.finishedAtMs,
|
||||
selection.evidence.finishedAtMs,
|
||||
(left, right) => Math.abs(Number(left) - Number(right)) <= toleranceMs,
|
||||
);
|
||||
const logMatches = compareDimension(
|
||||
dimensions.logArtifact,
|
||||
candidate.attempt.logArtifactId,
|
||||
selection.evidence.logArtifactId,
|
||||
);
|
||||
const compared = [true, exitMatches, startMatches, finishMatches, logMatches];
|
||||
const hasMismatch = compared.includes(false);
|
||||
return {
|
||||
category: hasMismatch ? 'field_mismatch' : 'matched',
|
||||
fullyComparable: compared.every((value) => value === true),
|
||||
};
|
||||
}
|
||||
|
||||
function sameCursor(
|
||||
left: LegacyShadowTerminalCursor | undefined,
|
||||
right: LegacyShadowTerminalCursor,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.createdAtMs === right.createdAtMs &&
|
||||
left.runId === right.runId
|
||||
);
|
||||
}
|
||||
|
||||
function validatePage(
|
||||
page: LegacyShadowTerminalPage,
|
||||
cursor: LegacyShadowTerminalCursor | undefined,
|
||||
limit: number,
|
||||
allowedOrigins: ReadonlySet<ExecutionOrigin>,
|
||||
windowStartMs: number,
|
||||
windowEndMs: number,
|
||||
): void {
|
||||
if (page.candidates.length > limit) {
|
||||
throw new Error('Legacy Shadow terminal source exceeded candidate limit');
|
||||
}
|
||||
if (
|
||||
page.evidence.length >
|
||||
Math.min(
|
||||
MAX_LEGACY_SHADOW_TERMINAL_EVIDENCE_PER_PAGE,
|
||||
page.candidates.length * 8,
|
||||
)
|
||||
) {
|
||||
throw new Error('Legacy Shadow terminal source exceeded evidence limit');
|
||||
}
|
||||
let previous = cursor;
|
||||
const runIds = new Set<string>();
|
||||
for (const candidate of page.candidates) {
|
||||
if (
|
||||
candidate.runId.length === 0 ||
|
||||
runIds.has(candidate.runId) ||
|
||||
!allowedOrigins.has(candidate.origin) ||
|
||||
!Number.isSafeInteger(candidate.createdAtMs) ||
|
||||
candidate.createdAtMs < windowStartMs ||
|
||||
candidate.createdAtMs >= windowEndMs ||
|
||||
(previous !== undefined &&
|
||||
(candidate.createdAtMs < previous.createdAtMs ||
|
||||
(candidate.createdAtMs === previous.createdAtMs &&
|
||||
candidate.runId <= previous.runId)))
|
||||
) {
|
||||
throw new Error('Legacy Shadow terminal source returned an invalid page');
|
||||
}
|
||||
runIds.add(candidate.runId);
|
||||
previous = { createdAtMs: candidate.createdAtMs, runId: candidate.runId };
|
||||
}
|
||||
if (page.truncated) {
|
||||
if (
|
||||
page.candidates.length !== limit ||
|
||||
page.nextCursor === undefined ||
|
||||
previous === undefined ||
|
||||
page.nextCursor.createdAtMs !== previous.createdAtMs ||
|
||||
page.nextCursor.runId !== previous.runId
|
||||
) {
|
||||
throw new Error(
|
||||
'Legacy Shadow terminal source returned an invalid cursor',
|
||||
);
|
||||
}
|
||||
} else if (page.nextCursor !== undefined) {
|
||||
throw new Error('Completed Legacy Shadow terminal page returned a cursor');
|
||||
}
|
||||
}
|
||||
|
||||
function permille(numerator: number, denominator: number): number {
|
||||
return Math.floor((numerator * 1_000) / denominator);
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit, timer-free audit for a closed Shadow cohort. The report deliberately
|
||||
* does not claim Legacy-to-Shadow capture coverage because RunningInstances have
|
||||
* no trustworthy execution-origin field in the 2.x schema.
|
||||
*/
|
||||
export class LegacyShadowTerminalDifferenceAuditor {
|
||||
constructor(private readonly source: LegacyShadowTerminalDifferenceSource) {}
|
||||
|
||||
async run(
|
||||
options: LegacyShadowTerminalAuditorOptions,
|
||||
): Promise<LegacyShadowTerminalDifferenceReport> {
|
||||
if (!['edge', 'standalone'].includes(options.profile)) {
|
||||
throw new RangeError('profile must be edge or standalone');
|
||||
}
|
||||
const projectId = options.projectId ?? 'default';
|
||||
if (projectId.length < 1 || projectId.length > 128) {
|
||||
throw new RangeError('projectId length is invalid');
|
||||
}
|
||||
const origins = [...new Set(options.origins)];
|
||||
if (origins.length < 1 || origins.length > MAX_CONFIGURED_ORIGINS) {
|
||||
throw new RangeError('origin count is outside its audit budget');
|
||||
}
|
||||
if (origins.some((origin) => !EXECUTION_ORIGIN_SET.has(origin))) {
|
||||
throw new RangeError('terminal audit origin is invalid');
|
||||
}
|
||||
assertSafeInteger('windowStartMs', options.windowStartMs);
|
||||
assertSafeInteger('windowEndMs', options.windowEndMs);
|
||||
if (options.windowStartMs >= options.windowEndMs) {
|
||||
throw new RangeError('terminal audit window must be non-empty');
|
||||
}
|
||||
const observedAtMs =
|
||||
options.observedAtMs ?? options.clock?.now() ?? Date.now();
|
||||
const minimumSettlingAgeMs = options.minimumSettlingAgeMs ?? 5 * 60_000;
|
||||
const correlationToleranceMs = options.correlationToleranceMs ?? 2_000;
|
||||
assertSafeInteger('observedAtMs', observedAtMs);
|
||||
assertSafeInteger('minimumSettlingAgeMs', minimumSettlingAgeMs);
|
||||
assertSafeInteger('correlationToleranceMs', correlationToleranceMs);
|
||||
if (correlationToleranceMs > MAX_CORRELATION_TOLERANCE_MS) {
|
||||
throw new RangeError('correlationToleranceMs exceeds its hard limit');
|
||||
}
|
||||
const budget = BUDGETS[options.profile];
|
||||
if (budget.pageSize > MAX_LEGACY_SHADOW_TERMINAL_PAGE_SIZE) {
|
||||
throw new RangeError('terminal audit Profile exceeds the source limit');
|
||||
}
|
||||
|
||||
const counts = emptyCounts();
|
||||
const dimensions = emptyDimensions();
|
||||
const originSummaries = new Map(
|
||||
origins.map((origin) => [origin, originSummary(origin)]),
|
||||
);
|
||||
const allowedOrigins = new Set(origins);
|
||||
let cursor: LegacyShadowTerminalCursor | undefined;
|
||||
let pages = 0;
|
||||
let scanned = 0;
|
||||
let fullyComparable = 0;
|
||||
let evidenceOverflowPages = 0;
|
||||
let remaining = false;
|
||||
let stopReason: LegacyShadowTerminalAuditStopReason = 'complete';
|
||||
|
||||
while (pages < budget.maxPages) {
|
||||
const page = await this.source.listCandidates({
|
||||
projectId,
|
||||
origins,
|
||||
windowStartMs: options.windowStartMs,
|
||||
windowEndMs: options.windowEndMs,
|
||||
observedAtMs,
|
||||
correlationToleranceMs,
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: budget.pageSize,
|
||||
});
|
||||
pages += 1;
|
||||
validatePage(
|
||||
page,
|
||||
cursor,
|
||||
budget.pageSize,
|
||||
allowedOrigins,
|
||||
options.windowStartMs,
|
||||
options.windowEndMs,
|
||||
);
|
||||
if (page.evidenceTruncated) evidenceOverflowPages += 1;
|
||||
|
||||
for (const candidate of page.candidates) {
|
||||
const result = classify(
|
||||
candidate,
|
||||
page.evidence,
|
||||
page.evidenceTruncated,
|
||||
correlationToleranceMs,
|
||||
dimensions,
|
||||
);
|
||||
counts[result.category] += 1;
|
||||
const perOrigin = originSummaries.get(candidate.origin)!;
|
||||
perOrigin.scanned += 1;
|
||||
perOrigin[result.category] += 1;
|
||||
scanned += 1;
|
||||
if (result.fullyComparable) fullyComparable += 1;
|
||||
}
|
||||
|
||||
if (!page.truncated) break;
|
||||
if (
|
||||
page.nextCursor === undefined ||
|
||||
sameCursor(cursor, page.nextCursor)
|
||||
) {
|
||||
remaining = true;
|
||||
stopReason = 'cursor_stalled';
|
||||
break;
|
||||
}
|
||||
cursor = page.nextCursor;
|
||||
if (pages === budget.maxPages) {
|
||||
remaining = true;
|
||||
stopReason = 'page_limit';
|
||||
}
|
||||
}
|
||||
|
||||
const windowClosed =
|
||||
options.windowEndMs <= observedAtMs - minimumSettlingAgeMs;
|
||||
const evidenceComplete = evidenceOverflowPages === 0;
|
||||
const assessment: LegacyShadowTerminalAuditAssessment = !windowClosed
|
||||
? 'window_open'
|
||||
: remaining || !evidenceComplete
|
||||
? 'incomplete'
|
||||
: scanned === 0
|
||||
? 'empty'
|
||||
: counts.matched === scanned
|
||||
? 'matched'
|
||||
: 'differences_found';
|
||||
const ratiosAvailable =
|
||||
windowClosed && !remaining && evidenceComplete && scanned > 0;
|
||||
|
||||
return {
|
||||
schema: 'qinglong/legacy-shadow-terminal-difference-report@v1',
|
||||
profile: options.profile,
|
||||
observedAtMs,
|
||||
window: {
|
||||
basis: 'shadow_run_created_at',
|
||||
startInclusiveMs: options.windowStartMs,
|
||||
endExclusiveMs: options.windowEndMs,
|
||||
minimumSettlingAgeMs,
|
||||
closed: windowClosed,
|
||||
},
|
||||
coverage: {
|
||||
direction: 'shadow_to_legacy',
|
||||
cohort: 'legacy_owned_shadow_runs',
|
||||
legacyWithoutShadow: 'not_measured',
|
||||
},
|
||||
budget: {
|
||||
pageSize: budget.pageSize,
|
||||
maxPages: budget.maxPages,
|
||||
maxCandidates: budget.pageSize * budget.maxPages,
|
||||
},
|
||||
pages,
|
||||
scanned,
|
||||
stopReason,
|
||||
remaining,
|
||||
evidenceComplete,
|
||||
evidenceOverflowPages,
|
||||
assessment,
|
||||
counts,
|
||||
byOrigin: [...originSummaries.values()],
|
||||
dimensions,
|
||||
...(ratiosAvailable
|
||||
? {
|
||||
terminalAgreementPermille: permille(counts.matched, scanned),
|
||||
fullyComparablePermille: permille(fullyComparable, scanned),
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import type {
|
||||
ExecutionOrigin,
|
||||
RunAttemptStatus,
|
||||
RunStatus,
|
||||
} from '../domain/run';
|
||||
|
||||
export const MAX_LEGACY_SHADOW_TERMINAL_PAGE_SIZE = 64;
|
||||
export const MAX_LEGACY_SHADOW_TERMINAL_EVIDENCE_PER_PAGE = 512;
|
||||
|
||||
export interface LegacyShadowTerminalCursor {
|
||||
createdAtMs: number;
|
||||
runId: string;
|
||||
}
|
||||
|
||||
export interface LegacyShadowTerminalAttemptEvidence {
|
||||
attemptId: string;
|
||||
status: RunAttemptStatus;
|
||||
pid?: number;
|
||||
logArtifactId?: string;
|
||||
createdAtMs: number;
|
||||
startedAtMs?: number;
|
||||
finishedAtMs?: number;
|
||||
exitCode?: number;
|
||||
}
|
||||
|
||||
export interface LegacyShadowTerminalCandidate {
|
||||
runId: string;
|
||||
legacyCronId?: number;
|
||||
origin: ExecutionOrigin;
|
||||
runStatus: RunStatus;
|
||||
createdAtMs: number;
|
||||
startedAtMs?: number;
|
||||
finishedAtMs?: number;
|
||||
attemptCount: number;
|
||||
attempt?: LegacyShadowTerminalAttemptEvidence;
|
||||
}
|
||||
|
||||
export type LegacyTerminalOutcome = 'succeeded' | 'failed' | 'stopped';
|
||||
|
||||
export interface LegacyTerminalEvidence {
|
||||
instanceId: number;
|
||||
legacyCronId: number;
|
||||
runId?: string;
|
||||
attemptId?: string;
|
||||
pid?: number;
|
||||
logArtifactId?: string;
|
||||
startedAtMs: number;
|
||||
finishedAtMs: number;
|
||||
outcome: LegacyTerminalOutcome;
|
||||
exitCode?: number;
|
||||
}
|
||||
|
||||
export interface LegacyShadowTerminalPage {
|
||||
candidates: readonly LegacyShadowTerminalCandidate[];
|
||||
evidence: readonly LegacyTerminalEvidence[];
|
||||
evidenceTruncated: boolean;
|
||||
truncated: boolean;
|
||||
nextCursor?: LegacyShadowTerminalCursor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only local Legacy authority used by the explicit terminal-difference
|
||||
* audit. Implementations must bound both candidate and evidence result sets.
|
||||
*/
|
||||
export interface LegacyShadowTerminalDifferenceSource {
|
||||
listCandidates(options: {
|
||||
projectId: string;
|
||||
origins: readonly ExecutionOrigin[];
|
||||
windowStartMs: number;
|
||||
windowEndMs: number;
|
||||
observedAtMs: number;
|
||||
correlationToleranceMs: number;
|
||||
cursor?: LegacyShadowTerminalCursor;
|
||||
limit: number;
|
||||
}): Promise<LegacyShadowTerminalPage>;
|
||||
}
|
||||
Reference in New Issue
Block a user