mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): reconcile legacy shadow runs on startup
This commit is contained in:
whyour
@@ -0,0 +1,490 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type { RunAttemptRecord, RunRecord, RunStatus } from '../domain/run';
|
||||
import {
|
||||
RUN_TRANSITIONS,
|
||||
isTerminalRunAttemptStatus,
|
||||
isTerminalRunStatus,
|
||||
} from '../domain/runStateMachine';
|
||||
import type {
|
||||
LegacyRunningInstanceEvidence,
|
||||
LegacyShadowStartupAttempt,
|
||||
LegacyShadowStartupCandidate,
|
||||
LegacyShadowStartupCursor,
|
||||
LegacyShadowStartupRecoverySource,
|
||||
} from '../ports/legacyShadowStartupRecovery';
|
||||
import { MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE } from '../ports/legacyShadowStartupRecovery';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import type { LegacyShadowRunWriter } from './legacyShadowRunWriter';
|
||||
import { RunCommandService } from './runCommandService';
|
||||
|
||||
export const MAX_LEGACY_SHADOW_STARTUP_PAGES = 64;
|
||||
|
||||
export interface LegacyShadowStartupClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface LegacyShadowStartupReconcileSummary {
|
||||
scanned: number;
|
||||
completed: number;
|
||||
cancelled: number;
|
||||
abandoned: number;
|
||||
markedLost: number;
|
||||
repaired: number;
|
||||
pending: number;
|
||||
ambiguous: number;
|
||||
skipped: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
nextCursor?: LegacyShadowStartupCursor;
|
||||
}
|
||||
|
||||
export type LegacyShadowStartupStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface LegacyShadowStartupSummary
|
||||
extends Omit<
|
||||
LegacyShadowStartupReconcileSummary,
|
||||
'truncated' | 'nextCursor'
|
||||
> {
|
||||
pages: number;
|
||||
stopReason: LegacyShadowStartupStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: LegacyShadowStartupCursor;
|
||||
}
|
||||
|
||||
type EvidenceSelection =
|
||||
| { status: 'matched'; evidence: LegacyRunningInstanceEvidence }
|
||||
| { status: 'none' | 'ambiguous' };
|
||||
|
||||
function sameCursor(
|
||||
left: LegacyShadowStartupCursor | undefined,
|
||||
right: LegacyShadowStartupCursor,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.createdAtMs === right.createdAtMs &&
|
||||
left.runId === right.runId
|
||||
);
|
||||
}
|
||||
|
||||
function selectEvidence(
|
||||
attempt: Pick<LegacyShadowStartupAttempt, 'pid' | 'logArtifactId'>,
|
||||
evidence: readonly LegacyRunningInstanceEvidence[],
|
||||
): EvidenceSelection {
|
||||
const byLog =
|
||||
attempt.logArtifactId === undefined
|
||||
? []
|
||||
: evidence.filter(
|
||||
(candidate) => candidate.logArtifactId === attempt.logArtifactId,
|
||||
);
|
||||
const byPid =
|
||||
attempt.pid === undefined
|
||||
? []
|
||||
: evidence.filter((candidate) => candidate.pid === attempt.pid);
|
||||
|
||||
if (attempt.logArtifactId !== undefined && attempt.pid !== undefined) {
|
||||
const byPidSet = new Set(byPid);
|
||||
const intersection = byLog.filter((candidate) => byPidSet.has(candidate));
|
||||
if (intersection.length === 1) {
|
||||
return { status: 'matched', evidence: intersection[0] };
|
||||
}
|
||||
if (
|
||||
intersection.length > 1 ||
|
||||
byLog.length > 1 ||
|
||||
byPid.length > 1 ||
|
||||
(byLog.length > 0 && byPid.length > 0)
|
||||
) {
|
||||
return { status: 'ambiguous' };
|
||||
}
|
||||
if (byLog.length === 1) {
|
||||
return { status: 'matched', evidence: byLog[0] };
|
||||
}
|
||||
if (byPid.length === 1) {
|
||||
return { status: 'matched', evidence: byPid[0] };
|
||||
}
|
||||
return { status: 'none' };
|
||||
}
|
||||
if (attempt.logArtifactId !== undefined) {
|
||||
if (byLog.length === 1) {
|
||||
return { status: 'matched', evidence: byLog[0] };
|
||||
}
|
||||
return { status: byLog.length > 1 ? 'ambiguous' : 'none' };
|
||||
}
|
||||
if (attempt.pid !== undefined) {
|
||||
if (byPid.length === 1) {
|
||||
return { status: 'matched', evidence: byPid[0] };
|
||||
}
|
||||
return { status: byPid.length > 1 ? 'ambiguous' : 'none' };
|
||||
}
|
||||
if (evidence.length === 1) {
|
||||
return { status: 'matched', evidence: evidence[0] };
|
||||
}
|
||||
return { status: evidence.length > 1 ? 'ambiguous' : 'none' };
|
||||
}
|
||||
|
||||
function terminalRunTarget(attempt: RunAttemptRecord): RunStatus | undefined {
|
||||
if (attempt.status === 'succeeded') return 'succeeded';
|
||||
if (attempt.status === 'failed') return 'failed';
|
||||
if (attempt.status === 'cancelled') return 'cancelled';
|
||||
if (attempt.status === 'timed_out') return 'timed_out';
|
||||
if (attempt.status === 'lost') return 'lost';
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Reconciles one bounded page after Legacy startup normalization. */
|
||||
export class LegacyShadowStartupReconciler {
|
||||
private readonly commands: RunCommandService;
|
||||
private readonly clock: LegacyShadowStartupClock;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly source: LegacyShadowStartupRecoverySource,
|
||||
private readonly writer: Pick<
|
||||
LegacyShadowRunWriter,
|
||||
'exited' | 'cancelled'
|
||||
>,
|
||||
options: {
|
||||
clock?: LegacyShadowStartupClock;
|
||||
createEventId?: () => string;
|
||||
} = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.commands = new RunCommandService(
|
||||
repository,
|
||||
options.createEventId ?? uuidV7,
|
||||
);
|
||||
}
|
||||
|
||||
async reconcileBatch(options: {
|
||||
origins: readonly RunRecord['executionOrigin'][];
|
||||
cursor?: LegacyShadowStartupCursor;
|
||||
limit?: number;
|
||||
}): Promise<LegacyShadowStartupReconcileSummary> {
|
||||
const enabledOrigins = [...new Set(options.origins)];
|
||||
const page = await this.source.listCandidates({
|
||||
origins: enabledOrigins,
|
||||
...(options.cursor === undefined ? {} : { cursor: options.cursor }),
|
||||
...(options.limit === undefined ? {} : { limit: options.limit }),
|
||||
});
|
||||
const summary: LegacyShadowStartupReconcileSummary = {
|
||||
scanned: page.candidates.length,
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 0,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
};
|
||||
const origins = new Set(enabledOrigins);
|
||||
for (const candidate of page.candidates) {
|
||||
try {
|
||||
await this.reconcileCandidate(candidate, origins, summary);
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
private async reconcileCandidate(
|
||||
candidate: LegacyShadowStartupCandidate,
|
||||
origins: ReadonlySet<RunRecord['executionOrigin']>,
|
||||
summary: LegacyShadowStartupReconcileSummary,
|
||||
): Promise<void> {
|
||||
const run = await this.repository.findRunById(candidate.runId);
|
||||
if (
|
||||
!run ||
|
||||
run.executionOwner !== 'legacy' ||
|
||||
!origins.has(run.executionOrigin) ||
|
||||
isTerminalRunStatus(run.status) ||
|
||||
run.status === 'lost'
|
||||
) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
}
|
||||
if (candidate.activeAttemptCount === 0) {
|
||||
await this.repairTerminalAttempt(run, summary);
|
||||
return;
|
||||
}
|
||||
if (candidate.activeAttemptCount !== 1 || !candidate.attempt) {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
}
|
||||
const attempt = await this.repository.findAttemptById(
|
||||
candidate.attempt.attemptId,
|
||||
);
|
||||
if (
|
||||
!attempt ||
|
||||
attempt.runId !== run.id ||
|
||||
isTerminalRunAttemptStatus(attempt.status)
|
||||
) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const evidence =
|
||||
candidate.legacyCronId === undefined
|
||||
? { evidence: [], truncated: false }
|
||||
: await this.source.listRunningInstanceEvidence({
|
||||
legacyCronId: candidate.legacyCronId,
|
||||
});
|
||||
if (evidence.truncated) {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
}
|
||||
const selected = selectEvidence(attempt, evidence.evidence);
|
||||
if (selected.status === 'ambiguous') {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
selected.status === 'matched' &&
|
||||
selected.evidence.finishedAtMs !== undefined
|
||||
) {
|
||||
const atMs = this.atOrAfter(run, attempt, selected.evidence.finishedAtMs);
|
||||
if (selected.evidence.outcome === 'stopped') {
|
||||
await this.writer.cancelled(
|
||||
{ runId: run.id, attemptId: attempt.id },
|
||||
{ atMs, reason: 'reconcile' },
|
||||
);
|
||||
summary.cancelled += 1;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
selected.evidence.outcome === 'succeeded' ||
|
||||
selected.evidence.outcome === 'failed'
|
||||
) {
|
||||
const succeeded = selected.evidence.outcome === 'succeeded';
|
||||
await this.writer.exited(
|
||||
{ runId: run.id, attemptId: attempt.id },
|
||||
{
|
||||
atMs,
|
||||
exitCode: succeeded
|
||||
? 0
|
||||
: selected.evidence.exitCode === undefined ||
|
||||
selected.evidence.exitCode === 0
|
||||
? 1
|
||||
: selected.evidence.exitCode,
|
||||
},
|
||||
);
|
||||
summary.completed += 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (run.executionOrigin === 'scheduled_system') {
|
||||
summary.pending += 1;
|
||||
return;
|
||||
}
|
||||
if (run.status === 'queued' && attempt.status === 'claimed') {
|
||||
await this.abandon(run, attempt);
|
||||
summary.abandoned += 1;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
(run.status === 'dispatching' || run.status === 'running') &&
|
||||
['claimed', 'starting', 'running'].includes(attempt.status)
|
||||
) {
|
||||
await this.markLost(run, attempt);
|
||||
summary.markedLost += 1;
|
||||
return;
|
||||
}
|
||||
summary.ambiguous += 1;
|
||||
}
|
||||
|
||||
private async repairTerminalAttempt(
|
||||
run: RunRecord,
|
||||
summary: LegacyShadowStartupReconcileSummary,
|
||||
): Promise<void> {
|
||||
const attempt = await this.repository.findLatestAttemptByRunId(run.id);
|
||||
const target = attempt ? terminalRunTarget(attempt) : undefined;
|
||||
if (
|
||||
!attempt ||
|
||||
target === undefined ||
|
||||
!RUN_TRANSITIONS[run.status].includes(target)
|
||||
) {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
}
|
||||
await this.commands.transitionRun({
|
||||
runId: run.id,
|
||||
to: target,
|
||||
expectedVersion: run.version,
|
||||
atMs: this.atOrAfter(run, attempt),
|
||||
...(attempt.errorCode === undefined
|
||||
? {}
|
||||
: { errorCode: attempt.errorCode }),
|
||||
...(attempt.errorSummary === undefined
|
||||
? {}
|
||||
: { errorSummary: attempt.errorSummary }),
|
||||
actor: { type: 'reconciler' },
|
||||
dedupeKey: `legacy-startup-repair:${attempt.id}:${target}`,
|
||||
});
|
||||
summary.repaired += 1;
|
||||
}
|
||||
|
||||
private async abandon(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
): Promise<void> {
|
||||
const atMs = this.atOrAfter(run, attempt);
|
||||
const attemptResult = await this.commands.transitionRunAttempt({
|
||||
runId: run.id,
|
||||
attemptId: attempt.id,
|
||||
to: 'cancelled',
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
errorCode: 'LEGACY_RECONCILE_ACCEPTANCE_ABANDONED',
|
||||
errorSummary: 'Legacy owner restarted before spawn was observed',
|
||||
actor: { type: 'reconciler' },
|
||||
dedupeKey: `legacy-startup-abandoned-attempt:${attempt.id}`,
|
||||
});
|
||||
await this.commands.transitionRun({
|
||||
runId: run.id,
|
||||
to: 'cancelled',
|
||||
expectedVersion: attemptResult.run.version,
|
||||
atMs,
|
||||
errorCode: 'LEGACY_RECONCILE_ACCEPTANCE_ABANDONED',
|
||||
errorSummary: 'Legacy owner restarted before spawn was observed',
|
||||
actor: { type: 'reconciler' },
|
||||
dedupeKey: `legacy-startup-abandoned-run:${attempt.id}`,
|
||||
});
|
||||
}
|
||||
|
||||
private async markLost(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
): Promise<void> {
|
||||
const atMs = this.atOrAfter(run, attempt);
|
||||
const attemptResult = await this.commands.transitionRunAttempt({
|
||||
runId: run.id,
|
||||
attemptId: attempt.id,
|
||||
to: 'lost',
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
errorCode: 'LEGACY_RECONCILE_OWNER_LOST',
|
||||
errorSummary: 'Legacy worker restarted without terminal evidence',
|
||||
actor: { type: 'reconciler' },
|
||||
dedupeKey: `legacy-startup-lost-attempt:${attempt.id}`,
|
||||
});
|
||||
await this.commands.transitionRun({
|
||||
runId: run.id,
|
||||
to: 'lost',
|
||||
expectedVersion: attemptResult.run.version,
|
||||
atMs,
|
||||
errorCode: 'LEGACY_RECONCILE_OWNER_LOST',
|
||||
errorSummary: 'Legacy worker restarted without terminal evidence',
|
||||
actor: { type: 'reconciler' },
|
||||
dedupeKey: `legacy-startup-lost-run:${attempt.id}`,
|
||||
});
|
||||
}
|
||||
|
||||
private atOrAfter(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
evidenceAtMs?: number,
|
||||
): number {
|
||||
return Math.max(
|
||||
this.clock.now(),
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs ?? 0,
|
||||
evidenceAtMs ?? 0,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs a complete but Profile-bounded, timer-free startup pass. */
|
||||
export class LegacyShadowStartupSupervisor {
|
||||
constructor(
|
||||
private readonly reconciler: Pick<
|
||||
LegacyShadowStartupReconciler,
|
||||
'reconcileBatch'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async run(options: {
|
||||
origins: readonly RunRecord['executionOrigin'][];
|
||||
cursor?: LegacyShadowStartupCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}): Promise<LegacyShadowStartupSummary> {
|
||||
const pageSize = options.pageSize ?? 8;
|
||||
const maxPages = options.maxPages ?? 1;
|
||||
if (
|
||||
!Number.isSafeInteger(pageSize) ||
|
||||
pageSize < 1 ||
|
||||
pageSize > MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'pageSize must be between 1 and MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(maxPages) ||
|
||||
maxPages < 1 ||
|
||||
maxPages > MAX_LEGACY_SHADOW_STARTUP_PAGES
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_LEGACY_SHADOW_STARTUP_PAGES',
|
||||
);
|
||||
}
|
||||
|
||||
const total: LegacyShadowStartupSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 0,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) {
|
||||
const page = await this.reconciler.reconcileBatch({
|
||||
origins: options.origins,
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.completed += page.completed;
|
||||
total.cancelled += page.cancelled;
|
||||
total.abandoned += page.abandoned;
|
||||
total.markedLost += page.markedLost;
|
||||
total.repaired += page.repaired;
|
||||
total.pending += page.pending;
|
||||
total.ambiguous += page.ambiguous;
|
||||
total.skipped += page.skipped;
|
||||
total.failed += page.failed;
|
||||
if (!page.truncated) return total;
|
||||
if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
cursor = page.nextCursor;
|
||||
if (pageNumber === maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return total;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user