mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): reconcile legacy shadow runs on startup
This commit is contained in:
whyour
@@ -0,0 +1,319 @@
|
||||
import { QueryTypes, type Sequelize } from 'sequelize';
|
||||
import {
|
||||
RUN_ATTEMPT_TABLE,
|
||||
RUN_TABLE,
|
||||
} from '../../../migrations/0002-run-schema';
|
||||
import { RUNNING_INSTANCE_TABLE } from '../../../migrations/0003-running-instance-run-reference';
|
||||
import type {
|
||||
ExecutionOrigin,
|
||||
RunAttemptStatus,
|
||||
RunStatus,
|
||||
} from '../../domain/run';
|
||||
import {
|
||||
MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE,
|
||||
MAX_LEGACY_SHADOW_STARTUP_EVIDENCE,
|
||||
type LegacyRunningInstanceEvidence,
|
||||
type LegacyRunningInstanceEvidencePage,
|
||||
type LegacyShadowStartupCandidate,
|
||||
type LegacyShadowStartupCursor,
|
||||
type LegacyShadowStartupPage,
|
||||
type LegacyShadowStartupRecoverySource,
|
||||
} from '../../ports/legacyShadowStartupRecovery';
|
||||
|
||||
interface RunRow {
|
||||
runId: string;
|
||||
legacyCronId: number | null;
|
||||
origin: string;
|
||||
runStatus: string;
|
||||
createdAtMs: number | string;
|
||||
}
|
||||
|
||||
interface AttemptCountRow {
|
||||
runId: string;
|
||||
activeAttemptCount: number | string;
|
||||
}
|
||||
|
||||
interface AttemptRow {
|
||||
attemptId: string;
|
||||
runId: string;
|
||||
status: string;
|
||||
pid: number | null;
|
||||
logArtifactId: string | null;
|
||||
createdAtMs: number | string;
|
||||
startedAtMs: number | string | null;
|
||||
}
|
||||
|
||||
interface RunningInstanceRow {
|
||||
pid: number | null;
|
||||
logPath: string | null;
|
||||
startedAt: number | string;
|
||||
finishedAt: number | string | null;
|
||||
status: number | string;
|
||||
exitCode: number | null;
|
||||
}
|
||||
|
||||
export type LegacyLogArtifactIdFactory = (logPath: string) => string;
|
||||
|
||||
function safeInteger(value: number | string, name: string): number {
|
||||
const parsed = typeof value === 'number' ? value : Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 0) {
|
||||
throw new TypeError(`${name} is invalid`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function secondsToMilliseconds(value: number | string, name: string): number {
|
||||
const seconds = safeInteger(value, name);
|
||||
const milliseconds = seconds * 1_000;
|
||||
if (!Number.isSafeInteger(milliseconds)) {
|
||||
throw new TypeError(`${name} is invalid`);
|
||||
}
|
||||
return milliseconds;
|
||||
}
|
||||
|
||||
function optionalPositiveInteger(
|
||||
value: number | null,
|
||||
name: string,
|
||||
): number | undefined {
|
||||
if (value === null) return undefined;
|
||||
const parsed = safeInteger(value, name);
|
||||
if (parsed < 1) throw new TypeError(`${name} is invalid`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function assertLimit(limit: number, maximum: number): void {
|
||||
if (!Number.isSafeInteger(limit) || limit < 1 || limit > maximum) {
|
||||
throw new RangeError(`limit must be between 1 and ${maximum}`);
|
||||
}
|
||||
}
|
||||
|
||||
function outcome(status: number): LegacyRunningInstanceEvidence['outcome'] {
|
||||
if (status === 0) return 'running';
|
||||
if (status === 1) return 'succeeded';
|
||||
if (status === 2) return 'stopped';
|
||||
if (status === 3) return 'failed';
|
||||
throw new TypeError('RunningInstance status is invalid');
|
||||
}
|
||||
|
||||
export class LegacySequelizeShadowStartupRecoverySource
|
||||
implements LegacyShadowStartupRecoverySource
|
||||
{
|
||||
constructor(
|
||||
private readonly database: Sequelize,
|
||||
private readonly createLogArtifactId: LegacyLogArtifactIdFactory,
|
||||
) {}
|
||||
|
||||
async listCandidates({
|
||||
origins,
|
||||
cursor,
|
||||
limit = MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE,
|
||||
}: {
|
||||
origins: readonly ExecutionOrigin[];
|
||||
cursor?: LegacyShadowStartupCursor;
|
||||
limit?: number;
|
||||
}): Promise<LegacyShadowStartupPage> {
|
||||
assertLimit(limit, MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE);
|
||||
const enabledOrigins = [...new Set(origins)];
|
||||
if (enabledOrigins.length === 0) {
|
||||
return { candidates: [], truncated: false };
|
||||
}
|
||||
if (
|
||||
cursor !== undefined &&
|
||||
(!Number.isSafeInteger(cursor.createdAtMs) ||
|
||||
cursor.createdAtMs < 0 ||
|
||||
cursor.runId.length === 0)
|
||||
) {
|
||||
throw new TypeError('Legacy Shadow startup cursor is invalid');
|
||||
}
|
||||
|
||||
const runRows = await this.database.query<RunRow>(
|
||||
`SELECT
|
||||
id AS "runId",
|
||||
legacy_cron_id AS "legacyCronId",
|
||||
execution_origin AS origin,
|
||||
status AS "runStatus",
|
||||
created_at_ms AS "createdAtMs"
|
||||
FROM ${RUN_TABLE}
|
||||
WHERE execution_owner = 'legacy'
|
||||
AND execution_origin IN (:origins)
|
||||
AND status IN ('queued', 'dispatching', 'running')
|
||||
${
|
||||
cursor === undefined
|
||||
? ''
|
||||
: `AND (
|
||||
created_at_ms > :cursorCreatedAtMs OR
|
||||
(created_at_ms = :cursorCreatedAtMs AND id > :cursorRunId)
|
||||
)`
|
||||
}
|
||||
ORDER BY created_at_ms ASC, id ASC
|
||||
LIMIT :fetchLimit`,
|
||||
{
|
||||
replacements: {
|
||||
origins: enabledOrigins,
|
||||
fetchLimit: limit + 1,
|
||||
...(cursor === undefined
|
||||
? {}
|
||||
: {
|
||||
cursorCreatedAtMs: cursor.createdAtMs,
|
||||
cursorRunId: cursor.runId,
|
||||
}),
|
||||
},
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
);
|
||||
const truncated = runRows.length > limit;
|
||||
const boundedRuns = runRows.slice(0, limit);
|
||||
if (boundedRuns.length === 0) {
|
||||
return { candidates: [], truncated };
|
||||
}
|
||||
|
||||
const runIds = boundedRuns.map((run) => run.runId);
|
||||
const countRows = await this.database.query<AttemptCountRow>(
|
||||
`SELECT run_id AS "runId", COUNT(*) AS "activeAttemptCount"
|
||||
FROM ${RUN_ATTEMPT_TABLE}
|
||||
WHERE run_id IN (:runIds)
|
||||
AND status IN ('claimed', 'starting', 'running')
|
||||
GROUP BY run_id`,
|
||||
{
|
||||
replacements: { runIds },
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
);
|
||||
const counts = new Map(
|
||||
countRows.map((row) => [
|
||||
row.runId,
|
||||
safeInteger(row.activeAttemptCount, 'activeAttemptCount'),
|
||||
]),
|
||||
);
|
||||
const singleAttemptRunIds = runIds.filter(
|
||||
(runId) => counts.get(runId) === 1,
|
||||
);
|
||||
const attemptRows =
|
||||
singleAttemptRunIds.length === 0
|
||||
? []
|
||||
: await this.database.query<AttemptRow>(
|
||||
`SELECT
|
||||
id AS "attemptId",
|
||||
run_id AS "runId",
|
||||
status,
|
||||
pid,
|
||||
log_artifact_id AS "logArtifactId",
|
||||
created_at_ms AS "createdAtMs",
|
||||
started_at_ms AS "startedAtMs"
|
||||
FROM ${RUN_ATTEMPT_TABLE}
|
||||
WHERE run_id IN (:runIds)
|
||||
AND status IN ('claimed', 'starting', 'running')`,
|
||||
{
|
||||
replacements: { runIds: singleAttemptRunIds },
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
);
|
||||
const attempts = new Map(attemptRows.map((row) => [row.runId, row]));
|
||||
const candidates: LegacyShadowStartupCandidate[] = boundedRuns.map(
|
||||
(run) => {
|
||||
const activeAttemptCount = counts.get(run.runId) ?? 0;
|
||||
const attempt = attempts.get(run.runId);
|
||||
return {
|
||||
runId: run.runId,
|
||||
...(run.legacyCronId === null
|
||||
? {}
|
||||
: {
|
||||
legacyCronId: optionalPositiveInteger(
|
||||
run.legacyCronId,
|
||||
'legacyCronId',
|
||||
),
|
||||
}),
|
||||
origin: run.origin as ExecutionOrigin,
|
||||
runStatus: run.runStatus as RunStatus,
|
||||
createdAtMs: safeInteger(run.createdAtMs, 'createdAtMs'),
|
||||
activeAttemptCount,
|
||||
...(attempt === undefined
|
||||
? {}
|
||||
: {
|
||||
attempt: {
|
||||
attemptId: attempt.attemptId,
|
||||
status: attempt.status as RunAttemptStatus,
|
||||
...(attempt.pid === null ? {} : { pid: attempt.pid }),
|
||||
...(attempt.logArtifactId === null
|
||||
? {}
|
||||
: { logArtifactId: attempt.logArtifactId }),
|
||||
createdAtMs: safeInteger(
|
||||
attempt.createdAtMs,
|
||||
'attempt.createdAtMs',
|
||||
),
|
||||
...(attempt.startedAtMs === null
|
||||
? {}
|
||||
: {
|
||||
startedAtMs: safeInteger(
|
||||
attempt.startedAtMs,
|
||||
'attempt.startedAtMs',
|
||||
),
|
||||
}),
|
||||
},
|
||||
}),
|
||||
};
|
||||
},
|
||||
);
|
||||
const last = candidates.at(-1);
|
||||
return {
|
||||
candidates,
|
||||
truncated,
|
||||
...(truncated && last
|
||||
? {
|
||||
nextCursor: {
|
||||
createdAtMs: last.createdAtMs,
|
||||
runId: last.runId,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
async listRunningInstanceEvidence({
|
||||
legacyCronId,
|
||||
limit = MAX_LEGACY_SHADOW_STARTUP_EVIDENCE,
|
||||
}: {
|
||||
legacyCronId: number;
|
||||
limit?: number;
|
||||
}): Promise<LegacyRunningInstanceEvidencePage> {
|
||||
if (!Number.isSafeInteger(legacyCronId) || legacyCronId < 1) {
|
||||
throw new RangeError('legacyCronId must be a positive safe integer');
|
||||
}
|
||||
assertLimit(limit, MAX_LEGACY_SHADOW_STARTUP_EVIDENCE);
|
||||
const rows = await this.database.query<RunningInstanceRow>(
|
||||
`SELECT
|
||||
pid,
|
||||
log_path AS "logPath",
|
||||
started_at AS "startedAt",
|
||||
finished_at AS "finishedAt",
|
||||
status,
|
||||
exit_code AS "exitCode"
|
||||
FROM ${RUNNING_INSTANCE_TABLE}
|
||||
WHERE cron_id = :legacyCronId
|
||||
ORDER BY started_at DESC, id DESC
|
||||
LIMIT :fetchLimit`,
|
||||
{
|
||||
replacements: { legacyCronId, fetchLimit: limit + 1 },
|
||||
type: QueryTypes.SELECT,
|
||||
},
|
||||
);
|
||||
const truncated = rows.length > limit;
|
||||
return {
|
||||
evidence: rows.slice(0, limit).map((row) => ({
|
||||
...(row.pid === null ? {} : { pid: row.pid }),
|
||||
...(row.logPath === null
|
||||
? {}
|
||||
: { logArtifactId: this.createLogArtifactId(row.logPath) }),
|
||||
startedAtMs: secondsToMilliseconds(row.startedAt, 'startedAt'),
|
||||
...(row.finishedAt === null
|
||||
? {}
|
||||
: {
|
||||
finishedAtMs: secondsToMilliseconds(row.finishedAt, 'finishedAt'),
|
||||
}),
|
||||
outcome: outcome(safeInteger(row.status, 'status')),
|
||||
...(row.exitCode === null ? {} : { exitCode: row.exitCode }),
|
||||
})),
|
||||
truncated,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import Logger from '../../../loaders/logger';
|
||||
import type { ExecutionOrigin } from '../../domain/run';
|
||||
import {
|
||||
parseDeploymentProfile,
|
||||
type DeploymentProfile,
|
||||
} from '../../domain/deploymentProfile';
|
||||
import { configuredLegacyShadowOrigins } from '../../compatibility/legacyExecutionBridge';
|
||||
import { createLegacyLogArtifactId } from '../../compatibility/legacyTaskRevision';
|
||||
import type { LegacyShadowStartupSummary } from '../../application/legacyShadowStartupReconciler';
|
||||
|
||||
export interface LegacyShadowStartupBudget {
|
||||
pageSize: number;
|
||||
maxPages: number;
|
||||
}
|
||||
|
||||
export interface LegacyShadowStartupRequest extends LegacyShadowStartupBudget {
|
||||
origins: readonly ExecutionOrigin[];
|
||||
profile: 'edge' | 'standalone';
|
||||
}
|
||||
|
||||
export type LegacyShadowStartupAudit =
|
||||
| {
|
||||
state: 'disabled';
|
||||
}
|
||||
| {
|
||||
state: 'profile_rejected';
|
||||
profile: 'cluster-control' | 'worker';
|
||||
}
|
||||
| {
|
||||
state: 'reconciled' | 'incomplete';
|
||||
profile: 'edge' | 'standalone';
|
||||
origins: number;
|
||||
summary: Omit<LegacyShadowStartupSummary, 'nextCursor'> & {
|
||||
resumeAvailable: boolean;
|
||||
};
|
||||
}
|
||||
| {
|
||||
state: 'failed';
|
||||
errorType: string;
|
||||
};
|
||||
|
||||
export interface BootstrapLegacyShadowStartupOptions {
|
||||
origins?: readonly ExecutionOrigin[];
|
||||
profile?: DeploymentProfile;
|
||||
execute?: (
|
||||
request: LegacyShadowStartupRequest,
|
||||
) => Promise<LegacyShadowStartupSummary>;
|
||||
audit?: (record: LegacyShadowStartupAudit) => void | Promise<void>;
|
||||
}
|
||||
|
||||
const BUDGETS: Readonly<
|
||||
Record<'edge' | 'standalone', LegacyShadowStartupBudget>
|
||||
> = {
|
||||
edge: { pageSize: 8, maxPages: 1 },
|
||||
standalone: { pageSize: 32, maxPages: 4 },
|
||||
};
|
||||
|
||||
function auditSummary(summary: LegacyShadowStartupSummary): Omit<
|
||||
LegacyShadowStartupSummary,
|
||||
'nextCursor'
|
||||
> & {
|
||||
resumeAvailable: boolean;
|
||||
} {
|
||||
const { nextCursor, ...bounded } = summary;
|
||||
return { ...bounded, resumeAvailable: nextCursor !== undefined };
|
||||
}
|
||||
|
||||
async function executeDefault(
|
||||
request: LegacyShadowStartupRequest,
|
||||
): Promise<LegacyShadowStartupSummary> {
|
||||
const [data, repositoryModule, sourceModule, writerModule, reconcilerModule] =
|
||||
await Promise.all([
|
||||
import('../../../data'),
|
||||
import('../legacy-sequelize/runRepository'),
|
||||
import('../legacy-sequelize/legacyShadowStartupRecoverySource'),
|
||||
import('../../application/legacyShadowRunWriter'),
|
||||
import('../../application/legacyShadowStartupReconciler'),
|
||||
]);
|
||||
const repository = new repositoryModule.LegacySequelizeRunRepository(
|
||||
data.sequelize,
|
||||
);
|
||||
const source = new sourceModule.LegacySequelizeShadowStartupRecoverySource(
|
||||
data.sequelize,
|
||||
createLegacyLogArtifactId,
|
||||
);
|
||||
const writer = new writerModule.LegacyShadowRunWriter(repository);
|
||||
const reconciler = new reconcilerModule.LegacyShadowStartupReconciler(
|
||||
repository,
|
||||
source,
|
||||
writer,
|
||||
);
|
||||
return new reconcilerModule.LegacyShadowStartupSupervisor(reconciler).run({
|
||||
origins: request.origins,
|
||||
pageSize: request.pageSize,
|
||||
maxPages: request.maxPages,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs once after Legacy startup normalization and before HTTP listen. Disabled
|
||||
* and non-local Profiles never import a Repository or touch the database.
|
||||
*/
|
||||
export async function bootstrapLegacyShadowStartupReconciliation(
|
||||
options: BootstrapLegacyShadowStartupOptions = {},
|
||||
): Promise<LegacyShadowStartupAudit> {
|
||||
const origins = [
|
||||
...new Set(options.origins ?? configuredLegacyShadowOrigins()),
|
||||
];
|
||||
const audit =
|
||||
options.audit ??
|
||||
((record: LegacyShadowStartupAudit) => {
|
||||
Logger.info(`[ql3-shadow-startup] ${JSON.stringify(record)}`);
|
||||
});
|
||||
if (origins.length === 0) {
|
||||
const record: LegacyShadowStartupAudit = { state: 'disabled' };
|
||||
try {
|
||||
await audit(record);
|
||||
} catch {
|
||||
// Shadow audit output must not affect Legacy startup.
|
||||
}
|
||||
return record;
|
||||
}
|
||||
|
||||
try {
|
||||
const profile =
|
||||
options.profile ??
|
||||
parseDeploymentProfile(process.env.QL_DEPLOYMENT_PROFILE);
|
||||
if (profile === 'cluster-control' || profile === 'worker') {
|
||||
const record: LegacyShadowStartupAudit = {
|
||||
state: 'profile_rejected',
|
||||
profile,
|
||||
};
|
||||
try {
|
||||
await audit(record);
|
||||
} catch {
|
||||
// Shadow audit output must not affect Legacy startup.
|
||||
}
|
||||
return record;
|
||||
}
|
||||
const summary = await (options.execute ?? executeDefault)({
|
||||
origins,
|
||||
profile,
|
||||
...BUDGETS[profile],
|
||||
});
|
||||
const record: LegacyShadowStartupAudit = {
|
||||
state: summary.remaining ? 'incomplete' : 'reconciled',
|
||||
profile,
|
||||
origins: origins.length,
|
||||
summary: auditSummary(summary),
|
||||
};
|
||||
try {
|
||||
await audit(record);
|
||||
} catch {
|
||||
// Shadow audit output must not affect Legacy startup.
|
||||
}
|
||||
return record;
|
||||
} catch (error) {
|
||||
const record: LegacyShadowStartupAudit = {
|
||||
state: 'failed',
|
||||
errorType: error instanceof Error ? error.name : 'unknown',
|
||||
};
|
||||
try {
|
||||
await audit(record);
|
||||
} catch {
|
||||
// Shadow audit output must not affect Legacy startup.
|
||||
}
|
||||
return record;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,490 @@
|
||||
import { v7 as uuidV7 } from 'uuid';
|
||||
import type { RunAttemptRecord, RunRecord, RunStatus } from '../domain/run';
|
||||
import {
|
||||
RUN_TRANSITIONS,
|
||||
isTerminalRunAttemptStatus,
|
||||
isTerminalRunStatus,
|
||||
} from '../domain/runStateMachine';
|
||||
import type {
|
||||
LegacyRunningInstanceEvidence,
|
||||
LegacyShadowStartupAttempt,
|
||||
LegacyShadowStartupCandidate,
|
||||
LegacyShadowStartupCursor,
|
||||
LegacyShadowStartupRecoverySource,
|
||||
} from '../ports/legacyShadowStartupRecovery';
|
||||
import { MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE } from '../ports/legacyShadowStartupRecovery';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import type { LegacyShadowRunWriter } from './legacyShadowRunWriter';
|
||||
import { RunCommandService } from './runCommandService';
|
||||
|
||||
export const MAX_LEGACY_SHADOW_STARTUP_PAGES = 64;
|
||||
|
||||
export interface LegacyShadowStartupClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface LegacyShadowStartupReconcileSummary {
|
||||
scanned: number;
|
||||
completed: number;
|
||||
cancelled: number;
|
||||
abandoned: number;
|
||||
markedLost: number;
|
||||
repaired: number;
|
||||
pending: number;
|
||||
ambiguous: number;
|
||||
skipped: number;
|
||||
failed: number;
|
||||
truncated: boolean;
|
||||
nextCursor?: LegacyShadowStartupCursor;
|
||||
}
|
||||
|
||||
export type LegacyShadowStartupStopReason =
|
||||
| 'complete'
|
||||
| 'page_limit'
|
||||
| 'cursor_stalled';
|
||||
|
||||
export interface LegacyShadowStartupSummary
|
||||
extends Omit<
|
||||
LegacyShadowStartupReconcileSummary,
|
||||
'truncated' | 'nextCursor'
|
||||
> {
|
||||
pages: number;
|
||||
stopReason: LegacyShadowStartupStopReason;
|
||||
remaining: boolean;
|
||||
nextCursor?: LegacyShadowStartupCursor;
|
||||
}
|
||||
|
||||
type EvidenceSelection =
|
||||
| { status: 'matched'; evidence: LegacyRunningInstanceEvidence }
|
||||
| { status: 'none' | 'ambiguous' };
|
||||
|
||||
function sameCursor(
|
||||
left: LegacyShadowStartupCursor | undefined,
|
||||
right: LegacyShadowStartupCursor,
|
||||
): boolean {
|
||||
return (
|
||||
left !== undefined &&
|
||||
left.createdAtMs === right.createdAtMs &&
|
||||
left.runId === right.runId
|
||||
);
|
||||
}
|
||||
|
||||
function selectEvidence(
|
||||
attempt: Pick<LegacyShadowStartupAttempt, 'pid' | 'logArtifactId'>,
|
||||
evidence: readonly LegacyRunningInstanceEvidence[],
|
||||
): EvidenceSelection {
|
||||
const byLog =
|
||||
attempt.logArtifactId === undefined
|
||||
? []
|
||||
: evidence.filter(
|
||||
(candidate) => candidate.logArtifactId === attempt.logArtifactId,
|
||||
);
|
||||
const byPid =
|
||||
attempt.pid === undefined
|
||||
? []
|
||||
: evidence.filter((candidate) => candidate.pid === attempt.pid);
|
||||
|
||||
if (attempt.logArtifactId !== undefined && attempt.pid !== undefined) {
|
||||
const byPidSet = new Set(byPid);
|
||||
const intersection = byLog.filter((candidate) => byPidSet.has(candidate));
|
||||
if (intersection.length === 1) {
|
||||
return { status: 'matched', evidence: intersection[0] };
|
||||
}
|
||||
if (
|
||||
intersection.length > 1 ||
|
||||
byLog.length > 1 ||
|
||||
byPid.length > 1 ||
|
||||
(byLog.length > 0 && byPid.length > 0)
|
||||
) {
|
||||
return { status: 'ambiguous' };
|
||||
}
|
||||
if (byLog.length === 1) {
|
||||
return { status: 'matched', evidence: byLog[0] };
|
||||
}
|
||||
if (byPid.length === 1) {
|
||||
return { status: 'matched', evidence: byPid[0] };
|
||||
}
|
||||
return { status: 'none' };
|
||||
}
|
||||
if (attempt.logArtifactId !== undefined) {
|
||||
if (byLog.length === 1) {
|
||||
return { status: 'matched', evidence: byLog[0] };
|
||||
}
|
||||
return { status: byLog.length > 1 ? 'ambiguous' : 'none' };
|
||||
}
|
||||
if (attempt.pid !== undefined) {
|
||||
if (byPid.length === 1) {
|
||||
return { status: 'matched', evidence: byPid[0] };
|
||||
}
|
||||
return { status: byPid.length > 1 ? 'ambiguous' : 'none' };
|
||||
}
|
||||
if (evidence.length === 1) {
|
||||
return { status: 'matched', evidence: evidence[0] };
|
||||
}
|
||||
return { status: evidence.length > 1 ? 'ambiguous' : 'none' };
|
||||
}
|
||||
|
||||
function terminalRunTarget(attempt: RunAttemptRecord): RunStatus | undefined {
|
||||
if (attempt.status === 'succeeded') return 'succeeded';
|
||||
if (attempt.status === 'failed') return 'failed';
|
||||
if (attempt.status === 'cancelled') return 'cancelled';
|
||||
if (attempt.status === 'timed_out') return 'timed_out';
|
||||
if (attempt.status === 'lost') return 'lost';
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Reconciles one bounded page after Legacy startup normalization. */
|
||||
export class LegacyShadowStartupReconciler {
|
||||
private readonly commands: RunCommandService;
|
||||
private readonly clock: LegacyShadowStartupClock;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly source: LegacyShadowStartupRecoverySource,
|
||||
private readonly writer: Pick<
|
||||
LegacyShadowRunWriter,
|
||||
'exited' | 'cancelled'
|
||||
>,
|
||||
options: {
|
||||
clock?: LegacyShadowStartupClock;
|
||||
createEventId?: () => string;
|
||||
} = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.commands = new RunCommandService(
|
||||
repository,
|
||||
options.createEventId ?? uuidV7,
|
||||
);
|
||||
}
|
||||
|
||||
async reconcileBatch(options: {
|
||||
origins: readonly RunRecord['executionOrigin'][];
|
||||
cursor?: LegacyShadowStartupCursor;
|
||||
limit?: number;
|
||||
}): Promise<LegacyShadowStartupReconcileSummary> {
|
||||
const enabledOrigins = [...new Set(options.origins)];
|
||||
const page = await this.source.listCandidates({
|
||||
origins: enabledOrigins,
|
||||
...(options.cursor === undefined ? {} : { cursor: options.cursor }),
|
||||
...(options.limit === undefined ? {} : { limit: options.limit }),
|
||||
});
|
||||
const summary: LegacyShadowStartupReconcileSummary = {
|
||||
scanned: page.candidates.length,
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 0,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
truncated: page.truncated,
|
||||
...(page.nextCursor === undefined ? {} : { nextCursor: page.nextCursor }),
|
||||
};
|
||||
const origins = new Set(enabledOrigins);
|
||||
for (const candidate of page.candidates) {
|
||||
try {
|
||||
await this.reconcileCandidate(candidate, origins, summary);
|
||||
} catch {
|
||||
summary.failed += 1;
|
||||
}
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
private async reconcileCandidate(
|
||||
candidate: LegacyShadowStartupCandidate,
|
||||
origins: ReadonlySet<RunRecord['executionOrigin']>,
|
||||
summary: LegacyShadowStartupReconcileSummary,
|
||||
): Promise<void> {
|
||||
const run = await this.repository.findRunById(candidate.runId);
|
||||
if (
|
||||
!run ||
|
||||
run.executionOwner !== 'legacy' ||
|
||||
!origins.has(run.executionOrigin) ||
|
||||
isTerminalRunStatus(run.status) ||
|
||||
run.status === 'lost'
|
||||
) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
}
|
||||
if (candidate.activeAttemptCount === 0) {
|
||||
await this.repairTerminalAttempt(run, summary);
|
||||
return;
|
||||
}
|
||||
if (candidate.activeAttemptCount !== 1 || !candidate.attempt) {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
}
|
||||
const attempt = await this.repository.findAttemptById(
|
||||
candidate.attempt.attemptId,
|
||||
);
|
||||
if (
|
||||
!attempt ||
|
||||
attempt.runId !== run.id ||
|
||||
isTerminalRunAttemptStatus(attempt.status)
|
||||
) {
|
||||
summary.skipped += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const evidence =
|
||||
candidate.legacyCronId === undefined
|
||||
? { evidence: [], truncated: false }
|
||||
: await this.source.listRunningInstanceEvidence({
|
||||
legacyCronId: candidate.legacyCronId,
|
||||
});
|
||||
if (evidence.truncated) {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
}
|
||||
const selected = selectEvidence(attempt, evidence.evidence);
|
||||
if (selected.status === 'ambiguous') {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
selected.status === 'matched' &&
|
||||
selected.evidence.finishedAtMs !== undefined
|
||||
) {
|
||||
const atMs = this.atOrAfter(run, attempt, selected.evidence.finishedAtMs);
|
||||
if (selected.evidence.outcome === 'stopped') {
|
||||
await this.writer.cancelled(
|
||||
{ runId: run.id, attemptId: attempt.id },
|
||||
{ atMs, reason: 'reconcile' },
|
||||
);
|
||||
summary.cancelled += 1;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
selected.evidence.outcome === 'succeeded' ||
|
||||
selected.evidence.outcome === 'failed'
|
||||
) {
|
||||
const succeeded = selected.evidence.outcome === 'succeeded';
|
||||
await this.writer.exited(
|
||||
{ runId: run.id, attemptId: attempt.id },
|
||||
{
|
||||
atMs,
|
||||
exitCode: succeeded
|
||||
? 0
|
||||
: selected.evidence.exitCode === undefined ||
|
||||
selected.evidence.exitCode === 0
|
||||
? 1
|
||||
: selected.evidence.exitCode,
|
||||
},
|
||||
);
|
||||
summary.completed += 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (run.executionOrigin === 'scheduled_system') {
|
||||
summary.pending += 1;
|
||||
return;
|
||||
}
|
||||
if (run.status === 'queued' && attempt.status === 'claimed') {
|
||||
await this.abandon(run, attempt);
|
||||
summary.abandoned += 1;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
(run.status === 'dispatching' || run.status === 'running') &&
|
||||
['claimed', 'starting', 'running'].includes(attempt.status)
|
||||
) {
|
||||
await this.markLost(run, attempt);
|
||||
summary.markedLost += 1;
|
||||
return;
|
||||
}
|
||||
summary.ambiguous += 1;
|
||||
}
|
||||
|
||||
private async repairTerminalAttempt(
|
||||
run: RunRecord,
|
||||
summary: LegacyShadowStartupReconcileSummary,
|
||||
): Promise<void> {
|
||||
const attempt = await this.repository.findLatestAttemptByRunId(run.id);
|
||||
const target = attempt ? terminalRunTarget(attempt) : undefined;
|
||||
if (
|
||||
!attempt ||
|
||||
target === undefined ||
|
||||
!RUN_TRANSITIONS[run.status].includes(target)
|
||||
) {
|
||||
summary.ambiguous += 1;
|
||||
return;
|
||||
}
|
||||
await this.commands.transitionRun({
|
||||
runId: run.id,
|
||||
to: target,
|
||||
expectedVersion: run.version,
|
||||
atMs: this.atOrAfter(run, attempt),
|
||||
...(attempt.errorCode === undefined
|
||||
? {}
|
||||
: { errorCode: attempt.errorCode }),
|
||||
...(attempt.errorSummary === undefined
|
||||
? {}
|
||||
: { errorSummary: attempt.errorSummary }),
|
||||
actor: { type: 'reconciler' },
|
||||
dedupeKey: `legacy-startup-repair:${attempt.id}:${target}`,
|
||||
});
|
||||
summary.repaired += 1;
|
||||
}
|
||||
|
||||
private async abandon(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
): Promise<void> {
|
||||
const atMs = this.atOrAfter(run, attempt);
|
||||
const attemptResult = await this.commands.transitionRunAttempt({
|
||||
runId: run.id,
|
||||
attemptId: attempt.id,
|
||||
to: 'cancelled',
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
errorCode: 'LEGACY_RECONCILE_ACCEPTANCE_ABANDONED',
|
||||
errorSummary: 'Legacy owner restarted before spawn was observed',
|
||||
actor: { type: 'reconciler' },
|
||||
dedupeKey: `legacy-startup-abandoned-attempt:${attempt.id}`,
|
||||
});
|
||||
await this.commands.transitionRun({
|
||||
runId: run.id,
|
||||
to: 'cancelled',
|
||||
expectedVersion: attemptResult.run.version,
|
||||
atMs,
|
||||
errorCode: 'LEGACY_RECONCILE_ACCEPTANCE_ABANDONED',
|
||||
errorSummary: 'Legacy owner restarted before spawn was observed',
|
||||
actor: { type: 'reconciler' },
|
||||
dedupeKey: `legacy-startup-abandoned-run:${attempt.id}`,
|
||||
});
|
||||
}
|
||||
|
||||
private async markLost(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
): Promise<void> {
|
||||
const atMs = this.atOrAfter(run, attempt);
|
||||
const attemptResult = await this.commands.transitionRunAttempt({
|
||||
runId: run.id,
|
||||
attemptId: attempt.id,
|
||||
to: 'lost',
|
||||
expectedRunVersion: run.version,
|
||||
atMs,
|
||||
errorCode: 'LEGACY_RECONCILE_OWNER_LOST',
|
||||
errorSummary: 'Legacy worker restarted without terminal evidence',
|
||||
actor: { type: 'reconciler' },
|
||||
dedupeKey: `legacy-startup-lost-attempt:${attempt.id}`,
|
||||
});
|
||||
await this.commands.transitionRun({
|
||||
runId: run.id,
|
||||
to: 'lost',
|
||||
expectedVersion: attemptResult.run.version,
|
||||
atMs,
|
||||
errorCode: 'LEGACY_RECONCILE_OWNER_LOST',
|
||||
errorSummary: 'Legacy worker restarted without terminal evidence',
|
||||
actor: { type: 'reconciler' },
|
||||
dedupeKey: `legacy-startup-lost-run:${attempt.id}`,
|
||||
});
|
||||
}
|
||||
|
||||
private atOrAfter(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
evidenceAtMs?: number,
|
||||
): number {
|
||||
return Math.max(
|
||||
this.clock.now(),
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs ?? 0,
|
||||
evidenceAtMs ?? 0,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs a complete but Profile-bounded, timer-free startup pass. */
|
||||
export class LegacyShadowStartupSupervisor {
|
||||
constructor(
|
||||
private readonly reconciler: Pick<
|
||||
LegacyShadowStartupReconciler,
|
||||
'reconcileBatch'
|
||||
>,
|
||||
) {}
|
||||
|
||||
async run(options: {
|
||||
origins: readonly RunRecord['executionOrigin'][];
|
||||
cursor?: LegacyShadowStartupCursor;
|
||||
pageSize?: number;
|
||||
maxPages?: number;
|
||||
}): Promise<LegacyShadowStartupSummary> {
|
||||
const pageSize = options.pageSize ?? 8;
|
||||
const maxPages = options.maxPages ?? 1;
|
||||
if (
|
||||
!Number.isSafeInteger(pageSize) ||
|
||||
pageSize < 1 ||
|
||||
pageSize > MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE
|
||||
) {
|
||||
throw new RangeError(
|
||||
'pageSize must be between 1 and MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE',
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(maxPages) ||
|
||||
maxPages < 1 ||
|
||||
maxPages > MAX_LEGACY_SHADOW_STARTUP_PAGES
|
||||
) {
|
||||
throw new RangeError(
|
||||
'maxPages must be between 1 and MAX_LEGACY_SHADOW_STARTUP_PAGES',
|
||||
);
|
||||
}
|
||||
|
||||
const total: LegacyShadowStartupSummary = {
|
||||
pages: 0,
|
||||
scanned: 0,
|
||||
completed: 0,
|
||||
cancelled: 0,
|
||||
abandoned: 0,
|
||||
markedLost: 0,
|
||||
repaired: 0,
|
||||
pending: 0,
|
||||
ambiguous: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
stopReason: 'complete',
|
||||
remaining: false,
|
||||
};
|
||||
let cursor = options.cursor;
|
||||
for (let pageNumber = 0; pageNumber < maxPages; pageNumber += 1) {
|
||||
const page = await this.reconciler.reconcileBatch({
|
||||
origins: options.origins,
|
||||
...(cursor === undefined ? {} : { cursor }),
|
||||
limit: pageSize,
|
||||
});
|
||||
total.pages += 1;
|
||||
total.scanned += page.scanned;
|
||||
total.completed += page.completed;
|
||||
total.cancelled += page.cancelled;
|
||||
total.abandoned += page.abandoned;
|
||||
total.markedLost += page.markedLost;
|
||||
total.repaired += page.repaired;
|
||||
total.pending += page.pending;
|
||||
total.ambiguous += page.ambiguous;
|
||||
total.skipped += page.skipped;
|
||||
total.failed += page.failed;
|
||||
if (!page.truncated) return total;
|
||||
if (!page.nextCursor || sameCursor(cursor, page.nextCursor)) {
|
||||
total.stopReason = 'cursor_stalled';
|
||||
total.remaining = true;
|
||||
return total;
|
||||
}
|
||||
cursor = page.nextCursor;
|
||||
if (pageNumber === maxPages - 1) {
|
||||
total.stopReason = 'page_limit';
|
||||
total.remaining = true;
|
||||
total.nextCursor = cursor;
|
||||
return total;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
}
|
||||
@@ -98,6 +98,10 @@ function readConfiguredOrigins(): ReadonlySet<ExecutionOrigin> {
|
||||
return configuredOrigins;
|
||||
}
|
||||
|
||||
export function configuredLegacyShadowOrigins(): readonly ExecutionOrigin[] {
|
||||
return [...readConfiguredOrigins()];
|
||||
}
|
||||
|
||||
async function createDefaultObserver(): Promise<LegacyExecutionObserver> {
|
||||
const [data, repositoryModule, observerModule, writerModule, rolloutModule] =
|
||||
await Promise.all([
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import type {
|
||||
ExecutionOrigin,
|
||||
RunAttemptStatus,
|
||||
RunStatus,
|
||||
} from '../domain/run';
|
||||
|
||||
export const MAX_LEGACY_SHADOW_STARTUP_BATCH_SIZE = 64;
|
||||
export const MAX_LEGACY_SHADOW_STARTUP_EVIDENCE = 8;
|
||||
|
||||
export interface LegacyShadowStartupCursor {
|
||||
createdAtMs: number;
|
||||
runId: string;
|
||||
}
|
||||
|
||||
export interface LegacyShadowStartupAttempt {
|
||||
attemptId: string;
|
||||
status: RunAttemptStatus;
|
||||
pid?: number;
|
||||
logArtifactId?: string;
|
||||
createdAtMs: number;
|
||||
startedAtMs?: number;
|
||||
}
|
||||
|
||||
export interface LegacyShadowStartupCandidate {
|
||||
runId: string;
|
||||
legacyCronId?: number;
|
||||
origin: ExecutionOrigin;
|
||||
runStatus: RunStatus;
|
||||
createdAtMs: number;
|
||||
activeAttemptCount: number;
|
||||
attempt?: LegacyShadowStartupAttempt;
|
||||
}
|
||||
|
||||
export interface LegacyShadowStartupPage {
|
||||
candidates: readonly LegacyShadowStartupCandidate[];
|
||||
truncated: boolean;
|
||||
nextCursor?: LegacyShadowStartupCursor;
|
||||
}
|
||||
|
||||
export type LegacyRunningInstanceOutcome =
|
||||
| 'running'
|
||||
| 'succeeded'
|
||||
| 'failed'
|
||||
| 'stopped';
|
||||
|
||||
export interface LegacyRunningInstanceEvidence {
|
||||
pid?: number;
|
||||
logArtifactId?: string;
|
||||
startedAtMs: number;
|
||||
finishedAtMs?: number;
|
||||
outcome: LegacyRunningInstanceOutcome;
|
||||
exitCode?: number;
|
||||
}
|
||||
|
||||
export interface LegacyRunningInstanceEvidencePage {
|
||||
evidence: readonly LegacyRunningInstanceEvidence[];
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
export interface LegacyShadowStartupRecoverySource {
|
||||
listCandidates(options: {
|
||||
origins: readonly ExecutionOrigin[];
|
||||
cursor?: LegacyShadowStartupCursor;
|
||||
limit?: number;
|
||||
}): Promise<LegacyShadowStartupPage>;
|
||||
|
||||
listRunningInstanceEvidence(options: {
|
||||
legacyCronId: number;
|
||||
limit?: number;
|
||||
}): Promise<LegacyRunningInstanceEvidencePage>;
|
||||
}
|
||||
Reference in New Issue
Block a user