mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 18:08:20 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,479 @@
|
||||
import { createHash, randomUUID, timingSafeEqual } from 'crypto';
|
||||
import type { LocalCompletionReceiptJournal } from '@qinglong/runtime-core/local-completion-receipt-journal';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
RunRepository,
|
||||
RunRepositoryTransaction,
|
||||
} from '@qinglong/runtime-core/run-repository';
|
||||
import {
|
||||
InvalidCompletionReceiptError,
|
||||
type CompletionReceipt,
|
||||
type CompletionReceiptStore,
|
||||
} from '@qinglong/local-process';
|
||||
import type { LocalWorkflowTaskExecutionRepository } from '../execution/workflowTaskExecution';
|
||||
|
||||
export const MAX_LOCAL_COMPLETION_QUARANTINE_RETENTION_MS = 24 * 60 * 60_000;
|
||||
|
||||
const ACTIVE_ATTEMPT_STATUSES = new Set(['claimed', 'starting', 'running']);
|
||||
const TERMINAL_ATTEMPT_STATUSES = new Set([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
'lost',
|
||||
]);
|
||||
const TERMINAL_RUN_STATUSES = new Set([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
'lost',
|
||||
]);
|
||||
|
||||
type MaintainedCompletionReceiptStore = CompletionReceiptStore & {
|
||||
quarantine?(attemptId: string): Promise<string | undefined>;
|
||||
quarantineReference?(attemptId: string): string;
|
||||
};
|
||||
|
||||
export type LocalCompletionDisposition =
|
||||
| 'completed'
|
||||
| 'already_terminal'
|
||||
| 'missing'
|
||||
| 'invalid'
|
||||
| 'stale';
|
||||
|
||||
export interface LocalCompletionReceiptProcessorOptions {
|
||||
readonly clock?: { now(): number };
|
||||
readonly createEventId?: () => string;
|
||||
readonly journal?: Pick<
|
||||
LocalCompletionReceiptJournal,
|
||||
'markQuarantined' | 'resolve'
|
||||
>;
|
||||
readonly quarantineRetentionMs?: number;
|
||||
readonly onDiagnostic?: (
|
||||
record: Readonly<{
|
||||
kind:
|
||||
| 'receipt_cleanup_failed'
|
||||
| 'receipt_quarantined'
|
||||
| 'journal_cleanup_failed';
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
}>,
|
||||
) => void | Promise<void>;
|
||||
readonly workflowTasks?: LocalWorkflowTaskExecutionRepository;
|
||||
}
|
||||
|
||||
interface AggregateSnapshot {
|
||||
readonly run: RunRecord;
|
||||
readonly attempt: RunAttemptRecord;
|
||||
}
|
||||
|
||||
interface TerminalMapping {
|
||||
readonly status: 'succeeded' | 'failed' | 'cancelled' | 'timed_out';
|
||||
readonly errorCode?: string;
|
||||
readonly errorSummary?: string;
|
||||
}
|
||||
|
||||
class LocalCompletionConcurrentWriteError extends Error {}
|
||||
|
||||
function timestamp(value: number, field: string): number {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new RangeError(`${field} must be a non-negative safe integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function atOrAfter(
|
||||
now: number,
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
): number {
|
||||
return Math.max(
|
||||
timestamp(now, 'Local completion observation'),
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs ?? 0,
|
||||
);
|
||||
}
|
||||
|
||||
function reserveEvent(run: RunRecord): Readonly<{
|
||||
run: RunRecord;
|
||||
sequence: number;
|
||||
}> {
|
||||
const version = run.version + 1;
|
||||
const sequence = run.eventSequence + 1;
|
||||
if (!Number.isSafeInteger(version) || !Number.isSafeInteger(sequence)) {
|
||||
throw new RangeError('Local completion aggregate counter overflowed');
|
||||
}
|
||||
return Object.freeze({
|
||||
run: { ...run, version, eventSequence: sequence },
|
||||
sequence,
|
||||
});
|
||||
}
|
||||
|
||||
function mapping(run: RunRecord, receipt: CompletionReceipt): TerminalMapping {
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
return run.cancelReason === 'timeout'
|
||||
? Object.freeze({
|
||||
status: 'timed_out' as const,
|
||||
errorCode: 'EXECUTION_TIMED_OUT',
|
||||
errorSummary: 'Execution exceeded its configured timeout',
|
||||
})
|
||||
: Object.freeze({
|
||||
status: 'cancelled' as const,
|
||||
errorCode: 'EXECUTION_CANCELLED',
|
||||
errorSummary: 'Execution was cancelled',
|
||||
});
|
||||
}
|
||||
return receipt.exitCode === 0
|
||||
? Object.freeze({ status: 'succeeded' as const })
|
||||
: Object.freeze({
|
||||
status: 'failed' as const,
|
||||
errorCode: 'EXECUTION_FAILED',
|
||||
errorSummary: 'Execution completed without success',
|
||||
});
|
||||
}
|
||||
|
||||
function authenticate(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
receipt: CompletionReceipt,
|
||||
terminal: boolean,
|
||||
): void {
|
||||
const allowedSequence = terminal
|
||||
? receipt.callbackSequence === attempt.callbackSequence ||
|
||||
receipt.callbackSequence === attempt.callbackSequence + 1
|
||||
: receipt.callbackSequence === attempt.callbackSequence + 1;
|
||||
if (
|
||||
run.executionOwner !== 'runtime' ||
|
||||
receipt.runId !== run.id ||
|
||||
receipt.attemptId !== attempt.id ||
|
||||
!allowedSequence ||
|
||||
receipt.startedAtMs < attempt.createdAtMs ||
|
||||
(attempt.startedAtMs !== undefined &&
|
||||
receipt.startedAtMs < attempt.startedAtMs) ||
|
||||
!attempt.callbackTokenHash ||
|
||||
!/^[a-f0-9]{64}$/.test(attempt.callbackTokenHash)
|
||||
) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt does not match durable Run authority',
|
||||
);
|
||||
}
|
||||
const expected = Buffer.from(attempt.callbackTokenHash, 'hex');
|
||||
const actual = createHash('sha256').update(receipt.token, 'utf8').digest();
|
||||
if (!timingSafeEqual(expected, actual)) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt token does not match durable authority',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function event(
|
||||
id: string,
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
sequence: number,
|
||||
type: string,
|
||||
dedupeKey: string,
|
||||
payload: Readonly<Record<string, unknown>>,
|
||||
atMs: number,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id,
|
||||
runId,
|
||||
attemptId,
|
||||
sequence,
|
||||
type,
|
||||
dedupeKey,
|
||||
actorType: 'executor',
|
||||
actorId: 'completion-receipt',
|
||||
payload,
|
||||
createdAtMs: atMs,
|
||||
};
|
||||
}
|
||||
|
||||
export class LocalCompletionReceiptProcessor {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createEventId: () => string;
|
||||
private readonly journal?: LocalCompletionReceiptProcessorOptions['journal'];
|
||||
private readonly quarantineRetentionMs: number;
|
||||
private readonly onDiagnostic?: LocalCompletionReceiptProcessorOptions['onDiagnostic'];
|
||||
private readonly workflowTasks:
|
||||
| LocalWorkflowTaskExecutionRepository
|
||||
| undefined;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly receipts: MaintainedCompletionReceiptStore,
|
||||
options: LocalCompletionReceiptProcessorOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
this.journal = options.journal;
|
||||
this.quarantineRetentionMs = options.quarantineRetentionMs ?? 60 * 60_000;
|
||||
if (
|
||||
!Number.isSafeInteger(this.quarantineRetentionMs) ||
|
||||
this.quarantineRetentionMs < 0 ||
|
||||
this.quarantineRetentionMs > MAX_LOCAL_COMPLETION_QUARANTINE_RETENTION_MS
|
||||
) {
|
||||
throw new RangeError('Local completion quarantine retention is invalid');
|
||||
}
|
||||
this.onDiagnostic = options.onDiagnostic;
|
||||
this.workflowTasks = options.workflowTasks;
|
||||
if (
|
||||
this.workflowTasks !== undefined &&
|
||||
typeof this.workflowTasks.complete !== 'function'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow Task completion repository is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async process(attemptId: string): Promise<LocalCompletionDisposition> {
|
||||
const snapshot = await this.load(attemptId);
|
||||
if (!snapshot || snapshot.attempt.executorType !== 'local_process') {
|
||||
return 'stale';
|
||||
}
|
||||
let receipt: CompletionReceipt | undefined;
|
||||
try {
|
||||
receipt = await this.receipts.read(attemptId);
|
||||
if (!receipt) return 'missing';
|
||||
authenticate(
|
||||
snapshot.run,
|
||||
snapshot.attempt,
|
||||
receipt,
|
||||
TERMINAL_RUN_STATUSES.has(snapshot.run.status) ||
|
||||
TERMINAL_ATTEMPT_STATUSES.has(snapshot.attempt.status),
|
||||
);
|
||||
} catch (error) {
|
||||
if (!(error instanceof InvalidCompletionReceiptError)) throw error;
|
||||
await this.quarantine(snapshot);
|
||||
return 'invalid';
|
||||
}
|
||||
|
||||
if (
|
||||
TERMINAL_RUN_STATUSES.has(snapshot.run.status) ||
|
||||
TERMINAL_ATTEMPT_STATUSES.has(snapshot.attempt.status)
|
||||
) {
|
||||
await this.cleanup(receipt);
|
||||
return 'already_terminal';
|
||||
}
|
||||
if (!ACTIVE_ATTEMPT_STATUSES.has(snapshot.attempt.status)) return 'stale';
|
||||
|
||||
if (snapshot.attempt.stepRunId !== undefined) {
|
||||
if (!this.workflowTasks) return 'stale';
|
||||
const terminal = mapping(snapshot.run, receipt);
|
||||
const disposition = await this.workflowTasks.complete({
|
||||
run: snapshot.run,
|
||||
attempt: snapshot.attempt,
|
||||
callbackSequence: receipt.callbackSequence,
|
||||
startedAtMs: receipt.startedAtMs,
|
||||
finishedAtMs: Math.max(
|
||||
atOrAfter(this.clock.now(), snapshot.run, snapshot.attempt),
|
||||
receipt.finishedAtMs,
|
||||
),
|
||||
exitCode: receipt.exitCode,
|
||||
terminalStatus: terminal.status,
|
||||
...(terminal.errorCode === undefined
|
||||
? {}
|
||||
: {
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
}),
|
||||
attemptEventId: this.createEventId(),
|
||||
syntheticStartMutationId: this.createEventId(),
|
||||
terminalStepMutationId: this.createEventId(),
|
||||
});
|
||||
if (
|
||||
disposition === 'completed' ||
|
||||
disposition === 'already_terminal'
|
||||
) {
|
||||
await this.cleanup(receipt);
|
||||
}
|
||||
return disposition;
|
||||
}
|
||||
|
||||
const completed = await this.repository.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(snapshot.run.id);
|
||||
const attempt = await transaction.findAttemptById(snapshot.attempt.id);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
run.version !== snapshot.run.version ||
|
||||
run.status !== snapshot.run.status ||
|
||||
attempt.status !== snapshot.attempt.status ||
|
||||
attempt.callbackSequence !== snapshot.attempt.callbackSequence ||
|
||||
attempt.runId !== run.id
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
authenticate(run, attempt, receipt, false);
|
||||
const terminal = mapping(run, receipt);
|
||||
const atMs = Math.max(
|
||||
atOrAfter(this.clock.now(), run, attempt),
|
||||
receipt.finishedAtMs,
|
||||
);
|
||||
const attemptReserved = reserveEvent(run);
|
||||
const nextAttempt: RunAttemptRecord = {
|
||||
...attempt,
|
||||
status: terminal.status,
|
||||
callbackSequence: receipt.callbackSequence,
|
||||
finishedAtMs: atMs,
|
||||
exitCode: receipt.exitCode,
|
||||
...(terminal.errorCode === undefined
|
||||
? {}
|
||||
: {
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
}),
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(attemptReserved.run, run.version))
|
||||
) {
|
||||
throw new LocalCompletionConcurrentWriteError();
|
||||
}
|
||||
if (
|
||||
!(await transaction.compareAndSetAttempt(nextAttempt, {
|
||||
status: attempt.status,
|
||||
callbackSequence: attempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new LocalCompletionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run.id,
|
||||
attempt.id,
|
||||
attemptReserved.sequence,
|
||||
`attempt.${terminal.status}`,
|
||||
`local-completion:${attempt.id}:${receipt.callbackSequence}:attempt`,
|
||||
Object.freeze({
|
||||
attempt_id: attempt.id,
|
||||
from_status: attempt.status,
|
||||
to_status: terminal.status,
|
||||
callback_sequence: receipt.callbackSequence,
|
||||
exit_code: receipt.exitCode,
|
||||
version: attemptReserved.run.version,
|
||||
}),
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
const runReserved = reserveEvent(attemptReserved.run);
|
||||
const nextRun: RunRecord = {
|
||||
...runReserved.run,
|
||||
status: terminal.status,
|
||||
finishedAtMs: atMs,
|
||||
...(terminal.errorCode === undefined
|
||||
? {}
|
||||
: {
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
}),
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
nextRun,
|
||||
attemptReserved.run.version,
|
||||
))
|
||||
) {
|
||||
throw new LocalCompletionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run.id,
|
||||
attempt.id,
|
||||
runReserved.sequence,
|
||||
`run.${terminal.status}`,
|
||||
`local-completion:${attempt.id}:${receipt.callbackSequence}:run`,
|
||||
Object.freeze({
|
||||
from_status: run.status,
|
||||
to_status: terminal.status,
|
||||
version: nextRun.version,
|
||||
}),
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
return true;
|
||||
});
|
||||
if (!completed) return 'stale';
|
||||
await this.cleanup(receipt);
|
||||
return 'completed';
|
||||
}
|
||||
|
||||
private load(attemptId: string): Promise<AggregateSnapshot | null> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const attempt = await transaction.findAttemptById(attemptId);
|
||||
if (!attempt) return null;
|
||||
const run = await transaction.findRunById(attempt.runId);
|
||||
return run ? Object.freeze({ run, attempt }) : null;
|
||||
});
|
||||
}
|
||||
|
||||
private async quarantine(snapshot: AggregateSnapshot): Promise<void> {
|
||||
const reference = this.receipts.quarantineReference?.(snapshot.attempt.id);
|
||||
if (reference && this.journal) {
|
||||
const updatedAtMs = atOrAfter(
|
||||
this.clock.now(),
|
||||
snapshot.run,
|
||||
snapshot.attempt,
|
||||
);
|
||||
const purgeAfterMs = timestamp(
|
||||
updatedAtMs + this.quarantineRetentionMs,
|
||||
'Local completion quarantine expiry',
|
||||
);
|
||||
await this.journal.markQuarantined({
|
||||
attemptId: snapshot.attempt.id,
|
||||
quarantineRef: reference,
|
||||
updatedAtMs,
|
||||
purgeAfterMs,
|
||||
});
|
||||
}
|
||||
const quarantined = await this.receipts.quarantine?.(snapshot.attempt.id);
|
||||
if (quarantined) {
|
||||
await this.diagnostic({
|
||||
kind: 'receipt_quarantined',
|
||||
runId: snapshot.run.id,
|
||||
attemptId: snapshot.attempt.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async cleanup(receipt: CompletionReceipt): Promise<void> {
|
||||
try {
|
||||
await this.receipts.remove(receipt.attemptId);
|
||||
} catch {
|
||||
await this.diagnostic({
|
||||
kind: 'receipt_cleanup_failed',
|
||||
runId: receipt.runId,
|
||||
attemptId: receipt.attemptId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!this.journal) return;
|
||||
try {
|
||||
await this.journal.resolve(receipt.attemptId);
|
||||
} catch {
|
||||
await this.diagnostic({
|
||||
kind: 'journal_cleanup_failed',
|
||||
runId: receipt.runId,
|
||||
attemptId: receipt.attemptId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async diagnostic(
|
||||
record: Parameters<NonNullable<typeof this.onDiagnostic>>[0],
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.onDiagnostic?.(Object.freeze(record));
|
||||
} catch {
|
||||
// Diagnostic failure cannot replace the durable completion result.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,696 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import {
|
||||
assertLocalExecutionControlLimit,
|
||||
normalizeLocalActiveExecutionCandidate,
|
||||
normalizeLocalExecutionControlCandidate,
|
||||
type LocalActiveExecutionCandidate,
|
||||
type LocalActiveExecutionCursor,
|
||||
type LocalExecutionControlCandidate,
|
||||
type LocalExecutionControlCursor,
|
||||
type LocalExecutionControlSource,
|
||||
} from '@qinglong/runtime-core/local-execution-control';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunCancellationReason,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
RunRepository,
|
||||
} from '@qinglong/runtime-core/run-repository';
|
||||
import type {
|
||||
LocalProcessController,
|
||||
LocalProcessStopResult,
|
||||
} from '@qinglong/local-process';
|
||||
import type { LocalCompletionReceiptProcessor } from './completion';
|
||||
import type { LocalWorkflowTaskExecutionRepository } from '../execution/workflowTaskExecution';
|
||||
|
||||
export const MAX_LOCAL_EXECUTION_CONTROL_PAGES = 16;
|
||||
|
||||
const ACTIVE_ATTEMPT_STATUSES = new Set(['claimed', 'starting', 'running']);
|
||||
const ACTIVE_RUN_STATUSES = new Set(['dispatching', 'running']);
|
||||
const TERMINAL_RUN_STATUSES = new Set([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
'lost',
|
||||
]);
|
||||
|
||||
export type LocalExecutionControlDisposition =
|
||||
| 'terminal'
|
||||
| 'cancel_requested'
|
||||
| 'stale'
|
||||
| 'remaining';
|
||||
|
||||
export interface LocalExecutionControlCoordinatorOptions {
|
||||
readonly clock?: { now(): number };
|
||||
readonly createEventId?: () => string;
|
||||
readonly workflowTasks?: LocalWorkflowTaskExecutionRepository;
|
||||
}
|
||||
|
||||
interface ActiveSnapshot {
|
||||
readonly run: RunRecord;
|
||||
readonly attempt: RunAttemptRecord;
|
||||
}
|
||||
|
||||
class LocalExecutionControlConcurrentWriteError extends Error {}
|
||||
|
||||
function timestamp(value: number, field: string): number {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new RangeError(`${field} must be a non-negative safe integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function reserveEvent(run: RunRecord): Readonly<{
|
||||
run: RunRecord;
|
||||
sequence: number;
|
||||
}> {
|
||||
const version = run.version + 1;
|
||||
const sequence = run.eventSequence + 1;
|
||||
if (!Number.isSafeInteger(version) || !Number.isSafeInteger(sequence)) {
|
||||
throw new RangeError(
|
||||
'Local execution control aggregate counter overflowed',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
run: { ...run, version, eventSequence: sequence },
|
||||
sequence,
|
||||
});
|
||||
}
|
||||
|
||||
function atOrAfter(
|
||||
now: number,
|
||||
run: RunRecord,
|
||||
attempt?: RunAttemptRecord,
|
||||
): number {
|
||||
return Math.max(
|
||||
timestamp(now, 'Local execution control observation'),
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt?.createdAtMs ?? 0,
|
||||
attempt?.startedAtMs ?? 0,
|
||||
);
|
||||
}
|
||||
|
||||
function event(
|
||||
id: string,
|
||||
run: RunRecord,
|
||||
attemptId: string,
|
||||
sequence: number,
|
||||
type: string,
|
||||
dedupeKey: string,
|
||||
payload: Readonly<Record<string, unknown>>,
|
||||
actorId: string,
|
||||
atMs: number,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id,
|
||||
runId: run.id,
|
||||
attemptId,
|
||||
sequence,
|
||||
type,
|
||||
dedupeKey,
|
||||
actorType: 'reconciler',
|
||||
actorId,
|
||||
payload,
|
||||
createdAtMs: atMs,
|
||||
};
|
||||
}
|
||||
|
||||
function terminalFor(reason: RunCancellationReason): Readonly<{
|
||||
status: 'cancelled' | 'timed_out';
|
||||
errorCode: string;
|
||||
errorSummary: string;
|
||||
}> {
|
||||
return reason === 'timeout'
|
||||
? Object.freeze({
|
||||
status: 'timed_out' as const,
|
||||
errorCode: 'EXECUTION_TIMED_OUT',
|
||||
errorSummary: 'Execution exceeded its configured timeout',
|
||||
})
|
||||
: Object.freeze({
|
||||
status: 'cancelled' as const,
|
||||
errorCode: 'EXECUTION_CANCELLED',
|
||||
errorSummary: 'Execution was cancelled',
|
||||
});
|
||||
}
|
||||
|
||||
function conclusiveStop(result: LocalProcessStopResult): boolean {
|
||||
return result.status === 'stopped' || result.status === 'already_exited';
|
||||
}
|
||||
|
||||
export class LocalExecutionControlCoordinator {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createEventId: () => string;
|
||||
private readonly workflowTasks:
|
||||
| LocalWorkflowTaskExecutionRepository
|
||||
| undefined;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly completions: Pick<
|
||||
LocalCompletionReceiptProcessor,
|
||||
'process'
|
||||
>,
|
||||
private readonly controller: Pick<LocalProcessController, 'stop'>,
|
||||
options: LocalExecutionControlCoordinatorOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
this.workflowTasks = options.workflowTasks;
|
||||
if (
|
||||
this.workflowTasks !== undefined &&
|
||||
(typeof this.workflowTasks.requestTimeout !== 'function' ||
|
||||
typeof this.workflowTasks.recordControlTerminal !== 'function')
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow Task execution control repository is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async process(
|
||||
value: LocalExecutionControlCandidate,
|
||||
): Promise<LocalExecutionControlDisposition> {
|
||||
let candidate = normalizeLocalExecutionControlCandidate(value);
|
||||
if (candidate.kind === 'deadline') {
|
||||
const current = await this.loadActive(
|
||||
candidate.runId,
|
||||
candidate.attemptId,
|
||||
);
|
||||
if (!current) return 'stale';
|
||||
const requested =
|
||||
current.attempt.stepRunId === undefined
|
||||
? await this.requestCancellation(
|
||||
candidate.runId,
|
||||
candidate.attemptId,
|
||||
'timeout',
|
||||
candidate.dueAtMs,
|
||||
true,
|
||||
)
|
||||
: this.workflowTasks === undefined
|
||||
? false
|
||||
: (await this.workflowTasks.requestTimeout({
|
||||
run: current.run,
|
||||
attempt: current.attempt,
|
||||
dueAtMs: candidate.dueAtMs,
|
||||
eventId: this.createEventId(),
|
||||
})) !== 'stale';
|
||||
if (!requested) return 'stale';
|
||||
candidate = Object.freeze({
|
||||
kind: 'cancellation',
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
dueAtMs: candidate.dueAtMs,
|
||||
cancelReason: 'timeout',
|
||||
});
|
||||
}
|
||||
|
||||
const completion = await this.completions.process(candidate.attemptId);
|
||||
if (completion === 'completed' || completion === 'already_terminal') {
|
||||
return 'terminal';
|
||||
}
|
||||
if (completion === 'invalid') return 'remaining';
|
||||
|
||||
const snapshot = await this.loadActive(
|
||||
candidate.runId,
|
||||
candidate.attemptId,
|
||||
);
|
||||
if (!snapshot) return 'stale';
|
||||
const reason =
|
||||
snapshot.attempt.stepRunId === undefined
|
||||
? snapshot.run.cancelReason
|
||||
: snapshot.run.cancelRequestedAtMs === undefined
|
||||
? candidate.cancelReason === 'timeout'
|
||||
? 'timeout'
|
||||
: undefined
|
||||
: snapshot.run.cancelReason;
|
||||
if (reason === undefined) return 'stale';
|
||||
if (snapshot.attempt.status === 'claimed') {
|
||||
return (await this.markTerminal(snapshot, reason))
|
||||
? 'terminal'
|
||||
: 'remaining';
|
||||
}
|
||||
if (!snapshot.attempt.executorHandle) return 'remaining';
|
||||
|
||||
const stopped = await this.controller.stop(snapshot.attempt.executorHandle);
|
||||
const lateCompletion = await this.completions.process(candidate.attemptId);
|
||||
if (
|
||||
lateCompletion === 'completed' ||
|
||||
lateCompletion === 'already_terminal'
|
||||
) {
|
||||
return 'terminal';
|
||||
}
|
||||
if (!conclusiveStop(stopped)) return 'remaining';
|
||||
return (await this.markTerminal(snapshot, reason))
|
||||
? 'terminal'
|
||||
: 'remaining';
|
||||
}
|
||||
|
||||
async requestShutdown(
|
||||
value: LocalActiveExecutionCandidate,
|
||||
requestedAtMs: number,
|
||||
): Promise<LocalExecutionControlDisposition> {
|
||||
const candidate = normalizeLocalActiveExecutionCandidate(value);
|
||||
const current = await this.loadActive(candidate.runId, candidate.attemptId);
|
||||
if (!current) return 'stale';
|
||||
const reason = current.run.cancelReason ?? 'shutdown';
|
||||
if (current.run.cancelRequestedAtMs === undefined) {
|
||||
const requested = await this.requestCancellation(
|
||||
candidate.runId,
|
||||
candidate.attemptId,
|
||||
reason,
|
||||
requestedAtMs,
|
||||
false,
|
||||
);
|
||||
if (!requested) return 'stale';
|
||||
}
|
||||
return this.process({
|
||||
kind: 'cancellation',
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
dueAtMs: current.run.cancelRequestedAtMs ?? requestedAtMs,
|
||||
cancelReason: reason,
|
||||
});
|
||||
}
|
||||
|
||||
private loadActive(
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
): Promise<ActiveSnapshot | null> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(runId);
|
||||
const attempt = await transaction.findAttemptById(attemptId);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
attempt.runId !== run.id ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
!ACTIVE_RUN_STATUSES.has(run.status) ||
|
||||
!ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
|
||||
attempt.executorType !== 'local_process'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
if (attempt.stepRunId === undefined) {
|
||||
const latest = await transaction.findLatestAttemptByRunId(run.id);
|
||||
if (latest?.id !== attempt.id) return null;
|
||||
}
|
||||
return Object.freeze({ run, attempt });
|
||||
});
|
||||
}
|
||||
|
||||
private requestCancellation(
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
reason: RunCancellationReason,
|
||||
requestedAtMs: number,
|
||||
requireDeadline: boolean,
|
||||
): Promise<boolean> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(runId);
|
||||
const attempt = await transaction.findAttemptById(attemptId);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
attempt.runId !== run.id ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
!ACTIVE_RUN_STATUSES.has(run.status) ||
|
||||
!ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
|
||||
attempt.executorType !== 'local_process' ||
|
||||
(attempt.stepRunId === undefined &&
|
||||
(await transaction.findLatestAttemptByRunId(run.id))?.id !==
|
||||
attempt.id)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
return run.cancelReason === reason;
|
||||
}
|
||||
const observedAtMs = timestamp(
|
||||
requestedAtMs,
|
||||
'Cancellation request time',
|
||||
);
|
||||
if (
|
||||
requireDeadline &&
|
||||
(attempt.deadlineAtMs === undefined ||
|
||||
attempt.deadlineAtMs > observedAtMs)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const atMs = atOrAfter(observedAtMs, run, attempt);
|
||||
const reserved = reserveEvent(run);
|
||||
const next: RunRecord = {
|
||||
...reserved.run,
|
||||
cancelRequestedAtMs: atMs,
|
||||
cancelReason: reason,
|
||||
};
|
||||
if (!(await transaction.compareAndSetRun(next, run.version))) {
|
||||
throw new LocalExecutionControlConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run,
|
||||
attempt.id,
|
||||
reserved.sequence,
|
||||
'run.cancel_requested',
|
||||
`local-control:${attempt.id}:${reason}:${run.version}`,
|
||||
Object.freeze({
|
||||
reason,
|
||||
from_status: run.status,
|
||||
version: next.version,
|
||||
}),
|
||||
requireDeadline ? 'local-deadline' : 'local-shutdown',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
private markTerminal(
|
||||
expected: ActiveSnapshot,
|
||||
reason: RunCancellationReason,
|
||||
): Promise<boolean> {
|
||||
if (expected.attempt.stepRunId !== undefined) {
|
||||
if (!this.workflowTasks) return Promise.resolve(false);
|
||||
const terminal = terminalFor(reason);
|
||||
const atMs = atOrAfter(this.clock.now(), expected.run, expected.attempt);
|
||||
return this.workflowTasks
|
||||
.recordControlTerminal({
|
||||
run: expected.run,
|
||||
attempt: expected.attempt,
|
||||
reason,
|
||||
terminalStatus: terminal.status,
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
finishedAtMs: atMs,
|
||||
attemptEventId: this.createEventId(),
|
||||
stepMutationId: this.createEventId(),
|
||||
})
|
||||
.then(
|
||||
(result) =>
|
||||
result === 'terminal' || result === 'already_terminal',
|
||||
);
|
||||
}
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(expected.run.id);
|
||||
const attempt = await transaction.findAttemptById(expected.attempt.id);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
run.version < expected.run.version ||
|
||||
attempt.runId !== run.id ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
!ACTIVE_RUN_STATUSES.has(run.status) ||
|
||||
!ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
|
||||
attempt.callbackSequence !== expected.attempt.callbackSequence ||
|
||||
attempt.executorHandle !== expected.attempt.executorHandle ||
|
||||
attempt.pid !== expected.attempt.pid ||
|
||||
run.cancelRequestedAtMs === undefined ||
|
||||
run.cancelReason !== reason ||
|
||||
(await transaction.findLatestAttemptByRunId(run.id))?.id !== attempt.id
|
||||
) {
|
||||
return TERMINAL_RUN_STATUSES.has(run?.status ?? 'created');
|
||||
}
|
||||
const terminal = terminalFor(reason);
|
||||
const atMs = atOrAfter(this.clock.now(), run, attempt);
|
||||
const attemptReserved = reserveEvent(run);
|
||||
const nextAttempt: RunAttemptRecord = {
|
||||
...attempt,
|
||||
status: terminal.status,
|
||||
finishedAtMs: atMs,
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(attemptReserved.run, run.version))
|
||||
) {
|
||||
throw new LocalExecutionControlConcurrentWriteError();
|
||||
}
|
||||
if (
|
||||
!(await transaction.compareAndSetAttempt(nextAttempt, {
|
||||
status: attempt.status,
|
||||
callbackSequence: attempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new LocalExecutionControlConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run,
|
||||
attempt.id,
|
||||
attemptReserved.sequence,
|
||||
`attempt.${terminal.status}`,
|
||||
`local-control:${attempt.id}:${attempt.callbackSequence}:attempt`,
|
||||
Object.freeze({
|
||||
attempt_id: attempt.id,
|
||||
from_status: attempt.status,
|
||||
to_status: terminal.status,
|
||||
reason,
|
||||
version: attemptReserved.run.version,
|
||||
}),
|
||||
'local-execution-control',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
const runReserved = reserveEvent(attemptReserved.run);
|
||||
const nextRun: RunRecord = {
|
||||
...runReserved.run,
|
||||
status: terminal.status,
|
||||
finishedAtMs: atMs,
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
nextRun,
|
||||
attemptReserved.run.version,
|
||||
))
|
||||
) {
|
||||
throw new LocalExecutionControlConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run,
|
||||
attempt.id,
|
||||
runReserved.sequence,
|
||||
`run.${terminal.status}`,
|
||||
`local-control:${attempt.id}:${attempt.callbackSequence}:run`,
|
||||
Object.freeze({
|
||||
from_status: run.status,
|
||||
to_status: terminal.status,
|
||||
reason,
|
||||
version: nextRun.version,
|
||||
}),
|
||||
'local-execution-control',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export interface LocalExecutionControlScanSummary {
|
||||
readonly scanned: number;
|
||||
readonly terminal: number;
|
||||
readonly cancelRequested: number;
|
||||
readonly stale: number;
|
||||
readonly remaining: number;
|
||||
readonly failed: number;
|
||||
readonly truncated: boolean;
|
||||
readonly nextCursor?: LocalExecutionControlCursor;
|
||||
}
|
||||
|
||||
export interface LocalExecutionDrainSummary {
|
||||
readonly scanned: number;
|
||||
readonly terminal: number;
|
||||
readonly remaining: number;
|
||||
readonly failed: number;
|
||||
readonly truncated: boolean;
|
||||
}
|
||||
|
||||
function controlCursor(candidate: LocalExecutionControlCandidate) {
|
||||
return Object.freeze({
|
||||
dueAtMs: candidate.dueAtMs,
|
||||
kind: candidate.kind,
|
||||
attemptId: candidate.attemptId,
|
||||
});
|
||||
}
|
||||
|
||||
function controlAdvances(
|
||||
previous: LocalExecutionControlCursor,
|
||||
next: LocalExecutionControlCursor,
|
||||
): boolean {
|
||||
return (
|
||||
next.dueAtMs > previous.dueAtMs ||
|
||||
(next.dueAtMs === previous.dueAtMs &&
|
||||
(next.kind > previous.kind ||
|
||||
(next.kind === previous.kind && next.attemptId > previous.attemptId)))
|
||||
);
|
||||
}
|
||||
|
||||
function activeCursor(candidate: LocalActiveExecutionCandidate) {
|
||||
return Object.freeze({
|
||||
attemptCreatedAtMs: candidate.attemptCreatedAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
});
|
||||
}
|
||||
|
||||
function activeAdvances(
|
||||
previous: LocalActiveExecutionCursor,
|
||||
next: LocalActiveExecutionCursor,
|
||||
): boolean {
|
||||
return (
|
||||
next.attemptCreatedAtMs > previous.attemptCreatedAtMs ||
|
||||
(next.attemptCreatedAtMs === previous.attemptCreatedAtMs &&
|
||||
next.attemptId > previous.attemptId)
|
||||
);
|
||||
}
|
||||
|
||||
export class LocalExecutionControlScanner {
|
||||
constructor(
|
||||
private readonly source: LocalExecutionControlSource,
|
||||
private readonly coordinator: Pick<
|
||||
LocalExecutionControlCoordinator,
|
||||
'process' | 'requestShutdown'
|
||||
>,
|
||||
private readonly clock: { now(): number } = { now: Date.now },
|
||||
) {}
|
||||
|
||||
async scan(options: {
|
||||
readonly limit: number;
|
||||
readonly cursor?: LocalExecutionControlCursor;
|
||||
}): Promise<LocalExecutionControlScanSummary> {
|
||||
assertLocalExecutionControlLimit(options.limit);
|
||||
const observedAtMs = timestamp(
|
||||
this.clock.now(),
|
||||
'Local execution control scan time',
|
||||
);
|
||||
const page = await this.source.listLocalExecutionControlCandidates({
|
||||
observedAtMs,
|
||||
limit: options.limit,
|
||||
...(options.cursor === undefined ? {} : { after: options.cursor }),
|
||||
});
|
||||
if (page.candidates.length > options.limit) {
|
||||
throw new RangeError('Local execution control source exceeded page size');
|
||||
}
|
||||
let previous = options.cursor;
|
||||
let terminal = 0;
|
||||
let cancelRequested = 0;
|
||||
let stale = 0;
|
||||
let remaining = 0;
|
||||
let failed = 0;
|
||||
for (const value of page.candidates) {
|
||||
const candidate = normalizeLocalExecutionControlCandidate(value);
|
||||
const cursor = controlCursor(candidate);
|
||||
if (previous && !controlAdvances(previous, cursor)) {
|
||||
throw new TypeError(
|
||||
'Local execution control page is not strictly ordered',
|
||||
);
|
||||
}
|
||||
previous = cursor;
|
||||
try {
|
||||
const disposition = await this.coordinator.process(candidate);
|
||||
if (disposition === 'terminal') terminal += 1;
|
||||
if (disposition === 'cancel_requested') cancelRequested += 1;
|
||||
if (disposition === 'stale') stale += 1;
|
||||
if (disposition === 'remaining') remaining += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
if (page.truncated && page.candidates.length === 0) {
|
||||
throw new TypeError(
|
||||
'Local execution control source returned an empty truncated page',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
scanned: page.candidates.length,
|
||||
terminal,
|
||||
cancelRequested,
|
||||
stale,
|
||||
remaining,
|
||||
failed,
|
||||
truncated: page.truncated,
|
||||
...(page.truncated && previous ? { nextCursor: previous } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
async drain(options: {
|
||||
readonly limit: number;
|
||||
readonly maxPages: number;
|
||||
}): Promise<LocalExecutionDrainSummary> {
|
||||
assertLocalExecutionControlLimit(options.limit);
|
||||
if (
|
||||
!Number.isSafeInteger(options.maxPages) ||
|
||||
options.maxPages < 1 ||
|
||||
options.maxPages > MAX_LOCAL_EXECUTION_CONTROL_PAGES
|
||||
) {
|
||||
throw new RangeError('Local execution drain page budget is invalid');
|
||||
}
|
||||
const requestedAtMs = timestamp(
|
||||
this.clock.now(),
|
||||
'Local execution drain time',
|
||||
);
|
||||
let cursor: LocalActiveExecutionCursor | undefined;
|
||||
let scanned = 0;
|
||||
let terminal = 0;
|
||||
let remaining = 0;
|
||||
let failed = 0;
|
||||
let truncated = false;
|
||||
for (let pageIndex = 0; pageIndex < options.maxPages; pageIndex += 1) {
|
||||
const page = await this.source.listLocalActiveExecutions({
|
||||
limit: options.limit,
|
||||
...(cursor === undefined ? {} : { after: cursor }),
|
||||
});
|
||||
if (page.candidates.length > options.limit) {
|
||||
throw new RangeError(
|
||||
'Local active execution source exceeded page size',
|
||||
);
|
||||
}
|
||||
let previous = cursor;
|
||||
for (const value of page.candidates) {
|
||||
const candidate = normalizeLocalActiveExecutionCandidate(value);
|
||||
const next = activeCursor(candidate);
|
||||
if (previous && !activeAdvances(previous, next)) {
|
||||
throw new TypeError(
|
||||
'Local active execution page is not strictly ordered',
|
||||
);
|
||||
}
|
||||
previous = next;
|
||||
scanned += 1;
|
||||
try {
|
||||
const disposition = await this.coordinator.requestShutdown(
|
||||
candidate,
|
||||
requestedAtMs,
|
||||
);
|
||||
if (disposition === 'terminal' || disposition === 'stale')
|
||||
terminal += 1;
|
||||
else remaining += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
truncated = page.truncated;
|
||||
if (!page.truncated) break;
|
||||
if (!previous) {
|
||||
throw new TypeError(
|
||||
'Local active execution source returned an empty truncated page',
|
||||
);
|
||||
}
|
||||
cursor = previous;
|
||||
}
|
||||
return Object.freeze({
|
||||
scanned,
|
||||
terminal,
|
||||
remaining,
|
||||
failed,
|
||||
truncated,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from './completion';
|
||||
export * from './control';
|
||||
export * from './lifecycle';
|
||||
@@ -0,0 +1,291 @@
|
||||
import type { LocalCompletionReceiptJournalCursor } from '@qinglong/runtime-core/local-completion-receipt-journal';
|
||||
import { assertLocalExecutionControlLimit } from '@qinglong/runtime-core/local-execution-control';
|
||||
import type {
|
||||
LocalCompletionReceiptCleanupScanner,
|
||||
LocalCompletionReceiptCleanupSummary,
|
||||
} from '@qinglong/local-process';
|
||||
import type { LocalCompletionReceiptProcessor } from './completion';
|
||||
import type {
|
||||
LocalExecutionControlScanSummary,
|
||||
LocalExecutionControlScanner,
|
||||
LocalExecutionDrainSummary,
|
||||
} from './control';
|
||||
|
||||
export const MAX_LOCAL_COMPLETION_NOTIFICATIONS = 64;
|
||||
|
||||
export interface LocalExecutionControlLifecycleOptions {
|
||||
readonly intervalMs: number;
|
||||
readonly pageSize: number;
|
||||
readonly cleanupIntervalMs: number;
|
||||
readonly cleanupPageSize: number;
|
||||
readonly stopTimeoutMs: number;
|
||||
readonly maxDrainPages: number;
|
||||
readonly maxNotifications?: number;
|
||||
readonly clock?: { now(): number };
|
||||
readonly onDiagnostic?: (
|
||||
error: unknown,
|
||||
summary?: LocalExecutionControlCycleSummary,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface LocalExecutionControlCycleSummary {
|
||||
readonly completions: number;
|
||||
readonly completionFailures: number;
|
||||
readonly control: LocalExecutionControlScanSummary;
|
||||
readonly cleanup?: LocalCompletionReceiptCleanupSummary;
|
||||
}
|
||||
|
||||
export interface LocalExecutionControlStopSummary {
|
||||
readonly status: 'stopped' | 'timed_out';
|
||||
readonly drain?: LocalExecutionDrainSummary;
|
||||
readonly cleanup?: LocalCompletionReceiptCleanupSummary;
|
||||
}
|
||||
|
||||
export class LocalExecutionControlLifecycle {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly maxNotifications: number;
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private inFlight: Promise<LocalExecutionControlCycleSummary> | undefined;
|
||||
private stopPromise: Promise<LocalExecutionControlStopSummary> | undefined;
|
||||
private running = false;
|
||||
private stopping = false;
|
||||
private kickQueued = false;
|
||||
private controlCursor:
|
||||
| NonNullable<LocalExecutionControlScanSummary['nextCursor']>
|
||||
| undefined;
|
||||
private cleanupCursor: LocalCompletionReceiptJournalCursor | undefined;
|
||||
private lastCleanupAtMs: number | undefined;
|
||||
private readonly pending = new Set<string>();
|
||||
|
||||
constructor(
|
||||
private readonly completions: Pick<
|
||||
LocalCompletionReceiptProcessor,
|
||||
'process'
|
||||
>,
|
||||
private readonly control: Pick<
|
||||
LocalExecutionControlScanner,
|
||||
'scan' | 'drain'
|
||||
>,
|
||||
private readonly cleanup: Pick<
|
||||
LocalCompletionReceiptCleanupScanner,
|
||||
'scan'
|
||||
>,
|
||||
private readonly options: LocalExecutionControlLifecycleOptions,
|
||||
) {
|
||||
if (
|
||||
!Number.isSafeInteger(options.intervalMs) ||
|
||||
options.intervalMs < 250 ||
|
||||
options.intervalMs > 60 * 60_000
|
||||
) {
|
||||
throw new RangeError('Local execution control interval is invalid');
|
||||
}
|
||||
assertLocalExecutionControlLimit(options.pageSize);
|
||||
if (
|
||||
!Number.isSafeInteger(options.cleanupIntervalMs) ||
|
||||
options.cleanupIntervalMs < 1_000 ||
|
||||
options.cleanupIntervalMs > 24 * 60 * 60_000
|
||||
) {
|
||||
throw new RangeError('Local execution cleanup interval is invalid');
|
||||
}
|
||||
assertLocalExecutionControlLimit(options.cleanupPageSize);
|
||||
if (
|
||||
!Number.isSafeInteger(options.stopTimeoutMs) ||
|
||||
options.stopTimeoutMs < 100 ||
|
||||
options.stopTimeoutMs > 30_000
|
||||
) {
|
||||
throw new RangeError('Local execution control stop timeout is invalid');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(options.maxDrainPages) ||
|
||||
options.maxDrainPages < 1 ||
|
||||
options.maxDrainPages > 16
|
||||
) {
|
||||
throw new RangeError('Local execution control drain budget is invalid');
|
||||
}
|
||||
this.maxNotifications =
|
||||
options.maxNotifications ?? MAX_LOCAL_COMPLETION_NOTIFICATIONS;
|
||||
if (
|
||||
!Number.isSafeInteger(this.maxNotifications) ||
|
||||
this.maxNotifications < 1 ||
|
||||
this.maxNotifications > MAX_LOCAL_COMPLETION_NOTIFICATIONS
|
||||
) {
|
||||
throw new RangeError('Local completion notification budget is invalid');
|
||||
}
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.running || this.stopping) return;
|
||||
this.running = true;
|
||||
this.schedule();
|
||||
}
|
||||
|
||||
notifyCompletion(attemptId: string): boolean {
|
||||
if (
|
||||
this.stopping ||
|
||||
typeof attemptId !== 'string' ||
|
||||
attemptId.length > 128
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
!this.pending.has(attemptId) &&
|
||||
this.pending.size >= this.maxNotifications
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
this.pending.add(attemptId);
|
||||
this.kick();
|
||||
return true;
|
||||
}
|
||||
|
||||
runOnce(forceCleanup = false): Promise<LocalExecutionControlCycleSummary> {
|
||||
if (this.inFlight) return this.inFlight;
|
||||
const work = this.cycle(forceCleanup).finally(() => {
|
||||
if (this.inFlight === work) this.inFlight = undefined;
|
||||
});
|
||||
this.inFlight = work;
|
||||
return work;
|
||||
}
|
||||
|
||||
stopAndDrain(): Promise<LocalExecutionControlStopSummary> {
|
||||
if (this.stopPromise) return this.stopPromise;
|
||||
this.stopping = true;
|
||||
this.running = false;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
|
||||
const stopWork = (async () => {
|
||||
await this.inFlight;
|
||||
await this.processPending();
|
||||
const drain = await this.control.drain({
|
||||
limit: this.options.pageSize,
|
||||
maxPages: this.options.maxDrainPages,
|
||||
});
|
||||
const cleanup = await this.cleanup.scan({
|
||||
limit: this.options.cleanupPageSize,
|
||||
...(this.cleanupCursor === undefined
|
||||
? {}
|
||||
: { cursor: this.cleanupCursor }),
|
||||
});
|
||||
this.cleanupCursor = cleanup.truncated ? cleanup.nextCursor : undefined;
|
||||
return Object.freeze({
|
||||
status:
|
||||
drain.remaining === 0 && drain.failed === 0 && !drain.truncated
|
||||
? ('stopped' as const)
|
||||
: ('timed_out' as const),
|
||||
drain,
|
||||
cleanup,
|
||||
});
|
||||
})();
|
||||
this.stopPromise = Promise.race([
|
||||
stopWork,
|
||||
new Promise<LocalExecutionControlStopSummary>((resolve) => {
|
||||
const timer = setTimeout(
|
||||
() => resolve(Object.freeze({ status: 'timed_out' as const })),
|
||||
this.options.stopTimeoutMs,
|
||||
);
|
||||
timer.unref?.();
|
||||
}),
|
||||
]);
|
||||
return this.stopPromise;
|
||||
}
|
||||
|
||||
private schedule(): void {
|
||||
if (!this.running || this.timer) return;
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = undefined;
|
||||
if (!this.running) return;
|
||||
void this.runOnce()
|
||||
.then((summary) => this.diagnostic(undefined, summary))
|
||||
.catch((error) => this.diagnostic(error))
|
||||
.finally(() => this.schedule());
|
||||
}, this.options.intervalMs);
|
||||
this.timer.unref?.();
|
||||
}
|
||||
|
||||
private kick(): void {
|
||||
if (this.kickQueued || this.inFlight || this.stopping) return;
|
||||
this.kickQueued = true;
|
||||
queueMicrotask(() => {
|
||||
this.kickQueued = false;
|
||||
if (this.stopping || this.inFlight) return;
|
||||
void this.runOnce()
|
||||
.then((summary) => this.diagnostic(undefined, summary))
|
||||
.catch((error) => this.diagnostic(error));
|
||||
});
|
||||
}
|
||||
|
||||
private async cycle(
|
||||
forceCleanup: boolean,
|
||||
): Promise<LocalExecutionControlCycleSummary> {
|
||||
const completion = await this.processPending();
|
||||
const control = await this.control.scan({
|
||||
limit: this.options.pageSize,
|
||||
...(this.controlCursor === undefined
|
||||
? {}
|
||||
: { cursor: this.controlCursor }),
|
||||
});
|
||||
this.controlCursor = control.truncated ? control.nextCursor : undefined;
|
||||
const now = this.clock.now();
|
||||
if (!Number.isSafeInteger(now) || now < 0) {
|
||||
throw new RangeError(
|
||||
'Local execution control lifecycle clock is invalid',
|
||||
);
|
||||
}
|
||||
let cleanup: LocalCompletionReceiptCleanupSummary | undefined;
|
||||
if (
|
||||
forceCleanup ||
|
||||
this.lastCleanupAtMs === undefined ||
|
||||
now - this.lastCleanupAtMs >= this.options.cleanupIntervalMs
|
||||
) {
|
||||
cleanup = await this.cleanup.scan({
|
||||
limit: this.options.cleanupPageSize,
|
||||
...(this.cleanupCursor === undefined
|
||||
? {}
|
||||
: { cursor: this.cleanupCursor }),
|
||||
});
|
||||
this.cleanupCursor = cleanup.truncated ? cleanup.nextCursor : undefined;
|
||||
this.lastCleanupAtMs = now;
|
||||
}
|
||||
if (this.pending.size > 0) this.kick();
|
||||
return Object.freeze({
|
||||
completions: completion.processed,
|
||||
completionFailures: completion.failed,
|
||||
control,
|
||||
...(cleanup === undefined ? {} : { cleanup }),
|
||||
});
|
||||
}
|
||||
|
||||
private async processPending(): Promise<
|
||||
Readonly<{
|
||||
processed: number;
|
||||
failed: number;
|
||||
}>
|
||||
> {
|
||||
const batch = [...this.pending].slice(0, this.maxNotifications);
|
||||
for (const attemptId of batch) this.pending.delete(attemptId);
|
||||
let processed = 0;
|
||||
let failed = 0;
|
||||
for (const attemptId of batch) {
|
||||
try {
|
||||
await this.completions.process(attemptId);
|
||||
processed += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
return Object.freeze({ processed, failed });
|
||||
}
|
||||
|
||||
private async diagnostic(
|
||||
error: unknown,
|
||||
summary?: LocalExecutionControlCycleSummary,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.options.onDiagnostic?.(error, summary);
|
||||
} catch {
|
||||
// Diagnostics cannot own execution-control liveness.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { constants } from 'node:fs';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import type { LocalDispatchCandidate } from '@qinglong/runtime-core/local-dispatch';
|
||||
import { normalizeLocalDispatchCandidate } from '@qinglong/runtime-core/local-dispatch';
|
||||
|
||||
export const MIN_LOCAL_ARTIFACT_MAXIMUM_BYTES = 64 * 1024;
|
||||
export const MAX_LOCAL_ARTIFACT_MAXIMUM_BYTES = 1024 * 1024 * 1024;
|
||||
export const MAX_LOCAL_ARTIFACT_MINIMUM_FREE_BYTES = 1024 ** 4;
|
||||
|
||||
export interface LocalArtifactCapacityPolicy {
|
||||
readonly maximumAttemptBytes: number;
|
||||
readonly minimumFreeBytes: number;
|
||||
}
|
||||
|
||||
export interface LocalArtifactCapacityProbe {
|
||||
inspect(directory: string): Promise<bigint>;
|
||||
}
|
||||
|
||||
export interface PreparedLocalArtifact {
|
||||
readonly logArtifactId: string;
|
||||
readonly output: Readonly<{
|
||||
filePath: string;
|
||||
maximumBytes: number;
|
||||
logArtifactId: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface LocalArtifactAllocator {
|
||||
prepare(candidate: LocalDispatchCandidate): Promise<PreparedLocalArtifact>;
|
||||
}
|
||||
|
||||
export class LocalArtifactCapacityUnavailableError extends Error {
|
||||
readonly code = 'LOCAL_ARTIFACT_CAPACITY_UNAVAILABLE';
|
||||
|
||||
constructor() {
|
||||
super('Local Artifact capacity is unavailable');
|
||||
this.name = 'LocalArtifactCapacityUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalArtifactIdentityConflictError extends Error {
|
||||
readonly code = 'LOCAL_ARTIFACT_IDENTITY_CONFLICT';
|
||||
|
||||
constructor() {
|
||||
super('Local Artifact identity is already occupied');
|
||||
this.name = 'LocalArtifactIdentityConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
class StatFsLocalArtifactCapacityProbe implements LocalArtifactCapacityProbe {
|
||||
async inspect(directory: string): Promise<bigint> {
|
||||
const stat = await fs.statfs(directory, { bigint: true });
|
||||
return stat.bavail * stat.bsize;
|
||||
}
|
||||
}
|
||||
|
||||
function assertAbsoluteRoot(value: string): string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
!path.isAbsolute(value) ||
|
||||
path.parse(value).root === value ||
|
||||
value.includes('\0') ||
|
||||
Buffer.byteLength(value, 'utf8') > 4096
|
||||
) {
|
||||
throw new TypeError('Local Artifact root is invalid');
|
||||
}
|
||||
return path.resolve(value);
|
||||
}
|
||||
|
||||
function normalizePolicy(
|
||||
policy: LocalArtifactCapacityPolicy,
|
||||
): Readonly<LocalArtifactCapacityPolicy> {
|
||||
if (!policy || typeof policy !== 'object' || Array.isArray(policy)) {
|
||||
throw new TypeError('Local Artifact capacity policy is invalid');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(policy.maximumAttemptBytes) ||
|
||||
policy.maximumAttemptBytes < MIN_LOCAL_ARTIFACT_MAXIMUM_BYTES ||
|
||||
policy.maximumAttemptBytes > MAX_LOCAL_ARTIFACT_MAXIMUM_BYTES ||
|
||||
!Number.isSafeInteger(policy.minimumFreeBytes) ||
|
||||
policy.minimumFreeBytes < 0 ||
|
||||
policy.minimumFreeBytes > MAX_LOCAL_ARTIFACT_MINIMUM_FREE_BYTES
|
||||
) {
|
||||
throw new RangeError('Local Artifact capacity policy is out of range');
|
||||
}
|
||||
return Object.freeze({ ...policy });
|
||||
}
|
||||
|
||||
async function ensurePrivateDirectory(directory: string): Promise<void> {
|
||||
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const stat = await fs.lstat(directory);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
||||
throw new TypeError('Local Artifact directory is unsafe');
|
||||
}
|
||||
await fs.chmod(directory, 0o700);
|
||||
}
|
||||
|
||||
export function localArtifactCapacityPolicyForProfile(
|
||||
profile: 'edge' | 'standalone',
|
||||
): Readonly<LocalArtifactCapacityPolicy> {
|
||||
if (profile === 'edge') {
|
||||
return Object.freeze({
|
||||
maximumAttemptBytes: 4 * 1024 * 1024,
|
||||
minimumFreeBytes: 32 * 1024 * 1024,
|
||||
});
|
||||
}
|
||||
if (profile === 'standalone') {
|
||||
return Object.freeze({
|
||||
maximumAttemptBytes: 64 * 1024 * 1024,
|
||||
minimumFreeBytes: 256 * 1024 * 1024,
|
||||
});
|
||||
}
|
||||
throw new TypeError('Local Artifact Profile is invalid');
|
||||
}
|
||||
|
||||
export function localArtifactId(candidate: LocalDispatchCandidate): string {
|
||||
const normalized = normalizeLocalDispatchCandidate(candidate);
|
||||
return `local-${createHash('sha256')
|
||||
.update(normalized.runId, 'utf8')
|
||||
.update('\0', 'utf8')
|
||||
.update(normalized.attemptId, 'utf8')
|
||||
.digest('hex')
|
||||
.slice(0, 30)}`;
|
||||
}
|
||||
|
||||
export class LocalFileArtifactAllocator implements LocalArtifactAllocator {
|
||||
private readonly root: string;
|
||||
private readonly policy: Readonly<LocalArtifactCapacityPolicy>;
|
||||
|
||||
constructor(
|
||||
root: string,
|
||||
policy: LocalArtifactCapacityPolicy,
|
||||
private readonly capacity: LocalArtifactCapacityProbe = new StatFsLocalArtifactCapacityProbe(),
|
||||
) {
|
||||
this.root = assertAbsoluteRoot(root);
|
||||
this.policy = normalizePolicy(policy);
|
||||
}
|
||||
|
||||
async prepare(
|
||||
candidate: LocalDispatchCandidate,
|
||||
): Promise<PreparedLocalArtifact> {
|
||||
const normalized = normalizeLocalDispatchCandidate(candidate);
|
||||
const logArtifactId = localArtifactId(normalized);
|
||||
const shard = logArtifactId.slice(6, 8);
|
||||
const directory = path.join(this.root, shard);
|
||||
await ensurePrivateDirectory(this.root);
|
||||
const availableBytes = await this.capacity.inspect(this.root);
|
||||
const requiredBytes =
|
||||
BigInt(this.policy.minimumFreeBytes) +
|
||||
BigInt(this.policy.maximumAttemptBytes);
|
||||
if (availableBytes < requiredBytes) {
|
||||
throw new LocalArtifactCapacityUnavailableError();
|
||||
}
|
||||
await ensurePrivateDirectory(directory);
|
||||
const filePath = path.join(directory, `${logArtifactId}.log`);
|
||||
let file: fs.FileHandle | undefined;
|
||||
try {
|
||||
file = await fs.open(
|
||||
filePath,
|
||||
constants.O_WRONLY |
|
||||
constants.O_CREAT |
|
||||
constants.O_APPEND |
|
||||
(constants.O_NOFOLLOW ?? 0),
|
||||
0o600,
|
||||
);
|
||||
const stat = await file.stat();
|
||||
if (!stat.isFile() || stat.size !== 0) {
|
||||
throw new LocalArtifactIdentityConflictError();
|
||||
}
|
||||
await file.chmod(0o600);
|
||||
} finally {
|
||||
await file?.close().catch(() => undefined);
|
||||
}
|
||||
return Object.freeze({
|
||||
logArtifactId,
|
||||
output: Object.freeze({
|
||||
filePath,
|
||||
maximumBytes: this.policy.maximumAttemptBytes,
|
||||
logArtifactId,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import {
|
||||
LOCAL_PROCESS_EXECUTOR_TYPE,
|
||||
assertLocalDispatchPageSize,
|
||||
normalizeLocalDispatchCandidate,
|
||||
type LocalDispatchCandidate,
|
||||
type LocalDispatchCandidateCursor,
|
||||
type LocalDispatchCandidateSource,
|
||||
} from '@qinglong/runtime-core/local-dispatch';
|
||||
import {
|
||||
LocalExecutionLaunchError,
|
||||
LocalExecutionRejectedError,
|
||||
type LocalExecutionStartCommand,
|
||||
type LocalExecutionStartResult,
|
||||
} from '../execution/coordinator';
|
||||
import type { LocalDispatchPlanSource } from './materializer';
|
||||
|
||||
export interface LocalDispatchActivator {
|
||||
start(
|
||||
command: LocalExecutionStartCommand,
|
||||
): Promise<LocalExecutionStartResult>;
|
||||
}
|
||||
|
||||
export interface LocalRunDispatcherOptions {
|
||||
readonly pageSize?: number;
|
||||
readonly maxPages?: number;
|
||||
readonly onCompletion?: (attemptId: string) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface LocalRunDispatcherStats {
|
||||
readonly pages: number;
|
||||
readonly candidatesScanned: number;
|
||||
readonly plansUnavailable: number;
|
||||
readonly activationRaces: number;
|
||||
}
|
||||
|
||||
export type LocalRunDispatcherResult =
|
||||
| Readonly<{
|
||||
status: 'activated';
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
}>
|
||||
| Readonly<{
|
||||
status: 'activation_failed';
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
}>
|
||||
| Readonly<{
|
||||
status: 'idle';
|
||||
reason:
|
||||
| 'no_candidates'
|
||||
| 'plans_unavailable'
|
||||
| 'activation_raced'
|
||||
| 'scan_budget_exhausted';
|
||||
stats: LocalRunDispatcherStats;
|
||||
truncated: boolean;
|
||||
}>;
|
||||
|
||||
function cursorOf(
|
||||
candidate: LocalDispatchCandidate,
|
||||
): LocalDispatchCandidateCursor {
|
||||
return Object.freeze({
|
||||
priority: candidate.priority,
|
||||
queuedAtMs: candidate.queuedAtMs,
|
||||
attemptCreatedAtMs: candidate.attemptCreatedAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
});
|
||||
}
|
||||
|
||||
function advances(
|
||||
previous: LocalDispatchCandidateCursor,
|
||||
next: LocalDispatchCandidateCursor,
|
||||
): boolean {
|
||||
return (
|
||||
next.priority < previous.priority ||
|
||||
(next.priority === previous.priority &&
|
||||
(next.queuedAtMs > previous.queuedAtMs ||
|
||||
(next.queuedAtMs === previous.queuedAtMs &&
|
||||
(next.attemptCreatedAtMs > previous.attemptCreatedAtMs ||
|
||||
(next.attemptCreatedAtMs === previous.attemptCreatedAtMs &&
|
||||
next.attemptId > previous.attemptId)))))
|
||||
);
|
||||
}
|
||||
|
||||
export class LocalRunDispatcher {
|
||||
private readonly pageSize: number;
|
||||
private readonly maxPages: number;
|
||||
private readonly onCompletion?: LocalRunDispatcherOptions['onCompletion'];
|
||||
|
||||
constructor(
|
||||
private readonly candidates: LocalDispatchCandidateSource,
|
||||
private readonly plans: LocalDispatchPlanSource,
|
||||
private readonly activator: LocalDispatchActivator,
|
||||
options: LocalRunDispatcherOptions = {},
|
||||
) {
|
||||
this.pageSize = options.pageSize ?? 8;
|
||||
this.maxPages = options.maxPages ?? 1;
|
||||
this.onCompletion = options.onCompletion;
|
||||
assertLocalDispatchPageSize(this.pageSize);
|
||||
if (
|
||||
!Number.isSafeInteger(this.maxPages) ||
|
||||
this.maxPages < 1 ||
|
||||
this.maxPages > 16
|
||||
) {
|
||||
throw new RangeError('Local dispatch maxPages must be between 1 and 16');
|
||||
}
|
||||
}
|
||||
|
||||
async dispatchOnce(): Promise<LocalRunDispatcherResult> {
|
||||
const stats = {
|
||||
pages: 0,
|
||||
candidatesScanned: 0,
|
||||
plansUnavailable: 0,
|
||||
activationRaces: 0,
|
||||
};
|
||||
const seen = new Set<string>();
|
||||
let after: LocalDispatchCandidateCursor | undefined;
|
||||
let truncated = false;
|
||||
for (let pageIndex = 0; pageIndex < this.maxPages; pageIndex += 1) {
|
||||
const page = await this.candidates.listLocalDispatchCandidates({
|
||||
limit: this.pageSize,
|
||||
...(after === undefined ? {} : { after }),
|
||||
});
|
||||
if (page.candidates.length > this.pageSize) {
|
||||
throw new RangeError('Local dispatch source exceeded its page size');
|
||||
}
|
||||
stats.pages += 1;
|
||||
truncated = page.truncated;
|
||||
let previous = after;
|
||||
for (const value of page.candidates) {
|
||||
const candidate = normalizeLocalDispatchCandidate(value);
|
||||
if (candidate.executorType !== LOCAL_PROCESS_EXECUTOR_TYPE) {
|
||||
throw new TypeError(
|
||||
'Local dispatch source returned another executor',
|
||||
);
|
||||
}
|
||||
const cursor = cursorOf(candidate);
|
||||
if (
|
||||
seen.has(candidate.attemptId) ||
|
||||
(previous !== undefined && !advances(previous, cursor))
|
||||
) {
|
||||
throw new TypeError('Local dispatch page is not strictly ordered');
|
||||
}
|
||||
seen.add(candidate.attemptId);
|
||||
previous = cursor;
|
||||
stats.candidatesScanned += 1;
|
||||
const plan = await this.plans.prepare(candidate);
|
||||
if (!plan) {
|
||||
stats.plansUnavailable += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const active = await this.activator.start(plan.command);
|
||||
void active.handle.completion
|
||||
.then(
|
||||
() => this.onCompletion?.(active.attempt.id),
|
||||
() => this.onCompletion?.(active.attempt.id),
|
||||
)
|
||||
.catch(() => undefined);
|
||||
return Object.freeze({
|
||||
status: 'activated' as const,
|
||||
runId: active.run.id,
|
||||
attemptId: active.attempt.id,
|
||||
stats: Object.freeze({ ...stats }),
|
||||
truncated,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof LocalExecutionRejectedError) {
|
||||
if (
|
||||
error.reason === 'aggregate_mismatch' ||
|
||||
error.reason === 'executor_mismatch'
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
stats.activationRaces += 1;
|
||||
continue;
|
||||
}
|
||||
if (error instanceof LocalExecutionLaunchError) {
|
||||
return Object.freeze({
|
||||
status: 'activation_failed' as const,
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
stats: Object.freeze({ ...stats }),
|
||||
truncated,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (!page.truncated) return this.idle(stats, false);
|
||||
const last = page.candidates.at(-1);
|
||||
if (!last) {
|
||||
throw new TypeError(
|
||||
'Local dispatch source reported an empty truncated page',
|
||||
);
|
||||
}
|
||||
after = cursorOf(last);
|
||||
}
|
||||
return Object.freeze({
|
||||
status: 'idle' as const,
|
||||
reason: 'scan_budget_exhausted' as const,
|
||||
stats: Object.freeze({ ...stats }),
|
||||
truncated: true,
|
||||
});
|
||||
}
|
||||
|
||||
private idle(
|
||||
stats: LocalRunDispatcherStats,
|
||||
truncated: boolean,
|
||||
): LocalRunDispatcherResult {
|
||||
const reason =
|
||||
stats.candidatesScanned === 0
|
||||
? 'no_candidates'
|
||||
: stats.plansUnavailable === stats.candidatesScanned
|
||||
? 'plans_unavailable'
|
||||
: 'activation_raced';
|
||||
return Object.freeze({
|
||||
status: 'idle' as const,
|
||||
reason,
|
||||
stats: Object.freeze({ ...stats }),
|
||||
truncated,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export * from './artifact';
|
||||
export * from './materializer';
|
||||
export * from './dispatcher';
|
||||
export type {
|
||||
LocalDispatchDefinitionWriter,
|
||||
LocalDispatchStore,
|
||||
LocalSecretEnvironmentProvider,
|
||||
} from '@qinglong/runtime-core/local-dispatch';
|
||||
@@ -0,0 +1,133 @@
|
||||
import type {
|
||||
LocalDispatchCandidate,
|
||||
LocalDispatchStore,
|
||||
LocalSecretEnvironmentProvider,
|
||||
} from '@qinglong/runtime-core/local-dispatch';
|
||||
import {
|
||||
MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES,
|
||||
MAX_LOCAL_DISPATCH_SECRET_REFS,
|
||||
normalizeLocalDispatchCandidate,
|
||||
normalizeLocalExecutionContextRecipe,
|
||||
normalizeLocalTaskExecutionRevision,
|
||||
} from '@qinglong/runtime-core/local-dispatch';
|
||||
import type { LocalExecutionStartCommand } from '../execution/coordinator';
|
||||
import type { LocalArtifactAllocator } from './artifact';
|
||||
|
||||
export interface LocalDispatchPlan {
|
||||
readonly command: LocalExecutionStartCommand;
|
||||
}
|
||||
|
||||
export interface LocalDispatchPlanSource {
|
||||
prepare(candidate: LocalDispatchCandidate): Promise<LocalDispatchPlan | null>;
|
||||
}
|
||||
|
||||
function assertEnvironmentValue(value: unknown): asserts value is string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.includes('\0') ||
|
||||
Buffer.byteLength(value, 'utf8') > 16 * 1024
|
||||
) {
|
||||
throw new TypeError('Local dispatch environment value is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalDispatchPlanMaterializer implements LocalDispatchPlanSource {
|
||||
constructor(
|
||||
private readonly definitions: Pick<
|
||||
LocalDispatchStore,
|
||||
'resolveLocalTaskExecutionRevision' | 'resolveLocalExecutionContextRecipe'
|
||||
>,
|
||||
private readonly artifacts: LocalArtifactAllocator,
|
||||
private readonly secrets?: LocalSecretEnvironmentProvider,
|
||||
) {}
|
||||
|
||||
async prepare(
|
||||
candidate: LocalDispatchCandidate,
|
||||
): Promise<LocalDispatchPlan | null> {
|
||||
const normalizedCandidate = normalizeLocalDispatchCandidate(candidate);
|
||||
const revision = await this.definitions.resolveLocalTaskExecutionRevision({
|
||||
projectId: normalizedCandidate.projectId,
|
||||
taskId: normalizedCandidate.taskId,
|
||||
taskRevision: normalizedCandidate.taskRevision,
|
||||
});
|
||||
if (!revision) return null;
|
||||
const normalizedRevision = normalizeLocalTaskExecutionRevision(revision);
|
||||
if (
|
||||
normalizedRevision.projectId !== normalizedCandidate.projectId ||
|
||||
normalizedRevision.taskId !== normalizedCandidate.taskId ||
|
||||
normalizedRevision.taskRevision !== normalizedCandidate.taskRevision ||
|
||||
normalizedRevision.executorType !== normalizedCandidate.executorType
|
||||
) {
|
||||
throw new TypeError('Local Task revision does not match its candidate');
|
||||
}
|
||||
const recipe = await this.definitions.resolveLocalExecutionContextRecipe(
|
||||
normalizedRevision.contextRef,
|
||||
);
|
||||
if (!recipe) return null;
|
||||
const normalizedRecipe = normalizeLocalExecutionContextRecipe(recipe);
|
||||
if (normalizedRecipe.contextRef !== normalizedRevision.contextRef) {
|
||||
throw new TypeError('Local context recipe does not match its revision');
|
||||
}
|
||||
const secretRefs = Object.freeze([
|
||||
...new Set(
|
||||
normalizedRecipe.environment.flatMap((binding) =>
|
||||
binding.kind === 'secret' ? [binding.secretRef] : [],
|
||||
),
|
||||
),
|
||||
]);
|
||||
if (secretRefs.length > MAX_LOCAL_DISPATCH_SECRET_REFS) {
|
||||
throw new RangeError('Local dispatch Secret reference budget exceeded');
|
||||
}
|
||||
let secretValues: readonly string[] = [];
|
||||
if (secretRefs.length > 0) {
|
||||
if (!this.secrets) return null;
|
||||
const resolved = await this.secrets.resolveLocalSecretEnvironment({
|
||||
candidate: normalizedCandidate,
|
||||
secretRefs,
|
||||
});
|
||||
if (!resolved) return null;
|
||||
if (resolved.length !== secretRefs.length) {
|
||||
throw new TypeError('Local Secret provider returned an invalid result');
|
||||
}
|
||||
secretValues = resolved;
|
||||
}
|
||||
const secretByRef = new Map(
|
||||
secretRefs.map((secretRef, index) => [secretRef, secretValues[index]]),
|
||||
);
|
||||
const environment: Record<string, string> = Object.create(null);
|
||||
let environmentBytes = 0;
|
||||
for (const binding of normalizedRecipe.environment) {
|
||||
const value =
|
||||
binding.kind === 'public'
|
||||
? binding.value
|
||||
: secretByRef.get(binding.secretRef);
|
||||
assertEnvironmentValue(value);
|
||||
environmentBytes +=
|
||||
Buffer.byteLength(binding.name, 'utf8') +
|
||||
Buffer.byteLength(value, 'utf8');
|
||||
if (environmentBytes > MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES) {
|
||||
throw new RangeError('Local dispatch environment byte budget exceeded');
|
||||
}
|
||||
environment[binding.name] = value;
|
||||
}
|
||||
const artifact = await this.artifacts.prepare(normalizedCandidate);
|
||||
return Object.freeze({
|
||||
command: Object.freeze({
|
||||
runId: normalizedCandidate.runId,
|
||||
attemptId: normalizedCandidate.attemptId,
|
||||
...(normalizedCandidate.stepRunId === undefined
|
||||
? {}
|
||||
: { stepRunId: normalizedCandidate.stepRunId }),
|
||||
command: normalizedRevision.command,
|
||||
environment: Object.freeze(environment),
|
||||
...(normalizedRevision.workingDirectory === undefined
|
||||
? {}
|
||||
: { workingDirectory: normalizedRevision.workingDirectory }),
|
||||
...(normalizedRevision.timeoutMs === undefined
|
||||
? {}
|
||||
: { timeoutMs: normalizedRevision.timeoutMs }),
|
||||
output: artifact.output,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,951 @@
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import { parseLocalProcessDurableHandle } from '@qinglong/local-process';
|
||||
import type {
|
||||
LocalProcessCommand,
|
||||
LocalProcessLaunchHandle,
|
||||
LocalProcessLaunchRequest,
|
||||
LocalProcessOutputPlan,
|
||||
LocalProcessStopResult,
|
||||
} from '@qinglong/local-process';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
RunRepository,
|
||||
RunRepositoryTransaction,
|
||||
} from '@qinglong/runtime-core/run-repository';
|
||||
import type { LocalWorkflowTaskExecutionRepository } from './workflowTaskExecution';
|
||||
|
||||
export const LOCAL_PROCESS_EXECUTOR_TYPE = 'local_process';
|
||||
export const MAX_LOCAL_EXECUTION_TIMEOUT_MS = 365 * 24 * 60 * 60_000;
|
||||
|
||||
export interface LocalExecutionStartCommand {
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly stepRunId?: string;
|
||||
readonly command: LocalProcessCommand;
|
||||
readonly environment?: Readonly<Record<string, string>>;
|
||||
readonly workingDirectory?: string;
|
||||
readonly output?: LocalProcessOutputPlan;
|
||||
readonly timeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface LocalExecutionStartResult {
|
||||
readonly run: RunRecord;
|
||||
readonly attempt: RunAttemptRecord;
|
||||
readonly handle: LocalProcessLaunchHandle;
|
||||
}
|
||||
|
||||
export interface LocalExecutionLauncher {
|
||||
start(request: LocalProcessLaunchRequest): Promise<LocalProcessLaunchHandle>;
|
||||
}
|
||||
|
||||
export interface LocalExecutionController {
|
||||
stop(durableHandle: string): Promise<LocalProcessStopResult>;
|
||||
}
|
||||
|
||||
export interface LocalExecutionCoordinatorOptions {
|
||||
readonly clock?: { now(): number };
|
||||
readonly createEventId?: () => string;
|
||||
readonly createCallbackToken?: () => string;
|
||||
readonly workflowTasks?: LocalWorkflowTaskExecutionRepository;
|
||||
}
|
||||
|
||||
interface AggregateSnapshot {
|
||||
readonly run: RunRecord;
|
||||
readonly attempt: RunAttemptRecord;
|
||||
}
|
||||
|
||||
interface PreparedAggregate extends AggregateSnapshot {
|
||||
readonly callbackSequence: number;
|
||||
readonly callbackTokenHash: string;
|
||||
}
|
||||
|
||||
interface EventDraft {
|
||||
readonly sequence: number;
|
||||
readonly type: string;
|
||||
readonly payload: Readonly<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export class LocalExecutionRejectedError extends Error {
|
||||
readonly code = 'LOCAL_EXECUTION_REJECTED';
|
||||
|
||||
constructor(readonly reason: string) {
|
||||
super(`Local execution rejected: ${reason}`);
|
||||
this.name = 'LocalExecutionRejectedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalExecutionConcurrentWriteError extends Error {
|
||||
readonly code = 'LOCAL_EXECUTION_CONCURRENT_WRITE';
|
||||
|
||||
constructor() {
|
||||
super('Local execution aggregate changed concurrently');
|
||||
this.name = 'LocalExecutionConcurrentWriteError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalExecutionLaunchError extends Error {
|
||||
readonly code = 'LOCAL_EXECUTION_LAUNCH_FAILED';
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
readonly snapshot: AggregateSnapshot,
|
||||
readonly cause?: unknown,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'LocalExecutionLaunchError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalExecutionOwnershipPersistenceError extends Error {
|
||||
readonly code = 'LOCAL_EXECUTION_OWNERSHIP_PERSISTENCE_FAILED';
|
||||
|
||||
constructor(
|
||||
readonly compensation: LocalProcessStopResult,
|
||||
readonly snapshot: AggregateSnapshot,
|
||||
readonly cause?: unknown,
|
||||
) {
|
||||
super('Local execution could not persist durable process ownership');
|
||||
this.name = 'LocalExecutionOwnershipPersistenceError';
|
||||
}
|
||||
}
|
||||
|
||||
function assertSafeTimestamp(value: number, field: string): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new TypeError(`${field} must be a non-negative safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertStartCommand(command: LocalExecutionStartCommand): void {
|
||||
if (!command || typeof command !== 'object' || Array.isArray(command)) {
|
||||
throw new TypeError('Local execution start command is invalid');
|
||||
}
|
||||
for (const [field, value] of [
|
||||
['runId', command.runId],
|
||||
['attemptId', command.attemptId],
|
||||
] as const) {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > 128 ||
|
||||
/[\0\r\n]/.test(value)
|
||||
) {
|
||||
throw new TypeError(`Local execution ${field} is invalid`);
|
||||
}
|
||||
}
|
||||
if (
|
||||
command.stepRunId !== undefined &&
|
||||
(typeof command.stepRunId !== 'string' ||
|
||||
command.stepRunId.length < 1 ||
|
||||
command.stepRunId.length > 128 ||
|
||||
/[\0\r\n]/.test(command.stepRunId))
|
||||
) {
|
||||
throw new TypeError('Local execution stepRunId is invalid');
|
||||
}
|
||||
if (
|
||||
command.timeoutMs !== undefined &&
|
||||
(!Number.isSafeInteger(command.timeoutMs) ||
|
||||
command.timeoutMs < 1 ||
|
||||
command.timeoutMs > MAX_LOCAL_EXECUTION_TIMEOUT_MS)
|
||||
) {
|
||||
throw new RangeError('Local execution timeout is invalid');
|
||||
}
|
||||
if (command.output !== undefined) {
|
||||
const output = command.output;
|
||||
if (
|
||||
!output ||
|
||||
typeof output !== 'object' ||
|
||||
Array.isArray(output) ||
|
||||
!path.isAbsolute(output.filePath) ||
|
||||
path.parse(output.filePath).root === output.filePath ||
|
||||
output.filePath.includes('\0') ||
|
||||
Buffer.byteLength(output.filePath, 'utf8') > 4096 ||
|
||||
!/^local-[0-9a-f]{30}$/.test(output.logArtifactId) ||
|
||||
path.basename(output.filePath) !== `${output.logArtifactId}.log` ||
|
||||
!Number.isSafeInteger(output.maximumBytes) ||
|
||||
output.maximumBytes < 64 * 1024 ||
|
||||
output.maximumBytes > 1024 * 1024 * 1024
|
||||
) {
|
||||
throw new TypeError('Local execution output plan is invalid');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function reserveEvent(run: RunRecord): Readonly<{
|
||||
run: RunRecord;
|
||||
sequence: number;
|
||||
}> {
|
||||
const version = run.version + 1;
|
||||
const sequence = run.eventSequence + 1;
|
||||
if (
|
||||
!Number.isSafeInteger(version) ||
|
||||
version < 1 ||
|
||||
!Number.isSafeInteger(sequence) ||
|
||||
sequence < 1
|
||||
) {
|
||||
throw new TypeError('Local execution version or sequence overflowed');
|
||||
}
|
||||
return Object.freeze({
|
||||
run: { ...run, version, eventSequence: sequence },
|
||||
sequence,
|
||||
});
|
||||
}
|
||||
|
||||
function event(
|
||||
id: string,
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
draft: EventDraft,
|
||||
dedupeKey: string,
|
||||
actorType: RunEventRecord['actorType'],
|
||||
actorId: string,
|
||||
atMs: number,
|
||||
): RunEventRecord {
|
||||
if (
|
||||
typeof id !== 'string' ||
|
||||
id.length < 1 ||
|
||||
id.length > 128 ||
|
||||
/[\0\r\n]/.test(id)
|
||||
) {
|
||||
throw new TypeError('Local execution event id is invalid');
|
||||
}
|
||||
return {
|
||||
id,
|
||||
runId,
|
||||
attemptId,
|
||||
sequence: draft.sequence,
|
||||
type: draft.type,
|
||||
dedupeKey,
|
||||
actorType,
|
||||
actorId,
|
||||
payload: draft.payload,
|
||||
createdAtMs: atMs,
|
||||
};
|
||||
}
|
||||
|
||||
function atOrAfter(
|
||||
now: number,
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
additional?: number,
|
||||
): number {
|
||||
assertSafeTimestamp(now, 'Local execution clock');
|
||||
if (additional !== undefined) {
|
||||
assertSafeTimestamp(additional, 'Local execution additional timestamp');
|
||||
}
|
||||
return Math.max(
|
||||
now,
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs ?? 0,
|
||||
additional ?? 0,
|
||||
);
|
||||
}
|
||||
|
||||
function assertClaimedAggregate(
|
||||
run: RunRecord | null,
|
||||
attempt: RunAttemptRecord | null,
|
||||
latest: RunAttemptRecord | null,
|
||||
): asserts run is RunRecord & object {
|
||||
if (!run) throw new LocalExecutionRejectedError('run_not_found');
|
||||
if (!attempt) throw new LocalExecutionRejectedError('attempt_not_found');
|
||||
if (
|
||||
attempt.runId !== run.id ||
|
||||
latest?.runId !== run.id ||
|
||||
latest.id !== attempt.id ||
|
||||
run.executionOwner !== 'runtime'
|
||||
) {
|
||||
throw new LocalExecutionRejectedError('aggregate_mismatch');
|
||||
}
|
||||
if (run.status !== 'queued') {
|
||||
throw new LocalExecutionRejectedError('run_not_queued');
|
||||
}
|
||||
if (attempt.status !== 'claimed') {
|
||||
throw new LocalExecutionRejectedError('attempt_not_claimed');
|
||||
}
|
||||
if (attempt.executorType !== LOCAL_PROCESS_EXECUTOR_TYPE) {
|
||||
throw new LocalExecutionRejectedError('executor_mismatch');
|
||||
}
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
throw new LocalExecutionRejectedError('cancellation_requested');
|
||||
}
|
||||
if (
|
||||
attempt.callbackSequence !== 0 ||
|
||||
attempt.callbackTokenHash !== undefined ||
|
||||
attempt.workerId !== undefined ||
|
||||
attempt.workerSessionId !== undefined ||
|
||||
attempt.workerGeneration !== undefined ||
|
||||
attempt.executorHandle !== undefined ||
|
||||
attempt.pid !== undefined ||
|
||||
attempt.logArtifactId !== undefined ||
|
||||
attempt.leaseToken !== undefined ||
|
||||
attempt.leaseTokenDigest !== undefined ||
|
||||
attempt.leaseGeneration !== undefined ||
|
||||
attempt.leaseVersion !== undefined ||
|
||||
attempt.leaseExpiresAtMs !== undefined ||
|
||||
attempt.offerId !== undefined ||
|
||||
attempt.deadlineAtMs !== undefined ||
|
||||
attempt.startedAtMs !== undefined ||
|
||||
attempt.finishedAtMs !== undefined ||
|
||||
attempt.exitCode !== undefined ||
|
||||
attempt.errorCode !== undefined ||
|
||||
attempt.errorSummary !== undefined
|
||||
) {
|
||||
throw new LocalExecutionRejectedError('stale_execution_authority');
|
||||
}
|
||||
}
|
||||
|
||||
function assertPreparedAggregate(
|
||||
current: AggregateSnapshot,
|
||||
expected: PreparedAggregate,
|
||||
): void {
|
||||
if (
|
||||
current.run.id !== expected.run.id ||
|
||||
current.run.version !== expected.run.version ||
|
||||
current.run.status !== 'dispatching' ||
|
||||
current.run.executionOwner !== 'runtime' ||
|
||||
current.run.cancelRequestedAtMs !== undefined ||
|
||||
current.attempt.id !== expected.attempt.id ||
|
||||
current.attempt.runId !== current.run.id ||
|
||||
current.attempt.status !== 'starting' ||
|
||||
current.attempt.executorType !== LOCAL_PROCESS_EXECUTOR_TYPE ||
|
||||
current.attempt.callbackSequence !== expected.attempt.callbackSequence ||
|
||||
current.attempt.callbackTokenHash !== expected.callbackTokenHash ||
|
||||
current.attempt.deadlineAtMs !== expected.attempt.deadlineAtMs ||
|
||||
current.attempt.logArtifactId !== expected.attempt.logArtifactId ||
|
||||
current.attempt.executorHandle !== undefined ||
|
||||
current.attempt.pid !== undefined ||
|
||||
current.attempt.startedAtMs !== undefined ||
|
||||
current.attempt.finishedAtMs !== undefined
|
||||
) {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
}
|
||||
|
||||
async function loadExact(
|
||||
transaction: RunRepositoryTransaction,
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
): Promise<AggregateSnapshot> {
|
||||
const [run, attempt, latest] = await Promise.all([
|
||||
transaction.findRunById(runId),
|
||||
transaction.findAttemptById(attemptId),
|
||||
transaction.findLatestAttemptByRunId(runId),
|
||||
]);
|
||||
if (!run || !attempt || latest?.id !== attempt.id) {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
export class LocalExecutionCoordinator {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createEventId: () => string;
|
||||
private readonly createCallbackToken: () => string;
|
||||
private readonly workflowTasks:
|
||||
| LocalWorkflowTaskExecutionRepository
|
||||
| undefined;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly launcher: LocalExecutionLauncher,
|
||||
private readonly controller: LocalExecutionController,
|
||||
options: LocalExecutionCoordinatorOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
this.createCallbackToken =
|
||||
options.createCallbackToken ??
|
||||
(() => randomBytes(32).toString('base64url'));
|
||||
this.workflowTasks = options.workflowTasks;
|
||||
if (
|
||||
this.workflowTasks !== undefined &&
|
||||
(typeof this.workflowTasks.prepare !== 'function' ||
|
||||
typeof this.workflowTasks.recordRunning !== 'function' ||
|
||||
typeof this.workflowTasks.recordStartFailure !== 'function')
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow Task execution repository is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async start(
|
||||
command: LocalExecutionStartCommand,
|
||||
): Promise<LocalExecutionStartResult> {
|
||||
assertStartCommand(command);
|
||||
const callbackToken = this.createCallbackToken();
|
||||
if (!/^[A-Za-z0-9_-]{32,128}$/.test(callbackToken)) {
|
||||
throw new TypeError('Local execution callback token factory is invalid');
|
||||
}
|
||||
const callbackTokenHash = createHash('sha256')
|
||||
.update(callbackToken)
|
||||
.digest('hex');
|
||||
const prepared = await this.prepare(command, callbackTokenHash);
|
||||
|
||||
let handle: LocalProcessLaunchHandle;
|
||||
try {
|
||||
handle = await this.launcher.start({
|
||||
runId: prepared.run.id,
|
||||
attemptId: prepared.attempt.id,
|
||||
callbackSequence: prepared.callbackSequence,
|
||||
callbackToken,
|
||||
command: command.command,
|
||||
...(command.environment === undefined
|
||||
? {}
|
||||
: { environment: command.environment }),
|
||||
...(command.workingDirectory === undefined
|
||||
? {}
|
||||
: { workingDirectory: command.workingDirectory }),
|
||||
...(command.output === undefined ? {} : { output: command.output }),
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
const failed = await this.recordTerminal(
|
||||
prepared,
|
||||
'failed',
|
||||
'EXECUTOR_START_FAILED',
|
||||
'Local process failed before durable ownership was established',
|
||||
);
|
||||
throw new LocalExecutionLaunchError(
|
||||
'Local process could not start',
|
||||
failed,
|
||||
error,
|
||||
);
|
||||
} catch (persistenceError) {
|
||||
if (persistenceError instanceof LocalExecutionLaunchError) {
|
||||
throw persistenceError;
|
||||
}
|
||||
throw new LocalExecutionLaunchError(
|
||||
'Local process could not start and failure state could not be persisted',
|
||||
prepared,
|
||||
new AggregateError([error, persistenceError]),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
this.assertHandle(handle, prepared);
|
||||
const running = await this.recordRunning(prepared, handle);
|
||||
return Object.freeze({ ...running, handle });
|
||||
} catch (error) {
|
||||
void handle.completion.catch(() => undefined);
|
||||
let compensation: LocalProcessStopResult;
|
||||
try {
|
||||
compensation = await this.controller.stop(handle.durableHandle);
|
||||
} catch {
|
||||
compensation = Object.freeze({
|
||||
status: 'unknown' as const,
|
||||
reason: 'signal_failed' as const,
|
||||
});
|
||||
}
|
||||
let snapshot: AggregateSnapshot = prepared;
|
||||
if (
|
||||
compensation.status === 'stopped' ||
|
||||
compensation.status === 'already_exited'
|
||||
) {
|
||||
try {
|
||||
snapshot = await this.recordTerminal(
|
||||
prepared,
|
||||
'lost',
|
||||
'EXECUTION_ACTIVATION_PERSISTENCE_FAILED',
|
||||
'Durable process ownership could not be persisted',
|
||||
);
|
||||
} catch {
|
||||
snapshot = await this.loadLatest(prepared);
|
||||
}
|
||||
} else {
|
||||
snapshot = await this.loadLatest(prepared);
|
||||
}
|
||||
throw new LocalExecutionOwnershipPersistenceError(
|
||||
compensation,
|
||||
snapshot,
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async prepare(
|
||||
command: LocalExecutionStartCommand,
|
||||
callbackTokenHash: string,
|
||||
): Promise<PreparedAggregate> {
|
||||
if (command.stepRunId !== undefined) {
|
||||
if (!this.workflowTasks) {
|
||||
throw new LocalExecutionRejectedError(
|
||||
'workflow_task_authority_unavailable',
|
||||
);
|
||||
}
|
||||
const observedAtMs = this.clock.now();
|
||||
assertSafeTimestamp(observedAtMs, 'Local execution clock');
|
||||
const deadlineAtMs =
|
||||
command.timeoutMs === undefined
|
||||
? undefined
|
||||
: observedAtMs + command.timeoutMs;
|
||||
if (
|
||||
deadlineAtMs !== undefined &&
|
||||
(!Number.isSafeInteger(deadlineAtMs) ||
|
||||
deadlineAtMs <= observedAtMs)
|
||||
) {
|
||||
throw new RangeError('Local execution deadline overflowed');
|
||||
}
|
||||
const result = await this.workflowTasks.prepare({
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
stepRunId: command.stepRunId,
|
||||
callbackTokenHash,
|
||||
...(deadlineAtMs === undefined ? {} : { deadlineAtMs }),
|
||||
...(command.output === undefined
|
||||
? {}
|
||||
: { logArtifactId: command.output.logArtifactId }),
|
||||
atMs: observedAtMs,
|
||||
eventId: this.createEventId(),
|
||||
});
|
||||
if (result.status === 'rejected') {
|
||||
throw new LocalExecutionRejectedError(result.reason);
|
||||
}
|
||||
const current = await this.loadWorkflowTaskSnapshot(
|
||||
command.runId,
|
||||
command.attemptId,
|
||||
result.snapshot,
|
||||
);
|
||||
return Object.freeze({
|
||||
...current,
|
||||
callbackSequence: current.attempt.callbackSequence + 1,
|
||||
callbackTokenHash,
|
||||
});
|
||||
}
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const [run, attempt, latest] = await Promise.all([
|
||||
transaction.findRunById(command.runId),
|
||||
transaction.findAttemptById(command.attemptId),
|
||||
transaction.findLatestAttemptByRunId(command.runId),
|
||||
]);
|
||||
assertClaimedAggregate(run, attempt, latest);
|
||||
if (!attempt) throw new LocalExecutionRejectedError('attempt_not_found');
|
||||
const atMs = atOrAfter(this.clock.now(), run, attempt);
|
||||
const deadlineAtMs =
|
||||
command.timeoutMs === undefined ? undefined : atMs + command.timeoutMs;
|
||||
if (
|
||||
deadlineAtMs !== undefined &&
|
||||
(!Number.isSafeInteger(deadlineAtMs) || deadlineAtMs <= atMs)
|
||||
) {
|
||||
throw new RangeError('Local execution deadline overflowed');
|
||||
}
|
||||
|
||||
const dispatching = reserveEvent(run);
|
||||
const nextDispatching: RunRecord = {
|
||||
...dispatching.run,
|
||||
status: 'dispatching',
|
||||
};
|
||||
if (!(await transaction.compareAndSetRun(nextDispatching, run.version))) {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run.id,
|
||||
attempt.id,
|
||||
{
|
||||
sequence: dispatching.sequence,
|
||||
type: 'run.dispatching',
|
||||
payload: Object.freeze({
|
||||
from_status: run.status,
|
||||
to_status: 'dispatching',
|
||||
version: nextDispatching.version,
|
||||
}),
|
||||
},
|
||||
`local-execution:run:${run.id}:${run.version}:dispatching`,
|
||||
'scheduler',
|
||||
'local-execution',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
|
||||
const starting = reserveEvent(nextDispatching);
|
||||
const nextStartingRun = starting.run;
|
||||
const nextAttempt: RunAttemptRecord = {
|
||||
...attempt,
|
||||
status: 'starting',
|
||||
callbackTokenHash,
|
||||
...(deadlineAtMs === undefined ? {} : { deadlineAtMs }),
|
||||
...(command.output === undefined
|
||||
? {}
|
||||
: { logArtifactId: command.output.logArtifactId }),
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
nextStartingRun,
|
||||
nextDispatching.version,
|
||||
)) ||
|
||||
!(await transaction.compareAndSetAttempt(nextAttempt, {
|
||||
status: attempt.status,
|
||||
callbackSequence: attempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run.id,
|
||||
attempt.id,
|
||||
{
|
||||
sequence: starting.sequence,
|
||||
type: 'attempt.starting',
|
||||
payload: Object.freeze({
|
||||
attempt_id: attempt.id,
|
||||
from_status: attempt.status,
|
||||
to_status: 'starting',
|
||||
version: nextStartingRun.version,
|
||||
...(deadlineAtMs === undefined
|
||||
? {}
|
||||
: { deadline_at_ms: deadlineAtMs }),
|
||||
}),
|
||||
},
|
||||
`local-execution:attempt:${attempt.id}:${attempt.callbackSequence}:starting`,
|
||||
'worker',
|
||||
LOCAL_PROCESS_EXECUTOR_TYPE,
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
const callbackSequence = attempt.callbackSequence + 1;
|
||||
if (!Number.isSafeInteger(callbackSequence)) {
|
||||
throw new TypeError('Local execution callback sequence overflowed');
|
||||
}
|
||||
return Object.freeze({
|
||||
run: nextStartingRun,
|
||||
attempt: nextAttempt,
|
||||
callbackSequence,
|
||||
callbackTokenHash,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async recordRunning(
|
||||
expected: PreparedAggregate,
|
||||
handle: LocalProcessLaunchHandle,
|
||||
): Promise<AggregateSnapshot> {
|
||||
if (expected.attempt.stepRunId !== undefined) {
|
||||
if (!this.workflowTasks) {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
const atMs = atOrAfter(
|
||||
handle.startedAtMs,
|
||||
expected.run,
|
||||
expected.attempt,
|
||||
);
|
||||
const result = await this.workflowTasks.recordRunning({
|
||||
run: expected.run,
|
||||
attempt: expected.attempt,
|
||||
callbackTokenHash: expected.callbackTokenHash,
|
||||
executorHandle: handle.durableHandle,
|
||||
pid: handle.pid,
|
||||
startedAtMs: atMs,
|
||||
attemptEventId: this.createEventId(),
|
||||
stepMutationId: this.createEventId(),
|
||||
});
|
||||
if (result.status === 'rejected') {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
return this.loadWorkflowTaskSnapshot(
|
||||
expected.run.id,
|
||||
expected.attempt.id,
|
||||
result.snapshot,
|
||||
);
|
||||
}
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const current = await loadExact(
|
||||
transaction,
|
||||
expected.run.id,
|
||||
expected.attempt.id,
|
||||
);
|
||||
assertPreparedAggregate(current, expected);
|
||||
const atMs = atOrAfter(
|
||||
handle.startedAtMs,
|
||||
current.run,
|
||||
current.attempt,
|
||||
);
|
||||
const attemptRunning = reserveEvent(current.run);
|
||||
const nextAttempt: RunAttemptRecord = {
|
||||
...current.attempt,
|
||||
status: 'running',
|
||||
executorHandle: handle.durableHandle,
|
||||
pid: handle.pid,
|
||||
startedAtMs: atMs,
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
attemptRunning.run,
|
||||
current.run.version,
|
||||
)) ||
|
||||
!(await transaction.compareAndSetAttempt(nextAttempt, {
|
||||
status: current.attempt.status,
|
||||
callbackSequence: current.attempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
current.run.id,
|
||||
current.attempt.id,
|
||||
{
|
||||
sequence: attemptRunning.sequence,
|
||||
type: 'attempt.running',
|
||||
payload: Object.freeze({
|
||||
attempt_id: current.attempt.id,
|
||||
from_status: current.attempt.status,
|
||||
to_status: 'running',
|
||||
pid: handle.pid,
|
||||
version: attemptRunning.run.version,
|
||||
}),
|
||||
},
|
||||
`local-execution:attempt:${current.attempt.id}:${current.attempt.callbackSequence}:running`,
|
||||
'executor',
|
||||
LOCAL_PROCESS_EXECUTOR_TYPE,
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
|
||||
const runRunning = reserveEvent(attemptRunning.run);
|
||||
const nextRun: RunRecord = {
|
||||
...runRunning.run,
|
||||
status: 'running',
|
||||
startedAtMs: atMs,
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
nextRun,
|
||||
attemptRunning.run.version,
|
||||
))
|
||||
) {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
current.run.id,
|
||||
current.attempt.id,
|
||||
{
|
||||
sequence: runRunning.sequence,
|
||||
type: 'run.running',
|
||||
payload: Object.freeze({
|
||||
from_status: current.run.status,
|
||||
to_status: 'running',
|
||||
version: nextRun.version,
|
||||
}),
|
||||
},
|
||||
`local-execution:run:${current.run.id}:${current.run.version}:running`,
|
||||
'executor',
|
||||
LOCAL_PROCESS_EXECUTOR_TYPE,
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
return Object.freeze({ run: nextRun, attempt: nextAttempt });
|
||||
});
|
||||
}
|
||||
|
||||
private async recordTerminal(
|
||||
expected: PreparedAggregate,
|
||||
status: 'failed' | 'lost',
|
||||
errorCode: string,
|
||||
errorSummary: string,
|
||||
): Promise<AggregateSnapshot> {
|
||||
if (expected.attempt.stepRunId !== undefined) {
|
||||
if (!this.workflowTasks) {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
const atMs = atOrAfter(
|
||||
this.clock.now(),
|
||||
expected.run,
|
||||
expected.attempt,
|
||||
);
|
||||
const result = await this.workflowTasks.recordStartFailure({
|
||||
run: expected.run,
|
||||
attempt: expected.attempt,
|
||||
callbackTokenHash: expected.callbackTokenHash,
|
||||
status: 'failed',
|
||||
errorCode,
|
||||
errorSummary,
|
||||
finishedAtMs: atMs,
|
||||
attemptEventId: this.createEventId(),
|
||||
stepMutationId: this.createEventId(),
|
||||
});
|
||||
if (result.status === 'rejected') {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
return this.loadWorkflowTaskSnapshot(
|
||||
expected.run.id,
|
||||
expected.attempt.id,
|
||||
result.snapshot,
|
||||
);
|
||||
}
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const current = await loadExact(
|
||||
transaction,
|
||||
expected.run.id,
|
||||
expected.attempt.id,
|
||||
);
|
||||
assertPreparedAggregate(current, expected);
|
||||
const atMs = atOrAfter(this.clock.now(), current.run, current.attempt);
|
||||
const attemptTerminal = reserveEvent(current.run);
|
||||
const nextAttempt: RunAttemptRecord = {
|
||||
...current.attempt,
|
||||
status,
|
||||
finishedAtMs: atMs,
|
||||
errorCode,
|
||||
errorSummary,
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
attemptTerminal.run,
|
||||
current.run.version,
|
||||
)) ||
|
||||
!(await transaction.compareAndSetAttempt(nextAttempt, {
|
||||
status: current.attempt.status,
|
||||
callbackSequence: current.attempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
current.run.id,
|
||||
current.attempt.id,
|
||||
{
|
||||
sequence: attemptTerminal.sequence,
|
||||
type: `attempt.${status}`,
|
||||
payload: Object.freeze({
|
||||
attempt_id: current.attempt.id,
|
||||
from_status: current.attempt.status,
|
||||
to_status: status,
|
||||
error_code: errorCode,
|
||||
version: attemptTerminal.run.version,
|
||||
}),
|
||||
},
|
||||
`local-execution:attempt:${current.attempt.id}:${current.attempt.callbackSequence}:${status}`,
|
||||
status === 'lost' ? 'reconciler' : 'executor',
|
||||
status === 'lost' ? 'local-execution' : LOCAL_PROCESS_EXECUTOR_TYPE,
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
|
||||
const runTerminal = reserveEvent(attemptTerminal.run);
|
||||
const nextRun: RunRecord = {
|
||||
...runTerminal.run,
|
||||
status,
|
||||
...(status === 'failed' ? { finishedAtMs: atMs } : {}),
|
||||
errorCode,
|
||||
errorSummary,
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
nextRun,
|
||||
attemptTerminal.run.version,
|
||||
))
|
||||
) {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
current.run.id,
|
||||
current.attempt.id,
|
||||
{
|
||||
sequence: runTerminal.sequence,
|
||||
type: `run.${status}`,
|
||||
payload: Object.freeze({
|
||||
from_status: current.run.status,
|
||||
to_status: status,
|
||||
error_code: errorCode,
|
||||
version: nextRun.version,
|
||||
}),
|
||||
},
|
||||
`local-execution:run:${current.run.id}:${current.run.version}:${status}`,
|
||||
status === 'lost' ? 'reconciler' : 'executor',
|
||||
status === 'lost' ? 'local-execution' : LOCAL_PROCESS_EXECUTOR_TYPE,
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
return Object.freeze({ run: nextRun, attempt: nextAttempt });
|
||||
});
|
||||
}
|
||||
|
||||
private assertHandle(
|
||||
handle: LocalProcessLaunchHandle,
|
||||
expected: PreparedAggregate,
|
||||
): void {
|
||||
const parsed =
|
||||
handle &&
|
||||
typeof handle === 'object' &&
|
||||
typeof handle.durableHandle === 'string'
|
||||
? parseLocalProcessDurableHandle(handle.durableHandle)
|
||||
: null;
|
||||
if (
|
||||
!handle ||
|
||||
typeof handle !== 'object' ||
|
||||
!Number.isSafeInteger(handle.pid) ||
|
||||
handle.pid < 1 ||
|
||||
!Number.isSafeInteger(handle.startedAtMs) ||
|
||||
handle.startedAtMs < 0 ||
|
||||
typeof handle.handleId !== 'string' ||
|
||||
handle.handleId.length < 1 ||
|
||||
typeof handle.durableHandle !== 'string' ||
|
||||
handle.durableHandle.length < 1 ||
|
||||
typeof handle.completion?.then !== 'function' ||
|
||||
!parsed ||
|
||||
parsed.handleId !== handle.handleId ||
|
||||
parsed.identity.pid !== handle.pid ||
|
||||
expected.run.id !== expected.attempt.runId
|
||||
) {
|
||||
throw new TypeError('Local process launcher returned an invalid handle');
|
||||
}
|
||||
}
|
||||
|
||||
private async loadLatest(
|
||||
fallback: AggregateSnapshot,
|
||||
): Promise<AggregateSnapshot> {
|
||||
const [run, attempt] = await Promise.all([
|
||||
this.repository.findRunById(fallback.run.id),
|
||||
this.repository.findAttemptById(fallback.attempt.id),
|
||||
]);
|
||||
return Object.freeze({
|
||||
run: run ?? fallback.run,
|
||||
attempt: attempt ?? fallback.attempt,
|
||||
});
|
||||
}
|
||||
|
||||
private async loadWorkflowTaskSnapshot(
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
expected: Readonly<{
|
||||
runVersion: number;
|
||||
runEventSequence: number;
|
||||
attemptStatus: RunAttemptRecord['status'];
|
||||
callbackSequence: number;
|
||||
}>,
|
||||
): Promise<AggregateSnapshot> {
|
||||
const [run, attempt] = await Promise.all([
|
||||
this.repository.findRunById(runId),
|
||||
this.repository.findAttemptById(attemptId),
|
||||
]);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
attempt.runId !== run.id ||
|
||||
attempt.stepRunId === undefined ||
|
||||
run.version !== expected.runVersion ||
|
||||
run.eventSequence !== expected.runEventSequence ||
|
||||
attempt.status !== expected.attemptStatus ||
|
||||
attempt.callbackSequence !== expected.callbackSequence
|
||||
) {
|
||||
throw new LocalExecutionConcurrentWriteError();
|
||||
}
|
||||
return Object.freeze({ run, attempt });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from './coordinator';
|
||||
export * from './workflowTaskExecution';
|
||||
@@ -0,0 +1,98 @@
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunRecord,
|
||||
} from '@qinglong/runtime-core/run-repository';
|
||||
|
||||
export type LocalWorkflowTaskTerminalStatus =
|
||||
| 'succeeded'
|
||||
| 'failed'
|
||||
| 'cancelled'
|
||||
| 'timed_out';
|
||||
|
||||
export interface LocalWorkflowTaskExecutionSnapshot {
|
||||
readonly runVersion: number;
|
||||
readonly runEventSequence: number;
|
||||
readonly attemptStatus: RunAttemptRecord['status'];
|
||||
readonly callbackSequence: number;
|
||||
}
|
||||
|
||||
export type LocalWorkflowTaskExecutionMutationResult =
|
||||
| Readonly<{
|
||||
status: 'applied';
|
||||
snapshot: Readonly<LocalWorkflowTaskExecutionSnapshot>;
|
||||
}>
|
||||
| Readonly<{
|
||||
status: 'rejected';
|
||||
reason:
|
||||
| 'aggregate_mismatch'
|
||||
| 'run_not_running'
|
||||
| 'attempt_not_claimed'
|
||||
| 'attempt_not_starting'
|
||||
| 'cancellation_requested'
|
||||
| 'stale_execution_authority';
|
||||
}>;
|
||||
|
||||
export interface LocalWorkflowTaskExecutionRepository {
|
||||
prepare(command: Readonly<{
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
stepRunId: string;
|
||||
callbackTokenHash: string;
|
||||
deadlineAtMs?: number;
|
||||
logArtifactId?: string;
|
||||
atMs: number;
|
||||
eventId: string;
|
||||
}>): Promise<Readonly<LocalWorkflowTaskExecutionMutationResult>>;
|
||||
recordRunning(command: Readonly<{
|
||||
run: Readonly<RunRecord>;
|
||||
attempt: Readonly<RunAttemptRecord>;
|
||||
callbackTokenHash: string;
|
||||
executorHandle: string;
|
||||
pid: number;
|
||||
startedAtMs: number;
|
||||
attemptEventId: string;
|
||||
stepMutationId: string;
|
||||
}>): Promise<Readonly<LocalWorkflowTaskExecutionMutationResult>>;
|
||||
recordStartFailure(command: Readonly<{
|
||||
run: Readonly<RunRecord>;
|
||||
attempt: Readonly<RunAttemptRecord>;
|
||||
callbackTokenHash: string;
|
||||
status: 'failed';
|
||||
errorCode: string;
|
||||
errorSummary: string;
|
||||
finishedAtMs: number;
|
||||
attemptEventId: string;
|
||||
stepMutationId: string;
|
||||
}>): Promise<Readonly<LocalWorkflowTaskExecutionMutationResult>>;
|
||||
complete(command: Readonly<{
|
||||
run: Readonly<RunRecord>;
|
||||
attempt: Readonly<RunAttemptRecord>;
|
||||
callbackSequence: number;
|
||||
startedAtMs: number;
|
||||
finishedAtMs: number;
|
||||
exitCode: number;
|
||||
terminalStatus: LocalWorkflowTaskTerminalStatus;
|
||||
errorCode?: string;
|
||||
errorSummary?: string;
|
||||
attemptEventId: string;
|
||||
syntheticStartMutationId: string;
|
||||
terminalStepMutationId: string;
|
||||
}>): Promise<'completed' | 'already_terminal' | 'stale'>;
|
||||
requestTimeout(command: Readonly<{
|
||||
run: Readonly<RunRecord>;
|
||||
attempt: Readonly<RunAttemptRecord>;
|
||||
dueAtMs: number;
|
||||
eventId: string;
|
||||
}>): Promise<'requested' | 'existing' | 'stale'>;
|
||||
recordControlTerminal(command: Readonly<{
|
||||
run: Readonly<RunRecord>;
|
||||
attempt: Readonly<RunAttemptRecord>;
|
||||
reason: 'user' | 'policy' | 'shutdown' | 'reconcile' | 'timeout';
|
||||
terminalStatus: 'cancelled' | 'timed_out';
|
||||
errorCode: string;
|
||||
errorSummary: string;
|
||||
finishedAtMs: number;
|
||||
attemptEventId: string;
|
||||
stepMutationId: string;
|
||||
}>): Promise<'terminal' | 'already_terminal' | 'stale'>;
|
||||
}
|
||||
@@ -0,0 +1,848 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import type {
|
||||
LocalRunStartupRecoveryCandidate,
|
||||
LocalRunStartupRecoveryPage,
|
||||
LocalRunStartupRecoverySource,
|
||||
} from '@qinglong/runtime-core/local-startup-recovery';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
RunRepository,
|
||||
RunRepositoryTransaction,
|
||||
} from '@qinglong/runtime-core/run-repository';
|
||||
import type { LocalCompletionReceiptJournal } from '@qinglong/runtime-core/local-completion-receipt-journal';
|
||||
import { LocalCompletionReceiptProcessor } from '../control/completion';
|
||||
import {
|
||||
type CompletionReceiptStore,
|
||||
type LocalPersistedExecutionInspection,
|
||||
type LocalPersistedExecutionInspector,
|
||||
} from '@qinglong/local-process';
|
||||
|
||||
export const MAX_LOCAL_RUN_RECOVERY_ITEMS = 256;
|
||||
export const MAX_LOCAL_RUN_RECEIPT_GRACE_MS = 5_000;
|
||||
export const MAX_LOCAL_RUN_QUARANTINE_RETENTION_MS = 24 * 60 * 60_000;
|
||||
|
||||
const ACTIVE_ATTEMPT_STATUSES = new Set(['claimed', 'starting', 'running']);
|
||||
const PAGE_KEYS = Object.freeze(['candidates', 'truncated']);
|
||||
const CANDIDATE_KEYS = Object.freeze([
|
||||
'activeAttemptCount',
|
||||
'runId',
|
||||
'runStatus',
|
||||
]);
|
||||
|
||||
export interface LocalRunStartupRecoverySummary {
|
||||
readonly safe: boolean;
|
||||
readonly scanned: number;
|
||||
readonly recovered: number;
|
||||
readonly remaining: number;
|
||||
readonly failed: number;
|
||||
readonly truncated: boolean;
|
||||
}
|
||||
|
||||
export interface LocalRunStartupRecoveryCoordinatorOptions {
|
||||
readonly receiptPublishGraceMs?: number;
|
||||
readonly clock?: { now(): number };
|
||||
readonly wait?: (delayMs: number) => Promise<void>;
|
||||
readonly createEventId?: () => string;
|
||||
readonly journal?: Pick<
|
||||
LocalCompletionReceiptJournal,
|
||||
'markQuarantined' | 'resolve'
|
||||
>;
|
||||
readonly quarantineRetentionMs?: number;
|
||||
readonly completionProcessor?: Pick<
|
||||
LocalCompletionReceiptProcessor,
|
||||
'process'
|
||||
>;
|
||||
readonly onDiagnostic?: (
|
||||
record: Readonly<{
|
||||
kind:
|
||||
| 'receipt_cleanup_failed'
|
||||
| 'receipt_quarantined'
|
||||
| 'journal_cleanup_failed';
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
}>,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
type CandidateDisposition =
|
||||
| Readonly<{ status: 'recovered' }>
|
||||
| Readonly<{ status: 'verified'; fingerprint: VerifiedFingerprint }>
|
||||
| Readonly<{ status: 'remaining' }>
|
||||
| Readonly<{ status: 'failed' }>;
|
||||
|
||||
interface AggregateSnapshot {
|
||||
readonly run: RunRecord;
|
||||
readonly attempt: RunAttemptRecord | null;
|
||||
}
|
||||
|
||||
interface VerifiedFingerprint {
|
||||
readonly runId: string;
|
||||
readonly runStatus: 'running';
|
||||
readonly runVersion: number;
|
||||
readonly attemptId: string;
|
||||
readonly attemptStatus: 'running';
|
||||
readonly callbackSequence: number;
|
||||
readonly executorHandle: string;
|
||||
readonly pid: number;
|
||||
}
|
||||
|
||||
interface EventDraft {
|
||||
readonly sequence: number;
|
||||
readonly type: string;
|
||||
readonly payload: Readonly<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
class LocalRunRecoveryConcurrentWriteError extends Error {}
|
||||
|
||||
function exactKeys(
|
||||
value: Readonly<Record<string, unknown>>,
|
||||
expected: readonly string[],
|
||||
): boolean {
|
||||
return (
|
||||
JSON.stringify(Object.keys(value).sort()) ===
|
||||
JSON.stringify([...expected].sort())
|
||||
);
|
||||
}
|
||||
|
||||
function assertSafeTimestamp(value: number, name: string): void {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new TypeError(`${name} must be a non-negative safe integer`);
|
||||
}
|
||||
}
|
||||
|
||||
function validatePage(value: unknown): LocalRunStartupRecoveryPage {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError('Local Run recovery page is invalid');
|
||||
}
|
||||
const page = value as unknown as Record<string, unknown>;
|
||||
if (
|
||||
!exactKeys(page, PAGE_KEYS) ||
|
||||
!Array.isArray(page.candidates) ||
|
||||
page.candidates.length > MAX_LOCAL_RUN_RECOVERY_ITEMS ||
|
||||
typeof page.truncated !== 'boolean'
|
||||
) {
|
||||
throw new TypeError('Local Run recovery page shape or bounds are invalid');
|
||||
}
|
||||
const candidates: LocalRunStartupRecoveryCandidate[] = [];
|
||||
const runIds = new Set<string>();
|
||||
for (const value of page.candidates) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError('Local Run recovery candidate is invalid');
|
||||
}
|
||||
const candidate = value as Record<string, unknown>;
|
||||
if (
|
||||
!exactKeys(candidate, CANDIDATE_KEYS) ||
|
||||
typeof candidate.runId !== 'string' ||
|
||||
candidate.runId.length < 1 ||
|
||||
candidate.runId.length > 128 ||
|
||||
/[\0\r\n]/.test(candidate.runId) ||
|
||||
(candidate.runStatus !== 'dispatching' &&
|
||||
candidate.runStatus !== 'running') ||
|
||||
!Number.isSafeInteger(candidate.activeAttemptCount) ||
|
||||
(candidate.activeAttemptCount as number) < 0 ||
|
||||
(candidate.activeAttemptCount as number) > MAX_LOCAL_RUN_RECOVERY_ITEMS ||
|
||||
runIds.has(candidate.runId)
|
||||
) {
|
||||
throw new TypeError('Local Run recovery candidate fields are invalid');
|
||||
}
|
||||
runIds.add(candidate.runId);
|
||||
candidates.push(
|
||||
Object.freeze({
|
||||
runId: candidate.runId,
|
||||
runStatus: candidate.runStatus,
|
||||
activeAttemptCount: candidate.activeAttemptCount as number,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
candidates: Object.freeze(candidates),
|
||||
truncated: page.truncated,
|
||||
});
|
||||
}
|
||||
|
||||
function reserveEvent(run: RunRecord): Readonly<{
|
||||
run: RunRecord;
|
||||
sequence: number;
|
||||
}> {
|
||||
const version = run.version + 1;
|
||||
const sequence = run.eventSequence + 1;
|
||||
if (
|
||||
!Number.isSafeInteger(version) ||
|
||||
version < 1 ||
|
||||
!Number.isSafeInteger(sequence) ||
|
||||
sequence < 1
|
||||
) {
|
||||
throw new TypeError('Local Run recovery version or sequence overflowed');
|
||||
}
|
||||
return Object.freeze({
|
||||
run: { ...run, version, eventSequence: sequence },
|
||||
sequence,
|
||||
});
|
||||
}
|
||||
|
||||
function event(
|
||||
id: string,
|
||||
runId: string,
|
||||
attemptId: string | undefined,
|
||||
draft: EventDraft,
|
||||
dedupeKey: string,
|
||||
actorType: RunEventRecord['actorType'],
|
||||
actorId: string,
|
||||
atMs: number,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id,
|
||||
runId,
|
||||
sequence: draft.sequence,
|
||||
type: draft.type,
|
||||
dedupeKey,
|
||||
actorType,
|
||||
actorId,
|
||||
...(attemptId === undefined ? {} : { attemptId }),
|
||||
payload: draft.payload,
|
||||
createdAtMs: atMs,
|
||||
};
|
||||
}
|
||||
|
||||
function atOrAfter(
|
||||
now: number,
|
||||
run: RunRecord,
|
||||
attempt?: RunAttemptRecord,
|
||||
): number {
|
||||
assertSafeTimestamp(now, 'Local Run recovery observation');
|
||||
return Math.max(
|
||||
now,
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt?.createdAtMs ?? 0,
|
||||
attempt?.startedAtMs ?? 0,
|
||||
);
|
||||
}
|
||||
|
||||
function fingerprint(snapshot: AggregateSnapshot): VerifiedFingerprint {
|
||||
const { run, attempt } = snapshot;
|
||||
if (
|
||||
run.status !== 'running' ||
|
||||
!attempt ||
|
||||
attempt.status !== 'running' ||
|
||||
attempt.executorType !== 'local_process' ||
|
||||
!attempt.executorHandle ||
|
||||
!Number.isSafeInteger(attempt.pid) ||
|
||||
(attempt.pid as number) < 1
|
||||
) {
|
||||
throw new TypeError('Verified local Run snapshot is not running');
|
||||
}
|
||||
return Object.freeze({
|
||||
runId: run.id,
|
||||
runStatus: 'running',
|
||||
runVersion: run.version,
|
||||
attemptId: attempt.id,
|
||||
attemptStatus: 'running',
|
||||
callbackSequence: attempt.callbackSequence,
|
||||
executorHandle: attempt.executorHandle,
|
||||
pid: attempt.pid as number,
|
||||
});
|
||||
}
|
||||
|
||||
function sameFingerprint(
|
||||
expected: VerifiedFingerprint,
|
||||
snapshot: AggregateSnapshot,
|
||||
): boolean {
|
||||
const attempt = snapshot.attempt;
|
||||
return (
|
||||
snapshot.run.id === expected.runId &&
|
||||
snapshot.run.status === expected.runStatus &&
|
||||
snapshot.run.version === expected.runVersion &&
|
||||
attempt?.id === expected.attemptId &&
|
||||
attempt.status === expected.attemptStatus &&
|
||||
attempt.callbackSequence === expected.callbackSequence &&
|
||||
attempt.executorHandle === expected.executorHandle &&
|
||||
attempt.pid === expected.pid
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* One bounded startup coordinator. It never scans receipt directories, starts
|
||||
* an Executor, or treats missing/invalid evidence as proof of process exit.
|
||||
*/
|
||||
export class LocalRunStartupRecoveryCoordinator {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly wait: (delayMs: number) => Promise<void>;
|
||||
private readonly receiptPublishGraceMs: number;
|
||||
private readonly createEventId: () => string;
|
||||
private readonly journal?: LocalRunStartupRecoveryCoordinatorOptions['journal'];
|
||||
private readonly quarantineRetentionMs: number;
|
||||
private readonly onDiagnostic?: LocalRunStartupRecoveryCoordinatorOptions['onDiagnostic'];
|
||||
private readonly completionProcessor: Pick<
|
||||
LocalCompletionReceiptProcessor,
|
||||
'process'
|
||||
>;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly source: LocalRunStartupRecoverySource,
|
||||
receipts: CompletionReceiptStore,
|
||||
private readonly inspector: LocalPersistedExecutionInspector,
|
||||
options: LocalRunStartupRecoveryCoordinatorOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.wait =
|
||||
options.wait ??
|
||||
((delayMs) =>
|
||||
new Promise((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
}));
|
||||
this.receiptPublishGraceMs = options.receiptPublishGraceMs ?? 0;
|
||||
if (
|
||||
!Number.isSafeInteger(this.receiptPublishGraceMs) ||
|
||||
this.receiptPublishGraceMs < 0 ||
|
||||
this.receiptPublishGraceMs > MAX_LOCAL_RUN_RECEIPT_GRACE_MS
|
||||
) {
|
||||
throw new RangeError(
|
||||
`receiptPublishGraceMs must be between 0 and ${MAX_LOCAL_RUN_RECEIPT_GRACE_MS}`,
|
||||
);
|
||||
}
|
||||
if (inspector.executorType !== 'local_process') {
|
||||
throw new TypeError('Local Run recovery inspector type is invalid');
|
||||
}
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
this.journal = options.journal;
|
||||
this.quarantineRetentionMs = options.quarantineRetentionMs ?? 60 * 60_000;
|
||||
if (
|
||||
!Number.isSafeInteger(this.quarantineRetentionMs) ||
|
||||
this.quarantineRetentionMs < 0 ||
|
||||
this.quarantineRetentionMs > MAX_LOCAL_RUN_QUARANTINE_RETENTION_MS
|
||||
) {
|
||||
throw new RangeError(
|
||||
`quarantineRetentionMs must be between 0 and ${MAX_LOCAL_RUN_QUARANTINE_RETENTION_MS}`,
|
||||
);
|
||||
}
|
||||
this.onDiagnostic = options.onDiagnostic;
|
||||
this.completionProcessor =
|
||||
options.completionProcessor ??
|
||||
new LocalCompletionReceiptProcessor(repository, receipts, {
|
||||
clock: this.clock,
|
||||
createEventId: this.createEventId,
|
||||
...(this.journal === undefined ? {} : { journal: this.journal }),
|
||||
quarantineRetentionMs: this.quarantineRetentionMs,
|
||||
...(this.onDiagnostic === undefined
|
||||
? {}
|
||||
: { onDiagnostic: this.onDiagnostic }),
|
||||
});
|
||||
}
|
||||
|
||||
async recover(): Promise<LocalRunStartupRecoverySummary> {
|
||||
const initial = validatePage(
|
||||
await this.source.inspectCandidates({
|
||||
limit: MAX_LOCAL_RUN_RECOVERY_ITEMS,
|
||||
}),
|
||||
);
|
||||
if (initial.truncated) {
|
||||
return Object.freeze({
|
||||
safe: false,
|
||||
scanned: initial.candidates.length,
|
||||
recovered: 0,
|
||||
remaining: initial.candidates.length,
|
||||
failed: 0,
|
||||
truncated: true,
|
||||
});
|
||||
}
|
||||
if (initial.candidates.length === 0) {
|
||||
return Object.freeze({
|
||||
safe: true,
|
||||
scanned: 0,
|
||||
recovered: 0,
|
||||
remaining: 0,
|
||||
failed: 0,
|
||||
truncated: false,
|
||||
});
|
||||
}
|
||||
|
||||
let recovered = 0;
|
||||
let remaining = 0;
|
||||
let failed = 0;
|
||||
const verified = new Map<string, VerifiedFingerprint>();
|
||||
for (const candidate of initial.candidates) {
|
||||
const disposition = await this.reconcile(candidate);
|
||||
if (disposition.status === 'recovered') recovered += 1;
|
||||
if (disposition.status === 'verified') {
|
||||
recovered += 1;
|
||||
verified.set(candidate.runId, disposition.fingerprint);
|
||||
}
|
||||
if (disposition.status === 'remaining') remaining += 1;
|
||||
if (disposition.status === 'failed') failed += 1;
|
||||
}
|
||||
|
||||
if (remaining === 0 && failed === 0) {
|
||||
const final = await this.verify(verified);
|
||||
if (final.remaining > 0 || final.failed > 0) {
|
||||
const unresolved = Math.min(
|
||||
initial.candidates.length,
|
||||
Math.max(1, final.remaining + final.failed),
|
||||
);
|
||||
recovered = Math.max(0, recovered - unresolved);
|
||||
failed = Math.min(unresolved, final.failed);
|
||||
remaining = unresolved - failed;
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
safe: remaining === 0 && failed === 0,
|
||||
scanned: initial.candidates.length,
|
||||
recovered,
|
||||
remaining,
|
||||
failed,
|
||||
truncated: false,
|
||||
});
|
||||
}
|
||||
|
||||
private async reconcile(
|
||||
candidate: LocalRunStartupRecoveryCandidate,
|
||||
): Promise<CandidateDisposition> {
|
||||
try {
|
||||
if (candidate.activeAttemptCount > 1) {
|
||||
return Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
let snapshot = await this.load(candidate.runId);
|
||||
if (
|
||||
!snapshot ||
|
||||
snapshot.run.executionOwner !== 'runtime' ||
|
||||
snapshot.run.status !== candidate.runStatus
|
||||
) {
|
||||
return Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
const attempt = snapshot.attempt;
|
||||
if (candidate.activeAttemptCount === 0) {
|
||||
if (snapshot.run.status !== 'dispatching' || attempt !== null) {
|
||||
return Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
return (await this.markRunLostWithoutAttempt(snapshot.run))
|
||||
? Object.freeze({ status: 'recovered' })
|
||||
: Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
if (
|
||||
!attempt ||
|
||||
!ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
|
||||
attempt.runId !== snapshot.run.id
|
||||
) {
|
||||
return Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
|
||||
const completion = await this.completionProcessor.process(attempt.id);
|
||||
if (completion === 'completed' || completion === 'already_terminal') {
|
||||
return Object.freeze({ status: 'recovered' });
|
||||
}
|
||||
if (completion === 'invalid')
|
||||
return Object.freeze({ status: 'remaining' });
|
||||
if (attempt.status === 'claimed') {
|
||||
return (await this.markAggregateLost(snapshot, 'unstarted_claim'))
|
||||
? Object.freeze({ status: 'recovered' })
|
||||
: Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
if (
|
||||
attempt.executorType !== 'local_process' ||
|
||||
!attempt.executorHandle ||
|
||||
!Number.isSafeInteger(attempt.pid) ||
|
||||
(attempt.pid as number) < 1
|
||||
) {
|
||||
return Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
|
||||
const inspection = await this.inspector.inspect(attempt.executorHandle);
|
||||
if (inspection.status === 'unknown') {
|
||||
return inspection.reason === 'provider_unavailable'
|
||||
? Object.freeze({ status: 'failed' })
|
||||
: Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
if (inspection.identityPid !== attempt.pid) {
|
||||
return Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
if (inspection.status === 'running') {
|
||||
snapshot = await this.markObservedRunning(snapshot);
|
||||
return snapshot
|
||||
? Object.freeze({
|
||||
status: 'verified',
|
||||
fingerprint: fingerprint(snapshot),
|
||||
})
|
||||
: Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
|
||||
if (this.receiptPublishGraceMs > 0) {
|
||||
await this.wait(this.receiptPublishGraceMs);
|
||||
const delayed = await this.load(candidate.runId);
|
||||
if (!delayed || !delayed.attempt || delayed.attempt.id !== attempt.id) {
|
||||
return Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
const lateCompletion = await this.completionProcessor.process(
|
||||
attempt.id,
|
||||
);
|
||||
if (lateCompletion === 'invalid') {
|
||||
return Object.freeze({ status: 'remaining' });
|
||||
}
|
||||
if (
|
||||
lateCompletion === 'completed' ||
|
||||
lateCompletion === 'already_terminal'
|
||||
) {
|
||||
return Object.freeze({ status: 'recovered' });
|
||||
}
|
||||
snapshot = delayed;
|
||||
}
|
||||
return (await this.markAggregateLost(snapshot, 'process_not_running'))
|
||||
? Object.freeze({ status: 'recovered' })
|
||||
: Object.freeze({ status: 'remaining' });
|
||||
} catch {
|
||||
return Object.freeze({ status: 'failed' });
|
||||
}
|
||||
}
|
||||
|
||||
private async verify(
|
||||
verified: ReadonlyMap<string, VerifiedFingerprint>,
|
||||
): Promise<Readonly<{ remaining: number; failed: number }>> {
|
||||
let remaining = 0;
|
||||
let failed = 0;
|
||||
let page: LocalRunStartupRecoveryPage;
|
||||
try {
|
||||
page = validatePage(
|
||||
await this.source.inspectCandidates({
|
||||
limit: MAX_LOCAL_RUN_RECOVERY_ITEMS,
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
return Object.freeze({ remaining: 0, failed: 1 });
|
||||
}
|
||||
if (page.truncated) {
|
||||
return Object.freeze({
|
||||
remaining: page.candidates.length,
|
||||
failed: 0,
|
||||
});
|
||||
}
|
||||
for (const candidate of page.candidates) {
|
||||
const expected = verified.get(candidate.runId);
|
||||
if (!expected || candidate.activeAttemptCount !== 1) {
|
||||
remaining += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const completion = await this.completionProcessor.process(
|
||||
expected.attemptId,
|
||||
);
|
||||
if (completion === 'completed' || completion === 'already_terminal') {
|
||||
continue;
|
||||
}
|
||||
if (completion === 'invalid') {
|
||||
remaining += 1;
|
||||
continue;
|
||||
}
|
||||
const snapshot = await this.load(candidate.runId);
|
||||
if (!snapshot || !sameFingerprint(expected, snapshot)) {
|
||||
remaining += 1;
|
||||
continue;
|
||||
}
|
||||
const inspection = await this.inspector.inspect(
|
||||
expected.executorHandle,
|
||||
);
|
||||
if (
|
||||
inspection.status !== 'running' ||
|
||||
inspection.identityPid !== expected.pid
|
||||
) {
|
||||
remaining += 1;
|
||||
}
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
return Object.freeze({ remaining, failed });
|
||||
}
|
||||
|
||||
private load(runId: string): Promise<AggregateSnapshot | null> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(runId);
|
||||
if (!run) return null;
|
||||
const attempt = await transaction.findLatestAttemptByRunId(runId);
|
||||
return Object.freeze({ run, attempt });
|
||||
});
|
||||
}
|
||||
|
||||
private async markRunLostWithoutAttempt(run: RunRecord): Promise<boolean> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const current = await transaction.findRunById(run.id);
|
||||
if (
|
||||
!current ||
|
||||
current.version !== run.version ||
|
||||
current.status !== 'dispatching' ||
|
||||
current.executionOwner !== 'runtime' ||
|
||||
(await transaction.findLatestAttemptByRunId(run.id)) !== null
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const atMs = atOrAfter(this.clock.now(), current);
|
||||
const reserved = reserveEvent(current);
|
||||
const next: RunRecord = {
|
||||
...reserved.run,
|
||||
status: 'lost',
|
||||
errorCode: 'RECOVERY_ATTEMPT_MISSING_BEFORE_START',
|
||||
errorSummary: 'Dispatching Run has no durable Attempt',
|
||||
};
|
||||
if (!(await transaction.compareAndSetRun(next, current.version))) {
|
||||
throw new LocalRunRecoveryConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
current.id,
|
||||
undefined,
|
||||
{
|
||||
sequence: reserved.sequence,
|
||||
type: 'run.lost',
|
||||
payload: Object.freeze({
|
||||
from_status: current.status,
|
||||
to_status: 'lost',
|
||||
error_code: next.errorCode,
|
||||
version: next.version,
|
||||
}),
|
||||
},
|
||||
`local-recovery:run:${current.id}:${current.version}:missing-attempt`,
|
||||
'reconciler',
|
||||
'local-startup',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
private async markAggregateLost(
|
||||
expected: AggregateSnapshot,
|
||||
reason: 'unstarted_claim' | 'process_not_running',
|
||||
): Promise<boolean> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const current = await this.reloadExact(transaction, expected);
|
||||
if (!current || !current.attempt) return false;
|
||||
const { run, attempt } = current;
|
||||
if (
|
||||
run.cancelRequestedAtMs !== undefined ||
|
||||
!ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
|
||||
(reason === 'unstarted_claim' && attempt.status !== 'claimed') ||
|
||||
(reason === 'process_not_running' &&
|
||||
attempt.status !== 'starting' &&
|
||||
attempt.status !== 'running')
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const errorCode =
|
||||
reason === 'unstarted_claim'
|
||||
? 'RECOVERY_UNSTARTED_CLAIM'
|
||||
: 'RECOVERY_PROCESS_NOT_RUNNING';
|
||||
const errorSummary =
|
||||
reason === 'unstarted_claim'
|
||||
? 'Claimed Attempt never crossed the start barrier'
|
||||
: 'Durable process identity proves the execution is not running';
|
||||
const atMs = atOrAfter(this.clock.now(), run, attempt);
|
||||
const attemptReserved = reserveEvent(run);
|
||||
const nextAttempt: RunAttemptRecord = {
|
||||
...attempt,
|
||||
status: 'lost',
|
||||
finishedAtMs: atMs,
|
||||
errorCode,
|
||||
errorSummary,
|
||||
};
|
||||
await this.persistRunAndAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
attemptReserved.run,
|
||||
nextAttempt,
|
||||
);
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run.id,
|
||||
attempt.id,
|
||||
{
|
||||
sequence: attemptReserved.sequence,
|
||||
type: 'attempt.lost',
|
||||
payload: Object.freeze({
|
||||
attempt_id: attempt.id,
|
||||
from_status: attempt.status,
|
||||
to_status: 'lost',
|
||||
error_code: errorCode,
|
||||
version: attemptReserved.run.version,
|
||||
}),
|
||||
},
|
||||
`local-recovery:attempt:${attempt.id}:${attempt.callbackSequence}:${reason}`,
|
||||
'reconciler',
|
||||
'local-startup',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
const runReserved = reserveEvent(attemptReserved.run);
|
||||
const nextRun: RunRecord = {
|
||||
...runReserved.run,
|
||||
status: 'lost',
|
||||
errorCode,
|
||||
errorSummary,
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
nextRun,
|
||||
attemptReserved.run.version,
|
||||
))
|
||||
) {
|
||||
throw new LocalRunRecoveryConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run.id,
|
||||
attempt.id,
|
||||
{
|
||||
sequence: runReserved.sequence,
|
||||
type: 'run.lost',
|
||||
payload: Object.freeze({
|
||||
from_status: run.status,
|
||||
to_status: 'lost',
|
||||
error_code: errorCode,
|
||||
version: nextRun.version,
|
||||
}),
|
||||
},
|
||||
`local-recovery:run:${run.id}:${attempt.id}:${run.version}:${reason}`,
|
||||
'reconciler',
|
||||
'local-startup',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
private async markObservedRunning(
|
||||
expected: AggregateSnapshot,
|
||||
): Promise<AggregateSnapshot | null> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const current = await this.reloadExact(transaction, expected);
|
||||
if (!current || !current.attempt) return null;
|
||||
let { run, attempt } = current;
|
||||
const originalRunStatus = run.status;
|
||||
const atMs = atOrAfter(this.clock.now(), run, attempt);
|
||||
if (attempt.status === 'starting') {
|
||||
const reserved = reserveEvent(run);
|
||||
const nextAttempt: RunAttemptRecord = {
|
||||
...attempt,
|
||||
status: 'running',
|
||||
startedAtMs: attempt.startedAtMs ?? atMs,
|
||||
};
|
||||
await this.persistRunAndAttempt(
|
||||
transaction,
|
||||
run,
|
||||
attempt,
|
||||
reserved.run,
|
||||
nextAttempt,
|
||||
);
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run.id,
|
||||
attempt.id,
|
||||
{
|
||||
sequence: reserved.sequence,
|
||||
type: 'attempt.running',
|
||||
payload: Object.freeze({
|
||||
attempt_id: attempt.id,
|
||||
from_status: attempt.status,
|
||||
to_status: 'running',
|
||||
evidence: 'durable_process_identity',
|
||||
version: reserved.run.version,
|
||||
}),
|
||||
},
|
||||
`local-recovery:attempt:${attempt.id}:${attempt.callbackSequence}:running`,
|
||||
'reconciler',
|
||||
'local-startup',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
run = reserved.run;
|
||||
attempt = nextAttempt;
|
||||
}
|
||||
if (run.status === 'dispatching') {
|
||||
const reserved = reserveEvent(run);
|
||||
const nextRun: RunRecord = {
|
||||
...reserved.run,
|
||||
status: 'running',
|
||||
startedAtMs: run.startedAtMs ?? atMs,
|
||||
};
|
||||
if (!(await transaction.compareAndSetRun(nextRun, run.version))) {
|
||||
throw new LocalRunRecoveryConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run.id,
|
||||
attempt.id,
|
||||
{
|
||||
sequence: reserved.sequence,
|
||||
type: 'run.running',
|
||||
payload: Object.freeze({
|
||||
from_status: originalRunStatus,
|
||||
to_status: 'running',
|
||||
evidence: 'durable_process_identity',
|
||||
version: nextRun.version,
|
||||
}),
|
||||
},
|
||||
`local-recovery:run:${run.id}:${attempt.id}:${run.version}:running`,
|
||||
'reconciler',
|
||||
'local-startup',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
run = nextRun;
|
||||
}
|
||||
if (run.status !== 'running' || attempt.status !== 'running') return null;
|
||||
return Object.freeze({ run, attempt });
|
||||
});
|
||||
}
|
||||
|
||||
private async reloadExact(
|
||||
transaction: RunRepositoryTransaction,
|
||||
expected: AggregateSnapshot,
|
||||
): Promise<AggregateSnapshot | null> {
|
||||
const run = await transaction.findRunById(expected.run.id);
|
||||
const expectedAttempt = expected.attempt;
|
||||
if (!run || run.version !== expected.run.version) return null;
|
||||
if (!expectedAttempt) {
|
||||
return Object.freeze({ run, attempt: null });
|
||||
}
|
||||
const attempt = await transaction.findAttemptById(expectedAttempt.id);
|
||||
if (
|
||||
!attempt ||
|
||||
attempt.runId !== run.id ||
|
||||
attempt.status !== expectedAttempt.status ||
|
||||
attempt.callbackSequence !== expectedAttempt.callbackSequence ||
|
||||
attempt.executorHandle !== expectedAttempt.executorHandle ||
|
||||
attempt.pid !== expectedAttempt.pid
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return Object.freeze({ run, attempt });
|
||||
}
|
||||
|
||||
private async persistRunAndAttempt(
|
||||
transaction: RunRepositoryTransaction,
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
nextRun: RunRecord,
|
||||
nextAttempt: RunAttemptRecord,
|
||||
): Promise<void> {
|
||||
if (!(await transaction.compareAndSetRun(nextRun, run.version))) {
|
||||
throw new LocalRunRecoveryConcurrentWriteError();
|
||||
}
|
||||
if (
|
||||
!(await transaction.compareAndSetAttempt(nextAttempt, {
|
||||
status: attempt.status,
|
||||
callbackSequence: attempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new LocalRunRecoveryConcurrentWriteError();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from './coordinator';
|
||||
export * from './workflowTask';
|
||||
@@ -0,0 +1,461 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunRecord,
|
||||
RunRepository,
|
||||
} from '@qinglong/runtime-core/run-repository';
|
||||
import type {
|
||||
LocalPersistedExecutionInspector,
|
||||
} from '@qinglong/local-process';
|
||||
|
||||
import type { LocalCompletionReceiptProcessor } from '../control/completion';
|
||||
import type { LocalWorkflowTaskExecutionRepository } from '../execution/workflowTaskExecution';
|
||||
|
||||
export const MAX_LOCAL_WORKFLOW_TASK_RECOVERY_ITEMS = 64;
|
||||
|
||||
export interface LocalWorkflowTaskRecoveryCandidate {
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly attemptCreatedAtMs: number;
|
||||
}
|
||||
|
||||
export interface LocalWorkflowTaskRecoveryPage {
|
||||
readonly candidates: readonly Readonly<LocalWorkflowTaskRecoveryCandidate>[];
|
||||
readonly truncated: boolean;
|
||||
}
|
||||
|
||||
export interface LocalWorkflowTaskRecoveryRepository {
|
||||
listRecoveryCandidates(command: Readonly<{
|
||||
limit: number;
|
||||
}>): Promise<Readonly<LocalWorkflowTaskRecoveryPage>>;
|
||||
recover(command: Readonly<{
|
||||
run: Readonly<RunRecord>;
|
||||
attempt: Readonly<RunAttemptRecord>;
|
||||
reason: 'unstarted_claim_expired' | 'execution_not_running';
|
||||
observedAtMs: number;
|
||||
}>): Promise<'requeued' | 'failed' | 'already_recovered' | 'stale'>;
|
||||
}
|
||||
|
||||
export interface LocalWorkflowTaskStartupRecoveryOptions {
|
||||
readonly receiptPublishGraceMs?: number;
|
||||
readonly clock?: { now(): number };
|
||||
readonly wait?: (delayMs: number) => Promise<void>;
|
||||
readonly createEventId?: () => string;
|
||||
}
|
||||
|
||||
export interface LocalWorkflowTaskStartupRecoverySummary {
|
||||
readonly safe: boolean;
|
||||
readonly scanned: number;
|
||||
readonly recovered: number;
|
||||
readonly verified: number;
|
||||
readonly remaining: number;
|
||||
readonly failed: number;
|
||||
readonly truncated: boolean;
|
||||
}
|
||||
|
||||
interface ActiveSnapshot {
|
||||
readonly run: Readonly<RunRecord>;
|
||||
readonly attempt: Readonly<RunAttemptRecord>;
|
||||
}
|
||||
|
||||
interface VerifiedFingerprint {
|
||||
readonly runId: string;
|
||||
readonly runVersion: number;
|
||||
readonly attemptId: string;
|
||||
readonly callbackSequence: number;
|
||||
readonly executorHandle: string;
|
||||
readonly pid: number;
|
||||
}
|
||||
|
||||
function timestamp(value: unknown, label: string): number {
|
||||
if (!Number.isSafeInteger(value) || (value as number) < 0) {
|
||||
throw new TypeError(`${label} is invalid`);
|
||||
}
|
||||
return value as number;
|
||||
}
|
||||
|
||||
function validatePage(
|
||||
value: Readonly<LocalWorkflowTaskRecoveryPage>,
|
||||
): Readonly<LocalWorkflowTaskRecoveryPage> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!Array.isArray(value.candidates) ||
|
||||
value.candidates.length > MAX_LOCAL_WORKFLOW_TASK_RECOVERY_ITEMS ||
|
||||
typeof value.truncated !== 'boolean'
|
||||
) {
|
||||
throw new TypeError('Local Workflow Task recovery page is invalid');
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
let previous:
|
||||
| Readonly<LocalWorkflowTaskRecoveryCandidate>
|
||||
| undefined;
|
||||
const candidates = value.candidates.map((candidate) => {
|
||||
if (
|
||||
!candidate ||
|
||||
typeof candidate !== 'object' ||
|
||||
Array.isArray(candidate) ||
|
||||
typeof candidate.runId !== 'string' ||
|
||||
candidate.runId.length < 1 ||
|
||||
typeof candidate.attemptId !== 'string' ||
|
||||
candidate.attemptId.length < 1 ||
|
||||
seen.has(candidate.attemptId)
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow Task recovery candidate is invalid',
|
||||
);
|
||||
}
|
||||
const attemptCreatedAtMs = timestamp(
|
||||
candidate.attemptCreatedAtMs,
|
||||
'Local Workflow Task recovery candidate timestamp',
|
||||
);
|
||||
const normalized = Object.freeze({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
attemptCreatedAtMs,
|
||||
});
|
||||
if (
|
||||
previous &&
|
||||
(normalized.attemptCreatedAtMs < previous.attemptCreatedAtMs ||
|
||||
(normalized.attemptCreatedAtMs === previous.attemptCreatedAtMs &&
|
||||
normalized.attemptId <= previous.attemptId))
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow Task recovery candidates are not ordered',
|
||||
);
|
||||
}
|
||||
seen.add(normalized.attemptId);
|
||||
previous = normalized;
|
||||
return normalized;
|
||||
});
|
||||
return Object.freeze({
|
||||
candidates: Object.freeze(candidates),
|
||||
truncated: value.truncated,
|
||||
});
|
||||
}
|
||||
|
||||
function fingerprint(snapshot: ActiveSnapshot): VerifiedFingerprint {
|
||||
if (
|
||||
snapshot.run.status !== 'running' ||
|
||||
snapshot.attempt.status !== 'running' ||
|
||||
!snapshot.attempt.stepRunId ||
|
||||
!snapshot.attempt.executorHandle ||
|
||||
!Number.isSafeInteger(snapshot.attempt.pid) ||
|
||||
snapshot.attempt.pid! < 1
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow Task recovery fingerprint is invalid',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
runId: snapshot.run.id,
|
||||
runVersion: snapshot.run.version,
|
||||
attemptId: snapshot.attempt.id,
|
||||
callbackSequence: snapshot.attempt.callbackSequence,
|
||||
executorHandle: snapshot.attempt.executorHandle,
|
||||
pid: snapshot.attempt.pid!,
|
||||
});
|
||||
}
|
||||
|
||||
export class LocalWorkflowTaskStartupRecoveryCoordinator {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly wait: (delayMs: number) => Promise<void>;
|
||||
private readonly receiptPublishGraceMs: number;
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly runs: RunRepository,
|
||||
private readonly recovery: LocalWorkflowTaskRecoveryRepository,
|
||||
private readonly workflowTasks: LocalWorkflowTaskExecutionRepository,
|
||||
private readonly completions: Pick<
|
||||
LocalCompletionReceiptProcessor,
|
||||
'process'
|
||||
>,
|
||||
private readonly inspector: LocalPersistedExecutionInspector,
|
||||
options: LocalWorkflowTaskStartupRecoveryOptions = {},
|
||||
) {
|
||||
if (
|
||||
typeof recovery?.listRecoveryCandidates !== 'function' ||
|
||||
typeof recovery?.recover !== 'function' ||
|
||||
typeof workflowTasks?.recordRunning !== 'function' ||
|
||||
typeof completions?.process !== 'function' ||
|
||||
inspector?.executorType !== 'local_process'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow Task startup recovery dependencies are invalid',
|
||||
);
|
||||
}
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.wait =
|
||||
options.wait ??
|
||||
((delayMs) =>
|
||||
new Promise((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
}));
|
||||
this.receiptPublishGraceMs = options.receiptPublishGraceMs ?? 0;
|
||||
if (
|
||||
!Number.isSafeInteger(this.receiptPublishGraceMs) ||
|
||||
this.receiptPublishGraceMs < 0 ||
|
||||
this.receiptPublishGraceMs > 5_000
|
||||
) {
|
||||
throw new RangeError(
|
||||
'Local Workflow Task receipt grace is invalid',
|
||||
);
|
||||
}
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
}
|
||||
|
||||
async recover(): Promise<Readonly<LocalWorkflowTaskStartupRecoverySummary>> {
|
||||
const initial = validatePage(
|
||||
await this.recovery.listRecoveryCandidates({
|
||||
limit: MAX_LOCAL_WORKFLOW_TASK_RECOVERY_ITEMS,
|
||||
}),
|
||||
);
|
||||
if (initial.truncated) {
|
||||
return Object.freeze({
|
||||
safe: false,
|
||||
scanned: initial.candidates.length,
|
||||
recovered: 0,
|
||||
verified: 0,
|
||||
remaining: initial.candidates.length,
|
||||
failed: 0,
|
||||
truncated: true,
|
||||
});
|
||||
}
|
||||
let recovered = 0;
|
||||
let remaining = 0;
|
||||
let failed = 0;
|
||||
const verified = new Map<string, VerifiedFingerprint>();
|
||||
for (const candidate of initial.candidates) {
|
||||
try {
|
||||
const result = await this.reconcile(candidate);
|
||||
if (result.status === 'recovered') recovered += 1;
|
||||
else if (result.status === 'verified') {
|
||||
verified.set(candidate.attemptId, result.fingerprint);
|
||||
} else if (result.status === 'remaining') remaining += 1;
|
||||
else failed += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
if (remaining === 0 && failed === 0) {
|
||||
const final = await this.verify(verified);
|
||||
remaining += final.remaining;
|
||||
failed += final.failed;
|
||||
}
|
||||
return Object.freeze({
|
||||
safe: remaining === 0 && failed === 0,
|
||||
scanned: initial.candidates.length,
|
||||
recovered,
|
||||
verified: verified.size,
|
||||
remaining,
|
||||
failed,
|
||||
truncated: false,
|
||||
});
|
||||
}
|
||||
|
||||
private async reconcile(
|
||||
candidate: Readonly<LocalWorkflowTaskRecoveryCandidate>,
|
||||
): Promise<
|
||||
| Readonly<{ status: 'recovered' }>
|
||||
| Readonly<{
|
||||
status: 'verified';
|
||||
fingerprint: VerifiedFingerprint;
|
||||
}>
|
||||
| Readonly<{ status: 'remaining' }>
|
||||
| Readonly<{ status: 'failed' }>
|
||||
> {
|
||||
let snapshot = await this.load(candidate);
|
||||
if (!snapshot) return Object.freeze({ status: 'remaining' as const });
|
||||
const completion = await this.completions.process(candidate.attemptId);
|
||||
if (completion === 'completed' || completion === 'already_terminal') {
|
||||
return Object.freeze({ status: 'recovered' as const });
|
||||
}
|
||||
if (completion === 'invalid') {
|
||||
return Object.freeze({ status: 'remaining' as const });
|
||||
}
|
||||
if (snapshot.attempt.status === 'claimed') {
|
||||
const result = await this.recovery.recover({
|
||||
...snapshot,
|
||||
reason: 'unstarted_claim_expired',
|
||||
observedAtMs: Math.max(
|
||||
timestamp(
|
||||
this.clock.now(),
|
||||
'Local Workflow Task recovery clock',
|
||||
),
|
||||
snapshot.attempt.createdAtMs,
|
||||
),
|
||||
});
|
||||
return result === 'requeued' || result === 'already_recovered'
|
||||
? Object.freeze({ status: 'recovered' as const })
|
||||
: Object.freeze({ status: 'remaining' as const });
|
||||
}
|
||||
if (
|
||||
(snapshot.attempt.status !== 'starting' &&
|
||||
snapshot.attempt.status !== 'running') ||
|
||||
!snapshot.attempt.executorHandle ||
|
||||
!Number.isSafeInteger(snapshot.attempt.pid) ||
|
||||
snapshot.attempt.pid! < 1
|
||||
) {
|
||||
return Object.freeze({ status: 'remaining' as const });
|
||||
}
|
||||
const inspection = await this.inspector.inspect(
|
||||
snapshot.attempt.executorHandle,
|
||||
);
|
||||
if (inspection.status === 'unknown') {
|
||||
return inspection.reason === 'provider_unavailable'
|
||||
? Object.freeze({ status: 'failed' as const })
|
||||
: Object.freeze({ status: 'remaining' as const });
|
||||
}
|
||||
if (inspection.identityPid !== snapshot.attempt.pid) {
|
||||
return Object.freeze({ status: 'remaining' as const });
|
||||
}
|
||||
if (inspection.status === 'running') {
|
||||
if (snapshot.attempt.status === 'starting') {
|
||||
if (!snapshot.attempt.callbackTokenHash) {
|
||||
return Object.freeze({ status: 'remaining' as const });
|
||||
}
|
||||
const marked = await this.workflowTasks.recordRunning({
|
||||
run: snapshot.run,
|
||||
attempt: snapshot.attempt,
|
||||
callbackTokenHash: snapshot.attempt.callbackTokenHash,
|
||||
executorHandle: snapshot.attempt.executorHandle,
|
||||
pid: snapshot.attempt.pid,
|
||||
startedAtMs: Math.max(
|
||||
timestamp(
|
||||
this.clock.now(),
|
||||
'Local Workflow Task recovery clock',
|
||||
),
|
||||
snapshot.attempt.createdAtMs,
|
||||
),
|
||||
attemptEventId: this.createEventId(),
|
||||
stepMutationId: this.createEventId(),
|
||||
});
|
||||
if (marked.status !== 'applied') {
|
||||
return Object.freeze({ status: 'remaining' as const });
|
||||
}
|
||||
snapshot = await this.load(candidate);
|
||||
if (!snapshot || snapshot.attempt.status !== 'running') {
|
||||
return Object.freeze({ status: 'remaining' as const });
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
status: 'verified' as const,
|
||||
fingerprint: fingerprint(snapshot),
|
||||
});
|
||||
}
|
||||
if (this.receiptPublishGraceMs > 0) {
|
||||
await this.wait(this.receiptPublishGraceMs);
|
||||
const late = await this.completions.process(candidate.attemptId);
|
||||
if (late === 'completed' || late === 'already_terminal') {
|
||||
return Object.freeze({ status: 'recovered' as const });
|
||||
}
|
||||
if (late === 'invalid') {
|
||||
return Object.freeze({ status: 'remaining' as const });
|
||||
}
|
||||
snapshot = await this.load(candidate);
|
||||
if (!snapshot) return Object.freeze({ status: 'recovered' as const });
|
||||
}
|
||||
const result = await this.recovery.recover({
|
||||
...snapshot,
|
||||
reason: 'execution_not_running',
|
||||
observedAtMs: Math.max(
|
||||
timestamp(
|
||||
this.clock.now(),
|
||||
'Local Workflow Task recovery clock',
|
||||
),
|
||||
snapshot.attempt.createdAtMs,
|
||||
snapshot.attempt.startedAtMs ?? 0,
|
||||
),
|
||||
});
|
||||
return result === 'failed' || result === 'already_recovered'
|
||||
? Object.freeze({ status: 'recovered' as const })
|
||||
: Object.freeze({ status: 'remaining' as const });
|
||||
}
|
||||
|
||||
private async verify(
|
||||
expected: ReadonlyMap<string, VerifiedFingerprint>,
|
||||
): Promise<Readonly<{ remaining: number; failed: number }>> {
|
||||
let page: Readonly<LocalWorkflowTaskRecoveryPage>;
|
||||
try {
|
||||
page = validatePage(
|
||||
await this.recovery.listRecoveryCandidates({
|
||||
limit: MAX_LOCAL_WORKFLOW_TASK_RECOVERY_ITEMS,
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
return Object.freeze({ remaining: 0, failed: 1 });
|
||||
}
|
||||
if (page.truncated) {
|
||||
return Object.freeze({
|
||||
remaining: page.candidates.length,
|
||||
failed: 0,
|
||||
});
|
||||
}
|
||||
let remaining = 0;
|
||||
let failed = 0;
|
||||
for (const candidate of page.candidates) {
|
||||
const fingerprintValue = expected.get(candidate.attemptId);
|
||||
if (!fingerprintValue) {
|
||||
remaining += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const snapshot = await this.load(candidate);
|
||||
if (
|
||||
!snapshot ||
|
||||
snapshot.run.version !== fingerprintValue.runVersion ||
|
||||
snapshot.attempt.status !== 'running' ||
|
||||
snapshot.attempt.callbackSequence !==
|
||||
fingerprintValue.callbackSequence ||
|
||||
snapshot.attempt.executorHandle !==
|
||||
fingerprintValue.executorHandle ||
|
||||
snapshot.attempt.pid !== fingerprintValue.pid
|
||||
) {
|
||||
remaining += 1;
|
||||
continue;
|
||||
}
|
||||
const inspection = await this.inspector.inspect(
|
||||
fingerprintValue.executorHandle,
|
||||
);
|
||||
if (
|
||||
inspection.status !== 'running' ||
|
||||
inspection.identityPid !== fingerprintValue.pid
|
||||
) {
|
||||
remaining += 1;
|
||||
}
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
return Object.freeze({ remaining, failed });
|
||||
}
|
||||
|
||||
private load(
|
||||
candidate: Readonly<LocalWorkflowTaskRecoveryCandidate>,
|
||||
): Promise<ActiveSnapshot | null> {
|
||||
return this.runs.transaction(async (transaction) => {
|
||||
const [run, attempt] = await Promise.all([
|
||||
transaction.findRunById(candidate.runId),
|
||||
transaction.findAttemptById(candidate.attemptId),
|
||||
]);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
run.status !== 'running' ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
run.cancelRequestedAtMs !== undefined ||
|
||||
attempt.runId !== run.id ||
|
||||
!attempt.stepRunId ||
|
||||
attempt.executorType !== 'local_process' ||
|
||||
!['claimed', 'starting', 'running'].includes(attempt.status) ||
|
||||
attempt.createdAtMs !== candidate.attemptCreatedAtMs
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return Object.freeze({ run, attempt });
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
assertLocalSchedulePageSize,
|
||||
resolveLocalScheduleDecision,
|
||||
type LocalCronNextOccurrence,
|
||||
type LocalScheduleStore,
|
||||
} from '@qinglong/runtime-core/local-scheduler';
|
||||
import { cronerLocalNextOccurrence } from './croner';
|
||||
|
||||
export interface LocalSchedulerCoordinatorOptions {
|
||||
readonly pageSize?: number;
|
||||
readonly misfireGraceMs?: number;
|
||||
readonly clock?: () => number;
|
||||
readonly createId?: () => string;
|
||||
readonly nextOccurrence?: LocalCronNextOccurrence;
|
||||
readonly onAdmitted?: (
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface LocalSchedulerCycleSummary {
|
||||
readonly observedAtMs: number;
|
||||
readonly scanned: number;
|
||||
readonly initialized: number;
|
||||
readonly skipped: number;
|
||||
readonly admitted: number;
|
||||
readonly raced: number;
|
||||
readonly truncated: boolean;
|
||||
}
|
||||
|
||||
export class LocalSchedulerCoordinator {
|
||||
private readonly pageSize: number;
|
||||
private readonly misfireGraceMs: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly createId: () => string;
|
||||
private readonly nextOccurrence: LocalCronNextOccurrence;
|
||||
private readonly onAdmitted?: LocalSchedulerCoordinatorOptions['onAdmitted'];
|
||||
|
||||
constructor(
|
||||
private readonly schedules: LocalScheduleStore,
|
||||
options: LocalSchedulerCoordinatorOptions = {},
|
||||
) {
|
||||
this.pageSize = options.pageSize ?? 8;
|
||||
this.misfireGraceMs = options.misfireGraceMs ?? 30_000;
|
||||
this.clock = options.clock ?? Date.now;
|
||||
this.createId = options.createId ?? randomUUID;
|
||||
this.nextOccurrence = options.nextOccurrence ?? cronerLocalNextOccurrence;
|
||||
this.onAdmitted = options.onAdmitted;
|
||||
assertLocalSchedulePageSize(this.pageSize);
|
||||
if (
|
||||
!Number.isSafeInteger(this.misfireGraceMs) ||
|
||||
this.misfireGraceMs < 0 ||
|
||||
this.misfireGraceMs > 5 * 60_000 ||
|
||||
typeof this.clock !== 'function' ||
|
||||
typeof this.createId !== 'function' ||
|
||||
typeof this.nextOccurrence !== 'function' ||
|
||||
(this.onAdmitted !== undefined && typeof this.onAdmitted !== 'function')
|
||||
) {
|
||||
throw new TypeError('Local scheduler options are invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async scheduleOnce(): Promise<LocalSchedulerCycleSummary> {
|
||||
const observedAtMs = this.clock();
|
||||
if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0) {
|
||||
throw new TypeError('Local scheduler clock is invalid');
|
||||
}
|
||||
const page = await this.schedules.listLocalScheduleCandidates({
|
||||
observedAtMs,
|
||||
limit: this.pageSize,
|
||||
});
|
||||
if (page.candidates.length > this.pageSize) {
|
||||
throw new RangeError('Local scheduler source exceeded its page size');
|
||||
}
|
||||
const stats = {
|
||||
observedAtMs,
|
||||
scanned: 0,
|
||||
initialized: 0,
|
||||
skipped: 0,
|
||||
admitted: 0,
|
||||
raced: 0,
|
||||
truncated: page.truncated,
|
||||
};
|
||||
for (const candidate of page.candidates) {
|
||||
stats.scanned += 1;
|
||||
const decision = resolveLocalScheduleDecision(
|
||||
candidate,
|
||||
observedAtMs,
|
||||
this.misfireGraceMs,
|
||||
this.nextOccurrence,
|
||||
);
|
||||
const admitted = decision.disposition === 'admit';
|
||||
const result = await this.schedules.commitLocalScheduleDecision({
|
||||
decision,
|
||||
...(admitted
|
||||
? {
|
||||
runId: this.createId(),
|
||||
attemptId: this.createId(),
|
||||
createdEventId: this.createId(),
|
||||
queuedEventId: this.createId(),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
if (result.status === 'raced') {
|
||||
stats.raced += 1;
|
||||
continue;
|
||||
}
|
||||
if (result.disposition === 'initialize') stats.initialized += 1;
|
||||
if (result.disposition === 'skip') stats.skipped += 1;
|
||||
if (result.status === 'admitted') {
|
||||
stats.admitted += 1;
|
||||
await this.onAdmitted?.(result.runId, result.attemptId);
|
||||
}
|
||||
}
|
||||
return Object.freeze(stats);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import type {
|
||||
LocalCronNextOccurrence,
|
||||
LocalCronSchedule,
|
||||
} from '@qinglong/runtime-core/local-scheduler';
|
||||
|
||||
interface CronerJob {
|
||||
nextRun(after: Date): Date | null;
|
||||
stop(): void;
|
||||
}
|
||||
|
||||
interface CronerConstructor {
|
||||
new (
|
||||
expression: string,
|
||||
options: Readonly<{
|
||||
timezone: string;
|
||||
paused: true;
|
||||
unref: true;
|
||||
}>,
|
||||
): CronerJob;
|
||||
}
|
||||
|
||||
export const cronerLocalNextOccurrence: LocalCronNextOccurrence = (
|
||||
schedule: LocalCronSchedule,
|
||||
afterMs: number,
|
||||
): number => {
|
||||
let job: CronerJob | undefined;
|
||||
try {
|
||||
const { Cron } = require('croner') as Readonly<{
|
||||
Cron: CronerConstructor;
|
||||
}>;
|
||||
job = new Cron(schedule.expression, {
|
||||
timezone: schedule.timezone,
|
||||
paused: true,
|
||||
unref: true,
|
||||
});
|
||||
const next = job.nextRun(new Date(afterMs));
|
||||
if (!(next instanceof Date)) {
|
||||
throw new Error('cron has no next occurrence');
|
||||
}
|
||||
return next.getTime();
|
||||
} finally {
|
||||
job?.stop();
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,16 @@
|
||||
export {
|
||||
LocalSchedulerCoordinator,
|
||||
type LocalSchedulerCoordinatorOptions,
|
||||
type LocalSchedulerCycleSummary,
|
||||
} from './coordinator';
|
||||
export {
|
||||
LocalSchedulerLifecycle,
|
||||
type LocalSchedulerLifecycleOptions,
|
||||
type LocalSchedulerLifecycleStopSummary,
|
||||
} from './lifecycle';
|
||||
export { cronerLocalNextOccurrence } from './croner';
|
||||
export {
|
||||
LocalWorkflowSchedulerCoordinator,
|
||||
type LocalWorkflowSchedulerCoordinatorOptions,
|
||||
type LocalWorkflowSchedulerCycleSummary,
|
||||
} from './workflowCoordinator';
|
||||
@@ -0,0 +1,128 @@
|
||||
import type {
|
||||
LocalSchedulerCoordinator,
|
||||
LocalSchedulerCycleSummary,
|
||||
} from './coordinator';
|
||||
|
||||
export interface LocalSchedulerLifecycleOptions {
|
||||
readonly intervalMs: number;
|
||||
readonly stopTimeoutMs: number;
|
||||
readonly onDiagnostic?: (
|
||||
error: unknown,
|
||||
summary?: LocalSchedulerCycleSummary,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface LocalSchedulerLifecycleStopSummary {
|
||||
readonly status: 'stopped' | 'timed_out';
|
||||
}
|
||||
|
||||
export class LocalSchedulerLifecycle {
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private inFlight: Promise<LocalSchedulerCycleSummary> | undefined;
|
||||
private stopPromise: Promise<LocalSchedulerLifecycleStopSummary> | undefined;
|
||||
private running = false;
|
||||
private stopping = false;
|
||||
|
||||
constructor(
|
||||
private readonly scheduler: Pick<LocalSchedulerCoordinator, 'scheduleOnce'>,
|
||||
private readonly options: LocalSchedulerLifecycleOptions,
|
||||
) {
|
||||
if (
|
||||
!scheduler ||
|
||||
typeof scheduler.scheduleOnce !== 'function' ||
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
!Number.isSafeInteger(options.intervalMs) ||
|
||||
options.intervalMs < 250 ||
|
||||
options.intervalMs > 60 * 60_000 ||
|
||||
!Number.isSafeInteger(options.stopTimeoutMs) ||
|
||||
options.stopTimeoutMs < 100 ||
|
||||
options.stopTimeoutMs > 30_000 ||
|
||||
(options.onDiagnostic !== undefined &&
|
||||
typeof options.onDiagnostic !== 'function')
|
||||
) {
|
||||
throw new TypeError('Local scheduler lifecycle options are invalid');
|
||||
}
|
||||
}
|
||||
|
||||
start(): 'started' {
|
||||
if (!this.running && !this.stopping) {
|
||||
this.running = true;
|
||||
this.schedule();
|
||||
}
|
||||
return 'started';
|
||||
}
|
||||
|
||||
runOnce(): Promise<LocalSchedulerCycleSummary> {
|
||||
if (this.stopping) {
|
||||
return Promise.reject(
|
||||
new Error('Local scheduler lifecycle is stopping'),
|
||||
);
|
||||
}
|
||||
if (this.inFlight) return this.inFlight;
|
||||
const work = this.scheduler.scheduleOnce().finally(() => {
|
||||
if (this.inFlight === work) this.inFlight = undefined;
|
||||
});
|
||||
this.inFlight = work;
|
||||
return work;
|
||||
}
|
||||
|
||||
stopAndDrain(): Promise<LocalSchedulerLifecycleStopSummary> {
|
||||
if (this.stopPromise) return this.stopPromise;
|
||||
this.stopping = true;
|
||||
this.running = false;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
|
||||
this.stopPromise = (async () => {
|
||||
const work = this.inFlight;
|
||||
if (!work) return Object.freeze({ status: 'stopped' as const });
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
work.then(
|
||||
() => Object.freeze({ status: 'stopped' as const }),
|
||||
() => Object.freeze({ status: 'stopped' as const }),
|
||||
),
|
||||
new Promise<LocalSchedulerLifecycleStopSummary>((resolve) => {
|
||||
timeout = setTimeout(
|
||||
() =>
|
||||
resolve(Object.freeze({ status: 'timed_out' as const })),
|
||||
this.options.stopTimeoutMs,
|
||||
);
|
||||
timeout.unref?.();
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
}
|
||||
})();
|
||||
return this.stopPromise;
|
||||
}
|
||||
|
||||
private schedule(): void {
|
||||
if (!this.running || this.timer) return;
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = undefined;
|
||||
if (!this.running) return;
|
||||
void this.runOnce()
|
||||
.then((summary) => this.diagnostic(undefined, summary))
|
||||
.catch((error) => this.diagnostic(error))
|
||||
.finally(() => this.schedule());
|
||||
}, this.options.intervalMs);
|
||||
this.timer.unref?.();
|
||||
}
|
||||
|
||||
private async diagnostic(
|
||||
error: unknown,
|
||||
summary?: LocalSchedulerCycleSummary,
|
||||
): Promise<void> {
|
||||
if (this.stopping) return;
|
||||
try {
|
||||
await this.options.onDiagnostic?.(error, summary);
|
||||
} catch {
|
||||
// Diagnostics cannot own or stop scheduling.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
import {
|
||||
ClusterRunCancellationConvergenceCoordinator,
|
||||
type ClusterRunCancellationConvergenceCycleResult,
|
||||
type ClusterRunCancellationConvergenceRepository,
|
||||
} from '@qinglong/runtime-core/cluster-run-cancellation-convergence';
|
||||
import type {
|
||||
PluginPackageWorkflowFrontierCursor,
|
||||
PluginPackageWorkflowFrontierRepository,
|
||||
} from '@qinglong/runtime-core/plugin-package-workflow-frontier';
|
||||
import type {
|
||||
PluginPackageWorkflowTaskAttemptAdmissionCursor,
|
||||
PluginPackageWorkflowTaskAttemptAdmissionRepository,
|
||||
} from '@qinglong/runtime-core/plugin-package-workflow-task-attempt-admission';
|
||||
|
||||
import type {
|
||||
LocalRunDispatcher,
|
||||
LocalRunDispatcherResult,
|
||||
} from '../dispatch/dispatcher';
|
||||
import type {
|
||||
LocalSchedulerCoordinator,
|
||||
LocalSchedulerCycleSummary,
|
||||
} from './coordinator';
|
||||
|
||||
export interface LocalWorkflowSchedulerCoordinatorOptions {
|
||||
readonly cancellationPageSize: number;
|
||||
readonly cancellationMaxPages: number;
|
||||
readonly frontierPageSize: number;
|
||||
readonly frontierMaxPages: number;
|
||||
readonly taskAttemptPageSize: number;
|
||||
readonly taskAttemptMaxPages: number;
|
||||
readonly maxDispatches: number;
|
||||
}
|
||||
|
||||
export interface LocalWorkflowSchedulerCycleSummary {
|
||||
readonly cancellation: Readonly<ClusterRunCancellationConvergenceCycleResult>;
|
||||
readonly frontierPages: number;
|
||||
readonly frontierScanned: number;
|
||||
readonly frontierAdvanced: number;
|
||||
readonly frontierTruncated: boolean;
|
||||
readonly taskAttemptPages: number;
|
||||
readonly taskAttemptsScanned: number;
|
||||
readonly taskAttemptsCreated: number;
|
||||
readonly taskAttemptsExisting: number;
|
||||
readonly taskAttemptsTruncated: boolean;
|
||||
readonly dispatches: number;
|
||||
readonly activated: number;
|
||||
readonly activationFailed: number;
|
||||
readonly dispatchIdle: boolean;
|
||||
}
|
||||
|
||||
function bounded(
|
||||
value: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
label: string,
|
||||
): number {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${label} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function nextFrontierCursor(
|
||||
current: PluginPackageWorkflowFrontierCursor | undefined,
|
||||
next: PluginPackageWorkflowFrontierCursor | undefined,
|
||||
): PluginPackageWorkflowFrontierCursor {
|
||||
if (
|
||||
!next ||
|
||||
(current !== undefined &&
|
||||
(next.admittedAtMs < current.admittedAtMs ||
|
||||
(next.admittedAtMs === current.admittedAtMs &&
|
||||
next.planDigest <= current.planDigest)))
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow frontier continuation did not advance',
|
||||
);
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function nextTaskAttemptCursor(
|
||||
current: PluginPackageWorkflowTaskAttemptAdmissionCursor | undefined,
|
||||
next: PluginPackageWorkflowTaskAttemptAdmissionCursor | undefined,
|
||||
): PluginPackageWorkflowTaskAttemptAdmissionCursor {
|
||||
if (
|
||||
!next ||
|
||||
(current !== undefined &&
|
||||
(next.readyAtMs < current.readyAtMs ||
|
||||
(next.readyAtMs === current.readyAtMs &&
|
||||
next.stepRunId <= current.stepRunId)))
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow Task Attempt continuation did not advance',
|
||||
);
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reuses the existing Local scheduler cadence. It owns no timer, connection,
|
||||
* watcher, or per-Workflow state.
|
||||
*/
|
||||
export class LocalWorkflowSchedulerCoordinator {
|
||||
private readonly cancellation: ClusterRunCancellationConvergenceCoordinator;
|
||||
private readonly frontierPageSize: number;
|
||||
private readonly frontierMaxPages: number;
|
||||
private readonly taskAttemptPageSize: number;
|
||||
private readonly taskAttemptMaxPages: number;
|
||||
private readonly maxDispatches: number;
|
||||
private inFlight:
|
||||
| Promise<Readonly<LocalSchedulerCycleSummary>>
|
||||
| undefined;
|
||||
private latestWorkflow:
|
||||
| Readonly<LocalWorkflowSchedulerCycleSummary>
|
||||
| undefined;
|
||||
|
||||
constructor(
|
||||
private readonly scheduler: Pick<LocalSchedulerCoordinator, 'scheduleOnce'>,
|
||||
cancellation: ClusterRunCancellationConvergenceRepository,
|
||||
private readonly frontier: PluginPackageWorkflowFrontierRepository,
|
||||
private readonly taskAttempts: PluginPackageWorkflowTaskAttemptAdmissionRepository,
|
||||
private readonly dispatcher: Pick<LocalRunDispatcher, 'dispatchOnce'>,
|
||||
options: LocalWorkflowSchedulerCoordinatorOptions,
|
||||
) {
|
||||
if (
|
||||
typeof scheduler?.scheduleOnce !== 'function' ||
|
||||
typeof cancellation?.convergePage !== 'function' ||
|
||||
typeof frontier?.listCandidates !== 'function' ||
|
||||
typeof frontier?.advance !== 'function' ||
|
||||
typeof taskAttempts?.listCandidates !== 'function' ||
|
||||
typeof taskAttempts?.admit !== 'function' ||
|
||||
typeof dispatcher?.dispatchOnce !== 'function' ||
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options)
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow scheduler coordinator is invalid',
|
||||
);
|
||||
}
|
||||
this.cancellation = new ClusterRunCancellationConvergenceCoordinator(
|
||||
cancellation,
|
||||
{
|
||||
pageSize: options.cancellationPageSize,
|
||||
maxPages: options.cancellationMaxPages,
|
||||
},
|
||||
);
|
||||
this.frontierPageSize = bounded(
|
||||
options.frontierPageSize,
|
||||
1,
|
||||
64,
|
||||
'Local Workflow frontier page size',
|
||||
);
|
||||
this.frontierMaxPages = bounded(
|
||||
options.frontierMaxPages,
|
||||
1,
|
||||
16,
|
||||
'Local Workflow frontier page limit',
|
||||
);
|
||||
this.taskAttemptPageSize = bounded(
|
||||
options.taskAttemptPageSize,
|
||||
1,
|
||||
64,
|
||||
'Local Workflow Task Attempt page size',
|
||||
);
|
||||
this.taskAttemptMaxPages = bounded(
|
||||
options.taskAttemptMaxPages,
|
||||
1,
|
||||
16,
|
||||
'Local Workflow Task Attempt page limit',
|
||||
);
|
||||
this.maxDispatches = bounded(
|
||||
options.maxDispatches,
|
||||
1,
|
||||
16,
|
||||
'Local Workflow dispatch limit',
|
||||
);
|
||||
}
|
||||
|
||||
scheduleOnce(): Promise<Readonly<LocalSchedulerCycleSummary>> {
|
||||
if (this.inFlight) return this.inFlight;
|
||||
const work = this.runCycle().finally(() => {
|
||||
if (this.inFlight === work) this.inFlight = undefined;
|
||||
});
|
||||
this.inFlight = work;
|
||||
return work;
|
||||
}
|
||||
|
||||
latestWorkflowSummary():
|
||||
| Readonly<LocalWorkflowSchedulerCycleSummary>
|
||||
| undefined {
|
||||
return this.latestWorkflow;
|
||||
}
|
||||
|
||||
private async runCycle(): Promise<Readonly<LocalSchedulerCycleSummary>> {
|
||||
const cancellation = await this.cancellation.reconcile();
|
||||
const scheduler = await this.scheduler.scheduleOnce();
|
||||
const frontier = await this.advanceFrontier();
|
||||
const taskAttempts = await this.admitTaskAttempts();
|
||||
const dispatch = await this.dispatch();
|
||||
this.latestWorkflow = Object.freeze({
|
||||
cancellation,
|
||||
...frontier,
|
||||
...taskAttempts,
|
||||
...dispatch,
|
||||
});
|
||||
return scheduler;
|
||||
}
|
||||
|
||||
private async advanceFrontier(): Promise<Readonly<{
|
||||
frontierPages: number;
|
||||
frontierScanned: number;
|
||||
frontierAdvanced: number;
|
||||
frontierTruncated: boolean;
|
||||
}>> {
|
||||
let frontierPages = 0;
|
||||
let frontierScanned = 0;
|
||||
let frontierAdvanced = 0;
|
||||
let frontierTruncated = false;
|
||||
let after: PluginPackageWorkflowFrontierCursor | undefined;
|
||||
for (
|
||||
let index = 0;
|
||||
index < this.frontierMaxPages;
|
||||
index += 1
|
||||
) {
|
||||
const page = await this.frontier.listCandidates({
|
||||
limit: this.frontierPageSize,
|
||||
...(after === undefined ? {} : { after }),
|
||||
});
|
||||
if (page.candidates.length > this.frontierPageSize) {
|
||||
throw new RangeError(
|
||||
'Local Workflow frontier exceeded its page size',
|
||||
);
|
||||
}
|
||||
frontierPages += 1;
|
||||
frontierScanned += page.candidates.length;
|
||||
for (const candidate of page.candidates) {
|
||||
await this.frontier.advance(candidate.runId);
|
||||
frontierAdvanced += 1;
|
||||
}
|
||||
frontierTruncated = page.truncated;
|
||||
if (!page.truncated) break;
|
||||
after = nextFrontierCursor(after, page.next);
|
||||
}
|
||||
return Object.freeze({
|
||||
frontierPages,
|
||||
frontierScanned,
|
||||
frontierAdvanced,
|
||||
frontierTruncated,
|
||||
});
|
||||
}
|
||||
|
||||
private async admitTaskAttempts(): Promise<Readonly<{
|
||||
taskAttemptPages: number;
|
||||
taskAttemptsScanned: number;
|
||||
taskAttemptsCreated: number;
|
||||
taskAttemptsExisting: number;
|
||||
taskAttemptsTruncated: boolean;
|
||||
}>> {
|
||||
let taskAttemptPages = 0;
|
||||
let taskAttemptsScanned = 0;
|
||||
let taskAttemptsCreated = 0;
|
||||
let taskAttemptsExisting = 0;
|
||||
let taskAttemptsTruncated = false;
|
||||
let after:
|
||||
| PluginPackageWorkflowTaskAttemptAdmissionCursor
|
||||
| undefined;
|
||||
for (
|
||||
let index = 0;
|
||||
index < this.taskAttemptMaxPages;
|
||||
index += 1
|
||||
) {
|
||||
const page = await this.taskAttempts.listCandidates({
|
||||
limit: this.taskAttemptPageSize,
|
||||
...(after === undefined ? {} : { after }),
|
||||
});
|
||||
if (page.candidates.length > this.taskAttemptPageSize) {
|
||||
throw new RangeError(
|
||||
'Local Workflow Task Attempt source exceeded its page size',
|
||||
);
|
||||
}
|
||||
taskAttemptPages += 1;
|
||||
taskAttemptsScanned += page.candidates.length;
|
||||
for (const candidate of page.candidates) {
|
||||
const admitted = await this.taskAttempts.admit(
|
||||
candidate.runId,
|
||||
candidate.stepRunId,
|
||||
);
|
||||
if (admitted.status === 'created') taskAttemptsCreated += 1;
|
||||
else taskAttemptsExisting += 1;
|
||||
}
|
||||
taskAttemptsTruncated = page.truncated;
|
||||
if (!page.truncated) break;
|
||||
after = nextTaskAttemptCursor(after, page.next);
|
||||
}
|
||||
return Object.freeze({
|
||||
taskAttemptPages,
|
||||
taskAttemptsScanned,
|
||||
taskAttemptsCreated,
|
||||
taskAttemptsExisting,
|
||||
taskAttemptsTruncated,
|
||||
});
|
||||
}
|
||||
|
||||
private async dispatch(): Promise<Readonly<{
|
||||
dispatches: number;
|
||||
activated: number;
|
||||
activationFailed: number;
|
||||
dispatchIdle: boolean;
|
||||
}>> {
|
||||
let dispatches = 0;
|
||||
let activated = 0;
|
||||
let activationFailed = 0;
|
||||
let dispatchIdle = false;
|
||||
for (let index = 0; index < this.maxDispatches; index += 1) {
|
||||
const result: LocalRunDispatcherResult =
|
||||
await this.dispatcher.dispatchOnce();
|
||||
dispatches += 1;
|
||||
if (result.status === 'activated') activated += 1;
|
||||
else if (result.status === 'activation_failed') {
|
||||
activationFailed += 1;
|
||||
} else {
|
||||
dispatchIdle = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
dispatches,
|
||||
activated,
|
||||
activationFailed,
|
||||
dispatchIdle,
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user