mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 18:08:20 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,479 @@
|
||||
import { createHash, randomUUID, timingSafeEqual } from 'crypto';
|
||||
import type { LocalCompletionReceiptJournal } from '@qinglong/runtime-core/local-completion-receipt-journal';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
RunRepository,
|
||||
RunRepositoryTransaction,
|
||||
} from '@qinglong/runtime-core/run-repository';
|
||||
import {
|
||||
InvalidCompletionReceiptError,
|
||||
type CompletionReceipt,
|
||||
type CompletionReceiptStore,
|
||||
} from '@qinglong/local-process';
|
||||
import type { LocalWorkflowTaskExecutionRepository } from '../execution/workflowTaskExecution';
|
||||
|
||||
export const MAX_LOCAL_COMPLETION_QUARANTINE_RETENTION_MS = 24 * 60 * 60_000;
|
||||
|
||||
const ACTIVE_ATTEMPT_STATUSES = new Set(['claimed', 'starting', 'running']);
|
||||
const TERMINAL_ATTEMPT_STATUSES = new Set([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
'lost',
|
||||
]);
|
||||
const TERMINAL_RUN_STATUSES = new Set([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
'lost',
|
||||
]);
|
||||
|
||||
type MaintainedCompletionReceiptStore = CompletionReceiptStore & {
|
||||
quarantine?(attemptId: string): Promise<string | undefined>;
|
||||
quarantineReference?(attemptId: string): string;
|
||||
};
|
||||
|
||||
export type LocalCompletionDisposition =
|
||||
| 'completed'
|
||||
| 'already_terminal'
|
||||
| 'missing'
|
||||
| 'invalid'
|
||||
| 'stale';
|
||||
|
||||
export interface LocalCompletionReceiptProcessorOptions {
|
||||
readonly clock?: { now(): number };
|
||||
readonly createEventId?: () => string;
|
||||
readonly journal?: Pick<
|
||||
LocalCompletionReceiptJournal,
|
||||
'markQuarantined' | 'resolve'
|
||||
>;
|
||||
readonly quarantineRetentionMs?: number;
|
||||
readonly onDiagnostic?: (
|
||||
record: Readonly<{
|
||||
kind:
|
||||
| 'receipt_cleanup_failed'
|
||||
| 'receipt_quarantined'
|
||||
| 'journal_cleanup_failed';
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
}>,
|
||||
) => void | Promise<void>;
|
||||
readonly workflowTasks?: LocalWorkflowTaskExecutionRepository;
|
||||
}
|
||||
|
||||
interface AggregateSnapshot {
|
||||
readonly run: RunRecord;
|
||||
readonly attempt: RunAttemptRecord;
|
||||
}
|
||||
|
||||
interface TerminalMapping {
|
||||
readonly status: 'succeeded' | 'failed' | 'cancelled' | 'timed_out';
|
||||
readonly errorCode?: string;
|
||||
readonly errorSummary?: string;
|
||||
}
|
||||
|
||||
class LocalCompletionConcurrentWriteError extends Error {}
|
||||
|
||||
function timestamp(value: number, field: string): number {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new RangeError(`${field} must be a non-negative safe integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function atOrAfter(
|
||||
now: number,
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
): number {
|
||||
return Math.max(
|
||||
timestamp(now, 'Local completion observation'),
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt.createdAtMs,
|
||||
attempt.startedAtMs ?? 0,
|
||||
);
|
||||
}
|
||||
|
||||
function reserveEvent(run: RunRecord): Readonly<{
|
||||
run: RunRecord;
|
||||
sequence: number;
|
||||
}> {
|
||||
const version = run.version + 1;
|
||||
const sequence = run.eventSequence + 1;
|
||||
if (!Number.isSafeInteger(version) || !Number.isSafeInteger(sequence)) {
|
||||
throw new RangeError('Local completion aggregate counter overflowed');
|
||||
}
|
||||
return Object.freeze({
|
||||
run: { ...run, version, eventSequence: sequence },
|
||||
sequence,
|
||||
});
|
||||
}
|
||||
|
||||
function mapping(run: RunRecord, receipt: CompletionReceipt): TerminalMapping {
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
return run.cancelReason === 'timeout'
|
||||
? Object.freeze({
|
||||
status: 'timed_out' as const,
|
||||
errorCode: 'EXECUTION_TIMED_OUT',
|
||||
errorSummary: 'Execution exceeded its configured timeout',
|
||||
})
|
||||
: Object.freeze({
|
||||
status: 'cancelled' as const,
|
||||
errorCode: 'EXECUTION_CANCELLED',
|
||||
errorSummary: 'Execution was cancelled',
|
||||
});
|
||||
}
|
||||
return receipt.exitCode === 0
|
||||
? Object.freeze({ status: 'succeeded' as const })
|
||||
: Object.freeze({
|
||||
status: 'failed' as const,
|
||||
errorCode: 'EXECUTION_FAILED',
|
||||
errorSummary: 'Execution completed without success',
|
||||
});
|
||||
}
|
||||
|
||||
function authenticate(
|
||||
run: RunRecord,
|
||||
attempt: RunAttemptRecord,
|
||||
receipt: CompletionReceipt,
|
||||
terminal: boolean,
|
||||
): void {
|
||||
const allowedSequence = terminal
|
||||
? receipt.callbackSequence === attempt.callbackSequence ||
|
||||
receipt.callbackSequence === attempt.callbackSequence + 1
|
||||
: receipt.callbackSequence === attempt.callbackSequence + 1;
|
||||
if (
|
||||
run.executionOwner !== 'runtime' ||
|
||||
receipt.runId !== run.id ||
|
||||
receipt.attemptId !== attempt.id ||
|
||||
!allowedSequence ||
|
||||
receipt.startedAtMs < attempt.createdAtMs ||
|
||||
(attempt.startedAtMs !== undefined &&
|
||||
receipt.startedAtMs < attempt.startedAtMs) ||
|
||||
!attempt.callbackTokenHash ||
|
||||
!/^[a-f0-9]{64}$/.test(attempt.callbackTokenHash)
|
||||
) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt does not match durable Run authority',
|
||||
);
|
||||
}
|
||||
const expected = Buffer.from(attempt.callbackTokenHash, 'hex');
|
||||
const actual = createHash('sha256').update(receipt.token, 'utf8').digest();
|
||||
if (!timingSafeEqual(expected, actual)) {
|
||||
throw new InvalidCompletionReceiptError(
|
||||
'Completion receipt token does not match durable authority',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function event(
|
||||
id: string,
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
sequence: number,
|
||||
type: string,
|
||||
dedupeKey: string,
|
||||
payload: Readonly<Record<string, unknown>>,
|
||||
atMs: number,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id,
|
||||
runId,
|
||||
attemptId,
|
||||
sequence,
|
||||
type,
|
||||
dedupeKey,
|
||||
actorType: 'executor',
|
||||
actorId: 'completion-receipt',
|
||||
payload,
|
||||
createdAtMs: atMs,
|
||||
};
|
||||
}
|
||||
|
||||
export class LocalCompletionReceiptProcessor {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createEventId: () => string;
|
||||
private readonly journal?: LocalCompletionReceiptProcessorOptions['journal'];
|
||||
private readonly quarantineRetentionMs: number;
|
||||
private readonly onDiagnostic?: LocalCompletionReceiptProcessorOptions['onDiagnostic'];
|
||||
private readonly workflowTasks:
|
||||
| LocalWorkflowTaskExecutionRepository
|
||||
| undefined;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly receipts: MaintainedCompletionReceiptStore,
|
||||
options: LocalCompletionReceiptProcessorOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
this.journal = options.journal;
|
||||
this.quarantineRetentionMs = options.quarantineRetentionMs ?? 60 * 60_000;
|
||||
if (
|
||||
!Number.isSafeInteger(this.quarantineRetentionMs) ||
|
||||
this.quarantineRetentionMs < 0 ||
|
||||
this.quarantineRetentionMs > MAX_LOCAL_COMPLETION_QUARANTINE_RETENTION_MS
|
||||
) {
|
||||
throw new RangeError('Local completion quarantine retention is invalid');
|
||||
}
|
||||
this.onDiagnostic = options.onDiagnostic;
|
||||
this.workflowTasks = options.workflowTasks;
|
||||
if (
|
||||
this.workflowTasks !== undefined &&
|
||||
typeof this.workflowTasks.complete !== 'function'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow Task completion repository is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async process(attemptId: string): Promise<LocalCompletionDisposition> {
|
||||
const snapshot = await this.load(attemptId);
|
||||
if (!snapshot || snapshot.attempt.executorType !== 'local_process') {
|
||||
return 'stale';
|
||||
}
|
||||
let receipt: CompletionReceipt | undefined;
|
||||
try {
|
||||
receipt = await this.receipts.read(attemptId);
|
||||
if (!receipt) return 'missing';
|
||||
authenticate(
|
||||
snapshot.run,
|
||||
snapshot.attempt,
|
||||
receipt,
|
||||
TERMINAL_RUN_STATUSES.has(snapshot.run.status) ||
|
||||
TERMINAL_ATTEMPT_STATUSES.has(snapshot.attempt.status),
|
||||
);
|
||||
} catch (error) {
|
||||
if (!(error instanceof InvalidCompletionReceiptError)) throw error;
|
||||
await this.quarantine(snapshot);
|
||||
return 'invalid';
|
||||
}
|
||||
|
||||
if (
|
||||
TERMINAL_RUN_STATUSES.has(snapshot.run.status) ||
|
||||
TERMINAL_ATTEMPT_STATUSES.has(snapshot.attempt.status)
|
||||
) {
|
||||
await this.cleanup(receipt);
|
||||
return 'already_terminal';
|
||||
}
|
||||
if (!ACTIVE_ATTEMPT_STATUSES.has(snapshot.attempt.status)) return 'stale';
|
||||
|
||||
if (snapshot.attempt.stepRunId !== undefined) {
|
||||
if (!this.workflowTasks) return 'stale';
|
||||
const terminal = mapping(snapshot.run, receipt);
|
||||
const disposition = await this.workflowTasks.complete({
|
||||
run: snapshot.run,
|
||||
attempt: snapshot.attempt,
|
||||
callbackSequence: receipt.callbackSequence,
|
||||
startedAtMs: receipt.startedAtMs,
|
||||
finishedAtMs: Math.max(
|
||||
atOrAfter(this.clock.now(), snapshot.run, snapshot.attempt),
|
||||
receipt.finishedAtMs,
|
||||
),
|
||||
exitCode: receipt.exitCode,
|
||||
terminalStatus: terminal.status,
|
||||
...(terminal.errorCode === undefined
|
||||
? {}
|
||||
: {
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
}),
|
||||
attemptEventId: this.createEventId(),
|
||||
syntheticStartMutationId: this.createEventId(),
|
||||
terminalStepMutationId: this.createEventId(),
|
||||
});
|
||||
if (
|
||||
disposition === 'completed' ||
|
||||
disposition === 'already_terminal'
|
||||
) {
|
||||
await this.cleanup(receipt);
|
||||
}
|
||||
return disposition;
|
||||
}
|
||||
|
||||
const completed = await this.repository.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(snapshot.run.id);
|
||||
const attempt = await transaction.findAttemptById(snapshot.attempt.id);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
run.version !== snapshot.run.version ||
|
||||
run.status !== snapshot.run.status ||
|
||||
attempt.status !== snapshot.attempt.status ||
|
||||
attempt.callbackSequence !== snapshot.attempt.callbackSequence ||
|
||||
attempt.runId !== run.id
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
authenticate(run, attempt, receipt, false);
|
||||
const terminal = mapping(run, receipt);
|
||||
const atMs = Math.max(
|
||||
atOrAfter(this.clock.now(), run, attempt),
|
||||
receipt.finishedAtMs,
|
||||
);
|
||||
const attemptReserved = reserveEvent(run);
|
||||
const nextAttempt: RunAttemptRecord = {
|
||||
...attempt,
|
||||
status: terminal.status,
|
||||
callbackSequence: receipt.callbackSequence,
|
||||
finishedAtMs: atMs,
|
||||
exitCode: receipt.exitCode,
|
||||
...(terminal.errorCode === undefined
|
||||
? {}
|
||||
: {
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
}),
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(attemptReserved.run, run.version))
|
||||
) {
|
||||
throw new LocalCompletionConcurrentWriteError();
|
||||
}
|
||||
if (
|
||||
!(await transaction.compareAndSetAttempt(nextAttempt, {
|
||||
status: attempt.status,
|
||||
callbackSequence: attempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new LocalCompletionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run.id,
|
||||
attempt.id,
|
||||
attemptReserved.sequence,
|
||||
`attempt.${terminal.status}`,
|
||||
`local-completion:${attempt.id}:${receipt.callbackSequence}:attempt`,
|
||||
Object.freeze({
|
||||
attempt_id: attempt.id,
|
||||
from_status: attempt.status,
|
||||
to_status: terminal.status,
|
||||
callback_sequence: receipt.callbackSequence,
|
||||
exit_code: receipt.exitCode,
|
||||
version: attemptReserved.run.version,
|
||||
}),
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
const runReserved = reserveEvent(attemptReserved.run);
|
||||
const nextRun: RunRecord = {
|
||||
...runReserved.run,
|
||||
status: terminal.status,
|
||||
finishedAtMs: atMs,
|
||||
...(terminal.errorCode === undefined
|
||||
? {}
|
||||
: {
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
}),
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
nextRun,
|
||||
attemptReserved.run.version,
|
||||
))
|
||||
) {
|
||||
throw new LocalCompletionConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run.id,
|
||||
attempt.id,
|
||||
runReserved.sequence,
|
||||
`run.${terminal.status}`,
|
||||
`local-completion:${attempt.id}:${receipt.callbackSequence}:run`,
|
||||
Object.freeze({
|
||||
from_status: run.status,
|
||||
to_status: terminal.status,
|
||||
version: nextRun.version,
|
||||
}),
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
return true;
|
||||
});
|
||||
if (!completed) return 'stale';
|
||||
await this.cleanup(receipt);
|
||||
return 'completed';
|
||||
}
|
||||
|
||||
private load(attemptId: string): Promise<AggregateSnapshot | null> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const attempt = await transaction.findAttemptById(attemptId);
|
||||
if (!attempt) return null;
|
||||
const run = await transaction.findRunById(attempt.runId);
|
||||
return run ? Object.freeze({ run, attempt }) : null;
|
||||
});
|
||||
}
|
||||
|
||||
private async quarantine(snapshot: AggregateSnapshot): Promise<void> {
|
||||
const reference = this.receipts.quarantineReference?.(snapshot.attempt.id);
|
||||
if (reference && this.journal) {
|
||||
const updatedAtMs = atOrAfter(
|
||||
this.clock.now(),
|
||||
snapshot.run,
|
||||
snapshot.attempt,
|
||||
);
|
||||
const purgeAfterMs = timestamp(
|
||||
updatedAtMs + this.quarantineRetentionMs,
|
||||
'Local completion quarantine expiry',
|
||||
);
|
||||
await this.journal.markQuarantined({
|
||||
attemptId: snapshot.attempt.id,
|
||||
quarantineRef: reference,
|
||||
updatedAtMs,
|
||||
purgeAfterMs,
|
||||
});
|
||||
}
|
||||
const quarantined = await this.receipts.quarantine?.(snapshot.attempt.id);
|
||||
if (quarantined) {
|
||||
await this.diagnostic({
|
||||
kind: 'receipt_quarantined',
|
||||
runId: snapshot.run.id,
|
||||
attemptId: snapshot.attempt.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async cleanup(receipt: CompletionReceipt): Promise<void> {
|
||||
try {
|
||||
await this.receipts.remove(receipt.attemptId);
|
||||
} catch {
|
||||
await this.diagnostic({
|
||||
kind: 'receipt_cleanup_failed',
|
||||
runId: receipt.runId,
|
||||
attemptId: receipt.attemptId,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!this.journal) return;
|
||||
try {
|
||||
await this.journal.resolve(receipt.attemptId);
|
||||
} catch {
|
||||
await this.diagnostic({
|
||||
kind: 'journal_cleanup_failed',
|
||||
runId: receipt.runId,
|
||||
attemptId: receipt.attemptId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async diagnostic(
|
||||
record: Parameters<NonNullable<typeof this.onDiagnostic>>[0],
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.onDiagnostic?.(Object.freeze(record));
|
||||
} catch {
|
||||
// Diagnostic failure cannot replace the durable completion result.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,696 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import {
|
||||
assertLocalExecutionControlLimit,
|
||||
normalizeLocalActiveExecutionCandidate,
|
||||
normalizeLocalExecutionControlCandidate,
|
||||
type LocalActiveExecutionCandidate,
|
||||
type LocalActiveExecutionCursor,
|
||||
type LocalExecutionControlCandidate,
|
||||
type LocalExecutionControlCursor,
|
||||
type LocalExecutionControlSource,
|
||||
} from '@qinglong/runtime-core/local-execution-control';
|
||||
import type {
|
||||
RunAttemptRecord,
|
||||
RunCancellationReason,
|
||||
RunEventRecord,
|
||||
RunRecord,
|
||||
RunRepository,
|
||||
} from '@qinglong/runtime-core/run-repository';
|
||||
import type {
|
||||
LocalProcessController,
|
||||
LocalProcessStopResult,
|
||||
} from '@qinglong/local-process';
|
||||
import type { LocalCompletionReceiptProcessor } from './completion';
|
||||
import type { LocalWorkflowTaskExecutionRepository } from '../execution/workflowTaskExecution';
|
||||
|
||||
export const MAX_LOCAL_EXECUTION_CONTROL_PAGES = 16;
|
||||
|
||||
const ACTIVE_ATTEMPT_STATUSES = new Set(['claimed', 'starting', 'running']);
|
||||
const ACTIVE_RUN_STATUSES = new Set(['dispatching', 'running']);
|
||||
const TERMINAL_RUN_STATUSES = new Set([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
'lost',
|
||||
]);
|
||||
|
||||
export type LocalExecutionControlDisposition =
|
||||
| 'terminal'
|
||||
| 'cancel_requested'
|
||||
| 'stale'
|
||||
| 'remaining';
|
||||
|
||||
export interface LocalExecutionControlCoordinatorOptions {
|
||||
readonly clock?: { now(): number };
|
||||
readonly createEventId?: () => string;
|
||||
readonly workflowTasks?: LocalWorkflowTaskExecutionRepository;
|
||||
}
|
||||
|
||||
interface ActiveSnapshot {
|
||||
readonly run: RunRecord;
|
||||
readonly attempt: RunAttemptRecord;
|
||||
}
|
||||
|
||||
class LocalExecutionControlConcurrentWriteError extends Error {}
|
||||
|
||||
function timestamp(value: number, field: string): number {
|
||||
if (!Number.isSafeInteger(value) || value < 0) {
|
||||
throw new RangeError(`${field} must be a non-negative safe integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function reserveEvent(run: RunRecord): Readonly<{
|
||||
run: RunRecord;
|
||||
sequence: number;
|
||||
}> {
|
||||
const version = run.version + 1;
|
||||
const sequence = run.eventSequence + 1;
|
||||
if (!Number.isSafeInteger(version) || !Number.isSafeInteger(sequence)) {
|
||||
throw new RangeError(
|
||||
'Local execution control aggregate counter overflowed',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
run: { ...run, version, eventSequence: sequence },
|
||||
sequence,
|
||||
});
|
||||
}
|
||||
|
||||
function atOrAfter(
|
||||
now: number,
|
||||
run: RunRecord,
|
||||
attempt?: RunAttemptRecord,
|
||||
): number {
|
||||
return Math.max(
|
||||
timestamp(now, 'Local execution control observation'),
|
||||
run.createdAtMs,
|
||||
run.startedAtMs ?? 0,
|
||||
attempt?.createdAtMs ?? 0,
|
||||
attempt?.startedAtMs ?? 0,
|
||||
);
|
||||
}
|
||||
|
||||
function event(
|
||||
id: string,
|
||||
run: RunRecord,
|
||||
attemptId: string,
|
||||
sequence: number,
|
||||
type: string,
|
||||
dedupeKey: string,
|
||||
payload: Readonly<Record<string, unknown>>,
|
||||
actorId: string,
|
||||
atMs: number,
|
||||
): RunEventRecord {
|
||||
return {
|
||||
id,
|
||||
runId: run.id,
|
||||
attemptId,
|
||||
sequence,
|
||||
type,
|
||||
dedupeKey,
|
||||
actorType: 'reconciler',
|
||||
actorId,
|
||||
payload,
|
||||
createdAtMs: atMs,
|
||||
};
|
||||
}
|
||||
|
||||
function terminalFor(reason: RunCancellationReason): Readonly<{
|
||||
status: 'cancelled' | 'timed_out';
|
||||
errorCode: string;
|
||||
errorSummary: string;
|
||||
}> {
|
||||
return reason === 'timeout'
|
||||
? Object.freeze({
|
||||
status: 'timed_out' as const,
|
||||
errorCode: 'EXECUTION_TIMED_OUT',
|
||||
errorSummary: 'Execution exceeded its configured timeout',
|
||||
})
|
||||
: Object.freeze({
|
||||
status: 'cancelled' as const,
|
||||
errorCode: 'EXECUTION_CANCELLED',
|
||||
errorSummary: 'Execution was cancelled',
|
||||
});
|
||||
}
|
||||
|
||||
function conclusiveStop(result: LocalProcessStopResult): boolean {
|
||||
return result.status === 'stopped' || result.status === 'already_exited';
|
||||
}
|
||||
|
||||
export class LocalExecutionControlCoordinator {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly createEventId: () => string;
|
||||
private readonly workflowTasks:
|
||||
| LocalWorkflowTaskExecutionRepository
|
||||
| undefined;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly completions: Pick<
|
||||
LocalCompletionReceiptProcessor,
|
||||
'process'
|
||||
>,
|
||||
private readonly controller: Pick<LocalProcessController, 'stop'>,
|
||||
options: LocalExecutionControlCoordinatorOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
this.workflowTasks = options.workflowTasks;
|
||||
if (
|
||||
this.workflowTasks !== undefined &&
|
||||
(typeof this.workflowTasks.requestTimeout !== 'function' ||
|
||||
typeof this.workflowTasks.recordControlTerminal !== 'function')
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local Workflow Task execution control repository is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async process(
|
||||
value: LocalExecutionControlCandidate,
|
||||
): Promise<LocalExecutionControlDisposition> {
|
||||
let candidate = normalizeLocalExecutionControlCandidate(value);
|
||||
if (candidate.kind === 'deadline') {
|
||||
const current = await this.loadActive(
|
||||
candidate.runId,
|
||||
candidate.attemptId,
|
||||
);
|
||||
if (!current) return 'stale';
|
||||
const requested =
|
||||
current.attempt.stepRunId === undefined
|
||||
? await this.requestCancellation(
|
||||
candidate.runId,
|
||||
candidate.attemptId,
|
||||
'timeout',
|
||||
candidate.dueAtMs,
|
||||
true,
|
||||
)
|
||||
: this.workflowTasks === undefined
|
||||
? false
|
||||
: (await this.workflowTasks.requestTimeout({
|
||||
run: current.run,
|
||||
attempt: current.attempt,
|
||||
dueAtMs: candidate.dueAtMs,
|
||||
eventId: this.createEventId(),
|
||||
})) !== 'stale';
|
||||
if (!requested) return 'stale';
|
||||
candidate = Object.freeze({
|
||||
kind: 'cancellation',
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
dueAtMs: candidate.dueAtMs,
|
||||
cancelReason: 'timeout',
|
||||
});
|
||||
}
|
||||
|
||||
const completion = await this.completions.process(candidate.attemptId);
|
||||
if (completion === 'completed' || completion === 'already_terminal') {
|
||||
return 'terminal';
|
||||
}
|
||||
if (completion === 'invalid') return 'remaining';
|
||||
|
||||
const snapshot = await this.loadActive(
|
||||
candidate.runId,
|
||||
candidate.attemptId,
|
||||
);
|
||||
if (!snapshot) return 'stale';
|
||||
const reason =
|
||||
snapshot.attempt.stepRunId === undefined
|
||||
? snapshot.run.cancelReason
|
||||
: snapshot.run.cancelRequestedAtMs === undefined
|
||||
? candidate.cancelReason === 'timeout'
|
||||
? 'timeout'
|
||||
: undefined
|
||||
: snapshot.run.cancelReason;
|
||||
if (reason === undefined) return 'stale';
|
||||
if (snapshot.attempt.status === 'claimed') {
|
||||
return (await this.markTerminal(snapshot, reason))
|
||||
? 'terminal'
|
||||
: 'remaining';
|
||||
}
|
||||
if (!snapshot.attempt.executorHandle) return 'remaining';
|
||||
|
||||
const stopped = await this.controller.stop(snapshot.attempt.executorHandle);
|
||||
const lateCompletion = await this.completions.process(candidate.attemptId);
|
||||
if (
|
||||
lateCompletion === 'completed' ||
|
||||
lateCompletion === 'already_terminal'
|
||||
) {
|
||||
return 'terminal';
|
||||
}
|
||||
if (!conclusiveStop(stopped)) return 'remaining';
|
||||
return (await this.markTerminal(snapshot, reason))
|
||||
? 'terminal'
|
||||
: 'remaining';
|
||||
}
|
||||
|
||||
async requestShutdown(
|
||||
value: LocalActiveExecutionCandidate,
|
||||
requestedAtMs: number,
|
||||
): Promise<LocalExecutionControlDisposition> {
|
||||
const candidate = normalizeLocalActiveExecutionCandidate(value);
|
||||
const current = await this.loadActive(candidate.runId, candidate.attemptId);
|
||||
if (!current) return 'stale';
|
||||
const reason = current.run.cancelReason ?? 'shutdown';
|
||||
if (current.run.cancelRequestedAtMs === undefined) {
|
||||
const requested = await this.requestCancellation(
|
||||
candidate.runId,
|
||||
candidate.attemptId,
|
||||
reason,
|
||||
requestedAtMs,
|
||||
false,
|
||||
);
|
||||
if (!requested) return 'stale';
|
||||
}
|
||||
return this.process({
|
||||
kind: 'cancellation',
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
dueAtMs: current.run.cancelRequestedAtMs ?? requestedAtMs,
|
||||
cancelReason: reason,
|
||||
});
|
||||
}
|
||||
|
||||
private loadActive(
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
): Promise<ActiveSnapshot | null> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(runId);
|
||||
const attempt = await transaction.findAttemptById(attemptId);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
attempt.runId !== run.id ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
!ACTIVE_RUN_STATUSES.has(run.status) ||
|
||||
!ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
|
||||
attempt.executorType !== 'local_process'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
if (attempt.stepRunId === undefined) {
|
||||
const latest = await transaction.findLatestAttemptByRunId(run.id);
|
||||
if (latest?.id !== attempt.id) return null;
|
||||
}
|
||||
return Object.freeze({ run, attempt });
|
||||
});
|
||||
}
|
||||
|
||||
private requestCancellation(
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
reason: RunCancellationReason,
|
||||
requestedAtMs: number,
|
||||
requireDeadline: boolean,
|
||||
): Promise<boolean> {
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(runId);
|
||||
const attempt = await transaction.findAttemptById(attemptId);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
attempt.runId !== run.id ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
!ACTIVE_RUN_STATUSES.has(run.status) ||
|
||||
!ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
|
||||
attempt.executorType !== 'local_process' ||
|
||||
(attempt.stepRunId === undefined &&
|
||||
(await transaction.findLatestAttemptByRunId(run.id))?.id !==
|
||||
attempt.id)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
return run.cancelReason === reason;
|
||||
}
|
||||
const observedAtMs = timestamp(
|
||||
requestedAtMs,
|
||||
'Cancellation request time',
|
||||
);
|
||||
if (
|
||||
requireDeadline &&
|
||||
(attempt.deadlineAtMs === undefined ||
|
||||
attempt.deadlineAtMs > observedAtMs)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const atMs = atOrAfter(observedAtMs, run, attempt);
|
||||
const reserved = reserveEvent(run);
|
||||
const next: RunRecord = {
|
||||
...reserved.run,
|
||||
cancelRequestedAtMs: atMs,
|
||||
cancelReason: reason,
|
||||
};
|
||||
if (!(await transaction.compareAndSetRun(next, run.version))) {
|
||||
throw new LocalExecutionControlConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run,
|
||||
attempt.id,
|
||||
reserved.sequence,
|
||||
'run.cancel_requested',
|
||||
`local-control:${attempt.id}:${reason}:${run.version}`,
|
||||
Object.freeze({
|
||||
reason,
|
||||
from_status: run.status,
|
||||
version: next.version,
|
||||
}),
|
||||
requireDeadline ? 'local-deadline' : 'local-shutdown',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
private markTerminal(
|
||||
expected: ActiveSnapshot,
|
||||
reason: RunCancellationReason,
|
||||
): Promise<boolean> {
|
||||
if (expected.attempt.stepRunId !== undefined) {
|
||||
if (!this.workflowTasks) return Promise.resolve(false);
|
||||
const terminal = terminalFor(reason);
|
||||
const atMs = atOrAfter(this.clock.now(), expected.run, expected.attempt);
|
||||
return this.workflowTasks
|
||||
.recordControlTerminal({
|
||||
run: expected.run,
|
||||
attempt: expected.attempt,
|
||||
reason,
|
||||
terminalStatus: terminal.status,
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
finishedAtMs: atMs,
|
||||
attemptEventId: this.createEventId(),
|
||||
stepMutationId: this.createEventId(),
|
||||
})
|
||||
.then(
|
||||
(result) =>
|
||||
result === 'terminal' || result === 'already_terminal',
|
||||
);
|
||||
}
|
||||
return this.repository.transaction(async (transaction) => {
|
||||
const run = await transaction.findRunById(expected.run.id);
|
||||
const attempt = await transaction.findAttemptById(expected.attempt.id);
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
run.version < expected.run.version ||
|
||||
attempt.runId !== run.id ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
!ACTIVE_RUN_STATUSES.has(run.status) ||
|
||||
!ACTIVE_ATTEMPT_STATUSES.has(attempt.status) ||
|
||||
attempt.callbackSequence !== expected.attempt.callbackSequence ||
|
||||
attempt.executorHandle !== expected.attempt.executorHandle ||
|
||||
attempt.pid !== expected.attempt.pid ||
|
||||
run.cancelRequestedAtMs === undefined ||
|
||||
run.cancelReason !== reason ||
|
||||
(await transaction.findLatestAttemptByRunId(run.id))?.id !== attempt.id
|
||||
) {
|
||||
return TERMINAL_RUN_STATUSES.has(run?.status ?? 'created');
|
||||
}
|
||||
const terminal = terminalFor(reason);
|
||||
const atMs = atOrAfter(this.clock.now(), run, attempt);
|
||||
const attemptReserved = reserveEvent(run);
|
||||
const nextAttempt: RunAttemptRecord = {
|
||||
...attempt,
|
||||
status: terminal.status,
|
||||
finishedAtMs: atMs,
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(attemptReserved.run, run.version))
|
||||
) {
|
||||
throw new LocalExecutionControlConcurrentWriteError();
|
||||
}
|
||||
if (
|
||||
!(await transaction.compareAndSetAttempt(nextAttempt, {
|
||||
status: attempt.status,
|
||||
callbackSequence: attempt.callbackSequence,
|
||||
}))
|
||||
) {
|
||||
throw new LocalExecutionControlConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run,
|
||||
attempt.id,
|
||||
attemptReserved.sequence,
|
||||
`attempt.${terminal.status}`,
|
||||
`local-control:${attempt.id}:${attempt.callbackSequence}:attempt`,
|
||||
Object.freeze({
|
||||
attempt_id: attempt.id,
|
||||
from_status: attempt.status,
|
||||
to_status: terminal.status,
|
||||
reason,
|
||||
version: attemptReserved.run.version,
|
||||
}),
|
||||
'local-execution-control',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
const runReserved = reserveEvent(attemptReserved.run);
|
||||
const nextRun: RunRecord = {
|
||||
...runReserved.run,
|
||||
status: terminal.status,
|
||||
finishedAtMs: atMs,
|
||||
errorCode: terminal.errorCode,
|
||||
errorSummary: terminal.errorSummary,
|
||||
};
|
||||
if (
|
||||
!(await transaction.compareAndSetRun(
|
||||
nextRun,
|
||||
attemptReserved.run.version,
|
||||
))
|
||||
) {
|
||||
throw new LocalExecutionControlConcurrentWriteError();
|
||||
}
|
||||
await transaction.appendEvent(
|
||||
event(
|
||||
this.createEventId(),
|
||||
run,
|
||||
attempt.id,
|
||||
runReserved.sequence,
|
||||
`run.${terminal.status}`,
|
||||
`local-control:${attempt.id}:${attempt.callbackSequence}:run`,
|
||||
Object.freeze({
|
||||
from_status: run.status,
|
||||
to_status: terminal.status,
|
||||
reason,
|
||||
version: nextRun.version,
|
||||
}),
|
||||
'local-execution-control',
|
||||
atMs,
|
||||
),
|
||||
);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export interface LocalExecutionControlScanSummary {
|
||||
readonly scanned: number;
|
||||
readonly terminal: number;
|
||||
readonly cancelRequested: number;
|
||||
readonly stale: number;
|
||||
readonly remaining: number;
|
||||
readonly failed: number;
|
||||
readonly truncated: boolean;
|
||||
readonly nextCursor?: LocalExecutionControlCursor;
|
||||
}
|
||||
|
||||
export interface LocalExecutionDrainSummary {
|
||||
readonly scanned: number;
|
||||
readonly terminal: number;
|
||||
readonly remaining: number;
|
||||
readonly failed: number;
|
||||
readonly truncated: boolean;
|
||||
}
|
||||
|
||||
function controlCursor(candidate: LocalExecutionControlCandidate) {
|
||||
return Object.freeze({
|
||||
dueAtMs: candidate.dueAtMs,
|
||||
kind: candidate.kind,
|
||||
attemptId: candidate.attemptId,
|
||||
});
|
||||
}
|
||||
|
||||
function controlAdvances(
|
||||
previous: LocalExecutionControlCursor,
|
||||
next: LocalExecutionControlCursor,
|
||||
): boolean {
|
||||
return (
|
||||
next.dueAtMs > previous.dueAtMs ||
|
||||
(next.dueAtMs === previous.dueAtMs &&
|
||||
(next.kind > previous.kind ||
|
||||
(next.kind === previous.kind && next.attemptId > previous.attemptId)))
|
||||
);
|
||||
}
|
||||
|
||||
function activeCursor(candidate: LocalActiveExecutionCandidate) {
|
||||
return Object.freeze({
|
||||
attemptCreatedAtMs: candidate.attemptCreatedAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
});
|
||||
}
|
||||
|
||||
function activeAdvances(
|
||||
previous: LocalActiveExecutionCursor,
|
||||
next: LocalActiveExecutionCursor,
|
||||
): boolean {
|
||||
return (
|
||||
next.attemptCreatedAtMs > previous.attemptCreatedAtMs ||
|
||||
(next.attemptCreatedAtMs === previous.attemptCreatedAtMs &&
|
||||
next.attemptId > previous.attemptId)
|
||||
);
|
||||
}
|
||||
|
||||
export class LocalExecutionControlScanner {
|
||||
constructor(
|
||||
private readonly source: LocalExecutionControlSource,
|
||||
private readonly coordinator: Pick<
|
||||
LocalExecutionControlCoordinator,
|
||||
'process' | 'requestShutdown'
|
||||
>,
|
||||
private readonly clock: { now(): number } = { now: Date.now },
|
||||
) {}
|
||||
|
||||
async scan(options: {
|
||||
readonly limit: number;
|
||||
readonly cursor?: LocalExecutionControlCursor;
|
||||
}): Promise<LocalExecutionControlScanSummary> {
|
||||
assertLocalExecutionControlLimit(options.limit);
|
||||
const observedAtMs = timestamp(
|
||||
this.clock.now(),
|
||||
'Local execution control scan time',
|
||||
);
|
||||
const page = await this.source.listLocalExecutionControlCandidates({
|
||||
observedAtMs,
|
||||
limit: options.limit,
|
||||
...(options.cursor === undefined ? {} : { after: options.cursor }),
|
||||
});
|
||||
if (page.candidates.length > options.limit) {
|
||||
throw new RangeError('Local execution control source exceeded page size');
|
||||
}
|
||||
let previous = options.cursor;
|
||||
let terminal = 0;
|
||||
let cancelRequested = 0;
|
||||
let stale = 0;
|
||||
let remaining = 0;
|
||||
let failed = 0;
|
||||
for (const value of page.candidates) {
|
||||
const candidate = normalizeLocalExecutionControlCandidate(value);
|
||||
const cursor = controlCursor(candidate);
|
||||
if (previous && !controlAdvances(previous, cursor)) {
|
||||
throw new TypeError(
|
||||
'Local execution control page is not strictly ordered',
|
||||
);
|
||||
}
|
||||
previous = cursor;
|
||||
try {
|
||||
const disposition = await this.coordinator.process(candidate);
|
||||
if (disposition === 'terminal') terminal += 1;
|
||||
if (disposition === 'cancel_requested') cancelRequested += 1;
|
||||
if (disposition === 'stale') stale += 1;
|
||||
if (disposition === 'remaining') remaining += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
if (page.truncated && page.candidates.length === 0) {
|
||||
throw new TypeError(
|
||||
'Local execution control source returned an empty truncated page',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
scanned: page.candidates.length,
|
||||
terminal,
|
||||
cancelRequested,
|
||||
stale,
|
||||
remaining,
|
||||
failed,
|
||||
truncated: page.truncated,
|
||||
...(page.truncated && previous ? { nextCursor: previous } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
async drain(options: {
|
||||
readonly limit: number;
|
||||
readonly maxPages: number;
|
||||
}): Promise<LocalExecutionDrainSummary> {
|
||||
assertLocalExecutionControlLimit(options.limit);
|
||||
if (
|
||||
!Number.isSafeInteger(options.maxPages) ||
|
||||
options.maxPages < 1 ||
|
||||
options.maxPages > MAX_LOCAL_EXECUTION_CONTROL_PAGES
|
||||
) {
|
||||
throw new RangeError('Local execution drain page budget is invalid');
|
||||
}
|
||||
const requestedAtMs = timestamp(
|
||||
this.clock.now(),
|
||||
'Local execution drain time',
|
||||
);
|
||||
let cursor: LocalActiveExecutionCursor | undefined;
|
||||
let scanned = 0;
|
||||
let terminal = 0;
|
||||
let remaining = 0;
|
||||
let failed = 0;
|
||||
let truncated = false;
|
||||
for (let pageIndex = 0; pageIndex < options.maxPages; pageIndex += 1) {
|
||||
const page = await this.source.listLocalActiveExecutions({
|
||||
limit: options.limit,
|
||||
...(cursor === undefined ? {} : { after: cursor }),
|
||||
});
|
||||
if (page.candidates.length > options.limit) {
|
||||
throw new RangeError(
|
||||
'Local active execution source exceeded page size',
|
||||
);
|
||||
}
|
||||
let previous = cursor;
|
||||
for (const value of page.candidates) {
|
||||
const candidate = normalizeLocalActiveExecutionCandidate(value);
|
||||
const next = activeCursor(candidate);
|
||||
if (previous && !activeAdvances(previous, next)) {
|
||||
throw new TypeError(
|
||||
'Local active execution page is not strictly ordered',
|
||||
);
|
||||
}
|
||||
previous = next;
|
||||
scanned += 1;
|
||||
try {
|
||||
const disposition = await this.coordinator.requestShutdown(
|
||||
candidate,
|
||||
requestedAtMs,
|
||||
);
|
||||
if (disposition === 'terminal' || disposition === 'stale')
|
||||
terminal += 1;
|
||||
else remaining += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
truncated = page.truncated;
|
||||
if (!page.truncated) break;
|
||||
if (!previous) {
|
||||
throw new TypeError(
|
||||
'Local active execution source returned an empty truncated page',
|
||||
);
|
||||
}
|
||||
cursor = previous;
|
||||
}
|
||||
return Object.freeze({
|
||||
scanned,
|
||||
terminal,
|
||||
remaining,
|
||||
failed,
|
||||
truncated,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from './completion';
|
||||
export * from './control';
|
||||
export * from './lifecycle';
|
||||
@@ -0,0 +1,291 @@
|
||||
import type { LocalCompletionReceiptJournalCursor } from '@qinglong/runtime-core/local-completion-receipt-journal';
|
||||
import { assertLocalExecutionControlLimit } from '@qinglong/runtime-core/local-execution-control';
|
||||
import type {
|
||||
LocalCompletionReceiptCleanupScanner,
|
||||
LocalCompletionReceiptCleanupSummary,
|
||||
} from '@qinglong/local-process';
|
||||
import type { LocalCompletionReceiptProcessor } from './completion';
|
||||
import type {
|
||||
LocalExecutionControlScanSummary,
|
||||
LocalExecutionControlScanner,
|
||||
LocalExecutionDrainSummary,
|
||||
} from './control';
|
||||
|
||||
export const MAX_LOCAL_COMPLETION_NOTIFICATIONS = 64;
|
||||
|
||||
export interface LocalExecutionControlLifecycleOptions {
|
||||
readonly intervalMs: number;
|
||||
readonly pageSize: number;
|
||||
readonly cleanupIntervalMs: number;
|
||||
readonly cleanupPageSize: number;
|
||||
readonly stopTimeoutMs: number;
|
||||
readonly maxDrainPages: number;
|
||||
readonly maxNotifications?: number;
|
||||
readonly clock?: { now(): number };
|
||||
readonly onDiagnostic?: (
|
||||
error: unknown,
|
||||
summary?: LocalExecutionControlCycleSummary,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface LocalExecutionControlCycleSummary {
|
||||
readonly completions: number;
|
||||
readonly completionFailures: number;
|
||||
readonly control: LocalExecutionControlScanSummary;
|
||||
readonly cleanup?: LocalCompletionReceiptCleanupSummary;
|
||||
}
|
||||
|
||||
export interface LocalExecutionControlStopSummary {
|
||||
readonly status: 'stopped' | 'timed_out';
|
||||
readonly drain?: LocalExecutionDrainSummary;
|
||||
readonly cleanup?: LocalCompletionReceiptCleanupSummary;
|
||||
}
|
||||
|
||||
export class LocalExecutionControlLifecycle {
|
||||
private readonly clock: { now(): number };
|
||||
private readonly maxNotifications: number;
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private inFlight: Promise<LocalExecutionControlCycleSummary> | undefined;
|
||||
private stopPromise: Promise<LocalExecutionControlStopSummary> | undefined;
|
||||
private running = false;
|
||||
private stopping = false;
|
||||
private kickQueued = false;
|
||||
private controlCursor:
|
||||
| NonNullable<LocalExecutionControlScanSummary['nextCursor']>
|
||||
| undefined;
|
||||
private cleanupCursor: LocalCompletionReceiptJournalCursor | undefined;
|
||||
private lastCleanupAtMs: number | undefined;
|
||||
private readonly pending = new Set<string>();
|
||||
|
||||
constructor(
|
||||
private readonly completions: Pick<
|
||||
LocalCompletionReceiptProcessor,
|
||||
'process'
|
||||
>,
|
||||
private readonly control: Pick<
|
||||
LocalExecutionControlScanner,
|
||||
'scan' | 'drain'
|
||||
>,
|
||||
private readonly cleanup: Pick<
|
||||
LocalCompletionReceiptCleanupScanner,
|
||||
'scan'
|
||||
>,
|
||||
private readonly options: LocalExecutionControlLifecycleOptions,
|
||||
) {
|
||||
if (
|
||||
!Number.isSafeInteger(options.intervalMs) ||
|
||||
options.intervalMs < 250 ||
|
||||
options.intervalMs > 60 * 60_000
|
||||
) {
|
||||
throw new RangeError('Local execution control interval is invalid');
|
||||
}
|
||||
assertLocalExecutionControlLimit(options.pageSize);
|
||||
if (
|
||||
!Number.isSafeInteger(options.cleanupIntervalMs) ||
|
||||
options.cleanupIntervalMs < 1_000 ||
|
||||
options.cleanupIntervalMs > 24 * 60 * 60_000
|
||||
) {
|
||||
throw new RangeError('Local execution cleanup interval is invalid');
|
||||
}
|
||||
assertLocalExecutionControlLimit(options.cleanupPageSize);
|
||||
if (
|
||||
!Number.isSafeInteger(options.stopTimeoutMs) ||
|
||||
options.stopTimeoutMs < 100 ||
|
||||
options.stopTimeoutMs > 30_000
|
||||
) {
|
||||
throw new RangeError('Local execution control stop timeout is invalid');
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(options.maxDrainPages) ||
|
||||
options.maxDrainPages < 1 ||
|
||||
options.maxDrainPages > 16
|
||||
) {
|
||||
throw new RangeError('Local execution control drain budget is invalid');
|
||||
}
|
||||
this.maxNotifications =
|
||||
options.maxNotifications ?? MAX_LOCAL_COMPLETION_NOTIFICATIONS;
|
||||
if (
|
||||
!Number.isSafeInteger(this.maxNotifications) ||
|
||||
this.maxNotifications < 1 ||
|
||||
this.maxNotifications > MAX_LOCAL_COMPLETION_NOTIFICATIONS
|
||||
) {
|
||||
throw new RangeError('Local completion notification budget is invalid');
|
||||
}
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.running || this.stopping) return;
|
||||
this.running = true;
|
||||
this.schedule();
|
||||
}
|
||||
|
||||
notifyCompletion(attemptId: string): boolean {
|
||||
if (
|
||||
this.stopping ||
|
||||
typeof attemptId !== 'string' ||
|
||||
attemptId.length > 128
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
!this.pending.has(attemptId) &&
|
||||
this.pending.size >= this.maxNotifications
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
this.pending.add(attemptId);
|
||||
this.kick();
|
||||
return true;
|
||||
}
|
||||
|
||||
runOnce(forceCleanup = false): Promise<LocalExecutionControlCycleSummary> {
|
||||
if (this.inFlight) return this.inFlight;
|
||||
const work = this.cycle(forceCleanup).finally(() => {
|
||||
if (this.inFlight === work) this.inFlight = undefined;
|
||||
});
|
||||
this.inFlight = work;
|
||||
return work;
|
||||
}
|
||||
|
||||
stopAndDrain(): Promise<LocalExecutionControlStopSummary> {
|
||||
if (this.stopPromise) return this.stopPromise;
|
||||
this.stopping = true;
|
||||
this.running = false;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
|
||||
const stopWork = (async () => {
|
||||
await this.inFlight;
|
||||
await this.processPending();
|
||||
const drain = await this.control.drain({
|
||||
limit: this.options.pageSize,
|
||||
maxPages: this.options.maxDrainPages,
|
||||
});
|
||||
const cleanup = await this.cleanup.scan({
|
||||
limit: this.options.cleanupPageSize,
|
||||
...(this.cleanupCursor === undefined
|
||||
? {}
|
||||
: { cursor: this.cleanupCursor }),
|
||||
});
|
||||
this.cleanupCursor = cleanup.truncated ? cleanup.nextCursor : undefined;
|
||||
return Object.freeze({
|
||||
status:
|
||||
drain.remaining === 0 && drain.failed === 0 && !drain.truncated
|
||||
? ('stopped' as const)
|
||||
: ('timed_out' as const),
|
||||
drain,
|
||||
cleanup,
|
||||
});
|
||||
})();
|
||||
this.stopPromise = Promise.race([
|
||||
stopWork,
|
||||
new Promise<LocalExecutionControlStopSummary>((resolve) => {
|
||||
const timer = setTimeout(
|
||||
() => resolve(Object.freeze({ status: 'timed_out' as const })),
|
||||
this.options.stopTimeoutMs,
|
||||
);
|
||||
timer.unref?.();
|
||||
}),
|
||||
]);
|
||||
return this.stopPromise;
|
||||
}
|
||||
|
||||
private schedule(): void {
|
||||
if (!this.running || this.timer) return;
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = undefined;
|
||||
if (!this.running) return;
|
||||
void this.runOnce()
|
||||
.then((summary) => this.diagnostic(undefined, summary))
|
||||
.catch((error) => this.diagnostic(error))
|
||||
.finally(() => this.schedule());
|
||||
}, this.options.intervalMs);
|
||||
this.timer.unref?.();
|
||||
}
|
||||
|
||||
private kick(): void {
|
||||
if (this.kickQueued || this.inFlight || this.stopping) return;
|
||||
this.kickQueued = true;
|
||||
queueMicrotask(() => {
|
||||
this.kickQueued = false;
|
||||
if (this.stopping || this.inFlight) return;
|
||||
void this.runOnce()
|
||||
.then((summary) => this.diagnostic(undefined, summary))
|
||||
.catch((error) => this.diagnostic(error));
|
||||
});
|
||||
}
|
||||
|
||||
private async cycle(
|
||||
forceCleanup: boolean,
|
||||
): Promise<LocalExecutionControlCycleSummary> {
|
||||
const completion = await this.processPending();
|
||||
const control = await this.control.scan({
|
||||
limit: this.options.pageSize,
|
||||
...(this.controlCursor === undefined
|
||||
? {}
|
||||
: { cursor: this.controlCursor }),
|
||||
});
|
||||
this.controlCursor = control.truncated ? control.nextCursor : undefined;
|
||||
const now = this.clock.now();
|
||||
if (!Number.isSafeInteger(now) || now < 0) {
|
||||
throw new RangeError(
|
||||
'Local execution control lifecycle clock is invalid',
|
||||
);
|
||||
}
|
||||
let cleanup: LocalCompletionReceiptCleanupSummary | undefined;
|
||||
if (
|
||||
forceCleanup ||
|
||||
this.lastCleanupAtMs === undefined ||
|
||||
now - this.lastCleanupAtMs >= this.options.cleanupIntervalMs
|
||||
) {
|
||||
cleanup = await this.cleanup.scan({
|
||||
limit: this.options.cleanupPageSize,
|
||||
...(this.cleanupCursor === undefined
|
||||
? {}
|
||||
: { cursor: this.cleanupCursor }),
|
||||
});
|
||||
this.cleanupCursor = cleanup.truncated ? cleanup.nextCursor : undefined;
|
||||
this.lastCleanupAtMs = now;
|
||||
}
|
||||
if (this.pending.size > 0) this.kick();
|
||||
return Object.freeze({
|
||||
completions: completion.processed,
|
||||
completionFailures: completion.failed,
|
||||
control,
|
||||
...(cleanup === undefined ? {} : { cleanup }),
|
||||
});
|
||||
}
|
||||
|
||||
private async processPending(): Promise<
|
||||
Readonly<{
|
||||
processed: number;
|
||||
failed: number;
|
||||
}>
|
||||
> {
|
||||
const batch = [...this.pending].slice(0, this.maxNotifications);
|
||||
for (const attemptId of batch) this.pending.delete(attemptId);
|
||||
let processed = 0;
|
||||
let failed = 0;
|
||||
for (const attemptId of batch) {
|
||||
try {
|
||||
await this.completions.process(attemptId);
|
||||
processed += 1;
|
||||
} catch {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
return Object.freeze({ processed, failed });
|
||||
}
|
||||
|
||||
private async diagnostic(
|
||||
error: unknown,
|
||||
summary?: LocalExecutionControlCycleSummary,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.options.onDiagnostic?.(error, summary);
|
||||
} catch {
|
||||
// Diagnostics cannot own execution-control liveness.
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user