mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,706 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import type {
|
||||
ExecutionContext,
|
||||
ExecutionHandle,
|
||||
ExecutionResult,
|
||||
ExecutionSpec,
|
||||
ExecutionStopReason,
|
||||
ExecutionStopResult,
|
||||
} from '../domain/execution';
|
||||
import type { RunAttemptRecord, RunRecord } from '../domain/run';
|
||||
import {
|
||||
assertAdmittedRunRetryPolicy,
|
||||
type RunRetryPolicyDefinition,
|
||||
} from '../domain/runRetryPolicy';
|
||||
import {
|
||||
MAX_LOG_ARTIFACT_ID_LENGTH,
|
||||
isTerminalRunAttemptStatus,
|
||||
isTerminalRunStatus,
|
||||
} from '../domain/runStateMachine';
|
||||
import type { Executor } from '../ports/executor';
|
||||
import type { CompletionReceiptJournal } from '../ports/completionReceiptJournal';
|
||||
import type { PrimaryRunIdempotencyLookup } from '../ports/primaryRunIdempotencyLookup';
|
||||
import type { RunRepository } from '../ports/runRepository';
|
||||
import type { RunRetryPolicyAdmission } from '../ports/runRetryPolicyAdmission';
|
||||
import { DuplicateIdempotencyKeyError } from '../domain/repositoryErrors';
|
||||
import {
|
||||
PrimaryRunCreator,
|
||||
type PrimaryRunDefinition,
|
||||
type PrimaryRunIdFactory,
|
||||
type PrimaryRunReference,
|
||||
} from './primaryRunCreator';
|
||||
import { RunCommandService } from './runCommandService';
|
||||
import {
|
||||
hashPrimaryCompletionToken,
|
||||
PrimaryRunCompletionService,
|
||||
} from './primaryRunCompletionService';
|
||||
|
||||
export interface PrimaryRunClock {
|
||||
now(): number;
|
||||
}
|
||||
|
||||
export interface PrimaryRunOrchestratorOptions {
|
||||
clock?: PrimaryRunClock;
|
||||
createId?: PrimaryRunIdFactory;
|
||||
idempotencyLookup?: PrimaryRunIdempotencyLookup;
|
||||
createCallbackToken?: () => string;
|
||||
completionReceiptJournal?: Pick<CompletionReceiptJournal, 'register'>;
|
||||
retryPolicyAdmission?: RunRetryPolicyAdmission;
|
||||
}
|
||||
|
||||
export interface PrimaryRunStartCommand {
|
||||
definition: Omit<PrimaryRunDefinition, 'acceptedAtMs' | 'retryPolicy'> & {
|
||||
acceptedAtMs?: number;
|
||||
};
|
||||
timeoutMs?: number;
|
||||
createSpec(reference: PrimaryRunReference): ExecutionSpec;
|
||||
context: ExecutionContext;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trusted local-dispatch input for an aggregate that already owns a claimed
|
||||
* Attempt. Callers must materialize the spec from the persisted Task revision.
|
||||
*/
|
||||
export interface PrimaryClaimedRunStartCommand {
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
timeoutMs?: number;
|
||||
createSpec(reference: PrimaryRunReference): ExecutionSpec;
|
||||
context: ExecutionContext;
|
||||
logArtifactId?: string;
|
||||
}
|
||||
|
||||
export interface PrimaryRunCompletion {
|
||||
run: RunRecord;
|
||||
attempt: RunAttemptRecord;
|
||||
result: ExecutionResult;
|
||||
}
|
||||
|
||||
export interface ActivePrimaryRun extends PrimaryRunReference {
|
||||
handle: ExecutionHandle;
|
||||
completion: Promise<PrimaryRunCompletion>;
|
||||
cancel(reason: ExecutionStopReason): Promise<ExecutionStopResult>;
|
||||
}
|
||||
|
||||
export class PrimaryRunLaunchError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly reference: PrimaryRunReference,
|
||||
options?: ErrorOptions,
|
||||
) {
|
||||
super(message, options);
|
||||
this.name = 'PrimaryRunLaunchError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunNotActiveError extends Error {
|
||||
constructor(readonly runId: string) {
|
||||
super(`Primary Run is not active: ${runId}`);
|
||||
this.name = 'PrimaryRunNotActiveError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunDuplicateRequestError extends Error {
|
||||
constructor(
|
||||
readonly projectId: string,
|
||||
readonly idempotencyKey: string,
|
||||
readonly existingRunId: string,
|
||||
) {
|
||||
super('A Primary Run already exists for this idempotent request');
|
||||
this.name = 'PrimaryRunDuplicateRequestError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunIdempotencyUnavailableError extends Error {
|
||||
constructor() {
|
||||
super('Primary Run idempotency lookup is not configured');
|
||||
this.name = 'PrimaryRunIdempotencyUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class PrimaryRunRetryPolicyAuthorityError extends Error {
|
||||
readonly code = 'PRIMARY_RUN_RETRY_POLICY_AUTHORITY_REQUIRED';
|
||||
|
||||
constructor() {
|
||||
super('Run requests cannot self-assert automatic retry safety');
|
||||
this.name = 'PrimaryRunRetryPolicyAuthorityError';
|
||||
}
|
||||
}
|
||||
|
||||
export type PrimaryClaimedRunRejectionReason =
|
||||
| 'run_not_found'
|
||||
| 'attempt_not_found'
|
||||
| 'aggregate_mismatch'
|
||||
| 'not_latest_attempt'
|
||||
| 'not_queued'
|
||||
| 'not_claimed'
|
||||
| 'stale_execution_authority'
|
||||
| 'executor_mismatch'
|
||||
| 'cancellation_requested'
|
||||
| 'already_active';
|
||||
|
||||
export class PrimaryClaimedRunRejectedError extends Error {
|
||||
readonly code = 'PRIMARY_CLAIMED_RUN_REJECTED';
|
||||
|
||||
constructor(readonly reason: PrimaryClaimedRunRejectionReason) {
|
||||
super(`Claimed Primary Run activation was rejected: ${reason}`);
|
||||
this.name = 'PrimaryClaimedRunRejectedError';
|
||||
}
|
||||
}
|
||||
|
||||
interface ActiveExecution {
|
||||
handle: ExecutionHandle;
|
||||
completion: Promise<PrimaryRunCompletion>;
|
||||
}
|
||||
|
||||
const ALREADY_EXITED_STOP_RESULT: ExecutionStopResult = {
|
||||
status: 'already_exited',
|
||||
termSignalSent: false,
|
||||
killSignalSent: false,
|
||||
};
|
||||
|
||||
/**
|
||||
* Serializes durable Run transitions around a single Executor side effect.
|
||||
* It deliberately owns no scheduler or HTTP routing policy.
|
||||
*/
|
||||
export class PrimaryRunOrchestrator {
|
||||
private readonly clock: PrimaryRunClock;
|
||||
private readonly creator: PrimaryRunCreator;
|
||||
private readonly commands: RunCommandService;
|
||||
private readonly completions: PrimaryRunCompletionService;
|
||||
private readonly createCallbackToken: () => string;
|
||||
private readonly idempotencyLookup?: PrimaryRunIdempotencyLookup;
|
||||
private readonly completionReceiptJournal?: Pick<
|
||||
CompletionReceiptJournal,
|
||||
'register'
|
||||
>;
|
||||
private readonly retryPolicyAdmission?: RunRetryPolicyAdmission;
|
||||
private readonly active = new Map<string, ActiveExecution>();
|
||||
|
||||
constructor(
|
||||
private readonly repository: RunRepository,
|
||||
private readonly executor: Executor,
|
||||
options: PrimaryRunOrchestratorOptions = {},
|
||||
) {
|
||||
this.clock = options.clock ?? { now: Date.now };
|
||||
this.creator = new PrimaryRunCreator(repository, options.createId);
|
||||
this.commands = new RunCommandService(repository, options.createId);
|
||||
this.completions = new PrimaryRunCompletionService(
|
||||
repository,
|
||||
options.createId,
|
||||
);
|
||||
this.createCallbackToken =
|
||||
options.createCallbackToken ??
|
||||
(() => randomBytes(32).toString('base64url'));
|
||||
this.idempotencyLookup = options.idempotencyLookup;
|
||||
this.completionReceiptJournal = options.completionReceiptJournal;
|
||||
this.retryPolicyAdmission = options.retryPolicyAdmission;
|
||||
}
|
||||
|
||||
async start(command: PrimaryRunStartCommand): Promise<ActivePrimaryRun> {
|
||||
this.assertTimeout(command.timeoutMs);
|
||||
if (
|
||||
Object.prototype.hasOwnProperty.call(command.definition, 'retryPolicy')
|
||||
) {
|
||||
throw new PrimaryRunRetryPolicyAuthorityError();
|
||||
}
|
||||
const retryPolicy = await this.admitRetryPolicy(command.definition);
|
||||
const acceptedAtMs = command.definition.acceptedAtMs ?? this.clock.now();
|
||||
const reference = await this.createPrimaryRun(
|
||||
{
|
||||
...command.definition,
|
||||
acceptedAtMs,
|
||||
...(retryPolicy === undefined ? {} : { retryPolicy }),
|
||||
},
|
||||
this.executor.type,
|
||||
);
|
||||
return this.activateReference(reference, command);
|
||||
}
|
||||
|
||||
async activateClaimed(
|
||||
command: PrimaryClaimedRunStartCommand,
|
||||
): Promise<ActivePrimaryRun> {
|
||||
this.assertTimeout(command.timeoutMs);
|
||||
this.assertLogArtifactId(command.logArtifactId);
|
||||
const reference = await this.loadClaimedReference(
|
||||
command.runId,
|
||||
command.attemptId,
|
||||
);
|
||||
return this.activateReference(reference, command);
|
||||
}
|
||||
|
||||
private async activateReference(
|
||||
initialReference: PrimaryRunReference,
|
||||
command: Pick<
|
||||
PrimaryClaimedRunStartCommand,
|
||||
'timeoutMs' | 'createSpec' | 'context' | 'logArtifactId'
|
||||
>,
|
||||
): Promise<ActivePrimaryRun> {
|
||||
const callbackToken = this.createCallbackToken();
|
||||
const callbackTokenHash = hashPrimaryCompletionToken(callbackToken);
|
||||
const callbackSequence = initialReference.attempt.callbackSequence + 1;
|
||||
let reference = initialReference;
|
||||
|
||||
reference = await this.prepareForSpawn(
|
||||
reference,
|
||||
command.timeoutMs,
|
||||
callbackTokenHash,
|
||||
command.logArtifactId,
|
||||
);
|
||||
|
||||
let spec: ExecutionSpec;
|
||||
let handle: ExecutionHandle;
|
||||
try {
|
||||
await this.completionReceiptJournal?.register({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
registeredAtMs: reference.attempt.createdAtMs,
|
||||
});
|
||||
spec = command.createSpec(reference);
|
||||
this.assertSpecMatches(reference, spec, command.timeoutMs);
|
||||
handle = await this.executor.start(spec, {
|
||||
...command.context,
|
||||
completionCallback: {
|
||||
token: callbackToken,
|
||||
callbackSequence,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
reference = await this.recordStartFailure(reference);
|
||||
throw new PrimaryRunLaunchError(
|
||||
'Primary Run could not start its Executor',
|
||||
reference,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
this.assertHandleMatches(reference, handle);
|
||||
reference = await this.recordRunning(reference, handle);
|
||||
} catch (error) {
|
||||
void handle.completion.catch(() => undefined);
|
||||
await this.compensateActivationFailure(reference, handle);
|
||||
const latest = await this.loadReference(reference);
|
||||
throw new PrimaryRunLaunchError(
|
||||
'Primary Run could not persist Executor ownership',
|
||||
latest,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
|
||||
const completion = handle.completion.then(
|
||||
(result) =>
|
||||
this.completions.complete({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
callbackSequence,
|
||||
result,
|
||||
source: { kind: 'executor', executorType: this.executor.type },
|
||||
}),
|
||||
() =>
|
||||
this.completions.complete({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
callbackSequence,
|
||||
result: {
|
||||
outcome: 'lost',
|
||||
startedAtMs: handle.startedAtMs,
|
||||
finishedAtMs: this.atOrAfter(handle.startedAtMs),
|
||||
errorCode: 'EXECUTOR_COMPLETION_REJECTED',
|
||||
errorSummary: 'Executor completion channel rejected',
|
||||
},
|
||||
source: { kind: 'executor', executorType: this.executor.type },
|
||||
}),
|
||||
);
|
||||
this.active.set(reference.run.id, { handle, completion });
|
||||
void completion.then(
|
||||
() => this.deleteActive(reference.run.id, handle),
|
||||
() => this.deleteActive(reference.run.id, handle),
|
||||
);
|
||||
|
||||
return {
|
||||
...reference,
|
||||
handle,
|
||||
completion,
|
||||
cancel: (reason) => this.cancel(reference.run.id, reason),
|
||||
};
|
||||
}
|
||||
|
||||
private async loadClaimedReference(
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const [run, attempt, latestAttempt] = await Promise.all([
|
||||
this.repository.findRunById(runId),
|
||||
this.repository.findAttemptById(attemptId),
|
||||
this.repository.findLatestAttemptByRunId(runId),
|
||||
]);
|
||||
if (!run) throw new PrimaryClaimedRunRejectedError('run_not_found');
|
||||
if (!attempt) {
|
||||
throw new PrimaryClaimedRunRejectedError('attempt_not_found');
|
||||
}
|
||||
if (
|
||||
attempt.runId !== run.id ||
|
||||
run.executionOwner !== 'runtime' ||
|
||||
latestAttempt?.runId !== run.id
|
||||
) {
|
||||
throw new PrimaryClaimedRunRejectedError('aggregate_mismatch');
|
||||
}
|
||||
if (latestAttempt.id !== attempt.id) {
|
||||
throw new PrimaryClaimedRunRejectedError('not_latest_attempt');
|
||||
}
|
||||
if (run.status !== 'queued') {
|
||||
throw new PrimaryClaimedRunRejectedError('not_queued');
|
||||
}
|
||||
if (attempt.status !== 'claimed') {
|
||||
throw new PrimaryClaimedRunRejectedError('not_claimed');
|
||||
}
|
||||
if (
|
||||
attempt.callbackSequence !== 0 ||
|
||||
attempt.callbackTokenHash !== undefined ||
|
||||
attempt.workerId !== undefined ||
|
||||
attempt.executorHandle !== undefined ||
|
||||
attempt.pid !== undefined ||
|
||||
attempt.logArtifactId !== undefined ||
|
||||
attempt.leaseToken !== undefined ||
|
||||
attempt.leaseExpiresAtMs !== undefined ||
|
||||
attempt.startedAtMs !== undefined ||
|
||||
attempt.finishedAtMs !== undefined ||
|
||||
attempt.exitCode !== undefined ||
|
||||
attempt.errorCode !== undefined ||
|
||||
attempt.errorSummary !== undefined
|
||||
) {
|
||||
throw new PrimaryClaimedRunRejectedError('stale_execution_authority');
|
||||
}
|
||||
if (attempt.executorType !== this.executor.type) {
|
||||
throw new PrimaryClaimedRunRejectedError('executor_mismatch');
|
||||
}
|
||||
if (run.cancelRequestedAtMs !== undefined) {
|
||||
throw new PrimaryClaimedRunRejectedError('cancellation_requested');
|
||||
}
|
||||
if (this.active.has(run.id)) {
|
||||
throw new PrimaryClaimedRunRejectedError('already_active');
|
||||
}
|
||||
return { run, attempt };
|
||||
}
|
||||
|
||||
private async admitRetryPolicy(
|
||||
definition: PrimaryRunStartCommand['definition'],
|
||||
): Promise<RunRetryPolicyDefinition | undefined> {
|
||||
if (!this.retryPolicyAdmission) return undefined;
|
||||
const admitted = await this.retryPolicyAdmission.resolve(
|
||||
Object.freeze({
|
||||
projectId: definition.projectId,
|
||||
taskId: definition.taskId,
|
||||
taskRevision: definition.taskRevision,
|
||||
triggerType: definition.triggerType,
|
||||
executionOrigin: definition.executionOrigin,
|
||||
}),
|
||||
);
|
||||
if (admitted === undefined) return undefined;
|
||||
const policy: RunRetryPolicyDefinition = {
|
||||
maxAttempts: admitted.maxAttempts,
|
||||
retryOnLost: admitted.retryOnLost,
|
||||
safety: admitted.safety,
|
||||
backoffBaseMs: admitted.backoffBaseMs,
|
||||
backoffMaxMs: admitted.backoffMaxMs,
|
||||
};
|
||||
assertAdmittedRunRetryPolicy(policy);
|
||||
return policy;
|
||||
}
|
||||
|
||||
async cancel(
|
||||
runId: string,
|
||||
reason: ExecutionStopReason,
|
||||
): Promise<ExecutionStopResult> {
|
||||
const active = this.active.get(runId);
|
||||
if (!active) throw new PrimaryRunNotActiveError(runId);
|
||||
const request = await this.commands.requestCancellation({
|
||||
runId,
|
||||
attemptId: active.handle.attemptId,
|
||||
atMs: this.atOrAfter(reason.requestedAtMs, active.handle.startedAtMs),
|
||||
reason: reason.kind,
|
||||
actor: this.cancellationActor(reason),
|
||||
});
|
||||
if (request.status === 'already_terminal') {
|
||||
return ALREADY_EXITED_STOP_RESULT;
|
||||
}
|
||||
return this.executor.stop(active.handle, reason);
|
||||
}
|
||||
|
||||
isActive(runId: string): boolean {
|
||||
return this.active.has(runId);
|
||||
}
|
||||
|
||||
private async createPrimaryRun(
|
||||
definition: PrimaryRunDefinition,
|
||||
executorType: Executor['type'],
|
||||
): Promise<PrimaryRunReference> {
|
||||
const key = definition.idempotencyKey;
|
||||
if (key === undefined) {
|
||||
return this.creator.create(definition, executorType);
|
||||
}
|
||||
if (!this.idempotencyLookup) {
|
||||
throw new PrimaryRunIdempotencyUnavailableError();
|
||||
}
|
||||
|
||||
const existingRunId = await this.idempotencyLookup.findRunId(
|
||||
definition.projectId,
|
||||
key,
|
||||
);
|
||||
if (existingRunId) {
|
||||
throw new PrimaryRunDuplicateRequestError(
|
||||
definition.projectId,
|
||||
key,
|
||||
existingRunId,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
return await this.creator.create(definition, executorType);
|
||||
} catch (error) {
|
||||
if (!(error instanceof DuplicateIdempotencyKeyError)) throw error;
|
||||
const racedRunId = await this.idempotencyLookup.findRunId(
|
||||
definition.projectId,
|
||||
key,
|
||||
);
|
||||
if (!racedRunId) throw error;
|
||||
throw new PrimaryRunDuplicateRequestError(
|
||||
definition.projectId,
|
||||
key,
|
||||
racedRunId,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async prepareForSpawn(
|
||||
reference: PrimaryRunReference,
|
||||
timeoutMs: number | undefined,
|
||||
callbackTokenHash: string,
|
||||
logArtifactId?: string,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const startingAtMs = this.atOrAfter(
|
||||
reference.run.createdAtMs,
|
||||
reference.attempt.createdAtMs,
|
||||
);
|
||||
const deadlineAtMs =
|
||||
timeoutMs === undefined ? undefined : startingAtMs + timeoutMs;
|
||||
if (deadlineAtMs !== undefined && !Number.isSafeInteger(deadlineAtMs)) {
|
||||
throw new RangeError('Primary Run deadline exceeds the supported range');
|
||||
}
|
||||
const dispatching = await this.commands.transitionRun({
|
||||
runId: reference.run.id,
|
||||
to: 'dispatching',
|
||||
expectedVersion: reference.run.version,
|
||||
atMs: startingAtMs,
|
||||
actor: { type: 'scheduler' },
|
||||
});
|
||||
const starting = await this.commands.transitionRunAttempt({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
to: 'starting',
|
||||
expectedRunVersion: dispatching.run.version,
|
||||
atMs: startingAtMs,
|
||||
...(deadlineAtMs === undefined ? {} : { deadlineAtMs }),
|
||||
callbackTokenHash,
|
||||
...(logArtifactId === undefined ? {} : { logArtifactId }),
|
||||
actor: { type: 'worker', id: this.executor.type },
|
||||
});
|
||||
return { run: starting.run, attempt: starting.attempt };
|
||||
}
|
||||
|
||||
private assertLogArtifactId(value: string | undefined): void {
|
||||
if (value === undefined) return;
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > MAX_LOG_ARTIFACT_ID_LENGTH ||
|
||||
!/^[A-Za-z0-9._:-]+$/.test(value)
|
||||
) {
|
||||
throw new TypeError('Primary Run logArtifactId is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
private async recordRunning(
|
||||
reference: PrimaryRunReference,
|
||||
handle: ExecutionHandle,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const runningAttempt = await this.commands.transitionRunAttempt({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
to: 'running',
|
||||
expectedRunVersion: reference.run.version,
|
||||
atMs: this.atOrAfter(
|
||||
reference.run.createdAtMs,
|
||||
reference.attempt.createdAtMs,
|
||||
handle.startedAtMs,
|
||||
),
|
||||
executorHandle: handle.durableHandle ?? handle.id,
|
||||
...(handle.pid === undefined ? {} : { pid: handle.pid }),
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
const runningRun = await this.commands.transitionRun({
|
||||
runId: reference.run.id,
|
||||
to: 'running',
|
||||
expectedVersion: runningAttempt.run.version,
|
||||
atMs: this.atOrAfter(
|
||||
runningAttempt.run.createdAtMs,
|
||||
runningAttempt.attempt.startedAtMs,
|
||||
),
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
return { run: runningRun.run, attempt: runningAttempt.attempt };
|
||||
}
|
||||
|
||||
private async recordStartFailure(
|
||||
reference: PrimaryRunReference,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const atMs = this.atOrAfter(
|
||||
reference.run.createdAtMs,
|
||||
reference.attempt.createdAtMs,
|
||||
);
|
||||
const failedAttempt = await this.commands.transitionRunAttempt({
|
||||
runId: reference.run.id,
|
||||
attemptId: reference.attempt.id,
|
||||
to: 'failed',
|
||||
expectedRunVersion: reference.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTOR_START_FAILED',
|
||||
errorSummary: 'Executor failed before ownership was established',
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
const failedRun = await this.commands.transitionRun({
|
||||
runId: reference.run.id,
|
||||
to: 'failed',
|
||||
expectedVersion: failedAttempt.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTOR_START_FAILED',
|
||||
errorSummary: 'Executor failed before ownership was established',
|
||||
actor: { type: 'executor', id: this.executor.type },
|
||||
});
|
||||
return { run: failedRun.run, attempt: failedAttempt.attempt };
|
||||
}
|
||||
|
||||
private async compensateActivationFailure(
|
||||
reference: PrimaryRunReference,
|
||||
handle: ExecutionHandle,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.executor.stop(handle, {
|
||||
kind: 'reconcile',
|
||||
requestedAtMs: this.atOrAfter(reference.run.createdAtMs),
|
||||
});
|
||||
} catch {
|
||||
// The durable state below remains lost when process ownership is unknown.
|
||||
}
|
||||
|
||||
try {
|
||||
let latest = await this.loadReference(reference);
|
||||
const atMs = this.atOrAfter(
|
||||
latest.run.createdAtMs,
|
||||
latest.run.startedAtMs,
|
||||
latest.attempt.createdAtMs,
|
||||
latest.attempt.startedAtMs,
|
||||
);
|
||||
if (!isTerminalRunAttemptStatus(latest.attempt.status)) {
|
||||
const attempt = await this.commands.transitionRunAttempt({
|
||||
runId: latest.run.id,
|
||||
attemptId: latest.attempt.id,
|
||||
to: 'lost',
|
||||
expectedRunVersion: latest.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTION_ACTIVATION_PERSISTENCE_FAILED',
|
||||
errorSummary: 'Executor ownership could not be persisted',
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
latest = { run: attempt.run, attempt: attempt.attempt };
|
||||
}
|
||||
if (!isTerminalRunStatus(latest.run.status)) {
|
||||
await this.commands.transitionRun({
|
||||
runId: latest.run.id,
|
||||
to: 'lost',
|
||||
expectedVersion: latest.run.version,
|
||||
atMs,
|
||||
errorCode: 'EXECUTION_ACTIVATION_PERSISTENCE_FAILED',
|
||||
errorSummary: 'Executor ownership could not be persisted',
|
||||
actor: { type: 'reconciler' },
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Original persistence failure is reported to the caller.
|
||||
}
|
||||
}
|
||||
|
||||
private async loadReference(
|
||||
fallback: PrimaryRunReference,
|
||||
): Promise<PrimaryRunReference> {
|
||||
const [run, attempt] = await Promise.all([
|
||||
this.repository.findRunById(fallback.run.id),
|
||||
this.repository.findAttemptById(fallback.attempt.id),
|
||||
]);
|
||||
return {
|
||||
run: run ?? fallback.run,
|
||||
attempt: attempt ?? fallback.attempt,
|
||||
};
|
||||
}
|
||||
|
||||
private assertSpecMatches(
|
||||
reference: PrimaryRunReference,
|
||||
spec: ExecutionSpec,
|
||||
timeoutMs?: number,
|
||||
): void {
|
||||
if (
|
||||
spec.runId !== reference.run.id ||
|
||||
spec.attemptId !== reference.attempt.id ||
|
||||
spec.projectId !== reference.run.projectId ||
|
||||
spec.taskId !== reference.run.taskId ||
|
||||
spec.taskRevision !== reference.run.taskRevision ||
|
||||
spec.timeoutMs !== timeoutMs
|
||||
) {
|
||||
throw new Error('ExecutionSpec does not match its persisted Primary Run');
|
||||
}
|
||||
}
|
||||
|
||||
private assertTimeout(timeoutMs: number | undefined): void {
|
||||
if (
|
||||
timeoutMs !== undefined &&
|
||||
(!Number.isSafeInteger(timeoutMs) || timeoutMs < 1)
|
||||
) {
|
||||
throw new RangeError('Primary Run timeoutMs must be a positive integer');
|
||||
}
|
||||
}
|
||||
|
||||
private assertHandleMatches(
|
||||
reference: PrimaryRunReference,
|
||||
handle: ExecutionHandle,
|
||||
): void {
|
||||
if (
|
||||
handle.runId !== reference.run.id ||
|
||||
handle.attemptId !== reference.attempt.id ||
|
||||
handle.executorType !== this.executor.type
|
||||
) {
|
||||
throw new Error(
|
||||
'Executor handle does not match its persisted Primary Run',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private atOrAfter(...timestamps: Array<number | undefined>): number {
|
||||
return Math.max(
|
||||
this.clock.now(),
|
||||
...timestamps.filter((value): value is number => value !== undefined),
|
||||
);
|
||||
}
|
||||
|
||||
private cancellationActor(reason: ExecutionStopReason): {
|
||||
type: 'user' | 'reconciler' | 'system';
|
||||
} {
|
||||
if (reason.kind === 'user') return { type: 'user' };
|
||||
if (reason.kind === 'reconcile') return { type: 'reconciler' };
|
||||
return { type: 'system' };
|
||||
}
|
||||
|
||||
private deleteActive(runId: string, handle: ExecutionHandle): void {
|
||||
if (this.active.get(runId)?.handle === handle) this.active.delete(runId);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user