mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 00:36:13 +08:00
451 lines
14 KiB
TypeScript
451 lines
14 KiB
TypeScript
import type { ExecutionHandle } from './execution';
|
|
import { cloneExecutionSpec } from './executionSpec';
|
|
import {
|
|
assertRunDispatchCandidate,
|
|
type RunDispatchCandidate,
|
|
} from './runDispatchCandidate';
|
|
import {
|
|
assertRunDispatchLeaseRecord,
|
|
type RunDispatchLeaseRecord,
|
|
} from './runDispatchLease';
|
|
import {
|
|
assertRunDispatchOfferId,
|
|
createExecutionSpecDigest,
|
|
createRunDispatchOfferId,
|
|
type ClaimedExecutionOffer,
|
|
} from './runDispatchOffer';
|
|
import {
|
|
MAX_EXECUTOR_HANDLE_LENGTH,
|
|
MAX_LOG_ARTIFACT_ID_LENGTH,
|
|
} from './runStateMachine';
|
|
import { WORKER_COMPLETION_RECEIPT_TOKEN_DIGEST_PATTERN } from './workerExecutionCompletionReceiptAuthentication';
|
|
|
|
export const WORKER_EXECUTION_OFFER_JOURNAL_STATES = [
|
|
'accepted',
|
|
'starting_acknowledged',
|
|
'launching',
|
|
'started',
|
|
'running_acknowledged',
|
|
'start_failed',
|
|
'start_failure_acknowledged',
|
|
'completion_acknowledged',
|
|
'recovery_required',
|
|
] as const;
|
|
|
|
export type WorkerExecutionOfferJournalState =
|
|
(typeof WORKER_EXECUTION_OFFER_JOURNAL_STATES)[number];
|
|
|
|
export const MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES = 192 * 1024;
|
|
export const MAX_WORKER_EXECUTION_OFFER_JOURNAL_ENTRIES = 1024;
|
|
export const MAX_WORKER_EXECUTION_OFFER_JOURNAL_PAGE_SIZE = 64;
|
|
|
|
export interface WorkerExecutionOfferJournalRecord {
|
|
schemaVersion: 1;
|
|
revision: number;
|
|
state: WorkerExecutionOfferJournalState;
|
|
offer: ClaimedExecutionOffer;
|
|
acceptedAtMs: number;
|
|
updatedAtMs: number;
|
|
executorHandle?: string;
|
|
executorStartedAtMs?: number;
|
|
logArtifactId?: string;
|
|
completionReceiptCallbackSequence?: number;
|
|
completionReceiptTokenDigest?: string;
|
|
completionAcknowledgedAtMs?: number;
|
|
recoveryReason?:
|
|
| 'launch_outcome_unknown'
|
|
| 'control_plane_already_running'
|
|
| 'control_plane_terminal'
|
|
| 'lease_lost_local_execution_stopped'
|
|
| 'lease_lost_local_execution_unverified';
|
|
}
|
|
|
|
export class InvalidWorkerExecutionOfferError extends TypeError {
|
|
constructor(message: string) {
|
|
super(`Worker execution offer is invalid: ${message}`);
|
|
this.name = 'InvalidWorkerExecutionOfferError';
|
|
}
|
|
}
|
|
|
|
export class WorkerExecutionOfferConflictError extends Error {
|
|
constructor(readonly offerId: string) {
|
|
super(`Worker execution offer ${offerId} conflicts with durable state`);
|
|
this.name = 'WorkerExecutionOfferConflictError';
|
|
}
|
|
}
|
|
|
|
function invalid(message: string): never {
|
|
throw new InvalidWorkerExecutionOfferError(message);
|
|
}
|
|
|
|
function safeInteger(value: unknown, name: string): number {
|
|
if (!Number.isSafeInteger(value) || (value as number) < 0) {
|
|
invalid(`${name} must be a non-negative safe integer`);
|
|
}
|
|
return value as number;
|
|
}
|
|
|
|
function boundedString(value: unknown, name: string, maximum: number): string {
|
|
if (
|
|
typeof value !== 'string' ||
|
|
value.length < 1 ||
|
|
value.length > maximum ||
|
|
/[\u0000-\u001f\u007f]/.test(value)
|
|
) {
|
|
invalid(`${name} is invalid`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function sameCandidate(
|
|
left: RunDispatchCandidate,
|
|
right: RunDispatchCandidate,
|
|
): boolean {
|
|
return (
|
|
left.runId === right.runId &&
|
|
left.attemptId === right.attemptId &&
|
|
left.projectId === right.projectId &&
|
|
left.taskId === right.taskId &&
|
|
left.taskRevision === right.taskRevision &&
|
|
left.executorType === right.executorType &&
|
|
left.priority === right.priority &&
|
|
left.queuedAtMs === right.queuedAtMs &&
|
|
left.attemptCreatedAtMs === right.attemptCreatedAtMs
|
|
);
|
|
}
|
|
|
|
function sameLeaseAuthority(
|
|
left: RunDispatchLeaseRecord,
|
|
right: RunDispatchLeaseRecord,
|
|
): boolean {
|
|
return (
|
|
left.attemptId === right.attemptId &&
|
|
left.runId === right.runId &&
|
|
left.workerId === right.workerId &&
|
|
left.workerSessionId === right.workerSessionId &&
|
|
left.workerGeneration === right.workerGeneration &&
|
|
left.leaseGeneration === right.leaseGeneration &&
|
|
left.leaseToken === right.leaseToken
|
|
);
|
|
}
|
|
|
|
export function assertClaimedExecutionOffer(
|
|
offer: ClaimedExecutionOffer,
|
|
): void {
|
|
if (!offer || typeof offer !== 'object' || Array.isArray(offer)) {
|
|
invalid('offer must be an object');
|
|
}
|
|
assertRunDispatchOfferId(offer.offerId);
|
|
assertRunDispatchCandidate(offer.candidate);
|
|
assertRunDispatchLeaseRecord(offer.lease);
|
|
if (offer.lease.status !== 'leased') invalid('lease must be active');
|
|
cloneExecutionSpec(offer.executionSpec);
|
|
if (
|
|
offer.deliveryKind !== 'new_claim' &&
|
|
offer.deliveryKind !== 'lease_recovery'
|
|
) {
|
|
invalid('deliveryKind is invalid');
|
|
}
|
|
if (
|
|
!offer.worker ||
|
|
typeof offer.worker !== 'object' ||
|
|
offer.worker.id !== offer.lease.workerId ||
|
|
offer.worker.sessionId !== offer.lease.workerSessionId ||
|
|
offer.worker.generation !== offer.lease.workerGeneration
|
|
) {
|
|
invalid('Worker target does not match the lease fence');
|
|
}
|
|
if (
|
|
offer.candidate.runId !== offer.lease.runId ||
|
|
offer.candidate.attemptId !== offer.lease.attemptId ||
|
|
offer.executionSpec.runId !== offer.candidate.runId ||
|
|
offer.executionSpec.attemptId !== offer.candidate.attemptId ||
|
|
offer.executionSpec.projectId !== offer.candidate.projectId ||
|
|
offer.executionSpec.taskId !== offer.candidate.taskId ||
|
|
offer.executionSpec.taskRevision !== offer.candidate.taskRevision
|
|
) {
|
|
invalid('Run, Attempt, Project, Task or revision identity drifted');
|
|
}
|
|
if (createRunDispatchOfferId(offer.lease) !== offer.offerId) {
|
|
invalid('offerId does not match the lease authority');
|
|
}
|
|
if (
|
|
createExecutionSpecDigest(offer.executionSpec) !== offer.executionSpecDigest
|
|
) {
|
|
invalid('ExecutionSpec digest does not match the payload');
|
|
}
|
|
if (
|
|
offer.placementScore !== undefined &&
|
|
(!Number.isFinite(offer.placementScore) || offer.placementScore < 0)
|
|
) {
|
|
invalid('placementScore is invalid');
|
|
}
|
|
}
|
|
|
|
export function cloneClaimedExecutionOffer(
|
|
offer: ClaimedExecutionOffer,
|
|
): ClaimedExecutionOffer {
|
|
assertClaimedExecutionOffer(offer);
|
|
return {
|
|
offerId: offer.offerId,
|
|
executionSpecDigest: offer.executionSpecDigest,
|
|
deliveryKind: offer.deliveryKind,
|
|
candidate: { ...offer.candidate },
|
|
worker: { ...offer.worker },
|
|
lease: { ...offer.lease },
|
|
executionSpec: cloneExecutionSpec(offer.executionSpec),
|
|
...(offer.placementScore === undefined
|
|
? {}
|
|
: { placementScore: offer.placementScore }),
|
|
};
|
|
}
|
|
|
|
export function assertSameWorkerExecutionOffer(
|
|
persisted: ClaimedExecutionOffer,
|
|
delivered: ClaimedExecutionOffer,
|
|
): void {
|
|
assertClaimedExecutionOffer(persisted);
|
|
assertClaimedExecutionOffer(delivered);
|
|
if (
|
|
persisted.offerId !== delivered.offerId ||
|
|
persisted.executionSpecDigest !== delivered.executionSpecDigest ||
|
|
!sameCandidate(persisted.candidate, delivered.candidate) ||
|
|
!sameLeaseAuthority(persisted.lease, delivered.lease) ||
|
|
persisted.worker.id !== delivered.worker.id ||
|
|
persisted.worker.sessionId !== delivered.worker.sessionId ||
|
|
persisted.worker.generation !== delivered.worker.generation
|
|
) {
|
|
throw new WorkerExecutionOfferConflictError(delivered.offerId);
|
|
}
|
|
}
|
|
|
|
export function mergeWorkerExecutionOffer(
|
|
persisted: ClaimedExecutionOffer,
|
|
delivered: ClaimedExecutionOffer,
|
|
): ClaimedExecutionOffer {
|
|
assertSameWorkerExecutionOffer(persisted, delivered);
|
|
const selectedLease =
|
|
delivered.lease.version > persisted.lease.version
|
|
? delivered.lease
|
|
: persisted.lease;
|
|
return cloneClaimedExecutionOffer({
|
|
...persisted,
|
|
deliveryKind: delivered.deliveryKind,
|
|
lease: selectedLease,
|
|
});
|
|
}
|
|
|
|
export function createWorkerExecutionOfferJournalRecord(
|
|
offer: ClaimedExecutionOffer,
|
|
acceptedAtMs: number,
|
|
): WorkerExecutionOfferJournalRecord {
|
|
safeInteger(acceptedAtMs, 'acceptedAtMs');
|
|
return {
|
|
schemaVersion: 1,
|
|
revision: 0,
|
|
state: 'accepted',
|
|
offer: cloneClaimedExecutionOffer(offer),
|
|
acceptedAtMs,
|
|
updatedAtMs: acceptedAtMs,
|
|
};
|
|
}
|
|
|
|
export function cloneWorkerExecutionOfferJournalRecord(
|
|
record: WorkerExecutionOfferJournalRecord,
|
|
): WorkerExecutionOfferJournalRecord {
|
|
assertWorkerExecutionOfferJournalRecord(record);
|
|
return {
|
|
schemaVersion: 1,
|
|
revision: record.revision,
|
|
state: record.state,
|
|
offer: cloneClaimedExecutionOffer(record.offer),
|
|
acceptedAtMs: record.acceptedAtMs,
|
|
updatedAtMs: record.updatedAtMs,
|
|
...(record.executorHandle === undefined
|
|
? {}
|
|
: { executorHandle: record.executorHandle }),
|
|
...(record.executorStartedAtMs === undefined
|
|
? {}
|
|
: { executorStartedAtMs: record.executorStartedAtMs }),
|
|
...(record.logArtifactId === undefined
|
|
? {}
|
|
: { logArtifactId: record.logArtifactId }),
|
|
...(record.completionReceiptCallbackSequence === undefined
|
|
? {}
|
|
: {
|
|
completionReceiptCallbackSequence:
|
|
record.completionReceiptCallbackSequence,
|
|
}),
|
|
...(record.completionReceiptTokenDigest === undefined
|
|
? {}
|
|
: {
|
|
completionReceiptTokenDigest: record.completionReceiptTokenDigest,
|
|
}),
|
|
...(record.completionAcknowledgedAtMs === undefined
|
|
? {}
|
|
: {
|
|
completionAcknowledgedAtMs: record.completionAcknowledgedAtMs,
|
|
}),
|
|
...(record.recoveryReason === undefined
|
|
? {}
|
|
: { recoveryReason: record.recoveryReason }),
|
|
};
|
|
}
|
|
|
|
export function assertWorkerExecutionOfferJournalRecord(
|
|
record: WorkerExecutionOfferJournalRecord,
|
|
): void {
|
|
if (!record || typeof record !== 'object' || Array.isArray(record)) {
|
|
invalid('journal record must be an object');
|
|
}
|
|
if (record.schemaVersion !== 1) invalid('schemaVersion is unsupported');
|
|
safeInteger(record.revision, 'revision');
|
|
safeInteger(record.acceptedAtMs, 'acceptedAtMs');
|
|
safeInteger(record.updatedAtMs, 'updatedAtMs');
|
|
if (record.updatedAtMs < record.acceptedAtMs) {
|
|
invalid('journal timestamps are inconsistent');
|
|
}
|
|
if (!WORKER_EXECUTION_OFFER_JOURNAL_STATES.includes(record.state)) {
|
|
invalid('journal state is invalid');
|
|
}
|
|
assertClaimedExecutionOffer(record.offer);
|
|
const hasExecutorMetadata =
|
|
record.executorHandle !== undefined ||
|
|
record.executorStartedAtMs !== undefined ||
|
|
record.logArtifactId !== undefined;
|
|
const requiresExecutorMetadata =
|
|
record.state === 'started' || record.state === 'running_acknowledged';
|
|
if (requiresExecutorMetadata || hasExecutorMetadata) {
|
|
boundedString(
|
|
record.executorHandle,
|
|
'executorHandle',
|
|
MAX_EXECUTOR_HANDLE_LENGTH,
|
|
);
|
|
safeInteger(record.executorStartedAtMs, 'executorStartedAtMs');
|
|
if (
|
|
!requiresExecutorMetadata &&
|
|
record.state !== 'recovery_required' &&
|
|
record.state !== 'completion_acknowledged'
|
|
) {
|
|
invalid('executor metadata is not allowed in this journal state');
|
|
}
|
|
}
|
|
if (record.logArtifactId !== undefined) {
|
|
boundedString(
|
|
record.logArtifactId,
|
|
'logArtifactId',
|
|
MAX_LOG_ARTIFACT_ID_LENGTH,
|
|
);
|
|
}
|
|
const hasCompletionCallbackSequence =
|
|
record.completionReceiptCallbackSequence !== undefined;
|
|
const hasCompletionTokenDigest =
|
|
record.completionReceiptTokenDigest !== undefined;
|
|
if (hasCompletionCallbackSequence !== hasCompletionTokenDigest) {
|
|
invalid('completion receipt authentication metadata must be complete');
|
|
}
|
|
if (hasCompletionCallbackSequence) {
|
|
if (
|
|
!Number.isSafeInteger(record.completionReceiptCallbackSequence) ||
|
|
record.completionReceiptCallbackSequence! < 1
|
|
) {
|
|
invalid('completionReceiptCallbackSequence must be positive');
|
|
}
|
|
if (
|
|
typeof record.completionReceiptTokenDigest !== 'string' ||
|
|
!WORKER_COMPLETION_RECEIPT_TOKEN_DIGEST_PATTERN.test(
|
|
record.completionReceiptTokenDigest,
|
|
)
|
|
) {
|
|
invalid('completionReceiptTokenDigest is invalid');
|
|
}
|
|
if (
|
|
record.state === 'accepted' ||
|
|
record.state === 'starting_acknowledged'
|
|
) {
|
|
invalid(
|
|
'completion receipt authentication is not allowed before launching',
|
|
);
|
|
}
|
|
}
|
|
if (record.state === 'completion_acknowledged') {
|
|
if (!hasCompletionCallbackSequence) {
|
|
invalid('completion acknowledgement requires authentication metadata');
|
|
}
|
|
safeInteger(
|
|
record.completionAcknowledgedAtMs,
|
|
'completionAcknowledgedAtMs',
|
|
);
|
|
if (record.completionAcknowledgedAtMs !== record.updatedAtMs) {
|
|
invalid('completion acknowledgement timestamp must match updatedAtMs');
|
|
}
|
|
} else if (record.completionAcknowledgedAtMs !== undefined) {
|
|
invalid(
|
|
'completionAcknowledgedAtMs is only allowed for completion_acknowledged',
|
|
);
|
|
}
|
|
if (record.state === 'recovery_required') {
|
|
if (
|
|
record.recoveryReason !== 'launch_outcome_unknown' &&
|
|
record.recoveryReason !== 'control_plane_already_running' &&
|
|
record.recoveryReason !== 'control_plane_terminal' &&
|
|
record.recoveryReason !== 'lease_lost_local_execution_stopped' &&
|
|
record.recoveryReason !== 'lease_lost_local_execution_unverified'
|
|
) {
|
|
invalid('recoveryReason is invalid');
|
|
}
|
|
} else if (record.recoveryReason !== undefined) {
|
|
invalid('recoveryReason is only allowed for recovery_required');
|
|
}
|
|
}
|
|
|
|
export function workerExecutionHandleMetadata(handle: ExecutionHandle): {
|
|
executorHandle: string;
|
|
executorStartedAtMs: number;
|
|
} {
|
|
const executorHandle = boundedString(
|
|
handle.durableHandle ?? handle.id,
|
|
'executorHandle',
|
|
MAX_EXECUTOR_HANDLE_LENGTH,
|
|
);
|
|
const executorStartedAtMs = safeInteger(handle.startedAtMs, 'startedAtMs');
|
|
return { executorHandle, executorStartedAtMs };
|
|
}
|
|
|
|
export function serializeWorkerExecutionOfferJournalRecord(
|
|
record: WorkerExecutionOfferJournalRecord,
|
|
): string {
|
|
const serialized = JSON.stringify(
|
|
cloneWorkerExecutionOfferJournalRecord(record),
|
|
);
|
|
if (
|
|
Buffer.byteLength(serialized, 'utf8') >
|
|
MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES
|
|
) {
|
|
invalid('journal record exceeds the byte limit');
|
|
}
|
|
return serialized;
|
|
}
|
|
|
|
export function parseWorkerExecutionOfferJournalRecord(
|
|
value: Uint8Array | string,
|
|
): WorkerExecutionOfferJournalRecord {
|
|
const bytes =
|
|
typeof value === 'string' ? Buffer.from(value) : Buffer.from(value);
|
|
if (
|
|
bytes.length < 2 ||
|
|
bytes.length > MAX_WORKER_EXECUTION_OFFER_RECORD_BYTES
|
|
) {
|
|
invalid('serialized journal record size is outside the allowed range');
|
|
}
|
|
let parsed: unknown;
|
|
try {
|
|
parsed = JSON.parse(bytes.toString('utf8'));
|
|
} catch {
|
|
return invalid('serialized journal record is not valid JSON');
|
|
}
|
|
return cloneWorkerExecutionOfferJournalRecord(
|
|
parsed as WorkerExecutionOfferJournalRecord,
|
|
);
|
|
}
|