open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-21 09:58:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): establish 3.0 incubation baseline

Browse Source
This commit is contained in:
whyour
2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-worker-runtime/src/remote-execution/executionContextMaterializer.ts
+304
View File
@@ -0,0 +1,304 @@
// Remote Execution owns bounded Secret and Artifact context materialization.
import {
MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES,
MAX_LOCAL_DISPATCH_SECRET_REFS,
} from '@qinglong/runtime-core/local-dispatch';
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { assertRunDispatchId } from '@qinglong/runtime-core/run-dispatch-lease';
import type {
MaterializedWorkerRemoteExecutionContext,
WorkerRemoteExecutionContextMaterializer,
WorkerRemoteExecutionOutputSink,
} from './executionInboxProcessor';
export interface WorkerRemoteSecretResolution {
readonly values: readonly Readonly<{
secretRef: string;
value: string;
}>[];
readonly dispose?: () => Promise<void> | void;
}
export interface WorkerRemoteSecretEnvironmentProvider {
resolve(request: Readonly<{
projectId: string;
taskId: string;
taskRevision: string;
runId: string;
attemptId: string;
offerId: string;
executionDigest: string;
secretRefs: readonly string[];
}>): Promise<WorkerRemoteSecretResolution | undefined>;
}
export interface WorkerRemoteLogArtifactPreparation {
readonly logArtifactId: string;
/** Transfers the prepared writer once; release must not close it afterwards. */
readonly takeOutput: () => WorkerRemoteExecutionOutputSink;
/** Releases only preparation resources; it must not delete a handed-off log. */
readonly release: () => Promise<void> | void;
}
export interface WorkerRemoteLogArtifactAllocator {
prepare(request: Readonly<{
projectId: string;
runId: string;
attemptId: string;
offerId: string;
}>): Promise<WorkerRemoteLogArtifactPreparation | undefined>;
}
export interface BoundedWorkerRemoteExecutionContextMaterializerOptions {
readonly artifacts: WorkerRemoteLogArtifactAllocator;
readonly secrets?: WorkerRemoteSecretEnvironmentProvider;
}
export class WorkerRemoteExecutionMaterializationError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'secret_unavailable'
| 'secret_response_invalid'
| 'environment_budget_exceeded'
| 'artifact_unavailable'
| 'artifact_response_invalid',
) {
super(`Worker remote execution materialization failed: ${reason}`);
this.name = 'WorkerRemoteExecutionMaterializationError';
}
}
function environmentValue(value: unknown): string {
if (
typeof value !== 'string' ||
value.includes('\0') ||
Buffer.byteLength(value, 'utf8') > 16 * 1024
) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
return value;
}
async function disposeQuietly(
operation: (() => Promise<void> | void) | undefined,
): Promise<void> {
await Promise.resolve().then(() => operation?.()).catch(() => undefined);
}
export class BoundedWorkerRemoteExecutionContextMaterializer
implements WorkerRemoteExecutionContextMaterializer {
private readonly artifacts: WorkerRemoteLogArtifactAllocator;
private readonly secrets?: WorkerRemoteSecretEnvironmentProvider;
constructor(options: BoundedWorkerRemoteExecutionContextMaterializerOptions) {
if (
!options ||
typeof options.artifacts?.prepare !== 'function' ||
(options.secrets !== undefined &&
typeof options.secrets.resolve !== 'function')
) {
throw new WorkerRemoteExecutionMaterializationError(
'invalid_configuration',
);
}
this.artifacts = options.artifacts;
this.secrets = options.secrets;
}
async prepare(input: Readonly<{
offer: ClusterRemoteExecutionOffer;
}>): Promise<MaterializedWorkerRemoteExecutionContext> {
let offer: ClusterRemoteExecutionOffer;
try {
offer = createClusterRemoteExecutionOffer(input?.offer);
} catch {
throw new WorkerRemoteExecutionMaterializationError(
'invalid_configuration',
);
}
const bindings = offer.executionRevision.environment;
const secretRefs = Object.freeze([
...new Set(bindings.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [])),
]);
if (secretRefs.length > MAX_LOCAL_DISPATCH_SECRET_REFS) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
);
}
let secretResolution: WorkerRemoteSecretResolution | undefined;
const secretByRef = new Map<string, string>();
if (secretRefs.length > 0) {
if (!this.secrets) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
);
}
try {
secretResolution = await this.secrets.resolve(Object.freeze({
projectId: offer.candidate.projectId,
taskId: offer.candidate.taskId,
taskRevision: offer.candidate.taskRevision,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs,
}));
} catch {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
);
}
if (!secretResolution) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
);
}
if (
Object.keys(secretResolution).some((key) =>
key !== 'values' && key !== 'dispose') ||
!Array.isArray(secretResolution.values) ||
secretResolution.values.length !== secretRefs.length ||
(secretResolution.dispose !== undefined &&
typeof secretResolution.dispose !== 'function')
) {
await disposeQuietly(secretResolution.dispose);
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
try {
for (const entry of secretResolution.values) {
if (
!entry ||
typeof entry !== 'object' ||
Object.keys(entry).length !== 2 ||
!Object.hasOwn(entry, 'secretRef') ||
!Object.hasOwn(entry, 'value') ||
typeof entry.secretRef !== 'string' ||
!secretRefs.includes(entry.secretRef) ||
secretByRef.has(entry.secretRef)
) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
secretByRef.set(entry.secretRef, environmentValue(entry.value));
}
} catch (error) {
await disposeQuietly(secretResolution.dispose);
throw error;
}
}
let environmentBytes = 0;
let environment: MaterializedWorkerRemoteExecutionContext['environment'];
try {
environment = Object.freeze(bindings.map((binding) => {
const value = binding.kind === 'public'
? binding.value
: secretByRef.get(binding.secretRef);
if (value === undefined) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
environmentBytes += Buffer.byteLength(binding.name, 'utf8') +
Buffer.byteLength(value, 'utf8');
if (environmentBytes > MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
);
}
return Object.freeze({ name: binding.name, value });
}));
} catch (error) {
await disposeQuietly(secretResolution?.dispose);
throw error;
}
let artifact: WorkerRemoteLogArtifactPreparation | undefined;
try {
artifact = await this.artifacts.prepare(Object.freeze({
projectId: offer.candidate.projectId,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
}));
} catch {
await disposeQuietly(secretResolution?.dispose);
throw new WorkerRemoteExecutionMaterializationError(
'artifact_unavailable',
);
}
if (!artifact) {
await disposeQuietly(secretResolution?.dispose);
throw new WorkerRemoteExecutionMaterializationError(
'artifact_unavailable',
);
}
try {
if (
Object.keys(artifact).length !== 3 ||
!Object.hasOwn(artifact, 'logArtifactId') ||
!Object.hasOwn(artifact, 'takeOutput') ||
!Object.hasOwn(artifact, 'release')
) {
throw new Error('invalid artifact preparation');
}
assertRunDispatchId('logArtifactId', artifact.logArtifactId);
if (
artifact.logArtifactId.length > 36 ||
typeof artifact.takeOutput !== 'function' ||
typeof artifact.release !== 'function'
) {
throw new Error('invalid artifact preparation');
}
} catch {
await disposeQuietly(artifact.release);
await disposeQuietly(secretResolution?.dispose);
throw new WorkerRemoteExecutionMaterializationError(
'artifact_response_invalid',
);
}
let disposed = false;
let outputTaken = false;
return Object.freeze({
environment,
logArtifactId: artifact.logArtifactId,
takeOutput() {
if (disposed || outputTaken) {
throw new WorkerRemoteExecutionMaterializationError(
'artifact_response_invalid',
);
}
const output = artifact!.takeOutput();
if (
!output ||
typeof output !== 'object' ||
output.logArtifactId !== artifact!.logArtifactId ||
typeof output.write !== 'function' ||
typeof output.close !== 'function'
) {
void Promise.resolve(output?.close?.()).catch(() => undefined);
throw new WorkerRemoteExecutionMaterializationError(
'artifact_response_invalid',
);
}
outputTaken = true;
return output;
},
async dispose() {
if (disposed) return;
disposed = true;
await Promise.all([
disposeQuietly(artifact!.release),
disposeQuietly(secretResolution?.dispose),
]);
},
});
}
}
packages/ql3-worker-runtime/src/remote-execution/executionInbox.ts
+379
View File
@@ -0,0 +1,379 @@
// Remote Execution owns the durable offer inbox authority and transition contract.
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
export const WORKER_REMOTE_EXECUTION_INBOX_STATES = [
'accepted',
'starting_acknowledged',
'launching',
'started',
'running_acknowledged',
'start_failed',
'start_failure_acknowledged',
'completion_acknowledged',
'recovery_required',
] as const;
export type WorkerRemoteExecutionInboxState =
(typeof WORKER_REMOTE_EXECUTION_INBOX_STATES)[number];
export type WorkerRemoteExecutionRecoveryReason =
| 'launch_outcome_unknown'
| 'control_plane_already_running'
| 'control_plane_terminal'
| 'lease_lost_local_execution_stopped'
| 'lease_lost_local_execution_unverified';
export interface WorkerRemoteExecutionInboxRecord {
readonly schemaVersion: 1;
readonly revision: number;
readonly state: WorkerRemoteExecutionInboxState;
readonly offer: ClusterRemoteExecutionOffer;
readonly acceptedAtMs: number;
readonly updatedAtMs: number;
readonly executorHandle?: string;
readonly executorStartedAtMs?: number;
readonly logArtifactId?: string;
readonly completionReceiptCallbackSequence?: number;
readonly completionReceiptTokenDigest?: string;
readonly completionAcknowledgedAtMs?: number;
readonly recoveryReason?: WorkerRemoteExecutionRecoveryReason;
}
export interface WorkerRemoteExecutionInboxPage {
readonly records: readonly WorkerRemoteExecutionInboxRecord[];
readonly nextAfterOfferId?: string;
}
export interface WorkerRemoteExecutionInbox {
readOffer(offerId: string): Promise<WorkerRemoteExecutionInboxRecord | undefined>;
replaceOffer(
record: WorkerRemoteExecutionInboxRecord,
expectedRevision: number,
): Promise<void>;
listOffers(options?: Readonly<{
afterOfferId?: string;
limit?: number;
}>): Promise<WorkerRemoteExecutionInboxPage>;
}
export class WorkerRemoteExecutionInboxError extends Error {
constructor(
readonly reason:
| 'invalid_record'
| 'authority_conflict'
| 'revision_conflict'
| 'invalid_transition',
) {
super(`Worker remote execution inbox failed: ${reason}`);
this.name = 'WorkerRemoteExecutionInboxError';
}
}
const STATES = new Set<string>(WORKER_REMOTE_EXECUTION_INBOX_STATES);
const SHA256 = /^[a-f0-9]{64}$/;
const RECOVERY_REASONS = new Set<string>([
'launch_outcome_unknown',
'control_plane_already_running',
'control_plane_terminal',
'lease_lost_local_execution_stopped',
'lease_lost_local_execution_unverified',
]);
const OPTIONAL_FIELDS = [
'executorHandle',
'executorStartedAtMs',
'logArtifactId',
'completionReceiptCallbackSequence',
'completionReceiptTokenDigest',
'completionAcknowledgedAtMs',
'recoveryReason',
] as const;
const BASE_FIELDS = [
'schemaVersion',
'revision',
'state',
'offer',
'acceptedAtMs',
'updatedAtMs',
] as const;
function invalid(): never {
throw new WorkerRemoteExecutionInboxError('invalid_record');
}
function safeInteger(value: unknown): number {
if (!Number.isSafeInteger(value) || (value as number) < 0) invalid();
return value as number;
}
function boundedText(value: unknown, maximum: number): string {
if (
typeof value !== 'string' ||
value.length < 1 ||
Buffer.byteLength(value, 'utf8') > maximum ||
/[\u0000-\u001f\u007f]/.test(value)
) {
invalid();
}
return value;
}
function sameOfferAuthority(
left: ClusterRemoteExecutionOffer,
right: ClusterRemoteExecutionOffer,
): boolean {
const first = createClusterRemoteExecutionOffer(left);
const second = createClusterRemoteExecutionOffer(right);
return (
first.offerId === second.offerId &&
first.executionDigest === second.executionDigest &&
first.deliveryKind === second.deliveryKind &&
JSON.stringify(first.candidate) === JSON.stringify(second.candidate) &&
JSON.stringify(first.worker) === JSON.stringify(second.worker) &&
first.lease.runId === second.lease.runId &&
first.lease.attemptId === second.lease.attemptId &&
first.lease.workerId === second.lease.workerId &&
first.lease.workerSessionId === second.lease.workerSessionId &&
first.lease.workerGeneration === second.lease.workerGeneration &&
first.lease.leaseGeneration === second.lease.leaseGeneration &&
first.lease.leaseTokenDigest === second.lease.leaseTokenDigest &&
first.leaseToken === second.leaseToken &&
JSON.stringify(first.executionRevision) ===
JSON.stringify(second.executionRevision)
);
}
function exactOptional<T extends keyof WorkerRemoteExecutionInboxRecord>(
value: WorkerRemoteExecutionInboxRecord,
key: T,
): WorkerRemoteExecutionInboxRecord[T] | undefined {
return Object.prototype.hasOwnProperty.call(value, key) ? value[key] : undefined;
}
export function normalizeWorkerRemoteExecutionInboxRecord(
value: WorkerRemoteExecutionInboxRecord,
): WorkerRemoteExecutionInboxRecord {
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
const allowed = new Set<string>([...BASE_FIELDS, ...OPTIONAL_FIELDS]);
const keys = Object.keys(value);
if (
value.schemaVersion !== 1 ||
BASE_FIELDS.some((key) => !keys.includes(key)) ||
keys.some((key) => !allowed.has(key)) ||
!STATES.has(value.state)
) {
invalid();
}
const revision = safeInteger(value.revision);
const acceptedAtMs = safeInteger(value.acceptedAtMs);
const updatedAtMs = safeInteger(value.updatedAtMs);
if (updatedAtMs < acceptedAtMs) invalid();
const offer = createClusterRemoteExecutionOffer(value.offer);
const executorHandle = exactOptional(value, 'executorHandle');
const executorStartedAtMs = exactOptional(value, 'executorStartedAtMs');
const logArtifactId = exactOptional(value, 'logArtifactId');
const callbackSequence = exactOptional(
value,
'completionReceiptCallbackSequence',
);
const tokenDigest = exactOptional(value, 'completionReceiptTokenDigest');
const completionAcknowledgedAtMs = exactOptional(
value,
'completionAcknowledgedAtMs',
);
const recoveryReason = exactOptional(value, 'recoveryReason');
if (executorHandle !== undefined && executorStartedAtMs === undefined) invalid();
if (executorHandle !== undefined) boundedText(executorHandle, 512);
if (executorStartedAtMs !== undefined) safeInteger(executorStartedAtMs);
if (logArtifactId !== undefined) boundedText(logArtifactId, 36);
if ((callbackSequence === undefined) !== (tokenDigest === undefined)) invalid();
if (
callbackSequence !== undefined &&
(safeInteger(callbackSequence) < 1 || callbackSequence > 2_147_483_647)
) {
invalid();
}
if (tokenDigest !== undefined && !SHA256.test(tokenDigest)) invalid();
if (completionAcknowledgedAtMs !== undefined) {
safeInteger(completionAcknowledgedAtMs);
}
if (
recoveryReason !== undefined &&
!RECOVERY_REASONS.has(recoveryReason)
) {
invalid();
}
const hasExecutor = executorHandle !== undefined;
const hasExecutorStartedAt = executorStartedAtMs !== undefined;
const hasLogArtifact = logArtifactId !== undefined;
const hasReceiptAuthentication = callbackSequence !== undefined;
const executorRequired = [
'started',
'running_acknowledged',
].includes(value.state);
const executorStartedAtRequired = [
'launching',
'started',
'running_acknowledged',
'completion_acknowledged',
].includes(value.state);
const executorStartedAtOptional = [
'start_failed',
'start_failure_acknowledged',
'recovery_required',
].includes(value.state);
const receiptAuthenticationRequired = [
'launching',
'started',
'running_acknowledged',
'completion_acknowledged',
].includes(value.state);
const receiptAuthenticationOptional = [
'start_failed',
'start_failure_acknowledged',
'recovery_required',
].includes(value.state);
const logArtifactRequired = [
'launching',
'started',
'running_acknowledged',
'completion_acknowledged',
].includes(value.state);
if (
(!['completion_acknowledged', 'recovery_required'].includes(value.state) &&
executorRequired !== hasExecutor) ||
(!executorStartedAtOptional &&
executorStartedAtRequired !== hasExecutorStartedAt) ||
(hasExecutor &&
!['started', 'running_acknowledged', 'completion_acknowledged', 'recovery_required']
.includes(value.state)) ||
(hasExecutorStartedAt &&
!['launching', 'started', 'running_acknowledged', 'start_failed',
'start_failure_acknowledged', 'completion_acknowledged',
'recovery_required'].includes(value.state)) ||
(!receiptAuthenticationOptional &&
receiptAuthenticationRequired !== hasReceiptAuthentication) ||
(hasReceiptAuthentication &&
!['launching', 'started', 'running_acknowledged', 'start_failed',
'start_failure_acknowledged', 'completion_acknowledged',
'recovery_required'].includes(value.state)) ||
(logArtifactRequired && !hasLogArtifact) ||
(hasLogArtifact && ['accepted', 'starting_acknowledged'].includes(value.state))
) {
invalid();
}
if (
completionAcknowledgedAtMs !== undefined !==
(value.state === 'completion_acknowledged') ||
recoveryReason !== undefined !== (value.state === 'recovery_required')
) {
invalid();
}
return Object.freeze({
schemaVersion: 1,
revision,
state: value.state,
offer,
acceptedAtMs,
updatedAtMs,
...(executorHandle === undefined ? {} : { executorHandle }),
...(executorStartedAtMs === undefined ? {} : { executorStartedAtMs }),
...(logArtifactId === undefined ? {} : { logArtifactId }),
...(callbackSequence === undefined
? {}
: { completionReceiptCallbackSequence: callbackSequence }),
...(tokenDigest === undefined
? {}
: { completionReceiptTokenDigest: tokenDigest }),
...(completionAcknowledgedAtMs === undefined
? {}
: { completionAcknowledgedAtMs }),
...(recoveryReason === undefined ? {} : { recoveryReason }),
});
}
export function createWorkerRemoteExecutionInboxRecord(
offer: ClusterRemoteExecutionOffer,
acceptedAtMs: number,
): WorkerRemoteExecutionInboxRecord {
return normalizeWorkerRemoteExecutionInboxRecord({
schemaVersion: 1,
revision: 0,
state: 'accepted',
offer,
acceptedAtMs,
updatedAtMs: acceptedAtMs,
});
}
function transitions(
...states: WorkerRemoteExecutionInboxState[]
): ReadonlySet<WorkerRemoteExecutionInboxState> {
return new Set(states);
}
const TRANSITIONS: Readonly<Record<
WorkerRemoteExecutionInboxState,
ReadonlySet<WorkerRemoteExecutionInboxState>
>> = Object.freeze({
accepted: transitions('accepted', 'starting_acknowledged', 'recovery_required'),
starting_acknowledged: transitions(
'starting_acknowledged', 'launching', 'start_failed', 'recovery_required',
),
launching: transitions(
'launching', 'started', 'start_failed', 'completion_acknowledged',
'recovery_required',
),
started: transitions(
'started', 'running_acknowledged', 'completion_acknowledged',
'recovery_required',
),
running_acknowledged: transitions(
'running_acknowledged', 'completion_acknowledged', 'recovery_required',
),
start_failed: transitions(
'start_failed', 'start_failure_acknowledged', 'recovery_required',
),
start_failure_acknowledged: transitions('start_failure_acknowledged'),
completion_acknowledged: transitions('completion_acknowledged'),
recovery_required: transitions('recovery_required', 'completion_acknowledged'),
});
export function assertWorkerRemoteExecutionInboxTransition(
previousValue: WorkerRemoteExecutionInboxRecord,
nextValue: WorkerRemoteExecutionInboxRecord,
): void {
const previous = normalizeWorkerRemoteExecutionInboxRecord(previousValue);
const next = normalizeWorkerRemoteExecutionInboxRecord(nextValue);
if (!sameOfferAuthority(previous.offer, next.offer)) {
throw new WorkerRemoteExecutionInboxError('authority_conflict');
}
if (
next.revision !== previous.revision + 1 ||
next.acceptedAtMs !== previous.acceptedAtMs ||
next.updatedAtMs < previous.updatedAtMs ||
next.offer.lease.version < previous.offer.lease.version ||
(next.offer.lease.version === previous.offer.lease.version &&
JSON.stringify(next.offer.lease) !== JSON.stringify(previous.offer.lease))
) {
throw new WorkerRemoteExecutionInboxError('revision_conflict');
}
if (!TRANSITIONS[previous.state].has(next.state)) {
throw new WorkerRemoteExecutionInboxError('invalid_transition');
}
for (const key of OPTIONAL_FIELDS) {
const before = exactOptional(previous, key);
const after = exactOptional(next, key);
if (
key === 'recoveryReason' &&
next.state === 'completion_acknowledged'
) continue;
if (before !== undefined && before !== after) {
throw new WorkerRemoteExecutionInboxError('invalid_transition');
}
}
}
packages/ql3-worker-runtime/src/remote-execution/executionInboxProcessor.ts
+649
View File
@@ -0,0 +1,649 @@
// Remote Execution owns offer activation, launch barriers, and recovery processing.
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import type {
AcknowledgeRemoteRunRunningCommand,
AcknowledgeRemoteRunStartingCommand,
FailRemoteRunStartCommand,
RemoteRunActivationResult,
} from '@qinglong/runtime-core/remote-activation';
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import {
normalizeWorkerRemoteExecutionInboxRecord,
type WorkerRemoteExecutionInbox,
type WorkerRemoteExecutionInboxRecord,
type WorkerRemoteExecutionRecoveryReason,
} from './executionInbox';
export interface WorkerRemoteExecutionSession {
readonly workerId: string;
readonly sessionId: string;
readonly generation: number;
readonly status: 'available' | 'draining' | 'offline';
readonly leaseExpiresAtMs: number;
}
export interface WorkerRemoteExecutionActivationClient {
acknowledgeStarting(
command: AcknowledgeRemoteRunStartingCommand,
): Promise<Readonly<RemoteRunActivationResult>>;
acknowledgeRunning(
command: AcknowledgeRemoteRunRunningCommand,
): Promise<Readonly<RemoteRunActivationResult>>;
failStart(
command: FailRemoteRunStartCommand,
): Promise<Readonly<RemoteRunActivationResult>>;
}
export interface WorkerRemoteExecutionCompletionCallback {
readonly sequence: number;
/** Ephemeral capability. Implementations must not persist or log it. */
readonly token: Uint8Array;
}
export type WorkerRemoteExecutionOutputStream = 'stdout' | 'stderr';
export interface WorkerRemoteExecutionOutputChunk {
readonly stream: WorkerRemoteExecutionOutputStream;
readonly chunk: Uint8Array;
readonly observedAtMs: number;
}
export interface WorkerRemoteExecutionOutputSink {
readonly logArtifactId: string;
write(output: WorkerRemoteExecutionOutputChunk): Promise<void>;
/** Flushes accepted bytes and releases the writer. Must be idempotent. */
close(): Promise<void>;
}
export interface MaterializedWorkerRemoteExecutionContext {
readonly environment: readonly Readonly<{
name: string;
value: string;
}>[];
readonly logArtifactId: string;
/** Transfers the prepared writer exactly once after the durable spawn barrier. */
readonly takeOutput: () => WorkerRemoteExecutionOutputSink;
readonly dispose?: () => Promise<void>;
}
export interface WorkerRemoteExecutionContextMaterializer {
prepare(input: Readonly<{
offer: ClusterRemoteExecutionOffer;
completionCallback: WorkerRemoteExecutionCompletionCallback;
}>): Promise<MaterializedWorkerRemoteExecutionContext>;
}
export interface WorkerRemoteExecutionLaunch {
readonly offerId: string;
readonly runId: string;
readonly attemptId: string;
/** Durable pre-spawn timestamp from the launching inbox barrier. */
readonly executorStartedAtMs: number;
readonly command: ClusterRemoteExecutionOffer['executionRevision']['command'];
readonly environment: MaterializedWorkerRemoteExecutionContext['environment'];
readonly workingDirectory?: string;
readonly timeoutMs?: number;
/** Durable database-clock timeout authority returned by starting ACK. */
readonly executionDeadlineAtMs?: number;
readonly logArtifactId: string;
/**
* Ownership transfers to the Executor when start() is called. The Executor
* must close it on every known terminal path, including explicit rejection.
*/
readonly output: WorkerRemoteExecutionOutputSink;
readonly completionCallback: WorkerRemoteExecutionCompletionCallback;
}
export interface WorkerRemoteExecutionExecutor {
/** A rejected result proves no execution started; a thrown error is unknown. */
start(launch: WorkerRemoteExecutionLaunch): Promise<
| Readonly<{
status: 'started';
executorHandle: string;
executorStartedAtMs: number;
}>
| Readonly<{ status: 'rejected' }>
>;
}
export type WorkerRemoteExecutionProcessResult = Readonly<{
status:
| 'running'
| 'already_running'
| 'start_failed'
| 'already_failed'
| 'already_completed'
| 'recovery_required';
offerId: string;
executorHandle?: string;
recoveryReason?: WorkerRemoteExecutionRecoveryReason;
}>;
export interface WorkerRemoteExecutionInboxProcessorOptions {
readonly inbox: WorkerRemoteExecutionInbox;
readonly activation: WorkerRemoteExecutionActivationClient;
readonly materializer: WorkerRemoteExecutionContextMaterializer;
readonly executor: WorkerRemoteExecutionExecutor;
readonly currentSession: () => WorkerRemoteExecutionSession | undefined;
readonly now?: () => number;
readonly randomCapability?: () => Uint8Array;
readonly eventId?: () => string;
}
export class WorkerRemoteExecutionProcessorError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'offer_missing'
| 'target_fenced'
| 'offer_expired'
| 'activation_response_invalid'
| 'materialized_context_invalid',
) {
super(`Worker remote execution processor failed: ${reason}`);
this.name = 'WorkerRemoteExecutionProcessorError';
}
}
const SHA256 = /^[a-f0-9]{64}$/;
function safeTime(value: number): number {
if (!Number.isSafeInteger(value) || value < 0) {
throw new WorkerRemoteExecutionProcessorError('invalid_configuration');
}
return value;
}
function boundedText(value: unknown, maximum: number): string {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
return value;
}
function validateEnvironment(
offer: ClusterRemoteExecutionOffer,
context: MaterializedWorkerRemoteExecutionContext,
): MaterializedWorkerRemoteExecutionContext['environment'] {
if (!context || typeof context !== 'object' || !Array.isArray(context.environment)) {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
const expected = offer.executionRevision.environment;
if (context.environment.length !== expected.length) {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
const values = new Map<string, string>();
for (const entry of context.environment) {
if (!entry || typeof entry !== 'object') {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
const name = boundedText(entry.name, 255);
if (
name.includes('=') ||
typeof entry.value !== 'string' ||
entry.value.includes('\0') ||
values.has(name)
) {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
values.set(name, entry.value);
}
const normalized = expected.map((binding) => {
const value = values.get(binding.name);
if (
value === undefined ||
(binding.kind === 'public' && value !== binding.value)
) {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
return Object.freeze({ name: binding.name, value });
});
return Object.freeze(normalized);
}
async function validateOutput(
context: MaterializedWorkerRemoteExecutionContext,
logArtifactId: string,
): Promise<WorkerRemoteExecutionOutputSink> {
if (typeof context.takeOutput !== 'function') {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
const output = context.takeOutput();
if (
!output ||
typeof output !== 'object' ||
output.logArtifactId !== logArtifactId ||
typeof output.write !== 'function' ||
typeof output.close !== 'function'
) {
if (
output &&
typeof output === 'object' &&
typeof (output as Partial<WorkerRemoteExecutionOutputSink>).close ===
'function'
) {
await Promise.resolve().then(() =>
(output as WorkerRemoteExecutionOutputSink).close()
).catch(() => undefined);
}
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
return output;
}
export class WorkerRemoteExecutionInboxProcessor {
private readonly inbox: WorkerRemoteExecutionInbox;
private readonly activation: WorkerRemoteExecutionActivationClient;
private readonly materializer: WorkerRemoteExecutionContextMaterializer;
private readonly executor: WorkerRemoteExecutionExecutor;
private readonly currentSessionProvider: () =>
WorkerRemoteExecutionSession | undefined;
private readonly nowProvider: () => number;
private readonly randomCapabilityProvider: () => Uint8Array;
private readonly eventIdProvider: () => string;
private readonly inFlight = new Map<string, Promise<WorkerRemoteExecutionProcessResult>>();
constructor(options: WorkerRemoteExecutionInboxProcessorOptions) {
if (
!options ||
typeof options.inbox?.readOffer !== 'function' ||
typeof options.inbox?.replaceOffer !== 'function' ||
typeof options.activation?.acknowledgeStarting !== 'function' ||
typeof options.activation?.acknowledgeRunning !== 'function' ||
typeof options.activation?.failStart !== 'function' ||
typeof options.materializer?.prepare !== 'function' ||
typeof options.executor?.start !== 'function' ||
typeof options.currentSession !== 'function'
) {
throw new WorkerRemoteExecutionProcessorError('invalid_configuration');
}
this.inbox = options.inbox;
this.activation = options.activation;
this.materializer = options.materializer;
this.executor = options.executor;
this.currentSessionProvider = options.currentSession;
this.nowProvider = options.now ?? Date.now;
this.randomCapabilityProvider = options.randomCapability ??
(() => randomBytes(32));
this.eventIdProvider = options.eventId ?? randomUUID;
}
process(offerId: string): Promise<WorkerRemoteExecutionProcessResult> {
const active = this.inFlight.get(offerId);
if (active) return active;
const operation = this.processOnce(offerId).finally(() => {
if (this.inFlight.get(offerId) === operation) this.inFlight.delete(offerId);
});
this.inFlight.set(offerId, operation);
return operation;
}
private async processOnce(
offerId: string,
): Promise<WorkerRemoteExecutionProcessResult> {
let record = await this.inbox.readOffer(offerId);
if (!record) {
throw new WorkerRemoteExecutionProcessorError('offer_missing');
}
record = normalizeWorkerRemoteExecutionInboxRecord(record);
if (record.state === 'completion_acknowledged') {
return Object.freeze({ status: 'already_completed', offerId });
}
if (record.state === 'running_acknowledged') {
return Object.freeze({
status: 'already_running',
offerId,
executorHandle: record.executorHandle,
});
}
if (record.state === 'start_failure_acknowledged') {
return Object.freeze({ status: 'already_failed', offerId });
}
if (record.state === 'recovery_required') return this.recoveryResult(record);
this.assertCurrentTarget(record.offer);
if (record.state === 'launching') {
record = await this.recover(record, 'launch_outcome_unknown');
return this.recoveryResult(record);
}
if (record.state === 'accepted') {
const starting = await this.activation.acknowledgeStarting({
...this.fence(record.offer),
eventId: this.eventId(),
});
this.assertActivation(record.offer, starting);
if (starting.status === 'already_running') {
record = await this.recover(record, 'control_plane_already_running');
return this.recoveryResult(record);
}
if (starting.status === 'already_terminal') {
record = await this.recover(record, 'control_plane_terminal');
return this.recoveryResult(record);
}
record = await this.replace(record, { state: 'starting_acknowledged' });
}
if (record.state === 'start_failed') {
return this.reportStartFailure(record);
}
if (record.state === 'starting_acknowledged') {
record = await this.launch(record);
if (record.state === 'start_failed') return this.reportStartFailure(record);
if (record.state === 'recovery_required') return this.recoveryResult(record);
}
if (record.state !== 'started') {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
const running = await this.activation.acknowledgeRunning({
...this.fence(record.offer),
attemptEventId: this.eventId(),
runEventId: this.eventId(),
executorHandle: record.executorHandle!,
...(record.logArtifactId === undefined
? {}
: { logArtifactId: record.logArtifactId }),
callbackSequence: record.completionReceiptCallbackSequence!,
callbackTokenDigest: record.completionReceiptTokenDigest!,
});
this.assertActivation(record.offer, running);
if (running.status === 'already_terminal') {
record = await this.recover(record, 'control_plane_terminal');
return this.recoveryResult(record);
}
if (
running.status !== 'applied' &&
running.status !== 'already_running'
) {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
record = await this.replace(record, { state: 'running_acknowledged' });
return Object.freeze({
status: running.status === 'already_running' ? 'already_running' : 'running',
offerId,
executorHandle: record.executorHandle,
});
}
private async launch(
record: WorkerRemoteExecutionInboxRecord,
): Promise<WorkerRemoteExecutionInboxRecord> {
const callback = await this.nextCallbackAuthority(record.offer);
const callbackSequence = callback.sequence;
const token = Buffer.from(this.randomCapabilityProvider());
if (token.byteLength !== 32) {
token.fill(0);
throw new WorkerRemoteExecutionProcessorError('invalid_configuration');
}
const tokenDigest = createHash('sha256').update(token).digest('hex');
let context: MaterializedWorkerRemoteExecutionContext | undefined;
try {
try {
context = await this.materializer.prepare({
offer: createClusterRemoteExecutionOffer(record.offer),
completionCallback: Object.freeze({
sequence: callbackSequence,
token,
}),
});
} catch {
return await this.replace(record, { state: 'start_failed' });
}
let environment: MaterializedWorkerRemoteExecutionContext['environment'];
let logArtifactId: string;
try {
environment = validateEnvironment(record.offer, context);
logArtifactId = boundedText(context.logArtifactId, 36);
if (typeof context.takeOutput !== 'function') {
throw new WorkerRemoteExecutionProcessorError(
'materialized_context_invalid',
);
}
} catch {
return await this.replace(record, { state: 'start_failed' });
}
record = await this.replace(record, {
state: 'launching',
executorStartedAtMs: this.now(),
logArtifactId,
completionReceiptCallbackSequence: callbackSequence,
completionReceiptTokenDigest: tokenDigest,
});
let output: WorkerRemoteExecutionOutputSink;
try {
output = await validateOutput(context, logArtifactId);
} catch {
return await this.replace(record, { state: 'start_failed' });
}
let outcome: Awaited<ReturnType<WorkerRemoteExecutionExecutor['start']>>;
try {
outcome = await this.executor.start(Object.freeze({
offerId: record.offer.offerId,
runId: record.offer.candidate.runId,
attemptId: record.offer.candidate.attemptId,
executorStartedAtMs: record.executorStartedAtMs!,
command: record.offer.executionRevision.command,
environment,
...(record.offer.executionRevision.workingDirectory === undefined
? {}
: { workingDirectory: record.offer.executionRevision.workingDirectory }),
...(record.offer.executionRevision.timeoutMs === undefined
? {}
: {
timeoutMs: record.offer.executionRevision.timeoutMs,
executionDeadlineAtMs: callback.deadlineAtMs,
}),
logArtifactId,
output,
completionCallback: Object.freeze({
sequence: callbackSequence,
token,
}),
}));
} catch {
return await this.recover(record, 'launch_outcome_unknown');
}
if (outcome?.status === 'rejected') {
await output.close().catch(() => undefined);
return await this.replace(record, { state: 'start_failed' });
}
if (outcome?.status !== 'started') {
return await this.recover(record, 'launch_outcome_unknown');
}
let executorHandle: string;
let executorStartedAtMs: number;
try {
executorHandle = boundedText(outcome.executorHandle, 512);
executorStartedAtMs = outcome.executorStartedAtMs;
if (
!Number.isSafeInteger(executorStartedAtMs) ||
executorStartedAtMs < 0 ||
executorStartedAtMs > this.now() ||
executorStartedAtMs !== record.executorStartedAtMs
) {
throw new WorkerRemoteExecutionProcessorError(
'materialized_context_invalid',
);
}
} catch {
return await this.recover(record, 'launch_outcome_unknown');
}
try {
return await this.replace(record, {
state: 'started',
executorHandle,
logArtifactId,
});
} catch {
return await this.recover(record, 'launch_outcome_unknown');
}
} finally {
token.fill(0);
await context?.dispose?.().catch(() => undefined);
}
}
private async nextCallbackAuthority(
offer: ClusterRemoteExecutionOffer,
): Promise<Readonly<{ sequence: number; deadlineAtMs?: number }>> {
const replay = await this.activation.acknowledgeStarting({
...this.fence(offer),
eventId: this.eventId(),
});
this.assertActivation(offer, replay);
if (
replay.status !== 'already_starting' &&
replay.status !== 'applied'
) {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
const sequence = replay.snapshot.callbackSequence + 1;
if (!Number.isSafeInteger(sequence) || sequence < 1 || sequence > 2_147_483_647) {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
return Object.freeze({
sequence,
...(replay.snapshot.deadlineAtMs === undefined
? {}
: { deadlineAtMs: replay.snapshot.deadlineAtMs }),
});
}
private async reportStartFailure(
record: WorkerRemoteExecutionInboxRecord,
): Promise<WorkerRemoteExecutionProcessResult> {
const result = await this.activation.failStart({
...this.fence(record.offer),
attemptEventId: this.eventId(),
runEventId: this.eventId(),
});
this.assertActivation(record.offer, result, true);
if (result.status === 'already_running') {
const recovery = await this.recover(
record,
'control_plane_already_running',
);
return this.recoveryResult(recovery);
}
if (result.status !== 'applied' && result.status !== 'already_terminal') {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
await this.replace(record, { state: 'start_failure_acknowledged' });
return Object.freeze({ status: 'start_failed', offerId: record.offer.offerId });
}
private assertCurrentTarget(offer: ClusterRemoteExecutionOffer): void {
const current = this.currentSessionProvider();
const now = this.now();
if (
!current ||
current.workerId !== offer.worker.workerId ||
current.sessionId !== offer.worker.sessionId ||
current.generation !== offer.worker.generation ||
current.status === 'offline' ||
(current.status === 'draining' && offer.deliveryKind === 'new_claim') ||
current.leaseExpiresAtMs <= now
) {
throw new WorkerRemoteExecutionProcessorError('target_fenced');
}
if (offer.lease.expiresAtMs <= now) {
throw new WorkerRemoteExecutionProcessorError('offer_expired');
}
}
private assertActivation(
offer: ClusterRemoteExecutionOffer,
result: Readonly<RemoteRunActivationResult>,
allowCompletedLease = false,
): void {
const snapshot = result?.snapshot;
if (
!['applied', 'already_starting', 'already_running', 'already_terminal']
.includes(result?.status) ||
!snapshot ||
snapshot.runId !== offer.candidate.runId ||
snapshot.attemptId !== offer.candidate.attemptId ||
snapshot.leaseGeneration !== offer.lease.leaseGeneration ||
!Number.isSafeInteger(snapshot.leaseVersion) ||
snapshot.leaseVersion < offer.lease.version ||
snapshot.leaseVersion > offer.lease.version + (allowCompletedLease ? 1 : 0) ||
!Number.isSafeInteger(snapshot.callbackSequence) ||
snapshot.callbackSequence < 0 ||
snapshot.callbackSequence > 2_147_483_647
|| (offer.executionRevision.timeoutMs === undefined) !==
(snapshot.deadlineAtMs === undefined)
|| (snapshot.deadlineAtMs !== undefined &&
(!Number.isSafeInteger(snapshot.deadlineAtMs) ||
snapshot.deadlineAtMs < 0))
) {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
}
private fence(offer: ClusterRemoteExecutionOffer) {
return Object.freeze({
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
workerId: offer.worker.workerId,
workerSessionId: offer.worker.sessionId,
workerGeneration: offer.worker.generation,
offerId: offer.offerId,
leaseGeneration: offer.lease.leaseGeneration,
leaseToken: offer.leaseToken,
expectedLeaseVersion: offer.lease.version,
});
}
private async replace(
previous: WorkerRemoteExecutionInboxRecord,
patch: Partial<WorkerRemoteExecutionInboxRecord>,
): Promise<WorkerRemoteExecutionInboxRecord> {
const next = normalizeWorkerRemoteExecutionInboxRecord({
...previous,
...patch,
schemaVersion: 1,
revision: previous.revision + 1,
updatedAtMs: Math.max(this.now(), previous.updatedAtMs),
});
await this.inbox.replaceOffer(next, previous.revision);
return next;
}
private recover(
record: WorkerRemoteExecutionInboxRecord,
recoveryReason: WorkerRemoteExecutionRecoveryReason,
): Promise<WorkerRemoteExecutionInboxRecord> {
return this.replace(record, { state: 'recovery_required', recoveryReason });
}
private recoveryResult(
record: WorkerRemoteExecutionInboxRecord,
): WorkerRemoteExecutionProcessResult {
return Object.freeze({
status: 'recovery_required',
offerId: record.offer.offerId,
recoveryReason: record.recoveryReason,
});
}
private eventId(): string {
const value = this.eventIdProvider();
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > 36 ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new WorkerRemoteExecutionProcessorError('invalid_configuration');
}
return value;
}
private now(): number {
return safeTime(this.nowProvider());
}
}
packages/ql3-worker-runtime/src/remote-execution/headlessExecutionLifecycle.ts
+386
View File
@@ -0,0 +1,386 @@
// Remote Execution owns the caller-driven offer supervision and drain lifecycle.
import type { WorkerRemoteExecutionInbox } from './executionInbox';
import type {
WorkerRemoteExecutionInboxProcessor,
WorkerRemoteExecutionProcessResult,
WorkerRemoteExecutionSession,
} from './executionInboxProcessor';
import type {
WorkerRemoteOfferPullCoordinator,
WorkerRemoteOfferPullResult,
} from './remoteOfferDelivery';
import type {
WorkerRemoteExecutionControlCoordinator,
WorkerRemoteExecutionControlResult,
} from '../execution/workerExecutionControlCoordinator';
export interface WorkerRemoteExecutionLifecycleJournal
extends WorkerRemoteExecutionInbox {
acquireOwnership(): Promise<void>;
releaseOwnership(): Promise<void>;
}
export interface WorkerRemoteExecutionHeadlessLifecycleOptions {
readonly journal: WorkerRemoteExecutionLifecycleJournal;
readonly offers: Pick<WorkerRemoteOfferPullCoordinator, 'pull'>;
readonly processor: Pick<WorkerRemoteExecutionInboxProcessor, 'process'>;
readonly control: Pick<WorkerRemoteExecutionControlCoordinator, 'reconcile'>;
readonly currentSession: () => WorkerRemoteExecutionSession | undefined;
readonly maximumRecordsPerTick?: number;
readonly maximumSupervisionRecordsPerTick?: number;
readonly now?: () => number;
}
export type WorkerRemoteExecutionLifecycleTickResult =
| Readonly<{
status: 'reconciling';
processed: number;
nextAfterOfferId: string;
}>
| Readonly<{
status: 'reconciled';
processed: number;
}>
| Readonly<{
status: 'recovery_required';
offerId: string;
}>
| Readonly<{
status: 'session_unavailable';
}>
| Readonly<{
status: 'draining';
}>
| Readonly<{
status: 'processed';
offerId: string;
execution: WorkerRemoteExecutionProcessResult;
pull?: WorkerRemoteOfferPullResult;
}>
| Readonly<{
status: 'pull_result';
pull: WorkerRemoteOfferPullResult;
}>;
export class WorkerRemoteExecutionLifecycleError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'inactive'
| 'stopping'
| 'draining',
) {
super(`Worker remote execution lifecycle failed: ${reason}`);
this.name = 'WorkerRemoteExecutionLifecycleError';
}
}
type Mode = 'inactive' | 'running' | 'stopping';
export class WorkerRemoteExecutionHeadlessLifecycle {
private readonly journal: WorkerRemoteExecutionLifecycleJournal;
private readonly offers: Pick<WorkerRemoteOfferPullCoordinator, 'pull'>;
private readonly processor: Pick<WorkerRemoteExecutionInboxProcessor, 'process'>;
private readonly control: Pick<
WorkerRemoteExecutionControlCoordinator,
'reconcile'
>;
private readonly currentSessionProvider: () =>
WorkerRemoteExecutionSession | undefined;
private readonly maximumRecordsPerTick: number;
private readonly maximumSupervisionRecordsPerTick: number;
private readonly nowProvider: () => number;
private mode: Mode = 'inactive';
private draining = false;
private startupAfterOfferId?: string;
private startupComplete = false;
private supervisionAfterOfferId?: string;
private retryOfferId?: string;
private recoveryOfferId?: string;
private inFlight?: Promise<WorkerRemoteExecutionLifecycleTickResult>;
private stopController?: AbortController;
private drainOperation?: Promise<void>;
private stopOperation?: Promise<void>;
constructor(options: WorkerRemoteExecutionHeadlessLifecycleOptions) {
if (
!options ||
typeof options.journal?.acquireOwnership !== 'function' ||
typeof options.journal?.releaseOwnership !== 'function' ||
typeof options.journal?.listOffers !== 'function' ||
typeof options.offers?.pull !== 'function' ||
typeof options.processor?.process !== 'function' ||
typeof options.control?.reconcile !== 'function' ||
typeof options.currentSession !== 'function'
) {
throw new WorkerRemoteExecutionLifecycleError('invalid_configuration');
}
const maximumRecordsPerTick = options.maximumRecordsPerTick ?? 16;
const maximumSupervisionRecordsPerTick =
options.maximumSupervisionRecordsPerTick ?? maximumRecordsPerTick;
if (
!Number.isSafeInteger(maximumRecordsPerTick) ||
maximumRecordsPerTick < 1 ||
maximumRecordsPerTick > 64
|| !Number.isSafeInteger(maximumSupervisionRecordsPerTick)
|| maximumSupervisionRecordsPerTick < 1
|| maximumSupervisionRecordsPerTick > 64
) {
throw new WorkerRemoteExecutionLifecycleError('invalid_configuration');
}
this.journal = options.journal;
this.offers = options.offers;
this.processor = options.processor;
this.control = options.control;
this.currentSessionProvider = options.currentSession;
this.maximumRecordsPerTick = maximumRecordsPerTick;
this.maximumSupervisionRecordsPerTick = maximumSupervisionRecordsPerTick;
this.nowProvider = options.now ?? Date.now;
}
async start(): Promise<'started' | 'already_started'> {
if (this.mode === 'running') return 'already_started';
if (this.mode === 'stopping') {
throw new WorkerRemoteExecutionLifecycleError('stopping');
}
await this.journal.acquireOwnership();
this.mode = 'running';
this.startupAfterOfferId = undefined;
this.startupComplete = false;
this.supervisionAfterOfferId = undefined;
this.retryOfferId = undefined;
this.recoveryOfferId = undefined;
this.draining = false;
this.drainOperation = undefined;
this.stopController = new AbortController();
return 'started';
}
beginDrain(): Promise<void> {
if (this.mode === 'stopping') {
return Promise.reject(
new WorkerRemoteExecutionLifecycleError('stopping'),
);
}
if (this.mode !== 'running' || !this.stopController) {
return Promise.reject(
new WorkerRemoteExecutionLifecycleError('inactive'),
);
}
if (this.drainOperation) return this.drainOperation;
if (this.draining) return Promise.resolve();
this.draining = true;
this.stopController.abort(
new WorkerRemoteExecutionLifecycleError('draining'),
);
const operation = (async () => {
await this.inFlight?.catch(() => undefined);
if (this.mode === 'running') this.stopController = new AbortController();
})().finally(() => {
if (this.drainOperation === operation) this.drainOperation = undefined;
});
this.drainOperation = operation;
return operation;
}
tick(signal?: AbortSignal): Promise<WorkerRemoteExecutionLifecycleTickResult> {
if (this.mode === 'stopping') {
return Promise.reject(
new WorkerRemoteExecutionLifecycleError('stopping'),
);
}
if (this.mode !== 'running' || !this.stopController) {
return Promise.reject(
new WorkerRemoteExecutionLifecycleError('inactive'),
);
}
if (this.inFlight) return this.inFlight;
const combinedSignal = signal === undefined
? this.stopController.signal
: AbortSignal.any([signal, this.stopController.signal]);
const operation = this.tickOnce(combinedSignal).finally(() => {
if (this.inFlight === operation) this.inFlight = undefined;
});
this.inFlight = operation;
return operation;
}
stop(): Promise<void> {
if (this.stopOperation) return this.stopOperation;
if (this.mode === 'inactive') return Promise.resolve();
this.mode = 'stopping';
this.stopController?.abort(
new WorkerRemoteExecutionLifecycleError('stopping'),
);
const operation = (async () => {
try {
await this.drainOperation?.catch(() => undefined);
await this.inFlight?.catch(() => undefined);
await this.journal.releaseOwnership();
} finally {
this.mode = 'inactive';
this.draining = false;
this.stopController = undefined;
this.drainOperation = undefined;
this.stopOperation = undefined;
}
})();
this.stopOperation = operation;
return operation;
}
private async tickOnce(
signal: AbortSignal,
): Promise<WorkerRemoteExecutionLifecycleTickResult> {
if (this.recoveryOfferId) {
return Object.freeze({
status: 'recovery_required' as const,
offerId: this.recoveryOfferId,
});
}
if (!this.startupComplete) {
return this.reconcileStartupPage();
}
if (this.retryOfferId) {
const offerId = this.retryOfferId;
const execution = await this.processor.process(offerId);
return this.observeExecution(offerId, execution);
}
const supervision = await this.supervisePage();
if (supervision) return supervision;
if (this.draining) {
return Object.freeze({ status: 'draining' as const });
}
const session = this.currentSessionProvider();
const now = this.nowProvider();
if (
!session ||
session.status !== 'available' ||
!Number.isSafeInteger(now) ||
now < 0 ||
session.leaseExpiresAtMs <= now
) {
return Object.freeze({ status: 'session_unavailable' as const });
}
if (signal.aborted) throw signal.reason;
const pull = await this.offers.pull(session, signal);
if (pull.status !== 'accepted' && pull.status !== 'replayed') {
return Object.freeze({ status: 'pull_result' as const, pull });
}
this.retryOfferId = pull.offerId;
const execution = await this.processor.process(pull.offerId);
return this.observeExecution(pull.offerId, execution, pull);
}
private async supervisePage(): Promise<
WorkerRemoteExecutionLifecycleTickResult | undefined
> {
const page = await this.journal.listOffers({
...(this.supervisionAfterOfferId === undefined
? {}
: { afterOfferId: this.supervisionAfterOfferId }),
limit: this.maximumSupervisionRecordsPerTick,
});
for (const record of page.records) {
if (record.state === 'recovery_required') {
this.recoveryOfferId = record.offer.offerId;
return Object.freeze({
status: 'recovery_required' as const,
offerId: record.offer.offerId,
});
}
if (
record.state !== 'launching' &&
record.state !== 'started' &&
record.state !== 'running_acknowledged'
) continue;
const control = await this.control.reconcile(record.offer.offerId);
if (this.controlRequiresRecovery(control)) {
this.recoveryOfferId = record.offer.offerId;
return Object.freeze({
status: 'recovery_required' as const,
offerId: record.offer.offerId,
});
}
}
this.supervisionAfterOfferId = page.nextAfterOfferId;
return undefined;
}
private controlRequiresRecovery(
result: WorkerRemoteExecutionControlResult,
): boolean {
return result.status === 'lease_expired' ||
result.status === 'terminal' ||
result.status === 'completion_terminal';
}
private async reconcileStartupPage(): Promise<
WorkerRemoteExecutionLifecycleTickResult
> {
const page = await this.journal.listOffers({
...(this.startupAfterOfferId === undefined
? {}
: { afterOfferId: this.startupAfterOfferId }),
limit: this.maximumRecordsPerTick,
});
let processed = 0;
for (const record of page.records) {
if (record.state === 'recovery_required') {
this.recoveryOfferId = record.offer.offerId;
return Object.freeze({
status: 'recovery_required' as const,
offerId: record.offer.offerId,
});
}
if (
record.state === 'accepted' ||
record.state === 'starting_acknowledged' ||
record.state === 'launching' ||
record.state === 'started' ||
record.state === 'start_failed'
) {
const execution = await this.processor.process(record.offer.offerId);
processed += 1;
const observed = this.observeExecution(record.offer.offerId, execution);
if (observed.status === 'recovery_required') return observed;
}
}
if (page.nextAfterOfferId !== undefined) {
this.startupAfterOfferId = page.nextAfterOfferId;
return Object.freeze({
status: 'reconciling' as const,
processed,
nextAfterOfferId: page.nextAfterOfferId,
});
}
this.startupAfterOfferId = undefined;
this.startupComplete = true;
return Object.freeze({
status: 'reconciled' as const,
processed,
});
}
private observeExecution(
offerId: string,
execution: WorkerRemoteExecutionProcessResult,
pull?: WorkerRemoteOfferPullResult,
): WorkerRemoteExecutionLifecycleTickResult {
if (execution.status === 'recovery_required') {
this.recoveryOfferId = offerId;
this.retryOfferId = undefined;
return Object.freeze({
status: 'recovery_required' as const,
offerId,
});
}
this.retryOfferId = undefined;
return Object.freeze({
status: 'processed' as const,
offerId,
execution,
...(pull === undefined ? {} : { pull }),
});
}
}
packages/ql3-worker-runtime/src/remote-execution/remoteOfferDelivery.ts
+458
View File
@@ -0,0 +1,458 @@
// Remote Execution owns stable offer claiming, delivery, and durable admission.
import { randomBytes, randomUUID } from 'node:crypto';
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import {
assertWorkerId,
assertWorkerSessionId,
} from '@qinglong/runtime-core/worker-session';
import {
InvalidRemoteExecutionOfferDeliveryError,
MAX_REMOTE_EXECUTION_OFFER_RESPONSE_BYTES,
normalizeRemoteExecutionOfferClaimAuthority,
parseRemoteExecutionOfferPullResponse,
type RemoteExecutionOfferClaimAuthority,
type RemoteExecutionOfferDeliveryStats,
type RemoteExecutionOfferIdleReason,
} from '@qinglong/runtime-core/remote-offer-delivery';
import {
normalizeWorkerRemoteExecutionInboxRecord,
type WorkerRemoteExecutionInboxRecord,
} from './executionInbox';
export const MAX_WORKER_REMOTE_OFFER_ATTEMPTS = 16;
export const MAX_WORKER_REMOTE_OFFER_BACKOFF_MS = 60_000;
export const MAX_WORKER_REMOTE_OFFER_INBOX_ENTRIES = 1024;
export const DEFAULT_WORKER_REMOTE_OFFER_INBOX_ENTRIES = 64;
export const MAX_WORKER_REMOTE_OFFER_RECORD_BYTES = 160 * 1024;
export interface WorkerRemoteOfferClaimRecord
extends RemoteExecutionOfferClaimAuthority {
readonly schemaVersion: 1;
readonly revision: number;
readonly attemptCount: number;
readonly createdAtMs: number;
readonly updatedAtMs: number;
readonly lastAttemptAtMs: number | null;
readonly nextAttemptAtMs: number;
}
/** @deprecated Use WorkerRemoteExecutionInboxRecord. */
export type WorkerRemoteOfferInboxRecord = WorkerRemoteExecutionInboxRecord;
export type WorkerRemoteOfferInboxAcceptResult = Readonly<{
status: 'accepted' | 'replayed';
record: WorkerRemoteOfferInboxRecord;
}>;
export interface WorkerRemoteOfferDeliveryJournal {
readPendingClaim(): Promise<WorkerRemoteOfferClaimRecord | undefined>;
createPendingClaim(
record: WorkerRemoteOfferClaimRecord,
): Promise<WorkerRemoteOfferClaimRecord>;
replacePendingClaim(
record: WorkerRemoteOfferClaimRecord,
expectedRevision: number,
): Promise<WorkerRemoteOfferClaimRecord>;
clearPendingClaim(offerId: string, expectedRevision: number): Promise<void>;
acceptOffer(
offer: ClusterRemoteExecutionOffer,
acceptedAtMs: number,
): Promise<WorkerRemoteOfferInboxAcceptResult>;
readOffer(offerId: string): Promise<WorkerRemoteOfferInboxRecord | undefined>;
}
export interface WorkerRemoteOfferTransport {
exchange(request: Readonly<{
path: string;
body: Readonly<{
workerGeneration: number;
offerId: string;
leaseToken: string;
}>;
maximumResponseBytes: number;
signal?: AbortSignal;
}>): Promise<Uint8Array | string>;
}
export interface WorkerRemoteOfferSession {
readonly workerId: string;
readonly sessionId: string;
readonly generation: number;
}
export interface WorkerRemoteOfferPullCoordinatorOptions {
readonly journal: WorkerRemoteOfferDeliveryJournal;
readonly transport: WorkerRemoteOfferTransport;
readonly currentSession: () => WorkerRemoteOfferSession | undefined;
readonly now?: () => number;
readonly random?: () => number;
readonly backoffBaseMs?: number;
}
export type WorkerRemoteOfferPullResult =
| Readonly<{
status: 'accepted' | 'replayed';
offerId: string;
stats: RemoteExecutionOfferDeliveryStats;
truncated: boolean;
}>
| Readonly<{
status: 'idle';
reason: RemoteExecutionOfferIdleReason;
stats: RemoteExecutionOfferDeliveryStats;
truncated: boolean;
}>
| Readonly<{
status: 'backoff' | 'unavailable' | 'invalid_response';
offerId: string;
nextAttemptAtMs: number;
}>;
export class WorkerRemoteOfferDeliveryError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'claim_conflict'
| 'claim_revision_conflict'
| 'offer_conflict'
| 'attempt_budget_exhausted',
) {
super(`Worker remote offer delivery failed: ${reason}`);
this.name = 'WorkerRemoteOfferDeliveryError';
}
}
function safeTime(value: number, label: string): number {
if (!Number.isSafeInteger(value) || value < 0) {
throw new WorkerRemoteOfferDeliveryError('invalid_configuration');
}
return value;
}
function safeRevision(value: number): number {
if (!Number.isSafeInteger(value) || value < 0) {
throw new WorkerRemoteOfferDeliveryError('claim_revision_conflict');
}
return value;
}
export function normalizeWorkerRemoteOfferClaimRecord(
value: WorkerRemoteOfferClaimRecord,
): WorkerRemoteOfferClaimRecord {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
const expected = [
'schemaVersion', 'revision', 'workerId', 'workerSessionId',
'workerGeneration', 'offerId', 'leaseToken', 'attemptCount',
'createdAtMs', 'updatedAtMs', 'lastAttemptAtMs', 'nextAttemptAtMs',
].sort();
const actual = Object.keys(value).sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index]) ||
value.schemaVersion !== 1
) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
const authority = normalizeRemoteExecutionOfferClaimAuthority({
workerId: value.workerId,
workerSessionId: value.workerSessionId,
workerGeneration: value.workerGeneration,
offerId: value.offerId,
leaseToken: value.leaseToken,
});
safeRevision(value.revision);
if (
!Number.isSafeInteger(value.attemptCount) ||
value.attemptCount < 0 ||
value.attemptCount > MAX_WORKER_REMOTE_OFFER_ATTEMPTS
) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
const createdAtMs = safeTime(value.createdAtMs, 'createdAtMs');
const updatedAtMs = safeTime(value.updatedAtMs, 'updatedAtMs');
const nextAttemptAtMs = safeTime(value.nextAttemptAtMs, 'nextAttemptAtMs');
if (
updatedAtMs < createdAtMs ||
nextAttemptAtMs < createdAtMs ||
(value.lastAttemptAtMs !== null &&
(!Number.isSafeInteger(value.lastAttemptAtMs) ||
value.lastAttemptAtMs < createdAtMs ||
value.lastAttemptAtMs > updatedAtMs))
) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
return Object.freeze({
schemaVersion: 1,
revision: value.revision,
...authority,
attemptCount: value.attemptCount,
createdAtMs,
updatedAtMs,
lastAttemptAtMs: value.lastAttemptAtMs,
nextAttemptAtMs,
});
}
export function createWorkerRemoteOfferClaimRecord(
authority: RemoteExecutionOfferClaimAuthority,
createdAtMs: number,
): WorkerRemoteOfferClaimRecord {
const normalized = normalizeRemoteExecutionOfferClaimAuthority(authority);
const now = safeTime(createdAtMs, 'createdAtMs');
return normalizeWorkerRemoteOfferClaimRecord({
schemaVersion: 1,
revision: 0,
...normalized,
attemptCount: 0,
createdAtMs: now,
updatedAtMs: now,
lastAttemptAtMs: null,
nextAttemptAtMs: now,
});
}
export function normalizeWorkerRemoteOfferInboxRecord(
value: WorkerRemoteOfferInboxRecord,
): WorkerRemoteOfferInboxRecord {
try {
return normalizeWorkerRemoteExecutionInboxRecord(value);
} catch {
throw new WorkerRemoteOfferDeliveryError('offer_conflict');
}
}
export function sameWorkerRemoteOfferAuthority(
left: ClusterRemoteExecutionOffer,
right: ClusterRemoteExecutionOffer,
): boolean {
const leftOffer = createClusterRemoteExecutionOffer(left);
const rightOffer = createClusterRemoteExecutionOffer(right);
return (
leftOffer.offerId === rightOffer.offerId &&
leftOffer.executionDigest === rightOffer.executionDigest &&
JSON.stringify(leftOffer.candidate) === JSON.stringify(rightOffer.candidate) &&
JSON.stringify(leftOffer.worker) === JSON.stringify(rightOffer.worker) &&
leftOffer.lease.runId === rightOffer.lease.runId &&
leftOffer.lease.attemptId === rightOffer.lease.attemptId &&
leftOffer.lease.leaseGeneration === rightOffer.lease.leaseGeneration &&
leftOffer.lease.leaseTokenDigest === rightOffer.lease.leaseTokenDigest &&
leftOffer.leaseToken === rightOffer.leaseToken &&
JSON.stringify(leftOffer.executionRevision) ===
JSON.stringify(rightOffer.executionRevision)
);
}
export class WorkerRemoteOfferPullCoordinator {
private readonly journal: WorkerRemoteOfferDeliveryJournal;
private readonly transport: WorkerRemoteOfferTransport;
private readonly currentSessionProvider: () =>
WorkerRemoteOfferSession | undefined;
private readonly nowProvider: () => number;
private readonly randomProvider: () => number;
private readonly backoffBaseMs: number;
private inFlight?: Readonly<{
session: WorkerRemoteOfferSession;
operation: Promise<WorkerRemoteOfferPullResult>;
}>;
constructor(options: WorkerRemoteOfferPullCoordinatorOptions) {
if (
!options ||
typeof options.journal?.readPendingClaim !== 'function' ||
typeof options.transport?.exchange !== 'function' ||
typeof options.currentSession !== 'function'
) {
throw new WorkerRemoteOfferDeliveryError('invalid_configuration');
}
const backoffBaseMs = options.backoffBaseMs ?? 1_000;
if (
!Number.isSafeInteger(backoffBaseMs) ||
backoffBaseMs < 100 ||
backoffBaseMs > MAX_WORKER_REMOTE_OFFER_BACKOFF_MS
) {
throw new WorkerRemoteOfferDeliveryError('invalid_configuration');
}
this.journal = options.journal;
this.transport = options.transport;
this.currentSessionProvider = options.currentSession;
this.nowProvider = options.now ?? Date.now;
this.randomProvider = options.random ?? Math.random;
this.backoffBaseMs = backoffBaseMs;
}
pull(
session: WorkerRemoteOfferSession,
signal?: AbortSignal,
): Promise<WorkerRemoteOfferPullResult> {
assertWorkerId(session.workerId);
assertWorkerSessionId(session.sessionId);
if (!Number.isSafeInteger(session.generation) || session.generation < 1) {
throw new WorkerRemoteOfferDeliveryError('invalid_configuration');
}
const normalizedSession = Object.freeze({
workerId: session.workerId,
sessionId: session.sessionId,
generation: session.generation,
});
this.assertCurrentSession(normalizedSession);
if (this.inFlight) {
if (
this.inFlight.session.workerId !== normalizedSession.workerId ||
this.inFlight.session.sessionId !== normalizedSession.sessionId ||
this.inFlight.session.generation !== normalizedSession.generation
) {
return Promise.reject(
new WorkerRemoteOfferDeliveryError('claim_conflict'),
);
}
return this.inFlight.operation;
}
const operation = this.pullOnce(normalizedSession, signal)
.finally(() => {
if (this.inFlight?.operation === operation) this.inFlight = undefined;
});
this.inFlight = Object.freeze({
session: normalizedSession,
operation,
});
return operation;
}
private async pullOnce(
session: WorkerRemoteOfferSession,
signal?: AbortSignal,
): Promise<WorkerRemoteOfferPullResult> {
const now = this.now();
let claim = await this.journal.readPendingClaim();
if (claim) {
if (
claim.workerId !== session.workerId ||
claim.workerSessionId !== session.sessionId ||
claim.workerGeneration !== session.generation
) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
} else {
const generated = normalizeRemoteExecutionOfferClaimAuthority({
workerId: session.workerId,
workerSessionId: session.sessionId,
workerGeneration: session.generation,
offerId: randomUUID(),
leaseToken: randomBytes(32).toString('base64url'),
});
claim = await this.journal.createPendingClaim(
createWorkerRemoteOfferClaimRecord(generated, now),
);
}
if (claim.nextAttemptAtMs > now) {
return Object.freeze({
status: 'backoff' as const,
offerId: claim.offerId,
nextAttemptAtMs: claim.nextAttemptAtMs,
});
}
if (claim.attemptCount >= MAX_WORKER_REMOTE_OFFER_ATTEMPTS) {
throw new WorkerRemoteOfferDeliveryError('attempt_budget_exhausted');
}
claim = await this.journal.replacePendingClaim(
normalizeWorkerRemoteOfferClaimRecord({
...claim,
revision: claim.revision + 1,
attemptCount: claim.attemptCount + 1,
lastAttemptAtMs: now,
updatedAtMs: now,
nextAttemptAtMs: now,
}),
claim.revision,
);
try {
const serialized = await this.transport.exchange({
path: `/api/v3/worker-ingress/workers/${claim.workerId}/sessions/${claim.workerSessionId}/offers`,
body: Object.freeze({
workerGeneration: claim.workerGeneration,
offerId: claim.offerId,
leaseToken: claim.leaseToken,
}),
maximumResponseBytes: MAX_REMOTE_EXECUTION_OFFER_RESPONSE_BYTES,
...(signal === undefined ? {} : { signal }),
});
const result = parseRemoteExecutionOfferPullResponse(serialized, {
workerId: claim.workerId,
workerSessionId: claim.workerSessionId,
workerGeneration: claim.workerGeneration,
offerId: claim.offerId,
leaseToken: claim.leaseToken,
});
if (result.status === 'idle') {
this.assertCurrentSession(session);
await this.journal.clearPendingClaim(claim.offerId, claim.revision);
return Object.freeze({
status: 'idle' as const,
reason: result.reason,
stats: result.stats,
truncated: result.truncated,
});
}
this.assertCurrentSession(session);
const accepted = await this.journal.acceptOffer(result.offer, this.now());
await this.journal.clearPendingClaim(claim.offerId, claim.revision);
return Object.freeze({
status: accepted.status,
offerId: accepted.record.offer.offerId,
stats: result.stats,
truncated: result.truncated,
});
} catch (error) {
if (signal?.aborted) throw signal.reason ?? error;
const nextAttemptAtMs = this.nextAttemptAt(claim.attemptCount, this.now());
await this.journal.replacePendingClaim(
normalizeWorkerRemoteOfferClaimRecord({
...claim,
revision: claim.revision + 1,
updatedAtMs: this.now(),
nextAttemptAtMs,
}),
claim.revision,
);
return Object.freeze({
status:
error instanceof InvalidRemoteExecutionOfferDeliveryError
? 'invalid_response' as const
: 'unavailable' as const,
offerId: claim.offerId,
nextAttemptAtMs,
});
}
}
private nextAttemptAt(attemptCount: number, now: number): number {
const random = this.randomProvider();
if (!Number.isFinite(random) || random < 0 || random >= 1) {
throw new WorkerRemoteOfferDeliveryError('invalid_configuration');
}
const ceiling = Math.min(
MAX_WORKER_REMOTE_OFFER_BACKOFF_MS,
this.backoffBaseMs * 2 ** Math.max(0, attemptCount - 1),
);
return safeTime(now + Math.floor(random * ceiling), 'nextAttemptAtMs');
}
private assertCurrentSession(expected: WorkerRemoteOfferSession): void {
const current = this.currentSessionProvider();
if (
!current ||
current.workerId !== expected.workerId ||
current.sessionId !== expected.sessionId ||
current.generation !== expected.generation
) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
}
private now(): number {
return safeTime(this.nowProvider(), 'now');
}
}
packages/ql3-worker-runtime/src/remote-execution/remoteOfferDeliveryEntrypoint.ts
+15
View File
@@ -0,0 +1,15 @@
// Remote Execution owns its public delivery composition and transport exports.
export * from './executionInbox';
export * from './executionInboxProcessor';
export * from './executionContextMaterializer';
export * from './headlessExecutionLifecycle';
export * from './remoteOfferDelivery';
export * from './remoteOfferFileJournal';
export * from './transport/remoteActivationHttpsClient';
export * from './transport/remoteOfferHttpsTransport';
export * from './transport/workerIngressHttpsClient';
export * from './transport/remoteSecretHttpsProvider';
export * from '../execution/workerFileLogArtifactAllocator';
export * from './transport/remoteWorkerCompletionHttpsClient';
export * from './transport/remoteWorkerLeaseControlHttpsClient';
export * from '../execution/workerExecutionControlCoordinator';
packages/ql3-worker-runtime/src/remote-execution/remoteOfferFileJournal.ts
+543
View File
@@ -0,0 +1,543 @@
// Remote Execution owns the private atomic offer journal and its single-owner fence.
import { randomBytes } from 'node:crypto';
import { constants } from 'node:fs';
import {
chmod,
link,
lstat,
mkdir,
open,
readdir,
rename,
rm,
unlink,
} from 'node:fs/promises';
import { isAbsolute, join } from 'node:path';
import { lock } from 'proper-lockfile';
import { assertRunDispatchId } from '@qinglong/runtime-core/run-dispatch-lease';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import {
DEFAULT_WORKER_REMOTE_OFFER_INBOX_ENTRIES,
MAX_WORKER_REMOTE_OFFER_INBOX_ENTRIES,
MAX_WORKER_REMOTE_OFFER_RECORD_BYTES,
normalizeWorkerRemoteOfferClaimRecord,
sameWorkerRemoteOfferAuthority,
type WorkerRemoteOfferClaimRecord,
type WorkerRemoteOfferDeliveryJournal,
type WorkerRemoteOfferInboxAcceptResult,
} from './remoteOfferDelivery';
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import {
assertWorkerRemoteExecutionInboxTransition,
createWorkerRemoteExecutionInboxRecord,
normalizeWorkerRemoteExecutionInboxRecord,
WorkerRemoteExecutionInboxError,
type WorkerRemoteExecutionInbox,
type WorkerRemoteExecutionInboxPage,
type WorkerRemoteExecutionInboxRecord,
} from './executionInbox';
const OFFER_FILE = /^([A-Za-z0-9._:-]{1,128})\.json$/;
const MIN_OWNERSHIP_STALE_MS = 5_000;
const MAX_OWNERSHIP_STALE_MS = 5 * 60_000;
export interface WorkerRemoteOfferFileJournalOptions {
readonly rootDirectory: string;
readonly maximumEntries?: number;
readonly ownershipStaleMs?: number;
}
export class WorkerRemoteOfferFileJournalError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'not_owned'
| 'already_owned'
| 'ownership_compromised'
| 'unsafe_storage'
| 'capacity_exhausted'
| 'claim_revision_conflict'
| 'offer_revision_conflict'
| 'invalid_transition'
| 'offer_conflict',
) {
super(`Worker remote offer file journal failed: ${reason}`);
this.name = 'WorkerRemoteOfferFileJournalError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
(error as NodeJS.ErrnoException).code === code
);
}
async function safeDirectory(path: string): Promise<void> {
try {
let created = false;
try {
await lstat(path);
} catch (error) {
if (!isCode(error, 'ENOENT')) throw error;
await mkdir(path, { recursive: true, mode: 0o700 });
created = true;
}
const stat = await lstat(path);
if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0) {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
if (created) await chmod(path, 0o700);
} catch (error) {
if (error instanceof WorkerRemoteOfferFileJournalError) throw error;
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
}
async function syncDirectory(path: string): Promise<void> {
const handle = await open(path, constants.O_RDONLY);
try {
await handle.sync();
} finally {
await handle.close();
}
}
function serialize(value: unknown): Buffer {
const bytes = Buffer.from(`${JSON.stringify(value)}\n`, 'utf8');
if (bytes.byteLength < 2 || bytes.byteLength > MAX_WORKER_REMOTE_OFFER_RECORD_BYTES) {
bytes.fill(0);
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
return bytes;
}
async function readJson(path: string): Promise<unknown> {
let handle;
try {
handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
const stat = await handle.stat();
if (
!stat.isFile() ||
stat.size < 2 ||
stat.size > MAX_WORKER_REMOTE_OFFER_RECORD_BYTES ||
(stat.mode & 0o077) !== 0
) {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
const bytes = await handle.readFile();
try {
return JSON.parse(bytes.toString('utf8')) as unknown;
} finally {
bytes.fill(0);
}
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
if (error instanceof WorkerRemoteOfferFileJournalError) throw error;
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
} finally {
await handle?.close().catch(() => undefined);
}
}
export class WorkerRemoteOfferFileJournal
implements WorkerRemoteOfferDeliveryJournal, WorkerRemoteExecutionInbox {
private readonly rootDirectory: string;
private readonly offersDirectory: string;
private readonly maximumEntries: number;
private readonly ownershipStaleMs: number;
private releaseOwnershipLock?: () => Promise<void>;
private compromised = false;
private mutationTail: Promise<void> = Promise.resolve();
constructor(options: WorkerRemoteOfferFileJournalOptions) {
if (
!options ||
typeof options.rootDirectory !== 'string' ||
!isAbsolute(options.rootDirectory) ||
options.rootDirectory.length > 4096 ||
/[\0\r\n]/.test(options.rootDirectory)
) {
throw new WorkerRemoteOfferFileJournalError('invalid_configuration');
}
const maximumEntries =
options.maximumEntries ?? DEFAULT_WORKER_REMOTE_OFFER_INBOX_ENTRIES;
if (
!Number.isSafeInteger(maximumEntries) ||
maximumEntries < 1 ||
maximumEntries > MAX_WORKER_REMOTE_OFFER_INBOX_ENTRIES
) {
throw new WorkerRemoteOfferFileJournalError('invalid_configuration');
}
const ownershipStaleMs = options.ownershipStaleMs ?? 30_000;
if (
!Number.isSafeInteger(ownershipStaleMs) ||
ownershipStaleMs < MIN_OWNERSHIP_STALE_MS ||
ownershipStaleMs > MAX_OWNERSHIP_STALE_MS
) {
throw new WorkerRemoteOfferFileJournalError('invalid_configuration');
}
this.rootDirectory = options.rootDirectory;
this.offersDirectory = join(options.rootDirectory, 'offers');
this.maximumEntries = maximumEntries;
this.ownershipStaleMs = ownershipStaleMs;
}
async acquireOwnership(): Promise<void> {
if (this.releaseOwnershipLock) {
throw new WorkerRemoteOfferFileJournalError('already_owned');
}
await safeDirectory(this.rootDirectory);
await safeDirectory(this.offersDirectory);
try {
this.compromised = false;
this.releaseOwnershipLock = await lock(this.rootDirectory, {
stale: this.ownershipStaleMs,
update: Math.floor(this.ownershipStaleMs / 2),
retries: 0,
realpath: true,
lockfilePath: join(this.rootDirectory, '.owner.lock'),
onCompromised: () => {
this.compromised = true;
this.releaseOwnershipLock = undefined;
},
});
} catch {
throw new WorkerRemoteOfferFileJournalError('already_owned');
}
}
async releaseOwnership(): Promise<void> {
this.assertOwned();
const release = this.releaseOwnershipLock!;
this.releaseOwnershipLock = undefined;
await this.mutationTail.catch(() => undefined);
try {
await release();
} catch {
throw new WorkerRemoteOfferFileJournalError('ownership_compromised');
}
}
async readPendingClaim(): Promise<WorkerRemoteOfferClaimRecord | undefined> {
this.assertOwned();
const value = await readJson(join(this.rootDirectory, 'pending-claim.json'));
if (value === undefined) return undefined;
return normalizeWorkerRemoteOfferClaimRecord(
value as WorkerRemoteOfferClaimRecord,
);
}
createPendingClaim(
record: WorkerRemoteOfferClaimRecord,
): Promise<WorkerRemoteOfferClaimRecord> {
return this.mutate(async () => {
const candidate = normalizeWorkerRemoteOfferClaimRecord(record);
const existing = await this.readPendingClaim();
if (existing) {
if (!this.sameClaim(existing, candidate)) {
throw new WorkerRemoteOfferFileJournalError('offer_conflict');
}
return existing;
}
await this.writeFirst(join(this.rootDirectory, 'pending-claim.json'), candidate);
return candidate;
});
}
replacePendingClaim(
record: WorkerRemoteOfferClaimRecord,
expectedRevision: number,
): Promise<WorkerRemoteOfferClaimRecord> {
return this.mutate(async () => {
const candidate = normalizeWorkerRemoteOfferClaimRecord(record);
const existing = await this.readPendingClaim();
if (
!existing ||
existing.revision !== expectedRevision ||
candidate.revision !== expectedRevision + 1 ||
!this.sameClaim(existing, candidate)
) {
throw new WorkerRemoteOfferFileJournalError('claim_revision_conflict');
}
await this.writeReplacement(
join(this.rootDirectory, 'pending-claim.json'),
candidate,
);
return candidate;
});
}
clearPendingClaim(offerId: string, expectedRevision: number): Promise<void> {
return this.mutate(async () => {
assertRunDispatchId('offerId', offerId);
const existing = await this.readPendingClaim();
if (
!existing ||
existing.offerId !== offerId ||
existing.revision !== expectedRevision
) {
throw new WorkerRemoteOfferFileJournalError('claim_revision_conflict');
}
try {
await unlink(join(this.rootDirectory, 'pending-claim.json'));
await syncDirectory(this.rootDirectory);
} catch {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
});
}
acceptOffer(
delivered: ClusterRemoteExecutionOffer,
acceptedAtMs: number,
): Promise<WorkerRemoteOfferInboxAcceptResult> {
return this.mutate(async () => {
const offer = createClusterRemoteExecutionOffer(delivered);
const existing = await this.readOffer(offer.offerId);
if (existing) {
if (!sameWorkerRemoteOfferAuthority(existing.offer, offer)) {
throw new WorkerRemoteOfferFileJournalError('offer_conflict');
}
if (offer.lease.version > existing.offer.lease.version) {
const updated = normalizeWorkerRemoteExecutionInboxRecord({
...existing,
revision: existing.revision + 1,
offer,
updatedAtMs: acceptedAtMs,
});
this.assertOfferTransition(existing, updated);
await this.writeReplacement(this.offerPath(offer.offerId), updated);
return Object.freeze({ status: 'replayed' as const, record: updated });
}
return Object.freeze({ status: 'replayed' as const, record: existing });
}
const names = await this.offerNames();
if (names.length >= this.maximumEntries) {
throw new WorkerRemoteOfferFileJournalError('capacity_exhausted');
}
const record = createWorkerRemoteExecutionInboxRecord(offer, acceptedAtMs);
await this.writeFirst(this.offerPath(offer.offerId), record);
return Object.freeze({ status: 'accepted' as const, record });
});
}
async readOffer(
offerId: string,
): Promise<WorkerRemoteExecutionInboxRecord | undefined> {
this.assertOwned();
const value = await readJson(this.offerPath(offerId));
if (value === undefined) return undefined;
return normalizeWorkerRemoteExecutionInboxRecord(
value as WorkerRemoteExecutionInboxRecord,
);
}
replaceOffer(
record: WorkerRemoteExecutionInboxRecord,
expectedRevision: number,
): Promise<void> {
return this.mutate(async () => {
const candidate = normalizeWorkerRemoteExecutionInboxRecord(record);
const existing = await this.readOffer(candidate.offer.offerId);
if (
!existing ||
existing.revision !== expectedRevision ||
candidate.revision !== expectedRevision + 1
) {
throw new WorkerRemoteOfferFileJournalError('offer_revision_conflict');
}
this.assertOfferTransition(existing, candidate);
await this.writeReplacement(this.offerPath(candidate.offer.offerId), candidate);
});
}
async listOffers(options: Readonly<{
afterOfferId?: string;
limit?: number;
}> = {}): Promise<WorkerRemoteExecutionInboxPage> {
this.assertOwned();
const limit = options.limit ?? 16;
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 64) {
throw new WorkerRemoteOfferFileJournalError('invalid_configuration');
}
if (options.afterOfferId !== undefined) {
this.offerPath(options.afterOfferId);
}
const names = await this.offerNames();
const selected = names
.filter((offerId) =>
options.afterOfferId === undefined || offerId > options.afterOfferId)
.slice(0, limit);
const records: WorkerRemoteExecutionInboxRecord[] = [];
for (const offerId of selected) {
const record = await this.readOffer(offerId);
if (!record) {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
records.push(record);
}
const hasMore = selected.length > 0 &&
names.some((offerId) => offerId > selected[selected.length - 1]!);
return Object.freeze({
records: Object.freeze(records),
...(hasMore
? { nextAfterOfferId: selected[selected.length - 1]! }
: {}),
});
}
private assertOfferTransition(
previous: WorkerRemoteExecutionInboxRecord,
next: WorkerRemoteExecutionInboxRecord,
): void {
try {
assertWorkerRemoteExecutionInboxTransition(previous, next);
} catch (error) {
if (
error instanceof WorkerRemoteExecutionInboxError &&
error.reason === 'revision_conflict'
) {
throw new WorkerRemoteOfferFileJournalError('offer_revision_conflict');
}
if (
error instanceof WorkerRemoteExecutionInboxError &&
error.reason === 'invalid_transition'
) {
throw new WorkerRemoteOfferFileJournalError('invalid_transition');
}
throw new WorkerRemoteOfferFileJournalError('offer_conflict');
}
}
private mutate<T>(operation: () => Promise<T>): Promise<T> {
this.assertOwned();
const result = this.mutationTail.then(operation, operation);
this.mutationTail = result.then(() => undefined, () => undefined);
return result;
}
private assertOwned(): void {
if (this.compromised) {
throw new WorkerRemoteOfferFileJournalError('ownership_compromised');
}
if (!this.releaseOwnershipLock) {
throw new WorkerRemoteOfferFileJournalError('not_owned');
}
}
private sameClaim(
left: WorkerRemoteOfferClaimRecord,
right: WorkerRemoteOfferClaimRecord,
): boolean {
return (
left.workerId === right.workerId &&
left.workerSessionId === right.workerSessionId &&
left.workerGeneration === right.workerGeneration &&
left.offerId === right.offerId &&
left.leaseToken === right.leaseToken
);
}
private offerPath(offerId: string): string {
assertRunDispatchId('offerId', offerId);
if (!/^[A-Za-z0-9._:-]+$/.test(offerId)) {
throw new WorkerRemoteOfferFileJournalError('offer_conflict');
}
return join(this.offersDirectory, `${offerId}.json`);
}
private async offerNames(): Promise<string[]> {
this.assertOwned();
let entries;
try {
entries = await readdir(this.offersDirectory, { withFileTypes: true });
} catch {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
const names: string[] = [];
for (const entry of entries) {
const match = OFFER_FILE.exec(entry.name);
if (!entry.isFile() || !match) {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
names.push(match[1]!);
}
return names.sort();
}
private temporary(target: string): string {
return join(
this.rootDirectory,
`.${target.split('/').at(-1)}.${randomBytes(16).toString('hex')}.tmp`,
);
}
private async writeFirst(target: string, value: unknown): Promise<void> {
const temporary = this.temporary(target);
const bytes = serialize(value);
try {
const handle = await open(
temporary,
constants.O_WRONLY |
constants.O_CREAT |
constants.O_EXCL |
(constants.O_NOFOLLOW ?? 0),
0o600,
);
try {
await handle.writeFile(bytes);
await handle.sync();
await handle.chmod(0o600);
} finally {
await handle.close();
}
await link(temporary, target);
await syncDirectory(target.startsWith(this.offersDirectory)
? this.offersDirectory
: this.rootDirectory);
} catch (error) {
if (isCode(error, 'EEXIST')) {
throw new WorkerRemoteOfferFileJournalError('offer_conflict');
}
if (error instanceof WorkerRemoteOfferFileJournalError) throw error;
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
} finally {
bytes.fill(0);
await rm(temporary, { force: true }).catch(() => undefined);
}
}
private async writeReplacement(target: string, value: unknown): Promise<void> {
const temporary = this.temporary(target);
const bytes = serialize(value);
try {
const handle = await open(
temporary,
constants.O_WRONLY |
constants.O_CREAT |
constants.O_EXCL |
(constants.O_NOFOLLOW ?? 0),
0o600,
);
try {
await handle.writeFile(bytes);
await handle.sync();
await handle.chmod(0o600);
} finally {
await handle.close();
}
await rename(temporary, target);
await syncDirectory(target.startsWith(this.offersDirectory)
? this.offersDirectory
: this.rootDirectory);
} catch (error) {
if (error instanceof WorkerRemoteOfferFileJournalError) throw error;
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
} finally {
bytes.fill(0);
await rm(temporary, { force: true }).catch(() => undefined);
}
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/remoteActivationHttpsClient.ts
+171
View File
@@ -0,0 +1,171 @@
// Remote Execution transport owns starting, running, and start-failure acknowledgements.
import {
assertAcknowledgeRemoteRunRunningCommand,
assertAcknowledgeRemoteRunStartingCommand,
assertFailRemoteRunStartCommand,
type AcknowledgeRemoteRunRunningCommand,
type AcknowledgeRemoteRunStartingCommand,
type FailRemoteRunStartCommand,
type RemoteRunActivationResult,
} from '@qinglong/runtime-core/remote-activation';
import {
MAX_REMOTE_RUN_ACTIVATION_RESPONSE_BYTES,
parseRemoteRunActivationResponse,
} from '@qinglong/runtime-core/remote-activation-delivery';
import type { WorkerRemoteExecutionActivationClient } from '../executionInboxProcessor';
import {
WorkerIngressHttpsClient,
WorkerIngressHttpsClientError,
} from './workerIngressHttpsClient';
type ActivationCommand =
| AcknowledgeRemoteRunStartingCommand
| AcknowledgeRemoteRunRunningCommand
| FailRemoteRunStartCommand;
export interface WorkerRemoteExecutionHttpsActivationClientOptions {
readonly client: WorkerIngressHttpsClient;
}
export class WorkerRemoteExecutionHttpsActivationError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'request_invalid'
| 'transport_unavailable'
| 'response_invalid',
options?: ErrorOptions,
) {
super(`Worker remote execution activation failed: ${reason}`, options);
this.name = 'WorkerRemoteExecutionHttpsActivationError';
}
}
function path(command: ActivationCommand, operation: string): string {
return '/api/v3/worker-ingress/workers/' + command.workerId +
'/sessions/' + command.workerSessionId + '/' + operation;
}
function fenceBody(command: ActivationCommand): Readonly<{
runId: string;
attemptId: string;
workerGeneration: number;
offerId: string;
leaseGeneration: number;
leaseToken: string;
expectedLeaseVersion: number;
}> {
return Object.freeze({
runId: command.runId,
attemptId: command.attemptId,
workerGeneration: command.workerGeneration,
offerId: command.offerId,
leaseGeneration: command.leaseGeneration,
leaseToken: command.leaseToken,
expectedLeaseVersion: command.expectedLeaseVersion,
});
}
export class WorkerRemoteExecutionHttpsActivationClient
implements WorkerRemoteExecutionActivationClient {
private readonly client: WorkerIngressHttpsClient;
constructor(options: WorkerRemoteExecutionHttpsActivationClientOptions) {
if (!options || !(options.client instanceof WorkerIngressHttpsClient)) {
throw new WorkerRemoteExecutionHttpsActivationError(
'invalid_configuration',
);
}
this.client = options.client;
}
async acknowledgeStarting(
command: AcknowledgeRemoteRunStartingCommand,
): Promise<Readonly<RemoteRunActivationResult>> {
try {
assertAcknowledgeRemoteRunStartingCommand(command);
} catch (error) {
throw new WorkerRemoteExecutionHttpsActivationError(
'request_invalid',
{ cause: error },
);
}
return this.exchange('starting', command, fenceBody(command));
}
async acknowledgeRunning(
command: AcknowledgeRemoteRunRunningCommand,
): Promise<Readonly<RemoteRunActivationResult>> {
try {
assertAcknowledgeRemoteRunRunningCommand(command);
} catch (error) {
throw new WorkerRemoteExecutionHttpsActivationError(
'request_invalid',
{ cause: error },
);
}
return this.exchange('running', command, Object.freeze({
...fenceBody(command),
executorHandle: command.executorHandle,
logArtifactId: command.logArtifactId ?? null,
callbackSequence: command.callbackSequence,
callbackTokenDigest: command.callbackTokenDigest,
}));
}
async failStart(
command: FailRemoteRunStartCommand,
): Promise<Readonly<RemoteRunActivationResult>> {
try {
assertFailRemoteRunStartCommand(command);
} catch (error) {
throw new WorkerRemoteExecutionHttpsActivationError(
'request_invalid',
{ cause: error },
);
}
return this.exchange('start-failure', command, fenceBody(command));
}
private async exchange(
operation: 'starting' | 'running' | 'start-failure',
command: ActivationCommand,
body: unknown,
): Promise<Readonly<RemoteRunActivationResult>> {
let serialized: Uint8Array;
try {
serialized = await this.client.postJson({
path: path(command, operation),
body,
maximumResponseBytes: MAX_REMOTE_RUN_ACTIVATION_RESPONSE_BYTES,
});
} catch (error) {
if (error instanceof WorkerIngressHttpsClientError) {
throw new WorkerRemoteExecutionHttpsActivationError(
'transport_unavailable',
{ cause: error },
);
}
throw error;
}
try {
const result = parseRemoteRunActivationResponse(serialized);
if (
result.snapshot.runId !== command.runId ||
result.snapshot.attemptId !== command.attemptId ||
result.snapshot.leaseGeneration !== command.leaseGeneration
) {
throw new TypeError('activation response authority mismatch');
}
return result;
} catch (error) {
throw new WorkerRemoteExecutionHttpsActivationError(
'response_invalid',
{ cause: error },
);
} finally {
Buffer.from(serialized.buffer, serialized.byteOffset, serialized.byteLength)
.fill(0);
}
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/remoteOfferHttpsTransport.ts
+125
View File
@@ -0,0 +1,125 @@
// Remote Execution transport owns bounded offer exchange over the shared mTLS client.
import type { Agent } from 'node:https';
import type { WorkerRemoteOfferTransport } from '../remoteOfferDelivery';
import {
WorkerIngressHttpsClient,
WorkerIngressHttpsClientError,
type WorkerIngressHttpsCredentialProvider,
type WorkerIngressHttpsCredentials,
type WorkerIngressHttpsRequestFactory,
} from './workerIngressHttpsClient';
const OFFER_PATH =
/^\/api\/v3\/worker-ingress\/workers\/[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\/sessions\/[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\/offers$/;
export type WorkerRemoteOfferHttpsCredentials = WorkerIngressHttpsCredentials;
export type WorkerRemoteOfferHttpsCredentialProvider =
WorkerIngressHttpsCredentialProvider;
export type WorkerRemoteOfferHttpsRequestFactory =
WorkerIngressHttpsRequestFactory;
export interface WorkerRemoteOfferHttpsTransportOptions {
readonly client?: WorkerIngressHttpsClient;
readonly origin?: string | URL;
readonly credentials?: WorkerRemoteOfferHttpsCredentialProvider;
readonly requestTimeoutMs?: number;
readonly agent?: Agent;
/** Injectable only for deterministic transport contract tests. */
readonly requestFactory?: WorkerRemoteOfferHttpsRequestFactory;
}
export class WorkerRemoteOfferHttpsTransportError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'credentials_unavailable'
| 'request_rejected'
| 'response_rejected'
| 'response_too_large'
| 'closed',
) {
super(`Worker remote offer HTTPS transport failed: ${reason}`);
this.name = 'WorkerRemoteOfferHttpsTransportError';
}
}
export class WorkerRemoteOfferHttpsTransport
implements WorkerRemoteOfferTransport {
private readonly client: WorkerIngressHttpsClient;
private readonly ownsClient: boolean;
private closed = false;
constructor(options: WorkerRemoteOfferHttpsTransportOptions) {
if (!options) {
throw new WorkerRemoteOfferHttpsTransportError('invalid_configuration');
}
if (options.client) {
if (
options.origin !== undefined ||
options.credentials !== undefined ||
options.requestTimeoutMs !== undefined ||
options.agent !== undefined ||
options.requestFactory !== undefined
) {
throw new WorkerRemoteOfferHttpsTransportError('invalid_configuration');
}
this.client = options.client;
this.ownsClient = false;
return;
}
if (options.origin === undefined || options.credentials === undefined) {
throw new WorkerRemoteOfferHttpsTransportError('invalid_configuration');
}
try {
this.client = new WorkerIngressHttpsClient({
origin: options.origin,
credentials: options.credentials,
...(options.requestTimeoutMs === undefined
? {}
: { requestTimeoutMs: options.requestTimeoutMs }),
...(options.agent === undefined ? {} : { agent: options.agent }),
...(options.requestFactory === undefined
? {}
: { requestFactory: options.requestFactory }),
});
} catch (error) {
if (error instanceof WorkerIngressHttpsClientError) {
throw new WorkerRemoteOfferHttpsTransportError(error.reason);
}
throw error;
}
this.ownsClient = true;
}
async exchange(request: Readonly<{
path: string;
body: Readonly<{
workerGeneration: number;
offerId: string;
leaseToken: string;
}>;
maximumResponseBytes: number;
signal?: AbortSignal;
}>): Promise<Uint8Array> {
if (this.closed) {
throw new WorkerRemoteOfferHttpsTransportError('closed');
}
if (!request || typeof request.path !== 'string' || !OFFER_PATH.test(request.path)) {
throw new WorkerRemoteOfferHttpsTransportError('request_rejected');
}
try {
return await this.client.postJson(request);
} catch (error) {
if (error instanceof WorkerIngressHttpsClientError) {
throw new WorkerRemoteOfferHttpsTransportError(error.reason);
}
throw error;
}
}
close(): void {
if (this.closed) return;
this.closed = true;
if (this.ownsClient) this.client.close();
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/remoteSecretHttpsProvider.ts
+138
View File
@@ -0,0 +1,138 @@
// Remote Execution transport owns capability-bound Secret delivery.
import {
MAX_REMOTE_SECRET_DELIVERY_REQUEST_BYTES,
MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES,
createRemoteWorkerSecretDeliveryRequestBody,
parseRemoteWorkerSecretDeliveryResponse,
} from '@qinglong/runtime-core/remote-secret-delivery';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import type { WorkerRemoteExecutionInbox } from '../executionInbox';
import type {
WorkerRemoteSecretEnvironmentProvider,
WorkerRemoteSecretResolution,
} from '../executionContextMaterializer';
import {
WorkerIngressHttpsClient,
type WorkerIngressHttpsPostRequest,
} from './workerIngressHttpsClient';
export class WorkerRemoteSecretHttpsProviderError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'offer_unavailable'
| 'authority_mismatch'
| 'delivery_unavailable'
| 'response_invalid',
) {
super(`Worker remote Secret HTTPS provider failed: ${reason}`);
this.name = 'WorkerRemoteSecretHttpsProviderError';
}
}
export interface WorkerRemoteSecretHttpsProviderOptions {
readonly client: Pick<WorkerIngressHttpsClient, 'postJson'>;
readonly inbox: Pick<WorkerRemoteExecutionInbox, 'readOffer'>;
}
export class WorkerRemoteSecretHttpsProvider
implements WorkerRemoteSecretEnvironmentProvider {
private readonly client: Pick<WorkerIngressHttpsClient, 'postJson'>;
private readonly inbox: Pick<WorkerRemoteExecutionInbox, 'readOffer'>;
constructor(options: WorkerRemoteSecretHttpsProviderOptions) {
if (
!options ||
typeof options.client?.postJson !== 'function' ||
typeof options.inbox?.readOffer !== 'function'
) throw new WorkerRemoteSecretHttpsProviderError('invalid_configuration');
this.client = options.client;
this.inbox = options.inbox;
}
async resolve(request: Parameters<WorkerRemoteSecretEnvironmentProvider['resolve']>[0])
: Promise<WorkerRemoteSecretResolution | undefined> {
let record;
try {
record = await this.inbox.readOffer(request.offerId);
} catch {
throw new WorkerRemoteSecretHttpsProviderError('offer_unavailable');
}
if (!record || record.state !== 'starting_acknowledged') {
throw new WorkerRemoteSecretHttpsProviderError('offer_unavailable');
}
let offer;
try {
offer = createClusterRemoteExecutionOffer(record.offer);
} catch {
throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
}
const expectedRefs = Object.freeze([
...new Set(offer.executionRevision.environment.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [])),
]);
if (
offer.offerId !== request.offerId ||
offer.executionDigest !== request.executionDigest ||
offer.candidate.projectId !== request.projectId ||
offer.candidate.taskId !== request.taskId ||
offer.candidate.taskRevision !== request.taskRevision ||
offer.candidate.runId !== request.runId ||
offer.candidate.attemptId !== request.attemptId ||
JSON.stringify(expectedRefs) !== JSON.stringify(request.secretRefs)
) throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
const path = `/api/v3/worker-ingress/workers/${offer.worker.workerId}` +
`/sessions/${offer.worker.sessionId}/secrets`;
const body = createRemoteWorkerSecretDeliveryRequestBody({
workerId: offer.worker.workerId,
workerSessionId: offer.worker.sessionId,
workerGeneration: offer.worker.generation,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
projectId: offer.candidate.projectId,
taskId: offer.candidate.taskId,
taskRevision: offer.candidate.taskRevision,
executionDigest: offer.executionDigest,
offerId: offer.offerId,
leaseGeneration: offer.lease.leaseGeneration,
leaseToken: offer.leaseToken,
expectedLeaseVersion: offer.lease.version,
secretRefs: expectedRefs,
});
let serialized: Uint8Array;
try {
const transportRequest: WorkerIngressHttpsPostRequest = {
path,
body,
maximumRequestBytes: MAX_REMOTE_SECRET_DELIVERY_REQUEST_BYTES,
maximumResponseBytes: MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES,
};
serialized = await this.client.postJson(transportRequest);
} catch {
throw new WorkerRemoteSecretHttpsProviderError('delivery_unavailable');
}
try {
const delivered = parseRemoteWorkerSecretDeliveryResponse(serialized, {
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs: expectedRefs,
});
const values = Object.freeze(delivered.values.map((entry) =>
Object.freeze({ secretRef: entry.secretRef, value: entry.value })));
return Object.freeze({
values,
dispose() {
// JavaScript strings cannot be zeroized. Drop all retained references;
// the transport bytes were already scrubbed by the parser.
},
});
} catch {
throw new WorkerRemoteSecretHttpsProviderError('response_invalid');
} finally {
if (Buffer.isBuffer(serialized)) serialized.fill(0);
}
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/remoteWorkerCompletionHttpsClient.ts
+214
View File
@@ -0,0 +1,214 @@
// Remote Execution transport owns Artifact upload and completion acknowledgement.
import {
MAX_REMOTE_WORKER_ARTIFACT_RESPONSE_BYTES,
MAX_REMOTE_WORKER_COMPLETION_REQUEST_BYTES,
MAX_REMOTE_WORKER_COMPLETION_RESPONSE_BYTES,
createRemoteWorkerArtifactUploadPreamble,
createRemoteWorkerCompletionRequestBody,
parseRemoteWorkerArtifactUploadResponse,
parseRemoteWorkerCompletionResponse,
type RemoteWorkerCompletionCommand,
} from '@qinglong/runtime-core/remote-worker-completion';
import type {
WorkerRemoteExecutionCompletionClient,
WorkerRemoteExecutionCompletionCommand,
WorkerRemoteExecutionCompletionResult,
WorkerRemoteLogArtifactUploadCommand,
WorkerRemoteLogArtifactUploadResult,
WorkerRemoteLogArtifactUploader,
} from '../../execution/workerCompletionCoordinator';
import type { WorkerIngressHttpsClient } from './workerIngressHttpsClient';
export class WorkerRemoteCompletionHttpsError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'request_invalid'
| 'transport_unavailable'
| 'response_invalid',
options?: ErrorOptions,
) {
super(`Worker remote completion HTTPS failed: ${reason}`, options);
this.name = 'WorkerRemoteCompletionHttpsError';
}
}
export interface WorkerRemoteArtifactHttpsUploaderOptions {
readonly client: Pick<WorkerIngressHttpsClient, 'postStream'>;
}
export interface WorkerRemoteExecutionHttpsCompletionClientOptions {
readonly client: Pick<WorkerIngressHttpsClient, 'postJson'>;
}
function path(
command: Readonly<{ workerId: string; workerSessionId: string }>,
operation: 'artifacts' | 'completion',
): string {
return `/api/v3/worker-ingress/workers/${command.workerId}` +
`/sessions/${command.workerSessionId}/${operation}`;
}
function erase(value: Uint8Array | undefined): void {
if (!value) return;
Buffer.from(value.buffer, value.byteOffset, value.byteLength).fill(0);
}
export class WorkerRemoteArtifactHttpsUploader
implements WorkerRemoteLogArtifactUploader {
private readonly client: Pick<WorkerIngressHttpsClient, 'postStream'>;
constructor(options: WorkerRemoteArtifactHttpsUploaderOptions) {
if (!options || typeof options.client?.postStream !== 'function') {
throw new WorkerRemoteCompletionHttpsError('invalid_configuration');
}
this.client = options.client;
}
async upload(
command: WorkerRemoteLogArtifactUploadCommand,
): Promise<Readonly<WorkerRemoteLogArtifactUploadResult>> {
let preamble: Buffer;
try {
preamble = createRemoteWorkerArtifactUploadPreamble({
workerId: command.workerId,
workerSessionId: command.workerSessionId,
workerGeneration: command.workerGeneration,
projectId: command.projectId,
runId: command.runId,
attemptId: command.attemptId,
offerId: command.offerId,
leaseGeneration: command.leaseGeneration,
leaseToken: command.leaseToken,
expectedLeaseVersion: command.expectedLeaseVersion,
logArtifactId: command.logArtifactId,
byteLength: command.byteLength,
...(command.truncated === undefined
? {}
: { truncated: command.truncated }),
});
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('request_invalid', {
cause: error,
});
}
let serialized: Uint8Array | undefined;
try {
const content = command.content;
serialized = await this.client.postStream({
path: path(command, 'artifacts'),
body: (async function* () {
yield preamble;
for await (const chunk of content) yield chunk;
})(),
byteLength: preamble.byteLength + command.byteLength,
maximumResponseBytes: MAX_REMOTE_WORKER_ARTIFACT_RESPONSE_BYTES,
});
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('transport_unavailable', {
cause: error,
});
} finally {
preamble.fill(0);
}
try {
const receipt = parseRemoteWorkerArtifactUploadResponse(serialized);
if (
receipt.projectId !== command.projectId ||
receipt.runId !== command.runId ||
receipt.attemptId !== command.attemptId ||
receipt.logArtifactId !== command.logArtifactId ||
receipt.byteLength !== command.byteLength ||
receipt.truncated !== command.truncated
) {
throw new TypeError('Artifact response authority does not match');
}
return Object.freeze({
status: receipt.status,
logArtifactId: receipt.logArtifactId,
byteLength: receipt.byteLength,
sha256: receipt.sha256,
});
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('response_invalid', {
cause: error,
});
} finally {
erase(serialized);
}
}
}
export class WorkerRemoteExecutionHttpsCompletionClient
implements WorkerRemoteExecutionCompletionClient {
private readonly client: Pick<WorkerIngressHttpsClient, 'postJson'>;
constructor(options: WorkerRemoteExecutionHttpsCompletionClientOptions) {
if (!options || typeof options.client?.postJson !== 'function') {
throw new WorkerRemoteCompletionHttpsError('invalid_configuration');
}
this.client = options.client;
}
async complete(
command: WorkerRemoteExecutionCompletionCommand,
): Promise<Readonly<WorkerRemoteExecutionCompletionResult>> {
let body;
try {
if (command.executorType !== 'remote_worker') {
throw new TypeError('completion executor type is invalid');
}
const wire: RemoteWorkerCompletionCommand = {
workerId: command.workerId,
workerSessionId: command.workerSessionId,
workerGeneration: command.workerGeneration,
projectId: command.projectId,
runId: command.runId,
attemptId: command.attemptId,
offerId: command.offerId,
leaseGeneration: command.leaseGeneration,
leaseToken: command.leaseToken,
expectedLeaseVersion: command.expectedLeaseVersion,
callbackSequence: command.callbackSequence,
callbackTokenDigest: command.callbackTokenDigest,
result: command.result,
artifact: command.artifact,
};
body = createRemoteWorkerCompletionRequestBody(wire);
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('request_invalid', {
cause: error,
});
}
let serialized: Uint8Array | undefined;
try {
serialized = await this.client.postJson({
path: path(command, 'completion'),
body,
maximumRequestBytes: MAX_REMOTE_WORKER_COMPLETION_REQUEST_BYTES,
maximumResponseBytes: MAX_REMOTE_WORKER_COMPLETION_RESPONSE_BYTES,
});
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('transport_unavailable', {
cause: error,
});
}
try {
const completed = parseRemoteWorkerCompletionResponse(serialized);
if (
completed.runId !== command.runId ||
completed.attemptId !== command.attemptId ||
completed.callbackSequence !== command.callbackSequence
) {
throw new TypeError('completion response authority does not match');
}
return completed;
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('response_invalid', {
cause: error,
});
} finally {
erase(serialized);
}
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/remoteWorkerLeaseControlHttpsClient.ts
+103
View File
@@ -0,0 +1,103 @@
// Remote Execution transport owns fenced lease-control exchange.
import {
MAX_REMOTE_WORKER_LEASE_CONTROL_REQUEST_BYTES,
MAX_REMOTE_WORKER_LEASE_CONTROL_RESPONSE_BYTES,
createRemoteWorkerLeaseControlRequestBody,
parseRemoteWorkerLeaseControlResponse,
type RemoteWorkerLeaseControlCommand,
type RemoteWorkerLeaseControlResult,
} from '@qinglong/runtime-core/remote-worker-lease-control';
import {
WorkerIngressHttpsClient,
WorkerIngressHttpsClientError,
} from './workerIngressHttpsClient';
export interface WorkerRemoteLeaseControlClient {
control(
command: RemoteWorkerLeaseControlCommand,
): Promise<Readonly<RemoteWorkerLeaseControlResult>>;
}
export interface WorkerRemoteLeaseControlHttpsClientOptions {
readonly client: WorkerIngressHttpsClient;
}
export class WorkerRemoteLeaseControlHttpsError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'request_invalid'
| 'transport_unavailable'
| 'response_invalid',
options?: ErrorOptions,
) {
super(`Worker remote lease control failed: ${reason}`, options);
this.name = 'WorkerRemoteLeaseControlHttpsError';
}
}
function path(command: RemoteWorkerLeaseControlCommand): string {
return '/api/v3/worker-ingress/workers/' + command.workerId +
'/sessions/' + command.workerSessionId + '/lease-control';
}
export class WorkerRemoteLeaseControlHttpsClient
implements WorkerRemoteLeaseControlClient {
private readonly client: WorkerIngressHttpsClient;
constructor(options: WorkerRemoteLeaseControlHttpsClientOptions) {
if (!options || !(options.client instanceof WorkerIngressHttpsClient)) {
throw new WorkerRemoteLeaseControlHttpsError('invalid_configuration');
}
this.client = options.client;
}
async control(
command: RemoteWorkerLeaseControlCommand,
): Promise<Readonly<RemoteWorkerLeaseControlResult>> {
let body;
try {
body = createRemoteWorkerLeaseControlRequestBody(command);
} catch (error) {
throw new WorkerRemoteLeaseControlHttpsError(
'request_invalid', { cause: error },
);
}
let serialized: Uint8Array;
try {
serialized = await this.client.postJson({
path: path(command),
body,
maximumRequestBytes: MAX_REMOTE_WORKER_LEASE_CONTROL_REQUEST_BYTES,
maximumResponseBytes: MAX_REMOTE_WORKER_LEASE_CONTROL_RESPONSE_BYTES,
});
} catch (error) {
if (error instanceof WorkerIngressHttpsClientError) {
throw new WorkerRemoteLeaseControlHttpsError(
'transport_unavailable', { cause: error },
);
}
throw error;
}
try {
const result = parseRemoteWorkerLeaseControlResponse(serialized);
if (
result.projectId !== command.projectId ||
result.runId !== command.runId ||
result.attemptId !== command.attemptId ||
result.offerId !== command.offerId ||
result.leaseGeneration !== command.leaseGeneration ||
(result.status !== 'terminal' &&
result.leaseVersion !== command.expectedLeaseVersion + 1)
) throw new TypeError('lease control response authority mismatch');
return result;
} catch (error) {
throw new WorkerRemoteLeaseControlHttpsError(
'response_invalid', { cause: error },
);
} finally {
Buffer.from(serialized.buffer, serialized.byteOffset, serialized.byteLength)
.fill(0);
}
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/workerIngressHttpsClient.ts
+641
View File
@@ -0,0 +1,641 @@
// Remote Execution transport owns the shared bounded TLS 1.3 Worker Ingress client.
import { Agent, request as nodeHttpsRequest } from 'node:https';
import type { RequestOptions } from 'node:https';
import type { ClientRequest, IncomingMessage } from 'node:http';
import { createHash } from 'node:crypto';
import { once } from 'node:events';
import { isIP } from 'node:net';
import { REMOTE_WORKER_ARTIFACT_CONTENT_TYPE } from '@qinglong/runtime-core/remote-worker-completion';
const AUTHORIZATION =
/^Worker ql3w_([A-Za-z0-9][A-Za-z0-9._:-]{0,63})_([A-Za-z0-9_-]{43})$/;
const WORKER_INGRESS_JSON_PATH =
/^\/api\/v3\/worker-ingress\/workers\/[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\/sessions\/[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\/(register|heartbeat|transition|offers|starting|running|start-failure|secrets|completion|lease-control)$/;
const WORKER_INGRESS_ARTIFACT_PATH =
/^\/api\/v3\/worker-ingress\/workers\/[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\/sessions\/[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\/artifacts$/;
const MAX_TLS_MATERIAL_BYTES = 1024 * 1024;
const MAX_REQUEST_BYTES = 4096;
const HARD_MAX_REQUEST_BYTES = 64 * 1024;
const HARD_MAX_STREAM_REQUEST_BYTES = 64 * 1024 * 1024 + 4 * 1024 + 4;
const MAX_RESPONSE_BYTES = 128 * 1024;
const CREDENTIAL_POOL_KEY = Symbol('qinglong.worker-ingress-credential-pool-key');
export const WORKER_INGRESS_ARTIFACT_CONTENT_TYPE =
REMOTE_WORKER_ARTIFACT_CONTENT_TYPE;
export interface WorkerIngressHttpsCredentials {
readonly authorization: string;
readonly certificateChainPem: string | Buffer;
readonly privateKeyPem: string | Buffer;
readonly trustAnchors: readonly (string | Buffer)[];
/** Erases provider-owned transient material after the client copies it. */
readonly dispose?: () => void;
}
export interface WorkerIngressHttpsCredentialProvider {
load(signal?: AbortSignal): Promise<WorkerIngressHttpsCredentials>;
}
export type WorkerIngressHttpsRequestFactory = (
options: RequestOptions,
callback: (response: IncomingMessage) => void,
) => ClientRequest;
export interface WorkerIngressHttpsClientOptions {
readonly origin: string | URL;
readonly credentials: WorkerIngressHttpsCredentialProvider;
readonly requestTimeoutMs?: number;
readonly agent?: Agent;
/** Injectable only for deterministic transport contract tests. */
readonly requestFactory?: WorkerIngressHttpsRequestFactory;
}
export interface WorkerIngressHttpsPostRequest {
readonly path: string;
readonly body: unknown;
readonly maximumResponseBytes: number;
/** Defaults to 4 KiB. Larger budgets are opt-in and capped at 64 KiB. */
readonly maximumRequestBytes?: number;
readonly signal?: AbortSignal;
}
export interface WorkerIngressHttpsStreamRequest {
readonly path: string;
readonly body: AsyncIterable<Uint8Array>;
readonly byteLength: number;
readonly maximumResponseBytes: number;
readonly signal?: AbortSignal;
}
export class WorkerIngressHttpsClientError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'credentials_unavailable'
| 'request_rejected'
| 'response_rejected'
| 'response_too_large'
| 'closed',
readonly httpStatus?: number,
) {
super(`Worker ingress HTTPS client failed: ${reason}`);
this.name = 'WorkerIngressHttpsClientError';
}
}
function boundedMaterial(value: string | Buffer): Buffer {
const bytes = Buffer.isBuffer(value)
? Buffer.from(value)
: Buffer.from(value, 'utf8');
if (bytes.byteLength < 1 || bytes.byteLength > MAX_TLS_MATERIAL_BYTES) {
bytes.fill(0);
throw new WorkerIngressHttpsClientError('credentials_unavailable');
}
return bytes;
}
function normalizeOrigin(value: string | URL): URL {
let origin: URL;
try {
origin = new URL(value);
} catch {
throw new WorkerIngressHttpsClientError('invalid_configuration');
}
if (
origin.protocol !== 'https:' ||
origin.username !== '' ||
origin.password !== '' ||
origin.pathname !== '/' ||
origin.search !== '' ||
origin.hash !== ''
) {
throw new WorkerIngressHttpsClientError('invalid_configuration');
}
return origin;
}
interface WorkerIngressHttpsCredentialMaterial {
readonly authorization: string;
readonly certificate: Buffer;
readonly privateKey: Buffer;
readonly trustAnchors: readonly Buffer[];
readonly poolKey: string;
}
interface WorkerIngressHttpsAgentRequestOptions extends RequestOptions {
readonly [CREDENTIAL_POOL_KEY]?: string;
}
class WorkerIngressHttpsAgent extends Agent {
override getName(options: RequestOptions = {}): string {
const poolKey = (options as WorkerIngressHttpsAgentRequestOptions)[
CREDENTIAL_POOL_KEY
];
if (poolKey === undefined) return super.getName(options);
// Node recomputes the HTTPS pool name when a socket becomes free. The
// request-local TLS Buffers have already been erased by then, so their
// mutable contents cannot safely participate in that name.
return `${super.getName({
...options,
ca: undefined,
cert: undefined,
key: undefined,
})}:qinglong:${poolKey}`;
}
}
function credentialPoolKey(
certificate: Buffer,
privateKey: Buffer,
trustAnchors: readonly Buffer[],
): string {
const hash = createHash('sha256');
for (const value of [certificate, privateKey, ...trustAnchors]) {
const length = Buffer.allocUnsafe(4);
length.writeUInt32BE(value.byteLength);
hash.update(length);
hash.update(value);
length.fill(0);
}
return hash.digest('base64url');
}
async function loadCredentialMaterial(
provider: WorkerIngressHttpsCredentialProvider,
signal?: AbortSignal,
): Promise<WorkerIngressHttpsCredentialMaterial> {
let loaded: WorkerIngressHttpsCredentials;
try {
loaded = await provider.load(signal);
} catch {
if (signal?.aborted) {
throw signal.reason ??
new WorkerIngressHttpsClientError('request_rejected');
}
throw new WorkerIngressHttpsClientError('credentials_unavailable');
}
let certificate: Buffer | undefined;
let privateKey: Buffer | undefined;
const trustAnchors: Buffer[] = [];
let failure: unknown;
try {
if (
!loaded ||
!AUTHORIZATION.test(loaded.authorization) ||
!Array.isArray(loaded.trustAnchors) ||
loaded.trustAnchors.length < 1 ||
loaded.trustAnchors.length > 8 ||
(loaded.dispose !== undefined && typeof loaded.dispose !== 'function')
) {
throw new WorkerIngressHttpsClientError('credentials_unavailable');
}
certificate = boundedMaterial(loaded.certificateChainPem);
privateKey = boundedMaterial(loaded.privateKeyPem);
for (const anchor of loaded.trustAnchors) {
trustAnchors.push(boundedMaterial(anchor));
}
} catch (error) {
failure = error;
}
try {
loaded?.dispose?.();
} catch {
failure ??= new WorkerIngressHttpsClientError('credentials_unavailable');
}
if (failure !== undefined || !certificate || !privateKey) {
certificate?.fill(0);
privateKey?.fill(0);
trustAnchors.forEach((value) => value.fill(0));
if (failure instanceof WorkerIngressHttpsClientError) throw failure;
throw new WorkerIngressHttpsClientError('credentials_unavailable');
}
return {
authorization: loaded.authorization,
certificate,
privateKey,
trustAnchors: Object.freeze(trustAnchors),
poolKey: credentialPoolKey(certificate, privateKey, trustAnchors),
};
}
function eraseCredentialMaterial(
material: WorkerIngressHttpsCredentialMaterial,
): void {
material.certificate.fill(0);
material.privateKey.fill(0);
material.trustAnchors.forEach((value) => value.fill(0));
}
export class WorkerIngressHttpsClient {
private readonly origin: URL;
private readonly credentials: WorkerIngressHttpsCredentialProvider;
private readonly requestTimeoutMs: number;
private readonly requestFactory: WorkerIngressHttpsRequestFactory;
private readonly agent: Agent;
private readonly ownsAgent: boolean;
private closed = false;
constructor(options: WorkerIngressHttpsClientOptions) {
if (
!options ||
typeof options.credentials?.load !== 'function' ||
(options.requestFactory !== undefined &&
typeof options.requestFactory !== 'function')
) {
throw new WorkerIngressHttpsClientError('invalid_configuration');
}
const requestTimeoutMs = options.requestTimeoutMs ?? 15_000;
if (
!Number.isSafeInteger(requestTimeoutMs) ||
requestTimeoutMs < 1_000 ||
requestTimeoutMs > 60_000
) {
throw new WorkerIngressHttpsClientError('invalid_configuration');
}
this.origin = normalizeOrigin(options.origin);
this.credentials = options.credentials;
this.requestTimeoutMs = requestTimeoutMs;
this.requestFactory = options.requestFactory ?? nodeHttpsRequest;
this.ownsAgent = options.agent === undefined;
this.agent = options.agent ?? new WorkerIngressHttpsAgent({
keepAlive: true,
maxSockets: 1,
maxFreeSockets: 1,
timeout: requestTimeoutMs,
});
}
async postJson(request: WorkerIngressHttpsPostRequest): Promise<Uint8Array> {
if (this.closed) throw new WorkerIngressHttpsClientError('closed');
if (
!request ||
typeof request.path !== 'string' ||
!WORKER_INGRESS_JSON_PATH.test(request.path) ||
!Number.isSafeInteger(request.maximumResponseBytes) ||
request.maximumResponseBytes < 2 ||
request.maximumResponseBytes > MAX_RESPONSE_BYTES ||
(request.maximumRequestBytes !== undefined &&
(!Number.isSafeInteger(request.maximumRequestBytes) ||
request.maximumRequestBytes < 2 ||
request.maximumRequestBytes > HARD_MAX_REQUEST_BYTES))
) {
throw new WorkerIngressHttpsClientError('request_rejected');
}
if (request.signal?.aborted) {
throw request.signal.reason ??
new WorkerIngressHttpsClientError('request_rejected');
}
const material = await loadCredentialMaterial(
this.credentials,
request.signal,
);
let body: Buffer;
try {
body = Buffer.from(JSON.stringify(request.body), 'utf8');
} catch {
eraseCredentialMaterial(material);
throw new WorkerIngressHttpsClientError('request_rejected');
}
const maximumRequestBytes = request.maximumRequestBytes ?? MAX_REQUEST_BYTES;
if (body.byteLength < 2 || body.byteLength > maximumRequestBytes) {
eraseCredentialMaterial(material);
body.fill(0);
throw new WorkerIngressHttpsClientError('request_rejected');
}
try {
return await this.perform(
request.path,
body,
request.maximumResponseBytes,
material.authorization,
material.certificate,
material.privateKey,
material.trustAnchors,
material.poolKey,
request.signal,
);
} finally {
eraseCredentialMaterial(material);
body.fill(0);
}
}
async postStream(
request: WorkerIngressHttpsStreamRequest,
): Promise<Uint8Array> {
if (this.closed) throw new WorkerIngressHttpsClientError('closed');
if (
!request ||
typeof request.path !== 'string' ||
!WORKER_INGRESS_ARTIFACT_PATH.test(request.path) ||
!request.body ||
typeof request.body[Symbol.asyncIterator] !== 'function' ||
!Number.isSafeInteger(request.byteLength) ||
request.byteLength < 1 ||
request.byteLength > HARD_MAX_STREAM_REQUEST_BYTES ||
!Number.isSafeInteger(request.maximumResponseBytes) ||
request.maximumResponseBytes < 2 ||
request.maximumResponseBytes > MAX_RESPONSE_BYTES
) {
throw new WorkerIngressHttpsClientError('request_rejected');
}
if (request.signal?.aborted) {
throw request.signal.reason ??
new WorkerIngressHttpsClientError('request_rejected');
}
const material = await loadCredentialMaterial(
this.credentials,
request.signal,
);
try {
return await this.performStream(
request.path,
request.body,
request.byteLength,
request.maximumResponseBytes,
material.authorization,
material.certificate,
material.privateKey,
material.trustAnchors,
material.poolKey,
request.signal,
);
} finally {
eraseCredentialMaterial(material);
}
}
close(): void {
if (this.closed) return;
this.closed = true;
if (this.ownsAgent) this.agent.destroy();
}
private perform(
path: string,
body: Buffer,
maximumResponseBytes: number,
authorization: string,
certificate: Buffer,
privateKey: Buffer,
trustAnchors: readonly Buffer[],
poolKey: string,
signal?: AbortSignal,
): Promise<Uint8Array> {
return new Promise((resolve, reject) => {
let settled = false;
const settle = (error?: unknown, bytes?: Buffer): void => {
if (settled) return;
settled = true;
signal?.removeEventListener('abort', abort);
if (error) reject(error);
else resolve(bytes!);
};
let clientRequest: ClientRequest;
const abort = (): void => {
clientRequest.destroy(
signal?.reason instanceof Error
? signal.reason
: new WorkerIngressHttpsClientError('request_rejected'),
);
};
try {
clientRequest = this.requestFactory({
protocol: 'https:',
hostname: this.origin.hostname,
port: this.origin.port || 443,
...(isIP(this.origin.hostname) === 0
? { servername: this.origin.hostname }
: {}),
method: 'POST',
path,
agent: this.agent,
minVersion: 'TLSv1.3',
rejectUnauthorized: true,
cert: certificate,
key: privateKey,
ca: [...trustAnchors],
[CREDENTIAL_POOL_KEY]: poolKey,
headers: {
accept: 'application/json',
authorization,
'content-type': 'application/json',
'content-length': String(body.byteLength),
},
} as WorkerIngressHttpsAgentRequestOptions, (response) => {
const contentType = response.headers['content-type'];
const contentEncoding = response.headers['content-encoding'];
const contentLength = response.headers['content-length'];
if (
response.statusCode !== 200 ||
typeof contentType !== 'string' ||
!/^application\/json(?:\s*;|$)/i.test(contentType) ||
(contentEncoding !== undefined && contentEncoding !== 'identity') ||
(contentLength !== undefined &&
(!/^\d+$/.test(contentLength) ||
Number(contentLength) > maximumResponseBytes))
) {
response.resume();
settle(new WorkerIngressHttpsClientError(
'response_rejected', response.statusCode,
));
return;
}
const chunks: Buffer[] = [];
let total = 0;
response.on('data', (chunk: Buffer | string) => {
if (settled) return;
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
total += bytes.byteLength;
if (total > maximumResponseBytes) {
chunks.forEach((value) => value.fill(0));
response.destroy();
settle(new WorkerIngressHttpsClientError('response_too_large'));
return;
}
chunks.push(Buffer.from(bytes));
});
response.once('end', () => {
if (settled) return;
const result = Buffer.concat(chunks, total);
chunks.forEach((value) => value.fill(0));
if (result.byteLength < 2) {
result.fill(0);
settle(new WorkerIngressHttpsClientError('response_rejected'));
return;
}
settle(undefined, result);
});
response.once('error', () => {
chunks.forEach((value) => value.fill(0));
settle(new WorkerIngressHttpsClientError(
'response_rejected', response.statusCode,
));
});
});
} catch {
settle(new WorkerIngressHttpsClientError('request_rejected'));
return;
}
clientRequest.once('error', (error) => {
if (signal?.aborted) settle(signal.reason ?? error);
else settle(new WorkerIngressHttpsClientError('request_rejected'));
});
clientRequest.setTimeout(this.requestTimeoutMs, () => {
clientRequest.destroy(
new WorkerIngressHttpsClientError('request_rejected'),
);
});
signal?.addEventListener('abort', abort, { once: true });
clientRequest.end(body);
});
}
private performStream(
path: string,
body: AsyncIterable<Uint8Array>,
byteLength: number,
maximumResponseBytes: number,
authorization: string,
certificate: Buffer,
privateKey: Buffer,
trustAnchors: readonly Buffer[],
poolKey: string,
signal?: AbortSignal,
): Promise<Uint8Array> {
return new Promise((resolve, reject) => {
let settled = false;
const settle = (error?: unknown, bytes?: Buffer): void => {
if (settled) return;
settled = true;
signal?.removeEventListener('abort', abort);
if (error) reject(error);
else resolve(bytes!);
};
let clientRequest: ClientRequest;
const abort = (): void => {
clientRequest.destroy(
signal?.reason instanceof Error
? signal.reason
: new WorkerIngressHttpsClientError('request_rejected'),
);
};
try {
clientRequest = this.requestFactory({
protocol: 'https:',
hostname: this.origin.hostname,
port: this.origin.port || 443,
...(isIP(this.origin.hostname) === 0
? { servername: this.origin.hostname }
: {}),
method: 'POST',
path,
agent: this.agent,
minVersion: 'TLSv1.3',
rejectUnauthorized: true,
cert: certificate,
key: privateKey,
ca: [...trustAnchors],
[CREDENTIAL_POOL_KEY]: poolKey,
headers: {
accept: 'application/json',
authorization,
'content-type': WORKER_INGRESS_ARTIFACT_CONTENT_TYPE,
'content-length': String(byteLength),
},
} as WorkerIngressHttpsAgentRequestOptions, (response) => {
const contentType = response.headers['content-type'];
const contentEncoding = response.headers['content-encoding'];
const contentLength = response.headers['content-length'];
if (
response.statusCode !== 200 ||
typeof contentType !== 'string' ||
!/^application\/json(?:\s*;|$)/i.test(contentType) ||
(contentEncoding !== undefined && contentEncoding !== 'identity') ||
(contentLength !== undefined &&
(!/^\d+$/.test(contentLength) ||
Number(contentLength) > maximumResponseBytes))
) {
response.resume();
clientRequest.destroy();
settle(new WorkerIngressHttpsClientError('response_rejected'));
return;
}
const chunks: Buffer[] = [];
let total = 0;
response.on('data', (chunk: Buffer | string) => {
if (settled) return;
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
total += bytes.byteLength;
if (total > maximumResponseBytes) {
chunks.forEach((value) => value.fill(0));
response.destroy();
settle(new WorkerIngressHttpsClientError('response_too_large'));
return;
}
chunks.push(Buffer.from(bytes));
});
response.once('end', () => {
if (settled) return;
const result = Buffer.concat(chunks, total);
chunks.forEach((value) => value.fill(0));
if (result.byteLength < 2) {
result.fill(0);
settle(new WorkerIngressHttpsClientError('response_rejected'));
return;
}
settle(undefined, result);
});
response.once('error', () => {
chunks.forEach((value) => value.fill(0));
settle(new WorkerIngressHttpsClientError('response_rejected'));
});
});
} catch {
settle(new WorkerIngressHttpsClientError('request_rejected'));
return;
}
clientRequest.once('error', (error) => {
if (signal?.aborted) settle(signal.reason ?? error);
else settle(new WorkerIngressHttpsClientError('request_rejected'));
});
clientRequest.setTimeout(this.requestTimeoutMs, () => {
clientRequest.destroy(
new WorkerIngressHttpsClientError('request_rejected'),
);
});
signal?.addEventListener('abort', abort, { once: true });
void (async () => {
let total = 0;
for await (const chunk of body) {
if (settled) return;
if (signal?.aborted) throw signal.reason;
if (!(chunk instanceof Uint8Array)) {
throw new WorkerIngressHttpsClientError('request_rejected');
}
total += chunk.byteLength;
if (total > byteLength) {
throw new WorkerIngressHttpsClientError('request_rejected');
}
if (!clientRequest.write(chunk)) {
await once(
clientRequest,
'drain',
signal ? { signal } : undefined,
);
}
}
if (total !== byteLength) {
throw new WorkerIngressHttpsClientError('request_rejected');
}
clientRequest.end();
})().catch((error: unknown) => {
clientRequest.destroy(
error instanceof Error
? error
: new WorkerIngressHttpsClientError('request_rejected'),
);
});
});
}
}