open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-20 16:07:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): establish 3.0 incubation baseline

Browse Source
This commit is contained in:
whyour
2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-worker-runtime/src/application-runtime/productionHeadlessApplication.ts
+709
View File
@@ -0,0 +1,709 @@
import path from 'node:path';
import { performance } from 'node:perf_hooks';
import {
CompletionReceiptFileStore,
LocalProcessController,
type LocalProcessControllerOptions,
} from '@qinglong/local-process';
import { BoundedWorkerRemoteExecutionContextMaterializer } from '../remote-execution/executionContextMaterializer';
import {
WorkerRemoteExecutionInboxProcessor,
type WorkerRemoteExecutionSession,
} from '../remote-execution/executionInboxProcessor';
import {
WorkerRemoteExecutionHeadlessLifecycle,
type WorkerRemoteExecutionLifecycleTickResult,
} from '../remote-execution/headlessExecutionLifecycle';
import { WorkerRemoteExecutionHttpsActivationClient } from '../remote-execution/transport/remoteActivationHttpsClient';
import {
DEFAULT_WORKER_REMOTE_OFFER_INBOX_ENTRIES,
MAX_WORKER_REMOTE_OFFER_INBOX_ENTRIES,
WorkerRemoteOfferPullCoordinator,
} from '../remote-execution/remoteOfferDelivery';
import { WorkerRemoteOfferFileJournal } from '../remote-execution/remoteOfferFileJournal';
import { WorkerRemoteOfferHttpsTransport } from '../remote-execution/transport/remoteOfferHttpsTransport';
import { WorkerRemoteSecretHttpsProvider } from '../remote-execution/transport/remoteSecretHttpsProvider';
import {
WorkerFileLogArtifactAllocator,
workerRemoteLogArtifactPolicy,
type WorkerRemoteLogArtifactProfile,
} from '../execution/workerFileLogArtifactAllocator';
import { WorkerIngressHttpsClient } from '../remote-execution/transport/workerIngressHttpsClient';
import {
WorkerRemoteArtifactHttpsUploader,
WorkerRemoteExecutionHttpsCompletionClient,
} from '../remote-execution/transport/remoteWorkerCompletionHttpsClient';
import { WorkerRemoteLeaseControlHttpsClient } from '../remote-execution/transport/remoteWorkerLeaseControlHttpsClient';
import { WorkerRemoteCompletionCoordinator } from '../execution/workerCompletionCoordinator';
import { WorkerRemoteExecutionControlCoordinator } from '../execution/workerExecutionControlCoordinator';
import {
WorkerInboxExecutionSpawnBarrier,
WorkerPosixExecutionExecutor,
type WorkerPosixExecutionExecutorOptions,
} from '../execution/workerPosixExecutionExecutor';
import type { WorkerIngressHttpsCredentialProvider } from '../remote-execution/transport/workerIngressHttpsClient';
import type { WorkerCertificateRenewalResult } from '../credential/workerCertificateRenewal';
const MIN_CADENCE_MS = 100;
const MAX_CADENCE_MS = 60_000;
const MIN_DRAIN_TIMEOUT_MS = 1_000;
const MAX_DRAIN_TIMEOUT_MS = 10 * 60_000;
const MIN_DRAIN_POLL_MS = 25;
const MAX_DRAIN_POLL_MS = 5_000;
const SETTLED_STATES = new Set([
'start_failure_acknowledged',
'completion_acknowledged',
]);
export interface ProductionWorkerSessionLifecycle {
current(): WorkerRemoteExecutionSession | undefined;
/** Optional product hook. Called only after bounded startup reconciliation. */
register?(): Promise<unknown>;
/** Optional caller-driven heartbeat step. Must not own a timer. */
tick?(): Promise<unknown>;
/** Immediately prevents new work when transport identity is unavailable. */
failClosed?(): void;
/** Resolves only after new work is durably disabled for the current Session. */
beginDrain(): Promise<void>;
/** Optional product hook. Called only after all execution records settle. */
disconnect?(): Promise<void>;
}
export interface ProductionWorkerCertificateRenewalLifecycle {
/** One bounded, caller-driven certificate maintenance step. */
run(): Promise<WorkerCertificateRenewalResult>;
}
export interface ProductionWorkerStorageOptions {
readonly journalRoot: string;
readonly logRoot: string;
readonly receiptRoot: string;
}
export interface ProductionWorkerHeadlessApplicationDisabledOptions {
readonly enabled?: false;
}
export interface ProductionWorkerHeadlessApplicationEnabledOptions {
readonly enabled: true;
readonly profile: string;
readonly capacityProfile: WorkerRemoteLogArtifactProfile;
readonly origin: string | URL;
readonly credentials: WorkerIngressHttpsCredentialProvider;
/** Shared product client. When supplied, this execution stack never closes it. */
readonly client?: WorkerIngressHttpsClient;
readonly session: ProductionWorkerSessionLifecycle;
/** Reuses this application's cadence; it must not own a timer or watcher. */
readonly certificateRenewal?: ProductionWorkerCertificateRenewalLifecycle;
readonly storage: ProductionWorkerStorageOptions;
readonly cadenceMs?: number;
readonly drainTimeoutMs?: number;
readonly drainPollMs?: number;
readonly maximumJournalEntries?: number;
readonly maximumRecordsPerTick?: number;
readonly maximumSupervisionRecordsPerTick?: number;
readonly requestTimeoutMs?: number;
readonly offerBackoffBaseMs?: number;
readonly ownershipStaleMs?: number;
readonly processController?: LocalProcessControllerOptions;
readonly launcherPath?: string;
readonly expectedLauncherSha256?: string;
readonly now?: () => number;
readonly diagnostic?: (
event: Readonly<{
code:
| 'tick_failed'
| 'session_tick_failed'
| 'certificate_renewal_failed'
| 'certificate_unavailable'
| 'recovery_required'
| 'drain_failed'
| 'disconnect_failed';
error?: unknown;
offerId?: string;
}>,
) => void | Promise<void>;
}
export type ProductionWorkerHeadlessApplicationOptions =
| ProductionWorkerHeadlessApplicationDisabledOptions
| ProductionWorkerHeadlessApplicationEnabledOptions;
export type ProductionWorkerHeadlessStopResult =
| 'stopped'
| 'drain_timed_out'
| 'recovery_required';
export type ProductionWorkerHeadlessApplicationResult =
| Readonly<{
status: 'disabled';
stop(): Promise<'stopped'>;
}>
| Readonly<{
status: 'active';
tick(): Promise<WorkerRemoteExecutionLifecycleTickResult>;
stop(): Promise<ProductionWorkerHeadlessStopResult>;
}>;
export interface ProductionWorkerHeadlessExecutionStack {
readonly journal: WorkerRemoteOfferFileJournal;
readonly lifecycle: WorkerRemoteExecutionHeadlessLifecycle;
readonly client: WorkerIngressHttpsClient;
readonly offerTransport: WorkerRemoteOfferHttpsTransport;
readonly ownsClient: boolean;
}
export class ProductionWorkerHeadlessApplicationError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'startup_recovery_required'
| 'startup_not_converged'
| 'certificate_unavailable'
| 'session_drain_unproven',
options?: ErrorOptions,
) {
super(`Production Worker headless application failed: ${reason}`, options);
this.name = 'ProductionWorkerHeadlessApplicationError';
}
}
interface NormalizedProductionWorkerOptions {
readonly cadenceMs: number;
readonly drainTimeoutMs: number;
readonly drainPollMs: number;
readonly maximumJournalEntries: number;
readonly maximumRecordsPerTick: number;
readonly maximumSupervisionRecordsPerTick: number;
}
function boundedInteger(
value: number,
minimum: number,
maximum: number,
): number {
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
throw new ProductionWorkerHeadlessApplicationError('invalid_configuration');
}
return value;
}
function storagePath(value: string): string {
if (
typeof value !== 'string' ||
!path.isAbsolute(value) ||
path.parse(value).root === value ||
path.normalize(value) !== value ||
value.includes('\0') ||
Buffer.byteLength(value, 'utf8') > 4096
) {
throw new ProductionWorkerHeadlessApplicationError('invalid_configuration');
}
return value;
}
function pathsOverlap(left: string, right: string): boolean {
const relative = path.relative(left, right);
return (
relative === '' ||
(!relative.startsWith(`..${path.sep}`) &&
relative !== '..' &&
!path.isAbsolute(relative))
);
}
function normalizeOptions(
options: ProductionWorkerHeadlessApplicationEnabledOptions,
): NormalizedProductionWorkerOptions {
if (
!options ||
options.enabled !== true ||
options.profile !== 'worker' ||
(options.capacityProfile !== 'edge' &&
options.capacityProfile !== 'node') ||
typeof options.credentials?.load !== 'function' ||
(options.client !== undefined &&
!(options.client instanceof WorkerIngressHttpsClient)) ||
typeof options.session?.current !== 'function' ||
typeof options.session?.beginDrain !== 'function' ||
(options.certificateRenewal !== undefined &&
(typeof options.certificateRenewal.run !== 'function' ||
typeof options.session.failClosed !== 'function')) ||
(options.now !== undefined && typeof options.now !== 'function') ||
(options.diagnostic !== undefined &&
typeof options.diagnostic !== 'function')
) {
throw new ProductionWorkerHeadlessApplicationError('invalid_configuration');
}
const roots = [
storagePath(options.storage?.journalRoot),
storagePath(options.storage?.logRoot),
storagePath(options.storage?.receiptRoot),
];
for (let left = 0; left < roots.length; left += 1) {
for (let right = left + 1; right < roots.length; right += 1) {
if (
pathsOverlap(roots[left]!, roots[right]!) ||
pathsOverlap(roots[right]!, roots[left]!)
) {
throw new ProductionWorkerHeadlessApplicationError(
'invalid_configuration',
);
}
}
}
const edge = options.capacityProfile === 'edge';
return Object.freeze({
cadenceMs: boundedInteger(
options.cadenceMs ?? (edge ? 2_000 : 500),
MIN_CADENCE_MS,
MAX_CADENCE_MS,
),
drainTimeoutMs: boundedInteger(
options.drainTimeoutMs ?? (edge ? 60_000 : 5 * 60_000),
MIN_DRAIN_TIMEOUT_MS,
MAX_DRAIN_TIMEOUT_MS,
),
drainPollMs: boundedInteger(
options.drainPollMs ?? (edge ? 500 : 100),
MIN_DRAIN_POLL_MS,
MAX_DRAIN_POLL_MS,
),
maximumJournalEntries: boundedInteger(
options.maximumJournalEntries ??
(edge ? DEFAULT_WORKER_REMOTE_OFFER_INBOX_ENTRIES : 256),
1,
MAX_WORKER_REMOTE_OFFER_INBOX_ENTRIES,
),
maximumRecordsPerTick: boundedInteger(
options.maximumRecordsPerTick ?? (edge ? 4 : 16),
1,
64,
),
maximumSupervisionRecordsPerTick: boundedInteger(
options.maximumSupervisionRecordsPerTick ?? (edge ? 4 : 32),
1,
64,
),
});
}
export function createProductionWorkerHeadlessExecutionStack(
options: ProductionWorkerHeadlessApplicationEnabledOptions,
): ProductionWorkerHeadlessExecutionStack {
const normalized = normalizeOptions(options);
const currentSession = () => options.session.current();
const journal = new WorkerRemoteOfferFileJournal({
rootDirectory: options.storage.journalRoot,
maximumEntries: normalized.maximumJournalEntries,
...(options.ownershipStaleMs === undefined
? {}
: { ownershipStaleMs: options.ownershipStaleMs }),
});
const ownsClient = options.client === undefined;
const client =
options.client ??
new WorkerIngressHttpsClient({
origin: options.origin,
credentials: options.credentials,
...(options.requestTimeoutMs === undefined
? {}
: { requestTimeoutMs: options.requestTimeoutMs }),
});
try {
const offerTransport = new WorkerRemoteOfferHttpsTransport({ client });
const offers = new WorkerRemoteOfferPullCoordinator({
journal,
transport: offerTransport,
currentSession,
...(options.now === undefined ? {} : { now: options.now }),
...(options.offerBackoffBaseMs === undefined
? {}
: { backoffBaseMs: options.offerBackoffBaseMs }),
});
const activation = new WorkerRemoteExecutionHttpsActivationClient({
client,
});
const secretProvider = new WorkerRemoteSecretHttpsProvider({
client,
inbox: journal,
});
const artifacts = new WorkerFileLogArtifactAllocator({
root: options.storage.logRoot,
policy: workerRemoteLogArtifactPolicy(options.capacityProfile),
});
const materializer = new BoundedWorkerRemoteExecutionContextMaterializer({
artifacts,
secrets: secretProvider,
});
const barrier = new WorkerInboxExecutionSpawnBarrier(journal);
const executorOptions: WorkerPosixExecutionExecutorOptions = {
barrier,
receiptRoot: options.storage.receiptRoot,
...(options.launcherPath === undefined
? {}
: { launcherPath: options.launcherPath }),
...(options.expectedLauncherSha256 === undefined
? {}
: { expectedLauncherSha256: options.expectedLauncherSha256 }),
};
const executor = new WorkerPosixExecutionExecutor(executorOptions);
const processor = new WorkerRemoteExecutionInboxProcessor({
inbox: journal,
activation,
materializer,
executor,
currentSession,
...(options.now === undefined ? {} : { now: options.now }),
});
const receipts = new CompletionReceiptFileStore(
options.storage.receiptRoot,
);
const uploader = new WorkerRemoteArtifactHttpsUploader({ client });
const completionClient = new WorkerRemoteExecutionHttpsCompletionClient({
client,
});
const completion = new WorkerRemoteCompletionCoordinator(
journal,
receipts,
artifacts,
uploader,
completionClient,
{
currentSession,
...(options.now === undefined ? {} : { now: options.now }),
},
);
const leaseControl = new WorkerRemoteLeaseControlHttpsClient({ client });
const processes = new LocalProcessController(
options.processController ?? {},
);
const control = new WorkerRemoteExecutionControlCoordinator(
journal,
completion,
leaseControl,
processes,
{
currentSession,
...(options.now === undefined ? {} : { now: options.now }),
},
);
const lifecycle = new WorkerRemoteExecutionHeadlessLifecycle({
journal,
offers,
processor,
control,
currentSession,
maximumRecordsPerTick: normalized.maximumRecordsPerTick,
maximumSupervisionRecordsPerTick:
normalized.maximumSupervisionRecordsPerTick,
...(options.now === undefined ? {} : { now: options.now }),
});
return Object.freeze({
journal,
lifecycle,
client,
offerTransport,
ownsClient,
});
} catch (error) {
if (ownsClient) client.close();
throw error;
}
}
async function inspectUnsettled(
journal: WorkerRemoteOfferFileJournal,
maximumEntries: number,
): Promise<
Readonly<{
unsettled: number;
recoveryOfferId?: string;
}>
> {
let afterOfferId: string | undefined;
let observed = 0;
let unsettled = 0;
do {
const page = await journal.listOffers({
...(afterOfferId === undefined ? {} : { afterOfferId }),
limit: 64,
});
observed += page.records.length;
if (observed > maximumEntries) {
throw new ProductionWorkerHeadlessApplicationError(
'invalid_configuration',
);
}
for (const record of page.records) {
if (record.state === 'recovery_required') {
return Object.freeze({
unsettled,
recoveryOfferId: record.offer.offerId,
});
}
if (!SETTLED_STATES.has(record.state)) unsettled += 1;
}
afterOfferId = page.nextAfterOfferId;
} while (afterOfferId !== undefined);
return Object.freeze({ unsettled });
}
function wait(delayMs: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, delayMs));
}
export async function startProductionWorkerHeadlessApplication(
options: ProductionWorkerHeadlessApplicationOptions,
): Promise<ProductionWorkerHeadlessApplicationResult> {
if (!options || options.enabled !== true) {
return Object.freeze({
status: 'disabled' as const,
async stop() {
return 'stopped' as const;
},
});
}
const stack = createProductionWorkerHeadlessExecutionStack(options);
return startProductionWorkerHeadlessApplicationWithStack(options, stack);
}
/**
* Starts one already-composed execution stack. Product composition uses this
* seam to share the exact HTTPS client with its timer-free Session lifecycle.
*/
export async function startProductionWorkerHeadlessApplicationWithStack(
options: ProductionWorkerHeadlessApplicationEnabledOptions,
stack: ProductionWorkerHeadlessExecutionStack,
): Promise<
Extract<ProductionWorkerHeadlessApplicationResult, { status: 'active' }>
> {
const normalized = normalizeOptions(options);
let lifecycleStarted = false;
let certificateUnavailable = false;
const emit = (
event: Readonly<{
code:
| 'tick_failed'
| 'session_tick_failed'
| 'certificate_renewal_failed'
| 'certificate_unavailable'
| 'recovery_required'
| 'drain_failed'
| 'disconnect_failed';
error?: unknown;
offerId?: string;
}>,
) => {
void Promise.resolve()
.then(() => options.diagnostic?.(event))
.catch(() => undefined);
};
const maintainCertificate = async (): Promise<boolean> => {
if (!options.certificateRenewal) return true;
let result: WorkerCertificateRenewalResult;
try {
result = await options.certificateRenewal.run();
} catch (error) {
if (!certificateUnavailable) {
certificateUnavailable = true;
options.session.failClosed!();
emit({ code: 'certificate_unavailable', error });
}
return false;
}
if (result.status === 'retry_scheduled') {
emit({ code: 'certificate_renewal_failed' });
}
if (result.status === 'unavailable') {
if (!certificateUnavailable) {
certificateUnavailable = true;
options.session.failClosed!();
emit({ code: 'certificate_unavailable' });
}
return false;
}
certificateUnavailable = false;
return true;
};
try {
await stack.lifecycle.start();
lifecycleStarted = true;
if (options.session.register !== undefined) {
const startupState = await inspectUnsettled(
stack.journal,
normalized.maximumJournalEntries,
);
if (
startupState.recoveryOfferId !== undefined ||
startupState.unsettled !== 0
) {
throw new ProductionWorkerHeadlessApplicationError(
'startup_recovery_required',
);
}
}
const maximumStartupTicks =
Math.ceil(
normalized.maximumJournalEntries / normalized.maximumRecordsPerTick,
) + 1;
let converged = false;
for (let tick = 0; tick < maximumStartupTicks; tick += 1) {
const result = await stack.lifecycle.tick();
if (result.status === 'recovery_required') {
throw new ProductionWorkerHeadlessApplicationError(
'startup_recovery_required',
);
}
if (result.status === 'reconciled') {
converged = true;
break;
}
if (result.status !== 'reconciling') {
throw new ProductionWorkerHeadlessApplicationError(
'startup_not_converged',
);
}
}
if (!converged) {
throw new ProductionWorkerHeadlessApplicationError(
'startup_not_converged',
);
}
if (!(await maintainCertificate())) {
throw new ProductionWorkerHeadlessApplicationError(
'certificate_unavailable',
);
}
await options.session.register?.();
let timer: NodeJS.Timeout | undefined;
let closed = false;
let tickOperation:
| Promise<WorkerRemoteExecutionLifecycleTickResult>
| undefined;
let stopOperation: Promise<ProductionWorkerHeadlessStopResult> | undefined;
const tickOnce =
async (): Promise<WorkerRemoteExecutionLifecycleTickResult> => {
if (await maintainCertificate()) {
await options.session.tick?.().catch((error) => {
emit({ code: 'session_tick_failed', error });
});
}
const result = await stack.lifecycle.tick();
if (result.status === 'recovery_required') {
emit({ code: 'recovery_required', offerId: result.offerId });
}
return result;
};
const tick = (): Promise<WorkerRemoteExecutionLifecycleTickResult> => {
if (tickOperation) return tickOperation;
const operation = tickOnce().finally(() => {
if (tickOperation === operation) tickOperation = undefined;
});
tickOperation = operation;
return operation;
};
const schedule = () => {
if (timer || closed) return;
timer = setInterval(() => {
void tick().catch((error) => emit({ code: 'tick_failed', error }));
}, normalized.cadenceMs);
timer.unref();
};
const unschedule = () => {
if (!timer) return;
clearInterval(timer);
timer = undefined;
};
const finish = async (): Promise<'stopped'> => {
unschedule();
await stack.lifecycle.stop();
stack.offerTransport.close();
if (stack.ownsClient) stack.client.close();
closed = true;
return 'stopped';
};
const drainAndStop =
async (): Promise<ProductionWorkerHeadlessStopResult> => {
unschedule();
await stack.lifecycle.beginDrain();
try {
await options.session.beginDrain();
} catch (error) {
emit({ code: 'drain_failed', error });
schedule();
throw error;
}
const session = options.session.current();
if (session?.status === 'available') {
schedule();
throw new ProductionWorkerHeadlessApplicationError(
'session_drain_unproven',
);
}
const deadline = performance.now() + normalized.drainTimeoutMs;
while (true) {
const tickResult = await tick().catch((error) => {
emit({ code: 'drain_failed', error });
return undefined;
});
if (tickResult?.status === 'recovery_required') {
schedule();
return 'recovery_required';
}
const state = await inspectUnsettled(
stack.journal,
normalized.maximumJournalEntries,
);
if (state.recoveryOfferId !== undefined) {
emit({
code: 'recovery_required',
offerId: state.recoveryOfferId,
});
schedule();
return 'recovery_required';
}
if (state.unsettled === 0) {
try {
await options.session.disconnect?.();
} catch (error) {
emit({ code: 'disconnect_failed', error });
schedule();
throw error;
}
return finish();
}
if (performance.now() >= deadline) {
schedule();
return 'drain_timed_out';
}
await wait(normalized.drainPollMs);
}
};
schedule();
return Object.freeze({
status: 'active' as const,
tick,
stop() {
if (closed) return Promise.resolve('stopped' as const);
if (stopOperation) return stopOperation;
const operation = drainAndStop().finally(() => {
if (stopOperation === operation && !closed) {
stopOperation = undefined;
}
});
stopOperation = operation;
return operation;
},
});
} catch (error) {
if (lifecycleStarted) {
await stack.lifecycle.stop().catch(() => undefined);
}
stack.offerTransport.close();
if (stack.ownsClient) stack.client.close();
throw error;
}
}
packages/ql3-worker-runtime/src/application-runtime/productionWorkerApplication.ts
+166
View File
@@ -0,0 +1,166 @@
import type { RemoteWorkerCapabilities } from '@qinglong/runtime-core/remote-dispatch';
import {
createProductionWorkerHeadlessExecutionStack,
startProductionWorkerHeadlessApplicationWithStack,
type ProductionWorkerHeadlessApplicationDisabledOptions,
type ProductionWorkerHeadlessApplicationEnabledOptions,
type ProductionWorkerHeadlessApplicationResult,
type ProductionWorkerSessionLifecycle,
} from './productionHeadlessApplication';
import { WorkerExecutionCapacityOracle } from '../session/workerExecutionCapacityOracle';
import { WorkerIngressHttpsClient } from '../remote-execution/transport/workerIngressHttpsClient';
import {
WorkerSessionCoordinator,
type WorkerSessionCoordinatorTickResult,
} from '../session/workerSessionCoordinator';
import { WorkerSessionHttpsClient } from '../session/workerSessionHttpsClient';
export type ProductionWorkerApplicationEnabledOptions = Omit<
ProductionWorkerHeadlessApplicationEnabledOptions,
'client' | 'session'
> &
Readonly<{
workerId: string;
capabilities: RemoteWorkerCapabilities;
maxConcurrentRuns: number;
leaseDurationMs?: number;
heartbeatIntervalMs?: number;
}>;
export type ProductionWorkerApplicationOptions =
| ProductionWorkerHeadlessApplicationDisabledOptions
| ProductionWorkerApplicationEnabledOptions;
class ManagedWorkerSessionLifecycle
implements ProductionWorkerSessionLifecycle
{
private oracle?: WorkerExecutionCapacityOracle;
constructor(private readonly coordinator: WorkerSessionCoordinator) {}
bind(oracle: WorkerExecutionCapacityOracle): void {
if (this.oracle !== undefined)
throw new TypeError('capacity already bound');
this.oracle = oracle;
}
current() {
return this.coordinator.current();
}
async register() {
const oracle = this.requiredOracle();
oracle.prepareRegistration();
try {
const registered = await this.coordinator.register();
oracle.activate();
return registered;
} catch (error) {
oracle.failClosed();
throw error;
}
}
async tick(): Promise<WorkerSessionCoordinatorTickResult> {
const result = await this.coordinator.tick();
if (result.status === 'heartbeat') this.requiredOracle().activate();
if (result.status === 'lease_expired') this.requiredOracle().failClosed();
return result;
}
failClosed(): void {
this.coordinator.failClosed();
this.requiredOracle().failClosed();
}
async beginDrain(): Promise<void> {
this.requiredOracle().beginDrain();
await this.coordinator.beginDrain();
}
async disconnect(): Promise<void> {
await this.coordinator.disconnect();
this.requiredOracle().offline();
}
private requiredOracle(): WorkerExecutionCapacityOracle {
if (!this.oracle) throw new TypeError('capacity is not bound');
return this.oracle;
}
}
/**
* Production composition root for one Remote Worker process. It owns exactly
* one HTTPS client/Agent and delegates all periodic work to the headless
* application's single cadence.
*/
export async function startProductionWorkerApplication(
options: ProductionWorkerApplicationOptions,
): Promise<ProductionWorkerHeadlessApplicationResult> {
if (!options || options.enabled !== true) {
return Object.freeze({
status: 'disabled' as const,
async stop() {
return 'stopped' as const;
},
});
}
const client = new WorkerIngressHttpsClient({
origin: options.origin,
credentials: options.credentials,
...(options.requestTimeoutMs === undefined
? {}
: { requestTimeoutMs: options.requestTimeoutMs }),
});
let oracle: WorkerExecutionCapacityOracle | undefined;
try {
const coordinator = new WorkerSessionCoordinator({
client: new WorkerSessionHttpsClient({ client }),
workerId: options.workerId,
capabilities: options.capabilities,
maxConcurrentRuns: options.maxConcurrentRuns,
availableSlots: () => oracle?.availableSlots() ?? 0,
...(options.leaseDurationMs === undefined
? {}
: { leaseDurationMs: options.leaseDurationMs }),
...(options.heartbeatIntervalMs === undefined
? {}
: { heartbeatIntervalMs: options.heartbeatIntervalMs }),
...(options.now === undefined ? {} : { now: options.now }),
});
const session = new ManagedWorkerSessionLifecycle(coordinator);
const executionOptions: ProductionWorkerHeadlessApplicationEnabledOptions =
{
...options,
client,
session,
};
const stack =
createProductionWorkerHeadlessExecutionStack(executionOptions);
oracle = new WorkerExecutionCapacityOracle({
journal: stack.journal,
maxConcurrentRuns: options.maxConcurrentRuns,
});
session.bind(oracle);
const application = await startProductionWorkerHeadlessApplicationWithStack(
executionOptions,
stack,
);
let closed = false;
return Object.freeze({
status: 'active' as const,
tick: application.tick,
async stop() {
const result = await application.stop();
if (result === 'stopped' && !closed) {
closed = true;
client.close();
}
return result;
},
});
} catch (error) {
client.close();
throw error;
}
}
packages/ql3-worker-runtime/src/credential/workerCertificateEnrollment.ts
+127
View File
@@ -0,0 +1,127 @@
// Credential ownership: generate bounded Worker certificate enrollment material.
import 'reflect-metadata';
import { createHash, webcrypto } from 'node:crypto';
import {
BasicConstraintsExtension,
ExtendedKeyUsage,
ExtendedKeyUsageExtension,
KeyUsageFlags,
KeyUsagesExtension,
PemConverter,
Pkcs10CertificateRequestGenerator,
} from '@peculiar/x509';
const WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const KEY_ALGORITHM = Object.freeze({
name: 'ECDSA',
namedCurve: 'P-256',
hash: 'SHA-256',
});
const MAX_PRIVATE_KEY_BYTES = 16 * 1024;
const MAX_CSR_BYTES = 16 * 1024;
export interface GenerateWorkerCertificateEnrollmentOptions {
readonly workerId: string;
}
export interface WorkerCertificateEnrollmentMaterial {
readonly algorithm: 'ECDSA_P256_SHA256';
readonly workerId: string;
readonly privateKeyPem: Buffer;
readonly certificateSigningRequestPem: string;
readonly publicKeySpkiSha256: string;
dispose(): void;
}
export class WorkerCertificateEnrollmentError extends TypeError {
constructor(message: string) {
super(`Worker certificate enrollment is invalid: ${message}`);
this.name = 'WorkerCertificateEnrollmentError';
}
}
function assertWorkerId(workerId: string): void {
if (typeof workerId !== 'string' || !WORKER_ID_PATTERN.test(workerId)) {
throw new WorkerCertificateEnrollmentError('workerId is invalid');
}
}
/**
* Generates a Worker-local P-256 key and a PKCS#10 request. This function does
* not contact a CA, persist the key or grant any Worker authority.
*/
export async function generateWorkerCertificateEnrollment(
options: GenerateWorkerCertificateEnrollmentOptions,
): Promise<WorkerCertificateEnrollmentMaterial> {
if (!options || typeof options !== 'object' || Array.isArray(options)) {
throw new WorkerCertificateEnrollmentError('options must be an object');
}
assertWorkerId(options.workerId);
let privateKeyPem: Buffer | undefined;
try {
const keys = await webcrypto.subtle.generateKey(KEY_ALGORITHM, true, [
'sign',
'verify',
]);
const request = await Pkcs10CertificateRequestGenerator.create(
{
name: `CN=${options.workerId}`,
keys: keys as unknown as CryptoKeyPair,
signingAlgorithm: KEY_ALGORITHM,
extensions: [
new BasicConstraintsExtension(false, undefined, true),
new ExtendedKeyUsageExtension([ExtendedKeyUsage.clientAuth], true),
new KeyUsagesExtension(KeyUsageFlags.digitalSignature, true),
],
},
webcrypto as unknown as Crypto,
);
if (!(await request.verify(webcrypto as unknown as Crypto))) {
throw new WorkerCertificateEnrollmentError(
'generated CSR signature is invalid',
);
}
const [privateKey, publicKey] = await Promise.all([
webcrypto.subtle.exportKey('pkcs8', keys.privateKey),
webcrypto.subtle.exportKey('spki', keys.publicKey),
]);
privateKeyPem = Buffer.from(
PemConverter.encode(privateKey, 'PRIVATE KEY'),
'ascii',
);
const certificateSigningRequestPem = request.toString('pem');
if (
privateKeyPem.byteLength < 1 ||
privateKeyPem.byteLength > MAX_PRIVATE_KEY_BYTES ||
Buffer.byteLength(certificateSigningRequestPem) < 1 ||
Buffer.byteLength(certificateSigningRequestPem) > MAX_CSR_BYTES
) {
throw new WorkerCertificateEnrollmentError(
'generated material exceeds its hard limit',
);
}
const publicKeySpkiSha256 = createHash('sha256')
.update(Buffer.from(publicKey))
.digest('hex');
let disposed = false;
const material: WorkerCertificateEnrollmentMaterial = {
algorithm: 'ECDSA_P256_SHA256',
workerId: options.workerId,
privateKeyPem,
certificateSigningRequestPem,
publicKeySpkiSha256,
dispose() {
if (disposed) return;
disposed = true;
privateKeyPem?.fill(0);
},
};
return Object.freeze(material);
} catch (error) {
privateKeyPem?.fill(0);
if (error instanceof WorkerCertificateEnrollmentError) throw error;
throw new WorkerCertificateEnrollmentError('key or CSR generation failed');
}
}
packages/ql3-worker-runtime/src/credential/workerCertificateIdentity.ts
+289
View File
@@ -0,0 +1,289 @@
// Credential ownership: validate Worker certificate identity and trust semantics.
import {
createHash,
createPrivateKey,
createPublicKey,
timingSafeEqual,
X509Certificate,
} from 'node:crypto';
const CERTIFICATE_PATTERN =
/-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g;
const CLIENT_AUTH_OID = '1.3.6.1.5.5.7.3.2';
const MAX_CERTIFICATE_MATERIAL_BYTES = 1024 * 1024;
const MAX_CERTIFICATES = 16;
const SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/;
export type WorkerCertificateIdentityFailureReason =
| 'invalid_material'
| 'key_mismatch'
| 'not_yet_valid'
| 'expired'
| 'insufficient_validity'
| 'not_client_auth'
| 'untrusted';
export class WorkerCertificateIdentityError extends Error {
constructor(readonly reason: WorkerCertificateIdentityFailureReason) {
super(`Worker certificate identity is unavailable: ${reason}`);
this.name = 'WorkerCertificateIdentityError';
}
}
export interface ValidateWorkerCertificateIdentityInput {
readonly privateKeyPem: string | Buffer;
readonly certificateChainPem: string | Buffer;
readonly trustAnchors: readonly (string | Buffer)[];
readonly now?: number;
readonly minimumRemainingValidityMs?: number;
}
export interface WorkerCertificateIdentitySummary {
readonly certificateSha256: string;
readonly publicKeySpkiSha256: string;
readonly serialNumber: string;
readonly notBeforeMs: number;
readonly notAfterMs: number;
}
interface ParsedCertificate {
readonly certificate: X509Certificate;
readonly fingerprint: string;
}
function materialBytes(
value: string | Buffer,
privateMaterial = false,
): Buffer {
if (typeof value !== 'string' && !Buffer.isBuffer(value)) {
throw new WorkerCertificateIdentityError('invalid_material');
}
const bytes = Buffer.isBuffer(value)
? Buffer.from(value)
: Buffer.from(value, 'utf8');
if (
bytes.byteLength < 1 ||
bytes.byteLength > MAX_CERTIFICATE_MATERIAL_BYTES
) {
bytes.fill(0);
throw new WorkerCertificateIdentityError('invalid_material');
}
if (privateMaterial && !bytes.includes(Buffer.from('PRIVATE KEY'))) {
bytes.fill(0);
throw new WorkerCertificateIdentityError('invalid_material');
}
return bytes;
}
function splitCertificates(
value: string | Buffer,
remaining: { count: number },
): ParsedCertificate[] {
const bytes = materialBytes(value);
try {
const pem = bytes.toString('utf8');
const matches = pem.match(CERTIFICATE_PATTERN);
if (!matches || matches.length === 0) {
throw new WorkerCertificateIdentityError('invalid_material');
}
const remainder = matches.reduce(
(candidate, match) => candidate.replace(match, ''),
pem,
);
remaining.count -= matches.length;
if (remainder.trim() !== '' || remaining.count < 0) {
throw new WorkerCertificateIdentityError('invalid_material');
}
return matches.map((match) => {
try {
const certificate = new X509Certificate(`${match}\n`);
return {
certificate,
fingerprint: createHash('sha256')
.update(certificate.raw)
.digest('hex'),
};
} catch {
throw new WorkerCertificateIdentityError('invalid_material');
}
});
} finally {
bytes.fill(0);
}
}
function certificateTime(
certificate: X509Certificate,
now: number,
): { notBeforeMs: number; notAfterMs: number } {
const notBeforeMs = Date.parse(certificate.validFrom);
const notAfterMs = Date.parse(certificate.validTo);
if (!Number.isFinite(notBeforeMs) || !Number.isFinite(notAfterMs)) {
throw new WorkerCertificateIdentityError('invalid_material');
}
if (now < notBeforeMs) {
throw new WorkerCertificateIdentityError('not_yet_valid');
}
if (now >= notAfterMs) {
throw new WorkerCertificateIdentityError('expired');
}
return { notBeforeMs, notAfterMs };
}
function signedBy(
certificate: X509Certificate,
issuer: X509Certificate,
): boolean {
try {
return (
certificate.checkIssued(issuer) && certificate.verify(issuer.publicKey)
);
} catch {
return false;
}
}
function reachesTrustAnchor(
certificate: ParsedCertificate,
intermediates: readonly ParsedCertificate[],
anchors: readonly ParsedCertificate[],
visited: ReadonlySet<string>,
): boolean {
if (
visited.has(certificate.fingerprint) ||
visited.size >= MAX_CERTIFICATES
) {
return false;
}
const nextVisited = new Set(visited);
nextVisited.add(certificate.fingerprint);
for (const anchor of anchors) {
if (signedBy(certificate.certificate, anchor.certificate)) return true;
}
for (const intermediate of intermediates) {
if (
!nextVisited.has(intermediate.fingerprint) &&
signedBy(certificate.certificate, intermediate.certificate) &&
reachesTrustAnchor(intermediate, intermediates, anchors, nextVisited)
) {
return true;
}
}
return false;
}
function matchingPublicKey(
privateKeyPem: Buffer,
certificate: X509Certificate,
): string {
let privateKey;
try {
privateKey = createPrivateKey(privateKeyPem);
} catch {
throw new WorkerCertificateIdentityError('invalid_material');
}
const key = createPublicKey(privateKey).export({
type: 'spki',
format: 'der',
});
const certificateKey = certificate.publicKey.export({
type: 'spki',
format: 'der',
});
if (
key.byteLength !== certificateKey.byteLength ||
!timingSafeEqual(key, certificateKey)
) {
throw new WorkerCertificateIdentityError('key_mismatch');
}
return createHash('sha256').update(key).digest('hex');
}
export function assertWorkerCertificateIdentitySummary(
value: WorkerCertificateIdentitySummary,
): void {
if (
!value ||
typeof value !== 'object' ||
!SHA256_HEX_PATTERN.test(value.certificateSha256) ||
!SHA256_HEX_PATTERN.test(value.publicKeySpkiSha256) ||
typeof value.serialNumber !== 'string' ||
!/^[A-Fa-f0-9]{1,128}$/.test(value.serialNumber) ||
!Number.isSafeInteger(value.notBeforeMs) ||
!Number.isSafeInteger(value.notAfterMs) ||
value.notBeforeMs < 0 ||
value.notAfterMs <= value.notBeforeMs
) {
throw new WorkerCertificateIdentityError('invalid_material');
}
}
/** Validates key possession, client-auth intent, validity and a bounded chain. */
export function validateWorkerCertificateIdentity(
input: ValidateWorkerCertificateIdentityInput,
): WorkerCertificateIdentitySummary {
if (!input || typeof input !== 'object' || Array.isArray(input)) {
throw new WorkerCertificateIdentityError('invalid_material');
}
const now = input.now ?? Date.now();
const minimumRemainingValidityMs = input.minimumRemainingValidityMs ?? 0;
if (
!Number.isSafeInteger(now) ||
now < 0 ||
!Number.isSafeInteger(minimumRemainingValidityMs) ||
minimumRemainingValidityMs < 0 ||
minimumRemainingValidityMs > 365 * 24 * 60 * 60_000
) {
throw new WorkerCertificateIdentityError('invalid_material');
}
if (
!Array.isArray(input.trustAnchors) ||
input.trustAnchors.length < 1 ||
input.trustAnchors.length > MAX_CERTIFICATES
) {
throw new WorkerCertificateIdentityError('invalid_material');
}
const remaining = { count: MAX_CERTIFICATES };
const chain = splitCertificates(input.certificateChainPem, remaining);
const anchors = input.trustAnchors.flatMap((anchor) =>
splitCertificates(anchor, remaining),
);
const [leaf, ...intermediates] = chain;
if (!leaf || anchors.length === 0 || leaf.certificate.ca) {
throw new WorkerCertificateIdentityError('invalid_material');
}
const leafTime = certificateTime(leaf.certificate, now);
if (leafTime.notAfterMs - now < minimumRemainingValidityMs) {
throw new WorkerCertificateIdentityError('insufficient_validity');
}
if (!leaf.certificate.keyUsage?.includes(CLIENT_AUTH_OID)) {
throw new WorkerCertificateIdentityError('not_client_auth');
}
for (const certificate of [...intermediates, ...anchors]) {
if (!certificate.certificate.ca) {
throw new WorkerCertificateIdentityError('invalid_material');
}
certificateTime(certificate.certificate, now);
}
if (!reachesTrustAnchor(leaf, intermediates, anchors, new Set())) {
throw new WorkerCertificateIdentityError('untrusted');
}
const privateKeyPem = materialBytes(input.privateKeyPem, true);
try {
const summary = Object.freeze({
certificateSha256: createHash('sha256')
.update(leaf.certificate.raw)
.digest('hex'),
publicKeySpkiSha256: matchingPublicKey(privateKeyPem, leaf.certificate),
serialNumber: leaf.certificate.serialNumber,
notBeforeMs: leafTime.notBeforeMs,
notAfterMs: leafTime.notAfterMs,
});
assertWorkerCertificateIdentitySummary(summary);
return summary;
} finally {
privateKeyPem.fill(0);
}
}
packages/ql3-worker-runtime/src/credential/workerCertificateRenewal.ts
+488
View File
@@ -0,0 +1,488 @@
// Credential ownership: coordinate Worker certificate renewal lifecycle state.
import type {
GenerateWorkerCertificateEnrollmentOptions,
WorkerCertificateEnrollmentMaterial,
} from './workerCertificateEnrollment';
import {
type ActiveWorkerCertificateIdentity,
type WorkerCertificateRenewalState,
type WorkerCertificateStore,
} from './workerCertificateStore';
const HOUR_MS = 60 * 60_000;
const DAY_MS = 24 * HOUR_MS;
const MAX_CERTIFICATE_MATERIAL_BYTES = 1024 * 1024;
const MAX_ENROLLMENT_MATERIAL_BYTES = 16 * 1024;
const MAX_CONSECUTIVE_FAILURES = 16;
export interface WorkerCertificateIssuer {
issue(input: {
readonly workerId: string;
readonly certificateSigningRequestPem: string;
readonly currentCertificateSha256?: string;
readonly signal: AbortSignal;
}): Promise<{ readonly certificateChainPem: string | Buffer }>;
}
export interface WorkerCertificateTrustAnchorProvider {
load(signal: AbortSignal): Promise<readonly (string | Buffer)[]>;
}
export interface WorkerCertificateRenewalPolicy {
readonly renewBeforeMs?: number;
readonly minimumIssuedValidityMs?: number;
readonly operationTimeoutMs?: number;
readonly backoffBaseMs?: number;
readonly backoffMaximumMs?: number;
}
export interface WorkerCertificateRenewalCoordinatorOptions {
readonly workerId: string;
readonly store: WorkerCertificateStore;
readonly issuer: WorkerCertificateIssuer;
readonly trustAnchors: WorkerCertificateTrustAnchorProvider;
readonly policy?: WorkerCertificateRenewalPolicy;
readonly now?: () => number;
readonly random?: () => number;
readonly prepareEnrollment?: (
options: GenerateWorkerCertificateEnrollmentOptions,
) => Promise<WorkerCertificateEnrollmentMaterial>;
}
export type WorkerCertificateRenewalResult =
| {
readonly status: 'not_due';
readonly identity: ActiveWorkerCertificateIdentity;
readonly renewAtMs: number;
}
| {
readonly status: 'renewed';
readonly identity: ActiveWorkerCertificateIdentity;
readonly cleanupPending: boolean;
}
| {
readonly status: 'backing_off';
readonly identity: ActiveWorkerCertificateIdentity;
readonly nextAttemptAtMs: number;
}
| {
readonly status: 'retry_scheduled';
readonly identity?: ActiveWorkerCertificateIdentity;
readonly reason: WorkerCertificateRenewalFailureReason;
readonly nextAttemptAtMs: number;
}
| {
readonly status: 'unavailable';
readonly nextAttemptAtMs: number;
};
export type WorkerCertificateRenewalFailureReason =
| 'trust_unavailable'
| 'enrollment_failed'
| 'issuance_failed'
| 'installation_failed'
| 'timed_out';
export class WorkerCertificateRenewalConfigurationError extends TypeError {
constructor(message: string) {
super(`Worker certificate renewal is invalid: ${message}`);
this.name = 'WorkerCertificateRenewalConfigurationError';
}
}
interface NormalizedPolicy {
readonly renewBeforeMs: number;
readonly minimumIssuedValidityMs: number;
readonly operationTimeoutMs: number;
readonly backoffBaseMs: number;
readonly backoffMaximumMs: number;
}
type EnrollmentFactory = NonNullable<
WorkerCertificateRenewalCoordinatorOptions['prepareEnrollment']
>;
function boundedInteger(
value: number | undefined,
fallback: number,
minimum: number,
maximum: number,
name: string,
): number {
const candidate = value ?? fallback;
if (
!Number.isSafeInteger(candidate) ||
candidate < minimum ||
candidate > maximum
) {
throw new WorkerCertificateRenewalConfigurationError(`${name} is invalid`);
}
return candidate;
}
function normalizePolicy(
policy: WorkerCertificateRenewalPolicy | undefined,
): NormalizedPolicy {
if (
policy !== undefined &&
(!policy || typeof policy !== 'object' || Array.isArray(policy))
) {
throw new WorkerCertificateRenewalConfigurationError(
'policy must be an object',
);
}
const renewBeforeMs = boundedInteger(
policy?.renewBeforeMs,
7 * DAY_MS,
HOUR_MS,
30 * DAY_MS,
'renewBeforeMs',
);
const minimumIssuedValidityMs = boundedInteger(
policy?.minimumIssuedValidityMs,
8 * DAY_MS,
HOUR_MS,
365 * DAY_MS,
'minimumIssuedValidityMs',
);
if (minimumIssuedValidityMs <= renewBeforeMs) {
throw new WorkerCertificateRenewalConfigurationError(
'minimumIssuedValidityMs must exceed renewBeforeMs',
);
}
const backoffBaseMs = boundedInteger(
policy?.backoffBaseMs,
30_000,
1_000,
HOUR_MS,
'backoffBaseMs',
);
const backoffMaximumMs = boundedInteger(
policy?.backoffMaximumMs,
6 * HOUR_MS,
backoffBaseMs,
6 * HOUR_MS,
'backoffMaximumMs',
);
return Object.freeze({
renewBeforeMs,
minimumIssuedValidityMs,
operationTimeoutMs: boundedInteger(
policy?.operationTimeoutMs,
30_000,
1_000,
120_000,
'operationTimeoutMs',
),
backoffBaseMs,
backoffMaximumMs,
});
}
function safeClock(clock: () => number): number {
const value = clock();
if (!Number.isSafeInteger(value) || value < 0) {
throw new WorkerCertificateRenewalConfigurationError(
'clock returned an invalid value',
);
}
return value;
}
function certificateMaterial(
value: string | Buffer,
): string | Buffer | undefined {
if (typeof value !== 'string' && !Buffer.isBuffer(value)) return undefined;
const size = Buffer.byteLength(value);
if (size < 1 || size > MAX_CERTIFICATE_MATERIAL_BYTES) return undefined;
return value;
}
function enrollmentMaterial(
value: WorkerCertificateEnrollmentMaterial,
workerId: string,
): WorkerCertificateEnrollmentMaterial {
if (
!value ||
typeof value !== 'object' ||
value.algorithm !== 'ECDSA_P256_SHA256' ||
value.workerId !== workerId ||
!Buffer.isBuffer(value.privateKeyPem) ||
value.privateKeyPem.byteLength < 1 ||
value.privateKeyPem.byteLength > MAX_ENROLLMENT_MATERIAL_BYTES ||
typeof value.certificateSigningRequestPem !== 'string' ||
Buffer.byteLength(value.certificateSigningRequestPem) < 1 ||
Buffer.byteLength(value.certificateSigningRequestPem) >
MAX_ENROLLMENT_MATERIAL_BYTES ||
typeof value.publicKeySpkiSha256 !== 'string' ||
!/^[a-f0-9]{64}$/.test(value.publicKeySpkiSha256) ||
typeof value.dispose !== 'function'
) {
if (Buffer.isBuffer(value?.privateKeyPem)) value.privateKeyPem.fill(0);
throw new Error('enrollment material is invalid');
}
return value;
}
async function defaultEnrollmentFactory(
options: GenerateWorkerCertificateEnrollmentOptions,
): Promise<WorkerCertificateEnrollmentMaterial> {
const enrollment = await import('./workerCertificateEnrollment');
return enrollment.generateWorkerCertificateEnrollment(options);
}
function timeoutSignal(
external: AbortSignal | undefined,
timeoutMs: number,
): { readonly operation: AbortSignal; readonly timeout: AbortSignal } {
const timeout = AbortSignal.timeout(timeoutMs);
return {
timeout,
operation: external ? AbortSignal.any([external, timeout]) : timeout,
};
}
/**
* Performs one explicitly triggered renewal check. It owns no timer, watcher or
* signal handler; edge and cluster profiles decide when to call it.
*/
export class WorkerCertificateRenewalCoordinator {
private readonly workerId: string;
private readonly store: WorkerCertificateStore;
private readonly issuer: WorkerCertificateIssuer;
private readonly trustAnchors: WorkerCertificateTrustAnchorProvider;
private readonly policy: NormalizedPolicy;
private readonly now: () => number;
private readonly random: () => number;
private readonly prepareEnrollment: EnrollmentFactory;
private inFlight?: Promise<WorkerCertificateRenewalResult>;
constructor(options: WorkerCertificateRenewalCoordinatorOptions) {
if (!options || typeof options !== 'object' || Array.isArray(options)) {
throw new WorkerCertificateRenewalConfigurationError(
'options must be an object',
);
}
if (
typeof options.workerId !== 'string' ||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(options.workerId)
) {
throw new WorkerCertificateRenewalConfigurationError(
'workerId is invalid',
);
}
if (
!options.store ||
typeof options.store.readActive !== 'function' ||
typeof options.store.install !== 'function' ||
typeof options.store.readRenewalState !== 'function' ||
typeof options.store.writeRenewalState !== 'function'
) {
throw new WorkerCertificateRenewalConfigurationError('store is invalid');
}
if (!options.issuer || typeof options.issuer.issue !== 'function') {
throw new WorkerCertificateRenewalConfigurationError('issuer is invalid');
}
if (
!options.trustAnchors ||
typeof options.trustAnchors.load !== 'function'
) {
throw new WorkerCertificateRenewalConfigurationError(
'trustAnchors is invalid',
);
}
if (options.now !== undefined && typeof options.now !== 'function') {
throw new WorkerCertificateRenewalConfigurationError('now is invalid');
}
if (options.random !== undefined && typeof options.random !== 'function') {
throw new WorkerCertificateRenewalConfigurationError('random is invalid');
}
if (
options.prepareEnrollment !== undefined &&
typeof options.prepareEnrollment !== 'function'
) {
throw new WorkerCertificateRenewalConfigurationError(
'prepareEnrollment is invalid',
);
}
this.workerId = options.workerId;
this.store = options.store;
this.issuer = options.issuer;
this.trustAnchors = options.trustAnchors;
this.policy = normalizePolicy(options.policy);
this.now = options.now ?? Date.now;
this.random = options.random ?? Math.random;
this.prepareEnrollment =
options.prepareEnrollment ?? defaultEnrollmentFactory;
}
run(signal?: AbortSignal): Promise<WorkerCertificateRenewalResult> {
if (signal !== undefined && !(signal instanceof AbortSignal)) {
return Promise.reject(
new WorkerCertificateRenewalConfigurationError('signal is invalid'),
);
}
if (this.inFlight) return this.inFlight;
const operation = this.performRun(signal).finally(() => {
if (this.inFlight === operation) this.inFlight = undefined;
});
this.inFlight = operation;
return operation;
}
private async performRun(
externalSignal: AbortSignal | undefined,
): Promise<WorkerCertificateRenewalResult> {
externalSignal?.throwIfAborted();
const observedAtMs = safeClock(this.now);
const renewalState = await this.store.readRenewalState();
const signals = timeoutSignal(
externalSignal,
this.policy.operationTimeoutMs,
);
let failureReason: WorkerCertificateRenewalFailureReason =
'trust_unavailable';
let current: ActiveWorkerCertificateIdentity | undefined;
let trustAnchors: readonly (string | Buffer)[] | undefined;
try {
trustAnchors = await this.trustAnchors.load(signals.operation);
signals.operation.throwIfAborted();
current = await this.store.readActive(trustAnchors, observedAtMs);
} catch (error) {
if (externalSignal?.aborted) throw externalSignal.reason ?? error;
if (signals.timeout.aborted) failureReason = 'timed_out';
}
if (
current &&
current.notAfterMs - observedAtMs > this.policy.renewBeforeMs
) {
return Object.freeze({
status: 'not_due',
identity: current,
renewAtMs: current.notAfterMs - this.policy.renewBeforeMs,
});
}
if (
renewalState.nextAttemptAtMs !== null &&
renewalState.nextAttemptAtMs > observedAtMs
) {
if (current) {
return Object.freeze({
status: 'backing_off',
identity: current,
nextAttemptAtMs: renewalState.nextAttemptAtMs,
});
}
return Object.freeze({
status: 'unavailable',
nextAttemptAtMs: renewalState.nextAttemptAtMs,
});
}
const attemptedAtMs = safeClock(this.now);
let enrollment: WorkerCertificateEnrollmentMaterial | undefined;
try {
if (!trustAnchors || signals.operation.aborted) {
if (signals.timeout.aborted) failureReason = 'timed_out';
throw new Error('trust is unavailable');
}
failureReason = 'enrollment_failed';
enrollment = enrollmentMaterial(
await this.prepareEnrollment({ workerId: this.workerId }),
this.workerId,
);
signals.operation.throwIfAborted();
failureReason = 'issuance_failed';
const issued = await this.issuer.issue({
workerId: this.workerId,
certificateSigningRequestPem: enrollment.certificateSigningRequestPem,
...(current
? { currentCertificateSha256: current.certificateSha256 }
: {}),
signal: signals.operation,
});
signals.operation.throwIfAborted();
const issuedMaterial = certificateMaterial(issued?.certificateChainPem);
if (!issuedMaterial) throw new Error('issued material is invalid');
failureReason = 'installation_failed';
const installedAtMs = safeClock(this.now);
const installed = await this.store.install({
privateKeyPem: enrollment.privateKeyPem,
certificateChainPem: issuedMaterial,
trustAnchors,
now: installedAtMs,
minimumRemainingValidityMs: this.policy.minimumIssuedValidityMs,
});
await this.store.writeRenewalState({
consecutiveFailures: 0,
nextAttemptAtMs: null,
lastAttemptAtMs: attemptedAtMs,
lastSuccessAtMs: installedAtMs,
});
return Object.freeze({
status: 'renewed',
identity: installed,
cleanupPending: installed.cleanupPending,
});
} catch (error) {
if (externalSignal?.aborted) throw externalSignal.reason ?? error;
if (signals.timeout.aborted) failureReason = 'timed_out';
const failedAtMs = safeClock(this.now);
const nextState = this.failedState(
renewalState,
attemptedAtMs,
failedAtMs,
);
await this.store.writeRenewalState(nextState);
if (!current || current.notAfterMs <= failedAtMs) {
return Object.freeze({
status: 'unavailable',
nextAttemptAtMs: nextState.nextAttemptAtMs!,
});
}
return Object.freeze({
status: 'retry_scheduled',
identity: current,
reason: failureReason,
nextAttemptAtMs: nextState.nextAttemptAtMs!,
});
} finally {
try {
enrollment?.dispose();
} finally {
enrollment?.privateKeyPem.fill(0);
}
}
}
private failedState(
previous: WorkerCertificateRenewalState,
attemptedAtMs: number,
failedAtMs: number,
): WorkerCertificateRenewalState {
const random = this.random();
if (!Number.isFinite(random) || random < 0 || random >= 1) {
throw new WorkerCertificateRenewalConfigurationError(
'random returned an invalid value',
);
}
const consecutiveFailures = Math.min(
MAX_CONSECUTIVE_FAILURES,
previous.consecutiveFailures + 1,
);
const exponential = Math.min(
this.policy.backoffMaximumMs,
this.policy.backoffBaseMs * 2 ** (consecutiveFailures - 1),
);
const delayMs = Math.max(1, Math.floor(exponential * (0.5 + random / 2)));
return Object.freeze({
consecutiveFailures,
nextAttemptAtMs: failedAtMs + delayMs,
lastAttemptAtMs: attemptedAtMs,
lastSuccessAtMs: previous.lastSuccessAtMs,
});
}
}
packages/ql3-worker-runtime/src/credential/workerCertificateStore.ts
+611
View File
@@ -0,0 +1,611 @@
// Credential ownership: persist active Worker certificate material atomically.
import { randomUUID } from 'node:crypto';
import { constants } from 'node:fs';
import {
chmod,
lstat,
mkdir,
open,
readdir,
rename,
rm,
} from 'node:fs/promises';
import { isAbsolute, join } from 'node:path';
import {
assertWorkerCertificateIdentitySummary,
validateWorkerCertificateIdentity,
type WorkerCertificateIdentitySummary,
} from './workerCertificateIdentity';
const MAX_IDENTITY_FILE_BYTES = 1024 * 1024;
const MAX_MANIFEST_BYTES = 4096;
const GENERATION_PATTERN =
/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/;
const MAX_GENERATIONS = 8;
export interface WorkerCertificateFileStoreOptions {
readonly rootDirectory: string;
readonly retainedGenerations?: number;
}
export interface InstallWorkerCertificateIdentityInput {
readonly privateKeyPem: string | Buffer;
readonly certificateChainPem: string | Buffer;
readonly trustAnchors: readonly (string | Buffer)[];
readonly now?: number;
readonly minimumRemainingValidityMs?: number;
}
export interface ActiveWorkerCertificateIdentity
extends WorkerCertificateIdentitySummary {
readonly generationId: string;
readonly installedAtMs: number;
readonly privateKeyFile: string;
readonly certificateChainFile: string;
}
export interface InstallWorkerCertificateIdentityResult
extends ActiveWorkerCertificateIdentity {
readonly cleanupPending: boolean;
}
export interface WorkerCertificateRenewalState {
readonly consecutiveFailures: number;
readonly nextAttemptAtMs: number | null;
readonly lastAttemptAtMs: number | null;
readonly lastSuccessAtMs: number | null;
}
export interface WorkerCertificateStore {
readActive(
trustAnchors: readonly (string | Buffer)[],
now?: number,
): Promise<ActiveWorkerCertificateIdentity | undefined>;
install(
input: InstallWorkerCertificateIdentityInput,
): Promise<InstallWorkerCertificateIdentityResult>;
readRenewalState(): Promise<WorkerCertificateRenewalState>;
writeRenewalState(state: WorkerCertificateRenewalState): Promise<void>;
}
export interface WorkerCertificateIdentityManifest
extends WorkerCertificateIdentitySummary {
readonly schemaVersion: 1;
readonly generationId: string;
readonly installedAtMs: number;
}
interface RenewalStateManifest extends WorkerCertificateRenewalState {
readonly schemaVersion: 1;
}
export class WorkerCertificateStoreError extends Error {
constructor(message: string) {
super(`Worker certificate store is unavailable: ${message}`);
this.name = 'WorkerCertificateStoreError';
}
}
function safeNow(value: number | undefined): number {
const now = value ?? Date.now();
if (!Number.isSafeInteger(now) || now < 0) {
throw new WorkerCertificateStoreError('observation time is invalid');
}
return now;
}
async function safeDirectory(path: string): Promise<void> {
let existed = true;
try {
await lstat(path);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
existed = false;
await mkdir(path, { recursive: true, mode: 0o700 });
}
const stat = await lstat(path);
if (
!stat.isDirectory() ||
stat.isSymbolicLink() ||
(stat.mode & 0o077) !== 0
) {
throw new WorkerCertificateStoreError('directory metadata is unsafe');
}
if (!existed) await chmod(path, 0o700);
}
async function syncDirectory(path: string): Promise<void> {
const handle = await open(path, constants.O_RDONLY);
try {
await handle.sync();
} finally {
await handle.close();
}
}
async function writeSyncedFile(
path: string,
bytes: Buffer,
mode: number,
): Promise<void> {
const flags =
constants.O_WRONLY |
constants.O_CREAT |
constants.O_EXCL |
(constants.O_NOFOLLOW ?? 0);
const handle = await open(path, flags, mode);
try {
await handle.writeFile(bytes);
await handle.sync();
await handle.chmod(mode);
} finally {
await handle.close();
}
}
async function readBoundedFile(
path: string,
maximumBytes: number,
): Promise<Buffer> {
const flags = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0);
let handle;
try {
handle = await open(path, flags);
const stat = await handle.stat();
if (
!stat.isFile() ||
stat.size < 1 ||
stat.size > maximumBytes ||
(stat.mode & 0o077) !== 0
) {
throw new WorkerCertificateStoreError('file metadata is unsafe');
}
const bytes = await handle.readFile();
if (bytes.byteLength < 1 || bytes.byteLength > maximumBytes) {
bytes.fill(0);
throw new WorkerCertificateStoreError('file size is unsafe');
}
return bytes;
} catch (error) {
if (error instanceof WorkerCertificateStoreError) throw error;
throw new WorkerCertificateStoreError('file is unavailable');
} finally {
await handle?.close().catch(() => undefined);
}
}
async function assertSafeExistingDirectory(path: string): Promise<void> {
let stat;
try {
stat = await lstat(path);
} catch {
throw new WorkerCertificateStoreError('directory is unavailable');
}
if (
!stat.isDirectory() ||
stat.isSymbolicLink() ||
(stat.mode & 0o077) !== 0
) {
throw new WorkerCertificateStoreError('directory metadata is unsafe');
}
}
function exactKeys(value: object, expected: readonly string[]): boolean {
const keys = Object.keys(value).sort();
const sorted = [...expected].sort();
return (
keys.length === sorted.length &&
keys.every((key, index) => key === sorted[index])
);
}
function identityManifest(value: unknown): WorkerCertificateIdentityManifest {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!exactKeys(value, [
'certificateSha256',
'generationId',
'installedAtMs',
'notAfterMs',
'notBeforeMs',
'publicKeySpkiSha256',
'schemaVersion',
'serialNumber',
])
) {
throw new WorkerCertificateStoreError('identity manifest is invalid');
}
const candidate = value as Partial<WorkerCertificateIdentityManifest>;
if (
candidate.schemaVersion !== 1 ||
typeof candidate.generationId !== 'string' ||
!GENERATION_PATTERN.test(candidate.generationId) ||
!Number.isSafeInteger(candidate.installedAtMs) ||
Number(candidate.installedAtMs) < 0
) {
throw new WorkerCertificateStoreError('identity manifest is invalid');
}
assertWorkerCertificateIdentitySummary(
candidate as WorkerCertificateIdentitySummary,
);
return Object.freeze(candidate as WorkerCertificateIdentityManifest);
}
function renewalStateManifest(value: unknown): RenewalStateManifest {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!exactKeys(value, [
'consecutiveFailures',
'lastAttemptAtMs',
'lastSuccessAtMs',
'nextAttemptAtMs',
'schemaVersion',
])
) {
throw new WorkerCertificateStoreError('renewal state is invalid');
}
const candidate = value as Partial<RenewalStateManifest>;
const optionalTime = (time: unknown): boolean =>
time === null || (Number.isSafeInteger(time) && Number(time) >= 0);
if (
candidate.schemaVersion !== 1 ||
!Number.isSafeInteger(candidate.consecutiveFailures) ||
Number(candidate.consecutiveFailures) < 0 ||
Number(candidate.consecutiveFailures) > 16 ||
!optionalTime(candidate.nextAttemptAtMs) ||
!optionalTime(candidate.lastAttemptAtMs) ||
!optionalTime(candidate.lastSuccessAtMs)
) {
throw new WorkerCertificateStoreError('renewal state is invalid');
}
return Object.freeze(candidate as RenewalStateManifest);
}
async function parseJsonFile<T>(
path: string,
parser: (value: unknown) => T,
): Promise<T> {
const bytes = await readBoundedFile(path, MAX_MANIFEST_BYTES);
try {
return parser(JSON.parse(bytes.toString('utf8')) as unknown);
} catch (error) {
if (error instanceof WorkerCertificateStoreError) throw error;
throw new WorkerCertificateStoreError('manifest JSON is invalid');
} finally {
bytes.fill(0);
}
}
export class WorkerCertificateFileStore implements WorkerCertificateStore {
private readonly rootDirectory: string;
private readonly generationsDirectory: string;
private readonly retainedGenerations: number;
private installing = false;
constructor(options: WorkerCertificateFileStoreOptions) {
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
typeof options.rootDirectory !== 'string' ||
!isAbsolute(options.rootDirectory) ||
options.rootDirectory.length > 4096 ||
/[\0\r\n]/.test(options.rootDirectory)
) {
throw new WorkerCertificateStoreError('rootDirectory is invalid');
}
const retainedGenerations = options.retainedGenerations ?? 2;
if (
!Number.isSafeInteger(retainedGenerations) ||
retainedGenerations < 1 ||
retainedGenerations > 4
) {
throw new WorkerCertificateStoreError(
'retainedGenerations must be between 1 and 4',
);
}
this.rootDirectory = options.rootDirectory;
this.generationsDirectory = join(options.rootDirectory, 'generations');
this.retainedGenerations = retainedGenerations;
}
private async initialize(): Promise<void> {
try {
await safeDirectory(this.rootDirectory);
await safeDirectory(this.generationsDirectory);
} catch (error) {
if (error instanceof WorkerCertificateStoreError) throw error;
throw new WorkerCertificateStoreError('directory is unavailable');
}
}
private async writeAtomicManifest(
name: string,
value: unknown,
onCommitted?: () => void,
): Promise<void> {
const temporary = join(this.rootDirectory, `.${name}.${randomUUID()}.tmp`);
const destination = join(this.rootDirectory, name);
const bytes = Buffer.from(`${JSON.stringify(value)}\n`, 'utf8');
try {
if (bytes.byteLength > MAX_MANIFEST_BYTES) {
throw new WorkerCertificateStoreError('manifest exceeds hard limit');
}
await writeSyncedFile(temporary, bytes, 0o600);
await rename(temporary, destination);
onCommitted?.();
await syncDirectory(this.rootDirectory);
} finally {
bytes.fill(0);
await rm(temporary, { force: true }).catch(() => undefined);
}
}
async readActiveSummary(): Promise<
WorkerCertificateIdentityManifest | undefined
> {
await this.initialize();
try {
return await parseJsonFile(
join(this.rootDirectory, 'active.json'),
identityManifest,
);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
if (
error instanceof WorkerCertificateStoreError &&
error.message.endsWith('file is unavailable')
) {
const stat = await lstat(join(this.rootDirectory, 'active.json')).catch(
() => undefined,
);
if (!stat) return undefined;
}
throw error;
}
}
async readActive(
trustAnchors: readonly (string | Buffer)[],
now: number = Date.now(),
): Promise<ActiveWorkerCertificateIdentity | undefined> {
const manifest = await this.readActiveSummary();
if (!manifest) return undefined;
const generationDirectory = join(
this.generationsDirectory,
manifest.generationId,
);
const privateKeyFile = join(generationDirectory, 'private-key.pem');
const certificateChainFile = join(
generationDirectory,
'certificate-chain.pem',
);
await assertSafeExistingDirectory(generationDirectory);
const privateKeyPem = await readBoundedFile(
privateKeyFile,
MAX_IDENTITY_FILE_BYTES,
);
let certificateChainPem: Buffer | undefined;
try {
certificateChainPem = await readBoundedFile(
certificateChainFile,
MAX_IDENTITY_FILE_BYTES,
);
const summary = validateWorkerCertificateIdentity({
privateKeyPem,
certificateChainPem,
trustAnchors,
now: safeNow(now),
});
if (
summary.certificateSha256 !== manifest.certificateSha256 ||
summary.publicKeySpkiSha256 !== manifest.publicKeySpkiSha256 ||
summary.serialNumber !== manifest.serialNumber ||
summary.notBeforeMs !== manifest.notBeforeMs ||
summary.notAfterMs !== manifest.notAfterMs
) {
throw new WorkerCertificateStoreError('active identity was modified');
}
return Object.freeze({
...summary,
generationId: manifest.generationId,
installedAtMs: manifest.installedAtMs,
privateKeyFile,
certificateChainFile,
});
} finally {
privateKeyPem.fill(0);
certificateChainPem?.fill(0);
}
}
async install(
input: InstallWorkerCertificateIdentityInput,
): Promise<InstallWorkerCertificateIdentityResult> {
if (this.installing) {
throw new WorkerCertificateStoreError('another install is in progress');
}
this.installing = true;
let stagingDirectory: string | undefined;
let generationDirectory: string | undefined;
let activated = false;
try {
if (!input || typeof input !== 'object' || Array.isArray(input)) {
throw new WorkerCertificateStoreError('install input is invalid');
}
const now = safeNow(input.now);
const summary = validateWorkerCertificateIdentity({ ...input, now });
await this.initialize();
const entries = await readdir(this.generationsDirectory, {
withFileTypes: true,
});
if (entries.length >= MAX_GENERATIONS) {
throw new WorkerCertificateStoreError(
'generation capacity requires maintenance',
);
}
const generationId = randomUUID();
generationDirectory = join(this.generationsDirectory, generationId);
stagingDirectory = join(
this.generationsDirectory,
`.staging-${generationId}`,
);
await mkdir(stagingDirectory, { mode: 0o700 });
const privateKeyBytes = Buffer.isBuffer(input.privateKeyPem)
? Buffer.from(input.privateKeyPem)
: Buffer.from(input.privateKeyPem, 'utf8');
const certificateBytes = Buffer.isBuffer(input.certificateChainPem)
? Buffer.from(input.certificateChainPem)
: Buffer.from(input.certificateChainPem, 'utf8');
const manifest: WorkerCertificateIdentityManifest = Object.freeze({
schemaVersion: 1,
generationId,
installedAtMs: now,
...summary,
});
const manifestBytes = Buffer.from(`${JSON.stringify(manifest)}\n`);
try {
await Promise.all([
writeSyncedFile(
join(stagingDirectory, 'private-key.pem'),
privateKeyBytes,
0o600,
),
writeSyncedFile(
join(stagingDirectory, 'certificate-chain.pem'),
certificateBytes,
0o600,
),
writeSyncedFile(
join(stagingDirectory, 'metadata.json'),
manifestBytes,
0o600,
),
]);
} finally {
privateKeyBytes.fill(0);
certificateBytes.fill(0);
manifestBytes.fill(0);
}
await syncDirectory(stagingDirectory);
await rename(stagingDirectory, generationDirectory);
stagingDirectory = undefined;
await syncDirectory(this.generationsDirectory);
await this.writeAtomicManifest('active.json', manifest, () => {
activated = true;
});
const cleanupPending = !(await this.pruneRetired(manifest.generationId));
return Object.freeze({
...summary,
generationId,
installedAtMs: now,
privateKeyFile: join(generationDirectory, 'private-key.pem'),
certificateChainFile: join(
generationDirectory,
'certificate-chain.pem',
),
cleanupPending,
});
} catch (error) {
if (error instanceof WorkerCertificateStoreError) throw error;
throw new WorkerCertificateStoreError('install failed');
} finally {
if (stagingDirectory) {
await rm(stagingDirectory, { recursive: true, force: true }).catch(
() => undefined,
);
}
if (generationDirectory && !activated) {
await rm(generationDirectory, { recursive: true, force: true }).catch(
() => undefined,
);
}
this.installing = false;
}
}
private async pruneRetired(activeGenerationId: string): Promise<boolean> {
try {
const entries = await readdir(this.generationsDirectory, {
withFileTypes: true,
});
const generations: Array<{
generationId: string;
installedAtMs: number;
}> = [];
for (const entry of entries) {
if (!entry.isDirectory() || !GENERATION_PATTERN.test(entry.name)) {
continue;
}
const metadata = await parseJsonFile(
join(this.generationsDirectory, entry.name, 'metadata.json'),
identityManifest,
);
if (metadata.generationId !== entry.name) return false;
generations.push({
generationId: entry.name,
installedAtMs: metadata.installedAtMs,
});
}
generations.sort(
(left, right) =>
right.installedAtMs - left.installedAtMs ||
right.generationId.localeCompare(left.generationId),
);
const keep = new Set(
generations
.filter((item) => item.generationId !== activeGenerationId)
.slice(0, Math.max(0, this.retainedGenerations - 1))
.map((item) => item.generationId),
);
keep.add(activeGenerationId);
for (const generation of generations) {
if (!keep.has(generation.generationId)) {
await rm(join(this.generationsDirectory, generation.generationId), {
recursive: true,
force: true,
});
}
}
await syncDirectory(this.generationsDirectory);
return true;
} catch {
return false;
}
}
async readRenewalState(): Promise<WorkerCertificateRenewalState> {
await this.initialize();
try {
const manifest = await parseJsonFile(
join(this.rootDirectory, 'renewal.json'),
renewalStateManifest,
);
const { schemaVersion: _schemaVersion, ...state } = manifest;
return Object.freeze(state);
} catch (error) {
const stat = await lstat(join(this.rootDirectory, 'renewal.json')).catch(
() => undefined,
);
if (!stat) {
return Object.freeze({
consecutiveFailures: 0,
nextAttemptAtMs: null,
lastAttemptAtMs: null,
lastSuccessAtMs: null,
});
}
throw error;
}
}
async writeRenewalState(state: WorkerCertificateRenewalState): Promise<void> {
await this.initialize();
const manifest = renewalStateManifest({ schemaVersion: 1, ...state });
await this.writeAtomicManifest('renewal.json', manifest);
}
}
packages/ql3-worker-runtime/src/credential/workerProductionCredentialProvider.ts
+260
View File
@@ -0,0 +1,260 @@
// Credential ownership: load production mTLS identity and bounded Worker token.
import { constants } from 'node:fs';
import { lstat, open } from 'node:fs/promises';
import { dirname, isAbsolute, normalize, parse } from 'node:path';
import { normalizeWorkerCredentialId } from '@qinglong/runtime-core/worker-credential';
import {
validateWorkerCertificateIdentity,
type WorkerCertificateIdentitySummary,
} from './workerCertificateIdentity';
import type {
WorkerCertificateStore,
} from './workerCertificateStore';
import type {
WorkerCertificateTrustAnchorProvider,
} from './workerCertificateRenewal';
import type {
WorkerIngressHttpsCredentialProvider,
WorkerIngressHttpsCredentials,
} from '../remote-execution/transport/workerIngressHttpsClient';
const MAX_CREDENTIAL_TOKEN_BYTES = 256;
const MAX_TLS_MATERIAL_BYTES = 1024 * 1024;
const CREDENTIAL_TOKEN =
/^ql3w_([A-Za-z0-9][A-Za-z0-9._:-]{0,63})_([A-Za-z0-9_-]{43})$/;
export interface WorkerProductionCredentialProviderOptions {
readonly certificateStore: Pick<WorkerCertificateStore, 'readActive'>;
readonly trustAnchors: WorkerCertificateTrustAnchorProvider;
/** Private, atomically replaceable file containing one ql3w token. */
readonly credentialTokenFile: string;
readonly expectedCredentialId?: string;
readonly now?: () => number;
}
export class WorkerProductionCredentialProviderError extends Error {
constructor(
readonly reason: 'invalid_configuration' | 'credentials_unavailable',
options?: ErrorOptions,
) {
super(`Worker production credentials failed: ${reason}`, options);
this.name = 'WorkerProductionCredentialProviderError';
}
}
function privateFilePath(value: string): string {
if (
typeof value !== 'string' ||
!isAbsolute(value) ||
parse(value).root === value ||
normalize(value) !== value ||
value.includes('\0') ||
Buffer.byteLength(value, 'utf8') > 4096
) throw new WorkerProductionCredentialProviderError('invalid_configuration');
return value;
}
function now(provider: () => number): number {
const value = provider();
if (!Number.isSafeInteger(value) || value < 0) {
throw new WorkerProductionCredentialProviderError('credentials_unavailable');
}
return value;
}
async function readPrivateFile(
path: string,
maximumBytes: number,
): Promise<Buffer> {
let handle;
try {
const parent = await lstat(dirname(path));
if (
!parent.isDirectory() ||
parent.isSymbolicLink() ||
(parent.mode & 0o077) !== 0
) throw new Error('unsafe parent');
handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
const stat = await handle.stat();
if (
!stat.isFile() ||
stat.size < 1 ||
stat.size > maximumBytes ||
(stat.mode & 0o077) !== 0
) throw new Error('unsafe file');
const bytes = await handle.readFile();
if (bytes.byteLength < 1 || bytes.byteLength > maximumBytes) {
bytes.fill(0);
throw new Error('unsafe bytes');
}
return bytes;
} catch (error) {
throw new WorkerProductionCredentialProviderError(
'credentials_unavailable', { cause: error },
);
} finally {
await handle?.close().catch(() => undefined);
}
}
function token(bytes: Buffer, expectedCredentialId?: string): string {
try {
let length = bytes.byteLength;
if (bytes[length - 1] === 0x0a) length -= 1;
const value = bytes.subarray(0, length).toString('ascii');
const match = CREDENTIAL_TOKEN.exec(value);
if (
!match ||
bytes.subarray(0, length).some((byte) => byte > 0x7f) ||
bytes.subarray(0, length).includes(0x0a) ||
(expectedCredentialId !== undefined &&
match[1] !== expectedCredentialId)
) throw new Error('token is invalid');
return value;
} catch (error) {
throw new WorkerProductionCredentialProviderError(
'credentials_unavailable', { cause: error },
);
} finally {
bytes.fill(0);
}
}
function sameIdentity(
expected: WorkerCertificateIdentitySummary,
actual: WorkerCertificateIdentitySummary,
): boolean {
return actual.certificateSha256 === expected.certificateSha256 &&
actual.publicKeySpkiSha256 === expected.publicKeySpkiSha256 &&
actual.serialNumber === expected.serialNumber &&
actual.notBeforeMs === expected.notBeforeMs &&
actual.notAfterMs === expected.notAfterMs;
}
function copyMaterial(value: string | Buffer): Buffer {
const bytes = Buffer.isBuffer(value)
? Buffer.from(value)
: Buffer.from(value, 'utf8');
if (bytes.byteLength < 1 || bytes.byteLength > MAX_TLS_MATERIAL_BYTES) {
bytes.fill(0);
throw new WorkerProductionCredentialProviderError(
'credentials_unavailable',
);
}
return bytes;
}
/**
* Loads the current certificate generation and ql3w token for every request.
* Atomic file replacement therefore rotates credentials without a watcher or
* a second Agent. Returned Buffer material is disposable by the HTTPS client.
*/
export class WorkerProductionCredentialProvider
implements WorkerIngressHttpsCredentialProvider {
private readonly certificateStore: Pick<WorkerCertificateStore, 'readActive'>;
private readonly trustAnchors: WorkerCertificateTrustAnchorProvider;
private readonly credentialTokenFile: string;
private readonly expectedCredentialId?: string;
private readonly nowProvider: () => number;
constructor(options: WorkerProductionCredentialProviderOptions) {
if (
!options ||
typeof options.certificateStore?.readActive !== 'function' ||
typeof options.trustAnchors?.load !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function')
) throw new WorkerProductionCredentialProviderError('invalid_configuration');
let expectedCredentialId: string | undefined;
try {
expectedCredentialId = options.expectedCredentialId === undefined
? undefined
: normalizeWorkerCredentialId(options.expectedCredentialId);
} catch (error) {
throw new WorkerProductionCredentialProviderError(
'invalid_configuration', { cause: error },
);
}
this.certificateStore = options.certificateStore;
this.trustAnchors = options.trustAnchors;
this.credentialTokenFile = privateFilePath(options.credentialTokenFile);
this.expectedCredentialId = expectedCredentialId;
this.nowProvider = options.now ?? Date.now;
}
async load(signal?: AbortSignal): Promise<WorkerIngressHttpsCredentials> {
const operationSignal = signal ?? new AbortController().signal;
operationSignal.throwIfAborted();
let certificate: Buffer | undefined;
let privateKey: Buffer | undefined;
const anchors: Buffer[] = [];
try {
const observedAtMs = now(this.nowProvider);
const trustAnchors = await this.trustAnchors.load(operationSignal);
operationSignal.throwIfAborted();
const active = await this.certificateStore.readActive(
trustAnchors,
observedAtMs,
);
if (!active) {
throw new WorkerProductionCredentialProviderError(
'credentials_unavailable',
);
}
certificate = await readPrivateFile(
active.certificateChainFile,
MAX_TLS_MATERIAL_BYTES,
);
privateKey = await readPrivateFile(
active.privateKeyFile,
MAX_TLS_MATERIAL_BYTES,
);
operationSignal.throwIfAborted();
const summary = validateWorkerCertificateIdentity({
certificateChainPem: certificate,
privateKeyPem: privateKey,
trustAnchors,
now: observedAtMs,
});
if (!sameIdentity(active, summary)) {
throw new WorkerProductionCredentialProviderError(
'credentials_unavailable',
);
}
const credentialToken = token(
await readPrivateFile(
this.credentialTokenFile,
MAX_CREDENTIAL_TOKEN_BYTES,
),
this.expectedCredentialId,
);
for (const anchor of trustAnchors) anchors.push(copyMaterial(anchor));
const disposableCertificate = certificate;
const disposablePrivateKey = privateKey;
certificate = undefined;
privateKey = undefined;
let disposed = false;
return Object.freeze({
authorization: `Worker ${credentialToken}`,
certificateChainPem: disposableCertificate,
privateKeyPem: disposablePrivateKey,
trustAnchors: Object.freeze(anchors),
dispose() {
if (disposed) return;
disposed = true;
disposableCertificate.fill(0);
disposablePrivateKey.fill(0);
anchors.forEach((anchor) => anchor.fill(0));
},
});
} catch (error) {
certificate?.fill(0);
privateKey?.fill(0);
anchors.forEach((anchor) => anchor.fill(0));
if (operationSignal.aborted) throw operationSignal.reason ?? error;
if (error instanceof WorkerProductionCredentialProviderError) throw error;
throw new WorkerProductionCredentialProviderError(
'credentials_unavailable', { cause: error },
);
}
}
}
packages/ql3-worker-runtime/src/execution/workerCompletionCoordinator.ts
+419
View File
@@ -0,0 +1,419 @@
// Worker Execution owns crash-replayable Artifact upload and completion convergence.
import { createHash, timingSafeEqual } from 'node:crypto';
import type {
CompletionReceipt,
CompletionReceiptStore,
} from '@qinglong/local-process';
import {
normalizeWorkerRemoteExecutionInboxRecord,
type WorkerRemoteExecutionInbox,
type WorkerRemoteExecutionInboxRecord,
} from '../remote-execution/executionInbox';
import type { WorkerRemoteExecutionSession } from '../remote-execution/executionInboxProcessor';
import type {
WorkerRemoteLogArtifactReadLease,
WorkerRemoteLogArtifactSource,
} from './workerFileLogArtifactAllocator';
import type { RemoteWorkerExecutionFence } from '@qinglong/runtime-core/remote-worker-completion';
const SHA256 = /^[a-f0-9]{64}$/;
const COMPLETION_EVIDENCE_STATES = new Set([
'launching',
'started',
'running_acknowledged',
'recovery_required',
]);
export interface WorkerRemoteLogArtifactUploadCommand {
readonly workerId: RemoteWorkerExecutionFence['workerId'];
readonly workerSessionId: RemoteWorkerExecutionFence['workerSessionId'];
readonly workerGeneration: RemoteWorkerExecutionFence['workerGeneration'];
readonly projectId: string;
readonly runId: string;
readonly attemptId: string;
readonly offerId: RemoteWorkerExecutionFence['offerId'];
readonly leaseGeneration: RemoteWorkerExecutionFence['leaseGeneration'];
readonly leaseToken: RemoteWorkerExecutionFence['leaseToken'];
readonly expectedLeaseVersion: RemoteWorkerExecutionFence['expectedLeaseVersion'];
readonly logArtifactId: string;
readonly byteLength: number;
readonly truncated: boolean | undefined;
readonly content: AsyncIterable<Uint8Array>;
}
export interface WorkerRemoteLogArtifactUploadResult {
readonly status: 'stored' | 'already_stored';
readonly logArtifactId: string;
readonly byteLength: number;
readonly sha256: string;
}
export interface WorkerRemoteLogArtifactUploader {
upload(
command: WorkerRemoteLogArtifactUploadCommand,
): Promise<Readonly<WorkerRemoteLogArtifactUploadResult>>;
}
export interface WorkerRemoteExecutionCompletionCommand {
readonly offerId: string;
readonly projectId: string;
readonly runId: string;
readonly attemptId: string;
readonly callbackSequence: number;
readonly callbackTokenDigest: string;
readonly result: Readonly<{
outcome: 'succeeded' | 'failed';
startedAtMs: number;
finishedAtMs: number;
exitCode: number;
}>;
readonly artifact: Readonly<{
logArtifactId: string;
byteLength: number;
sha256: string;
truncated: boolean | undefined;
}>;
readonly executorType: string;
readonly workerId: string;
readonly workerSessionId: string;
readonly workerGeneration: number;
readonly leaseGeneration: number;
readonly leaseToken: string;
readonly expectedLeaseVersion: number;
}
export interface WorkerRemoteExecutionCompletionResult {
readonly status: 'applied' | 'already_completed' | 'already_terminal';
readonly runId: string;
readonly attemptId: string;
readonly callbackSequence: number;
}
export interface WorkerRemoteExecutionCompletionClient {
complete(
command: WorkerRemoteExecutionCompletionCommand,
): Promise<Readonly<WorkerRemoteExecutionCompletionResult>>;
}
export type WorkerRemoteCompletionStatus =
| 'not_found'
| 'deferred'
| 'receipt_missing'
| 'receipt_unavailable'
| 'receipt_invalid'
| 'artifact_missing'
| 'completion_acknowledged'
| 'already_completed'
| 'control_plane_terminal';
export interface WorkerRemoteCompletionResult {
readonly offerId: string;
readonly status: WorkerRemoteCompletionStatus;
readonly receiptCleanup?: 'removed' | 'already_absent' | 'pending';
}
export interface WorkerRemoteCompletionCoordinatorOptions {
readonly currentSession: () => WorkerRemoteExecutionSession | undefined;
readonly now?: () => number;
}
export class WorkerRemoteCompletionCoordinatorError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'artifact_response_invalid'
| 'completion_response_invalid',
) {
super(`Worker remote completion coordination failed: ${reason}`);
this.name = 'WorkerRemoteCompletionCoordinatorError';
}
}
/**
* Crash-replayable Worker completion pipeline. The local receipt is an
* authenticated capability, the log is uploaded before terminal mutation,
* and receipt deletion happens only after the inbox completion barrier.
*/
export class WorkerRemoteCompletionCoordinator {
private readonly currentSessionProvider: () =>
WorkerRemoteExecutionSession | undefined;
private readonly nowProvider: () => number;
private readonly inFlight = new Map<
string,
Promise<WorkerRemoteCompletionResult>
>();
constructor(
private readonly inbox: Pick<
WorkerRemoteExecutionInbox,
'readOffer' | 'replaceOffer'
>,
private readonly receipts: Pick<CompletionReceiptStore, 'read' | 'remove'>,
private readonly artifacts: WorkerRemoteLogArtifactSource,
private readonly uploader: WorkerRemoteLogArtifactUploader,
private readonly completion: WorkerRemoteExecutionCompletionClient,
options: WorkerRemoteCompletionCoordinatorOptions,
) {
if (
typeof inbox?.readOffer !== 'function' ||
typeof inbox?.replaceOffer !== 'function' ||
typeof receipts?.read !== 'function' ||
typeof receipts?.remove !== 'function' ||
typeof artifacts?.open !== 'function' ||
typeof uploader?.upload !== 'function' ||
typeof completion?.complete !== 'function' ||
typeof options?.currentSession !== 'function'
) {
throw new WorkerRemoteCompletionCoordinatorError('invalid_configuration');
}
this.currentSessionProvider = options.currentSession;
this.nowProvider = options.now ?? Date.now;
}
recover(offerId: string): Promise<WorkerRemoteCompletionResult> {
const active = this.inFlight.get(offerId);
if (active) return active;
const operation = this.process(offerId).finally(() => {
if (this.inFlight.get(offerId) === operation) this.inFlight.delete(offerId);
});
this.inFlight.set(offerId, operation);
return operation;
}
private async process(offerId: string): Promise<WorkerRemoteCompletionResult> {
const value = await this.inbox.readOffer(offerId);
if (!value) return Object.freeze({ offerId, status: 'not_found' });
let record = normalizeWorkerRemoteExecutionInboxRecord(value);
if (record.state === 'completion_acknowledged') {
return Object.freeze({
offerId,
status: 'already_completed',
receiptCleanup: await this.cleanup(record.offer.candidate.attemptId),
});
}
if (!this.canSubmit(record)) {
return Object.freeze({ offerId, status: 'deferred' });
}
let receipt: CompletionReceipt | undefined;
try {
receipt = await this.receipts.read(record.offer.candidate.attemptId);
} catch {
return Object.freeze({ offerId, status: 'receipt_unavailable' });
}
if (!receipt) return Object.freeze({ offerId, status: 'receipt_missing' });
try {
this.authenticate(record, receipt);
} catch {
return Object.freeze({ offerId, status: 'receipt_invalid' });
}
const artifact = await this.artifacts.open({
runId: record.offer.candidate.runId,
attemptId: record.offer.candidate.attemptId,
logArtifactId: record.logArtifactId!,
});
if (!artifact) return Object.freeze({ offerId, status: 'artifact_missing' });
const uploaded = await this.upload(record, artifact);
const completed = await this.completion.complete(Object.freeze({
offerId: record.offer.offerId,
projectId: record.offer.candidate.projectId,
runId: record.offer.candidate.runId,
attemptId: record.offer.candidate.attemptId,
callbackSequence: receipt.callbackSequence,
callbackTokenDigest: record.completionReceiptTokenDigest!,
result: Object.freeze({
outcome: receipt.exitCode === 0 ? 'succeeded' as const : 'failed' as const,
startedAtMs: receipt.startedAtMs,
finishedAtMs: receipt.finishedAtMs,
exitCode: receipt.exitCode,
}),
artifact: Object.freeze({
logArtifactId: uploaded.logArtifactId,
byteLength: uploaded.byteLength,
sha256: uploaded.sha256,
truncated: artifact.truncated,
}),
executorType: record.offer.candidate.executorType,
workerId: record.offer.lease.workerId,
workerSessionId: record.offer.lease.workerSessionId,
workerGeneration: record.offer.lease.workerGeneration,
leaseGeneration: record.offer.lease.leaseGeneration,
leaseToken: record.offer.leaseToken,
expectedLeaseVersion: record.offer.lease.version,
}));
this.assertCompletionResponse(record, receipt, completed);
if (completed.status === 'already_terminal') {
return Object.freeze({ offerId, status: 'control_plane_terminal' });
}
record = await this.markAcknowledged(record);
return Object.freeze({
offerId,
status: 'completion_acknowledged',
receiptCleanup: await this.cleanup(record.offer.candidate.attemptId),
});
}
private canSubmit(record: WorkerRemoteExecutionInboxRecord): boolean {
if (!COMPLETION_EVIDENCE_STATES.has(record.state)) return false;
if (
record.state === 'recovery_required' &&
record.recoveryReason !== 'launch_outcome_unknown'
) return false;
const current = this.currentSessionProvider();
const now = this.now();
return Boolean(
current &&
current.workerId === record.offer.worker.workerId &&
current.sessionId === record.offer.worker.sessionId &&
current.generation === record.offer.worker.generation &&
current.status !== 'offline' &&
current.leaseExpiresAtMs > now &&
record.offer.lease.expiresAtMs > now &&
record.executorStartedAtMs !== undefined &&
record.logArtifactId !== undefined &&
record.completionReceiptCallbackSequence !== undefined &&
record.completionReceiptTokenDigest !== undefined
);
}
private authenticate(
record: WorkerRemoteExecutionInboxRecord,
receipt: CompletionReceipt,
): void {
if (
receipt.runId !== record.offer.candidate.runId ||
receipt.attemptId !== record.offer.candidate.attemptId ||
receipt.callbackSequence !== record.completionReceiptCallbackSequence ||
receipt.startedAtMs !== record.executorStartedAtMs ||
receipt.finishedAtMs > this.now() ||
!record.completionReceiptTokenDigest ||
!SHA256.test(record.completionReceiptTokenDigest)
) throw new Error('Completion receipt authority does not match');
const token = Buffer.from(receipt.token, 'base64url');
try {
if (token.byteLength !== 32 || token.toString('base64url') !== receipt.token) {
throw new Error('Completion receipt capability is not canonical');
}
const expected = Buffer.from(record.completionReceiptTokenDigest, 'hex');
const actual = createHash('sha256').update(token).digest();
if (!timingSafeEqual(expected, actual)) {
throw new Error('Completion receipt capability does not match');
}
} finally {
token.fill(0);
}
}
private async upload(
record: WorkerRemoteExecutionInboxRecord,
artifact: WorkerRemoteLogArtifactReadLease,
): Promise<WorkerRemoteLogArtifactUploadResult> {
const digest = createHash('sha256');
let observedBytes = 0;
const content = (async function* () {
for await (const chunk of artifact.chunks()) {
observedBytes += chunk.byteLength;
digest.update(chunk);
yield chunk;
}
})();
let result: Readonly<WorkerRemoteLogArtifactUploadResult>;
try {
result = await this.uploader.upload(Object.freeze({
workerId: record.offer.lease.workerId,
workerSessionId: record.offer.lease.workerSessionId,
workerGeneration: record.offer.lease.workerGeneration,
projectId: record.offer.candidate.projectId,
runId: record.offer.candidate.runId,
attemptId: record.offer.candidate.attemptId,
offerId: record.offer.offerId,
leaseGeneration: record.offer.lease.leaseGeneration,
leaseToken: record.offer.leaseToken,
expectedLeaseVersion: record.offer.lease.version,
logArtifactId: artifact.logArtifactId,
byteLength: artifact.byteLength,
truncated: artifact.truncated,
content,
}));
} finally {
await artifact.close();
}
const actualDigest = digest.digest('hex');
if (
observedBytes !== artifact.byteLength ||
(result?.status !== 'stored' && result?.status !== 'already_stored') ||
result.logArtifactId !== artifact.logArtifactId ||
result.byteLength !== artifact.byteLength ||
!SHA256.test(result.sha256) ||
result.sha256 !== actualDigest
) {
throw new WorkerRemoteCompletionCoordinatorError(
'artifact_response_invalid',
);
}
return Object.freeze({ ...result });
}
private assertCompletionResponse(
record: WorkerRemoteExecutionInboxRecord,
receipt: CompletionReceipt,
result: Readonly<WorkerRemoteExecutionCompletionResult>,
): void {
if (
!['applied', 'already_completed', 'already_terminal'].includes(result?.status) ||
result.runId !== record.offer.candidate.runId ||
result.attemptId !== record.offer.candidate.attemptId ||
result.callbackSequence !== receipt.callbackSequence
) {
throw new WorkerRemoteCompletionCoordinatorError(
'completion_response_invalid',
);
}
}
private async markAcknowledged(
record: WorkerRemoteExecutionInboxRecord,
): Promise<WorkerRemoteExecutionInboxRecord> {
const updatedAtMs = Math.max(this.now(), record.updatedAtMs);
const { recoveryReason: _recoveryReason, ...authority } = record;
const next = normalizeWorkerRemoteExecutionInboxRecord({
...authority,
revision: record.revision + 1,
state: 'completion_acknowledged',
updatedAtMs,
completionAcknowledgedAtMs: updatedAtMs,
});
try {
await this.inbox.replaceOffer(next, record.revision);
return next;
} catch (error) {
const current = await this.inbox.readOffer(record.offer.offerId);
if (current) {
const normalized = normalizeWorkerRemoteExecutionInboxRecord(current);
if (normalized.state === 'completion_acknowledged') return normalized;
}
throw error;
}
}
private async cleanup(
attemptId: string,
): Promise<'removed' | 'already_absent' | 'pending'> {
try {
return (await this.receipts.remove(attemptId))
? 'removed'
: 'already_absent';
} catch {
return 'pending';
}
}
private now(): number {
const value = this.nowProvider();
if (!Number.isSafeInteger(value) || value < 0) {
throw new WorkerRemoteCompletionCoordinatorError('invalid_configuration');
}
return value;
}
}
packages/ql3-worker-runtime/src/execution/workerExecutionControlCoordinator.ts
+415
View File
@@ -0,0 +1,415 @@
// Worker Execution owns lease renewal, stop fencing, and completion convergence.
import type {
LocalProcessController,
LocalProcessStopResult,
} from '@qinglong/local-process';
import type { RemoteWorkerLeaseControlResult } from '@qinglong/runtime-core/remote-worker-lease-control';
import {
normalizeWorkerRemoteExecutionInboxRecord,
type WorkerRemoteExecutionInbox,
type WorkerRemoteExecutionInboxRecord,
} from '../remote-execution/executionInbox';
import type { WorkerRemoteExecutionSession } from '../remote-execution/executionInboxProcessor';
import type {
WorkerRemoteCompletionResult,
WorkerRemoteCompletionStatus,
} from './workerCompletionCoordinator';
import type { WorkerRemoteLeaseControlClient } from '../remote-execution/transport/remoteWorkerLeaseControlHttpsClient';
export interface WorkerRemoteExecutionControlCoordinatorOptions {
readonly currentSession: () => WorkerRemoteExecutionSession | undefined;
readonly now?: () => number;
}
export type WorkerRemoteExecutionControlResult = Readonly<{
readonly offerId: string;
readonly completionStatus?: WorkerRemoteCompletionStatus;
} & (
| { readonly status: 'not_found' }
| { readonly status: 'completion_acknowledged' }
| {
readonly status: 'renewed';
readonly leaseVersion: number;
readonly expiresAtMs: number;
}
| {
readonly status: 'stop_requested' | 'stop_unverified';
readonly reason: NonNullable<RemoteWorkerLeaseControlResult['stop']>['reason'];
readonly leaseVersion: number;
readonly expiresAtMs: number;
readonly stop: LocalProcessStopResult;
}
| {
readonly status: 'terminal';
readonly terminalStatus: NonNullable<
RemoteWorkerLeaseControlResult['terminalStatus']
>;
readonly stop: LocalProcessStopResult;
}
| {
readonly status: 'completion_terminal';
readonly stop: LocalProcessStopResult;
}
| {
readonly status: 'lease_expired';
readonly stop: LocalProcessStopResult;
readonly recoveryReason:
| 'lease_lost_local_execution_stopped'
| 'lease_lost_local_execution_unverified';
}
| { readonly status: 'session_unavailable' }
)>;
export class WorkerRemoteExecutionControlCoordinatorError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'invalid_clock'
| 'lease_response_invalid',
options?: ErrorOptions,
) {
super(`Worker remote execution control failed: ${reason}`, options);
this.name = 'WorkerRemoteExecutionControlCoordinatorError';
}
}
const CONTROL_PLANE_TERMINAL_TRANSITIONS = new Set([
'accepted',
'starting_acknowledged',
'launching',
'started',
'running_acknowledged',
'start_failed',
'recovery_required',
]);
/**
* One bounded, timer-free supervision step for an accepted remote execution.
* Completion evidence is replayed first; live authority is then renewed or an
* exact durable process identity is stopped before recovery evidence is stored.
*/
export class WorkerRemoteExecutionControlCoordinator {
private readonly currentSessionProvider: () =>
WorkerRemoteExecutionSession | undefined;
private readonly nowProvider: () => number;
private readonly inFlight = new Map<
string,
Promise<WorkerRemoteExecutionControlResult>
>();
constructor(
private readonly inbox: Pick<
WorkerRemoteExecutionInbox,
'readOffer' | 'replaceOffer'
>,
private readonly completion: Pick<
{ recover(offerId: string): Promise<WorkerRemoteCompletionResult> },
'recover'
>,
private readonly leaseControl: WorkerRemoteLeaseControlClient,
private readonly processes: Pick<LocalProcessController, 'stop'>,
options: WorkerRemoteExecutionControlCoordinatorOptions,
) {
if (
typeof inbox?.readOffer !== 'function' ||
typeof inbox?.replaceOffer !== 'function' ||
typeof completion?.recover !== 'function' ||
typeof leaseControl?.control !== 'function' ||
typeof processes?.stop !== 'function' ||
typeof options?.currentSession !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function')
) {
throw new WorkerRemoteExecutionControlCoordinatorError(
'invalid_configuration',
);
}
this.currentSessionProvider = options.currentSession;
this.nowProvider = options.now ?? Date.now;
}
reconcile(offerId: string): Promise<WorkerRemoteExecutionControlResult> {
const active = this.inFlight.get(offerId);
if (active) return active;
const operation = this.reconcileOnce(offerId).finally(() => {
if (this.inFlight.get(offerId) === operation) this.inFlight.delete(offerId);
});
this.inFlight.set(offerId, operation);
return operation;
}
private async reconcileOnce(
offerId: string,
): Promise<WorkerRemoteExecutionControlResult> {
const value = await this.inbox.readOffer(offerId);
if (!value) return Object.freeze({ offerId, status: 'not_found' as const });
let record = normalizeWorkerRemoteExecutionInboxRecord(value);
if (record.state === 'completion_acknowledged') {
return Object.freeze({
offerId,
status: 'completion_acknowledged' as const,
});
}
let replay: WorkerRemoteCompletionResult | undefined;
try {
replay = await this.completion.recover(offerId);
} catch {
// Completion and lease transports are intentionally isolated: a failed
// Artifact upload must not prevent the Worker from retaining authority.
}
if (
replay?.status === 'completion_acknowledged' ||
replay?.status === 'already_completed'
) {
return Object.freeze({
offerId,
status: 'completion_acknowledged' as const,
completionStatus: replay.status,
});
}
record = await this.readRequired(offerId);
const completionStatus = replay?.status;
if (replay?.status === 'control_plane_terminal') {
const stop = await this.stop(record);
await this.markControlPlaneTerminal(record);
return Object.freeze({
offerId,
status: 'completion_terminal' as const,
stop,
completionStatus,
});
}
const now = this.now();
if (record.offer.lease.expiresAtMs <= now) {
const stop = await this.stop(record);
const recoveryReason = this.stopWasConclusive(stop)
? 'lease_lost_local_execution_stopped' as const
: 'lease_lost_local_execution_unverified' as const;
await this.markLeaseLost(record, recoveryReason);
return Object.freeze({
offerId,
status: 'lease_expired' as const,
stop,
recoveryReason,
...(completionStatus === undefined ? {} : { completionStatus }),
});
}
if (!this.sessionMatches(record, now)) {
return Object.freeze({
offerId,
status: 'session_unavailable' as const,
...(completionStatus === undefined ? {} : { completionStatus }),
});
}
const control = await this.leaseControl.control(Object.freeze({
workerId: record.offer.lease.workerId,
workerSessionId: record.offer.lease.workerSessionId,
workerGeneration: record.offer.lease.workerGeneration,
projectId: record.offer.candidate.projectId,
runId: record.offer.candidate.runId,
attemptId: record.offer.candidate.attemptId,
offerId: record.offer.offerId,
leaseGeneration: record.offer.lease.leaseGeneration,
leaseToken: record.offer.leaseToken,
expectedLeaseVersion: record.offer.lease.version,
}));
this.assertAuthority(record, control);
if (control.status === 'terminal') {
const stop = await this.stop(record);
await this.markControlPlaneTerminal(record);
return Object.freeze({
offerId,
status: 'terminal' as const,
terminalStatus: control.terminalStatus!,
stop,
...(completionStatus === undefined ? {} : { completionStatus }),
});
}
record = await this.persistRenewedLease(record, control);
if (control.status === 'renewed') {
return Object.freeze({
offerId,
status: 'renewed' as const,
leaseVersion: record.offer.lease.version,
expiresAtMs: record.offer.lease.expiresAtMs,
...(completionStatus === undefined ? {} : { completionStatus }),
});
}
const stop = await this.stop(record);
return Object.freeze({
offerId,
status: this.stopWasConclusive(stop)
? 'stop_requested' as const
: 'stop_unverified' as const,
reason: control.stop!.reason,
leaseVersion: record.offer.lease.version,
expiresAtMs: record.offer.lease.expiresAtMs,
stop,
...(completionStatus === undefined ? {} : { completionStatus }),
});
}
private assertAuthority(
record: WorkerRemoteExecutionInboxRecord,
result: Readonly<RemoteWorkerLeaseControlResult>,
): void {
if (
result.projectId !== record.offer.candidate.projectId ||
result.runId !== record.offer.candidate.runId ||
result.attemptId !== record.offer.candidate.attemptId ||
result.offerId !== record.offer.offerId ||
result.leaseGeneration !== record.offer.lease.leaseGeneration ||
(result.status !== 'terminal' &&
result.leaseVersion !== record.offer.lease.version + 1)
) {
throw new WorkerRemoteExecutionControlCoordinatorError(
'lease_response_invalid',
);
}
}
private async persistRenewedLease(
record: WorkerRemoteExecutionInboxRecord,
result: Readonly<RemoteWorkerLeaseControlResult>,
): Promise<WorkerRemoteExecutionInboxRecord> {
if (
result.leaseVersion === undefined ||
result.renewedAtMs === undefined ||
result.expiresAtMs === undefined ||
result.renewedAtMs < record.offer.lease.renewedAtMs ||
result.renewedAtMs < record.offer.lease.updatedAtMs ||
result.expiresAtMs <= result.renewedAtMs
) {
throw new WorkerRemoteExecutionControlCoordinatorError(
'lease_response_invalid',
);
}
const next = normalizeWorkerRemoteExecutionInboxRecord({
...record,
revision: record.revision + 1,
updatedAtMs: Math.max(record.updatedAtMs, result.renewedAtMs),
offer: {
...record.offer,
lease: {
...record.offer.lease,
version: result.leaseVersion,
renewedAtMs: result.renewedAtMs,
expiresAtMs: result.expiresAtMs,
updatedAtMs: result.renewedAtMs,
},
},
});
try {
await this.inbox.replaceOffer(next, record.revision);
return next;
} catch (error) {
const current = await this.inbox.readOffer(record.offer.offerId);
if (current) {
const normalized = normalizeWorkerRemoteExecutionInboxRecord(current);
if (normalized.offer.lease.version >= result.leaseVersion) {
return normalized;
}
}
throw error;
}
}
private async markLeaseLost(
record: WorkerRemoteExecutionInboxRecord,
recoveryReason:
| 'lease_lost_local_execution_stopped'
| 'lease_lost_local_execution_unverified',
): Promise<void> {
if (record.state === 'recovery_required') return;
if (record.state === 'start_failure_acknowledged') return;
await this.replaceRecovery(record, recoveryReason);
}
private async markControlPlaneTerminal(
record: WorkerRemoteExecutionInboxRecord,
): Promise<void> {
if (
record.state === 'recovery_required' ||
!CONTROL_PLANE_TERMINAL_TRANSITIONS.has(record.state)
) return;
await this.replaceRecovery(record, 'control_plane_terminal');
}
private async replaceRecovery(
record: WorkerRemoteExecutionInboxRecord,
recoveryReason: WorkerRemoteExecutionInboxRecord['recoveryReason'],
): Promise<void> {
const next = normalizeWorkerRemoteExecutionInboxRecord({
...record,
revision: record.revision + 1,
state: 'recovery_required',
recoveryReason,
updatedAtMs: Math.max(record.updatedAtMs, this.now()),
});
try {
await this.inbox.replaceOffer(next, record.revision);
} catch (error) {
const current = await this.inbox.readOffer(record.offer.offerId);
if (
current &&
normalizeWorkerRemoteExecutionInboxRecord(current).state ===
'recovery_required'
) return;
throw error;
}
}
private async stop(
record: WorkerRemoteExecutionInboxRecord,
): Promise<LocalProcessStopResult> {
if (!record.executorHandle) {
return record.state === 'launching'
? Object.freeze({ status: 'unknown' as const, reason: 'invalid_handle' as const })
: Object.freeze({ status: 'already_exited' as const });
}
return this.processes.stop(record.executorHandle);
}
private stopWasConclusive(result: LocalProcessStopResult): boolean {
return result.status === 'stopped' || result.status === 'already_exited';
}
private sessionMatches(
record: WorkerRemoteExecutionInboxRecord,
now: number,
): boolean {
const session = this.currentSessionProvider();
return Boolean(
session &&
session.workerId === record.offer.worker.workerId &&
session.sessionId === record.offer.worker.sessionId &&
session.generation === record.offer.worker.generation &&
session.status !== 'offline' &&
session.leaseExpiresAtMs > now
);
}
private async readRequired(
offerId: string,
): Promise<WorkerRemoteExecutionInboxRecord> {
const value = await this.inbox.readOffer(offerId);
if (!value) {
throw new WorkerRemoteExecutionControlCoordinatorError(
'invalid_configuration',
);
}
return normalizeWorkerRemoteExecutionInboxRecord(value);
}
private now(): number {
const value = this.nowProvider();
if (!Number.isSafeInteger(value) || value < 0) {
throw new WorkerRemoteExecutionControlCoordinatorError('invalid_clock');
}
return value;
}
}
packages/ql3-worker-runtime/src/execution/workerFileLogArtifactAllocator.ts
+642
View File
@@ -0,0 +1,642 @@
// Worker Execution owns bounded local log allocation, output, and read leases.
import { createHash } from 'node:crypto';
import { constants } from 'node:fs';
import fs, { type FileHandle } from 'node:fs/promises';
import path from 'node:path';
import { assertRunDispatchId } from '@qinglong/runtime-core/run-dispatch-lease';
import type {
WorkerRemoteExecutionOutputChunk,
WorkerRemoteExecutionOutputSink,
} from '../remote-execution/executionInboxProcessor';
import type {
WorkerRemoteLogArtifactAllocator,
WorkerRemoteLogArtifactPreparation,
} from '../remote-execution/executionContextMaterializer';
const MEBIBYTE = 1024 * 1024;
const MAXIMUM_POLICY_BYTES = 1024 * MEBIBYTE;
const MAXIMUM_RESERVE_BYTES = 1024 * 1024 * MEBIBYTE;
const ARTIFACT_ID_PREFIX = 'wlog-';
const ARTIFACT_ID_DIGEST_LENGTH = 30;
const ARTIFACT_ID_DOMAIN = 'qinglong/worker-log-artifact@v1';
const WORKER_FILE_LOG_OUTPUT_PLAN = Symbol('worker-file-log-output-plan');
export type WorkerRemoteLogArtifactProfile = 'edge' | 'node';
export interface WorkerRemoteLogArtifactPolicy {
readonly maximumAttemptBytes: number;
readonly minimumFreeBytes: number;
readonly maximumWriteChunkBytes: number;
}
export interface WorkerRemoteLogArtifactCapacityProbe {
availableBytes(root: string): Promise<bigint>;
}
export interface WorkerFileLogArtifactAllocatorOptions {
readonly root: string;
readonly policy: WorkerRemoteLogArtifactPolicy;
readonly capacity?: WorkerRemoteLogArtifactCapacityProbe;
}
export interface WorkerFileLogOutputPlan {
readonly filePath: string;
readonly maximumBytes: number;
readonly logArtifactId: string;
}
export interface WorkerRemoteLogArtifactReadRequest {
readonly runId: string;
readonly attemptId: string;
readonly logArtifactId: string;
}
export interface WorkerRemoteLogArtifactReadLease {
readonly logArtifactId: string;
readonly byteLength: number;
/** Undefined means the launcher's bounded truncation fact was unavailable. */
readonly truncated: boolean | undefined;
chunks(): AsyncIterable<Uint8Array>;
close(): Promise<void>;
}
export interface WorkerRemoteLogArtifactSource {
open(
request: WorkerRemoteLogArtifactReadRequest,
): Promise<WorkerRemoteLogArtifactReadLease | undefined>;
}
type PlannedWorkerRemoteExecutionOutputSink = WorkerRemoteExecutionOutputSink & {
readonly [WORKER_FILE_LOG_OUTPUT_PLAN]?: () => WorkerFileLogOutputPlan;
};
/** Adapter-private path capability; it is non-enumerable on the output sink. */
export function workerFileLogOutputPlan(
output: WorkerRemoteExecutionOutputSink,
): Readonly<WorkerFileLogOutputPlan> | undefined {
return (output as PlannedWorkerRemoteExecutionOutputSink)[
WORKER_FILE_LOG_OUTPUT_PLAN
]?.();
}
export class WorkerRemoteLogArtifactError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'invalid_request'
| 'capacity_unavailable'
| 'unsafe_path'
| 'quota_exceeded'
| 'invalid_output'
| 'closed',
) {
super(`Worker remote log Artifact failed: ${reason}`);
this.name = 'WorkerRemoteLogArtifactError';
}
}
export function workerRemoteLogArtifactPolicy(
profile: WorkerRemoteLogArtifactProfile,
): Readonly<WorkerRemoteLogArtifactPolicy> {
if (profile === 'edge') {
return Object.freeze({
maximumAttemptBytes: 4 * MEBIBYTE,
minimumFreeBytes: 32 * MEBIBYTE,
maximumWriteChunkBytes: MEBIBYTE,
});
}
if (profile === 'node') {
return Object.freeze({
maximumAttemptBytes: 64 * MEBIBYTE,
minimumFreeBytes: 256 * MEBIBYTE,
maximumWriteChunkBytes: MEBIBYTE,
});
}
throw new WorkerRemoteLogArtifactError('invalid_configuration');
}
function normalizePolicy(
policy: WorkerRemoteLogArtifactPolicy,
): Readonly<WorkerRemoteLogArtifactPolicy> {
if (
!policy ||
!Number.isSafeInteger(policy.maximumAttemptBytes) ||
policy.maximumAttemptBytes < 1 ||
policy.maximumAttemptBytes > MAXIMUM_POLICY_BYTES ||
!Number.isSafeInteger(policy.minimumFreeBytes) ||
policy.minimumFreeBytes < 0 ||
policy.minimumFreeBytes > MAXIMUM_RESERVE_BYTES ||
!Number.isSafeInteger(policy.maximumWriteChunkBytes) ||
policy.maximumWriteChunkBytes < 1 ||
policy.maximumWriteChunkBytes > MEBIBYTE
) {
throw new WorkerRemoteLogArtifactError('invalid_configuration');
}
return Object.freeze({
maximumAttemptBytes: policy.maximumAttemptBytes,
minimumFreeBytes: policy.minimumFreeBytes,
maximumWriteChunkBytes: policy.maximumWriteChunkBytes,
});
}
function currentUid(): number | undefined {
return typeof process.getuid === 'function' ? process.getuid() : undefined;
}
function assertOwnedOrdinaryFile(stat: Awaited<ReturnType<FileHandle['stat']>>): void {
const uid = currentUid();
if (
!stat.isFile() ||
stat.nlink !== 1 ||
(uid !== undefined && stat.uid !== uid)
) {
throw new WorkerRemoteLogArtifactError('unsafe_path');
}
}
async function privateDirectory(directory: string): Promise<Readonly<{
dev: number;
ino: number;
}>> {
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
const before = await fs.lstat(directory);
const uid = currentUid();
if (
!before.isDirectory() ||
before.isSymbolicLink() ||
(uid !== undefined && before.uid !== uid)
) {
throw new WorkerRemoteLogArtifactError('unsafe_path');
}
await fs.chmod(directory, 0o700);
const after = await fs.lstat(directory);
if (
!after.isDirectory() ||
after.isSymbolicLink() ||
after.dev !== before.dev ||
after.ino !== before.ino ||
(after.mode & 0o777) !== 0o700 ||
(uid !== undefined && after.uid !== uid)
) {
throw new WorkerRemoteLogArtifactError('unsafe_path');
}
return Object.freeze({ dev: after.dev, ino: after.ino });
}
function requestId(name: string, value: unknown): string {
try {
if (typeof value !== 'string') {
throw new TypeError('invalid ID');
}
assertRunDispatchId(name, value);
return value;
} catch {
throw new WorkerRemoteLogArtifactError('invalid_request');
}
}
export function createWorkerRemoteLogArtifactId(request: Readonly<{
projectId: string;
runId: string;
attemptId: string;
offerId: string;
}>): string {
if (!request || typeof request !== 'object') {
throw new WorkerRemoteLogArtifactError('invalid_request');
}
const values = [
requestId('projectId', request.projectId),
requestId('runId', request.runId),
requestId('attemptId', request.attemptId),
requestId('offerId', request.offerId),
];
const digest = createHash('sha256');
digest.update(ARTIFACT_ID_DOMAIN, 'utf8');
for (const value of values) {
digest.update('\0', 'utf8');
digest.update(value, 'utf8');
}
const artifactId = `${ARTIFACT_ID_PREFIX}${digest.digest('hex').slice(
0,
ARTIFACT_ID_DIGEST_LENGTH,
)}`;
assertRunDispatchId('logArtifactId', artifactId);
return artifactId;
}
class FileSystemCapacityProbe implements WorkerRemoteLogArtifactCapacityProbe {
async availableBytes(root: string): Promise<bigint> {
const stat = await fs.statfs(root, { bigint: true });
return stat.bavail * stat.bsize;
}
}
class WorkerFileLogOutput implements WorkerRemoteExecutionOutputSink {
private pending: Promise<unknown> = Promise.resolve();
private closeOperation: Promise<void> | undefined;
private state: 'open' | 'closing' | 'closed' = 'open';
private remainingBytes: number;
constructor(
readonly logArtifactId: string,
private readonly file: FileHandle,
maximumBytes: number,
existingBytes: number,
private readonly maximumWriteChunkBytes: number,
outputFilePath: string,
) {
this.remainingBytes = maximumBytes - existingBytes;
Object.defineProperty(this, WORKER_FILE_LOG_OUTPUT_PLAN, {
configurable: false,
enumerable: false,
writable: false,
value: () => Object.freeze({
filePath: outputFilePath,
maximumBytes,
logArtifactId,
}),
});
}
write(output: WorkerRemoteExecutionOutputChunk): Promise<void> {
if (this.state !== 'open') {
return Promise.reject(new WorkerRemoteLogArtifactError('closed'));
}
if (
!output ||
typeof output !== 'object' ||
(output.stream !== 'stdout' && output.stream !== 'stderr') ||
!(output.chunk instanceof Uint8Array) ||
output.chunk.byteLength > this.maximumWriteChunkBytes ||
!Number.isSafeInteger(output.observedAtMs) ||
output.observedAtMs < 0
) {
return Promise.reject(new WorkerRemoteLogArtifactError('invalid_output'));
}
const chunk = Buffer.from(output.chunk);
const operation = this.pending.then(async () => {
if (chunk.byteLength === 0) return;
if (this.remainingBytes <= 0) {
throw new WorkerRemoteLogArtifactError('quota_exceeded');
}
const accepted = chunk.subarray(
0,
Math.min(chunk.byteLength, this.remainingBytes),
);
let offset = 0;
while (offset < accepted.byteLength) {
const result = await this.file.write(accepted.subarray(offset));
if (result.bytesWritten < 1) {
throw new Error('Worker remote log Artifact write made no progress');
}
offset += result.bytesWritten;
this.remainingBytes -= result.bytesWritten;
}
if (accepted.byteLength !== chunk.byteLength) {
throw new WorkerRemoteLogArtifactError('quota_exceeded');
}
});
this.pending = operation.catch(() => undefined);
return operation;
}
close(): Promise<void> {
if (this.closeOperation) return this.closeOperation;
this.state = 'closing';
this.closeOperation = (async () => {
await this.pending.catch(() => undefined);
let syncError: unknown;
try {
await this.file.datasync();
} catch (error) {
syncError = error;
}
try {
await this.file.close();
} finally {
this.state = 'closed';
}
if (syncError !== undefined) throw syncError;
})();
return this.closeOperation;
}
}
class WorkerFileLogReadLease implements WorkerRemoteLogArtifactReadLease {
private consumed = false;
private closeOperation: Promise<void> | undefined;
constructor(
readonly logArtifactId: string,
readonly byteLength: number,
readonly truncated: boolean | undefined,
private readonly file: FileHandle,
private readonly chunkBytes: number,
) {}
chunks(): AsyncIterable<Uint8Array> {
if (this.consumed) {
throw new WorkerRemoteLogArtifactError('closed');
}
this.consumed = true;
const file = this.file;
const byteLength = this.byteLength;
const chunkBytes = this.chunkBytes;
const close = () => this.close();
return (async function* () {
let offset = 0;
try {
while (offset < byteLength) {
const buffer = Buffer.allocUnsafe(
Math.min(chunkBytes, byteLength - offset),
);
const result = await file.read(buffer, 0, buffer.length, offset);
if (result.bytesRead < 1) {
throw new WorkerRemoteLogArtifactError('invalid_output');
}
offset += result.bytesRead;
yield Buffer.from(buffer.subarray(0, result.bytesRead));
}
} finally {
await close();
}
})();
}
close(): Promise<void> {
this.closeOperation ??= this.file.close();
return this.closeOperation;
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
'code' in error &&
(error as NodeJS.ErrnoException).code === code
);
}
async function readTruncationFact(
directory: string,
request: WorkerRemoteLogArtifactReadRequest,
maximumBytes: number,
): Promise<boolean | undefined> {
const target = path.join(
directory,
`.${request.logArtifactId}.log.truncated.json`,
);
let file: FileHandle;
try {
file = await fs.open(
target,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
if (isCode(error, 'ELOOP')) {
throw new WorkerRemoteLogArtifactError('unsafe_path');
}
throw error;
}
try {
const stat = await file.stat();
assertOwnedOrdinaryFile(stat);
if (stat.size < 1 || stat.size > 4096 || (stat.mode & 0o777) !== 0o600) {
throw new WorkerRemoteLogArtifactError('invalid_output');
}
const bytes = Buffer.allocUnsafe(stat.size + 1);
const result = await file.read(bytes, 0, bytes.length, 0);
if (result.bytesRead !== stat.size) {
throw new WorkerRemoteLogArtifactError('invalid_output');
}
let value: unknown;
try {
value = JSON.parse(bytes.subarray(0, result.bytesRead).toString('utf8'));
} catch {
throw new WorkerRemoteLogArtifactError('invalid_output');
}
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new WorkerRemoteLogArtifactError('invalid_output');
}
const record = value as Record<string, unknown>;
const keys = Object.keys(record).sort();
if (
JSON.stringify(keys) !== JSON.stringify([
'attemptId', 'logArtifactId', 'maximumBytes', 'observedAtMs',
'quotaReached', 'runId', 'schemaVersion',
]) ||
record.schemaVersion !== 1 ||
record.runId !== request.runId ||
record.attemptId !== request.attemptId ||
record.logArtifactId !== request.logArtifactId ||
record.maximumBytes !== maximumBytes ||
typeof record.quotaReached !== 'boolean' ||
!Number.isSafeInteger(record.observedAtMs) ||
(record.observedAtMs as number) < 0
) {
throw new WorkerRemoteLogArtifactError('invalid_output');
}
return record.quotaReached;
} finally {
await file.close();
}
}
export class WorkerFileLogArtifactAllocator
implements WorkerRemoteLogArtifactAllocator, WorkerRemoteLogArtifactSource {
private readonly root: string;
private readonly policy: Readonly<WorkerRemoteLogArtifactPolicy>;
private readonly capacity: WorkerRemoteLogArtifactCapacityProbe;
constructor(options: WorkerFileLogArtifactAllocatorOptions) {
if (
!options ||
typeof options.root !== 'string' ||
!path.isAbsolute(options.root) ||
options.root.includes('\0') ||
(options.capacity !== undefined &&
typeof options.capacity.availableBytes !== 'function')
) {
throw new WorkerRemoteLogArtifactError('invalid_configuration');
}
this.root = path.resolve(options.root);
this.policy = normalizePolicy(options.policy);
this.capacity = options.capacity ?? new FileSystemCapacityProbe();
}
async prepare(request: Readonly<{
projectId: string;
runId: string;
attemptId: string;
offerId: string;
}>): Promise<WorkerRemoteLogArtifactPreparation> {
const logArtifactId = createWorkerRemoteLogArtifactId(request);
const rootIdentity = await privateDirectory(this.root);
let availableBytes: bigint;
try {
availableBytes = await this.capacity.availableBytes(this.root);
} catch {
throw new WorkerRemoteLogArtifactError('capacity_unavailable');
}
if (typeof availableBytes !== 'bigint' || availableBytes < 0n) {
throw new WorkerRemoteLogArtifactError('capacity_unavailable');
}
const requiredBytes = BigInt(this.policy.minimumFreeBytes) +
BigInt(this.policy.maximumAttemptBytes);
if (availableBytes < requiredBytes) {
throw new WorkerRemoteLogArtifactError('capacity_unavailable');
}
const rootAfterCapacity = await fs.lstat(this.root);
if (
rootAfterCapacity.dev !== rootIdentity.dev ||
rootAfterCapacity.ino !== rootIdentity.ino
) {
throw new WorkerRemoteLogArtifactError('unsafe_path');
}
const shard = logArtifactId.slice(
ARTIFACT_ID_PREFIX.length,
ARTIFACT_ID_PREFIX.length + 2,
);
const directory = path.join(this.root, shard);
const directoryIdentity = await privateDirectory(directory);
const outputFilePath = path.join(directory, `${logArtifactId}.log`);
let file: FileHandle;
try {
file = await fs.open(
outputFilePath,
constants.O_WRONLY |
constants.O_CREAT |
constants.O_APPEND |
(constants.O_NOFOLLOW ?? 0),
0o600,
);
} catch (error) {
if ((error as NodeJS.ErrnoException)?.code === 'ELOOP') {
throw new WorkerRemoteLogArtifactError('unsafe_path');
}
throw error;
}
try {
const stat = await file.stat();
assertOwnedOrdinaryFile(stat);
if (
!Number.isSafeInteger(stat.size) ||
stat.size < 0 ||
stat.size > this.policy.maximumAttemptBytes
) {
throw new WorkerRemoteLogArtifactError('quota_exceeded');
}
await file.chmod(0o600);
const pathStat = await fs.lstat(outputFilePath);
const directoryAfterOpen = await fs.lstat(directory);
if (
pathStat.isSymbolicLink() ||
pathStat.dev !== stat.dev ||
pathStat.ino !== stat.ino ||
(pathStat.mode & 0o777) !== 0o600 ||
directoryAfterOpen.dev !== directoryIdentity.dev ||
directoryAfterOpen.ino !== directoryIdentity.ino
) {
throw new WorkerRemoteLogArtifactError('unsafe_path');
}
const output = new WorkerFileLogOutput(
logArtifactId,
file,
this.policy.maximumAttemptBytes,
stat.size,
this.policy.maximumWriteChunkBytes,
outputFilePath,
);
let ownership: 'prepared' | 'handed_off' | 'released' = 'prepared';
return Object.freeze({
logArtifactId,
takeOutput() {
if (ownership !== 'prepared') {
throw new WorkerRemoteLogArtifactError('closed');
}
ownership = 'handed_off';
return output;
},
async release() {
if (ownership === 'released' || ownership === 'handed_off') return;
ownership = 'released';
await output.close();
},
});
} catch (error) {
await file.close().catch(() => undefined);
throw error;
}
}
async open(
request: WorkerRemoteLogArtifactReadRequest,
): Promise<WorkerRemoteLogArtifactReadLease | undefined> {
const runId = requestId('runId', request?.runId);
const attemptId = requestId('attemptId', request?.attemptId);
const logArtifactId = requestId('logArtifactId', request?.logArtifactId);
if (!/^wlog-[0-9a-f]{30}$/.test(logArtifactId)) {
throw new WorkerRemoteLogArtifactError('invalid_request');
}
const rootIdentity = await privateDirectory(this.root);
const directory = path.join(this.root, logArtifactId.slice(5, 7));
const directoryIdentity = await privateDirectory(directory);
const target = path.join(directory, `${logArtifactId}.log`);
let file: FileHandle;
try {
file = await fs.open(
target,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
if (isCode(error, 'ELOOP')) {
throw new WorkerRemoteLogArtifactError('unsafe_path');
}
throw error;
}
try {
const stat = await file.stat();
assertOwnedOrdinaryFile(stat);
if (
!Number.isSafeInteger(stat.size) ||
stat.size < 0 ||
stat.size > this.policy.maximumAttemptBytes ||
(stat.mode & 0o777) !== 0o600
) {
throw new WorkerRemoteLogArtifactError('invalid_output');
}
const pathStat = await fs.lstat(target);
const rootAfterOpen = await fs.lstat(this.root);
const directoryAfterOpen = await fs.lstat(directory);
if (
pathStat.isSymbolicLink() ||
pathStat.dev !== stat.dev ||
pathStat.ino !== stat.ino ||
rootAfterOpen.dev !== rootIdentity.dev ||
rootAfterOpen.ino !== rootIdentity.ino ||
directoryAfterOpen.dev !== directoryIdentity.dev ||
directoryAfterOpen.ino !== directoryIdentity.ino
) {
throw new WorkerRemoteLogArtifactError('unsafe_path');
}
const truncated = await readTruncationFact(
directory,
{ runId, attemptId, logArtifactId },
this.policy.maximumAttemptBytes,
);
return new WorkerFileLogReadLease(
logArtifactId,
stat.size,
truncated,
file,
Math.min(this.policy.maximumWriteChunkBytes, 64 * 1024),
);
} catch (error) {
await file.close().catch(() => undefined);
throw error;
}
}
}
packages/ql3-worker-runtime/src/execution/workerPosixExecutionExecutor.ts
+215
View File
@@ -0,0 +1,215 @@
// Worker Execution owns the fenced POSIX launch adapter and spawn barrier.
import { createHash } from 'node:crypto';
import {
LocalProcessLaunchError,
LocalProcessLauncher,
type LocalProcessIdentityProvider,
} from '@qinglong/local-process';
import { assertRunDispatchId } from '@qinglong/runtime-core/run-dispatch-lease';
import {
normalizeWorkerRemoteExecutionInboxRecord,
type WorkerRemoteExecutionInbox,
} from '../remote-execution/executionInbox';
import type {
WorkerRemoteExecutionExecutor,
WorkerRemoteExecutionLaunch,
} from '../remote-execution/executionInboxProcessor';
import { workerFileLogOutputPlan } from './workerFileLogArtifactAllocator';
export interface WorkerRemoteExecutionSpawnBarrier {
verify(input: Readonly<{
offerId: string;
runId: string;
attemptId: string;
callbackSequence: number;
callbackTokenDigest: string;
logArtifactId: string;
executorStartedAtMs: number;
}>): Promise<void>;
}
export class WorkerInboxExecutionSpawnBarrier
implements WorkerRemoteExecutionSpawnBarrier {
constructor(private readonly inbox: Pick<WorkerRemoteExecutionInbox, 'readOffer'>) {
if (!inbox || typeof inbox.readOffer !== 'function') {
throw new TypeError('Worker execution spawn barrier inbox is invalid');
}
}
async verify(input: Readonly<{
offerId: string;
runId: string;
attemptId: string;
callbackSequence: number;
callbackTokenDigest: string;
logArtifactId: string;
executorStartedAtMs: number;
}>): Promise<void> {
const record = await this.inbox.readOffer(input.offerId);
if (!record) throw new Error('Worker execution spawn barrier is missing');
const current = normalizeWorkerRemoteExecutionInboxRecord(record);
if (
current.state !== 'launching' ||
current.offer.offerId !== input.offerId ||
current.offer.candidate.runId !== input.runId ||
current.offer.candidate.attemptId !== input.attemptId ||
current.completionReceiptCallbackSequence !== input.callbackSequence ||
current.completionReceiptTokenDigest !== input.callbackTokenDigest ||
current.logArtifactId !== input.logArtifactId ||
current.executorStartedAtMs !== input.executorStartedAtMs
) {
throw new Error('Worker execution spawn barrier authority drifted');
}
}
}
export interface WorkerPosixExecutionExecutorOptions {
readonly barrier: WorkerRemoteExecutionSpawnBarrier;
readonly receiptRoot: string;
readonly launcherPath?: string;
readonly expectedLauncherSha256?: string;
readonly identityProvider?: LocalProcessIdentityProvider;
readonly clock?: { now(): number };
readonly createHandleId?: () => string;
}
export class WorkerPosixExecutionExecutor
implements WorkerRemoteExecutionExecutor {
private readonly options: WorkerPosixExecutionExecutorOptions;
constructor(options: WorkerPosixExecutionExecutorOptions) {
if (!options || typeof options.barrier?.verify !== 'function') {
throw new TypeError('Worker POSIX Executor barrier is invalid');
}
this.options = options;
}
async start(launch: WorkerRemoteExecutionLaunch): Promise<
| Readonly<{
status: 'started';
executorHandle: string;
executorStartedAtMs: number;
}>
| Readonly<{ status: 'rejected' }>
> {
let callbackToken: Buffer | undefined;
try {
assertRunDispatchId('offerId', launch.offerId);
assertRunDispatchId('runId', launch.runId);
assertRunDispatchId('attemptId', launch.attemptId);
assertRunDispatchId('logArtifactId', launch.logArtifactId);
if (
(launch.timeoutMs === undefined) !==
(launch.executionDeadlineAtMs === undefined) ||
(launch.executionDeadlineAtMs !== undefined &&
(!Number.isSafeInteger(launch.executionDeadlineAtMs) ||
launch.executionDeadlineAtMs < 0))
) return this.reject(launch);
const outputPlan = workerFileLogOutputPlan(launch.output);
if (
!outputPlan ||
outputPlan.logArtifactId !== launch.logArtifactId
) return this.reject(launch);
callbackToken = Buffer.from(launch.completionCallback.token);
if (callbackToken.byteLength !== 32) return this.reject(launch);
const callbackTokenDigest = createHash('sha256')
.update(callbackToken)
.digest('hex');
const environment: Record<string, string> = {};
for (const entry of launch.environment) {
if (Object.hasOwn(environment, entry.name)) return this.reject(launch);
environment[entry.name] = entry.value;
}
await launch.output.close();
const launcher = new LocalProcessLauncher(
{
register: async (registered: Readonly<{
runId: string;
attemptId: string;
registeredAtMs: number;
}>) => {
if (
registered.runId !== launch.runId ||
registered.attemptId !== launch.attemptId
) throw new Error('Worker POSIX Executor journal authority drifted');
await this.options.barrier.verify(Object.freeze({
offerId: launch.offerId,
runId: launch.runId,
attemptId: launch.attemptId,
callbackSequence: launch.completionCallback.sequence,
callbackTokenDigest,
logArtifactId: launch.logArtifactId,
executorStartedAtMs: launch.executorStartedAtMs,
}));
},
},
{
receiptRoot: this.options.receiptRoot,
...(this.options.launcherPath === undefined
? {}
: { launcherPath: this.options.launcherPath }),
...(this.options.expectedLauncherSha256 === undefined
? {}
: { expectedLauncherSha256: this.options.expectedLauncherSha256 }),
...(this.options.identityProvider === undefined
? {}
: { identityProvider: this.options.identityProvider }),
...(this.options.clock === undefined
? {}
: { clock: this.options.clock }),
...(this.options.createHandleId === undefined
? {}
: { createHandleId: this.options.createHandleId }),
},
);
const command = launch.command.kind === 'argv'
? Object.freeze({
kind: 'argv' as const,
file: launch.command.file,
args: launch.command.args,
})
: Object.freeze({
kind: 'shell' as const,
command: launch.command.command,
...(launch.command.shell === undefined
? {}
: { shell: launch.command.shell as '/bin/sh' | '/bin/bash' }),
});
const handle = await launcher.start({
runId: launch.runId,
attemptId: launch.attemptId,
startedAtMs: launch.executorStartedAtMs,
callbackSequence: launch.completionCallback.sequence,
callbackToken: callbackToken.toString('base64url'),
command,
environment,
...(launch.workingDirectory === undefined
? {}
: { workingDirectory: launch.workingDirectory }),
output: outputPlan,
});
void handle.completion;
return Object.freeze({
status: 'started' as const,
executorHandle: handle.durableHandle,
executorStartedAtMs: handle.startedAtMs,
});
} catch (error) {
await launch?.output?.close?.().catch(() => undefined);
if (
error instanceof LocalProcessLaunchError &&
error.spawnOutcome === 'unknown'
) throw error;
return Object.freeze({ status: 'rejected' as const });
} finally {
callbackToken?.fill(0);
}
}
private async reject(
launch: WorkerRemoteExecutionLaunch,
): Promise<Readonly<{ status: 'rejected' }>> {
await launch.output.close().catch(() => undefined);
return Object.freeze({ status: 'rejected' as const });
}
}
packages/ql3-worker-runtime/src/index.ts
+7
View File
@@ -0,0 +1,7 @@
export type {
GenerateWorkerCertificateEnrollmentOptions,
WorkerCertificateEnrollmentMaterial,
} from './credential/workerCertificateEnrollment';
export * from './credential/workerCertificateIdentity';
export * from './credential/workerCertificateStore';
export * from './credential/workerCertificateRenewal';
packages/ql3-worker-runtime/src/process/workerProcessApplication.ts
+304
View File
@@ -0,0 +1,304 @@
// Worker Process owns configuration-to-runtime activation, signals, and shutdown.
import type { WorkerIngressHttpsCredentialProvider } from '../remote-execution/transport/workerIngressHttpsClient';
import {
startProductionWorkerApplication,
type ProductionWorkerApplicationEnabledOptions,
type ProductionWorkerApplicationOptions,
} from '../application-runtime/productionWorkerApplication';
import {
loadWorkerProcessConfig,
type EnabledWorkerProcessConfig,
type WorkerProcessEnvironment,
} from './workerProcessConfig';
import { createWorkerProcessCredentialProvider } from './workerProcessIdentity';
import type {
ProductionWorkerCertificateRenewalLifecycle,
ProductionWorkerHeadlessApplicationResult,
ProductionWorkerHeadlessStopResult,
} from '../application-runtime/productionHeadlessApplication';
export type WorkerProcessSignal = 'SIGINT' | 'SIGTERM';
export interface WorkerProcessSignalSource {
subscribe(listener: (signal: WorkerProcessSignal) => void): () => void;
}
export interface WorkerProcessEvent {
readonly schemaVersion: 1;
readonly component: 'qinglong3-worker';
readonly level: 'info' | 'error';
readonly event:
| 'starting'
| 'active'
| 'shutdown_requested'
| 'shutdown_deferred'
| 'stopped'
| 'runtime_diagnostic';
readonly workerId?: string;
readonly capacityProfile?: 'edge' | 'node';
readonly signal?: WorkerProcessSignal;
readonly stopResult?: ProductionWorkerHeadlessStopResult;
readonly diagnostic?: Readonly<{
readonly code:
| 'tick_failed'
| 'session_tick_failed'
| 'certificate_renewal_failed'
| 'certificate_unavailable'
| 'recovery_required'
| 'drain_failed'
| 'disconnect_failed';
}>;
}
export type WorkerProcessStarter = (
options: ProductionWorkerApplicationOptions,
) => Promise<ProductionWorkerHeadlessApplicationResult>;
export type WorkerProcessCredentialFactory = (
config: EnabledWorkerProcessConfig['identity'],
) => Promise<Readonly<WorkerIngressHttpsCredentialProvider>>;
export type WorkerProcessCertificateRenewalFactory = (
config: EnabledWorkerProcessConfig,
credentials: Readonly<WorkerIngressHttpsCredentialProvider>,
) => Promise<ProductionWorkerCertificateRenewalLifecycle | undefined>;
export interface ProductionWorkerProcessOptions {
readonly environment: WorkerProcessEnvironment;
readonly signals: WorkerProcessSignalSource;
readonly emit: (event: WorkerProcessEvent) => void | Promise<void>;
readonly start?: WorkerProcessStarter;
readonly createCredentials?: WorkerProcessCredentialFactory;
/** Deployment-owned CA adapter; omitted profiles keep zero renewal cost. */
readonly createCertificateRenewal?: WorkerProcessCertificateRenewalFactory;
readonly waitBeforeStopRetry?: () => Promise<void>;
}
export class WorkerProcessError extends Error {
readonly code:
| 'QL3_WORKER_PROCESS_DISABLED'
| 'QL3_WORKER_PROCESS_ACTIVATION_FAILED';
constructor(
code: WorkerProcessError['code'],
message: string,
options?: ErrorOptions,
) {
super(message, options);
this.name = 'WorkerProcessError';
this.code = code;
}
}
function delay(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 1_000));
}
function productionOptions(
config: EnabledWorkerProcessConfig,
credentials: Readonly<WorkerIngressHttpsCredentialProvider>,
certificateRenewal: ProductionWorkerCertificateRenewalLifecycle | undefined,
diagnostic: NonNullable<
ProductionWorkerApplicationEnabledOptions['diagnostic']
>,
): ProductionWorkerApplicationEnabledOptions {
return Object.freeze({
enabled: true,
profile: 'worker',
capacityProfile: config.capacityProfile,
origin: config.origin,
credentials,
...(certificateRenewal === undefined ? {} : { certificateRenewal }),
workerId: config.workerId,
capabilities: config.capabilities,
maxConcurrentRuns: config.maxConcurrentRuns,
storage: config.storage,
cadenceMs: config.lifecycle.cadenceMs,
leaseDurationMs: config.lifecycle.leaseDurationMs,
heartbeatIntervalMs: config.lifecycle.heartbeatIntervalMs,
drainTimeoutMs: config.lifecycle.drainTimeoutMs,
drainPollMs: config.lifecycle.drainPollMs,
requestTimeoutMs: config.lifecycle.requestTimeoutMs,
maximumJournalEntries: config.lifecycle.maximumJournalEntries,
maximumRecordsPerTick: config.lifecycle.maximumRecordsPerTick,
maximumSupervisionRecordsPerTick:
config.lifecycle.maximumSupervisionRecordsPerTick,
...(config.executor.launcherPath === undefined
? {}
: {
launcherPath: config.executor.launcherPath,
expectedLauncherSha256: config.executor.expectedLauncherSha256!,
}),
diagnostic,
});
}
/**
* Owns one production Worker process. Incomplete drain never releases the
* application or returns success; it keeps the existing owner/Agent alive and
* retries the same proof-bearing stop operation with a ref'ed shutdown wait.
*/
export async function runProductionWorkerProcess(
options: ProductionWorkerProcessOptions,
): Promise<'stopped'> {
if (
!options ||
typeof options !== 'object' ||
typeof options.emit !== 'function' ||
typeof options.signals?.subscribe !== 'function' ||
(options.start !== undefined && typeof options.start !== 'function') ||
(options.createCredentials !== undefined &&
typeof options.createCredentials !== 'function') ||
(options.createCertificateRenewal !== undefined &&
typeof options.createCertificateRenewal !== 'function') ||
(options.waitBeforeStopRetry !== undefined &&
typeof options.waitBeforeStopRetry !== 'function')
) {
throw new TypeError('Worker process options are invalid');
}
let resolveSignal: ((signal: WorkerProcessSignal) => void) | undefined;
const requestedSignal = new Promise<WorkerProcessSignal>((resolve) => {
resolveSignal = resolve;
});
let acceptedSignal = false;
const unsubscribe = options.signals.subscribe((signal) => {
if (acceptedSignal) return;
acceptedSignal = true;
resolveSignal?.(signal);
});
// A pending Promise does not keep Node's event loop alive. The Worker owns
// the process until an OS shutdown signal arrives, so retain one inexpensive
// referenced handle and release it with the rest of the signal authority.
const lifecycleReference = setInterval(() => undefined, 2_147_483_647);
const emit = (event: WorkerProcessEvent): void => {
void Promise.resolve(options.emit(event)).catch(() => undefined);
};
try {
const config = await loadWorkerProcessConfig(options.environment);
if (!config.enabled) {
throw new WorkerProcessError(
'QL3_WORKER_PROCESS_DISABLED',
'The Worker process requires an enabled worker profile',
);
}
emit(
Object.freeze({
schemaVersion: 1,
component: 'qinglong3-worker',
level: 'info',
event: 'starting',
workerId: config.workerId,
capacityProfile: config.capacityProfile,
}),
);
const credentials = await (
options.createCredentials ?? createWorkerProcessCredentialProvider
)(config.identity);
if (!credentials || typeof credentials.load !== 'function') {
throw new WorkerProcessError(
'QL3_WORKER_PROCESS_ACTIVATION_FAILED',
'Worker credential provider is invalid',
);
}
const certificateRenewal = await options.createCertificateRenewal?.(
config,
credentials,
);
if (
certificateRenewal !== undefined &&
typeof certificateRenewal.run !== 'function'
) {
throw new WorkerProcessError(
'QL3_WORKER_PROCESS_ACTIVATION_FAILED',
'Worker certificate renewal lifecycle is invalid',
);
}
const application = await (
options.start ?? startProductionWorkerApplication
)(
productionOptions(config, credentials, certificateRenewal, (fact) => {
emit(
Object.freeze({
schemaVersion: 1,
component: 'qinglong3-worker',
level: 'error',
event: 'runtime_diagnostic',
workerId: config.workerId,
capacityProfile: config.capacityProfile,
diagnostic: Object.freeze({ code: fact.code }),
}),
);
}),
);
if (application.status !== 'active') {
throw new WorkerProcessError(
'QL3_WORKER_PROCESS_ACTIVATION_FAILED',
'Worker application did not activate',
);
}
emit(
Object.freeze({
schemaVersion: 1,
component: 'qinglong3-worker',
level: 'info',
event: 'active',
workerId: config.workerId,
capacityProfile: config.capacityProfile,
}),
);
const signal = await requestedSignal;
emit(
Object.freeze({
schemaVersion: 1,
component: 'qinglong3-worker',
level: 'info',
event: 'shutdown_requested',
workerId: config.workerId,
capacityProfile: config.capacityProfile,
signal,
}),
);
const waitBeforeRetry = options.waitBeforeStopRetry ?? delay;
while (true) {
let result: ProductionWorkerHeadlessStopResult;
try {
result = await application.stop();
} catch {
result = 'recovery_required';
}
if (result === 'stopped') {
emit(
Object.freeze({
schemaVersion: 1,
component: 'qinglong3-worker',
level: 'info',
event: 'stopped',
workerId: config.workerId,
capacityProfile: config.capacityProfile,
stopResult: result,
}),
);
return result;
}
emit(
Object.freeze({
schemaVersion: 1,
component: 'qinglong3-worker',
level: 'error',
event: 'shutdown_deferred',
workerId: config.workerId,
capacityProfile: config.capacityProfile,
stopResult: result,
}),
);
await waitBeforeRetry();
}
} finally {
clearInterval(lifecycleReference);
unsubscribe();
resolveSignal = undefined;
}
}
packages/ql3-worker-runtime/src/process/workerProcessCli.ts
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env node
// Worker Process owns the OS signal and diagnostic stream CLI adapter.
import {
runProductionWorkerProcess,
type WorkerProcessSignal,
type WorkerProcessSignalSource,
} from './workerProcessApplication';
function signalSource(): WorkerProcessSignalSource {
return Object.freeze({
subscribe(listener: (signal: WorkerProcessSignal) => void): () => void {
const onInterrupt = () => listener('SIGINT');
const onTerminate = () => listener('SIGTERM');
process.once('SIGINT', onInterrupt);
process.once('SIGTERM', onTerminate);
return () => {
process.off('SIGINT', onInterrupt);
process.off('SIGTERM', onTerminate);
};
},
});
}
async function main(): Promise<void> {
await runProductionWorkerProcess({
environment: process.env,
signals: signalSource(),
emit(event) {
const output = `${JSON.stringify(event)}\n`;
if (event.level === 'error') process.stderr.write(output);
else process.stdout.write(output);
},
});
}
void main().catch((error: unknown) => {
const candidate = error as {
readonly name?: unknown;
readonly code?: unknown;
};
process.stderr.write(
`${JSON.stringify({
schemaVersion: 1,
component: 'qinglong3-worker',
level: 'error',
event: 'process_failed',
name:
typeof candidate?.name === 'string'
? candidate.name.slice(0, 128)
: 'Error',
...(typeof candidate?.code === 'string'
? { code: candidate.code.slice(0, 128) }
: {}),
})}\n`,
);
process.exitCode = 1;
});
packages/ql3-worker-runtime/src/process/workerProcessConfig.ts
+448
View File
@@ -0,0 +1,448 @@
// Worker Process owns bounded environment-to-runtime configuration mapping.
import { constants } from 'node:fs';
import { open } from 'node:fs/promises';
import {
isAbsolute,
normalize,
parse,
} from 'node:path';
import {
canonicalRemoteWorkerCapabilities,
type RemoteWorkerCapabilities,
} from '@qinglong/runtime-core/remote-dispatch';
const MAX_CAPABILITIES_FILE_BYTES = 20 * 1024;
const WORKER_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const CREDENTIAL_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/;
const SHA256 = /^[0-9a-f]{64}$/;
export type WorkerProcessEnvironment = Readonly<
Record<string, string | undefined>
>;
export interface DisabledWorkerProcessConfig {
readonly enabled: false;
readonly profile: string;
}
export interface EnabledWorkerProcessConfig {
readonly enabled: true;
readonly profile: 'worker';
readonly capacityProfile: 'edge' | 'node';
readonly workerId: string;
readonly origin: string;
readonly capabilities: RemoteWorkerCapabilities;
readonly maxConcurrentRuns: number;
readonly storage: Readonly<{
readonly journalRoot: string;
readonly logRoot: string;
readonly receiptRoot: string;
}>;
readonly identity: Readonly<{
readonly certificateStoreRoot: string;
readonly trustAnchorFile: string;
readonly credentialTokenFile: string;
readonly expectedCredentialId?: string;
readonly bootstrap?: Readonly<{
readonly privateKeyFile: string;
readonly certificateChainFile: string;
}>;
}>;
readonly lifecycle: Readonly<{
readonly cadenceMs: number;
readonly leaseDurationMs: number;
readonly heartbeatIntervalMs: number;
readonly drainTimeoutMs: number;
readonly drainPollMs: number;
readonly requestTimeoutMs: number;
readonly maximumJournalEntries: number;
readonly maximumRecordsPerTick: number;
readonly maximumSupervisionRecordsPerTick: number;
}>;
readonly executor: Readonly<{
readonly launcherPath?: string;
readonly expectedLauncherSha256?: string;
}>;
}
export type WorkerProcessConfig =
| DisabledWorkerProcessConfig
| EnabledWorkerProcessConfig;
export class WorkerProcessConfigError extends TypeError {
readonly code = 'QL3_WORKER_PROCESS_CONFIG_INVALID';
constructor(message: string, options?: ErrorOptions) {
super(`Worker process configuration is invalid: ${message}`, options);
this.name = 'WorkerProcessConfigError';
}
}
function booleanValue(
environment: WorkerProcessEnvironment,
name: string,
fallback: boolean,
): boolean {
const value = environment[name];
if (value === undefined || value === '') return fallback;
if (value === 'true') return true;
if (value === 'false') return false;
throw new WorkerProcessConfigError(`${name} must be true or false`);
}
function boundedValue(
environment: WorkerProcessEnvironment,
name: string,
maximumBytes: number,
required = false,
): string | undefined {
const value = environment[name];
if (value === undefined || value === '') {
if (required) throw new WorkerProcessConfigError(`${name} is required`);
return undefined;
}
if (
Buffer.byteLength(value, 'utf8') > maximumBytes ||
/[\0\r\n]/.test(value)
) {
throw new WorkerProcessConfigError(`${name} is invalid`);
}
return value;
}
function integerValue(
environment: WorkerProcessEnvironment,
name: string,
fallback: number,
minimum: number,
maximum: number,
): number {
const value = environment[name];
if (value === undefined || value === '') return fallback;
if (!/^(0|[1-9]\d*)$/.test(value)) {
throw new WorkerProcessConfigError(`${name} must be an integer`);
}
const parsed = Number(value);
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
throw new WorkerProcessConfigError(
`${name} must be between ${minimum} and ${maximum}`,
);
}
return parsed;
}
function absolutePath(
environment: WorkerProcessEnvironment,
name: string,
required = true,
): string | undefined {
const value = boundedValue(environment, name, 4096, required);
if (value === undefined) return undefined;
if (
!isAbsolute(value) ||
parse(value).root === value ||
normalize(value) !== value
) {
throw new WorkerProcessConfigError(`${name} must be a normalized absolute path`);
}
return value;
}
function origin(environment: WorkerProcessEnvironment): string {
const value = boundedValue(
environment,
'QL3_WORKER_CONTROL_ORIGIN',
2048,
true,
)!;
let parsed: URL;
try {
parsed = new URL(value);
} catch (error) {
throw new WorkerProcessConfigError(
'QL3_WORKER_CONTROL_ORIGIN is invalid',
{ cause: error },
);
}
if (
parsed.protocol !== 'https:' ||
parsed.username !== '' ||
parsed.password !== '' ||
parsed.pathname !== '/' ||
parsed.search !== '' ||
parsed.hash !== ''
) {
throw new WorkerProcessConfigError(
'QL3_WORKER_CONTROL_ORIGIN must be an HTTPS origin',
);
}
return parsed.origin;
}
async function capabilities(path: string): Promise<RemoteWorkerCapabilities> {
let handle;
let bytes: Buffer | undefined;
try {
handle = await open(
path,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
const stat = await handle.stat();
if (
!stat.isFile() ||
stat.size < 2 ||
stat.size > MAX_CAPABILITIES_FILE_BYTES ||
(stat.mode & 0o022) !== 0
) {
throw new Error('unsafe capabilities metadata');
}
bytes = await handle.readFile();
if (
bytes.byteLength < 2 ||
bytes.byteLength > MAX_CAPABILITIES_FILE_BYTES
) {
throw new Error('unsafe capabilities size');
}
const parsed = JSON.parse(
new TextDecoder('utf-8', { fatal: true }).decode(bytes),
) as unknown;
return canonicalRemoteWorkerCapabilities(parsed).capabilities;
} catch (error) {
throw new WorkerProcessConfigError(
'QL3_WORKER_CAPABILITIES_FILE is unavailable or invalid',
{ cause: error },
);
} finally {
bytes?.fill(0);
await handle?.close().catch(() => undefined);
}
}
export async function loadWorkerProcessConfig(
environment: WorkerProcessEnvironment,
): Promise<WorkerProcessConfig> {
if (
!environment ||
typeof environment !== 'object' ||
Array.isArray(environment)
) {
throw new WorkerProcessConfigError('environment must be an object');
}
const enabled = booleanValue(
environment,
'QL3_WORKER_RUNTIME_ENABLED',
false,
);
const profile = environment.QL_DEPLOYMENT_PROFILE ?? 'standalone';
if (!enabled) return Object.freeze({ enabled: false, profile });
if (profile !== 'worker') {
throw new WorkerProcessConfigError(
'enabled runtime requires QL_DEPLOYMENT_PROFILE=worker',
);
}
const capacityProfile =
boundedValue(environment, 'QL3_WORKER_CAPACITY_PROFILE', 8) ?? 'edge';
if (capacityProfile !== 'edge' && capacityProfile !== 'node') {
throw new WorkerProcessConfigError(
'QL3_WORKER_CAPACITY_PROFILE must be edge or node',
);
}
const edge = capacityProfile === 'edge';
const workerId = boundedValue(
environment,
'QL3_WORKER_ID',
128,
true,
)!;
if (!WORKER_ID.test(workerId)) {
throw new WorkerProcessConfigError('QL3_WORKER_ID is invalid');
}
const expectedCredentialId = boundedValue(
environment,
'QL3_WORKER_EXPECTED_CREDENTIAL_ID',
64,
);
if (
expectedCredentialId !== undefined &&
!CREDENTIAL_ID.test(expectedCredentialId)
) {
throw new WorkerProcessConfigError(
'QL3_WORKER_EXPECTED_CREDENTIAL_ID is invalid',
);
}
const bootstrapPrivateKeyFile = absolutePath(
environment,
'QL3_WORKER_IDENTITY_BOOTSTRAP_PRIVATE_KEY_FILE',
false,
);
const bootstrapCertificateChainFile = absolutePath(
environment,
'QL3_WORKER_IDENTITY_BOOTSTRAP_CERTIFICATE_FILE',
false,
);
if (
(bootstrapPrivateKeyFile === undefined) !==
(bootstrapCertificateChainFile === undefined)
) {
throw new WorkerProcessConfigError(
'Worker identity bootstrap key and certificate must be configured together',
);
}
const launcherPath = absolutePath(
environment,
'QL3_WORKER_LAUNCHER_PATH',
false,
);
const expectedLauncherSha256 = boundedValue(
environment,
'QL3_WORKER_LAUNCHER_SHA256',
64,
);
if (
(launcherPath === undefined) !==
(expectedLauncherSha256 === undefined) ||
(expectedLauncherSha256 !== undefined &&
!SHA256.test(expectedLauncherSha256))
) {
throw new WorkerProcessConfigError(
'Worker launcher path and SHA-256 must be configured together',
);
}
const capabilitiesFile = absolutePath(
environment,
'QL3_WORKER_CAPABILITIES_FILE',
)!;
const leaseDurationMs = integerValue(
environment,
'QL3_WORKER_SESSION_LEASE_DURATION_MS',
45_000,
15_000,
10 * 60_000,
);
const heartbeatIntervalMs = integerValue(
environment,
'QL3_WORKER_HEARTBEAT_INTERVAL_MS',
10_000,
5_000,
5 * 60_000,
);
if (heartbeatIntervalMs * 2 > leaseDurationMs) {
throw new WorkerProcessConfigError(
'Worker heartbeat interval must fit twice inside the Session lease',
);
}
return Object.freeze({
enabled: true,
profile: 'worker',
capacityProfile,
workerId,
origin: origin(environment),
capabilities: await capabilities(capabilitiesFile),
maxConcurrentRuns: integerValue(
environment,
'QL3_WORKER_MAX_CONCURRENT_RUNS',
edge ? 1 : 8,
1,
edge ? 4 : 64,
),
storage: Object.freeze({
journalRoot: absolutePath(
environment,
'QL3_WORKER_JOURNAL_ROOT',
)!,
logRoot: absolutePath(environment, 'QL3_WORKER_LOG_ROOT')!,
receiptRoot: absolutePath(
environment,
'QL3_WORKER_RECEIPT_ROOT',
)!,
}),
identity: Object.freeze({
certificateStoreRoot: absolutePath(
environment,
'QL3_WORKER_CERTIFICATE_STORE_ROOT',
)!,
trustAnchorFile: absolutePath(
environment,
'QL3_WORKER_TRUST_ANCHOR_FILE',
)!,
credentialTokenFile: absolutePath(
environment,
'QL3_WORKER_CREDENTIAL_TOKEN_FILE',
)!,
...(expectedCredentialId === undefined
? {}
: { expectedCredentialId }),
...(bootstrapPrivateKeyFile === undefined
? {}
: {
bootstrap: Object.freeze({
privateKeyFile: bootstrapPrivateKeyFile,
certificateChainFile: bootstrapCertificateChainFile!,
}),
}),
}),
lifecycle: Object.freeze({
cadenceMs: integerValue(
environment,
'QL3_WORKER_CADENCE_MS',
edge ? 2_000 : 500,
100,
60_000,
),
leaseDurationMs,
heartbeatIntervalMs,
drainTimeoutMs: integerValue(
environment,
'QL3_WORKER_DRAIN_TIMEOUT_MS',
edge ? 60_000 : 5 * 60_000,
1_000,
10 * 60_000,
),
drainPollMs: integerValue(
environment,
'QL3_WORKER_DRAIN_POLL_MS',
edge ? 500 : 100,
25,
5_000,
),
requestTimeoutMs: integerValue(
environment,
'QL3_WORKER_REQUEST_TIMEOUT_MS',
15_000,
100,
120_000,
),
maximumJournalEntries: integerValue(
environment,
'QL3_WORKER_MAXIMUM_JOURNAL_ENTRIES',
edge ? 64 : 256,
1,
1024,
),
maximumRecordsPerTick: integerValue(
environment,
'QL3_WORKER_MAXIMUM_RECORDS_PER_TICK',
edge ? 4 : 16,
1,
64,
),
maximumSupervisionRecordsPerTick: integerValue(
environment,
'QL3_WORKER_MAXIMUM_SUPERVISION_RECORDS_PER_TICK',
edge ? 4 : 32,
1,
64,
),
}),
executor: Object.freeze({
...(launcherPath === undefined
? {}
: {
launcherPath,
expectedLauncherSha256: expectedLauncherSha256!,
}),
}),
});
}
packages/ql3-worker-runtime/src/process/workerProcessIdentity.ts
+151
View File
@@ -0,0 +1,151 @@
// Worker Process owns private bootstrap material and active credential composition.
import { constants } from 'node:fs';
import { open } from 'node:fs/promises';
import type {
WorkerCertificateTrustAnchorProvider,
} from '../credential/workerCertificateRenewal';
import { WorkerCertificateFileStore } from '../credential/workerCertificateStore';
import { validateWorkerCertificateIdentity } from '../credential/workerCertificateIdentity';
import { WorkerProductionCredentialProvider } from '../credential/workerProductionCredentialProvider';
import type { EnabledWorkerProcessConfig } from './workerProcessConfig';
const MAX_TLS_MATERIAL_BYTES = 1024 * 1024;
export class WorkerProcessIdentityError extends Error {
readonly code = 'QL3_WORKER_PROCESS_IDENTITY_UNAVAILABLE';
constructor(message: string, options?: ErrorOptions) {
super(`Worker process identity is unavailable: ${message}`, options);
this.name = 'WorkerProcessIdentityError';
}
}
async function readMaterial(
path: string,
privateMaterial: boolean,
): Promise<Buffer> {
let handle;
try {
handle = await open(
path,
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
);
const stat = await handle.stat();
if (
!stat.isFile() ||
stat.size < 1 ||
stat.size > MAX_TLS_MATERIAL_BYTES ||
(stat.mode & 0o022) !== 0 ||
(privateMaterial && (stat.mode & 0o077) !== 0)
) {
throw new Error('unsafe identity material metadata');
}
const bytes = await handle.readFile();
if (
bytes.byteLength < 1 ||
bytes.byteLength > MAX_TLS_MATERIAL_BYTES
) {
bytes.fill(0);
throw new Error('unsafe identity material size');
}
return bytes;
} catch (error) {
throw new WorkerProcessIdentityError('material read failed', {
cause: error,
});
} finally {
await handle?.close().catch(() => undefined);
}
}
export class WorkerTrustAnchorFileProvider
implements WorkerCertificateTrustAnchorProvider
{
constructor(private readonly path: string) {
if (typeof path !== 'string' || path.length < 1) {
throw new WorkerProcessIdentityError('trust anchor path is invalid');
}
}
async load(signal: AbortSignal): Promise<readonly Buffer[]> {
if (!(signal instanceof AbortSignal)) {
throw new WorkerProcessIdentityError('trust signal is invalid');
}
signal.throwIfAborted();
const bytes = await readMaterial(this.path, false);
try {
signal.throwIfAborted();
return Object.freeze([bytes]);
} catch (error) {
bytes.fill(0);
throw error;
}
}
}
/**
* Verifies or bootstraps one durable Worker certificate store, then returns
* the per-request credential provider. Bootstrap material is optional and is
* only installed when its validated leaf differs from the active generation.
*/
export async function createWorkerProcessCredentialProvider(
config: EnabledWorkerProcessConfig['identity'],
): Promise<Readonly<WorkerProductionCredentialProvider>> {
if (!config || typeof config !== 'object') {
throw new WorkerProcessIdentityError('configuration is invalid');
}
const store = new WorkerCertificateFileStore({
rootDirectory: config.certificateStoreRoot,
retainedGenerations: 2,
});
const trustAnchors = new WorkerTrustAnchorFileProvider(
config.trustAnchorFile,
);
const signal = new AbortController().signal;
let anchors: readonly Buffer[] | undefined;
let privateKey: Buffer | undefined;
let certificate: Buffer | undefined;
try {
anchors = await trustAnchors.load(signal);
const active = await store.readActive(anchors);
if (config.bootstrap !== undefined) {
[privateKey, certificate] = await Promise.all([
readMaterial(config.bootstrap.privateKeyFile, true),
readMaterial(config.bootstrap.certificateChainFile, false),
]);
const bootstrap = validateWorkerCertificateIdentity({
privateKeyPem: privateKey,
certificateChainPem: certificate,
trustAnchors: anchors,
});
if (active?.certificateSha256 !== bootstrap.certificateSha256) {
await store.install({
privateKeyPem: privateKey,
certificateChainPem: certificate,
trustAnchors: anchors,
});
}
} else if (!active) {
throw new WorkerProcessIdentityError(
'no active identity or bootstrap material',
);
}
return new WorkerProductionCredentialProvider({
certificateStore: store,
trustAnchors,
credentialTokenFile: config.credentialTokenFile,
...(config.expectedCredentialId === undefined
? {}
: { expectedCredentialId: config.expectedCredentialId }),
});
} catch (error) {
if (error instanceof WorkerProcessIdentityError) throw error;
throw new WorkerProcessIdentityError('activation failed', {
cause: error,
});
} finally {
privateKey?.fill(0);
certificate?.fill(0);
anchors?.forEach((anchor) => anchor.fill(0));
}
}
packages/ql3-worker-runtime/src/remote-execution/executionContextMaterializer.ts
+304
View File
@@ -0,0 +1,304 @@
// Remote Execution owns bounded Secret and Artifact context materialization.
import {
MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES,
MAX_LOCAL_DISPATCH_SECRET_REFS,
} from '@qinglong/runtime-core/local-dispatch';
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { assertRunDispatchId } from '@qinglong/runtime-core/run-dispatch-lease';
import type {
MaterializedWorkerRemoteExecutionContext,
WorkerRemoteExecutionContextMaterializer,
WorkerRemoteExecutionOutputSink,
} from './executionInboxProcessor';
export interface WorkerRemoteSecretResolution {
readonly values: readonly Readonly<{
secretRef: string;
value: string;
}>[];
readonly dispose?: () => Promise<void> | void;
}
export interface WorkerRemoteSecretEnvironmentProvider {
resolve(request: Readonly<{
projectId: string;
taskId: string;
taskRevision: string;
runId: string;
attemptId: string;
offerId: string;
executionDigest: string;
secretRefs: readonly string[];
}>): Promise<WorkerRemoteSecretResolution | undefined>;
}
export interface WorkerRemoteLogArtifactPreparation {
readonly logArtifactId: string;
/** Transfers the prepared writer once; release must not close it afterwards. */
readonly takeOutput: () => WorkerRemoteExecutionOutputSink;
/** Releases only preparation resources; it must not delete a handed-off log. */
readonly release: () => Promise<void> | void;
}
export interface WorkerRemoteLogArtifactAllocator {
prepare(request: Readonly<{
projectId: string;
runId: string;
attemptId: string;
offerId: string;
}>): Promise<WorkerRemoteLogArtifactPreparation | undefined>;
}
export interface BoundedWorkerRemoteExecutionContextMaterializerOptions {
readonly artifacts: WorkerRemoteLogArtifactAllocator;
readonly secrets?: WorkerRemoteSecretEnvironmentProvider;
}
export class WorkerRemoteExecutionMaterializationError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'secret_unavailable'
| 'secret_response_invalid'
| 'environment_budget_exceeded'
| 'artifact_unavailable'
| 'artifact_response_invalid',
) {
super(`Worker remote execution materialization failed: ${reason}`);
this.name = 'WorkerRemoteExecutionMaterializationError';
}
}
function environmentValue(value: unknown): string {
if (
typeof value !== 'string' ||
value.includes('\0') ||
Buffer.byteLength(value, 'utf8') > 16 * 1024
) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
return value;
}
async function disposeQuietly(
operation: (() => Promise<void> | void) | undefined,
): Promise<void> {
await Promise.resolve().then(() => operation?.()).catch(() => undefined);
}
export class BoundedWorkerRemoteExecutionContextMaterializer
implements WorkerRemoteExecutionContextMaterializer {
private readonly artifacts: WorkerRemoteLogArtifactAllocator;
private readonly secrets?: WorkerRemoteSecretEnvironmentProvider;
constructor(options: BoundedWorkerRemoteExecutionContextMaterializerOptions) {
if (
!options ||
typeof options.artifacts?.prepare !== 'function' ||
(options.secrets !== undefined &&
typeof options.secrets.resolve !== 'function')
) {
throw new WorkerRemoteExecutionMaterializationError(
'invalid_configuration',
);
}
this.artifacts = options.artifacts;
this.secrets = options.secrets;
}
async prepare(input: Readonly<{
offer: ClusterRemoteExecutionOffer;
}>): Promise<MaterializedWorkerRemoteExecutionContext> {
let offer: ClusterRemoteExecutionOffer;
try {
offer = createClusterRemoteExecutionOffer(input?.offer);
} catch {
throw new WorkerRemoteExecutionMaterializationError(
'invalid_configuration',
);
}
const bindings = offer.executionRevision.environment;
const secretRefs = Object.freeze([
...new Set(bindings.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [])),
]);
if (secretRefs.length > MAX_LOCAL_DISPATCH_SECRET_REFS) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
);
}
let secretResolution: WorkerRemoteSecretResolution | undefined;
const secretByRef = new Map<string, string>();
if (secretRefs.length > 0) {
if (!this.secrets) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
);
}
try {
secretResolution = await this.secrets.resolve(Object.freeze({
projectId: offer.candidate.projectId,
taskId: offer.candidate.taskId,
taskRevision: offer.candidate.taskRevision,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs,
}));
} catch {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
);
}
if (!secretResolution) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_unavailable',
);
}
if (
Object.keys(secretResolution).some((key) =>
key !== 'values' && key !== 'dispose') ||
!Array.isArray(secretResolution.values) ||
secretResolution.values.length !== secretRefs.length ||
(secretResolution.dispose !== undefined &&
typeof secretResolution.dispose !== 'function')
) {
await disposeQuietly(secretResolution.dispose);
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
try {
for (const entry of secretResolution.values) {
if (
!entry ||
typeof entry !== 'object' ||
Object.keys(entry).length !== 2 ||
!Object.hasOwn(entry, 'secretRef') ||
!Object.hasOwn(entry, 'value') ||
typeof entry.secretRef !== 'string' ||
!secretRefs.includes(entry.secretRef) ||
secretByRef.has(entry.secretRef)
) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
secretByRef.set(entry.secretRef, environmentValue(entry.value));
}
} catch (error) {
await disposeQuietly(secretResolution.dispose);
throw error;
}
}
let environmentBytes = 0;
let environment: MaterializedWorkerRemoteExecutionContext['environment'];
try {
environment = Object.freeze(bindings.map((binding) => {
const value = binding.kind === 'public'
? binding.value
: secretByRef.get(binding.secretRef);
if (value === undefined) {
throw new WorkerRemoteExecutionMaterializationError(
'secret_response_invalid',
);
}
environmentBytes += Buffer.byteLength(binding.name, 'utf8') +
Buffer.byteLength(value, 'utf8');
if (environmentBytes > MAX_LOCAL_DISPATCH_ENVIRONMENT_BYTES) {
throw new WorkerRemoteExecutionMaterializationError(
'environment_budget_exceeded',
);
}
return Object.freeze({ name: binding.name, value });
}));
} catch (error) {
await disposeQuietly(secretResolution?.dispose);
throw error;
}
let artifact: WorkerRemoteLogArtifactPreparation | undefined;
try {
artifact = await this.artifacts.prepare(Object.freeze({
projectId: offer.candidate.projectId,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
}));
} catch {
await disposeQuietly(secretResolution?.dispose);
throw new WorkerRemoteExecutionMaterializationError(
'artifact_unavailable',
);
}
if (!artifact) {
await disposeQuietly(secretResolution?.dispose);
throw new WorkerRemoteExecutionMaterializationError(
'artifact_unavailable',
);
}
try {
if (
Object.keys(artifact).length !== 3 ||
!Object.hasOwn(artifact, 'logArtifactId') ||
!Object.hasOwn(artifact, 'takeOutput') ||
!Object.hasOwn(artifact, 'release')
) {
throw new Error('invalid artifact preparation');
}
assertRunDispatchId('logArtifactId', artifact.logArtifactId);
if (
artifact.logArtifactId.length > 36 ||
typeof artifact.takeOutput !== 'function' ||
typeof artifact.release !== 'function'
) {
throw new Error('invalid artifact preparation');
}
} catch {
await disposeQuietly(artifact.release);
await disposeQuietly(secretResolution?.dispose);
throw new WorkerRemoteExecutionMaterializationError(
'artifact_response_invalid',
);
}
let disposed = false;
let outputTaken = false;
return Object.freeze({
environment,
logArtifactId: artifact.logArtifactId,
takeOutput() {
if (disposed || outputTaken) {
throw new WorkerRemoteExecutionMaterializationError(
'artifact_response_invalid',
);
}
const output = artifact!.takeOutput();
if (
!output ||
typeof output !== 'object' ||
output.logArtifactId !== artifact!.logArtifactId ||
typeof output.write !== 'function' ||
typeof output.close !== 'function'
) {
void Promise.resolve(output?.close?.()).catch(() => undefined);
throw new WorkerRemoteExecutionMaterializationError(
'artifact_response_invalid',
);
}
outputTaken = true;
return output;
},
async dispose() {
if (disposed) return;
disposed = true;
await Promise.all([
disposeQuietly(artifact!.release),
disposeQuietly(secretResolution?.dispose),
]);
},
});
}
}
packages/ql3-worker-runtime/src/remote-execution/executionInbox.ts
+379
View File
@@ -0,0 +1,379 @@
// Remote Execution owns the durable offer inbox authority and transition contract.
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
export const WORKER_REMOTE_EXECUTION_INBOX_STATES = [
'accepted',
'starting_acknowledged',
'launching',
'started',
'running_acknowledged',
'start_failed',
'start_failure_acknowledged',
'completion_acknowledged',
'recovery_required',
] as const;
export type WorkerRemoteExecutionInboxState =
(typeof WORKER_REMOTE_EXECUTION_INBOX_STATES)[number];
export type WorkerRemoteExecutionRecoveryReason =
| 'launch_outcome_unknown'
| 'control_plane_already_running'
| 'control_plane_terminal'
| 'lease_lost_local_execution_stopped'
| 'lease_lost_local_execution_unverified';
export interface WorkerRemoteExecutionInboxRecord {
readonly schemaVersion: 1;
readonly revision: number;
readonly state: WorkerRemoteExecutionInboxState;
readonly offer: ClusterRemoteExecutionOffer;
readonly acceptedAtMs: number;
readonly updatedAtMs: number;
readonly executorHandle?: string;
readonly executorStartedAtMs?: number;
readonly logArtifactId?: string;
readonly completionReceiptCallbackSequence?: number;
readonly completionReceiptTokenDigest?: string;
readonly completionAcknowledgedAtMs?: number;
readonly recoveryReason?: WorkerRemoteExecutionRecoveryReason;
}
export interface WorkerRemoteExecutionInboxPage {
readonly records: readonly WorkerRemoteExecutionInboxRecord[];
readonly nextAfterOfferId?: string;
}
export interface WorkerRemoteExecutionInbox {
readOffer(offerId: string): Promise<WorkerRemoteExecutionInboxRecord | undefined>;
replaceOffer(
record: WorkerRemoteExecutionInboxRecord,
expectedRevision: number,
): Promise<void>;
listOffers(options?: Readonly<{
afterOfferId?: string;
limit?: number;
}>): Promise<WorkerRemoteExecutionInboxPage>;
}
export class WorkerRemoteExecutionInboxError extends Error {
constructor(
readonly reason:
| 'invalid_record'
| 'authority_conflict'
| 'revision_conflict'
| 'invalid_transition',
) {
super(`Worker remote execution inbox failed: ${reason}`);
this.name = 'WorkerRemoteExecutionInboxError';
}
}
const STATES = new Set<string>(WORKER_REMOTE_EXECUTION_INBOX_STATES);
const SHA256 = /^[a-f0-9]{64}$/;
const RECOVERY_REASONS = new Set<string>([
'launch_outcome_unknown',
'control_plane_already_running',
'control_plane_terminal',
'lease_lost_local_execution_stopped',
'lease_lost_local_execution_unverified',
]);
const OPTIONAL_FIELDS = [
'executorHandle',
'executorStartedAtMs',
'logArtifactId',
'completionReceiptCallbackSequence',
'completionReceiptTokenDigest',
'completionAcknowledgedAtMs',
'recoveryReason',
] as const;
const BASE_FIELDS = [
'schemaVersion',
'revision',
'state',
'offer',
'acceptedAtMs',
'updatedAtMs',
] as const;
function invalid(): never {
throw new WorkerRemoteExecutionInboxError('invalid_record');
}
function safeInteger(value: unknown): number {
if (!Number.isSafeInteger(value) || (value as number) < 0) invalid();
return value as number;
}
function boundedText(value: unknown, maximum: number): string {
if (
typeof value !== 'string' ||
value.length < 1 ||
Buffer.byteLength(value, 'utf8') > maximum ||
/[\u0000-\u001f\u007f]/.test(value)
) {
invalid();
}
return value;
}
function sameOfferAuthority(
left: ClusterRemoteExecutionOffer,
right: ClusterRemoteExecutionOffer,
): boolean {
const first = createClusterRemoteExecutionOffer(left);
const second = createClusterRemoteExecutionOffer(right);
return (
first.offerId === second.offerId &&
first.executionDigest === second.executionDigest &&
first.deliveryKind === second.deliveryKind &&
JSON.stringify(first.candidate) === JSON.stringify(second.candidate) &&
JSON.stringify(first.worker) === JSON.stringify(second.worker) &&
first.lease.runId === second.lease.runId &&
first.lease.attemptId === second.lease.attemptId &&
first.lease.workerId === second.lease.workerId &&
first.lease.workerSessionId === second.lease.workerSessionId &&
first.lease.workerGeneration === second.lease.workerGeneration &&
first.lease.leaseGeneration === second.lease.leaseGeneration &&
first.lease.leaseTokenDigest === second.lease.leaseTokenDigest &&
first.leaseToken === second.leaseToken &&
JSON.stringify(first.executionRevision) ===
JSON.stringify(second.executionRevision)
);
}
function exactOptional<T extends keyof WorkerRemoteExecutionInboxRecord>(
value: WorkerRemoteExecutionInboxRecord,
key: T,
): WorkerRemoteExecutionInboxRecord[T] | undefined {
return Object.prototype.hasOwnProperty.call(value, key) ? value[key] : undefined;
}
export function normalizeWorkerRemoteExecutionInboxRecord(
value: WorkerRemoteExecutionInboxRecord,
): WorkerRemoteExecutionInboxRecord {
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
const allowed = new Set<string>([...BASE_FIELDS, ...OPTIONAL_FIELDS]);
const keys = Object.keys(value);
if (
value.schemaVersion !== 1 ||
BASE_FIELDS.some((key) => !keys.includes(key)) ||
keys.some((key) => !allowed.has(key)) ||
!STATES.has(value.state)
) {
invalid();
}
const revision = safeInteger(value.revision);
const acceptedAtMs = safeInteger(value.acceptedAtMs);
const updatedAtMs = safeInteger(value.updatedAtMs);
if (updatedAtMs < acceptedAtMs) invalid();
const offer = createClusterRemoteExecutionOffer(value.offer);
const executorHandle = exactOptional(value, 'executorHandle');
const executorStartedAtMs = exactOptional(value, 'executorStartedAtMs');
const logArtifactId = exactOptional(value, 'logArtifactId');
const callbackSequence = exactOptional(
value,
'completionReceiptCallbackSequence',
);
const tokenDigest = exactOptional(value, 'completionReceiptTokenDigest');
const completionAcknowledgedAtMs = exactOptional(
value,
'completionAcknowledgedAtMs',
);
const recoveryReason = exactOptional(value, 'recoveryReason');
if (executorHandle !== undefined && executorStartedAtMs === undefined) invalid();
if (executorHandle !== undefined) boundedText(executorHandle, 512);
if (executorStartedAtMs !== undefined) safeInteger(executorStartedAtMs);
if (logArtifactId !== undefined) boundedText(logArtifactId, 36);
if ((callbackSequence === undefined) !== (tokenDigest === undefined)) invalid();
if (
callbackSequence !== undefined &&
(safeInteger(callbackSequence) < 1 || callbackSequence > 2_147_483_647)
) {
invalid();
}
if (tokenDigest !== undefined && !SHA256.test(tokenDigest)) invalid();
if (completionAcknowledgedAtMs !== undefined) {
safeInteger(completionAcknowledgedAtMs);
}
if (
recoveryReason !== undefined &&
!RECOVERY_REASONS.has(recoveryReason)
) {
invalid();
}
const hasExecutor = executorHandle !== undefined;
const hasExecutorStartedAt = executorStartedAtMs !== undefined;
const hasLogArtifact = logArtifactId !== undefined;
const hasReceiptAuthentication = callbackSequence !== undefined;
const executorRequired = [
'started',
'running_acknowledged',
].includes(value.state);
const executorStartedAtRequired = [
'launching',
'started',
'running_acknowledged',
'completion_acknowledged',
].includes(value.state);
const executorStartedAtOptional = [
'start_failed',
'start_failure_acknowledged',
'recovery_required',
].includes(value.state);
const receiptAuthenticationRequired = [
'launching',
'started',
'running_acknowledged',
'completion_acknowledged',
].includes(value.state);
const receiptAuthenticationOptional = [
'start_failed',
'start_failure_acknowledged',
'recovery_required',
].includes(value.state);
const logArtifactRequired = [
'launching',
'started',
'running_acknowledged',
'completion_acknowledged',
].includes(value.state);
if (
(!['completion_acknowledged', 'recovery_required'].includes(value.state) &&
executorRequired !== hasExecutor) ||
(!executorStartedAtOptional &&
executorStartedAtRequired !== hasExecutorStartedAt) ||
(hasExecutor &&
!['started', 'running_acknowledged', 'completion_acknowledged', 'recovery_required']
.includes(value.state)) ||
(hasExecutorStartedAt &&
!['launching', 'started', 'running_acknowledged', 'start_failed',
'start_failure_acknowledged', 'completion_acknowledged',
'recovery_required'].includes(value.state)) ||
(!receiptAuthenticationOptional &&
receiptAuthenticationRequired !== hasReceiptAuthentication) ||
(hasReceiptAuthentication &&
!['launching', 'started', 'running_acknowledged', 'start_failed',
'start_failure_acknowledged', 'completion_acknowledged',
'recovery_required'].includes(value.state)) ||
(logArtifactRequired && !hasLogArtifact) ||
(hasLogArtifact && ['accepted', 'starting_acknowledged'].includes(value.state))
) {
invalid();
}
if (
completionAcknowledgedAtMs !== undefined !==
(value.state === 'completion_acknowledged') ||
recoveryReason !== undefined !== (value.state === 'recovery_required')
) {
invalid();
}
return Object.freeze({
schemaVersion: 1,
revision,
state: value.state,
offer,
acceptedAtMs,
updatedAtMs,
...(executorHandle === undefined ? {} : { executorHandle }),
...(executorStartedAtMs === undefined ? {} : { executorStartedAtMs }),
...(logArtifactId === undefined ? {} : { logArtifactId }),
...(callbackSequence === undefined
? {}
: { completionReceiptCallbackSequence: callbackSequence }),
...(tokenDigest === undefined
? {}
: { completionReceiptTokenDigest: tokenDigest }),
...(completionAcknowledgedAtMs === undefined
? {}
: { completionAcknowledgedAtMs }),
...(recoveryReason === undefined ? {} : { recoveryReason }),
});
}
export function createWorkerRemoteExecutionInboxRecord(
offer: ClusterRemoteExecutionOffer,
acceptedAtMs: number,
): WorkerRemoteExecutionInboxRecord {
return normalizeWorkerRemoteExecutionInboxRecord({
schemaVersion: 1,
revision: 0,
state: 'accepted',
offer,
acceptedAtMs,
updatedAtMs: acceptedAtMs,
});
}
function transitions(
...states: WorkerRemoteExecutionInboxState[]
): ReadonlySet<WorkerRemoteExecutionInboxState> {
return new Set(states);
}
const TRANSITIONS: Readonly<Record<
WorkerRemoteExecutionInboxState,
ReadonlySet<WorkerRemoteExecutionInboxState>
>> = Object.freeze({
accepted: transitions('accepted', 'starting_acknowledged', 'recovery_required'),
starting_acknowledged: transitions(
'starting_acknowledged', 'launching', 'start_failed', 'recovery_required',
),
launching: transitions(
'launching', 'started', 'start_failed', 'completion_acknowledged',
'recovery_required',
),
started: transitions(
'started', 'running_acknowledged', 'completion_acknowledged',
'recovery_required',
),
running_acknowledged: transitions(
'running_acknowledged', 'completion_acknowledged', 'recovery_required',
),
start_failed: transitions(
'start_failed', 'start_failure_acknowledged', 'recovery_required',
),
start_failure_acknowledged: transitions('start_failure_acknowledged'),
completion_acknowledged: transitions('completion_acknowledged'),
recovery_required: transitions('recovery_required', 'completion_acknowledged'),
});
export function assertWorkerRemoteExecutionInboxTransition(
previousValue: WorkerRemoteExecutionInboxRecord,
nextValue: WorkerRemoteExecutionInboxRecord,
): void {
const previous = normalizeWorkerRemoteExecutionInboxRecord(previousValue);
const next = normalizeWorkerRemoteExecutionInboxRecord(nextValue);
if (!sameOfferAuthority(previous.offer, next.offer)) {
throw new WorkerRemoteExecutionInboxError('authority_conflict');
}
if (
next.revision !== previous.revision + 1 ||
next.acceptedAtMs !== previous.acceptedAtMs ||
next.updatedAtMs < previous.updatedAtMs ||
next.offer.lease.version < previous.offer.lease.version ||
(next.offer.lease.version === previous.offer.lease.version &&
JSON.stringify(next.offer.lease) !== JSON.stringify(previous.offer.lease))
) {
throw new WorkerRemoteExecutionInboxError('revision_conflict');
}
if (!TRANSITIONS[previous.state].has(next.state)) {
throw new WorkerRemoteExecutionInboxError('invalid_transition');
}
for (const key of OPTIONAL_FIELDS) {
const before = exactOptional(previous, key);
const after = exactOptional(next, key);
if (
key === 'recoveryReason' &&
next.state === 'completion_acknowledged'
) continue;
if (before !== undefined && before !== after) {
throw new WorkerRemoteExecutionInboxError('invalid_transition');
}
}
}
packages/ql3-worker-runtime/src/remote-execution/executionInboxProcessor.ts
+649
View File
@@ -0,0 +1,649 @@
// Remote Execution owns offer activation, launch barriers, and recovery processing.
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import type {
AcknowledgeRemoteRunRunningCommand,
AcknowledgeRemoteRunStartingCommand,
FailRemoteRunStartCommand,
RemoteRunActivationResult,
} from '@qinglong/runtime-core/remote-activation';
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import {
normalizeWorkerRemoteExecutionInboxRecord,
type WorkerRemoteExecutionInbox,
type WorkerRemoteExecutionInboxRecord,
type WorkerRemoteExecutionRecoveryReason,
} from './executionInbox';
export interface WorkerRemoteExecutionSession {
readonly workerId: string;
readonly sessionId: string;
readonly generation: number;
readonly status: 'available' | 'draining' | 'offline';
readonly leaseExpiresAtMs: number;
}
export interface WorkerRemoteExecutionActivationClient {
acknowledgeStarting(
command: AcknowledgeRemoteRunStartingCommand,
): Promise<Readonly<RemoteRunActivationResult>>;
acknowledgeRunning(
command: AcknowledgeRemoteRunRunningCommand,
): Promise<Readonly<RemoteRunActivationResult>>;
failStart(
command: FailRemoteRunStartCommand,
): Promise<Readonly<RemoteRunActivationResult>>;
}
export interface WorkerRemoteExecutionCompletionCallback {
readonly sequence: number;
/** Ephemeral capability. Implementations must not persist or log it. */
readonly token: Uint8Array;
}
export type WorkerRemoteExecutionOutputStream = 'stdout' | 'stderr';
export interface WorkerRemoteExecutionOutputChunk {
readonly stream: WorkerRemoteExecutionOutputStream;
readonly chunk: Uint8Array;
readonly observedAtMs: number;
}
export interface WorkerRemoteExecutionOutputSink {
readonly logArtifactId: string;
write(output: WorkerRemoteExecutionOutputChunk): Promise<void>;
/** Flushes accepted bytes and releases the writer. Must be idempotent. */
close(): Promise<void>;
}
export interface MaterializedWorkerRemoteExecutionContext {
readonly environment: readonly Readonly<{
name: string;
value: string;
}>[];
readonly logArtifactId: string;
/** Transfers the prepared writer exactly once after the durable spawn barrier. */
readonly takeOutput: () => WorkerRemoteExecutionOutputSink;
readonly dispose?: () => Promise<void>;
}
export interface WorkerRemoteExecutionContextMaterializer {
prepare(input: Readonly<{
offer: ClusterRemoteExecutionOffer;
completionCallback: WorkerRemoteExecutionCompletionCallback;
}>): Promise<MaterializedWorkerRemoteExecutionContext>;
}
export interface WorkerRemoteExecutionLaunch {
readonly offerId: string;
readonly runId: string;
readonly attemptId: string;
/** Durable pre-spawn timestamp from the launching inbox barrier. */
readonly executorStartedAtMs: number;
readonly command: ClusterRemoteExecutionOffer['executionRevision']['command'];
readonly environment: MaterializedWorkerRemoteExecutionContext['environment'];
readonly workingDirectory?: string;
readonly timeoutMs?: number;
/** Durable database-clock timeout authority returned by starting ACK. */
readonly executionDeadlineAtMs?: number;
readonly logArtifactId: string;
/**
* Ownership transfers to the Executor when start() is called. The Executor
* must close it on every known terminal path, including explicit rejection.
*/
readonly output: WorkerRemoteExecutionOutputSink;
readonly completionCallback: WorkerRemoteExecutionCompletionCallback;
}
export interface WorkerRemoteExecutionExecutor {
/** A rejected result proves no execution started; a thrown error is unknown. */
start(launch: WorkerRemoteExecutionLaunch): Promise<
| Readonly<{
status: 'started';
executorHandle: string;
executorStartedAtMs: number;
}>
| Readonly<{ status: 'rejected' }>
>;
}
export type WorkerRemoteExecutionProcessResult = Readonly<{
status:
| 'running'
| 'already_running'
| 'start_failed'
| 'already_failed'
| 'already_completed'
| 'recovery_required';
offerId: string;
executorHandle?: string;
recoveryReason?: WorkerRemoteExecutionRecoveryReason;
}>;
export interface WorkerRemoteExecutionInboxProcessorOptions {
readonly inbox: WorkerRemoteExecutionInbox;
readonly activation: WorkerRemoteExecutionActivationClient;
readonly materializer: WorkerRemoteExecutionContextMaterializer;
readonly executor: WorkerRemoteExecutionExecutor;
readonly currentSession: () => WorkerRemoteExecutionSession | undefined;
readonly now?: () => number;
readonly randomCapability?: () => Uint8Array;
readonly eventId?: () => string;
}
export class WorkerRemoteExecutionProcessorError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'offer_missing'
| 'target_fenced'
| 'offer_expired'
| 'activation_response_invalid'
| 'materialized_context_invalid',
) {
super(`Worker remote execution processor failed: ${reason}`);
this.name = 'WorkerRemoteExecutionProcessorError';
}
}
const SHA256 = /^[a-f0-9]{64}$/;
function safeTime(value: number): number {
if (!Number.isSafeInteger(value) || value < 0) {
throw new WorkerRemoteExecutionProcessorError('invalid_configuration');
}
return value;
}
function boundedText(value: unknown, maximum: number): string {
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
return value;
}
function validateEnvironment(
offer: ClusterRemoteExecutionOffer,
context: MaterializedWorkerRemoteExecutionContext,
): MaterializedWorkerRemoteExecutionContext['environment'] {
if (!context || typeof context !== 'object' || !Array.isArray(context.environment)) {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
const expected = offer.executionRevision.environment;
if (context.environment.length !== expected.length) {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
const values = new Map<string, string>();
for (const entry of context.environment) {
if (!entry || typeof entry !== 'object') {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
const name = boundedText(entry.name, 255);
if (
name.includes('=') ||
typeof entry.value !== 'string' ||
entry.value.includes('\0') ||
values.has(name)
) {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
values.set(name, entry.value);
}
const normalized = expected.map((binding) => {
const value = values.get(binding.name);
if (
value === undefined ||
(binding.kind === 'public' && value !== binding.value)
) {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
return Object.freeze({ name: binding.name, value });
});
return Object.freeze(normalized);
}
async function validateOutput(
context: MaterializedWorkerRemoteExecutionContext,
logArtifactId: string,
): Promise<WorkerRemoteExecutionOutputSink> {
if (typeof context.takeOutput !== 'function') {
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
const output = context.takeOutput();
if (
!output ||
typeof output !== 'object' ||
output.logArtifactId !== logArtifactId ||
typeof output.write !== 'function' ||
typeof output.close !== 'function'
) {
if (
output &&
typeof output === 'object' &&
typeof (output as Partial<WorkerRemoteExecutionOutputSink>).close ===
'function'
) {
await Promise.resolve().then(() =>
(output as WorkerRemoteExecutionOutputSink).close()
).catch(() => undefined);
}
throw new WorkerRemoteExecutionProcessorError('materialized_context_invalid');
}
return output;
}
export class WorkerRemoteExecutionInboxProcessor {
private readonly inbox: WorkerRemoteExecutionInbox;
private readonly activation: WorkerRemoteExecutionActivationClient;
private readonly materializer: WorkerRemoteExecutionContextMaterializer;
private readonly executor: WorkerRemoteExecutionExecutor;
private readonly currentSessionProvider: () =>
WorkerRemoteExecutionSession | undefined;
private readonly nowProvider: () => number;
private readonly randomCapabilityProvider: () => Uint8Array;
private readonly eventIdProvider: () => string;
private readonly inFlight = new Map<string, Promise<WorkerRemoteExecutionProcessResult>>();
constructor(options: WorkerRemoteExecutionInboxProcessorOptions) {
if (
!options ||
typeof options.inbox?.readOffer !== 'function' ||
typeof options.inbox?.replaceOffer !== 'function' ||
typeof options.activation?.acknowledgeStarting !== 'function' ||
typeof options.activation?.acknowledgeRunning !== 'function' ||
typeof options.activation?.failStart !== 'function' ||
typeof options.materializer?.prepare !== 'function' ||
typeof options.executor?.start !== 'function' ||
typeof options.currentSession !== 'function'
) {
throw new WorkerRemoteExecutionProcessorError('invalid_configuration');
}
this.inbox = options.inbox;
this.activation = options.activation;
this.materializer = options.materializer;
this.executor = options.executor;
this.currentSessionProvider = options.currentSession;
this.nowProvider = options.now ?? Date.now;
this.randomCapabilityProvider = options.randomCapability ??
(() => randomBytes(32));
this.eventIdProvider = options.eventId ?? randomUUID;
}
process(offerId: string): Promise<WorkerRemoteExecutionProcessResult> {
const active = this.inFlight.get(offerId);
if (active) return active;
const operation = this.processOnce(offerId).finally(() => {
if (this.inFlight.get(offerId) === operation) this.inFlight.delete(offerId);
});
this.inFlight.set(offerId, operation);
return operation;
}
private async processOnce(
offerId: string,
): Promise<WorkerRemoteExecutionProcessResult> {
let record = await this.inbox.readOffer(offerId);
if (!record) {
throw new WorkerRemoteExecutionProcessorError('offer_missing');
}
record = normalizeWorkerRemoteExecutionInboxRecord(record);
if (record.state === 'completion_acknowledged') {
return Object.freeze({ status: 'already_completed', offerId });
}
if (record.state === 'running_acknowledged') {
return Object.freeze({
status: 'already_running',
offerId,
executorHandle: record.executorHandle,
});
}
if (record.state === 'start_failure_acknowledged') {
return Object.freeze({ status: 'already_failed', offerId });
}
if (record.state === 'recovery_required') return this.recoveryResult(record);
this.assertCurrentTarget(record.offer);
if (record.state === 'launching') {
record = await this.recover(record, 'launch_outcome_unknown');
return this.recoveryResult(record);
}
if (record.state === 'accepted') {
const starting = await this.activation.acknowledgeStarting({
...this.fence(record.offer),
eventId: this.eventId(),
});
this.assertActivation(record.offer, starting);
if (starting.status === 'already_running') {
record = await this.recover(record, 'control_plane_already_running');
return this.recoveryResult(record);
}
if (starting.status === 'already_terminal') {
record = await this.recover(record, 'control_plane_terminal');
return this.recoveryResult(record);
}
record = await this.replace(record, { state: 'starting_acknowledged' });
}
if (record.state === 'start_failed') {
return this.reportStartFailure(record);
}
if (record.state === 'starting_acknowledged') {
record = await this.launch(record);
if (record.state === 'start_failed') return this.reportStartFailure(record);
if (record.state === 'recovery_required') return this.recoveryResult(record);
}
if (record.state !== 'started') {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
const running = await this.activation.acknowledgeRunning({
...this.fence(record.offer),
attemptEventId: this.eventId(),
runEventId: this.eventId(),
executorHandle: record.executorHandle!,
...(record.logArtifactId === undefined
? {}
: { logArtifactId: record.logArtifactId }),
callbackSequence: record.completionReceiptCallbackSequence!,
callbackTokenDigest: record.completionReceiptTokenDigest!,
});
this.assertActivation(record.offer, running);
if (running.status === 'already_terminal') {
record = await this.recover(record, 'control_plane_terminal');
return this.recoveryResult(record);
}
if (
running.status !== 'applied' &&
running.status !== 'already_running'
) {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
record = await this.replace(record, { state: 'running_acknowledged' });
return Object.freeze({
status: running.status === 'already_running' ? 'already_running' : 'running',
offerId,
executorHandle: record.executorHandle,
});
}
private async launch(
record: WorkerRemoteExecutionInboxRecord,
): Promise<WorkerRemoteExecutionInboxRecord> {
const callback = await this.nextCallbackAuthority(record.offer);
const callbackSequence = callback.sequence;
const token = Buffer.from(this.randomCapabilityProvider());
if (token.byteLength !== 32) {
token.fill(0);
throw new WorkerRemoteExecutionProcessorError('invalid_configuration');
}
const tokenDigest = createHash('sha256').update(token).digest('hex');
let context: MaterializedWorkerRemoteExecutionContext | undefined;
try {
try {
context = await this.materializer.prepare({
offer: createClusterRemoteExecutionOffer(record.offer),
completionCallback: Object.freeze({
sequence: callbackSequence,
token,
}),
});
} catch {
return await this.replace(record, { state: 'start_failed' });
}
let environment: MaterializedWorkerRemoteExecutionContext['environment'];
let logArtifactId: string;
try {
environment = validateEnvironment(record.offer, context);
logArtifactId = boundedText(context.logArtifactId, 36);
if (typeof context.takeOutput !== 'function') {
throw new WorkerRemoteExecutionProcessorError(
'materialized_context_invalid',
);
}
} catch {
return await this.replace(record, { state: 'start_failed' });
}
record = await this.replace(record, {
state: 'launching',
executorStartedAtMs: this.now(),
logArtifactId,
completionReceiptCallbackSequence: callbackSequence,
completionReceiptTokenDigest: tokenDigest,
});
let output: WorkerRemoteExecutionOutputSink;
try {
output = await validateOutput(context, logArtifactId);
} catch {
return await this.replace(record, { state: 'start_failed' });
}
let outcome: Awaited<ReturnType<WorkerRemoteExecutionExecutor['start']>>;
try {
outcome = await this.executor.start(Object.freeze({
offerId: record.offer.offerId,
runId: record.offer.candidate.runId,
attemptId: record.offer.candidate.attemptId,
executorStartedAtMs: record.executorStartedAtMs!,
command: record.offer.executionRevision.command,
environment,
...(record.offer.executionRevision.workingDirectory === undefined
? {}
: { workingDirectory: record.offer.executionRevision.workingDirectory }),
...(record.offer.executionRevision.timeoutMs === undefined
? {}
: {
timeoutMs: record.offer.executionRevision.timeoutMs,
executionDeadlineAtMs: callback.deadlineAtMs,
}),
logArtifactId,
output,
completionCallback: Object.freeze({
sequence: callbackSequence,
token,
}),
}));
} catch {
return await this.recover(record, 'launch_outcome_unknown');
}
if (outcome?.status === 'rejected') {
await output.close().catch(() => undefined);
return await this.replace(record, { state: 'start_failed' });
}
if (outcome?.status !== 'started') {
return await this.recover(record, 'launch_outcome_unknown');
}
let executorHandle: string;
let executorStartedAtMs: number;
try {
executorHandle = boundedText(outcome.executorHandle, 512);
executorStartedAtMs = outcome.executorStartedAtMs;
if (
!Number.isSafeInteger(executorStartedAtMs) ||
executorStartedAtMs < 0 ||
executorStartedAtMs > this.now() ||
executorStartedAtMs !== record.executorStartedAtMs
) {
throw new WorkerRemoteExecutionProcessorError(
'materialized_context_invalid',
);
}
} catch {
return await this.recover(record, 'launch_outcome_unknown');
}
try {
return await this.replace(record, {
state: 'started',
executorHandle,
logArtifactId,
});
} catch {
return await this.recover(record, 'launch_outcome_unknown');
}
} finally {
token.fill(0);
await context?.dispose?.().catch(() => undefined);
}
}
private async nextCallbackAuthority(
offer: ClusterRemoteExecutionOffer,
): Promise<Readonly<{ sequence: number; deadlineAtMs?: number }>> {
const replay = await this.activation.acknowledgeStarting({
...this.fence(offer),
eventId: this.eventId(),
});
this.assertActivation(offer, replay);
if (
replay.status !== 'already_starting' &&
replay.status !== 'applied'
) {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
const sequence = replay.snapshot.callbackSequence + 1;
if (!Number.isSafeInteger(sequence) || sequence < 1 || sequence > 2_147_483_647) {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
return Object.freeze({
sequence,
...(replay.snapshot.deadlineAtMs === undefined
? {}
: { deadlineAtMs: replay.snapshot.deadlineAtMs }),
});
}
private async reportStartFailure(
record: WorkerRemoteExecutionInboxRecord,
): Promise<WorkerRemoteExecutionProcessResult> {
const result = await this.activation.failStart({
...this.fence(record.offer),
attemptEventId: this.eventId(),
runEventId: this.eventId(),
});
this.assertActivation(record.offer, result, true);
if (result.status === 'already_running') {
const recovery = await this.recover(
record,
'control_plane_already_running',
);
return this.recoveryResult(recovery);
}
if (result.status !== 'applied' && result.status !== 'already_terminal') {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
await this.replace(record, { state: 'start_failure_acknowledged' });
return Object.freeze({ status: 'start_failed', offerId: record.offer.offerId });
}
private assertCurrentTarget(offer: ClusterRemoteExecutionOffer): void {
const current = this.currentSessionProvider();
const now = this.now();
if (
!current ||
current.workerId !== offer.worker.workerId ||
current.sessionId !== offer.worker.sessionId ||
current.generation !== offer.worker.generation ||
current.status === 'offline' ||
(current.status === 'draining' && offer.deliveryKind === 'new_claim') ||
current.leaseExpiresAtMs <= now
) {
throw new WorkerRemoteExecutionProcessorError('target_fenced');
}
if (offer.lease.expiresAtMs <= now) {
throw new WorkerRemoteExecutionProcessorError('offer_expired');
}
}
private assertActivation(
offer: ClusterRemoteExecutionOffer,
result: Readonly<RemoteRunActivationResult>,
allowCompletedLease = false,
): void {
const snapshot = result?.snapshot;
if (
!['applied', 'already_starting', 'already_running', 'already_terminal']
.includes(result?.status) ||
!snapshot ||
snapshot.runId !== offer.candidate.runId ||
snapshot.attemptId !== offer.candidate.attemptId ||
snapshot.leaseGeneration !== offer.lease.leaseGeneration ||
!Number.isSafeInteger(snapshot.leaseVersion) ||
snapshot.leaseVersion < offer.lease.version ||
snapshot.leaseVersion > offer.lease.version + (allowCompletedLease ? 1 : 0) ||
!Number.isSafeInteger(snapshot.callbackSequence) ||
snapshot.callbackSequence < 0 ||
snapshot.callbackSequence > 2_147_483_647
|| (offer.executionRevision.timeoutMs === undefined) !==
(snapshot.deadlineAtMs === undefined)
|| (snapshot.deadlineAtMs !== undefined &&
(!Number.isSafeInteger(snapshot.deadlineAtMs) ||
snapshot.deadlineAtMs < 0))
) {
throw new WorkerRemoteExecutionProcessorError('activation_response_invalid');
}
}
private fence(offer: ClusterRemoteExecutionOffer) {
return Object.freeze({
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
workerId: offer.worker.workerId,
workerSessionId: offer.worker.sessionId,
workerGeneration: offer.worker.generation,
offerId: offer.offerId,
leaseGeneration: offer.lease.leaseGeneration,
leaseToken: offer.leaseToken,
expectedLeaseVersion: offer.lease.version,
});
}
private async replace(
previous: WorkerRemoteExecutionInboxRecord,
patch: Partial<WorkerRemoteExecutionInboxRecord>,
): Promise<WorkerRemoteExecutionInboxRecord> {
const next = normalizeWorkerRemoteExecutionInboxRecord({
...previous,
...patch,
schemaVersion: 1,
revision: previous.revision + 1,
updatedAtMs: Math.max(this.now(), previous.updatedAtMs),
});
await this.inbox.replaceOffer(next, previous.revision);
return next;
}
private recover(
record: WorkerRemoteExecutionInboxRecord,
recoveryReason: WorkerRemoteExecutionRecoveryReason,
): Promise<WorkerRemoteExecutionInboxRecord> {
return this.replace(record, { state: 'recovery_required', recoveryReason });
}
private recoveryResult(
record: WorkerRemoteExecutionInboxRecord,
): WorkerRemoteExecutionProcessResult {
return Object.freeze({
status: 'recovery_required',
offerId: record.offer.offerId,
recoveryReason: record.recoveryReason,
});
}
private eventId(): string {
const value = this.eventIdProvider();
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > 36 ||
/[\u0000-\u001f\u007f]/.test(value)
) {
throw new WorkerRemoteExecutionProcessorError('invalid_configuration');
}
return value;
}
private now(): number {
return safeTime(this.nowProvider());
}
}
packages/ql3-worker-runtime/src/remote-execution/headlessExecutionLifecycle.ts
+386
View File
@@ -0,0 +1,386 @@
// Remote Execution owns the caller-driven offer supervision and drain lifecycle.
import type { WorkerRemoteExecutionInbox } from './executionInbox';
import type {
WorkerRemoteExecutionInboxProcessor,
WorkerRemoteExecutionProcessResult,
WorkerRemoteExecutionSession,
} from './executionInboxProcessor';
import type {
WorkerRemoteOfferPullCoordinator,
WorkerRemoteOfferPullResult,
} from './remoteOfferDelivery';
import type {
WorkerRemoteExecutionControlCoordinator,
WorkerRemoteExecutionControlResult,
} from '../execution/workerExecutionControlCoordinator';
export interface WorkerRemoteExecutionLifecycleJournal
extends WorkerRemoteExecutionInbox {
acquireOwnership(): Promise<void>;
releaseOwnership(): Promise<void>;
}
export interface WorkerRemoteExecutionHeadlessLifecycleOptions {
readonly journal: WorkerRemoteExecutionLifecycleJournal;
readonly offers: Pick<WorkerRemoteOfferPullCoordinator, 'pull'>;
readonly processor: Pick<WorkerRemoteExecutionInboxProcessor, 'process'>;
readonly control: Pick<WorkerRemoteExecutionControlCoordinator, 'reconcile'>;
readonly currentSession: () => WorkerRemoteExecutionSession | undefined;
readonly maximumRecordsPerTick?: number;
readonly maximumSupervisionRecordsPerTick?: number;
readonly now?: () => number;
}
export type WorkerRemoteExecutionLifecycleTickResult =
| Readonly<{
status: 'reconciling';
processed: number;
nextAfterOfferId: string;
}>
| Readonly<{
status: 'reconciled';
processed: number;
}>
| Readonly<{
status: 'recovery_required';
offerId: string;
}>
| Readonly<{
status: 'session_unavailable';
}>
| Readonly<{
status: 'draining';
}>
| Readonly<{
status: 'processed';
offerId: string;
execution: WorkerRemoteExecutionProcessResult;
pull?: WorkerRemoteOfferPullResult;
}>
| Readonly<{
status: 'pull_result';
pull: WorkerRemoteOfferPullResult;
}>;
export class WorkerRemoteExecutionLifecycleError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'inactive'
| 'stopping'
| 'draining',
) {
super(`Worker remote execution lifecycle failed: ${reason}`);
this.name = 'WorkerRemoteExecutionLifecycleError';
}
}
type Mode = 'inactive' | 'running' | 'stopping';
export class WorkerRemoteExecutionHeadlessLifecycle {
private readonly journal: WorkerRemoteExecutionLifecycleJournal;
private readonly offers: Pick<WorkerRemoteOfferPullCoordinator, 'pull'>;
private readonly processor: Pick<WorkerRemoteExecutionInboxProcessor, 'process'>;
private readonly control: Pick<
WorkerRemoteExecutionControlCoordinator,
'reconcile'
>;
private readonly currentSessionProvider: () =>
WorkerRemoteExecutionSession | undefined;
private readonly maximumRecordsPerTick: number;
private readonly maximumSupervisionRecordsPerTick: number;
private readonly nowProvider: () => number;
private mode: Mode = 'inactive';
private draining = false;
private startupAfterOfferId?: string;
private startupComplete = false;
private supervisionAfterOfferId?: string;
private retryOfferId?: string;
private recoveryOfferId?: string;
private inFlight?: Promise<WorkerRemoteExecutionLifecycleTickResult>;
private stopController?: AbortController;
private drainOperation?: Promise<void>;
private stopOperation?: Promise<void>;
constructor(options: WorkerRemoteExecutionHeadlessLifecycleOptions) {
if (
!options ||
typeof options.journal?.acquireOwnership !== 'function' ||
typeof options.journal?.releaseOwnership !== 'function' ||
typeof options.journal?.listOffers !== 'function' ||
typeof options.offers?.pull !== 'function' ||
typeof options.processor?.process !== 'function' ||
typeof options.control?.reconcile !== 'function' ||
typeof options.currentSession !== 'function'
) {
throw new WorkerRemoteExecutionLifecycleError('invalid_configuration');
}
const maximumRecordsPerTick = options.maximumRecordsPerTick ?? 16;
const maximumSupervisionRecordsPerTick =
options.maximumSupervisionRecordsPerTick ?? maximumRecordsPerTick;
if (
!Number.isSafeInteger(maximumRecordsPerTick) ||
maximumRecordsPerTick < 1 ||
maximumRecordsPerTick > 64
|| !Number.isSafeInteger(maximumSupervisionRecordsPerTick)
|| maximumSupervisionRecordsPerTick < 1
|| maximumSupervisionRecordsPerTick > 64
) {
throw new WorkerRemoteExecutionLifecycleError('invalid_configuration');
}
this.journal = options.journal;
this.offers = options.offers;
this.processor = options.processor;
this.control = options.control;
this.currentSessionProvider = options.currentSession;
this.maximumRecordsPerTick = maximumRecordsPerTick;
this.maximumSupervisionRecordsPerTick = maximumSupervisionRecordsPerTick;
this.nowProvider = options.now ?? Date.now;
}
async start(): Promise<'started' | 'already_started'> {
if (this.mode === 'running') return 'already_started';
if (this.mode === 'stopping') {
throw new WorkerRemoteExecutionLifecycleError('stopping');
}
await this.journal.acquireOwnership();
this.mode = 'running';
this.startupAfterOfferId = undefined;
this.startupComplete = false;
this.supervisionAfterOfferId = undefined;
this.retryOfferId = undefined;
this.recoveryOfferId = undefined;
this.draining = false;
this.drainOperation = undefined;
this.stopController = new AbortController();
return 'started';
}
beginDrain(): Promise<void> {
if (this.mode === 'stopping') {
return Promise.reject(
new WorkerRemoteExecutionLifecycleError('stopping'),
);
}
if (this.mode !== 'running' || !this.stopController) {
return Promise.reject(
new WorkerRemoteExecutionLifecycleError('inactive'),
);
}
if (this.drainOperation) return this.drainOperation;
if (this.draining) return Promise.resolve();
this.draining = true;
this.stopController.abort(
new WorkerRemoteExecutionLifecycleError('draining'),
);
const operation = (async () => {
await this.inFlight?.catch(() => undefined);
if (this.mode === 'running') this.stopController = new AbortController();
})().finally(() => {
if (this.drainOperation === operation) this.drainOperation = undefined;
});
this.drainOperation = operation;
return operation;
}
tick(signal?: AbortSignal): Promise<WorkerRemoteExecutionLifecycleTickResult> {
if (this.mode === 'stopping') {
return Promise.reject(
new WorkerRemoteExecutionLifecycleError('stopping'),
);
}
if (this.mode !== 'running' || !this.stopController) {
return Promise.reject(
new WorkerRemoteExecutionLifecycleError('inactive'),
);
}
if (this.inFlight) return this.inFlight;
const combinedSignal = signal === undefined
? this.stopController.signal
: AbortSignal.any([signal, this.stopController.signal]);
const operation = this.tickOnce(combinedSignal).finally(() => {
if (this.inFlight === operation) this.inFlight = undefined;
});
this.inFlight = operation;
return operation;
}
stop(): Promise<void> {
if (this.stopOperation) return this.stopOperation;
if (this.mode === 'inactive') return Promise.resolve();
this.mode = 'stopping';
this.stopController?.abort(
new WorkerRemoteExecutionLifecycleError('stopping'),
);
const operation = (async () => {
try {
await this.drainOperation?.catch(() => undefined);
await this.inFlight?.catch(() => undefined);
await this.journal.releaseOwnership();
} finally {
this.mode = 'inactive';
this.draining = false;
this.stopController = undefined;
this.drainOperation = undefined;
this.stopOperation = undefined;
}
})();
this.stopOperation = operation;
return operation;
}
private async tickOnce(
signal: AbortSignal,
): Promise<WorkerRemoteExecutionLifecycleTickResult> {
if (this.recoveryOfferId) {
return Object.freeze({
status: 'recovery_required' as const,
offerId: this.recoveryOfferId,
});
}
if (!this.startupComplete) {
return this.reconcileStartupPage();
}
if (this.retryOfferId) {
const offerId = this.retryOfferId;
const execution = await this.processor.process(offerId);
return this.observeExecution(offerId, execution);
}
const supervision = await this.supervisePage();
if (supervision) return supervision;
if (this.draining) {
return Object.freeze({ status: 'draining' as const });
}
const session = this.currentSessionProvider();
const now = this.nowProvider();
if (
!session ||
session.status !== 'available' ||
!Number.isSafeInteger(now) ||
now < 0 ||
session.leaseExpiresAtMs <= now
) {
return Object.freeze({ status: 'session_unavailable' as const });
}
if (signal.aborted) throw signal.reason;
const pull = await this.offers.pull(session, signal);
if (pull.status !== 'accepted' && pull.status !== 'replayed') {
return Object.freeze({ status: 'pull_result' as const, pull });
}
this.retryOfferId = pull.offerId;
const execution = await this.processor.process(pull.offerId);
return this.observeExecution(pull.offerId, execution, pull);
}
private async supervisePage(): Promise<
WorkerRemoteExecutionLifecycleTickResult | undefined
> {
const page = await this.journal.listOffers({
...(this.supervisionAfterOfferId === undefined
? {}
: { afterOfferId: this.supervisionAfterOfferId }),
limit: this.maximumSupervisionRecordsPerTick,
});
for (const record of page.records) {
if (record.state === 'recovery_required') {
this.recoveryOfferId = record.offer.offerId;
return Object.freeze({
status: 'recovery_required' as const,
offerId: record.offer.offerId,
});
}
if (
record.state !== 'launching' &&
record.state !== 'started' &&
record.state !== 'running_acknowledged'
) continue;
const control = await this.control.reconcile(record.offer.offerId);
if (this.controlRequiresRecovery(control)) {
this.recoveryOfferId = record.offer.offerId;
return Object.freeze({
status: 'recovery_required' as const,
offerId: record.offer.offerId,
});
}
}
this.supervisionAfterOfferId = page.nextAfterOfferId;
return undefined;
}
private controlRequiresRecovery(
result: WorkerRemoteExecutionControlResult,
): boolean {
return result.status === 'lease_expired' ||
result.status === 'terminal' ||
result.status === 'completion_terminal';
}
private async reconcileStartupPage(): Promise<
WorkerRemoteExecutionLifecycleTickResult
> {
const page = await this.journal.listOffers({
...(this.startupAfterOfferId === undefined
? {}
: { afterOfferId: this.startupAfterOfferId }),
limit: this.maximumRecordsPerTick,
});
let processed = 0;
for (const record of page.records) {
if (record.state === 'recovery_required') {
this.recoveryOfferId = record.offer.offerId;
return Object.freeze({
status: 'recovery_required' as const,
offerId: record.offer.offerId,
});
}
if (
record.state === 'accepted' ||
record.state === 'starting_acknowledged' ||
record.state === 'launching' ||
record.state === 'started' ||
record.state === 'start_failed'
) {
const execution = await this.processor.process(record.offer.offerId);
processed += 1;
const observed = this.observeExecution(record.offer.offerId, execution);
if (observed.status === 'recovery_required') return observed;
}
}
if (page.nextAfterOfferId !== undefined) {
this.startupAfterOfferId = page.nextAfterOfferId;
return Object.freeze({
status: 'reconciling' as const,
processed,
nextAfterOfferId: page.nextAfterOfferId,
});
}
this.startupAfterOfferId = undefined;
this.startupComplete = true;
return Object.freeze({
status: 'reconciled' as const,
processed,
});
}
private observeExecution(
offerId: string,
execution: WorkerRemoteExecutionProcessResult,
pull?: WorkerRemoteOfferPullResult,
): WorkerRemoteExecutionLifecycleTickResult {
if (execution.status === 'recovery_required') {
this.recoveryOfferId = offerId;
this.retryOfferId = undefined;
return Object.freeze({
status: 'recovery_required' as const,
offerId,
});
}
this.retryOfferId = undefined;
return Object.freeze({
status: 'processed' as const,
offerId,
execution,
...(pull === undefined ? {} : { pull }),
});
}
}
packages/ql3-worker-runtime/src/remote-execution/remoteOfferDelivery.ts
+458
View File
@@ -0,0 +1,458 @@
// Remote Execution owns stable offer claiming, delivery, and durable admission.
import { randomBytes, randomUUID } from 'node:crypto';
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import {
assertWorkerId,
assertWorkerSessionId,
} from '@qinglong/runtime-core/worker-session';
import {
InvalidRemoteExecutionOfferDeliveryError,
MAX_REMOTE_EXECUTION_OFFER_RESPONSE_BYTES,
normalizeRemoteExecutionOfferClaimAuthority,
parseRemoteExecutionOfferPullResponse,
type RemoteExecutionOfferClaimAuthority,
type RemoteExecutionOfferDeliveryStats,
type RemoteExecutionOfferIdleReason,
} from '@qinglong/runtime-core/remote-offer-delivery';
import {
normalizeWorkerRemoteExecutionInboxRecord,
type WorkerRemoteExecutionInboxRecord,
} from './executionInbox';
export const MAX_WORKER_REMOTE_OFFER_ATTEMPTS = 16;
export const MAX_WORKER_REMOTE_OFFER_BACKOFF_MS = 60_000;
export const MAX_WORKER_REMOTE_OFFER_INBOX_ENTRIES = 1024;
export const DEFAULT_WORKER_REMOTE_OFFER_INBOX_ENTRIES = 64;
export const MAX_WORKER_REMOTE_OFFER_RECORD_BYTES = 160 * 1024;
export interface WorkerRemoteOfferClaimRecord
extends RemoteExecutionOfferClaimAuthority {
readonly schemaVersion: 1;
readonly revision: number;
readonly attemptCount: number;
readonly createdAtMs: number;
readonly updatedAtMs: number;
readonly lastAttemptAtMs: number | null;
readonly nextAttemptAtMs: number;
}
/** @deprecated Use WorkerRemoteExecutionInboxRecord. */
export type WorkerRemoteOfferInboxRecord = WorkerRemoteExecutionInboxRecord;
export type WorkerRemoteOfferInboxAcceptResult = Readonly<{
status: 'accepted' | 'replayed';
record: WorkerRemoteOfferInboxRecord;
}>;
export interface WorkerRemoteOfferDeliveryJournal {
readPendingClaim(): Promise<WorkerRemoteOfferClaimRecord | undefined>;
createPendingClaim(
record: WorkerRemoteOfferClaimRecord,
): Promise<WorkerRemoteOfferClaimRecord>;
replacePendingClaim(
record: WorkerRemoteOfferClaimRecord,
expectedRevision: number,
): Promise<WorkerRemoteOfferClaimRecord>;
clearPendingClaim(offerId: string, expectedRevision: number): Promise<void>;
acceptOffer(
offer: ClusterRemoteExecutionOffer,
acceptedAtMs: number,
): Promise<WorkerRemoteOfferInboxAcceptResult>;
readOffer(offerId: string): Promise<WorkerRemoteOfferInboxRecord | undefined>;
}
export interface WorkerRemoteOfferTransport {
exchange(request: Readonly<{
path: string;
body: Readonly<{
workerGeneration: number;
offerId: string;
leaseToken: string;
}>;
maximumResponseBytes: number;
signal?: AbortSignal;
}>): Promise<Uint8Array | string>;
}
export interface WorkerRemoteOfferSession {
readonly workerId: string;
readonly sessionId: string;
readonly generation: number;
}
export interface WorkerRemoteOfferPullCoordinatorOptions {
readonly journal: WorkerRemoteOfferDeliveryJournal;
readonly transport: WorkerRemoteOfferTransport;
readonly currentSession: () => WorkerRemoteOfferSession | undefined;
readonly now?: () => number;
readonly random?: () => number;
readonly backoffBaseMs?: number;
}
export type WorkerRemoteOfferPullResult =
| Readonly<{
status: 'accepted' | 'replayed';
offerId: string;
stats: RemoteExecutionOfferDeliveryStats;
truncated: boolean;
}>
| Readonly<{
status: 'idle';
reason: RemoteExecutionOfferIdleReason;
stats: RemoteExecutionOfferDeliveryStats;
truncated: boolean;
}>
| Readonly<{
status: 'backoff' | 'unavailable' | 'invalid_response';
offerId: string;
nextAttemptAtMs: number;
}>;
export class WorkerRemoteOfferDeliveryError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'claim_conflict'
| 'claim_revision_conflict'
| 'offer_conflict'
| 'attempt_budget_exhausted',
) {
super(`Worker remote offer delivery failed: ${reason}`);
this.name = 'WorkerRemoteOfferDeliveryError';
}
}
function safeTime(value: number, label: string): number {
if (!Number.isSafeInteger(value) || value < 0) {
throw new WorkerRemoteOfferDeliveryError('invalid_configuration');
}
return value;
}
function safeRevision(value: number): number {
if (!Number.isSafeInteger(value) || value < 0) {
throw new WorkerRemoteOfferDeliveryError('claim_revision_conflict');
}
return value;
}
export function normalizeWorkerRemoteOfferClaimRecord(
value: WorkerRemoteOfferClaimRecord,
): WorkerRemoteOfferClaimRecord {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
const expected = [
'schemaVersion', 'revision', 'workerId', 'workerSessionId',
'workerGeneration', 'offerId', 'leaseToken', 'attemptCount',
'createdAtMs', 'updatedAtMs', 'lastAttemptAtMs', 'nextAttemptAtMs',
].sort();
const actual = Object.keys(value).sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index]) ||
value.schemaVersion !== 1
) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
const authority = normalizeRemoteExecutionOfferClaimAuthority({
workerId: value.workerId,
workerSessionId: value.workerSessionId,
workerGeneration: value.workerGeneration,
offerId: value.offerId,
leaseToken: value.leaseToken,
});
safeRevision(value.revision);
if (
!Number.isSafeInteger(value.attemptCount) ||
value.attemptCount < 0 ||
value.attemptCount > MAX_WORKER_REMOTE_OFFER_ATTEMPTS
) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
const createdAtMs = safeTime(value.createdAtMs, 'createdAtMs');
const updatedAtMs = safeTime(value.updatedAtMs, 'updatedAtMs');
const nextAttemptAtMs = safeTime(value.nextAttemptAtMs, 'nextAttemptAtMs');
if (
updatedAtMs < createdAtMs ||
nextAttemptAtMs < createdAtMs ||
(value.lastAttemptAtMs !== null &&
(!Number.isSafeInteger(value.lastAttemptAtMs) ||
value.lastAttemptAtMs < createdAtMs ||
value.lastAttemptAtMs > updatedAtMs))
) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
return Object.freeze({
schemaVersion: 1,
revision: value.revision,
...authority,
attemptCount: value.attemptCount,
createdAtMs,
updatedAtMs,
lastAttemptAtMs: value.lastAttemptAtMs,
nextAttemptAtMs,
});
}
export function createWorkerRemoteOfferClaimRecord(
authority: RemoteExecutionOfferClaimAuthority,
createdAtMs: number,
): WorkerRemoteOfferClaimRecord {
const normalized = normalizeRemoteExecutionOfferClaimAuthority(authority);
const now = safeTime(createdAtMs, 'createdAtMs');
return normalizeWorkerRemoteOfferClaimRecord({
schemaVersion: 1,
revision: 0,
...normalized,
attemptCount: 0,
createdAtMs: now,
updatedAtMs: now,
lastAttemptAtMs: null,
nextAttemptAtMs: now,
});
}
export function normalizeWorkerRemoteOfferInboxRecord(
value: WorkerRemoteOfferInboxRecord,
): WorkerRemoteOfferInboxRecord {
try {
return normalizeWorkerRemoteExecutionInboxRecord(value);
} catch {
throw new WorkerRemoteOfferDeliveryError('offer_conflict');
}
}
export function sameWorkerRemoteOfferAuthority(
left: ClusterRemoteExecutionOffer,
right: ClusterRemoteExecutionOffer,
): boolean {
const leftOffer = createClusterRemoteExecutionOffer(left);
const rightOffer = createClusterRemoteExecutionOffer(right);
return (
leftOffer.offerId === rightOffer.offerId &&
leftOffer.executionDigest === rightOffer.executionDigest &&
JSON.stringify(leftOffer.candidate) === JSON.stringify(rightOffer.candidate) &&
JSON.stringify(leftOffer.worker) === JSON.stringify(rightOffer.worker) &&
leftOffer.lease.runId === rightOffer.lease.runId &&
leftOffer.lease.attemptId === rightOffer.lease.attemptId &&
leftOffer.lease.leaseGeneration === rightOffer.lease.leaseGeneration &&
leftOffer.lease.leaseTokenDigest === rightOffer.lease.leaseTokenDigest &&
leftOffer.leaseToken === rightOffer.leaseToken &&
JSON.stringify(leftOffer.executionRevision) ===
JSON.stringify(rightOffer.executionRevision)
);
}
export class WorkerRemoteOfferPullCoordinator {
private readonly journal: WorkerRemoteOfferDeliveryJournal;
private readonly transport: WorkerRemoteOfferTransport;
private readonly currentSessionProvider: () =>
WorkerRemoteOfferSession | undefined;
private readonly nowProvider: () => number;
private readonly randomProvider: () => number;
private readonly backoffBaseMs: number;
private inFlight?: Readonly<{
session: WorkerRemoteOfferSession;
operation: Promise<WorkerRemoteOfferPullResult>;
}>;
constructor(options: WorkerRemoteOfferPullCoordinatorOptions) {
if (
!options ||
typeof options.journal?.readPendingClaim !== 'function' ||
typeof options.transport?.exchange !== 'function' ||
typeof options.currentSession !== 'function'
) {
throw new WorkerRemoteOfferDeliveryError('invalid_configuration');
}
const backoffBaseMs = options.backoffBaseMs ?? 1_000;
if (
!Number.isSafeInteger(backoffBaseMs) ||
backoffBaseMs < 100 ||
backoffBaseMs > MAX_WORKER_REMOTE_OFFER_BACKOFF_MS
) {
throw new WorkerRemoteOfferDeliveryError('invalid_configuration');
}
this.journal = options.journal;
this.transport = options.transport;
this.currentSessionProvider = options.currentSession;
this.nowProvider = options.now ?? Date.now;
this.randomProvider = options.random ?? Math.random;
this.backoffBaseMs = backoffBaseMs;
}
pull(
session: WorkerRemoteOfferSession,
signal?: AbortSignal,
): Promise<WorkerRemoteOfferPullResult> {
assertWorkerId(session.workerId);
assertWorkerSessionId(session.sessionId);
if (!Number.isSafeInteger(session.generation) || session.generation < 1) {
throw new WorkerRemoteOfferDeliveryError('invalid_configuration');
}
const normalizedSession = Object.freeze({
workerId: session.workerId,
sessionId: session.sessionId,
generation: session.generation,
});
this.assertCurrentSession(normalizedSession);
if (this.inFlight) {
if (
this.inFlight.session.workerId !== normalizedSession.workerId ||
this.inFlight.session.sessionId !== normalizedSession.sessionId ||
this.inFlight.session.generation !== normalizedSession.generation
) {
return Promise.reject(
new WorkerRemoteOfferDeliveryError('claim_conflict'),
);
}
return this.inFlight.operation;
}
const operation = this.pullOnce(normalizedSession, signal)
.finally(() => {
if (this.inFlight?.operation === operation) this.inFlight = undefined;
});
this.inFlight = Object.freeze({
session: normalizedSession,
operation,
});
return operation;
}
private async pullOnce(
session: WorkerRemoteOfferSession,
signal?: AbortSignal,
): Promise<WorkerRemoteOfferPullResult> {
const now = this.now();
let claim = await this.journal.readPendingClaim();
if (claim) {
if (
claim.workerId !== session.workerId ||
claim.workerSessionId !== session.sessionId ||
claim.workerGeneration !== session.generation
) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
} else {
const generated = normalizeRemoteExecutionOfferClaimAuthority({
workerId: session.workerId,
workerSessionId: session.sessionId,
workerGeneration: session.generation,
offerId: randomUUID(),
leaseToken: randomBytes(32).toString('base64url'),
});
claim = await this.journal.createPendingClaim(
createWorkerRemoteOfferClaimRecord(generated, now),
);
}
if (claim.nextAttemptAtMs > now) {
return Object.freeze({
status: 'backoff' as const,
offerId: claim.offerId,
nextAttemptAtMs: claim.nextAttemptAtMs,
});
}
if (claim.attemptCount >= MAX_WORKER_REMOTE_OFFER_ATTEMPTS) {
throw new WorkerRemoteOfferDeliveryError('attempt_budget_exhausted');
}
claim = await this.journal.replacePendingClaim(
normalizeWorkerRemoteOfferClaimRecord({
...claim,
revision: claim.revision + 1,
attemptCount: claim.attemptCount + 1,
lastAttemptAtMs: now,
updatedAtMs: now,
nextAttemptAtMs: now,
}),
claim.revision,
);
try {
const serialized = await this.transport.exchange({
path: `/api/v3/worker-ingress/workers/${claim.workerId}/sessions/${claim.workerSessionId}/offers`,
body: Object.freeze({
workerGeneration: claim.workerGeneration,
offerId: claim.offerId,
leaseToken: claim.leaseToken,
}),
maximumResponseBytes: MAX_REMOTE_EXECUTION_OFFER_RESPONSE_BYTES,
...(signal === undefined ? {} : { signal }),
});
const result = parseRemoteExecutionOfferPullResponse(serialized, {
workerId: claim.workerId,
workerSessionId: claim.workerSessionId,
workerGeneration: claim.workerGeneration,
offerId: claim.offerId,
leaseToken: claim.leaseToken,
});
if (result.status === 'idle') {
this.assertCurrentSession(session);
await this.journal.clearPendingClaim(claim.offerId, claim.revision);
return Object.freeze({
status: 'idle' as const,
reason: result.reason,
stats: result.stats,
truncated: result.truncated,
});
}
this.assertCurrentSession(session);
const accepted = await this.journal.acceptOffer(result.offer, this.now());
await this.journal.clearPendingClaim(claim.offerId, claim.revision);
return Object.freeze({
status: accepted.status,
offerId: accepted.record.offer.offerId,
stats: result.stats,
truncated: result.truncated,
});
} catch (error) {
if (signal?.aborted) throw signal.reason ?? error;
const nextAttemptAtMs = this.nextAttemptAt(claim.attemptCount, this.now());
await this.journal.replacePendingClaim(
normalizeWorkerRemoteOfferClaimRecord({
...claim,
revision: claim.revision + 1,
updatedAtMs: this.now(),
nextAttemptAtMs,
}),
claim.revision,
);
return Object.freeze({
status:
error instanceof InvalidRemoteExecutionOfferDeliveryError
? 'invalid_response' as const
: 'unavailable' as const,
offerId: claim.offerId,
nextAttemptAtMs,
});
}
}
private nextAttemptAt(attemptCount: number, now: number): number {
const random = this.randomProvider();
if (!Number.isFinite(random) || random < 0 || random >= 1) {
throw new WorkerRemoteOfferDeliveryError('invalid_configuration');
}
const ceiling = Math.min(
MAX_WORKER_REMOTE_OFFER_BACKOFF_MS,
this.backoffBaseMs * 2 ** Math.max(0, attemptCount - 1),
);
return safeTime(now + Math.floor(random * ceiling), 'nextAttemptAtMs');
}
private assertCurrentSession(expected: WorkerRemoteOfferSession): void {
const current = this.currentSessionProvider();
if (
!current ||
current.workerId !== expected.workerId ||
current.sessionId !== expected.sessionId ||
current.generation !== expected.generation
) {
throw new WorkerRemoteOfferDeliveryError('claim_conflict');
}
}
private now(): number {
return safeTime(this.nowProvider(), 'now');
}
}
packages/ql3-worker-runtime/src/remote-execution/remoteOfferDeliveryEntrypoint.ts
+15
View File
@@ -0,0 +1,15 @@
// Remote Execution owns its public delivery composition and transport exports.
export * from './executionInbox';
export * from './executionInboxProcessor';
export * from './executionContextMaterializer';
export * from './headlessExecutionLifecycle';
export * from './remoteOfferDelivery';
export * from './remoteOfferFileJournal';
export * from './transport/remoteActivationHttpsClient';
export * from './transport/remoteOfferHttpsTransport';
export * from './transport/workerIngressHttpsClient';
export * from './transport/remoteSecretHttpsProvider';
export * from '../execution/workerFileLogArtifactAllocator';
export * from './transport/remoteWorkerCompletionHttpsClient';
export * from './transport/remoteWorkerLeaseControlHttpsClient';
export * from '../execution/workerExecutionControlCoordinator';
packages/ql3-worker-runtime/src/remote-execution/remoteOfferFileJournal.ts
+543
View File
@@ -0,0 +1,543 @@
// Remote Execution owns the private atomic offer journal and its single-owner fence.
import { randomBytes } from 'node:crypto';
import { constants } from 'node:fs';
import {
chmod,
link,
lstat,
mkdir,
open,
readdir,
rename,
rm,
unlink,
} from 'node:fs/promises';
import { isAbsolute, join } from 'node:path';
import { lock } from 'proper-lockfile';
import { assertRunDispatchId } from '@qinglong/runtime-core/run-dispatch-lease';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import {
DEFAULT_WORKER_REMOTE_OFFER_INBOX_ENTRIES,
MAX_WORKER_REMOTE_OFFER_INBOX_ENTRIES,
MAX_WORKER_REMOTE_OFFER_RECORD_BYTES,
normalizeWorkerRemoteOfferClaimRecord,
sameWorkerRemoteOfferAuthority,
type WorkerRemoteOfferClaimRecord,
type WorkerRemoteOfferDeliveryJournal,
type WorkerRemoteOfferInboxAcceptResult,
} from './remoteOfferDelivery';
import type { ClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import {
assertWorkerRemoteExecutionInboxTransition,
createWorkerRemoteExecutionInboxRecord,
normalizeWorkerRemoteExecutionInboxRecord,
WorkerRemoteExecutionInboxError,
type WorkerRemoteExecutionInbox,
type WorkerRemoteExecutionInboxPage,
type WorkerRemoteExecutionInboxRecord,
} from './executionInbox';
const OFFER_FILE = /^([A-Za-z0-9._:-]{1,128})\.json$/;
const MIN_OWNERSHIP_STALE_MS = 5_000;
const MAX_OWNERSHIP_STALE_MS = 5 * 60_000;
export interface WorkerRemoteOfferFileJournalOptions {
readonly rootDirectory: string;
readonly maximumEntries?: number;
readonly ownershipStaleMs?: number;
}
export class WorkerRemoteOfferFileJournalError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'not_owned'
| 'already_owned'
| 'ownership_compromised'
| 'unsafe_storage'
| 'capacity_exhausted'
| 'claim_revision_conflict'
| 'offer_revision_conflict'
| 'invalid_transition'
| 'offer_conflict',
) {
super(`Worker remote offer file journal failed: ${reason}`);
this.name = 'WorkerRemoteOfferFileJournalError';
}
}
function isCode(error: unknown, code: string): boolean {
return (
typeof error === 'object' &&
error !== null &&
(error as NodeJS.ErrnoException).code === code
);
}
async function safeDirectory(path: string): Promise<void> {
try {
let created = false;
try {
await lstat(path);
} catch (error) {
if (!isCode(error, 'ENOENT')) throw error;
await mkdir(path, { recursive: true, mode: 0o700 });
created = true;
}
const stat = await lstat(path);
if (!stat.isDirectory() || stat.isSymbolicLink() || (stat.mode & 0o077) !== 0) {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
if (created) await chmod(path, 0o700);
} catch (error) {
if (error instanceof WorkerRemoteOfferFileJournalError) throw error;
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
}
async function syncDirectory(path: string): Promise<void> {
const handle = await open(path, constants.O_RDONLY);
try {
await handle.sync();
} finally {
await handle.close();
}
}
function serialize(value: unknown): Buffer {
const bytes = Buffer.from(`${JSON.stringify(value)}\n`, 'utf8');
if (bytes.byteLength < 2 || bytes.byteLength > MAX_WORKER_REMOTE_OFFER_RECORD_BYTES) {
bytes.fill(0);
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
return bytes;
}
async function readJson(path: string): Promise<unknown> {
let handle;
try {
handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
const stat = await handle.stat();
if (
!stat.isFile() ||
stat.size < 2 ||
stat.size > MAX_WORKER_REMOTE_OFFER_RECORD_BYTES ||
(stat.mode & 0o077) !== 0
) {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
const bytes = await handle.readFile();
try {
return JSON.parse(bytes.toString('utf8')) as unknown;
} finally {
bytes.fill(0);
}
} catch (error) {
if (isCode(error, 'ENOENT')) return undefined;
if (error instanceof WorkerRemoteOfferFileJournalError) throw error;
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
} finally {
await handle?.close().catch(() => undefined);
}
}
export class WorkerRemoteOfferFileJournal
implements WorkerRemoteOfferDeliveryJournal, WorkerRemoteExecutionInbox {
private readonly rootDirectory: string;
private readonly offersDirectory: string;
private readonly maximumEntries: number;
private readonly ownershipStaleMs: number;
private releaseOwnershipLock?: () => Promise<void>;
private compromised = false;
private mutationTail: Promise<void> = Promise.resolve();
constructor(options: WorkerRemoteOfferFileJournalOptions) {
if (
!options ||
typeof options.rootDirectory !== 'string' ||
!isAbsolute(options.rootDirectory) ||
options.rootDirectory.length > 4096 ||
/[\0\r\n]/.test(options.rootDirectory)
) {
throw new WorkerRemoteOfferFileJournalError('invalid_configuration');
}
const maximumEntries =
options.maximumEntries ?? DEFAULT_WORKER_REMOTE_OFFER_INBOX_ENTRIES;
if (
!Number.isSafeInteger(maximumEntries) ||
maximumEntries < 1 ||
maximumEntries > MAX_WORKER_REMOTE_OFFER_INBOX_ENTRIES
) {
throw new WorkerRemoteOfferFileJournalError('invalid_configuration');
}
const ownershipStaleMs = options.ownershipStaleMs ?? 30_000;
if (
!Number.isSafeInteger(ownershipStaleMs) ||
ownershipStaleMs < MIN_OWNERSHIP_STALE_MS ||
ownershipStaleMs > MAX_OWNERSHIP_STALE_MS
) {
throw new WorkerRemoteOfferFileJournalError('invalid_configuration');
}
this.rootDirectory = options.rootDirectory;
this.offersDirectory = join(options.rootDirectory, 'offers');
this.maximumEntries = maximumEntries;
this.ownershipStaleMs = ownershipStaleMs;
}
async acquireOwnership(): Promise<void> {
if (this.releaseOwnershipLock) {
throw new WorkerRemoteOfferFileJournalError('already_owned');
}
await safeDirectory(this.rootDirectory);
await safeDirectory(this.offersDirectory);
try {
this.compromised = false;
this.releaseOwnershipLock = await lock(this.rootDirectory, {
stale: this.ownershipStaleMs,
update: Math.floor(this.ownershipStaleMs / 2),
retries: 0,
realpath: true,
lockfilePath: join(this.rootDirectory, '.owner.lock'),
onCompromised: () => {
this.compromised = true;
this.releaseOwnershipLock = undefined;
},
});
} catch {
throw new WorkerRemoteOfferFileJournalError('already_owned');
}
}
async releaseOwnership(): Promise<void> {
this.assertOwned();
const release = this.releaseOwnershipLock!;
this.releaseOwnershipLock = undefined;
await this.mutationTail.catch(() => undefined);
try {
await release();
} catch {
throw new WorkerRemoteOfferFileJournalError('ownership_compromised');
}
}
async readPendingClaim(): Promise<WorkerRemoteOfferClaimRecord | undefined> {
this.assertOwned();
const value = await readJson(join(this.rootDirectory, 'pending-claim.json'));
if (value === undefined) return undefined;
return normalizeWorkerRemoteOfferClaimRecord(
value as WorkerRemoteOfferClaimRecord,
);
}
createPendingClaim(
record: WorkerRemoteOfferClaimRecord,
): Promise<WorkerRemoteOfferClaimRecord> {
return this.mutate(async () => {
const candidate = normalizeWorkerRemoteOfferClaimRecord(record);
const existing = await this.readPendingClaim();
if (existing) {
if (!this.sameClaim(existing, candidate)) {
throw new WorkerRemoteOfferFileJournalError('offer_conflict');
}
return existing;
}
await this.writeFirst(join(this.rootDirectory, 'pending-claim.json'), candidate);
return candidate;
});
}
replacePendingClaim(
record: WorkerRemoteOfferClaimRecord,
expectedRevision: number,
): Promise<WorkerRemoteOfferClaimRecord> {
return this.mutate(async () => {
const candidate = normalizeWorkerRemoteOfferClaimRecord(record);
const existing = await this.readPendingClaim();
if (
!existing ||
existing.revision !== expectedRevision ||
candidate.revision !== expectedRevision + 1 ||
!this.sameClaim(existing, candidate)
) {
throw new WorkerRemoteOfferFileJournalError('claim_revision_conflict');
}
await this.writeReplacement(
join(this.rootDirectory, 'pending-claim.json'),
candidate,
);
return candidate;
});
}
clearPendingClaim(offerId: string, expectedRevision: number): Promise<void> {
return this.mutate(async () => {
assertRunDispatchId('offerId', offerId);
const existing = await this.readPendingClaim();
if (
!existing ||
existing.offerId !== offerId ||
existing.revision !== expectedRevision
) {
throw new WorkerRemoteOfferFileJournalError('claim_revision_conflict');
}
try {
await unlink(join(this.rootDirectory, 'pending-claim.json'));
await syncDirectory(this.rootDirectory);
} catch {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
});
}
acceptOffer(
delivered: ClusterRemoteExecutionOffer,
acceptedAtMs: number,
): Promise<WorkerRemoteOfferInboxAcceptResult> {
return this.mutate(async () => {
const offer = createClusterRemoteExecutionOffer(delivered);
const existing = await this.readOffer(offer.offerId);
if (existing) {
if (!sameWorkerRemoteOfferAuthority(existing.offer, offer)) {
throw new WorkerRemoteOfferFileJournalError('offer_conflict');
}
if (offer.lease.version > existing.offer.lease.version) {
const updated = normalizeWorkerRemoteExecutionInboxRecord({
...existing,
revision: existing.revision + 1,
offer,
updatedAtMs: acceptedAtMs,
});
this.assertOfferTransition(existing, updated);
await this.writeReplacement(this.offerPath(offer.offerId), updated);
return Object.freeze({ status: 'replayed' as const, record: updated });
}
return Object.freeze({ status: 'replayed' as const, record: existing });
}
const names = await this.offerNames();
if (names.length >= this.maximumEntries) {
throw new WorkerRemoteOfferFileJournalError('capacity_exhausted');
}
const record = createWorkerRemoteExecutionInboxRecord(offer, acceptedAtMs);
await this.writeFirst(this.offerPath(offer.offerId), record);
return Object.freeze({ status: 'accepted' as const, record });
});
}
async readOffer(
offerId: string,
): Promise<WorkerRemoteExecutionInboxRecord | undefined> {
this.assertOwned();
const value = await readJson(this.offerPath(offerId));
if (value === undefined) return undefined;
return normalizeWorkerRemoteExecutionInboxRecord(
value as WorkerRemoteExecutionInboxRecord,
);
}
replaceOffer(
record: WorkerRemoteExecutionInboxRecord,
expectedRevision: number,
): Promise<void> {
return this.mutate(async () => {
const candidate = normalizeWorkerRemoteExecutionInboxRecord(record);
const existing = await this.readOffer(candidate.offer.offerId);
if (
!existing ||
existing.revision !== expectedRevision ||
candidate.revision !== expectedRevision + 1
) {
throw new WorkerRemoteOfferFileJournalError('offer_revision_conflict');
}
this.assertOfferTransition(existing, candidate);
await this.writeReplacement(this.offerPath(candidate.offer.offerId), candidate);
});
}
async listOffers(options: Readonly<{
afterOfferId?: string;
limit?: number;
}> = {}): Promise<WorkerRemoteExecutionInboxPage> {
this.assertOwned();
const limit = options.limit ?? 16;
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 64) {
throw new WorkerRemoteOfferFileJournalError('invalid_configuration');
}
if (options.afterOfferId !== undefined) {
this.offerPath(options.afterOfferId);
}
const names = await this.offerNames();
const selected = names
.filter((offerId) =>
options.afterOfferId === undefined || offerId > options.afterOfferId)
.slice(0, limit);
const records: WorkerRemoteExecutionInboxRecord[] = [];
for (const offerId of selected) {
const record = await this.readOffer(offerId);
if (!record) {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
records.push(record);
}
const hasMore = selected.length > 0 &&
names.some((offerId) => offerId > selected[selected.length - 1]!);
return Object.freeze({
records: Object.freeze(records),
...(hasMore
? { nextAfterOfferId: selected[selected.length - 1]! }
: {}),
});
}
private assertOfferTransition(
previous: WorkerRemoteExecutionInboxRecord,
next: WorkerRemoteExecutionInboxRecord,
): void {
try {
assertWorkerRemoteExecutionInboxTransition(previous, next);
} catch (error) {
if (
error instanceof WorkerRemoteExecutionInboxError &&
error.reason === 'revision_conflict'
) {
throw new WorkerRemoteOfferFileJournalError('offer_revision_conflict');
}
if (
error instanceof WorkerRemoteExecutionInboxError &&
error.reason === 'invalid_transition'
) {
throw new WorkerRemoteOfferFileJournalError('invalid_transition');
}
throw new WorkerRemoteOfferFileJournalError('offer_conflict');
}
}
private mutate<T>(operation: () => Promise<T>): Promise<T> {
this.assertOwned();
const result = this.mutationTail.then(operation, operation);
this.mutationTail = result.then(() => undefined, () => undefined);
return result;
}
private assertOwned(): void {
if (this.compromised) {
throw new WorkerRemoteOfferFileJournalError('ownership_compromised');
}
if (!this.releaseOwnershipLock) {
throw new WorkerRemoteOfferFileJournalError('not_owned');
}
}
private sameClaim(
left: WorkerRemoteOfferClaimRecord,
right: WorkerRemoteOfferClaimRecord,
): boolean {
return (
left.workerId === right.workerId &&
left.workerSessionId === right.workerSessionId &&
left.workerGeneration === right.workerGeneration &&
left.offerId === right.offerId &&
left.leaseToken === right.leaseToken
);
}
private offerPath(offerId: string): string {
assertRunDispatchId('offerId', offerId);
if (!/^[A-Za-z0-9._:-]+$/.test(offerId)) {
throw new WorkerRemoteOfferFileJournalError('offer_conflict');
}
return join(this.offersDirectory, `${offerId}.json`);
}
private async offerNames(): Promise<string[]> {
this.assertOwned();
let entries;
try {
entries = await readdir(this.offersDirectory, { withFileTypes: true });
} catch {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
const names: string[] = [];
for (const entry of entries) {
const match = OFFER_FILE.exec(entry.name);
if (!entry.isFile() || !match) {
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
}
names.push(match[1]!);
}
return names.sort();
}
private temporary(target: string): string {
return join(
this.rootDirectory,
`.${target.split('/').at(-1)}.${randomBytes(16).toString('hex')}.tmp`,
);
}
private async writeFirst(target: string, value: unknown): Promise<void> {
const temporary = this.temporary(target);
const bytes = serialize(value);
try {
const handle = await open(
temporary,
constants.O_WRONLY |
constants.O_CREAT |
constants.O_EXCL |
(constants.O_NOFOLLOW ?? 0),
0o600,
);
try {
await handle.writeFile(bytes);
await handle.sync();
await handle.chmod(0o600);
} finally {
await handle.close();
}
await link(temporary, target);
await syncDirectory(target.startsWith(this.offersDirectory)
? this.offersDirectory
: this.rootDirectory);
} catch (error) {
if (isCode(error, 'EEXIST')) {
throw new WorkerRemoteOfferFileJournalError('offer_conflict');
}
if (error instanceof WorkerRemoteOfferFileJournalError) throw error;
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
} finally {
bytes.fill(0);
await rm(temporary, { force: true }).catch(() => undefined);
}
}
private async writeReplacement(target: string, value: unknown): Promise<void> {
const temporary = this.temporary(target);
const bytes = serialize(value);
try {
const handle = await open(
temporary,
constants.O_WRONLY |
constants.O_CREAT |
constants.O_EXCL |
(constants.O_NOFOLLOW ?? 0),
0o600,
);
try {
await handle.writeFile(bytes);
await handle.sync();
await handle.chmod(0o600);
} finally {
await handle.close();
}
await rename(temporary, target);
await syncDirectory(target.startsWith(this.offersDirectory)
? this.offersDirectory
: this.rootDirectory);
} catch (error) {
if (error instanceof WorkerRemoteOfferFileJournalError) throw error;
throw new WorkerRemoteOfferFileJournalError('unsafe_storage');
} finally {
bytes.fill(0);
await rm(temporary, { force: true }).catch(() => undefined);
}
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/remoteActivationHttpsClient.ts
+171
View File
@@ -0,0 +1,171 @@
// Remote Execution transport owns starting, running, and start-failure acknowledgements.
import {
assertAcknowledgeRemoteRunRunningCommand,
assertAcknowledgeRemoteRunStartingCommand,
assertFailRemoteRunStartCommand,
type AcknowledgeRemoteRunRunningCommand,
type AcknowledgeRemoteRunStartingCommand,
type FailRemoteRunStartCommand,
type RemoteRunActivationResult,
} from '@qinglong/runtime-core/remote-activation';
import {
MAX_REMOTE_RUN_ACTIVATION_RESPONSE_BYTES,
parseRemoteRunActivationResponse,
} from '@qinglong/runtime-core/remote-activation-delivery';
import type { WorkerRemoteExecutionActivationClient } from '../executionInboxProcessor';
import {
WorkerIngressHttpsClient,
WorkerIngressHttpsClientError,
} from './workerIngressHttpsClient';
type ActivationCommand =
| AcknowledgeRemoteRunStartingCommand
| AcknowledgeRemoteRunRunningCommand
| FailRemoteRunStartCommand;
export interface WorkerRemoteExecutionHttpsActivationClientOptions {
readonly client: WorkerIngressHttpsClient;
}
export class WorkerRemoteExecutionHttpsActivationError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'request_invalid'
| 'transport_unavailable'
| 'response_invalid',
options?: ErrorOptions,
) {
super(`Worker remote execution activation failed: ${reason}`, options);
this.name = 'WorkerRemoteExecutionHttpsActivationError';
}
}
function path(command: ActivationCommand, operation: string): string {
return '/api/v3/worker-ingress/workers/' + command.workerId +
'/sessions/' + command.workerSessionId + '/' + operation;
}
function fenceBody(command: ActivationCommand): Readonly<{
runId: string;
attemptId: string;
workerGeneration: number;
offerId: string;
leaseGeneration: number;
leaseToken: string;
expectedLeaseVersion: number;
}> {
return Object.freeze({
runId: command.runId,
attemptId: command.attemptId,
workerGeneration: command.workerGeneration,
offerId: command.offerId,
leaseGeneration: command.leaseGeneration,
leaseToken: command.leaseToken,
expectedLeaseVersion: command.expectedLeaseVersion,
});
}
export class WorkerRemoteExecutionHttpsActivationClient
implements WorkerRemoteExecutionActivationClient {
private readonly client: WorkerIngressHttpsClient;
constructor(options: WorkerRemoteExecutionHttpsActivationClientOptions) {
if (!options || !(options.client instanceof WorkerIngressHttpsClient)) {
throw new WorkerRemoteExecutionHttpsActivationError(
'invalid_configuration',
);
}
this.client = options.client;
}
async acknowledgeStarting(
command: AcknowledgeRemoteRunStartingCommand,
): Promise<Readonly<RemoteRunActivationResult>> {
try {
assertAcknowledgeRemoteRunStartingCommand(command);
} catch (error) {
throw new WorkerRemoteExecutionHttpsActivationError(
'request_invalid',
{ cause: error },
);
}
return this.exchange('starting', command, fenceBody(command));
}
async acknowledgeRunning(
command: AcknowledgeRemoteRunRunningCommand,
): Promise<Readonly<RemoteRunActivationResult>> {
try {
assertAcknowledgeRemoteRunRunningCommand(command);
} catch (error) {
throw new WorkerRemoteExecutionHttpsActivationError(
'request_invalid',
{ cause: error },
);
}
return this.exchange('running', command, Object.freeze({
...fenceBody(command),
executorHandle: command.executorHandle,
logArtifactId: command.logArtifactId ?? null,
callbackSequence: command.callbackSequence,
callbackTokenDigest: command.callbackTokenDigest,
}));
}
async failStart(
command: FailRemoteRunStartCommand,
): Promise<Readonly<RemoteRunActivationResult>> {
try {
assertFailRemoteRunStartCommand(command);
} catch (error) {
throw new WorkerRemoteExecutionHttpsActivationError(
'request_invalid',
{ cause: error },
);
}
return this.exchange('start-failure', command, fenceBody(command));
}
private async exchange(
operation: 'starting' | 'running' | 'start-failure',
command: ActivationCommand,
body: unknown,
): Promise<Readonly<RemoteRunActivationResult>> {
let serialized: Uint8Array;
try {
serialized = await this.client.postJson({
path: path(command, operation),
body,
maximumResponseBytes: MAX_REMOTE_RUN_ACTIVATION_RESPONSE_BYTES,
});
} catch (error) {
if (error instanceof WorkerIngressHttpsClientError) {
throw new WorkerRemoteExecutionHttpsActivationError(
'transport_unavailable',
{ cause: error },
);
}
throw error;
}
try {
const result = parseRemoteRunActivationResponse(serialized);
if (
result.snapshot.runId !== command.runId ||
result.snapshot.attemptId !== command.attemptId ||
result.snapshot.leaseGeneration !== command.leaseGeneration
) {
throw new TypeError('activation response authority mismatch');
}
return result;
} catch (error) {
throw new WorkerRemoteExecutionHttpsActivationError(
'response_invalid',
{ cause: error },
);
} finally {
Buffer.from(serialized.buffer, serialized.byteOffset, serialized.byteLength)
.fill(0);
}
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/remoteOfferHttpsTransport.ts
+125
View File
@@ -0,0 +1,125 @@
// Remote Execution transport owns bounded offer exchange over the shared mTLS client.
import type { Agent } from 'node:https';
import type { WorkerRemoteOfferTransport } from '../remoteOfferDelivery';
import {
WorkerIngressHttpsClient,
WorkerIngressHttpsClientError,
type WorkerIngressHttpsCredentialProvider,
type WorkerIngressHttpsCredentials,
type WorkerIngressHttpsRequestFactory,
} from './workerIngressHttpsClient';
const OFFER_PATH =
/^\/api\/v3\/worker-ingress\/workers\/[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\/sessions\/[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\/offers$/;
export type WorkerRemoteOfferHttpsCredentials = WorkerIngressHttpsCredentials;
export type WorkerRemoteOfferHttpsCredentialProvider =
WorkerIngressHttpsCredentialProvider;
export type WorkerRemoteOfferHttpsRequestFactory =
WorkerIngressHttpsRequestFactory;
export interface WorkerRemoteOfferHttpsTransportOptions {
readonly client?: WorkerIngressHttpsClient;
readonly origin?: string | URL;
readonly credentials?: WorkerRemoteOfferHttpsCredentialProvider;
readonly requestTimeoutMs?: number;
readonly agent?: Agent;
/** Injectable only for deterministic transport contract tests. */
readonly requestFactory?: WorkerRemoteOfferHttpsRequestFactory;
}
export class WorkerRemoteOfferHttpsTransportError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'credentials_unavailable'
| 'request_rejected'
| 'response_rejected'
| 'response_too_large'
| 'closed',
) {
super(`Worker remote offer HTTPS transport failed: ${reason}`);
this.name = 'WorkerRemoteOfferHttpsTransportError';
}
}
export class WorkerRemoteOfferHttpsTransport
implements WorkerRemoteOfferTransport {
private readonly client: WorkerIngressHttpsClient;
private readonly ownsClient: boolean;
private closed = false;
constructor(options: WorkerRemoteOfferHttpsTransportOptions) {
if (!options) {
throw new WorkerRemoteOfferHttpsTransportError('invalid_configuration');
}
if (options.client) {
if (
options.origin !== undefined ||
options.credentials !== undefined ||
options.requestTimeoutMs !== undefined ||
options.agent !== undefined ||
options.requestFactory !== undefined
) {
throw new WorkerRemoteOfferHttpsTransportError('invalid_configuration');
}
this.client = options.client;
this.ownsClient = false;
return;
}
if (options.origin === undefined || options.credentials === undefined) {
throw new WorkerRemoteOfferHttpsTransportError('invalid_configuration');
}
try {
this.client = new WorkerIngressHttpsClient({
origin: options.origin,
credentials: options.credentials,
...(options.requestTimeoutMs === undefined
? {}
: { requestTimeoutMs: options.requestTimeoutMs }),
...(options.agent === undefined ? {} : { agent: options.agent }),
...(options.requestFactory === undefined
? {}
: { requestFactory: options.requestFactory }),
});
} catch (error) {
if (error instanceof WorkerIngressHttpsClientError) {
throw new WorkerRemoteOfferHttpsTransportError(error.reason);
}
throw error;
}
this.ownsClient = true;
}
async exchange(request: Readonly<{
path: string;
body: Readonly<{
workerGeneration: number;
offerId: string;
leaseToken: string;
}>;
maximumResponseBytes: number;
signal?: AbortSignal;
}>): Promise<Uint8Array> {
if (this.closed) {
throw new WorkerRemoteOfferHttpsTransportError('closed');
}
if (!request || typeof request.path !== 'string' || !OFFER_PATH.test(request.path)) {
throw new WorkerRemoteOfferHttpsTransportError('request_rejected');
}
try {
return await this.client.postJson(request);
} catch (error) {
if (error instanceof WorkerIngressHttpsClientError) {
throw new WorkerRemoteOfferHttpsTransportError(error.reason);
}
throw error;
}
}
close(): void {
if (this.closed) return;
this.closed = true;
if (this.ownsClient) this.client.close();
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/remoteSecretHttpsProvider.ts
+138
View File
@@ -0,0 +1,138 @@
// Remote Execution transport owns capability-bound Secret delivery.
import {
MAX_REMOTE_SECRET_DELIVERY_REQUEST_BYTES,
MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES,
createRemoteWorkerSecretDeliveryRequestBody,
parseRemoteWorkerSecretDeliveryResponse,
} from '@qinglong/runtime-core/remote-secret-delivery';
import { createClusterRemoteExecutionOffer } from '@qinglong/runtime-core/remote-dispatch';
import type { WorkerRemoteExecutionInbox } from '../executionInbox';
import type {
WorkerRemoteSecretEnvironmentProvider,
WorkerRemoteSecretResolution,
} from '../executionContextMaterializer';
import {
WorkerIngressHttpsClient,
type WorkerIngressHttpsPostRequest,
} from './workerIngressHttpsClient';
export class WorkerRemoteSecretHttpsProviderError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'offer_unavailable'
| 'authority_mismatch'
| 'delivery_unavailable'
| 'response_invalid',
) {
super(`Worker remote Secret HTTPS provider failed: ${reason}`);
this.name = 'WorkerRemoteSecretHttpsProviderError';
}
}
export interface WorkerRemoteSecretHttpsProviderOptions {
readonly client: Pick<WorkerIngressHttpsClient, 'postJson'>;
readonly inbox: Pick<WorkerRemoteExecutionInbox, 'readOffer'>;
}
export class WorkerRemoteSecretHttpsProvider
implements WorkerRemoteSecretEnvironmentProvider {
private readonly client: Pick<WorkerIngressHttpsClient, 'postJson'>;
private readonly inbox: Pick<WorkerRemoteExecutionInbox, 'readOffer'>;
constructor(options: WorkerRemoteSecretHttpsProviderOptions) {
if (
!options ||
typeof options.client?.postJson !== 'function' ||
typeof options.inbox?.readOffer !== 'function'
) throw new WorkerRemoteSecretHttpsProviderError('invalid_configuration');
this.client = options.client;
this.inbox = options.inbox;
}
async resolve(request: Parameters<WorkerRemoteSecretEnvironmentProvider['resolve']>[0])
: Promise<WorkerRemoteSecretResolution | undefined> {
let record;
try {
record = await this.inbox.readOffer(request.offerId);
} catch {
throw new WorkerRemoteSecretHttpsProviderError('offer_unavailable');
}
if (!record || record.state !== 'starting_acknowledged') {
throw new WorkerRemoteSecretHttpsProviderError('offer_unavailable');
}
let offer;
try {
offer = createClusterRemoteExecutionOffer(record.offer);
} catch {
throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
}
const expectedRefs = Object.freeze([
...new Set(offer.executionRevision.environment.flatMap((binding) =>
binding.kind === 'secret' ? [binding.secretRef] : [])),
]);
if (
offer.offerId !== request.offerId ||
offer.executionDigest !== request.executionDigest ||
offer.candidate.projectId !== request.projectId ||
offer.candidate.taskId !== request.taskId ||
offer.candidate.taskRevision !== request.taskRevision ||
offer.candidate.runId !== request.runId ||
offer.candidate.attemptId !== request.attemptId ||
JSON.stringify(expectedRefs) !== JSON.stringify(request.secretRefs)
) throw new WorkerRemoteSecretHttpsProviderError('authority_mismatch');
const path = `/api/v3/worker-ingress/workers/${offer.worker.workerId}` +
`/sessions/${offer.worker.sessionId}/secrets`;
const body = createRemoteWorkerSecretDeliveryRequestBody({
workerId: offer.worker.workerId,
workerSessionId: offer.worker.sessionId,
workerGeneration: offer.worker.generation,
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
projectId: offer.candidate.projectId,
taskId: offer.candidate.taskId,
taskRevision: offer.candidate.taskRevision,
executionDigest: offer.executionDigest,
offerId: offer.offerId,
leaseGeneration: offer.lease.leaseGeneration,
leaseToken: offer.leaseToken,
expectedLeaseVersion: offer.lease.version,
secretRefs: expectedRefs,
});
let serialized: Uint8Array;
try {
const transportRequest: WorkerIngressHttpsPostRequest = {
path,
body,
maximumRequestBytes: MAX_REMOTE_SECRET_DELIVERY_REQUEST_BYTES,
maximumResponseBytes: MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES,
};
serialized = await this.client.postJson(transportRequest);
} catch {
throw new WorkerRemoteSecretHttpsProviderError('delivery_unavailable');
}
try {
const delivered = parseRemoteWorkerSecretDeliveryResponse(serialized, {
runId: offer.candidate.runId,
attemptId: offer.candidate.attemptId,
offerId: offer.offerId,
executionDigest: offer.executionDigest,
secretRefs: expectedRefs,
});
const values = Object.freeze(delivered.values.map((entry) =>
Object.freeze({ secretRef: entry.secretRef, value: entry.value })));
return Object.freeze({
values,
dispose() {
// JavaScript strings cannot be zeroized. Drop all retained references;
// the transport bytes were already scrubbed by the parser.
},
});
} catch {
throw new WorkerRemoteSecretHttpsProviderError('response_invalid');
} finally {
if (Buffer.isBuffer(serialized)) serialized.fill(0);
}
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/remoteWorkerCompletionHttpsClient.ts
+214
View File
@@ -0,0 +1,214 @@
// Remote Execution transport owns Artifact upload and completion acknowledgement.
import {
MAX_REMOTE_WORKER_ARTIFACT_RESPONSE_BYTES,
MAX_REMOTE_WORKER_COMPLETION_REQUEST_BYTES,
MAX_REMOTE_WORKER_COMPLETION_RESPONSE_BYTES,
createRemoteWorkerArtifactUploadPreamble,
createRemoteWorkerCompletionRequestBody,
parseRemoteWorkerArtifactUploadResponse,
parseRemoteWorkerCompletionResponse,
type RemoteWorkerCompletionCommand,
} from '@qinglong/runtime-core/remote-worker-completion';
import type {
WorkerRemoteExecutionCompletionClient,
WorkerRemoteExecutionCompletionCommand,
WorkerRemoteExecutionCompletionResult,
WorkerRemoteLogArtifactUploadCommand,
WorkerRemoteLogArtifactUploadResult,
WorkerRemoteLogArtifactUploader,
} from '../../execution/workerCompletionCoordinator';
import type { WorkerIngressHttpsClient } from './workerIngressHttpsClient';
export class WorkerRemoteCompletionHttpsError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'request_invalid'
| 'transport_unavailable'
| 'response_invalid',
options?: ErrorOptions,
) {
super(`Worker remote completion HTTPS failed: ${reason}`, options);
this.name = 'WorkerRemoteCompletionHttpsError';
}
}
export interface WorkerRemoteArtifactHttpsUploaderOptions {
readonly client: Pick<WorkerIngressHttpsClient, 'postStream'>;
}
export interface WorkerRemoteExecutionHttpsCompletionClientOptions {
readonly client: Pick<WorkerIngressHttpsClient, 'postJson'>;
}
function path(
command: Readonly<{ workerId: string; workerSessionId: string }>,
operation: 'artifacts' | 'completion',
): string {
return `/api/v3/worker-ingress/workers/${command.workerId}` +
`/sessions/${command.workerSessionId}/${operation}`;
}
function erase(value: Uint8Array | undefined): void {
if (!value) return;
Buffer.from(value.buffer, value.byteOffset, value.byteLength).fill(0);
}
export class WorkerRemoteArtifactHttpsUploader
implements WorkerRemoteLogArtifactUploader {
private readonly client: Pick<WorkerIngressHttpsClient, 'postStream'>;
constructor(options: WorkerRemoteArtifactHttpsUploaderOptions) {
if (!options || typeof options.client?.postStream !== 'function') {
throw new WorkerRemoteCompletionHttpsError('invalid_configuration');
}
this.client = options.client;
}
async upload(
command: WorkerRemoteLogArtifactUploadCommand,
): Promise<Readonly<WorkerRemoteLogArtifactUploadResult>> {
let preamble: Buffer;
try {
preamble = createRemoteWorkerArtifactUploadPreamble({
workerId: command.workerId,
workerSessionId: command.workerSessionId,
workerGeneration: command.workerGeneration,
projectId: command.projectId,
runId: command.runId,
attemptId: command.attemptId,
offerId: command.offerId,
leaseGeneration: command.leaseGeneration,
leaseToken: command.leaseToken,
expectedLeaseVersion: command.expectedLeaseVersion,
logArtifactId: command.logArtifactId,
byteLength: command.byteLength,
...(command.truncated === undefined
? {}
: { truncated: command.truncated }),
});
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('request_invalid', {
cause: error,
});
}
let serialized: Uint8Array | undefined;
try {
const content = command.content;
serialized = await this.client.postStream({
path: path(command, 'artifacts'),
body: (async function* () {
yield preamble;
for await (const chunk of content) yield chunk;
})(),
byteLength: preamble.byteLength + command.byteLength,
maximumResponseBytes: MAX_REMOTE_WORKER_ARTIFACT_RESPONSE_BYTES,
});
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('transport_unavailable', {
cause: error,
});
} finally {
preamble.fill(0);
}
try {
const receipt = parseRemoteWorkerArtifactUploadResponse(serialized);
if (
receipt.projectId !== command.projectId ||
receipt.runId !== command.runId ||
receipt.attemptId !== command.attemptId ||
receipt.logArtifactId !== command.logArtifactId ||
receipt.byteLength !== command.byteLength ||
receipt.truncated !== command.truncated
) {
throw new TypeError('Artifact response authority does not match');
}
return Object.freeze({
status: receipt.status,
logArtifactId: receipt.logArtifactId,
byteLength: receipt.byteLength,
sha256: receipt.sha256,
});
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('response_invalid', {
cause: error,
});
} finally {
erase(serialized);
}
}
}
export class WorkerRemoteExecutionHttpsCompletionClient
implements WorkerRemoteExecutionCompletionClient {
private readonly client: Pick<WorkerIngressHttpsClient, 'postJson'>;
constructor(options: WorkerRemoteExecutionHttpsCompletionClientOptions) {
if (!options || typeof options.client?.postJson !== 'function') {
throw new WorkerRemoteCompletionHttpsError('invalid_configuration');
}
this.client = options.client;
}
async complete(
command: WorkerRemoteExecutionCompletionCommand,
): Promise<Readonly<WorkerRemoteExecutionCompletionResult>> {
let body;
try {
if (command.executorType !== 'remote_worker') {
throw new TypeError('completion executor type is invalid');
}
const wire: RemoteWorkerCompletionCommand = {
workerId: command.workerId,
workerSessionId: command.workerSessionId,
workerGeneration: command.workerGeneration,
projectId: command.projectId,
runId: command.runId,
attemptId: command.attemptId,
offerId: command.offerId,
leaseGeneration: command.leaseGeneration,
leaseToken: command.leaseToken,
expectedLeaseVersion: command.expectedLeaseVersion,
callbackSequence: command.callbackSequence,
callbackTokenDigest: command.callbackTokenDigest,
result: command.result,
artifact: command.artifact,
};
body = createRemoteWorkerCompletionRequestBody(wire);
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('request_invalid', {
cause: error,
});
}
let serialized: Uint8Array | undefined;
try {
serialized = await this.client.postJson({
path: path(command, 'completion'),
body,
maximumRequestBytes: MAX_REMOTE_WORKER_COMPLETION_REQUEST_BYTES,
maximumResponseBytes: MAX_REMOTE_WORKER_COMPLETION_RESPONSE_BYTES,
});
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('transport_unavailable', {
cause: error,
});
}
try {
const completed = parseRemoteWorkerCompletionResponse(serialized);
if (
completed.runId !== command.runId ||
completed.attemptId !== command.attemptId ||
completed.callbackSequence !== command.callbackSequence
) {
throw new TypeError('completion response authority does not match');
}
return completed;
} catch (error) {
throw new WorkerRemoteCompletionHttpsError('response_invalid', {
cause: error,
});
} finally {
erase(serialized);
}
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/remoteWorkerLeaseControlHttpsClient.ts
+103
View File
@@ -0,0 +1,103 @@
// Remote Execution transport owns fenced lease-control exchange.
import {
MAX_REMOTE_WORKER_LEASE_CONTROL_REQUEST_BYTES,
MAX_REMOTE_WORKER_LEASE_CONTROL_RESPONSE_BYTES,
createRemoteWorkerLeaseControlRequestBody,
parseRemoteWorkerLeaseControlResponse,
type RemoteWorkerLeaseControlCommand,
type RemoteWorkerLeaseControlResult,
} from '@qinglong/runtime-core/remote-worker-lease-control';
import {
WorkerIngressHttpsClient,
WorkerIngressHttpsClientError,
} from './workerIngressHttpsClient';
export interface WorkerRemoteLeaseControlClient {
control(
command: RemoteWorkerLeaseControlCommand,
): Promise<Readonly<RemoteWorkerLeaseControlResult>>;
}
export interface WorkerRemoteLeaseControlHttpsClientOptions {
readonly client: WorkerIngressHttpsClient;
}
export class WorkerRemoteLeaseControlHttpsError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'request_invalid'
| 'transport_unavailable'
| 'response_invalid',
options?: ErrorOptions,
) {
super(`Worker remote lease control failed: ${reason}`, options);
this.name = 'WorkerRemoteLeaseControlHttpsError';
}
}
function path(command: RemoteWorkerLeaseControlCommand): string {
return '/api/v3/worker-ingress/workers/' + command.workerId +
'/sessions/' + command.workerSessionId + '/lease-control';
}
export class WorkerRemoteLeaseControlHttpsClient
implements WorkerRemoteLeaseControlClient {
private readonly client: WorkerIngressHttpsClient;
constructor(options: WorkerRemoteLeaseControlHttpsClientOptions) {
if (!options || !(options.client instanceof WorkerIngressHttpsClient)) {
throw new WorkerRemoteLeaseControlHttpsError('invalid_configuration');
}
this.client = options.client;
}
async control(
command: RemoteWorkerLeaseControlCommand,
): Promise<Readonly<RemoteWorkerLeaseControlResult>> {
let body;
try {
body = createRemoteWorkerLeaseControlRequestBody(command);
} catch (error) {
throw new WorkerRemoteLeaseControlHttpsError(
'request_invalid', { cause: error },
);
}
let serialized: Uint8Array;
try {
serialized = await this.client.postJson({
path: path(command),
body,
maximumRequestBytes: MAX_REMOTE_WORKER_LEASE_CONTROL_REQUEST_BYTES,
maximumResponseBytes: MAX_REMOTE_WORKER_LEASE_CONTROL_RESPONSE_BYTES,
});
} catch (error) {
if (error instanceof WorkerIngressHttpsClientError) {
throw new WorkerRemoteLeaseControlHttpsError(
'transport_unavailable', { cause: error },
);
}
throw error;
}
try {
const result = parseRemoteWorkerLeaseControlResponse(serialized);
if (
result.projectId !== command.projectId ||
result.runId !== command.runId ||
result.attemptId !== command.attemptId ||
result.offerId !== command.offerId ||
result.leaseGeneration !== command.leaseGeneration ||
(result.status !== 'terminal' &&
result.leaseVersion !== command.expectedLeaseVersion + 1)
) throw new TypeError('lease control response authority mismatch');
return result;
} catch (error) {
throw new WorkerRemoteLeaseControlHttpsError(
'response_invalid', { cause: error },
);
} finally {
Buffer.from(serialized.buffer, serialized.byteOffset, serialized.byteLength)
.fill(0);
}
}
}
packages/ql3-worker-runtime/src/remote-execution/transport/workerIngressHttpsClient.ts
+641
View File
@@ -0,0 +1,641 @@
// Remote Execution transport owns the shared bounded TLS 1.3 Worker Ingress client.
import { Agent, request as nodeHttpsRequest } from 'node:https';
import type { RequestOptions } from 'node:https';
import type { ClientRequest, IncomingMessage } from 'node:http';
import { createHash } from 'node:crypto';
import { once } from 'node:events';
import { isIP } from 'node:net';
import { REMOTE_WORKER_ARTIFACT_CONTENT_TYPE } from '@qinglong/runtime-core/remote-worker-completion';
const AUTHORIZATION =
/^Worker ql3w_([A-Za-z0-9][A-Za-z0-9._:-]{0,63})_([A-Za-z0-9_-]{43})$/;
const WORKER_INGRESS_JSON_PATH =
/^\/api\/v3\/worker-ingress\/workers\/[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\/sessions\/[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\/(register|heartbeat|transition|offers|starting|running|start-failure|secrets|completion|lease-control)$/;
const WORKER_INGRESS_ARTIFACT_PATH =
/^\/api\/v3\/worker-ingress\/workers\/[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\/sessions\/[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\/artifacts$/;
const MAX_TLS_MATERIAL_BYTES = 1024 * 1024;
const MAX_REQUEST_BYTES = 4096;
const HARD_MAX_REQUEST_BYTES = 64 * 1024;
const HARD_MAX_STREAM_REQUEST_BYTES = 64 * 1024 * 1024 + 4 * 1024 + 4;
const MAX_RESPONSE_BYTES = 128 * 1024;
const CREDENTIAL_POOL_KEY = Symbol('qinglong.worker-ingress-credential-pool-key');
export const WORKER_INGRESS_ARTIFACT_CONTENT_TYPE =
REMOTE_WORKER_ARTIFACT_CONTENT_TYPE;
export interface WorkerIngressHttpsCredentials {
readonly authorization: string;
readonly certificateChainPem: string | Buffer;
readonly privateKeyPem: string | Buffer;
readonly trustAnchors: readonly (string | Buffer)[];
/** Erases provider-owned transient material after the client copies it. */
readonly dispose?: () => void;
}
export interface WorkerIngressHttpsCredentialProvider {
load(signal?: AbortSignal): Promise<WorkerIngressHttpsCredentials>;
}
export type WorkerIngressHttpsRequestFactory = (
options: RequestOptions,
callback: (response: IncomingMessage) => void,
) => ClientRequest;
export interface WorkerIngressHttpsClientOptions {
readonly origin: string | URL;
readonly credentials: WorkerIngressHttpsCredentialProvider;
readonly requestTimeoutMs?: number;
readonly agent?: Agent;
/** Injectable only for deterministic transport contract tests. */
readonly requestFactory?: WorkerIngressHttpsRequestFactory;
}
export interface WorkerIngressHttpsPostRequest {
readonly path: string;
readonly body: unknown;
readonly maximumResponseBytes: number;
/** Defaults to 4 KiB. Larger budgets are opt-in and capped at 64 KiB. */
readonly maximumRequestBytes?: number;
readonly signal?: AbortSignal;
}
export interface WorkerIngressHttpsStreamRequest {
readonly path: string;
readonly body: AsyncIterable<Uint8Array>;
readonly byteLength: number;
readonly maximumResponseBytes: number;
readonly signal?: AbortSignal;
}
export class WorkerIngressHttpsClientError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'credentials_unavailable'
| 'request_rejected'
| 'response_rejected'
| 'response_too_large'
| 'closed',
readonly httpStatus?: number,
) {
super(`Worker ingress HTTPS client failed: ${reason}`);
this.name = 'WorkerIngressHttpsClientError';
}
}
function boundedMaterial(value: string | Buffer): Buffer {
const bytes = Buffer.isBuffer(value)
? Buffer.from(value)
: Buffer.from(value, 'utf8');
if (bytes.byteLength < 1 || bytes.byteLength > MAX_TLS_MATERIAL_BYTES) {
bytes.fill(0);
throw new WorkerIngressHttpsClientError('credentials_unavailable');
}
return bytes;
}
function normalizeOrigin(value: string | URL): URL {
let origin: URL;
try {
origin = new URL(value);
} catch {
throw new WorkerIngressHttpsClientError('invalid_configuration');
}
if (
origin.protocol !== 'https:' ||
origin.username !== '' ||
origin.password !== '' ||
origin.pathname !== '/' ||
origin.search !== '' ||
origin.hash !== ''
) {
throw new WorkerIngressHttpsClientError('invalid_configuration');
}
return origin;
}
interface WorkerIngressHttpsCredentialMaterial {
readonly authorization: string;
readonly certificate: Buffer;
readonly privateKey: Buffer;
readonly trustAnchors: readonly Buffer[];
readonly poolKey: string;
}
interface WorkerIngressHttpsAgentRequestOptions extends RequestOptions {
readonly [CREDENTIAL_POOL_KEY]?: string;
}
class WorkerIngressHttpsAgent extends Agent {
override getName(options: RequestOptions = {}): string {
const poolKey = (options as WorkerIngressHttpsAgentRequestOptions)[
CREDENTIAL_POOL_KEY
];
if (poolKey === undefined) return super.getName(options);
// Node recomputes the HTTPS pool name when a socket becomes free. The
// request-local TLS Buffers have already been erased by then, so their
// mutable contents cannot safely participate in that name.
return `${super.getName({
...options,
ca: undefined,
cert: undefined,
key: undefined,
})}:qinglong:${poolKey}`;
}
}
function credentialPoolKey(
certificate: Buffer,
privateKey: Buffer,
trustAnchors: readonly Buffer[],
): string {
const hash = createHash('sha256');
for (const value of [certificate, privateKey, ...trustAnchors]) {
const length = Buffer.allocUnsafe(4);
length.writeUInt32BE(value.byteLength);
hash.update(length);
hash.update(value);
length.fill(0);
}
return hash.digest('base64url');
}
async function loadCredentialMaterial(
provider: WorkerIngressHttpsCredentialProvider,
signal?: AbortSignal,
): Promise<WorkerIngressHttpsCredentialMaterial> {
let loaded: WorkerIngressHttpsCredentials;
try {
loaded = await provider.load(signal);
} catch {
if (signal?.aborted) {
throw signal.reason ??
new WorkerIngressHttpsClientError('request_rejected');
}
throw new WorkerIngressHttpsClientError('credentials_unavailable');
}
let certificate: Buffer | undefined;
let privateKey: Buffer | undefined;
const trustAnchors: Buffer[] = [];
let failure: unknown;
try {
if (
!loaded ||
!AUTHORIZATION.test(loaded.authorization) ||
!Array.isArray(loaded.trustAnchors) ||
loaded.trustAnchors.length < 1 ||
loaded.trustAnchors.length > 8 ||
(loaded.dispose !== undefined && typeof loaded.dispose !== 'function')
) {
throw new WorkerIngressHttpsClientError('credentials_unavailable');
}
certificate = boundedMaterial(loaded.certificateChainPem);
privateKey = boundedMaterial(loaded.privateKeyPem);
for (const anchor of loaded.trustAnchors) {
trustAnchors.push(boundedMaterial(anchor));
}
} catch (error) {
failure = error;
}
try {
loaded?.dispose?.();
} catch {
failure ??= new WorkerIngressHttpsClientError('credentials_unavailable');
}
if (failure !== undefined || !certificate || !privateKey) {
certificate?.fill(0);
privateKey?.fill(0);
trustAnchors.forEach((value) => value.fill(0));
if (failure instanceof WorkerIngressHttpsClientError) throw failure;
throw new WorkerIngressHttpsClientError('credentials_unavailable');
}
return {
authorization: loaded.authorization,
certificate,
privateKey,
trustAnchors: Object.freeze(trustAnchors),
poolKey: credentialPoolKey(certificate, privateKey, trustAnchors),
};
}
function eraseCredentialMaterial(
material: WorkerIngressHttpsCredentialMaterial,
): void {
material.certificate.fill(0);
material.privateKey.fill(0);
material.trustAnchors.forEach((value) => value.fill(0));
}
export class WorkerIngressHttpsClient {
private readonly origin: URL;
private readonly credentials: WorkerIngressHttpsCredentialProvider;
private readonly requestTimeoutMs: number;
private readonly requestFactory: WorkerIngressHttpsRequestFactory;
private readonly agent: Agent;
private readonly ownsAgent: boolean;
private closed = false;
constructor(options: WorkerIngressHttpsClientOptions) {
if (
!options ||
typeof options.credentials?.load !== 'function' ||
(options.requestFactory !== undefined &&
typeof options.requestFactory !== 'function')
) {
throw new WorkerIngressHttpsClientError('invalid_configuration');
}
const requestTimeoutMs = options.requestTimeoutMs ?? 15_000;
if (
!Number.isSafeInteger(requestTimeoutMs) ||
requestTimeoutMs < 1_000 ||
requestTimeoutMs > 60_000
) {
throw new WorkerIngressHttpsClientError('invalid_configuration');
}
this.origin = normalizeOrigin(options.origin);
this.credentials = options.credentials;
this.requestTimeoutMs = requestTimeoutMs;
this.requestFactory = options.requestFactory ?? nodeHttpsRequest;
this.ownsAgent = options.agent === undefined;
this.agent = options.agent ?? new WorkerIngressHttpsAgent({
keepAlive: true,
maxSockets: 1,
maxFreeSockets: 1,
timeout: requestTimeoutMs,
});
}
async postJson(request: WorkerIngressHttpsPostRequest): Promise<Uint8Array> {
if (this.closed) throw new WorkerIngressHttpsClientError('closed');
if (
!request ||
typeof request.path !== 'string' ||
!WORKER_INGRESS_JSON_PATH.test(request.path) ||
!Number.isSafeInteger(request.maximumResponseBytes) ||
request.maximumResponseBytes < 2 ||
request.maximumResponseBytes > MAX_RESPONSE_BYTES ||
(request.maximumRequestBytes !== undefined &&
(!Number.isSafeInteger(request.maximumRequestBytes) ||
request.maximumRequestBytes < 2 ||
request.maximumRequestBytes > HARD_MAX_REQUEST_BYTES))
) {
throw new WorkerIngressHttpsClientError('request_rejected');
}
if (request.signal?.aborted) {
throw request.signal.reason ??
new WorkerIngressHttpsClientError('request_rejected');
}
const material = await loadCredentialMaterial(
this.credentials,
request.signal,
);
let body: Buffer;
try {
body = Buffer.from(JSON.stringify(request.body), 'utf8');
} catch {
eraseCredentialMaterial(material);
throw new WorkerIngressHttpsClientError('request_rejected');
}
const maximumRequestBytes = request.maximumRequestBytes ?? MAX_REQUEST_BYTES;
if (body.byteLength < 2 || body.byteLength > maximumRequestBytes) {
eraseCredentialMaterial(material);
body.fill(0);
throw new WorkerIngressHttpsClientError('request_rejected');
}
try {
return await this.perform(
request.path,
body,
request.maximumResponseBytes,
material.authorization,
material.certificate,
material.privateKey,
material.trustAnchors,
material.poolKey,
request.signal,
);
} finally {
eraseCredentialMaterial(material);
body.fill(0);
}
}
async postStream(
request: WorkerIngressHttpsStreamRequest,
): Promise<Uint8Array> {
if (this.closed) throw new WorkerIngressHttpsClientError('closed');
if (
!request ||
typeof request.path !== 'string' ||
!WORKER_INGRESS_ARTIFACT_PATH.test(request.path) ||
!request.body ||
typeof request.body[Symbol.asyncIterator] !== 'function' ||
!Number.isSafeInteger(request.byteLength) ||
request.byteLength < 1 ||
request.byteLength > HARD_MAX_STREAM_REQUEST_BYTES ||
!Number.isSafeInteger(request.maximumResponseBytes) ||
request.maximumResponseBytes < 2 ||
request.maximumResponseBytes > MAX_RESPONSE_BYTES
) {
throw new WorkerIngressHttpsClientError('request_rejected');
}
if (request.signal?.aborted) {
throw request.signal.reason ??
new WorkerIngressHttpsClientError('request_rejected');
}
const material = await loadCredentialMaterial(
this.credentials,
request.signal,
);
try {
return await this.performStream(
request.path,
request.body,
request.byteLength,
request.maximumResponseBytes,
material.authorization,
material.certificate,
material.privateKey,
material.trustAnchors,
material.poolKey,
request.signal,
);
} finally {
eraseCredentialMaterial(material);
}
}
close(): void {
if (this.closed) return;
this.closed = true;
if (this.ownsAgent) this.agent.destroy();
}
private perform(
path: string,
body: Buffer,
maximumResponseBytes: number,
authorization: string,
certificate: Buffer,
privateKey: Buffer,
trustAnchors: readonly Buffer[],
poolKey: string,
signal?: AbortSignal,
): Promise<Uint8Array> {
return new Promise((resolve, reject) => {
let settled = false;
const settle = (error?: unknown, bytes?: Buffer): void => {
if (settled) return;
settled = true;
signal?.removeEventListener('abort', abort);
if (error) reject(error);
else resolve(bytes!);
};
let clientRequest: ClientRequest;
const abort = (): void => {
clientRequest.destroy(
signal?.reason instanceof Error
? signal.reason
: new WorkerIngressHttpsClientError('request_rejected'),
);
};
try {
clientRequest = this.requestFactory({
protocol: 'https:',
hostname: this.origin.hostname,
port: this.origin.port || 443,
...(isIP(this.origin.hostname) === 0
? { servername: this.origin.hostname }
: {}),
method: 'POST',
path,
agent: this.agent,
minVersion: 'TLSv1.3',
rejectUnauthorized: true,
cert: certificate,
key: privateKey,
ca: [...trustAnchors],
[CREDENTIAL_POOL_KEY]: poolKey,
headers: {
accept: 'application/json',
authorization,
'content-type': 'application/json',
'content-length': String(body.byteLength),
},
} as WorkerIngressHttpsAgentRequestOptions, (response) => {
const contentType = response.headers['content-type'];
const contentEncoding = response.headers['content-encoding'];
const contentLength = response.headers['content-length'];
if (
response.statusCode !== 200 ||
typeof contentType !== 'string' ||
!/^application\/json(?:\s*;|$)/i.test(contentType) ||
(contentEncoding !== undefined && contentEncoding !== 'identity') ||
(contentLength !== undefined &&
(!/^\d+$/.test(contentLength) ||
Number(contentLength) > maximumResponseBytes))
) {
response.resume();
settle(new WorkerIngressHttpsClientError(
'response_rejected', response.statusCode,
));
return;
}
const chunks: Buffer[] = [];
let total = 0;
response.on('data', (chunk: Buffer | string) => {
if (settled) return;
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
total += bytes.byteLength;
if (total > maximumResponseBytes) {
chunks.forEach((value) => value.fill(0));
response.destroy();
settle(new WorkerIngressHttpsClientError('response_too_large'));
return;
}
chunks.push(Buffer.from(bytes));
});
response.once('end', () => {
if (settled) return;
const result = Buffer.concat(chunks, total);
chunks.forEach((value) => value.fill(0));
if (result.byteLength < 2) {
result.fill(0);
settle(new WorkerIngressHttpsClientError('response_rejected'));
return;
}
settle(undefined, result);
});
response.once('error', () => {
chunks.forEach((value) => value.fill(0));
settle(new WorkerIngressHttpsClientError(
'response_rejected', response.statusCode,
));
});
});
} catch {
settle(new WorkerIngressHttpsClientError('request_rejected'));
return;
}
clientRequest.once('error', (error) => {
if (signal?.aborted) settle(signal.reason ?? error);
else settle(new WorkerIngressHttpsClientError('request_rejected'));
});
clientRequest.setTimeout(this.requestTimeoutMs, () => {
clientRequest.destroy(
new WorkerIngressHttpsClientError('request_rejected'),
);
});
signal?.addEventListener('abort', abort, { once: true });
clientRequest.end(body);
});
}
private performStream(
path: string,
body: AsyncIterable<Uint8Array>,
byteLength: number,
maximumResponseBytes: number,
authorization: string,
certificate: Buffer,
privateKey: Buffer,
trustAnchors: readonly Buffer[],
poolKey: string,
signal?: AbortSignal,
): Promise<Uint8Array> {
return new Promise((resolve, reject) => {
let settled = false;
const settle = (error?: unknown, bytes?: Buffer): void => {
if (settled) return;
settled = true;
signal?.removeEventListener('abort', abort);
if (error) reject(error);
else resolve(bytes!);
};
let clientRequest: ClientRequest;
const abort = (): void => {
clientRequest.destroy(
signal?.reason instanceof Error
? signal.reason
: new WorkerIngressHttpsClientError('request_rejected'),
);
};
try {
clientRequest = this.requestFactory({
protocol: 'https:',
hostname: this.origin.hostname,
port: this.origin.port || 443,
...(isIP(this.origin.hostname) === 0
? { servername: this.origin.hostname }
: {}),
method: 'POST',
path,
agent: this.agent,
minVersion: 'TLSv1.3',
rejectUnauthorized: true,
cert: certificate,
key: privateKey,
ca: [...trustAnchors],
[CREDENTIAL_POOL_KEY]: poolKey,
headers: {
accept: 'application/json',
authorization,
'content-type': WORKER_INGRESS_ARTIFACT_CONTENT_TYPE,
'content-length': String(byteLength),
},
} as WorkerIngressHttpsAgentRequestOptions, (response) => {
const contentType = response.headers['content-type'];
const contentEncoding = response.headers['content-encoding'];
const contentLength = response.headers['content-length'];
if (
response.statusCode !== 200 ||
typeof contentType !== 'string' ||
!/^application\/json(?:\s*;|$)/i.test(contentType) ||
(contentEncoding !== undefined && contentEncoding !== 'identity') ||
(contentLength !== undefined &&
(!/^\d+$/.test(contentLength) ||
Number(contentLength) > maximumResponseBytes))
) {
response.resume();
clientRequest.destroy();
settle(new WorkerIngressHttpsClientError('response_rejected'));
return;
}
const chunks: Buffer[] = [];
let total = 0;
response.on('data', (chunk: Buffer | string) => {
if (settled) return;
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
total += bytes.byteLength;
if (total > maximumResponseBytes) {
chunks.forEach((value) => value.fill(0));
response.destroy();
settle(new WorkerIngressHttpsClientError('response_too_large'));
return;
}
chunks.push(Buffer.from(bytes));
});
response.once('end', () => {
if (settled) return;
const result = Buffer.concat(chunks, total);
chunks.forEach((value) => value.fill(0));
if (result.byteLength < 2) {
result.fill(0);
settle(new WorkerIngressHttpsClientError('response_rejected'));
return;
}
settle(undefined, result);
});
response.once('error', () => {
chunks.forEach((value) => value.fill(0));
settle(new WorkerIngressHttpsClientError('response_rejected'));
});
});
} catch {
settle(new WorkerIngressHttpsClientError('request_rejected'));
return;
}
clientRequest.once('error', (error) => {
if (signal?.aborted) settle(signal.reason ?? error);
else settle(new WorkerIngressHttpsClientError('request_rejected'));
});
clientRequest.setTimeout(this.requestTimeoutMs, () => {
clientRequest.destroy(
new WorkerIngressHttpsClientError('request_rejected'),
);
});
signal?.addEventListener('abort', abort, { once: true });
void (async () => {
let total = 0;
for await (const chunk of body) {
if (settled) return;
if (signal?.aborted) throw signal.reason;
if (!(chunk instanceof Uint8Array)) {
throw new WorkerIngressHttpsClientError('request_rejected');
}
total += chunk.byteLength;
if (total > byteLength) {
throw new WorkerIngressHttpsClientError('request_rejected');
}
if (!clientRequest.write(chunk)) {
await once(
clientRequest,
'drain',
signal ? { signal } : undefined,
);
}
}
if (total !== byteLength) {
throw new WorkerIngressHttpsClientError('request_rejected');
}
clientRequest.end();
})().catch((error: unknown) => {
clientRequest.destroy(
error instanceof Error
? error
: new WorkerIngressHttpsClientError('request_rejected'),
);
});
});
}
}
packages/ql3-worker-runtime/src/session/workerExecutionCapacityOracle.ts
+151
View File
@@ -0,0 +1,151 @@
// Session ownership: derive advertised capacity from the durable execution journal.
import { assertWorkerConcurrency } from '@qinglong/runtime-core/worker-session';
import type {
WorkerRemoteExecutionInboxPage,
WorkerRemoteExecutionInboxRecord,
} from '../remote-execution/executionInbox';
import {
MAX_WORKER_REMOTE_OFFER_INBOX_ENTRIES,
type WorkerRemoteOfferClaimRecord,
} from '../remote-execution/remoteOfferDelivery';
const SETTLED_STATES = new Set<WorkerRemoteExecutionInboxRecord['state']>([
'start_failure_acknowledged',
'completion_acknowledged',
]);
export interface WorkerExecutionCapacityJournal {
listOffers(options: Readonly<{
afterOfferId?: string;
limit?: number;
}>): Promise<WorkerRemoteExecutionInboxPage>;
readPendingClaim(): Promise<WorkerRemoteOfferClaimRecord | undefined>;
}
export interface WorkerExecutionCapacityOracleOptions {
readonly journal: WorkerExecutionCapacityJournal;
readonly maxConcurrentRuns: number;
}
export type WorkerExecutionCapacityMode =
| 'reconciling'
| 'registering'
| 'active'
| 'draining'
| 'recovery_required'
| 'offline';
export class WorkerExecutionCapacityOracleError extends Error {
constructor(readonly reason: 'invalid_configuration' | 'invalid_transition') {
super(`Worker execution capacity oracle failed: ${reason}`);
this.name = 'WorkerExecutionCapacityOracleError';
}
}
/**
* Derives advertised capacity exclusively from the owned execution journal.
* It has no timer and never treats a deployment-supplied slot count as truth.
*/
export class WorkerExecutionCapacityOracle {
private readonly journal: WorkerExecutionCapacityJournal;
private readonly maxConcurrentRuns: number;
private modeValue: WorkerExecutionCapacityMode = 'reconciling';
private operation?: Promise<number>;
constructor(options: WorkerExecutionCapacityOracleOptions) {
if (
!options ||
typeof options.journal?.listOffers !== 'function' ||
typeof options.journal?.readPendingClaim !== 'function'
) throw new WorkerExecutionCapacityOracleError('invalid_configuration');
try {
assertWorkerConcurrency(options.maxConcurrentRuns, 0);
} catch {
throw new WorkerExecutionCapacityOracleError('invalid_configuration');
}
this.journal = options.journal;
this.maxConcurrentRuns = options.maxConcurrentRuns;
}
mode(): WorkerExecutionCapacityMode {
return this.modeValue;
}
prepareRegistration(): void {
this.transition('reconciling', 'registering');
}
activate(): void {
this.transition('registering', 'active');
}
beginDrain(): void {
if (
this.modeValue === 'draining' ||
this.modeValue === 'offline'
) return;
this.transition('active', 'draining');
}
failClosed(): void {
this.modeValue = 'recovery_required';
}
offline(): void {
if (this.modeValue === 'offline') return;
this.transition('draining', 'offline');
}
availableSlots(): Promise<number> {
if (
this.modeValue !== 'registering' &&
this.modeValue !== 'active'
) return Promise.resolve(0);
if (this.operation) return this.operation;
const operation = this.readAvailableSlots().finally(() => {
if (this.operation === operation) this.operation = undefined;
});
this.operation = operation;
return operation;
}
private async readAvailableSlots(): Promise<number> {
let afterOfferId: string | undefined;
let observed = 0;
const active = new Set<string>();
do {
const page = await this.journal.listOffers({
...(afterOfferId === undefined ? {} : { afterOfferId }),
limit: 64,
});
observed += page.records.length;
if (observed > MAX_WORKER_REMOTE_OFFER_INBOX_ENTRIES) {
this.failClosed();
return 0;
}
for (const record of page.records) {
if (record.state === 'recovery_required') {
this.failClosed();
return 0;
}
if (!SETTLED_STATES.has(record.state)) {
active.add(record.offer.offerId);
}
}
afterOfferId = page.nextAfterOfferId;
} while (afterOfferId !== undefined);
const pending = await this.journal.readPendingClaim();
if (pending !== undefined) active.add(pending.offerId);
return Math.max(0, this.maxConcurrentRuns - active.size);
}
private transition(
expected: WorkerExecutionCapacityMode,
next: WorkerExecutionCapacityMode,
): void {
if (this.modeValue !== expected) {
throw new WorkerExecutionCapacityOracleError('invalid_transition');
}
this.modeValue = next;
}
}
packages/ql3-worker-runtime/src/session/workerSessionCoordinator.ts
+359
View File
@@ -0,0 +1,359 @@
// Session ownership: coordinate registration, heartbeat, drain, and lease fencing.
import { randomBytes } from 'node:crypto';
import {
assertWorkerConcurrency,
assertWorkerId,
assertWorkerSessionLeaseDuration,
type WorkerSessionStatus,
} from '@qinglong/runtime-core/worker-session';
import {
canonicalRemoteWorkerCapabilities,
type RemoteWorkerCapabilities,
} from '@qinglong/runtime-core/remote-dispatch';
import type { WorkerRemoteExecutionSession } from '../remote-execution/executionInboxProcessor';
import {
WorkerSessionHttpsClient,
WorkerSessionHttpsClientError,
} from './workerSessionHttpsClient';
export const MIN_WORKER_PRODUCT_HEARTBEAT_INTERVAL_MS = 5_000;
export const MAX_WORKER_PRODUCT_HEARTBEAT_INTERVAL_MS = 5 * 60_000;
export interface WorkerSessionCoordinatorOptions {
readonly client: WorkerSessionHttpsClient;
readonly workerId: string;
readonly capabilities: RemoteWorkerCapabilities;
readonly maxConcurrentRuns: number;
readonly availableSlots: () => number | Promise<number>;
readonly leaseDurationMs?: number;
readonly heartbeatIntervalMs?: number;
readonly now?: () => number;
readonly createSessionId?: (nowMs: number) => string;
}
export interface WorkerSessionCoordinatorRecord
extends WorkerRemoteExecutionSession {
readonly version: number;
readonly nextHeartbeatAtMs: number;
}
export type WorkerSessionCoordinatorTickResult = Readonly<
| { status: 'inactive' | 'not_due' | 'lease_expired' }
| { status: 'heartbeat'; session: WorkerSessionCoordinatorRecord }
>;
export class WorkerSessionCoordinatorError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'already_registered'
| 'inactive'
| 'lease_expired'
| 'invalid_capacity'
| 'response_invalid',
options?: ErrorOptions,
) {
super(`Worker Session coordinator failed: ${reason}`, options);
this.name = 'WorkerSessionCoordinatorError';
}
}
function safeNow(provider: () => number): number {
const value = provider();
if (!Number.isSafeInteger(value) || value < 0) {
throw new WorkerSessionCoordinatorError('invalid_configuration');
}
return value;
}
function uuidV7(nowMs: number): string {
if (!Number.isSafeInteger(nowMs) || nowMs < 0 || nowMs > 0xffffffffffff) {
throw new WorkerSessionCoordinatorError('invalid_configuration');
}
const bytes = randomBytes(16);
let timestamp = nowMs;
for (let index = 5; index >= 0; index -= 1) {
bytes[index] = timestamp % 256;
timestamp = Math.floor(timestamp / 256);
}
bytes[6] = (bytes[6]! & 0x0f) | 0x70;
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
const hex = bytes.toString('hex');
bytes.fill(0);
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-` +
`${hex.slice(16, 20)}-${hex.slice(20)}`;
}
function executionStatus(
status: WorkerSessionStatus,
): WorkerRemoteExecutionSession['status'] {
if (status === 'online') return 'available';
return status;
}
export class WorkerSessionCoordinator {
private readonly client: WorkerSessionHttpsClient;
private readonly workerId: string;
private readonly capabilitiesJson: string;
private readonly capabilitiesHash: string;
private readonly maxConcurrentRuns: number;
private readonly availableSlotsProvider: () => number | Promise<number>;
private readonly leaseDurationMs: number;
private readonly heartbeatIntervalMs: number;
private readonly nowProvider: () => number;
private readonly createSessionId: (nowMs: number) => string;
private session?: WorkerSessionCoordinatorRecord;
private blocked = false;
private operation?: Promise<unknown>;
constructor(options: WorkerSessionCoordinatorOptions) {
if (
!options ||
!(options.client instanceof WorkerSessionHttpsClient) ||
typeof options.availableSlots !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function') ||
(options.createSessionId !== undefined &&
typeof options.createSessionId !== 'function')
) throw new WorkerSessionCoordinatorError('invalid_configuration');
try {
assertWorkerId(options.workerId);
assertWorkerConcurrency(options.maxConcurrentRuns, 0);
} catch (error) {
throw new WorkerSessionCoordinatorError(
'invalid_configuration', { cause: error },
);
}
const leaseDurationMs = options.leaseDurationMs ?? 45_000;
const heartbeatIntervalMs = options.heartbeatIntervalMs ?? 10_000;
try {
assertWorkerSessionLeaseDuration(leaseDurationMs);
} catch (error) {
throw new WorkerSessionCoordinatorError(
'invalid_configuration', { cause: error },
);
}
if (
!Number.isSafeInteger(heartbeatIntervalMs) ||
heartbeatIntervalMs < MIN_WORKER_PRODUCT_HEARTBEAT_INTERVAL_MS ||
heartbeatIntervalMs > MAX_WORKER_PRODUCT_HEARTBEAT_INTERVAL_MS ||
heartbeatIntervalMs * 2 > leaseDurationMs
) throw new WorkerSessionCoordinatorError('invalid_configuration');
let canonical;
try {
canonical = canonicalRemoteWorkerCapabilities(options.capabilities);
} catch (error) {
throw new WorkerSessionCoordinatorError(
'invalid_configuration', { cause: error },
);
}
this.client = options.client;
this.workerId = options.workerId;
this.capabilitiesJson = canonical.json;
this.capabilitiesHash = canonical.hash;
this.maxConcurrentRuns = options.maxConcurrentRuns;
this.availableSlotsProvider = options.availableSlots;
this.leaseDurationMs = leaseDurationMs;
this.heartbeatIntervalMs = heartbeatIntervalMs;
this.nowProvider = options.now ?? Date.now;
this.createSessionId = options.createSessionId ?? uuidV7;
}
current(): WorkerRemoteExecutionSession | undefined {
const session = this.session;
if (
!session ||
this.blocked ||
session.leaseExpiresAtMs <= safeNow(this.nowProvider)
) {
return undefined;
}
return Object.freeze({
workerId: session.workerId,
sessionId: session.sessionId,
generation: session.generation,
status: session.status,
leaseExpiresAtMs: session.leaseExpiresAtMs,
});
}
currentRecord(): WorkerSessionCoordinatorRecord | undefined {
return this.session ? Object.freeze({ ...this.session }) : undefined;
}
/**
* Removes the Session from execution admission without destroying its
* durable identity. A later authenticated exchange is the only operation
* that can clear this transport fence.
*/
failClosed(): void {
this.blocked = true;
}
register(): Promise<WorkerSessionCoordinatorRecord> {
return this.serial(async () => {
if (this.session) {
throw new WorkerSessionCoordinatorError('already_registered');
}
const now = safeNow(this.nowProvider);
const sessionId = this.createSessionId(now);
const availableSlots = await this.readAvailableSlots();
const response = await this.exchange(() => this.client.register({
workerId: this.workerId,
sessionId,
capabilitiesJson: this.capabilitiesJson,
capabilitiesHash: this.capabilitiesHash,
maxConcurrentRuns: this.maxConcurrentRuns,
availableSlots,
leaseDurationMs: this.leaseDurationMs,
}));
if (response.leaseExpiresAtMs <= now) {
throw new WorkerSessionCoordinatorError('response_invalid');
}
return this.accept(response, now);
});
}
tick(): Promise<WorkerSessionCoordinatorTickResult> {
return this.serial(async () => {
const session = this.session;
if (!session) return Object.freeze({ status: 'inactive' as const });
const now = safeNow(this.nowProvider);
if (session.leaseExpiresAtMs <= now) {
return Object.freeze({ status: 'lease_expired' as const });
}
if (now < session.nextHeartbeatAtMs) {
return Object.freeze({ status: 'not_due' as const });
}
const availableSlots = session.status === 'available'
? await this.readAvailableSlots()
: 0;
const response = await this.exchange(() => this.client.heartbeat({
workerId: session.workerId,
sessionId: session.sessionId,
generation: session.generation,
expectedVersion: session.version,
availableSlots,
leaseDurationMs: this.leaseDurationMs,
}));
return Object.freeze({
status: 'heartbeat' as const,
session: this.accept(response, now),
});
});
}
beginDrain(): Promise<void> {
return this.serial(async () => {
const session = this.requireLiveSession();
if (session.status === 'draining') return;
if (session.status === 'offline') return;
const now = safeNow(this.nowProvider);
const response = await this.exchange(() => this.client.transition({
workerId: session.workerId,
sessionId: session.sessionId,
generation: session.generation,
expectedVersion: session.version,
status: 'draining',
}));
this.accept(response, now);
});
}
disconnect(): Promise<void> {
return this.serial(async () => {
const session = this.requireLiveSession();
if (session.status === 'offline') return;
if (session.status !== 'draining') {
throw new WorkerSessionCoordinatorError('inactive');
}
const now = safeNow(this.nowProvider);
const response = await this.exchange(() => this.client.transition({
workerId: session.workerId,
sessionId: session.sessionId,
generation: session.generation,
expectedVersion: session.version,
status: 'offline',
}));
this.accept(response, now);
});
}
private accept(
response: Readonly<{
workerId: string;
sessionId: string;
generation: number;
version: number;
status: WorkerSessionStatus;
leaseExpiresAtMs: number;
}>,
now: number,
): WorkerSessionCoordinatorRecord {
const previous = this.session;
if (
response.workerId !== this.workerId ||
(previous !== undefined &&
(response.sessionId !== previous.sessionId ||
response.generation !== previous.generation ||
response.version !== previous.version + 1)) ||
response.leaseExpiresAtMs < now
) throw new WorkerSessionCoordinatorError('response_invalid');
const record = Object.freeze({
workerId: response.workerId,
sessionId: response.sessionId,
generation: response.generation,
version: response.version,
status: executionStatus(response.status),
leaseExpiresAtMs: response.leaseExpiresAtMs,
nextHeartbeatAtMs: now + this.heartbeatIntervalMs,
});
this.session = record;
return record;
}
private requireLiveSession(): WorkerSessionCoordinatorRecord {
const session = this.session;
if (!session) throw new WorkerSessionCoordinatorError('inactive');
if (session.leaseExpiresAtMs <= safeNow(this.nowProvider)) {
throw new WorkerSessionCoordinatorError('lease_expired');
}
return session;
}
private async readAvailableSlots(): Promise<number> {
const availableSlots = await this.availableSlotsProvider();
try {
assertWorkerConcurrency(this.maxConcurrentRuns, availableSlots);
} catch (error) {
throw new WorkerSessionCoordinatorError(
'invalid_capacity', { cause: error },
);
}
return availableSlots;
}
private async exchange<T>(operation: () => Promise<T>): Promise<T> {
try {
const result = await operation();
this.blocked = false;
return result;
} catch (error) {
if (
error instanceof WorkerSessionHttpsClientError &&
(error.reason === 'credential_rejected' ||
error.reason === 'session_fenced')
) this.blocked = true;
throw error;
}
}
private serial<T>(operation: () => Promise<T>): Promise<T> {
const current = (this.operation ?? Promise.resolve())
.catch(() => undefined)
.then(operation);
this.operation = current;
return current.finally(() => {
if (this.operation === current) this.operation = undefined;
});
}
}
packages/ql3-worker-runtime/src/session/workerSessionHttpsClient.ts
+215
View File
@@ -0,0 +1,215 @@
// Session ownership: exchange bounded register, heartbeat, and transition messages.
import {
MAX_WORKER_SESSION_REGISTER_REQUEST_BYTES,
MAX_WORKER_SESSION_REQUEST_BYTES,
MAX_WORKER_SESSION_RESPONSE_BYTES,
createWorkerSessionHeartbeatRequestBody,
createWorkerSessionRegisterRequestBody,
createWorkerSessionTransitionRequestBody,
parseWorkerSessionHeartbeatResponseBody,
parseWorkerSessionRegisterResponseBody,
parseWorkerSessionTransitionResponseBody,
type WorkerSessionHeartbeatResponseBody,
type WorkerSessionRegisterResponseBody,
type WorkerSessionTransitionResponseBody,
} from '@qinglong/runtime-core/worker-session-transport';
import type {
HeartbeatWorkerSessionCommand,
RegisterWorkerSessionCommand,
TransitionWorkerSessionCommand,
} from '@qinglong/runtime-core/worker-session';
import {
WorkerIngressHttpsClient,
WorkerIngressHttpsClientError,
} from '../remote-execution/transport/workerIngressHttpsClient';
export interface WorkerSessionHttpsClientOptions {
readonly client: WorkerIngressHttpsClient;
}
export class WorkerSessionHttpsClientError extends Error {
constructor(
readonly reason:
| 'invalid_configuration'
| 'request_invalid'
| 'credential_rejected'
| 'session_fenced'
| 'transport_unavailable'
| 'response_invalid',
options?: ErrorOptions,
) {
super(`Worker Session HTTPS client failed: ${reason}`, options);
this.name = 'WorkerSessionHttpsClientError';
}
}
function path(
command: Readonly<{ workerId: string; sessionId: string }>,
operation: 'register' | 'heartbeat' | 'transition',
): string {
return '/api/v3/worker-ingress/workers/' + command.workerId +
'/sessions/' + command.sessionId + '/' + operation;
}
function parseJson(bytes: Uint8Array): unknown {
return JSON.parse(
Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString('utf8'),
) as unknown;
}
export class WorkerSessionHttpsClient {
private readonly client: WorkerIngressHttpsClient;
constructor(options: WorkerSessionHttpsClientOptions) {
if (!options || !(options.client instanceof WorkerIngressHttpsClient)) {
throw new WorkerSessionHttpsClientError('invalid_configuration');
}
this.client = options.client;
}
async register(
command: RegisterWorkerSessionCommand,
signal?: AbortSignal,
): Promise<WorkerSessionRegisterResponseBody> {
let body;
try {
body = createWorkerSessionRegisterRequestBody(command);
} catch (error) {
throw new WorkerSessionHttpsClientError(
'request_invalid', { cause: error },
);
}
const response = await this.exchange(
path(command, 'register'),
body,
MAX_WORKER_SESSION_REGISTER_REQUEST_BYTES,
signal,
);
try {
const result = parseWorkerSessionRegisterResponseBody(parseJson(response));
if (
result.workerId !== command.workerId ||
result.sessionId !== command.sessionId ||
result.status !== 'online'
) throw new TypeError('register response authority mismatch');
return result;
} catch (error) {
throw new WorkerSessionHttpsClientError(
'response_invalid', { cause: error },
);
} finally {
Buffer.from(response.buffer, response.byteOffset, response.byteLength)
.fill(0);
}
}
async heartbeat(
command: HeartbeatWorkerSessionCommand,
signal?: AbortSignal,
): Promise<WorkerSessionHeartbeatResponseBody> {
let body;
try {
body = createWorkerSessionHeartbeatRequestBody(command);
} catch (error) {
throw new WorkerSessionHttpsClientError(
'request_invalid', { cause: error },
);
}
const response = await this.exchange(
path(command, 'heartbeat'),
body,
MAX_WORKER_SESSION_REQUEST_BYTES,
signal,
);
try {
const result = parseWorkerSessionHeartbeatResponseBody(parseJson(response));
if (
result.workerId !== command.workerId ||
result.sessionId !== command.sessionId ||
result.generation !== command.generation ||
result.version !== command.expectedVersion + 1 ||
result.status === 'offline'
) throw new TypeError('heartbeat response authority mismatch');
return result;
} catch (error) {
throw new WorkerSessionHttpsClientError(
'response_invalid', { cause: error },
);
} finally {
Buffer.from(response.buffer, response.byteOffset, response.byteLength)
.fill(0);
}
}
async transition(
command: TransitionWorkerSessionCommand,
signal?: AbortSignal,
): Promise<WorkerSessionTransitionResponseBody> {
let body;
try {
body = createWorkerSessionTransitionRequestBody(command);
} catch (error) {
throw new WorkerSessionHttpsClientError(
'request_invalid', { cause: error },
);
}
const response = await this.exchange(
path(command, 'transition'),
body,
MAX_WORKER_SESSION_REQUEST_BYTES,
signal,
);
try {
const result = parseWorkerSessionTransitionResponseBody(parseJson(response));
if (
result.workerId !== command.workerId ||
result.sessionId !== command.sessionId ||
result.generation !== command.generation ||
result.version !== command.expectedVersion + 1 ||
result.status !== command.status
) throw new TypeError('transition response authority mismatch');
return result;
} catch (error) {
throw new WorkerSessionHttpsClientError(
'response_invalid', { cause: error },
);
} finally {
Buffer.from(response.buffer, response.byteOffset, response.byteLength)
.fill(0);
}
}
private async exchange(
requestPath: string,
body: unknown,
maximumRequestBytes: number,
signal?: AbortSignal,
): Promise<Uint8Array> {
try {
return await this.client.postJson({
path: requestPath,
body,
maximumRequestBytes,
maximumResponseBytes: MAX_WORKER_SESSION_RESPONSE_BYTES,
...(signal === undefined ? {} : { signal }),
});
} catch (error) {
if (error instanceof WorkerIngressHttpsClientError) {
if (error.httpStatus === 401 || error.httpStatus === 403) {
throw new WorkerSessionHttpsClientError(
'credential_rejected', { cause: error },
);
}
if (error.httpStatus === 409) {
throw new WorkerSessionHttpsClientError(
'session_fenced', { cause: error },
);
}
throw new WorkerSessionHttpsClientError(
'transport_unavailable', { cause: error },
);
}
throw error;
}
}
}