mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 09:58:46 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,304 @@
|
||||
// Worker Process owns configuration-to-runtime activation, signals, and shutdown.
|
||||
import type { WorkerIngressHttpsCredentialProvider } from '../remote-execution/transport/workerIngressHttpsClient';
|
||||
import {
|
||||
startProductionWorkerApplication,
|
||||
type ProductionWorkerApplicationEnabledOptions,
|
||||
type ProductionWorkerApplicationOptions,
|
||||
} from '../application-runtime/productionWorkerApplication';
|
||||
import {
|
||||
loadWorkerProcessConfig,
|
||||
type EnabledWorkerProcessConfig,
|
||||
type WorkerProcessEnvironment,
|
||||
} from './workerProcessConfig';
|
||||
import { createWorkerProcessCredentialProvider } from './workerProcessIdentity';
|
||||
import type {
|
||||
ProductionWorkerCertificateRenewalLifecycle,
|
||||
ProductionWorkerHeadlessApplicationResult,
|
||||
ProductionWorkerHeadlessStopResult,
|
||||
} from '../application-runtime/productionHeadlessApplication';
|
||||
|
||||
export type WorkerProcessSignal = 'SIGINT' | 'SIGTERM';
|
||||
|
||||
export interface WorkerProcessSignalSource {
|
||||
subscribe(listener: (signal: WorkerProcessSignal) => void): () => void;
|
||||
}
|
||||
|
||||
export interface WorkerProcessEvent {
|
||||
readonly schemaVersion: 1;
|
||||
readonly component: 'qinglong3-worker';
|
||||
readonly level: 'info' | 'error';
|
||||
readonly event:
|
||||
| 'starting'
|
||||
| 'active'
|
||||
| 'shutdown_requested'
|
||||
| 'shutdown_deferred'
|
||||
| 'stopped'
|
||||
| 'runtime_diagnostic';
|
||||
readonly workerId?: string;
|
||||
readonly capacityProfile?: 'edge' | 'node';
|
||||
readonly signal?: WorkerProcessSignal;
|
||||
readonly stopResult?: ProductionWorkerHeadlessStopResult;
|
||||
readonly diagnostic?: Readonly<{
|
||||
readonly code:
|
||||
| 'tick_failed'
|
||||
| 'session_tick_failed'
|
||||
| 'certificate_renewal_failed'
|
||||
| 'certificate_unavailable'
|
||||
| 'recovery_required'
|
||||
| 'drain_failed'
|
||||
| 'disconnect_failed';
|
||||
}>;
|
||||
}
|
||||
|
||||
export type WorkerProcessStarter = (
|
||||
options: ProductionWorkerApplicationOptions,
|
||||
) => Promise<ProductionWorkerHeadlessApplicationResult>;
|
||||
|
||||
export type WorkerProcessCredentialFactory = (
|
||||
config: EnabledWorkerProcessConfig['identity'],
|
||||
) => Promise<Readonly<WorkerIngressHttpsCredentialProvider>>;
|
||||
|
||||
export type WorkerProcessCertificateRenewalFactory = (
|
||||
config: EnabledWorkerProcessConfig,
|
||||
credentials: Readonly<WorkerIngressHttpsCredentialProvider>,
|
||||
) => Promise<ProductionWorkerCertificateRenewalLifecycle | undefined>;
|
||||
|
||||
export interface ProductionWorkerProcessOptions {
|
||||
readonly environment: WorkerProcessEnvironment;
|
||||
readonly signals: WorkerProcessSignalSource;
|
||||
readonly emit: (event: WorkerProcessEvent) => void | Promise<void>;
|
||||
readonly start?: WorkerProcessStarter;
|
||||
readonly createCredentials?: WorkerProcessCredentialFactory;
|
||||
/** Deployment-owned CA adapter; omitted profiles keep zero renewal cost. */
|
||||
readonly createCertificateRenewal?: WorkerProcessCertificateRenewalFactory;
|
||||
readonly waitBeforeStopRetry?: () => Promise<void>;
|
||||
}
|
||||
|
||||
export class WorkerProcessError extends Error {
|
||||
readonly code:
|
||||
| 'QL3_WORKER_PROCESS_DISABLED'
|
||||
| 'QL3_WORKER_PROCESS_ACTIVATION_FAILED';
|
||||
|
||||
constructor(
|
||||
code: WorkerProcessError['code'],
|
||||
message: string,
|
||||
options?: ErrorOptions,
|
||||
) {
|
||||
super(message, options);
|
||||
this.name = 'WorkerProcessError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
function delay(): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 1_000));
|
||||
}
|
||||
|
||||
function productionOptions(
|
||||
config: EnabledWorkerProcessConfig,
|
||||
credentials: Readonly<WorkerIngressHttpsCredentialProvider>,
|
||||
certificateRenewal: ProductionWorkerCertificateRenewalLifecycle | undefined,
|
||||
diagnostic: NonNullable<
|
||||
ProductionWorkerApplicationEnabledOptions['diagnostic']
|
||||
>,
|
||||
): ProductionWorkerApplicationEnabledOptions {
|
||||
return Object.freeze({
|
||||
enabled: true,
|
||||
profile: 'worker',
|
||||
capacityProfile: config.capacityProfile,
|
||||
origin: config.origin,
|
||||
credentials,
|
||||
...(certificateRenewal === undefined ? {} : { certificateRenewal }),
|
||||
workerId: config.workerId,
|
||||
capabilities: config.capabilities,
|
||||
maxConcurrentRuns: config.maxConcurrentRuns,
|
||||
storage: config.storage,
|
||||
cadenceMs: config.lifecycle.cadenceMs,
|
||||
leaseDurationMs: config.lifecycle.leaseDurationMs,
|
||||
heartbeatIntervalMs: config.lifecycle.heartbeatIntervalMs,
|
||||
drainTimeoutMs: config.lifecycle.drainTimeoutMs,
|
||||
drainPollMs: config.lifecycle.drainPollMs,
|
||||
requestTimeoutMs: config.lifecycle.requestTimeoutMs,
|
||||
maximumJournalEntries: config.lifecycle.maximumJournalEntries,
|
||||
maximumRecordsPerTick: config.lifecycle.maximumRecordsPerTick,
|
||||
maximumSupervisionRecordsPerTick:
|
||||
config.lifecycle.maximumSupervisionRecordsPerTick,
|
||||
...(config.executor.launcherPath === undefined
|
||||
? {}
|
||||
: {
|
||||
launcherPath: config.executor.launcherPath,
|
||||
expectedLauncherSha256: config.executor.expectedLauncherSha256!,
|
||||
}),
|
||||
diagnostic,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns one production Worker process. Incomplete drain never releases the
|
||||
* application or returns success; it keeps the existing owner/Agent alive and
|
||||
* retries the same proof-bearing stop operation with a ref'ed shutdown wait.
|
||||
*/
|
||||
export async function runProductionWorkerProcess(
|
||||
options: ProductionWorkerProcessOptions,
|
||||
): Promise<'stopped'> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
typeof options.emit !== 'function' ||
|
||||
typeof options.signals?.subscribe !== 'function' ||
|
||||
(options.start !== undefined && typeof options.start !== 'function') ||
|
||||
(options.createCredentials !== undefined &&
|
||||
typeof options.createCredentials !== 'function') ||
|
||||
(options.createCertificateRenewal !== undefined &&
|
||||
typeof options.createCertificateRenewal !== 'function') ||
|
||||
(options.waitBeforeStopRetry !== undefined &&
|
||||
typeof options.waitBeforeStopRetry !== 'function')
|
||||
) {
|
||||
throw new TypeError('Worker process options are invalid');
|
||||
}
|
||||
|
||||
let resolveSignal: ((signal: WorkerProcessSignal) => void) | undefined;
|
||||
const requestedSignal = new Promise<WorkerProcessSignal>((resolve) => {
|
||||
resolveSignal = resolve;
|
||||
});
|
||||
let acceptedSignal = false;
|
||||
const unsubscribe = options.signals.subscribe((signal) => {
|
||||
if (acceptedSignal) return;
|
||||
acceptedSignal = true;
|
||||
resolveSignal?.(signal);
|
||||
});
|
||||
// A pending Promise does not keep Node's event loop alive. The Worker owns
|
||||
// the process until an OS shutdown signal arrives, so retain one inexpensive
|
||||
// referenced handle and release it with the rest of the signal authority.
|
||||
const lifecycleReference = setInterval(() => undefined, 2_147_483_647);
|
||||
const emit = (event: WorkerProcessEvent): void => {
|
||||
void Promise.resolve(options.emit(event)).catch(() => undefined);
|
||||
};
|
||||
|
||||
try {
|
||||
const config = await loadWorkerProcessConfig(options.environment);
|
||||
if (!config.enabled) {
|
||||
throw new WorkerProcessError(
|
||||
'QL3_WORKER_PROCESS_DISABLED',
|
||||
'The Worker process requires an enabled worker profile',
|
||||
);
|
||||
}
|
||||
emit(
|
||||
Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-worker',
|
||||
level: 'info',
|
||||
event: 'starting',
|
||||
workerId: config.workerId,
|
||||
capacityProfile: config.capacityProfile,
|
||||
}),
|
||||
);
|
||||
const credentials = await (
|
||||
options.createCredentials ?? createWorkerProcessCredentialProvider
|
||||
)(config.identity);
|
||||
if (!credentials || typeof credentials.load !== 'function') {
|
||||
throw new WorkerProcessError(
|
||||
'QL3_WORKER_PROCESS_ACTIVATION_FAILED',
|
||||
'Worker credential provider is invalid',
|
||||
);
|
||||
}
|
||||
const certificateRenewal = await options.createCertificateRenewal?.(
|
||||
config,
|
||||
credentials,
|
||||
);
|
||||
if (
|
||||
certificateRenewal !== undefined &&
|
||||
typeof certificateRenewal.run !== 'function'
|
||||
) {
|
||||
throw new WorkerProcessError(
|
||||
'QL3_WORKER_PROCESS_ACTIVATION_FAILED',
|
||||
'Worker certificate renewal lifecycle is invalid',
|
||||
);
|
||||
}
|
||||
const application = await (
|
||||
options.start ?? startProductionWorkerApplication
|
||||
)(
|
||||
productionOptions(config, credentials, certificateRenewal, (fact) => {
|
||||
emit(
|
||||
Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-worker',
|
||||
level: 'error',
|
||||
event: 'runtime_diagnostic',
|
||||
workerId: config.workerId,
|
||||
capacityProfile: config.capacityProfile,
|
||||
diagnostic: Object.freeze({ code: fact.code }),
|
||||
}),
|
||||
);
|
||||
}),
|
||||
);
|
||||
if (application.status !== 'active') {
|
||||
throw new WorkerProcessError(
|
||||
'QL3_WORKER_PROCESS_ACTIVATION_FAILED',
|
||||
'Worker application did not activate',
|
||||
);
|
||||
}
|
||||
emit(
|
||||
Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-worker',
|
||||
level: 'info',
|
||||
event: 'active',
|
||||
workerId: config.workerId,
|
||||
capacityProfile: config.capacityProfile,
|
||||
}),
|
||||
);
|
||||
|
||||
const signal = await requestedSignal;
|
||||
emit(
|
||||
Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-worker',
|
||||
level: 'info',
|
||||
event: 'shutdown_requested',
|
||||
workerId: config.workerId,
|
||||
capacityProfile: config.capacityProfile,
|
||||
signal,
|
||||
}),
|
||||
);
|
||||
const waitBeforeRetry = options.waitBeforeStopRetry ?? delay;
|
||||
while (true) {
|
||||
let result: ProductionWorkerHeadlessStopResult;
|
||||
try {
|
||||
result = await application.stop();
|
||||
} catch {
|
||||
result = 'recovery_required';
|
||||
}
|
||||
if (result === 'stopped') {
|
||||
emit(
|
||||
Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-worker',
|
||||
level: 'info',
|
||||
event: 'stopped',
|
||||
workerId: config.workerId,
|
||||
capacityProfile: config.capacityProfile,
|
||||
stopResult: result,
|
||||
}),
|
||||
);
|
||||
return result;
|
||||
}
|
||||
emit(
|
||||
Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-worker',
|
||||
level: 'error',
|
||||
event: 'shutdown_deferred',
|
||||
workerId: config.workerId,
|
||||
capacityProfile: config.capacityProfile,
|
||||
stopResult: result,
|
||||
}),
|
||||
);
|
||||
await waitBeforeRetry();
|
||||
}
|
||||
} finally {
|
||||
clearInterval(lifecycleReference);
|
||||
unsubscribe();
|
||||
resolveSignal = undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// Worker Process owns the OS signal and diagnostic stream CLI adapter.
|
||||
import {
|
||||
runProductionWorkerProcess,
|
||||
type WorkerProcessSignal,
|
||||
type WorkerProcessSignalSource,
|
||||
} from './workerProcessApplication';
|
||||
|
||||
function signalSource(): WorkerProcessSignalSource {
|
||||
return Object.freeze({
|
||||
subscribe(listener: (signal: WorkerProcessSignal) => void): () => void {
|
||||
const onInterrupt = () => listener('SIGINT');
|
||||
const onTerminate = () => listener('SIGTERM');
|
||||
process.once('SIGINT', onInterrupt);
|
||||
process.once('SIGTERM', onTerminate);
|
||||
return () => {
|
||||
process.off('SIGINT', onInterrupt);
|
||||
process.off('SIGTERM', onTerminate);
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
await runProductionWorkerProcess({
|
||||
environment: process.env,
|
||||
signals: signalSource(),
|
||||
emit(event) {
|
||||
const output = `${JSON.stringify(event)}\n`;
|
||||
if (event.level === 'error') process.stderr.write(output);
|
||||
else process.stdout.write(output);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
void main().catch((error: unknown) => {
|
||||
const candidate = error as {
|
||||
readonly name?: unknown;
|
||||
readonly code?: unknown;
|
||||
};
|
||||
process.stderr.write(
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-worker',
|
||||
level: 'error',
|
||||
event: 'process_failed',
|
||||
name:
|
||||
typeof candidate?.name === 'string'
|
||||
? candidate.name.slice(0, 128)
|
||||
: 'Error',
|
||||
...(typeof candidate?.code === 'string'
|
||||
? { code: candidate.code.slice(0, 128) }
|
||||
: {}),
|
||||
})}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,448 @@
|
||||
// Worker Process owns bounded environment-to-runtime configuration mapping.
|
||||
import { constants } from 'node:fs';
|
||||
import { open } from 'node:fs/promises';
|
||||
import {
|
||||
isAbsolute,
|
||||
normalize,
|
||||
parse,
|
||||
} from 'node:path';
|
||||
import {
|
||||
canonicalRemoteWorkerCapabilities,
|
||||
type RemoteWorkerCapabilities,
|
||||
} from '@qinglong/runtime-core/remote-dispatch';
|
||||
|
||||
const MAX_CAPABILITIES_FILE_BYTES = 20 * 1024;
|
||||
const WORKER_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const CREDENTIAL_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/;
|
||||
const SHA256 = /^[0-9a-f]{64}$/;
|
||||
|
||||
export type WorkerProcessEnvironment = Readonly<
|
||||
Record<string, string | undefined>
|
||||
>;
|
||||
|
||||
export interface DisabledWorkerProcessConfig {
|
||||
readonly enabled: false;
|
||||
readonly profile: string;
|
||||
}
|
||||
|
||||
export interface EnabledWorkerProcessConfig {
|
||||
readonly enabled: true;
|
||||
readonly profile: 'worker';
|
||||
readonly capacityProfile: 'edge' | 'node';
|
||||
readonly workerId: string;
|
||||
readonly origin: string;
|
||||
readonly capabilities: RemoteWorkerCapabilities;
|
||||
readonly maxConcurrentRuns: number;
|
||||
readonly storage: Readonly<{
|
||||
readonly journalRoot: string;
|
||||
readonly logRoot: string;
|
||||
readonly receiptRoot: string;
|
||||
}>;
|
||||
readonly identity: Readonly<{
|
||||
readonly certificateStoreRoot: string;
|
||||
readonly trustAnchorFile: string;
|
||||
readonly credentialTokenFile: string;
|
||||
readonly expectedCredentialId?: string;
|
||||
readonly bootstrap?: Readonly<{
|
||||
readonly privateKeyFile: string;
|
||||
readonly certificateChainFile: string;
|
||||
}>;
|
||||
}>;
|
||||
readonly lifecycle: Readonly<{
|
||||
readonly cadenceMs: number;
|
||||
readonly leaseDurationMs: number;
|
||||
readonly heartbeatIntervalMs: number;
|
||||
readonly drainTimeoutMs: number;
|
||||
readonly drainPollMs: number;
|
||||
readonly requestTimeoutMs: number;
|
||||
readonly maximumJournalEntries: number;
|
||||
readonly maximumRecordsPerTick: number;
|
||||
readonly maximumSupervisionRecordsPerTick: number;
|
||||
}>;
|
||||
readonly executor: Readonly<{
|
||||
readonly launcherPath?: string;
|
||||
readonly expectedLauncherSha256?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export type WorkerProcessConfig =
|
||||
| DisabledWorkerProcessConfig
|
||||
| EnabledWorkerProcessConfig;
|
||||
|
||||
export class WorkerProcessConfigError extends TypeError {
|
||||
readonly code = 'QL3_WORKER_PROCESS_CONFIG_INVALID';
|
||||
|
||||
constructor(message: string, options?: ErrorOptions) {
|
||||
super(`Worker process configuration is invalid: ${message}`, options);
|
||||
this.name = 'WorkerProcessConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
function booleanValue(
|
||||
environment: WorkerProcessEnvironment,
|
||||
name: string,
|
||||
fallback: boolean,
|
||||
): boolean {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return fallback;
|
||||
if (value === 'true') return true;
|
||||
if (value === 'false') return false;
|
||||
throw new WorkerProcessConfigError(`${name} must be true or false`);
|
||||
}
|
||||
|
||||
function boundedValue(
|
||||
environment: WorkerProcessEnvironment,
|
||||
name: string,
|
||||
maximumBytes: number,
|
||||
required = false,
|
||||
): string | undefined {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') {
|
||||
if (required) throw new WorkerProcessConfigError(`${name} is required`);
|
||||
return undefined;
|
||||
}
|
||||
if (
|
||||
Buffer.byteLength(value, 'utf8') > maximumBytes ||
|
||||
/[\0\r\n]/.test(value)
|
||||
) {
|
||||
throw new WorkerProcessConfigError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function integerValue(
|
||||
environment: WorkerProcessEnvironment,
|
||||
name: string,
|
||||
fallback: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return fallback;
|
||||
if (!/^(0|[1-9]\d*)$/.test(value)) {
|
||||
throw new WorkerProcessConfigError(`${name} must be an integer`);
|
||||
}
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
|
||||
throw new WorkerProcessConfigError(
|
||||
`${name} must be between ${minimum} and ${maximum}`,
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function absolutePath(
|
||||
environment: WorkerProcessEnvironment,
|
||||
name: string,
|
||||
required = true,
|
||||
): string | undefined {
|
||||
const value = boundedValue(environment, name, 4096, required);
|
||||
if (value === undefined) return undefined;
|
||||
if (
|
||||
!isAbsolute(value) ||
|
||||
parse(value).root === value ||
|
||||
normalize(value) !== value
|
||||
) {
|
||||
throw new WorkerProcessConfigError(`${name} must be a normalized absolute path`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function origin(environment: WorkerProcessEnvironment): string {
|
||||
const value = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_CONTROL_ORIGIN',
|
||||
2048,
|
||||
true,
|
||||
)!;
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(value);
|
||||
} catch (error) {
|
||||
throw new WorkerProcessConfigError(
|
||||
'QL3_WORKER_CONTROL_ORIGIN is invalid',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
if (
|
||||
parsed.protocol !== 'https:' ||
|
||||
parsed.username !== '' ||
|
||||
parsed.password !== '' ||
|
||||
parsed.pathname !== '/' ||
|
||||
parsed.search !== '' ||
|
||||
parsed.hash !== ''
|
||||
) {
|
||||
throw new WorkerProcessConfigError(
|
||||
'QL3_WORKER_CONTROL_ORIGIN must be an HTTPS origin',
|
||||
);
|
||||
}
|
||||
return parsed.origin;
|
||||
}
|
||||
|
||||
async function capabilities(path: string): Promise<RemoteWorkerCapabilities> {
|
||||
let handle;
|
||||
let bytes: Buffer | undefined;
|
||||
try {
|
||||
handle = await open(
|
||||
path,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
const stat = await handle.stat();
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.size < 2 ||
|
||||
stat.size > MAX_CAPABILITIES_FILE_BYTES ||
|
||||
(stat.mode & 0o022) !== 0
|
||||
) {
|
||||
throw new Error('unsafe capabilities metadata');
|
||||
}
|
||||
bytes = await handle.readFile();
|
||||
if (
|
||||
bytes.byteLength < 2 ||
|
||||
bytes.byteLength > MAX_CAPABILITIES_FILE_BYTES
|
||||
) {
|
||||
throw new Error('unsafe capabilities size');
|
||||
}
|
||||
const parsed = JSON.parse(
|
||||
new TextDecoder('utf-8', { fatal: true }).decode(bytes),
|
||||
) as unknown;
|
||||
return canonicalRemoteWorkerCapabilities(parsed).capabilities;
|
||||
} catch (error) {
|
||||
throw new WorkerProcessConfigError(
|
||||
'QL3_WORKER_CAPABILITIES_FILE is unavailable or invalid',
|
||||
{ cause: error },
|
||||
);
|
||||
} finally {
|
||||
bytes?.fill(0);
|
||||
await handle?.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadWorkerProcessConfig(
|
||||
environment: WorkerProcessEnvironment,
|
||||
): Promise<WorkerProcessConfig> {
|
||||
if (
|
||||
!environment ||
|
||||
typeof environment !== 'object' ||
|
||||
Array.isArray(environment)
|
||||
) {
|
||||
throw new WorkerProcessConfigError('environment must be an object');
|
||||
}
|
||||
const enabled = booleanValue(
|
||||
environment,
|
||||
'QL3_WORKER_RUNTIME_ENABLED',
|
||||
false,
|
||||
);
|
||||
const profile = environment.QL_DEPLOYMENT_PROFILE ?? 'standalone';
|
||||
if (!enabled) return Object.freeze({ enabled: false, profile });
|
||||
if (profile !== 'worker') {
|
||||
throw new WorkerProcessConfigError(
|
||||
'enabled runtime requires QL_DEPLOYMENT_PROFILE=worker',
|
||||
);
|
||||
}
|
||||
|
||||
const capacityProfile =
|
||||
boundedValue(environment, 'QL3_WORKER_CAPACITY_PROFILE', 8) ?? 'edge';
|
||||
if (capacityProfile !== 'edge' && capacityProfile !== 'node') {
|
||||
throw new WorkerProcessConfigError(
|
||||
'QL3_WORKER_CAPACITY_PROFILE must be edge or node',
|
||||
);
|
||||
}
|
||||
const edge = capacityProfile === 'edge';
|
||||
const workerId = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ID',
|
||||
128,
|
||||
true,
|
||||
)!;
|
||||
if (!WORKER_ID.test(workerId)) {
|
||||
throw new WorkerProcessConfigError('QL3_WORKER_ID is invalid');
|
||||
}
|
||||
const expectedCredentialId = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_EXPECTED_CREDENTIAL_ID',
|
||||
64,
|
||||
);
|
||||
if (
|
||||
expectedCredentialId !== undefined &&
|
||||
!CREDENTIAL_ID.test(expectedCredentialId)
|
||||
) {
|
||||
throw new WorkerProcessConfigError(
|
||||
'QL3_WORKER_EXPECTED_CREDENTIAL_ID is invalid',
|
||||
);
|
||||
}
|
||||
const bootstrapPrivateKeyFile = absolutePath(
|
||||
environment,
|
||||
'QL3_WORKER_IDENTITY_BOOTSTRAP_PRIVATE_KEY_FILE',
|
||||
false,
|
||||
);
|
||||
const bootstrapCertificateChainFile = absolutePath(
|
||||
environment,
|
||||
'QL3_WORKER_IDENTITY_BOOTSTRAP_CERTIFICATE_FILE',
|
||||
false,
|
||||
);
|
||||
if (
|
||||
(bootstrapPrivateKeyFile === undefined) !==
|
||||
(bootstrapCertificateChainFile === undefined)
|
||||
) {
|
||||
throw new WorkerProcessConfigError(
|
||||
'Worker identity bootstrap key and certificate must be configured together',
|
||||
);
|
||||
}
|
||||
const launcherPath = absolutePath(
|
||||
environment,
|
||||
'QL3_WORKER_LAUNCHER_PATH',
|
||||
false,
|
||||
);
|
||||
const expectedLauncherSha256 = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_LAUNCHER_SHA256',
|
||||
64,
|
||||
);
|
||||
if (
|
||||
(launcherPath === undefined) !==
|
||||
(expectedLauncherSha256 === undefined) ||
|
||||
(expectedLauncherSha256 !== undefined &&
|
||||
!SHA256.test(expectedLauncherSha256))
|
||||
) {
|
||||
throw new WorkerProcessConfigError(
|
||||
'Worker launcher path and SHA-256 must be configured together',
|
||||
);
|
||||
}
|
||||
const capabilitiesFile = absolutePath(
|
||||
environment,
|
||||
'QL3_WORKER_CAPABILITIES_FILE',
|
||||
)!;
|
||||
const leaseDurationMs = integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_SESSION_LEASE_DURATION_MS',
|
||||
45_000,
|
||||
15_000,
|
||||
10 * 60_000,
|
||||
);
|
||||
const heartbeatIntervalMs = integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_HEARTBEAT_INTERVAL_MS',
|
||||
10_000,
|
||||
5_000,
|
||||
5 * 60_000,
|
||||
);
|
||||
if (heartbeatIntervalMs * 2 > leaseDurationMs) {
|
||||
throw new WorkerProcessConfigError(
|
||||
'Worker heartbeat interval must fit twice inside the Session lease',
|
||||
);
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
enabled: true,
|
||||
profile: 'worker',
|
||||
capacityProfile,
|
||||
workerId,
|
||||
origin: origin(environment),
|
||||
capabilities: await capabilities(capabilitiesFile),
|
||||
maxConcurrentRuns: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_MAX_CONCURRENT_RUNS',
|
||||
edge ? 1 : 8,
|
||||
1,
|
||||
edge ? 4 : 64,
|
||||
),
|
||||
storage: Object.freeze({
|
||||
journalRoot: absolutePath(
|
||||
environment,
|
||||
'QL3_WORKER_JOURNAL_ROOT',
|
||||
)!,
|
||||
logRoot: absolutePath(environment, 'QL3_WORKER_LOG_ROOT')!,
|
||||
receiptRoot: absolutePath(
|
||||
environment,
|
||||
'QL3_WORKER_RECEIPT_ROOT',
|
||||
)!,
|
||||
}),
|
||||
identity: Object.freeze({
|
||||
certificateStoreRoot: absolutePath(
|
||||
environment,
|
||||
'QL3_WORKER_CERTIFICATE_STORE_ROOT',
|
||||
)!,
|
||||
trustAnchorFile: absolutePath(
|
||||
environment,
|
||||
'QL3_WORKER_TRUST_ANCHOR_FILE',
|
||||
)!,
|
||||
credentialTokenFile: absolutePath(
|
||||
environment,
|
||||
'QL3_WORKER_CREDENTIAL_TOKEN_FILE',
|
||||
)!,
|
||||
...(expectedCredentialId === undefined
|
||||
? {}
|
||||
: { expectedCredentialId }),
|
||||
...(bootstrapPrivateKeyFile === undefined
|
||||
? {}
|
||||
: {
|
||||
bootstrap: Object.freeze({
|
||||
privateKeyFile: bootstrapPrivateKeyFile,
|
||||
certificateChainFile: bootstrapCertificateChainFile!,
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
lifecycle: Object.freeze({
|
||||
cadenceMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_CADENCE_MS',
|
||||
edge ? 2_000 : 500,
|
||||
100,
|
||||
60_000,
|
||||
),
|
||||
leaseDurationMs,
|
||||
heartbeatIntervalMs,
|
||||
drainTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_DRAIN_TIMEOUT_MS',
|
||||
edge ? 60_000 : 5 * 60_000,
|
||||
1_000,
|
||||
10 * 60_000,
|
||||
),
|
||||
drainPollMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_DRAIN_POLL_MS',
|
||||
edge ? 500 : 100,
|
||||
25,
|
||||
5_000,
|
||||
),
|
||||
requestTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_REQUEST_TIMEOUT_MS',
|
||||
15_000,
|
||||
100,
|
||||
120_000,
|
||||
),
|
||||
maximumJournalEntries: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_MAXIMUM_JOURNAL_ENTRIES',
|
||||
edge ? 64 : 256,
|
||||
1,
|
||||
1024,
|
||||
),
|
||||
maximumRecordsPerTick: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_MAXIMUM_RECORDS_PER_TICK',
|
||||
edge ? 4 : 16,
|
||||
1,
|
||||
64,
|
||||
),
|
||||
maximumSupervisionRecordsPerTick: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_MAXIMUM_SUPERVISION_RECORDS_PER_TICK',
|
||||
edge ? 4 : 32,
|
||||
1,
|
||||
64,
|
||||
),
|
||||
}),
|
||||
executor: Object.freeze({
|
||||
...(launcherPath === undefined
|
||||
? {}
|
||||
: {
|
||||
launcherPath,
|
||||
expectedLauncherSha256: expectedLauncherSha256!,
|
||||
}),
|
||||
}),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
// Worker Process owns private bootstrap material and active credential composition.
|
||||
import { constants } from 'node:fs';
|
||||
import { open } from 'node:fs/promises';
|
||||
import type {
|
||||
WorkerCertificateTrustAnchorProvider,
|
||||
} from '../credential/workerCertificateRenewal';
|
||||
import { WorkerCertificateFileStore } from '../credential/workerCertificateStore';
|
||||
import { validateWorkerCertificateIdentity } from '../credential/workerCertificateIdentity';
|
||||
import { WorkerProductionCredentialProvider } from '../credential/workerProductionCredentialProvider';
|
||||
import type { EnabledWorkerProcessConfig } from './workerProcessConfig';
|
||||
|
||||
const MAX_TLS_MATERIAL_BYTES = 1024 * 1024;
|
||||
|
||||
export class WorkerProcessIdentityError extends Error {
|
||||
readonly code = 'QL3_WORKER_PROCESS_IDENTITY_UNAVAILABLE';
|
||||
|
||||
constructor(message: string, options?: ErrorOptions) {
|
||||
super(`Worker process identity is unavailable: ${message}`, options);
|
||||
this.name = 'WorkerProcessIdentityError';
|
||||
}
|
||||
}
|
||||
|
||||
async function readMaterial(
|
||||
path: string,
|
||||
privateMaterial: boolean,
|
||||
): Promise<Buffer> {
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(
|
||||
path,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
const stat = await handle.stat();
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.size < 1 ||
|
||||
stat.size > MAX_TLS_MATERIAL_BYTES ||
|
||||
(stat.mode & 0o022) !== 0 ||
|
||||
(privateMaterial && (stat.mode & 0o077) !== 0)
|
||||
) {
|
||||
throw new Error('unsafe identity material metadata');
|
||||
}
|
||||
const bytes = await handle.readFile();
|
||||
if (
|
||||
bytes.byteLength < 1 ||
|
||||
bytes.byteLength > MAX_TLS_MATERIAL_BYTES
|
||||
) {
|
||||
bytes.fill(0);
|
||||
throw new Error('unsafe identity material size');
|
||||
}
|
||||
return bytes;
|
||||
} catch (error) {
|
||||
throw new WorkerProcessIdentityError('material read failed', {
|
||||
cause: error,
|
||||
});
|
||||
} finally {
|
||||
await handle?.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export class WorkerTrustAnchorFileProvider
|
||||
implements WorkerCertificateTrustAnchorProvider
|
||||
{
|
||||
constructor(private readonly path: string) {
|
||||
if (typeof path !== 'string' || path.length < 1) {
|
||||
throw new WorkerProcessIdentityError('trust anchor path is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async load(signal: AbortSignal): Promise<readonly Buffer[]> {
|
||||
if (!(signal instanceof AbortSignal)) {
|
||||
throw new WorkerProcessIdentityError('trust signal is invalid');
|
||||
}
|
||||
signal.throwIfAborted();
|
||||
const bytes = await readMaterial(this.path, false);
|
||||
try {
|
||||
signal.throwIfAborted();
|
||||
return Object.freeze([bytes]);
|
||||
} catch (error) {
|
||||
bytes.fill(0);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies or bootstraps one durable Worker certificate store, then returns
|
||||
* the per-request credential provider. Bootstrap material is optional and is
|
||||
* only installed when its validated leaf differs from the active generation.
|
||||
*/
|
||||
export async function createWorkerProcessCredentialProvider(
|
||||
config: EnabledWorkerProcessConfig['identity'],
|
||||
): Promise<Readonly<WorkerProductionCredentialProvider>> {
|
||||
if (!config || typeof config !== 'object') {
|
||||
throw new WorkerProcessIdentityError('configuration is invalid');
|
||||
}
|
||||
const store = new WorkerCertificateFileStore({
|
||||
rootDirectory: config.certificateStoreRoot,
|
||||
retainedGenerations: 2,
|
||||
});
|
||||
const trustAnchors = new WorkerTrustAnchorFileProvider(
|
||||
config.trustAnchorFile,
|
||||
);
|
||||
const signal = new AbortController().signal;
|
||||
let anchors: readonly Buffer[] | undefined;
|
||||
let privateKey: Buffer | undefined;
|
||||
let certificate: Buffer | undefined;
|
||||
try {
|
||||
anchors = await trustAnchors.load(signal);
|
||||
const active = await store.readActive(anchors);
|
||||
if (config.bootstrap !== undefined) {
|
||||
[privateKey, certificate] = await Promise.all([
|
||||
readMaterial(config.bootstrap.privateKeyFile, true),
|
||||
readMaterial(config.bootstrap.certificateChainFile, false),
|
||||
]);
|
||||
const bootstrap = validateWorkerCertificateIdentity({
|
||||
privateKeyPem: privateKey,
|
||||
certificateChainPem: certificate,
|
||||
trustAnchors: anchors,
|
||||
});
|
||||
if (active?.certificateSha256 !== bootstrap.certificateSha256) {
|
||||
await store.install({
|
||||
privateKeyPem: privateKey,
|
||||
certificateChainPem: certificate,
|
||||
trustAnchors: anchors,
|
||||
});
|
||||
}
|
||||
} else if (!active) {
|
||||
throw new WorkerProcessIdentityError(
|
||||
'no active identity or bootstrap material',
|
||||
);
|
||||
}
|
||||
return new WorkerProductionCredentialProvider({
|
||||
certificateStore: store,
|
||||
trustAnchors,
|
||||
credentialTokenFile: config.credentialTokenFile,
|
||||
...(config.expectedCredentialId === undefined
|
||||
? {}
|
||||
: { expectedCredentialId: config.expectedCredentialId }),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof WorkerProcessIdentityError) throw error;
|
||||
throw new WorkerProcessIdentityError('activation failed', {
|
||||
cause: error,
|
||||
});
|
||||
} finally {
|
||||
privateKey?.fill(0);
|
||||
certificate?.fill(0);
|
||||
anchors?.forEach((anchor) => anchor.fill(0));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user