mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 19:29:13 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,99 @@
|
||||
// Cluster Control Worker Ingress boundary; keep production PostgreSQL composition explicit.
|
||||
import {
|
||||
assertPostgresWorkerIngressSchemaReady,
|
||||
PostgresSecurityAuditRepository,
|
||||
PostgresWorkerCredentialRepository,
|
||||
PostgresWorkerExecutionAttestationRepository,
|
||||
PostgresWorkerSessionRepository,
|
||||
} from '@qinglong/cluster-postgres/worker-ingress';
|
||||
import {
|
||||
startClusterWorkerIngressApplication,
|
||||
type ClusterWorkerIngressApplicationResult,
|
||||
} from './workerIngressApplication';
|
||||
import {
|
||||
createClusterWorkerIngressDatabaseOpener,
|
||||
createClusterWorkerIngressHttpOptions,
|
||||
type EnabledClusterWorkerIngressConfig,
|
||||
} from './workerIngressConfig';
|
||||
import {
|
||||
createWorkerCredentialAuthenticator,
|
||||
} from './workerCredentialAuthenticator';
|
||||
import {
|
||||
createWorkerIngressAdmissionPipeline,
|
||||
} from './workerIngressPipeline';
|
||||
import type { ClusterWorkerRuntimePort } from '../remote-execution/workerRuntimePort';
|
||||
|
||||
export interface ProductionClusterWorkerIngressOptions {
|
||||
readonly config: EnabledClusterWorkerIngressConfig;
|
||||
readonly runtime: ClusterWorkerRuntimePort;
|
||||
readonly onPoolError?: (error: Error) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the reviewed Worker-facing listener. The worker-ingress Pool is used
|
||||
* only for authentication, Session, attestation and audit authority. Every
|
||||
* Run/Attempt/Lease mutation crosses the injected runtime capability port.
|
||||
*/
|
||||
export async function startProductionClusterWorkerIngress(
|
||||
options: ProductionClusterWorkerIngressOptions,
|
||||
): Promise<ClusterWorkerIngressApplicationResult> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
!options.config?.enabled ||
|
||||
!options.runtime
|
||||
) {
|
||||
throw new TypeError('Production Worker ingress options are invalid');
|
||||
}
|
||||
if (
|
||||
options.onPoolError !== undefined &&
|
||||
typeof options.onPoolError !== 'function'
|
||||
) {
|
||||
throw new TypeError('Production Worker ingress Pool error sink is invalid');
|
||||
}
|
||||
const http = await createClusterWorkerIngressHttpOptions(options.config);
|
||||
const openDatabase = createClusterWorkerIngressDatabaseOpener(
|
||||
options.config,
|
||||
(error) => options.onPoolError?.(error),
|
||||
);
|
||||
return startClusterWorkerIngressApplication({
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
workerCredentialPepper:
|
||||
options.config.security.workerCredentialPepper,
|
||||
openDatabase,
|
||||
http,
|
||||
async create({ database, workerCredentialPepper }) {
|
||||
const report = await assertPostgresWorkerIngressSchemaReady(
|
||||
database.pool,
|
||||
);
|
||||
return Object.freeze({
|
||||
evidence: Object.freeze({
|
||||
contractName: report.contractName,
|
||||
contractVersion: report.contractVersion,
|
||||
serverMajor: report.serverMajor,
|
||||
migrationIds: Object.freeze([...report.migrationIds]),
|
||||
}),
|
||||
pipeline: createWorkerIngressAdmissionPipeline({
|
||||
authenticator: createWorkerCredentialAuthenticator(
|
||||
new PostgresWorkerCredentialRepository(database.pool),
|
||||
workerCredentialPepper,
|
||||
),
|
||||
workers: new PostgresWorkerSessionRepository(database.pool),
|
||||
attestations: new PostgresWorkerExecutionAttestationRepository(
|
||||
database.pool,
|
||||
),
|
||||
audit: new PostgresSecurityAuditRepository(database.pool),
|
||||
offers: options.runtime.offers,
|
||||
activation: options.runtime.activation,
|
||||
...(options.runtime.secrets === undefined
|
||||
? {}
|
||||
: { secrets: options.runtime.secrets }),
|
||||
artifacts: options.runtime.artifacts,
|
||||
completion: options.runtime.completion,
|
||||
leaseControl: options.runtime.leaseControl,
|
||||
}),
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Cluster Control Worker Ingress boundary; keep Worker credential authentication explicit.
|
||||
import { timingSafeEqual } from 'node:crypto';
|
||||
import {
|
||||
WorkerCredentialUnavailableError,
|
||||
normalizeWorkerCredentialRecord,
|
||||
type WorkerCredentialRepository,
|
||||
} from '@qinglong/runtime-core/worker-credential';
|
||||
import {
|
||||
assertWorkerCredentialPepper,
|
||||
workerCredentialSecretDigest,
|
||||
} from '@qinglong/runtime-core/worker-credential-token';
|
||||
import type { ClusterControlAdmissionMetadata } from '../transport/httpSurface';
|
||||
|
||||
export interface AuthenticatedWorkerPrincipal {
|
||||
readonly workerId: string;
|
||||
readonly credentialId: string;
|
||||
readonly credentialVersion: number;
|
||||
readonly authenticationId: string;
|
||||
readonly authenticatedAtMs: number;
|
||||
readonly expiresAtMs: number;
|
||||
}
|
||||
|
||||
export interface WorkerCredentialAuthenticator {
|
||||
authenticate(
|
||||
metadata: ClusterControlAdmissionMetadata,
|
||||
): Promise<Readonly<AuthenticatedWorkerPrincipal> | null>;
|
||||
}
|
||||
|
||||
const AUTHORIZATION =
|
||||
/^Worker ql3w_([A-Za-z0-9][A-Za-z0-9._:-]{0,63})_([A-Za-z0-9_-]{43})$/;
|
||||
|
||||
export function createWorkerCredentialAuthenticator(
|
||||
repository: WorkerCredentialRepository,
|
||||
pepper: string,
|
||||
options: Readonly<{ now?: () => number; principalTtlMs?: number }> = {},
|
||||
): WorkerCredentialAuthenticator {
|
||||
if (!repository || typeof repository.resolve !== 'function') {
|
||||
throw new TypeError('Worker credential authenticator repository is invalid');
|
||||
}
|
||||
assertWorkerCredentialPepper(pepper);
|
||||
const now = options.now ?? Date.now;
|
||||
const principalTtlMs = options.principalTtlMs ?? 60_000;
|
||||
if (
|
||||
!Number.isSafeInteger(principalTtlMs) ||
|
||||
principalTtlMs < 1_000 ||
|
||||
principalTtlMs > 300_000
|
||||
) {
|
||||
throw new RangeError('Worker credential principal TTL is invalid');
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
async authenticate(metadata: ClusterControlAdmissionMetadata) {
|
||||
const header = metadata.headers.authorization;
|
||||
if (typeof header !== 'string') return null;
|
||||
const match = AUTHORIZATION.exec(header);
|
||||
if (!match) return null;
|
||||
let presented: Buffer | undefined;
|
||||
try {
|
||||
presented = Buffer.from(
|
||||
workerCredentialSecretDigest(pepper, match[1]!, match[2]!),
|
||||
'hex',
|
||||
);
|
||||
const candidate = await repository.resolve(match[1]!);
|
||||
if (metadata.signal.aborted) throw new WorkerCredentialUnavailableError();
|
||||
const record = candidate ? normalizeWorkerCredentialRecord(candidate) : null;
|
||||
const stored = record
|
||||
? Buffer.from(record.secretDigest, 'hex')
|
||||
: Buffer.alloc(32);
|
||||
const matches = timingSafeEqual(presented, stored);
|
||||
stored.fill(0);
|
||||
if (!record || !matches) return null;
|
||||
const nowMs = now();
|
||||
if (
|
||||
!Number.isSafeInteger(nowMs) ||
|
||||
nowMs < 0 ||
|
||||
record.state !== 'active' ||
|
||||
record.notBeforeAtMs > nowMs ||
|
||||
record.expiresAtMs <= nowMs
|
||||
) return null;
|
||||
return Object.freeze({
|
||||
workerId: record.workerId,
|
||||
credentialId: record.credentialId,
|
||||
credentialVersion: record.version,
|
||||
authenticationId: `worker_credential:${record.credentialId}:${record.version}`,
|
||||
authenticatedAtMs: nowMs,
|
||||
expiresAtMs: Math.min(record.expiresAtMs, nowMs + principalTtlMs),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof WorkerCredentialUnavailableError) throw error;
|
||||
throw new WorkerCredentialUnavailableError();
|
||||
} finally {
|
||||
presented?.fill(0);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
// Cluster Control Worker Ingress boundary; keep listener lifecycle authority explicit.
|
||||
import type {
|
||||
ClusterControlReadinessEvidence,
|
||||
ClusterControlAdmissionDisposer,
|
||||
DeploymentProfile,
|
||||
OpenPostgresDatabase,
|
||||
PostgresDatabaseResource,
|
||||
} from '@qinglong/runtime-core';
|
||||
import { assertWorkerCredentialPepper } from '@qinglong/runtime-core/worker-credential-token';
|
||||
import { MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES } from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import {
|
||||
startClusterControlHttpSurface,
|
||||
type ClusterControlAdmissionPipeline,
|
||||
type ClusterControlHttpAddress,
|
||||
type ClusterControlHttpSurfaceOptions,
|
||||
type ClusterControlMutualTlsOptions,
|
||||
} from '../transport/httpSurface';
|
||||
|
||||
export interface ClusterWorkerIngressAssemblyInput {
|
||||
readonly database: PostgresDatabaseResource;
|
||||
readonly workerCredentialPepper: string;
|
||||
}
|
||||
|
||||
export interface ClusterWorkerIngressAssembly {
|
||||
readonly evidence: ClusterControlReadinessEvidence;
|
||||
readonly pipeline: ClusterControlAdmissionPipeline;
|
||||
}
|
||||
|
||||
export interface ClusterWorkerIngressApplicationOptions {
|
||||
readonly enabled?: boolean;
|
||||
readonly profile: DeploymentProfile;
|
||||
readonly workerCredentialPepper?: string;
|
||||
readonly openDatabase: OpenPostgresDatabase;
|
||||
readonly http: ClusterControlHttpSurfaceOptions;
|
||||
readonly create: (
|
||||
input: ClusterWorkerIngressAssemblyInput,
|
||||
) => ClusterWorkerIngressAssembly | Promise<ClusterWorkerIngressAssembly>;
|
||||
}
|
||||
|
||||
export type ClusterWorkerIngressApplicationResult =
|
||||
| { readonly status: 'disabled'; stop(): Promise<'stopped'> }
|
||||
| {
|
||||
readonly status: 'active';
|
||||
readonly protocol: 'https';
|
||||
readonly transport: 'mutual-tls';
|
||||
readonly address: ClusterControlHttpAddress;
|
||||
readonly evidence: ClusterControlReadinessEvidence;
|
||||
reloadTransport(options: ClusterControlMutualTlsOptions): number;
|
||||
stop(): Promise<'stopped'>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Separate Worker-facing composition root. It owns a dedicated listener and a
|
||||
* worker-ingress database resource. Storage readiness and repositories are
|
||||
* supplied by the outer composition root, so this transport layer cannot
|
||||
* acquire Project Policy, dispatch, recovery-claim or DDL authority itself.
|
||||
*/
|
||||
export async function startClusterWorkerIngressApplication(
|
||||
options: ClusterWorkerIngressApplicationOptions,
|
||||
): Promise<ClusterWorkerIngressApplicationResult> {
|
||||
if (!(options.enabled ?? false)) {
|
||||
return Object.freeze({
|
||||
status: 'disabled',
|
||||
async stop() {
|
||||
return 'stopped' as const;
|
||||
},
|
||||
});
|
||||
}
|
||||
if (options.profile !== 'cluster-control') {
|
||||
throw new TypeError('Worker ingress requires cluster-control profile');
|
||||
}
|
||||
if (typeof options.create !== 'function') {
|
||||
throw new TypeError('Worker ingress assembly factory is required');
|
||||
}
|
||||
if (!options.http?.mutualTls) {
|
||||
throw new TypeError('Worker ingress requires mutual TLS');
|
||||
}
|
||||
assertWorkerCredentialPepper(options.workerCredentialPepper ?? '');
|
||||
const bodyLimit = options.http.maxBodyBytes ?? 64 * 1024;
|
||||
if (
|
||||
!Number.isSafeInteger(bodyLimit) ||
|
||||
bodyLimit < 1024 ||
|
||||
bodyLimit > 64 * 1024
|
||||
) {
|
||||
throw new RangeError(
|
||||
'Worker ingress body limit must be between 1 KiB and 64 KiB',
|
||||
);
|
||||
}
|
||||
|
||||
let database: PostgresDatabaseResource | undefined;
|
||||
const http = await startClusterControlHttpSurface({
|
||||
...options.http,
|
||||
maxBodyBytes: bodyLimit,
|
||||
maxResponseBytes: Math.min(
|
||||
options.http.maxResponseBytes ?? MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES,
|
||||
MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES,
|
||||
),
|
||||
maxInFlightRequests: Math.min(options.http.maxInFlightRequests ?? 64, 256),
|
||||
});
|
||||
let disposeAdmission: ClusterControlAdmissionDisposer | undefined;
|
||||
try {
|
||||
database = await options.openDatabase();
|
||||
const assembly = await options.create({
|
||||
database,
|
||||
workerCredentialPepper: options.workerCredentialPepper!,
|
||||
});
|
||||
disposeAdmission = http.installAdmission(
|
||||
assembly.evidence,
|
||||
assembly.pipeline,
|
||||
);
|
||||
let stopPromise: Promise<'stopped'> | undefined;
|
||||
return Object.freeze({
|
||||
status: 'active' as const,
|
||||
protocol: 'https' as const,
|
||||
transport: 'mutual-tls' as const,
|
||||
address: http.address,
|
||||
evidence: assembly.evidence,
|
||||
reloadTransport(mutualTls: ClusterControlMutualTlsOptions) {
|
||||
return http.reloadMutualTls(mutualTls);
|
||||
},
|
||||
stop() {
|
||||
stopPromise ??= (async () => {
|
||||
let primary: unknown;
|
||||
try {
|
||||
await disposeAdmission?.();
|
||||
} catch (error) {
|
||||
primary = error;
|
||||
}
|
||||
try {
|
||||
await database?.close();
|
||||
} catch (error) {
|
||||
primary ??= error;
|
||||
}
|
||||
try {
|
||||
await http.close();
|
||||
} catch (error) {
|
||||
primary ??= error;
|
||||
}
|
||||
if (primary) throw primary;
|
||||
return 'stopped' as const;
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await disposeAdmission?.();
|
||||
} catch {
|
||||
/* preserve root */
|
||||
}
|
||||
try {
|
||||
await database?.close();
|
||||
} catch {
|
||||
/* preserve root */
|
||||
}
|
||||
try {
|
||||
await http.close();
|
||||
} catch {
|
||||
/* preserve root */
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export * from './workerCredentialAuthenticator';
|
||||
export * from './workerIngressPipeline';
|
||||
export * from '../remote-execution/remoteRunActivationService';
|
||||
export * from '../remote-execution/remoteWorkerSecretDeliveryService';
|
||||
export * from '../remote-execution/remoteWorkerCompletionService';
|
||||
export * from '../remote-execution/remoteWorkerLeaseControlService';
|
||||
@@ -0,0 +1,864 @@
|
||||
// Cluster Control Worker Ingress boundary; keep fail-closed deployment configuration explicit.
|
||||
import {
|
||||
createPrivateKey,
|
||||
createPublicKey,
|
||||
timingSafeEqual,
|
||||
X509Certificate,
|
||||
type KeyObject,
|
||||
} from 'node:crypto';
|
||||
import { constants } from 'node:fs';
|
||||
import { open } from 'node:fs/promises';
|
||||
import { isAbsolute } from 'node:path';
|
||||
import type {
|
||||
DeploymentProfile,
|
||||
OpenPostgresDatabase,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
createPostgresDatabaseOpener,
|
||||
isPostgresTlsDnsServername,
|
||||
loadPostgresConnectionEnvironment,
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
type PostgresConnectionOptions,
|
||||
type PostgresPoolOptions,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
import type {
|
||||
ClusterControlHttpSurfaceOptions,
|
||||
ClusterControlMutualTlsOptions,
|
||||
} from '../transport/httpSurface';
|
||||
|
||||
export type ClusterWorkerIngressEnvironment = Readonly<
|
||||
Record<string, string | undefined>
|
||||
>;
|
||||
|
||||
export interface DisabledClusterWorkerIngressConfig {
|
||||
readonly enabled: false;
|
||||
readonly profile: DeploymentProfile;
|
||||
}
|
||||
|
||||
export interface ClusterWorkerArtifactS3Config {
|
||||
readonly bucket: string;
|
||||
readonly region: string;
|
||||
readonly prefix?: string;
|
||||
readonly expectedBucketOwner?: string;
|
||||
readonly endpoint?: string;
|
||||
readonly forcePathStyle: boolean;
|
||||
readonly encryption:
|
||||
| Readonly<{ readonly mode: 's3' }>
|
||||
| Readonly<{ readonly mode: 'kms'; readonly keyId: string }>;
|
||||
}
|
||||
|
||||
export interface ClusterWorkerMountedSecretConfig {
|
||||
readonly provider: 'mounted-files';
|
||||
readonly rootDirectory: string;
|
||||
}
|
||||
|
||||
export interface EnabledClusterWorkerIngressConfig {
|
||||
readonly enabled: true;
|
||||
readonly profile: 'cluster-control';
|
||||
readonly http: Omit<ClusterControlHttpSurfaceOptions, 'mutualTls'>;
|
||||
readonly transport: Readonly<{
|
||||
readonly privateKeyFile: string;
|
||||
readonly certificateFile: string;
|
||||
readonly clientCertificateAuthorityFile: string;
|
||||
readonly clientCertificateRevocationListFile?: string;
|
||||
}>;
|
||||
readonly database: Readonly<{
|
||||
readonly connection: PostgresConnectionOptions;
|
||||
readonly pool: PostgresPoolOptions;
|
||||
}>;
|
||||
readonly security: Readonly<{
|
||||
readonly workerCredentialPepper: string;
|
||||
}>;
|
||||
readonly artifact: Readonly<ClusterWorkerArtifactS3Config>;
|
||||
readonly secret?: Readonly<ClusterWorkerMountedSecretConfig>;
|
||||
}
|
||||
|
||||
export type ClusterWorkerIngressConfig =
|
||||
| DisabledClusterWorkerIngressConfig
|
||||
| EnabledClusterWorkerIngressConfig;
|
||||
|
||||
export class ClusterWorkerIngressConfigError extends TypeError {
|
||||
constructor(message: string) {
|
||||
super(`Worker ingress configuration is invalid: ${message}`);
|
||||
this.name = 'ClusterWorkerIngressConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
const PROFILES = new Set<DeploymentProfile>([
|
||||
'edge',
|
||||
'standalone',
|
||||
'cluster-control',
|
||||
'worker',
|
||||
]);
|
||||
const MAX_TLS_FILE_BYTES = 1024 * 1024;
|
||||
|
||||
function booleanValue(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
name: string,
|
||||
defaultValue: boolean,
|
||||
): boolean {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return defaultValue;
|
||||
if (value === 'true') return true;
|
||||
if (value === 'false') return false;
|
||||
throw new ClusterWorkerIngressConfigError(`${name} must be true or false`);
|
||||
}
|
||||
|
||||
function integerValue(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
name: string,
|
||||
defaultValue: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return defaultValue;
|
||||
if (!/^\d+$/.test(value)) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} must be an integer`);
|
||||
}
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
`${name} must be between ${minimum} and ${maximum}`,
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function boundedValue(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
name: string,
|
||||
maximumLength: number,
|
||||
required = false,
|
||||
): string | undefined {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') {
|
||||
if (required) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} is required`);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
if (value.length > maximumLength || /[\0\r\n]/.test(value)) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function deploymentProfile(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): DeploymentProfile {
|
||||
const value = environment.QL_DEPLOYMENT_PROFILE ?? 'standalone';
|
||||
if (!PROFILES.has(value as DeploymentProfile)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL_DEPLOYMENT_PROFILE is invalid',
|
||||
);
|
||||
}
|
||||
return value as DeploymentProfile;
|
||||
}
|
||||
|
||||
function absoluteFile(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
name: string,
|
||||
): string {
|
||||
const value = boundedValue(environment, name, 4096, true)!;
|
||||
if (!isAbsolute(value)) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} must be absolute`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function optionalAbsoluteFile(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
name: string,
|
||||
): string | undefined {
|
||||
const value = boundedValue(environment, name, 4096);
|
||||
if (value === undefined) return undefined;
|
||||
if (!isAbsolute(value)) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} must be absolute`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function workerCredentialPepper(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): string {
|
||||
const value = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_CREDENTIAL_PEPPER',
|
||||
64,
|
||||
true,
|
||||
)!;
|
||||
if (!/^[A-Za-z0-9_-]{43}$/.test(value)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_CREDENTIAL_PEPPER must be canonical base64url for 32 bytes',
|
||||
);
|
||||
}
|
||||
const decoded = Buffer.from(value, 'base64url');
|
||||
const canonical =
|
||||
decoded.byteLength === 32 && decoded.toString('base64url') === value;
|
||||
decoded.fill(0);
|
||||
if (!canonical) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_CREDENTIAL_PEPPER must be canonical base64url for 32 bytes',
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function databaseConnection(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): PostgresConnectionOptions {
|
||||
let connection: PostgresConnectionOptions;
|
||||
try {
|
||||
connection = loadPostgresConnectionEnvironment(environment, {
|
||||
connectionString: 'QL3_POSTGRES_WORKER_INGRESS_URL',
|
||||
host: 'QL3_POSTGRES_WORKER_INGRESS_HOST',
|
||||
port: 'QL3_POSTGRES_WORKER_INGRESS_PORT',
|
||||
database: 'QL3_POSTGRES_WORKER_INGRESS_DATABASE',
|
||||
user: 'QL3_POSTGRES_WORKER_INGRESS_USER',
|
||||
password: 'QL3_POSTGRES_WORKER_INGRESS_PASSWORD',
|
||||
});
|
||||
} catch (error) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: 'PostgreSQL Worker ingress connection is invalid',
|
||||
);
|
||||
}
|
||||
const mode =
|
||||
environment.QL3_WORKER_INGRESS_POSTGRES_TLS_MODE ?? 'verify-full';
|
||||
if (mode !== 'verify-full' && mode !== 'disable') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_MODE must be verify-full or disable',
|
||||
);
|
||||
}
|
||||
if (
|
||||
mode === 'disable' &&
|
||||
!booleanValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_ALLOW_INSECURE',
|
||||
false,
|
||||
)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'disabling PostgreSQL TLS requires QL3_WORKER_INGRESS_POSTGRES_ALLOW_INSECURE=true',
|
||||
);
|
||||
}
|
||||
const servername = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_SERVERNAME',
|
||||
253,
|
||||
);
|
||||
if (mode === 'verify-full' && !isPostgresTlsDnsServername(servername)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_SERVERNAME must be an explicit DNS name for verify-full',
|
||||
);
|
||||
}
|
||||
const certificateAuthorityFile = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_CA_FILE',
|
||||
4096,
|
||||
);
|
||||
if (mode === 'disable' && certificateAuthorityFile !== undefined) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_CA_FILE cannot be used when TLS is disabled',
|
||||
);
|
||||
}
|
||||
let certificateAuthority: string | undefined;
|
||||
if (certificateAuthorityFile !== undefined) {
|
||||
try {
|
||||
certificateAuthority = loadPostgresCertificateAuthorityFile(
|
||||
certificateAuthorityFile,
|
||||
);
|
||||
} catch {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_CA_FILE must contain a bounded trusted CA bundle',
|
||||
);
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
...connection,
|
||||
tls:
|
||||
mode === 'disable'
|
||||
? Object.freeze({ mode: 'disable' as const })
|
||||
: Object.freeze({
|
||||
mode: 'verify-full' as const,
|
||||
...(certificateAuthority === undefined
|
||||
? {}
|
||||
: { ca: certificateAuthority }),
|
||||
servername: servername!,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function workerArtifactS3(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): Readonly<ClusterWorkerArtifactS3Config> {
|
||||
const bucket = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_BUCKET',
|
||||
63,
|
||||
true,
|
||||
)!;
|
||||
if (
|
||||
!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) ||
|
||||
bucket.includes('..') ||
|
||||
/^\d{1,3}(?:\.\d{1,3}){3}$/.test(bucket)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_BUCKET is invalid',
|
||||
);
|
||||
}
|
||||
const region = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_REGION',
|
||||
63,
|
||||
true,
|
||||
)!;
|
||||
if (!/^[a-z0-9][a-z0-9-]{0,62}$/.test(region)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_REGION is invalid',
|
||||
);
|
||||
}
|
||||
const prefix = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_PREFIX',
|
||||
255,
|
||||
);
|
||||
if (
|
||||
prefix !== undefined &&
|
||||
(
|
||||
!/^[A-Za-z0-9][A-Za-z0-9/_=-]{0,254}$/.test(prefix) ||
|
||||
prefix.startsWith('/') ||
|
||||
prefix.endsWith('/') ||
|
||||
prefix.includes('//') ||
|
||||
prefix.split('/').some((segment) => segment === '.' || segment === '..')
|
||||
)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_PREFIX is invalid',
|
||||
);
|
||||
}
|
||||
const expectedBucketOwner = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_EXPECTED_BUCKET_OWNER',
|
||||
12,
|
||||
);
|
||||
if (
|
||||
expectedBucketOwner !== undefined &&
|
||||
!/^\d{12}$/.test(expectedBucketOwner)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_EXPECTED_BUCKET_OWNER must be 12 digits',
|
||||
);
|
||||
}
|
||||
const endpointValue = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_ENDPOINT',
|
||||
2048,
|
||||
);
|
||||
let endpoint: string | undefined;
|
||||
if (endpointValue !== undefined) {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(endpointValue);
|
||||
} catch {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_ENDPOINT is invalid',
|
||||
);
|
||||
}
|
||||
const allowInsecure = booleanValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_ALLOW_INSECURE',
|
||||
false,
|
||||
);
|
||||
if (
|
||||
(parsed.protocol !== 'https:' &&
|
||||
!(parsed.protocol === 'http:' && allowInsecure)) ||
|
||||
parsed.username !== '' ||
|
||||
parsed.password !== '' ||
|
||||
parsed.search !== '' ||
|
||||
parsed.hash !== '' ||
|
||||
parsed.pathname !== '/'
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_ENDPOINT must be an origin URL; HTTP requires explicit insecure opt-in',
|
||||
);
|
||||
}
|
||||
endpoint = parsed.origin;
|
||||
}
|
||||
const encryptionMode =
|
||||
boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_ENCRYPTION',
|
||||
3,
|
||||
) ?? 's3';
|
||||
if (encryptionMode !== 's3' && encryptionMode !== 'kms') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_ENCRYPTION must be s3 or kms',
|
||||
);
|
||||
}
|
||||
const keyId = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_KMS_KEY_ID',
|
||||
2048,
|
||||
);
|
||||
if (
|
||||
(encryptionMode === 'kms' && keyId === undefined) ||
|
||||
(encryptionMode === 's3' && keyId !== undefined)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_KMS_KEY_ID must be present exactly for kms encryption',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
bucket,
|
||||
region,
|
||||
...(prefix === undefined ? {} : { prefix }),
|
||||
...(expectedBucketOwner === undefined
|
||||
? {}
|
||||
: { expectedBucketOwner }),
|
||||
...(endpoint === undefined ? {} : { endpoint }),
|
||||
forcePathStyle: booleanValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_FORCE_PATH_STYLE',
|
||||
false,
|
||||
),
|
||||
encryption:
|
||||
encryptionMode === 's3'
|
||||
? Object.freeze({ mode: 's3' as const })
|
||||
: Object.freeze({ mode: 'kms' as const, keyId: keyId! }),
|
||||
});
|
||||
}
|
||||
|
||||
function workerSecret(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): Readonly<ClusterWorkerMountedSecretConfig> | undefined {
|
||||
const provider = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_SECRET_PROVIDER',
|
||||
32,
|
||||
);
|
||||
if (provider === undefined || provider === 'disabled') return undefined;
|
||||
if (provider !== 'mounted-files') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_SECRET_PROVIDER must be disabled or mounted-files',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
provider,
|
||||
rootDirectory: absoluteFile(
|
||||
environment,
|
||||
'QL3_WORKER_SECRET_ROOT_DIRECTORY',
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the Profile gate before reading database, Worker secret or TLS file
|
||||
* configuration. Disabled edge/standalone installs therefore remain free of
|
||||
* Worker ingress credential and filesystem requirements.
|
||||
*/
|
||||
export function loadClusterWorkerIngressConfig(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): ClusterWorkerIngressConfig {
|
||||
if (
|
||||
!environment ||
|
||||
typeof environment !== 'object' ||
|
||||
Array.isArray(environment)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError('environment must be an object');
|
||||
}
|
||||
const profile = deploymentProfile(environment);
|
||||
const enabled = booleanValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_ENABLED',
|
||||
false,
|
||||
);
|
||||
if (!enabled) return Object.freeze({ enabled: false, profile });
|
||||
if (profile !== 'cluster-control') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'enabled ingress requires QL_DEPLOYMENT_PROFILE=cluster-control',
|
||||
);
|
||||
}
|
||||
|
||||
const applicationName =
|
||||
boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_APPLICATION_NAME',
|
||||
63,
|
||||
) ?? 'qinglong-worker-ingress';
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,62}$/.test(applicationName)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_INGRESS_POSTGRES_APPLICATION_NAME is invalid',
|
||||
);
|
||||
}
|
||||
const host =
|
||||
boundedValue(environment, 'QL3_WORKER_INGRESS_HOST', 253) ?? '0.0.0.0';
|
||||
const secret = workerSecret(environment);
|
||||
|
||||
return Object.freeze({
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
http: Object.freeze({
|
||||
host,
|
||||
port: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_PORT',
|
||||
5801,
|
||||
1,
|
||||
65_535,
|
||||
),
|
||||
maxBodyBytes: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_MAX_BODY_BYTES',
|
||||
64 * 1024,
|
||||
1024,
|
||||
64 * 1024,
|
||||
),
|
||||
maxResponseBytes: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_MAX_RESPONSE_BYTES',
|
||||
64 * 1024,
|
||||
1024,
|
||||
64 * 1024,
|
||||
),
|
||||
maxInFlightRequests: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_MAX_IN_FLIGHT',
|
||||
64,
|
||||
1,
|
||||
256,
|
||||
),
|
||||
authenticationRateWindowMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_AUTH_RATE_WINDOW_MS',
|
||||
60_000,
|
||||
1_000,
|
||||
60 * 60_000,
|
||||
),
|
||||
authenticationRatePerPeer: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_AUTH_RATE_PER_PEER',
|
||||
120,
|
||||
1,
|
||||
1_000_000,
|
||||
),
|
||||
authenticationRateGlobal: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_AUTH_RATE_GLOBAL',
|
||||
1_200,
|
||||
1,
|
||||
1_000_000,
|
||||
),
|
||||
authenticationRateMaxPeers: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_AUTH_RATE_MAX_PEERS',
|
||||
4_096,
|
||||
1,
|
||||
65_536,
|
||||
),
|
||||
requestTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_REQUEST_TIMEOUT_MS',
|
||||
15_000,
|
||||
100,
|
||||
120_000,
|
||||
),
|
||||
drainTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_DRAIN_TIMEOUT_MS',
|
||||
10_000,
|
||||
100,
|
||||
120_000,
|
||||
),
|
||||
}),
|
||||
transport: Object.freeze({
|
||||
privateKeyFile: absoluteFile(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_TLS_PRIVATE_KEY_FILE',
|
||||
),
|
||||
certificateFile: absoluteFile(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_TLS_CERTIFICATE_FILE',
|
||||
),
|
||||
clientCertificateAuthorityFile: absoluteFile(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_TLS_CLIENT_CA_FILE',
|
||||
),
|
||||
...(() => {
|
||||
const clientCertificateRevocationListFile = optionalAbsoluteFile(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_TLS_CLIENT_CRL_FILE',
|
||||
);
|
||||
return clientCertificateRevocationListFile === undefined
|
||||
? {}
|
||||
: { clientCertificateRevocationListFile };
|
||||
})(),
|
||||
}),
|
||||
database: Object.freeze({
|
||||
connection: databaseConnection(environment),
|
||||
pool: Object.freeze({
|
||||
applicationName,
|
||||
maxConnections: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_MAX_CONNECTIONS',
|
||||
4,
|
||||
1,
|
||||
16,
|
||||
),
|
||||
connectionTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_CONNECTION_TIMEOUT_MS',
|
||||
5_000,
|
||||
100,
|
||||
60_000,
|
||||
),
|
||||
}),
|
||||
}),
|
||||
security: Object.freeze({
|
||||
workerCredentialPepper: workerCredentialPepper(environment),
|
||||
}),
|
||||
artifact: workerArtifactS3(environment),
|
||||
...(secret === undefined ? {} : { secret }),
|
||||
});
|
||||
}
|
||||
|
||||
async function readTlsFile(
|
||||
path: string,
|
||||
privateMaterial: boolean,
|
||||
): Promise<Buffer> {
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(path, constants.O_RDONLY);
|
||||
const stat = await handle.stat();
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.size < 1 ||
|
||||
stat.size > MAX_TLS_FILE_BYTES ||
|
||||
(privateMaterial && (stat.mode & 0o022) !== 0)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError('TLS file metadata is unsafe');
|
||||
}
|
||||
const bytes = await handle.readFile();
|
||||
if (bytes.byteLength < 1 || bytes.byteLength > MAX_TLS_FILE_BYTES) {
|
||||
bytes.fill(0);
|
||||
throw new ClusterWorkerIngressConfigError('TLS file size is unsafe');
|
||||
}
|
||||
return bytes;
|
||||
} catch (error) {
|
||||
if (error instanceof ClusterWorkerIngressConfigError) throw error;
|
||||
throw new ClusterWorkerIngressConfigError('TLS material is unavailable');
|
||||
} finally {
|
||||
await handle?.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
function activeCertificate(
|
||||
name: string,
|
||||
bytes: Buffer,
|
||||
now: number,
|
||||
): X509Certificate {
|
||||
let certificate: X509Certificate;
|
||||
try {
|
||||
certificate = new X509Certificate(bytes);
|
||||
} catch {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
`${name} is not an X.509 certificate`,
|
||||
);
|
||||
}
|
||||
const validFrom = Date.parse(certificate.validFrom);
|
||||
const validTo = Date.parse(certificate.validTo);
|
||||
if (
|
||||
!Number.isFinite(validFrom) ||
|
||||
!Number.isFinite(validTo) ||
|
||||
now < validFrom ||
|
||||
now >= validTo
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} is not currently valid`);
|
||||
}
|
||||
return certificate;
|
||||
}
|
||||
|
||||
function activeCertificateAuthorities(
|
||||
bytes: Buffer,
|
||||
now: number,
|
||||
): readonly Buffer[] {
|
||||
const pem = bytes.toString('utf8');
|
||||
const matches = pem.match(
|
||||
/-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g,
|
||||
);
|
||||
if (!matches || matches.length < 1 || matches.length > 16) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS client certificate authority bundle must contain 1 to 16 PEM certificates',
|
||||
);
|
||||
}
|
||||
const remainder = matches.reduce(
|
||||
(value, certificate) => value.replace(certificate, ''),
|
||||
pem,
|
||||
);
|
||||
if (remainder.trim() !== '') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS client certificate authority bundle contains unsupported data',
|
||||
);
|
||||
}
|
||||
const authorities: Buffer[] = [];
|
||||
try {
|
||||
for (const match of matches) {
|
||||
const authorityBytes = Buffer.from(`${match}\n`, 'utf8');
|
||||
const authority = activeCertificate(
|
||||
'TLS client certificate authority',
|
||||
authorityBytes,
|
||||
now,
|
||||
);
|
||||
if (!authority.ca) {
|
||||
authorityBytes.fill(0);
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS client certificate authority is not a CA',
|
||||
);
|
||||
}
|
||||
authorities.push(authorityBytes);
|
||||
}
|
||||
return Object.freeze(authorities);
|
||||
} catch (error) {
|
||||
for (const authority of authorities) authority.fill(0);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function certificateRevocationList(bytes: Buffer): Buffer {
|
||||
const value = bytes.toString('utf8').trim();
|
||||
if (
|
||||
!value.startsWith('-----BEGIN X509 CRL-----') ||
|
||||
!value.endsWith('-----END X509 CRL-----')
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS client certificate revocation list is not a PEM CRL',
|
||||
);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
function matchingPrivateKey(
|
||||
privateKey: KeyObject,
|
||||
certificate: X509Certificate,
|
||||
): boolean {
|
||||
const key = createPublicKey(privateKey).export({
|
||||
type: 'spki',
|
||||
format: 'der',
|
||||
});
|
||||
const certificateKey = certificate.publicKey.export({
|
||||
type: 'spki',
|
||||
format: 'der',
|
||||
});
|
||||
return (
|
||||
key.byteLength === certificateKey.byteLength &&
|
||||
timingSafeEqual(key, certificateKey)
|
||||
);
|
||||
}
|
||||
|
||||
export async function loadClusterWorkerIngressMutualTls(
|
||||
config: EnabledClusterWorkerIngressConfig,
|
||||
now: number = Date.now(),
|
||||
): Promise<ClusterControlMutualTlsOptions> {
|
||||
if (!config?.enabled || config.profile !== 'cluster-control') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS material requires an enabled Worker ingress config',
|
||||
);
|
||||
}
|
||||
if (!Number.isSafeInteger(now) || now < 0) {
|
||||
throw new ClusterWorkerIngressConfigError('observation time is invalid');
|
||||
}
|
||||
const keyBytes = await readTlsFile(config.transport.privateKeyFile, true);
|
||||
let certificateBytes: Buffer | undefined;
|
||||
let clientAuthorityBundleBytes: Buffer | undefined;
|
||||
let certificateRevocationListBytes: Buffer | undefined;
|
||||
let clientCertificateAuthorities: readonly Buffer[] = Object.freeze([]);
|
||||
try {
|
||||
let privateKey: KeyObject;
|
||||
try {
|
||||
privateKey = createPrivateKey(keyBytes);
|
||||
} catch {
|
||||
throw new ClusterWorkerIngressConfigError('TLS private key is invalid');
|
||||
}
|
||||
certificateBytes = await readTlsFile(
|
||||
config.transport.certificateFile,
|
||||
false,
|
||||
);
|
||||
clientAuthorityBundleBytes = await readTlsFile(
|
||||
config.transport.clientCertificateAuthorityFile,
|
||||
false,
|
||||
);
|
||||
const certificate = activeCertificate(
|
||||
'TLS server certificate',
|
||||
certificateBytes,
|
||||
now,
|
||||
);
|
||||
clientCertificateAuthorities = activeCertificateAuthorities(
|
||||
clientAuthorityBundleBytes,
|
||||
now,
|
||||
);
|
||||
if (!matchingPrivateKey(privateKey, certificate)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS private key does not match the server certificate',
|
||||
);
|
||||
}
|
||||
if (config.transport.clientCertificateRevocationListFile !== undefined) {
|
||||
certificateRevocationListBytes = certificateRevocationList(
|
||||
await readTlsFile(
|
||||
config.transport.clientCertificateRevocationListFile,
|
||||
false,
|
||||
),
|
||||
);
|
||||
}
|
||||
const mutualTls: ClusterControlMutualTlsOptions = Object.freeze({
|
||||
privateKey: keyBytes,
|
||||
certificateChain: certificateBytes,
|
||||
clientCertificateAuthorities,
|
||||
...(certificateRevocationListBytes === undefined
|
||||
? {}
|
||||
: {
|
||||
certificateRevocationLists: Object.freeze([
|
||||
certificateRevocationListBytes,
|
||||
]),
|
||||
}),
|
||||
});
|
||||
clientAuthorityBundleBytes.fill(0);
|
||||
return mutualTls;
|
||||
} catch (error) {
|
||||
keyBytes.fill(0);
|
||||
certificateBytes?.fill(0);
|
||||
clientAuthorityBundleBytes?.fill(0);
|
||||
certificateRevocationListBytes?.fill(0);
|
||||
for (const authority of clientCertificateAuthorities) authority.fill(0);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function createClusterWorkerIngressHttpOptions(
|
||||
config: EnabledClusterWorkerIngressConfig,
|
||||
now: number = Date.now(),
|
||||
): Promise<ClusterControlHttpSurfaceOptions> {
|
||||
const mutualTls = await loadClusterWorkerIngressMutualTls(config, now);
|
||||
return Object.freeze({ ...config.http, mutualTls });
|
||||
}
|
||||
|
||||
export function createClusterWorkerIngressDatabaseOpener(
|
||||
config: EnabledClusterWorkerIngressConfig,
|
||||
onPoolError: (error: Error) => void,
|
||||
): OpenPostgresDatabase {
|
||||
if (!config?.enabled || config.profile !== 'cluster-control') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'database opener requires an enabled Worker ingress config',
|
||||
);
|
||||
}
|
||||
if (typeof onPoolError !== 'function') {
|
||||
throw new ClusterWorkerIngressConfigError('onPoolError must be a function');
|
||||
}
|
||||
return createPostgresDatabaseOpener({
|
||||
role: 'worker-ingress',
|
||||
connection: config.database.connection,
|
||||
pool: config.database.pool,
|
||||
onPoolError,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,584 @@
|
||||
// Cluster Control Worker Ingress boundary; keep authenticated admission routing explicit.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
WorkerSessionConflictError,
|
||||
WorkerSessionFenceRejectedError,
|
||||
} from '@qinglong/runtime-core';
|
||||
import type {
|
||||
AuthenticatedWorkerSessionRepository,
|
||||
} from '@qinglong/runtime-core/worker-credential-delivery';
|
||||
import {
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
WorkerCredentialDeliveryUnavailableError,
|
||||
} from '@qinglong/runtime-core/worker-credential-delivery';
|
||||
import {
|
||||
InvalidWorkerSessionTransportError,
|
||||
createWorkerSessionHeartbeatResponseBody,
|
||||
createWorkerSessionRegisterResponseBody,
|
||||
createWorkerSessionTransitionResponseBody,
|
||||
parseWorkerSessionHeartbeatRequestBody,
|
||||
parseWorkerSessionRegisterRequestBody,
|
||||
parseWorkerSessionTransitionRequestBody,
|
||||
} from '@qinglong/runtime-core/worker-session-transport';
|
||||
import {
|
||||
WorkerExecutionAttestationFenceRejectedError,
|
||||
WorkerExecutionAttestationUnavailableError,
|
||||
type WorkerExecutionAttestationRepository,
|
||||
} from '@qinglong/runtime-core/worker-attestation';
|
||||
import {
|
||||
WorkerCredentialUnavailableError,
|
||||
} from '@qinglong/runtime-core/worker-credential';
|
||||
import {
|
||||
normalizeSecurityAuditRecord,
|
||||
type SecurityAuditSink,
|
||||
} from '@qinglong/runtime-core/security-audit';
|
||||
import type {
|
||||
ClusterControlAdmissionMetadata,
|
||||
ClusterControlAdmissionPipeline,
|
||||
ClusterControlAdmissionResponse,
|
||||
ClusterControlStreamingAdmissionBody,
|
||||
} from '../transport/httpSurface';
|
||||
import type {
|
||||
AuthenticatedWorkerPrincipal,
|
||||
WorkerCredentialAuthenticator,
|
||||
} from './workerCredentialAuthenticator';
|
||||
import {
|
||||
ClusterRemoteWorkerOfferFenceRejectedError,
|
||||
type ClusterRemoteWorkerOfferClaimService,
|
||||
} from '../remote-execution/remoteWorkerDispatcher';
|
||||
import {
|
||||
RemoteRunActivationFenceRejectedError,
|
||||
RemoteRunActivationUnavailableError,
|
||||
} from '@qinglong/runtime-core/remote-activation';
|
||||
import {
|
||||
createRemoteRunActivationResponseBody,
|
||||
InvalidRemoteRunActivationDeliveryError,
|
||||
} from '@qinglong/runtime-core/remote-activation-delivery';
|
||||
import type { ClusterRemoteRunActivationService } from '../remote-execution/remoteRunActivationService';
|
||||
import {
|
||||
createRemoteExecutionOfferPullBody,
|
||||
InvalidRemoteExecutionOfferDeliveryError,
|
||||
} from '@qinglong/runtime-core/remote-offer-delivery';
|
||||
import {
|
||||
createRemoteWorkerSecretDeliveryResponseBody,
|
||||
InvalidRemoteWorkerSecretDeliveryError,
|
||||
REMOTE_SECRET_DELIVERY_SCHEMA,
|
||||
RemoteWorkerSecretDeliveryFenceRejectedError,
|
||||
RemoteWorkerSecretDeliveryUnavailableError,
|
||||
} from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import type { ClusterRemoteWorkerSecretDeliveryService } from '../remote-execution/remoteWorkerSecretDeliveryService';
|
||||
import {
|
||||
InvalidRemoteWorkerCompletionError,
|
||||
MAX_REMOTE_WORKER_ARTIFACT_BYTES,
|
||||
MAX_REMOTE_WORKER_ARTIFACT_HEADER_BYTES,
|
||||
REMOTE_WORKER_ARTIFACT_CONTENT_TYPE,
|
||||
RemoteWorkerCompletionFenceRejectedError,
|
||||
RemoteWorkerCompletionUnavailableError,
|
||||
createRemoteWorkerArtifactUploadResponseBody,
|
||||
createRemoteWorkerCompletionResponseBody,
|
||||
parseRemoteWorkerCompletionRequestBody,
|
||||
} from '@qinglong/runtime-core/remote-worker-completion';
|
||||
import type {
|
||||
ClusterRemoteWorkerArtifactService,
|
||||
ClusterRemoteWorkerCompletionService,
|
||||
} from '../remote-execution/remoteWorkerCompletionService';
|
||||
import {
|
||||
InvalidRemoteWorkerLeaseControlError,
|
||||
RemoteWorkerLeaseControlFenceRejectedError,
|
||||
RemoteWorkerLeaseControlUnavailableError,
|
||||
createRemoteWorkerLeaseControlResponseBody,
|
||||
parseRemoteWorkerLeaseControlRequestBody,
|
||||
} from '@qinglong/runtime-core/remote-worker-lease-control';
|
||||
import type { ClusterRemoteWorkerLeaseControlService } from '../remote-execution/remoteWorkerLeaseControlService';
|
||||
|
||||
export interface WorkerIngressPipelineOptions {
|
||||
readonly authenticator: WorkerCredentialAuthenticator;
|
||||
readonly workers: AuthenticatedWorkerSessionRepository;
|
||||
readonly attestations: WorkerExecutionAttestationRepository;
|
||||
readonly audit: SecurityAuditSink;
|
||||
readonly offers?: Pick<ClusterRemoteWorkerOfferClaimService, 'claimNext'>;
|
||||
readonly activation?: Pick<
|
||||
ClusterRemoteRunActivationService,
|
||||
'acknowledgeStarting' | 'acknowledgeRunning' | 'failStart'
|
||||
>;
|
||||
readonly secrets?: Pick<ClusterRemoteWorkerSecretDeliveryService, 'deliver'>;
|
||||
readonly artifacts?: Pick<ClusterRemoteWorkerArtifactService, 'upload'>;
|
||||
readonly completion?: Pick<ClusterRemoteWorkerCompletionService, 'complete'>;
|
||||
readonly leaseControl?: Pick<ClusterRemoteWorkerLeaseControlService, 'control'>;
|
||||
readonly now?: () => number;
|
||||
}
|
||||
|
||||
type Operation =
|
||||
| 'register'
|
||||
| 'heartbeat'
|
||||
| 'transition'
|
||||
| 'attestations'
|
||||
| 'offers'
|
||||
| 'starting'
|
||||
| 'running'
|
||||
| 'start-failure'
|
||||
| 'secrets'
|
||||
| 'artifacts'
|
||||
| 'completion'
|
||||
| 'lease-control';
|
||||
|
||||
interface ResolvedRoute {
|
||||
readonly workerId: string;
|
||||
readonly sessionId: string;
|
||||
readonly operation: Operation;
|
||||
}
|
||||
|
||||
const ROUTE = /^\/api\/v3\/worker-ingress\/workers\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/sessions\/([0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\/(register|heartbeat|transition|attestations|offers|starting|running|start-failure|secrets|artifacts|completion|lease-control)$/;
|
||||
|
||||
function failure(statusCode: number, code: string): Error {
|
||||
return Object.assign(new Error(code), { statusCode, code });
|
||||
}
|
||||
|
||||
function route(metadata: ClusterControlAdmissionMetadata): ResolvedRoute {
|
||||
if (metadata.method !== 'POST' || Object.keys(metadata.query).length !== 0) {
|
||||
throw failure(404, 'worker_route_not_found');
|
||||
}
|
||||
const match = ROUTE.exec(metadata.path);
|
||||
if (!match) throw failure(404, 'worker_route_not_found');
|
||||
return Object.freeze({
|
||||
workerId: match[1]!,
|
||||
sessionId: match[2]!,
|
||||
operation: match[3]! as Operation,
|
||||
});
|
||||
}
|
||||
|
||||
function objectBody(body: unknown | null, keys: readonly string[]): Record<string, unknown> {
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body)) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
const actual = Object.keys(body).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) throw failure(400, 'invalid_worker_request');
|
||||
return body as Record<string, unknown>;
|
||||
}
|
||||
|
||||
async function audit(
|
||||
sink: SecurityAuditSink,
|
||||
metadata: ClusterControlAdmissionMetadata,
|
||||
operation: Operation,
|
||||
principal: Readonly<AuthenticatedWorkerPrincipal> | null,
|
||||
outcome: 'authentication_rejected' | 'authentication_unavailable' | 'allowed',
|
||||
now: () => number,
|
||||
): Promise<void> {
|
||||
await sink.record(normalizeSecurityAuditRecord({
|
||||
eventId: randomUUID(),
|
||||
requestId: metadata.requestId,
|
||||
operationId: `worker.${operation}`,
|
||||
projectId: null,
|
||||
subject: principal ? { type: 'worker', id: principal.workerId } : null,
|
||||
authenticationId: principal?.authenticationId ?? null,
|
||||
outcome,
|
||||
reasons: [outcome === 'allowed' ? 'worker_credential' : outcome],
|
||||
fence: null,
|
||||
occurredAtMs: now(),
|
||||
}));
|
||||
}
|
||||
|
||||
function response(statusCode: number, body: unknown): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body });
|
||||
}
|
||||
|
||||
function mapIngressFailure(error: unknown): never {
|
||||
if (error && typeof error === 'object' && 'statusCode' in error) throw error;
|
||||
if (
|
||||
error instanceof WorkerSessionConflictError ||
|
||||
error instanceof WorkerSessionFenceRejectedError ||
|
||||
error instanceof WorkerCredentialDeliveryConflictError
|
||||
) throw failure(409, 'worker_session_fenced');
|
||||
if (error instanceof WorkerExecutionAttestationFenceRejectedError) {
|
||||
throw failure(409, 'worker_attestation_fenced');
|
||||
}
|
||||
if (error instanceof ClusterRemoteWorkerOfferFenceRejectedError) {
|
||||
throw failure(409, 'worker_offer_fenced');
|
||||
}
|
||||
if (error instanceof RemoteRunActivationFenceRejectedError) {
|
||||
throw failure(409, 'worker_activation_fenced');
|
||||
}
|
||||
if (error instanceof RemoteWorkerSecretDeliveryFenceRejectedError) {
|
||||
throw failure(409, 'worker_secret_delivery_fenced');
|
||||
}
|
||||
if (error instanceof RemoteWorkerCompletionFenceRejectedError) {
|
||||
throw failure(409, 'worker_completion_fenced');
|
||||
}
|
||||
if (error instanceof RemoteWorkerLeaseControlFenceRejectedError) {
|
||||
throw failure(409, 'worker_lease_control_fenced');
|
||||
}
|
||||
if (error instanceof InvalidRemoteWorkerCompletionError) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
if (error instanceof InvalidRemoteWorkerLeaseControlError) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
if (error instanceof InvalidWorkerSessionTransportError) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
if (
|
||||
error instanceof InvalidRemoteExecutionOfferDeliveryError ||
|
||||
error instanceof InvalidRemoteRunActivationDeliveryError ||
|
||||
error instanceof InvalidRemoteWorkerSecretDeliveryError
|
||||
) throw failure(503, 'worker_ingress_unavailable');
|
||||
if (
|
||||
error instanceof WorkerExecutionAttestationUnavailableError ||
|
||||
error instanceof WorkerCredentialUnavailableError ||
|
||||
error instanceof WorkerCredentialDeliveryUnavailableError ||
|
||||
error instanceof RemoteRunActivationUnavailableError ||
|
||||
error instanceof RemoteWorkerSecretDeliveryUnavailableError ||
|
||||
error instanceof RemoteWorkerCompletionUnavailableError ||
|
||||
error instanceof RemoteWorkerLeaseControlUnavailableError
|
||||
) throw failure(503, 'worker_ingress_unavailable');
|
||||
if (error instanceof TypeError || error instanceof RangeError) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
throw failure(503, 'worker_ingress_unavailable');
|
||||
}
|
||||
|
||||
export function createWorkerIngressAdmissionPipeline(
|
||||
options: WorkerIngressPipelineOptions,
|
||||
): ClusterControlAdmissionPipeline {
|
||||
if (
|
||||
!options ||
|
||||
typeof options.authenticator?.authenticate !== 'function' ||
|
||||
typeof options.workers?.register !== 'function' ||
|
||||
typeof options.workers?.heartbeatAuthenticated !== 'function' ||
|
||||
typeof options.workers?.transitionAuthenticated !== 'function' ||
|
||||
typeof options.attestations?.submit !== 'function' ||
|
||||
typeof options.audit?.record !== 'function' ||
|
||||
(options.offers !== undefined &&
|
||||
typeof options.offers.claimNext !== 'function') ||
|
||||
(options.activation !== undefined &&
|
||||
(typeof options.activation.acknowledgeStarting !== 'function' ||
|
||||
typeof options.activation.acknowledgeRunning !== 'function' ||
|
||||
typeof options.activation.failStart !== 'function')) ||
|
||||
(options.secrets !== undefined &&
|
||||
typeof options.secrets.deliver !== 'function') ||
|
||||
(options.artifacts !== undefined &&
|
||||
typeof options.artifacts.upload !== 'function') ||
|
||||
(options.completion !== undefined &&
|
||||
typeof options.completion.complete !== 'function') ||
|
||||
(options.leaseControl !== undefined &&
|
||||
typeof options.leaseControl.control !== 'function')
|
||||
) throw new TypeError('Worker ingress pipeline options are invalid');
|
||||
const now = options.now ?? Date.now;
|
||||
|
||||
return Object.freeze({
|
||||
async prepare(metadata: ClusterControlAdmissionMetadata) {
|
||||
const resolved = route(metadata);
|
||||
let principal: Readonly<AuthenticatedWorkerPrincipal> | null;
|
||||
try {
|
||||
principal = await options.authenticator.authenticate(metadata);
|
||||
} catch {
|
||||
try { await audit(options.audit, metadata, resolved.operation, null, 'authentication_unavailable', now); } catch { /* fail below */ }
|
||||
throw failure(503, 'worker_authentication_unavailable');
|
||||
}
|
||||
if (!principal || principal.workerId !== resolved.workerId) {
|
||||
try { await audit(options.audit, metadata, resolved.operation, null, 'authentication_rejected', now); } catch { throw failure(503, 'worker_audit_unavailable'); }
|
||||
throw failure(401, 'worker_authentication_required');
|
||||
}
|
||||
try {
|
||||
await audit(options.audit, metadata, resolved.operation, principal, 'allowed', now);
|
||||
} catch {
|
||||
throw failure(503, 'worker_audit_unavailable');
|
||||
}
|
||||
|
||||
if (resolved.operation === 'artifacts') {
|
||||
return Object.freeze({
|
||||
bodyMode: 'stream' as const,
|
||||
contentType: REMOTE_WORKER_ARTIFACT_CONTENT_TYPE,
|
||||
maximumBodyBytes:
|
||||
4 + MAX_REMOTE_WORKER_ARTIFACT_HEADER_BYTES +
|
||||
MAX_REMOTE_WORKER_ARTIFACT_BYTES,
|
||||
async handleStream(body: ClusterControlStreamingAdmissionBody) {
|
||||
try {
|
||||
if (!options.artifacts) {
|
||||
throw failure(503, 'worker_artifact_unavailable');
|
||||
}
|
||||
const receipt = await options.artifacts.upload({
|
||||
workerId: resolved.workerId,
|
||||
workerSessionId: resolved.sessionId,
|
||||
contentLength: body.contentLength,
|
||||
chunks: body.chunks,
|
||||
signal: metadata.signal,
|
||||
});
|
||||
return response(
|
||||
200,
|
||||
createRemoteWorkerArtifactUploadResponseBody(receipt),
|
||||
);
|
||||
} catch (error) {
|
||||
return mapIngressFailure(error);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
async handle(body: unknown | null) {
|
||||
try {
|
||||
if (resolved.operation === 'register') {
|
||||
const command = parseWorkerSessionRegisterRequestBody(body, {
|
||||
workerId: resolved.workerId,
|
||||
sessionId: resolved.sessionId,
|
||||
});
|
||||
const result = await options.workers.register(command);
|
||||
return response(
|
||||
200,
|
||||
createWorkerSessionRegisterResponseBody(result),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'heartbeat') {
|
||||
const command = parseWorkerSessionHeartbeatRequestBody(body, {
|
||||
workerId: resolved.workerId,
|
||||
sessionId: resolved.sessionId,
|
||||
});
|
||||
const worker = await options.workers.heartbeatAuthenticated(
|
||||
command,
|
||||
{
|
||||
workerId: principal.workerId,
|
||||
credentialId: principal.credentialId,
|
||||
credentialVersion: principal.credentialVersion,
|
||||
},
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
createWorkerSessionHeartbeatResponseBody(worker),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'transition') {
|
||||
const command = parseWorkerSessionTransitionRequestBody(body, {
|
||||
workerId: resolved.workerId,
|
||||
sessionId: resolved.sessionId,
|
||||
});
|
||||
const worker = await options.workers.transitionAuthenticated(
|
||||
command,
|
||||
{
|
||||
workerId: principal.workerId,
|
||||
credentialId: principal.credentialId,
|
||||
credentialVersion: principal.credentialVersion,
|
||||
},
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
createWorkerSessionTransitionResponseBody(worker),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'offers') {
|
||||
if (!options.offers) {
|
||||
throw failure(503, 'worker_offer_unavailable');
|
||||
}
|
||||
const value = objectBody(body, [
|
||||
'workerGeneration', 'offerId', 'leaseToken',
|
||||
]);
|
||||
const result = await options.offers.claimNext(
|
||||
{ workerId: resolved.workerId },
|
||||
{
|
||||
workerSessionId: resolved.sessionId,
|
||||
workerGeneration: value.workerGeneration as number,
|
||||
offerId: value.offerId as string,
|
||||
leaseToken: value.leaseToken as string,
|
||||
},
|
||||
);
|
||||
return response(200, createRemoteExecutionOfferPullBody(result));
|
||||
}
|
||||
if (
|
||||
resolved.operation === 'starting' ||
|
||||
resolved.operation === 'start-failure'
|
||||
) {
|
||||
if (!options.activation) {
|
||||
throw failure(503, 'worker_activation_unavailable');
|
||||
}
|
||||
const value = objectBody(body, [
|
||||
'runId', 'attemptId', 'workerGeneration', 'offerId',
|
||||
'leaseGeneration', 'leaseToken', 'expectedLeaseVersion',
|
||||
]);
|
||||
const command = {
|
||||
runId: value.runId as string,
|
||||
attemptId: value.attemptId as string,
|
||||
workerSessionId: resolved.sessionId,
|
||||
workerGeneration: value.workerGeneration as number,
|
||||
offerId: value.offerId as string,
|
||||
leaseGeneration: value.leaseGeneration as number,
|
||||
leaseToken: value.leaseToken as string,
|
||||
expectedLeaseVersion: value.expectedLeaseVersion as number,
|
||||
};
|
||||
const activation = resolved.operation === 'starting'
|
||||
? await options.activation.acknowledgeStarting(
|
||||
{ workerId: resolved.workerId }, command,
|
||||
)
|
||||
: await options.activation.failStart(
|
||||
{ workerId: resolved.workerId }, command,
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
createRemoteRunActivationResponseBody(activation),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'running') {
|
||||
if (!options.activation) {
|
||||
throw failure(503, 'worker_activation_unavailable');
|
||||
}
|
||||
const value = objectBody(body, [
|
||||
'runId', 'attemptId', 'workerGeneration', 'offerId',
|
||||
'leaseGeneration', 'leaseToken', 'expectedLeaseVersion',
|
||||
'executorHandle', 'logArtifactId', 'callbackSequence',
|
||||
'callbackTokenDigest',
|
||||
]);
|
||||
if (
|
||||
value.logArtifactId !== null &&
|
||||
typeof value.logArtifactId !== 'string'
|
||||
) throw failure(400, 'invalid_worker_request');
|
||||
const activation = await options.activation.acknowledgeRunning(
|
||||
{ workerId: resolved.workerId },
|
||||
{
|
||||
runId: value.runId as string,
|
||||
attemptId: value.attemptId as string,
|
||||
workerSessionId: resolved.sessionId,
|
||||
workerGeneration: value.workerGeneration as number,
|
||||
offerId: value.offerId as string,
|
||||
leaseGeneration: value.leaseGeneration as number,
|
||||
leaseToken: value.leaseToken as string,
|
||||
expectedLeaseVersion: value.expectedLeaseVersion as number,
|
||||
executorHandle: value.executorHandle as string,
|
||||
callbackSequence: value.callbackSequence as number,
|
||||
callbackTokenDigest: value.callbackTokenDigest as string,
|
||||
...(value.logArtifactId === null
|
||||
? {}
|
||||
: { logArtifactId: value.logArtifactId }),
|
||||
},
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
createRemoteRunActivationResponseBody(activation),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'secrets') {
|
||||
if (!options.secrets) {
|
||||
throw failure(503, 'worker_secret_delivery_unavailable');
|
||||
}
|
||||
const value = objectBody(body, [
|
||||
'schema', 'runId', 'attemptId', 'projectId', 'taskId',
|
||||
'taskRevision', 'executionDigest', 'workerGeneration',
|
||||
'offerId', 'leaseGeneration', 'leaseToken',
|
||||
'expectedLeaseVersion', 'secretRefs',
|
||||
]);
|
||||
if (value.schema !== REMOTE_SECRET_DELIVERY_SCHEMA) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
const delivered = await options.secrets.deliver(
|
||||
{ workerId: resolved.workerId },
|
||||
{
|
||||
workerSessionId: resolved.sessionId,
|
||||
workerGeneration: value.workerGeneration as number,
|
||||
runId: value.runId as string,
|
||||
attemptId: value.attemptId as string,
|
||||
projectId: value.projectId as string,
|
||||
taskId: value.taskId as string,
|
||||
taskRevision: value.taskRevision as string,
|
||||
executionDigest: value.executionDigest as string,
|
||||
offerId: value.offerId as string,
|
||||
leaseGeneration: value.leaseGeneration as number,
|
||||
leaseToken: value.leaseToken as string,
|
||||
expectedLeaseVersion: value.expectedLeaseVersion as number,
|
||||
secretRefs: value.secretRefs as string[],
|
||||
},
|
||||
);
|
||||
try {
|
||||
const responseBody = createRemoteWorkerSecretDeliveryResponseBody(
|
||||
delivered,
|
||||
value.secretRefs as string[],
|
||||
);
|
||||
if (
|
||||
responseBody.runId !== value.runId ||
|
||||
responseBody.attemptId !== value.attemptId ||
|
||||
responseBody.offerId !== value.offerId ||
|
||||
responseBody.executionDigest !== value.executionDigest
|
||||
) throw new InvalidRemoteWorkerSecretDeliveryError(
|
||||
'service response authority does not match request',
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
responseBody,
|
||||
);
|
||||
} finally {
|
||||
try { await delivered.dispose?.(); } catch { /* response remains valid */ }
|
||||
}
|
||||
}
|
||||
if (resolved.operation === 'completion') {
|
||||
if (!options.completion) {
|
||||
throw failure(503, 'worker_completion_unavailable');
|
||||
}
|
||||
const command = parseRemoteWorkerCompletionRequestBody(body, {
|
||||
workerId: resolved.workerId,
|
||||
workerSessionId: resolved.sessionId,
|
||||
});
|
||||
const completed = await options.completion.complete(
|
||||
command,
|
||||
metadata.signal,
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
createRemoteWorkerCompletionResponseBody(completed),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'lease-control') {
|
||||
if (!options.leaseControl) {
|
||||
throw failure(503, 'worker_lease_control_unavailable');
|
||||
}
|
||||
const command = parseRemoteWorkerLeaseControlRequestBody(body, {
|
||||
workerId: resolved.workerId,
|
||||
workerSessionId: resolved.sessionId,
|
||||
});
|
||||
return response(
|
||||
200,
|
||||
createRemoteWorkerLeaseControlResponseBody(
|
||||
await options.leaseControl.control(command),
|
||||
),
|
||||
);
|
||||
}
|
||||
const value = objectBody(body, [
|
||||
'attestationId', 'runId', 'attemptId', 'sequence', 'state',
|
||||
'workerGeneration', 'leaseTokenDigest', 'leaseGeneration',
|
||||
'leaseVersion', 'offerId', 'callbackSequence', 'executorHandle',
|
||||
'journalRevision',
|
||||
]);
|
||||
if (value.workerGeneration === undefined) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
const result = await options.attestations.submit({
|
||||
attestationId: value.attestationId as string,
|
||||
runId: value.runId as string,
|
||||
attemptId: value.attemptId as string,
|
||||
sequence: value.sequence as number,
|
||||
state: value.state as 'running' | 'stopped',
|
||||
workerId: resolved.workerId,
|
||||
workerSessionId: resolved.sessionId,
|
||||
workerGeneration: value.workerGeneration as number,
|
||||
leaseTokenDigest: value.leaseTokenDigest as string,
|
||||
leaseGeneration: value.leaseGeneration as number,
|
||||
leaseVersion: value.leaseVersion as number,
|
||||
offerId: value.offerId as string,
|
||||
callbackSequence: value.callbackSequence as number,
|
||||
executorHandle: value.executorHandle as string,
|
||||
journalRevision: value.journalRevision as number,
|
||||
});
|
||||
return response(result.status === 'created' ? 201 : 200, {
|
||||
attestationId: result.attestation.attestationId,
|
||||
sequence: result.attestation.sequence,
|
||||
state: result.attestation.state,
|
||||
receivedAtMs: result.attestation.receivedAtMs,
|
||||
replay: result.status === 'existing',
|
||||
});
|
||||
} catch (error) {
|
||||
return mapIngressFailure(error);
|
||||
}
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user