feat(ql3): establish 3.0 incubation baseline

packages/ql3-worker-runtime/src/credential/workerCertificateEnrollment.ts

@@ -0,0 +1,127 @@
+// Credential ownership: generate bounded Worker certificate enrollment material.
+import 'reflect-metadata';
+
+import { createHash, webcrypto } from 'node:crypto';
+import {
+  BasicConstraintsExtension,
+  ExtendedKeyUsage,
+  ExtendedKeyUsageExtension,
+  KeyUsageFlags,
+  KeyUsagesExtension,
+  PemConverter,
+  Pkcs10CertificateRequestGenerator,
+} from '@peculiar/x509';
+
+const WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+const KEY_ALGORITHM = Object.freeze({
+  name: 'ECDSA',
+  namedCurve: 'P-256',
+  hash: 'SHA-256',
+});
+const MAX_PRIVATE_KEY_BYTES = 16 * 1024;
+const MAX_CSR_BYTES = 16 * 1024;
+
+export interface GenerateWorkerCertificateEnrollmentOptions {
+  readonly workerId: string;
+}
+
+export interface WorkerCertificateEnrollmentMaterial {
+  readonly algorithm: 'ECDSA_P256_SHA256';
+  readonly workerId: string;
+  readonly privateKeyPem: Buffer;
+  readonly certificateSigningRequestPem: string;
+  readonly publicKeySpkiSha256: string;
+  dispose(): void;
+}
+
+export class WorkerCertificateEnrollmentError extends TypeError {
+  constructor(message: string) {
+    super(`Worker certificate enrollment is invalid: ${message}`);
+    this.name = 'WorkerCertificateEnrollmentError';
+  }
+}
+
+function assertWorkerId(workerId: string): void {
+  if (typeof workerId !== 'string' || !WORKER_ID_PATTERN.test(workerId)) {
+    throw new WorkerCertificateEnrollmentError('workerId is invalid');
+  }
+}
+
+/**
+ * Generates a Worker-local P-256 key and a PKCS#10 request. This function does
+ * not contact a CA, persist the key or grant any Worker authority.
+ */
+export async function generateWorkerCertificateEnrollment(
+  options: GenerateWorkerCertificateEnrollmentOptions,
+): Promise<WorkerCertificateEnrollmentMaterial> {
+  if (!options || typeof options !== 'object' || Array.isArray(options)) {
+    throw new WorkerCertificateEnrollmentError('options must be an object');
+  }
+  assertWorkerId(options.workerId);
+
+  let privateKeyPem: Buffer | undefined;
+  try {
+    const keys = await webcrypto.subtle.generateKey(KEY_ALGORITHM, true, [
+      'sign',
+      'verify',
+    ]);
+    const request = await Pkcs10CertificateRequestGenerator.create(
+      {
+        name: `CN=${options.workerId}`,
+        keys: keys as unknown as CryptoKeyPair,
+        signingAlgorithm: KEY_ALGORITHM,
+        extensions: [
+          new BasicConstraintsExtension(false, undefined, true),
+          new ExtendedKeyUsageExtension([ExtendedKeyUsage.clientAuth], true),
+          new KeyUsagesExtension(KeyUsageFlags.digitalSignature, true),
+        ],
+      },
+      webcrypto as unknown as Crypto,
+    );
+    if (!(await request.verify(webcrypto as unknown as Crypto))) {
+      throw new WorkerCertificateEnrollmentError(
+        'generated CSR signature is invalid',
+      );
+    }
+    const [privateKey, publicKey] = await Promise.all([
+      webcrypto.subtle.exportKey('pkcs8', keys.privateKey),
+      webcrypto.subtle.exportKey('spki', keys.publicKey),
+    ]);
+    privateKeyPem = Buffer.from(
+      PemConverter.encode(privateKey, 'PRIVATE KEY'),
+      'ascii',
+    );
+    const certificateSigningRequestPem = request.toString('pem');
+    if (
+      privateKeyPem.byteLength < 1 ||
+      privateKeyPem.byteLength > MAX_PRIVATE_KEY_BYTES ||
+      Buffer.byteLength(certificateSigningRequestPem) < 1 ||
+      Buffer.byteLength(certificateSigningRequestPem) > MAX_CSR_BYTES
+    ) {
+      throw new WorkerCertificateEnrollmentError(
+        'generated material exceeds its hard limit',
+      );
+    }
+    const publicKeySpkiSha256 = createHash('sha256')
+      .update(Buffer.from(publicKey))
+      .digest('hex');
+    let disposed = false;
+    const material: WorkerCertificateEnrollmentMaterial = {
+      algorithm: 'ECDSA_P256_SHA256',
+      workerId: options.workerId,
+      privateKeyPem,
+      certificateSigningRequestPem,
+      publicKeySpkiSha256,
+      dispose() {
+        if (disposed) return;
+        disposed = true;
+        privateKeyPem?.fill(0);
+      },
+    };
+    return Object.freeze(material);
+  } catch (error) {
+    privateKeyPem?.fill(0);
+    if (error instanceof WorkerCertificateEnrollmentError) throw error;
+    throw new WorkerCertificateEnrollmentError('key or CSR generation failed');
+  }
+}

packages/ql3-worker-runtime/src/credential/workerCertificateIdentity.ts

@@ -0,0 +1,289 @@
+// Credential ownership: validate Worker certificate identity and trust semantics.
+import {
+  createHash,
+  createPrivateKey,
+  createPublicKey,
+  timingSafeEqual,
+  X509Certificate,
+} from 'node:crypto';
+
+const CERTIFICATE_PATTERN =
+  /-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g;
+const CLIENT_AUTH_OID = '1.3.6.1.5.5.7.3.2';
+const MAX_CERTIFICATE_MATERIAL_BYTES = 1024 * 1024;
+const MAX_CERTIFICATES = 16;
+const SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/;
+
+export type WorkerCertificateIdentityFailureReason =
+  | 'invalid_material'
+  | 'key_mismatch'
+  | 'not_yet_valid'
+  | 'expired'
+  | 'insufficient_validity'
+  | 'not_client_auth'
+  | 'untrusted';
+
+export class WorkerCertificateIdentityError extends Error {
+  constructor(readonly reason: WorkerCertificateIdentityFailureReason) {
+    super(`Worker certificate identity is unavailable: ${reason}`);
+    this.name = 'WorkerCertificateIdentityError';
+  }
+}
+
+export interface ValidateWorkerCertificateIdentityInput {
+  readonly privateKeyPem: string | Buffer;
+  readonly certificateChainPem: string | Buffer;
+  readonly trustAnchors: readonly (string | Buffer)[];
+  readonly now?: number;
+  readonly minimumRemainingValidityMs?: number;
+}
+
+export interface WorkerCertificateIdentitySummary {
+  readonly certificateSha256: string;
+  readonly publicKeySpkiSha256: string;
+  readonly serialNumber: string;
+  readonly notBeforeMs: number;
+  readonly notAfterMs: number;
+}
+
+interface ParsedCertificate {
+  readonly certificate: X509Certificate;
+  readonly fingerprint: string;
+}
+
+function materialBytes(
+  value: string | Buffer,
+  privateMaterial = false,
+): Buffer {
+  if (typeof value !== 'string' && !Buffer.isBuffer(value)) {
+    throw new WorkerCertificateIdentityError('invalid_material');
+  }
+  const bytes = Buffer.isBuffer(value)
+    ? Buffer.from(value)
+    : Buffer.from(value, 'utf8');
+  if (
+    bytes.byteLength < 1 ||
+    bytes.byteLength > MAX_CERTIFICATE_MATERIAL_BYTES
+  ) {
+    bytes.fill(0);
+    throw new WorkerCertificateIdentityError('invalid_material');
+  }
+  if (privateMaterial && !bytes.includes(Buffer.from('PRIVATE KEY'))) {
+    bytes.fill(0);
+    throw new WorkerCertificateIdentityError('invalid_material');
+  }
+  return bytes;
+}
+
+function splitCertificates(
+  value: string | Buffer,
+  remaining: { count: number },
+): ParsedCertificate[] {
+  const bytes = materialBytes(value);
+  try {
+    const pem = bytes.toString('utf8');
+    const matches = pem.match(CERTIFICATE_PATTERN);
+    if (!matches || matches.length === 0) {
+      throw new WorkerCertificateIdentityError('invalid_material');
+    }
+    const remainder = matches.reduce(
+      (candidate, match) => candidate.replace(match, ''),
+      pem,
+    );
+    remaining.count -= matches.length;
+    if (remainder.trim() !== '' || remaining.count < 0) {
+      throw new WorkerCertificateIdentityError('invalid_material');
+    }
+    return matches.map((match) => {
+      try {
+        const certificate = new X509Certificate(`${match}\n`);
+        return {
+          certificate,
+          fingerprint: createHash('sha256')
+            .update(certificate.raw)
+            .digest('hex'),
+        };
+      } catch {
+        throw new WorkerCertificateIdentityError('invalid_material');
+      }
+    });
+  } finally {
+    bytes.fill(0);
+  }
+}
+
+function certificateTime(
+  certificate: X509Certificate,
+  now: number,
+): { notBeforeMs: number; notAfterMs: number } {
+  const notBeforeMs = Date.parse(certificate.validFrom);
+  const notAfterMs = Date.parse(certificate.validTo);
+  if (!Number.isFinite(notBeforeMs) || !Number.isFinite(notAfterMs)) {
+    throw new WorkerCertificateIdentityError('invalid_material');
+  }
+  if (now < notBeforeMs) {
+    throw new WorkerCertificateIdentityError('not_yet_valid');
+  }
+  if (now >= notAfterMs) {
+    throw new WorkerCertificateIdentityError('expired');
+  }
+  return { notBeforeMs, notAfterMs };
+}
+
+function signedBy(
+  certificate: X509Certificate,
+  issuer: X509Certificate,
+): boolean {
+  try {
+    return (
+      certificate.checkIssued(issuer) && certificate.verify(issuer.publicKey)
+    );
+  } catch {
+    return false;
+  }
+}
+
+function reachesTrustAnchor(
+  certificate: ParsedCertificate,
+  intermediates: readonly ParsedCertificate[],
+  anchors: readonly ParsedCertificate[],
+  visited: ReadonlySet<string>,
+): boolean {
+  if (
+    visited.has(certificate.fingerprint) ||
+    visited.size >= MAX_CERTIFICATES
+  ) {
+    return false;
+  }
+  const nextVisited = new Set(visited);
+  nextVisited.add(certificate.fingerprint);
+  for (const anchor of anchors) {
+    if (signedBy(certificate.certificate, anchor.certificate)) return true;
+  }
+  for (const intermediate of intermediates) {
+    if (
+      !nextVisited.has(intermediate.fingerprint) &&
+      signedBy(certificate.certificate, intermediate.certificate) &&
+      reachesTrustAnchor(intermediate, intermediates, anchors, nextVisited)
+    ) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function matchingPublicKey(
+  privateKeyPem: Buffer,
+  certificate: X509Certificate,
+): string {
+  let privateKey;
+  try {
+    privateKey = createPrivateKey(privateKeyPem);
+  } catch {
+    throw new WorkerCertificateIdentityError('invalid_material');
+  }
+  const key = createPublicKey(privateKey).export({
+    type: 'spki',
+    format: 'der',
+  });
+  const certificateKey = certificate.publicKey.export({
+    type: 'spki',
+    format: 'der',
+  });
+  if (
+    key.byteLength !== certificateKey.byteLength ||
+    !timingSafeEqual(key, certificateKey)
+  ) {
+    throw new WorkerCertificateIdentityError('key_mismatch');
+  }
+  return createHash('sha256').update(key).digest('hex');
+}
+
+export function assertWorkerCertificateIdentitySummary(
+  value: WorkerCertificateIdentitySummary,
+): void {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    !SHA256_HEX_PATTERN.test(value.certificateSha256) ||
+    !SHA256_HEX_PATTERN.test(value.publicKeySpkiSha256) ||
+    typeof value.serialNumber !== 'string' ||
+    !/^[A-Fa-f0-9]{1,128}$/.test(value.serialNumber) ||
+    !Number.isSafeInteger(value.notBeforeMs) ||
+    !Number.isSafeInteger(value.notAfterMs) ||
+    value.notBeforeMs < 0 ||
+    value.notAfterMs <= value.notBeforeMs
+  ) {
+    throw new WorkerCertificateIdentityError('invalid_material');
+  }
+}
+
+/** Validates key possession, client-auth intent, validity and a bounded chain. */
+export function validateWorkerCertificateIdentity(
+  input: ValidateWorkerCertificateIdentityInput,
+): WorkerCertificateIdentitySummary {
+  if (!input || typeof input !== 'object' || Array.isArray(input)) {
+    throw new WorkerCertificateIdentityError('invalid_material');
+  }
+  const now = input.now ?? Date.now();
+  const minimumRemainingValidityMs = input.minimumRemainingValidityMs ?? 0;
+  if (
+    !Number.isSafeInteger(now) ||
+    now < 0 ||
+    !Number.isSafeInteger(minimumRemainingValidityMs) ||
+    minimumRemainingValidityMs < 0 ||
+    minimumRemainingValidityMs > 365 * 24 * 60 * 60_000
+  ) {
+    throw new WorkerCertificateIdentityError('invalid_material');
+  }
+  if (
+    !Array.isArray(input.trustAnchors) ||
+    input.trustAnchors.length < 1 ||
+    input.trustAnchors.length > MAX_CERTIFICATES
+  ) {
+    throw new WorkerCertificateIdentityError('invalid_material');
+  }
+
+  const remaining = { count: MAX_CERTIFICATES };
+  const chain = splitCertificates(input.certificateChainPem, remaining);
+  const anchors = input.trustAnchors.flatMap((anchor) =>
+    splitCertificates(anchor, remaining),
+  );
+  const [leaf, ...intermediates] = chain;
+  if (!leaf || anchors.length === 0 || leaf.certificate.ca) {
+    throw new WorkerCertificateIdentityError('invalid_material');
+  }
+  const leafTime = certificateTime(leaf.certificate, now);
+  if (leafTime.notAfterMs - now < minimumRemainingValidityMs) {
+    throw new WorkerCertificateIdentityError('insufficient_validity');
+  }
+  if (!leaf.certificate.keyUsage?.includes(CLIENT_AUTH_OID)) {
+    throw new WorkerCertificateIdentityError('not_client_auth');
+  }
+  for (const certificate of [...intermediates, ...anchors]) {
+    if (!certificate.certificate.ca) {
+      throw new WorkerCertificateIdentityError('invalid_material');
+    }
+    certificateTime(certificate.certificate, now);
+  }
+  if (!reachesTrustAnchor(leaf, intermediates, anchors, new Set())) {
+    throw new WorkerCertificateIdentityError('untrusted');
+  }
+
+  const privateKeyPem = materialBytes(input.privateKeyPem, true);
+  try {
+    const summary = Object.freeze({
+      certificateSha256: createHash('sha256')
+        .update(leaf.certificate.raw)
+        .digest('hex'),
+      publicKeySpkiSha256: matchingPublicKey(privateKeyPem, leaf.certificate),
+      serialNumber: leaf.certificate.serialNumber,
+      notBeforeMs: leafTime.notBeforeMs,
+      notAfterMs: leafTime.notAfterMs,
+    });
+    assertWorkerCertificateIdentitySummary(summary);
+    return summary;
+  } finally {
+    privateKeyPem.fill(0);
+  }
+}

packages/ql3-worker-runtime/src/credential/workerCertificateRenewal.ts

@@ -0,0 +1,488 @@
+// Credential ownership: coordinate Worker certificate renewal lifecycle state.
+import type {
+  GenerateWorkerCertificateEnrollmentOptions,
+  WorkerCertificateEnrollmentMaterial,
+} from './workerCertificateEnrollment';
+import {
+  type ActiveWorkerCertificateIdentity,
+  type WorkerCertificateRenewalState,
+  type WorkerCertificateStore,
+} from './workerCertificateStore';
+
+const HOUR_MS = 60 * 60_000;
+const DAY_MS = 24 * HOUR_MS;
+const MAX_CERTIFICATE_MATERIAL_BYTES = 1024 * 1024;
+const MAX_ENROLLMENT_MATERIAL_BYTES = 16 * 1024;
+const MAX_CONSECUTIVE_FAILURES = 16;
+
+export interface WorkerCertificateIssuer {
+  issue(input: {
+    readonly workerId: string;
+    readonly certificateSigningRequestPem: string;
+    readonly currentCertificateSha256?: string;
+    readonly signal: AbortSignal;
+  }): Promise<{ readonly certificateChainPem: string | Buffer }>;
+}
+
+export interface WorkerCertificateTrustAnchorProvider {
+  load(signal: AbortSignal): Promise<readonly (string | Buffer)[]>;
+}
+
+export interface WorkerCertificateRenewalPolicy {
+  readonly renewBeforeMs?: number;
+  readonly minimumIssuedValidityMs?: number;
+  readonly operationTimeoutMs?: number;
+  readonly backoffBaseMs?: number;
+  readonly backoffMaximumMs?: number;
+}
+
+export interface WorkerCertificateRenewalCoordinatorOptions {
+  readonly workerId: string;
+  readonly store: WorkerCertificateStore;
+  readonly issuer: WorkerCertificateIssuer;
+  readonly trustAnchors: WorkerCertificateTrustAnchorProvider;
+  readonly policy?: WorkerCertificateRenewalPolicy;
+  readonly now?: () => number;
+  readonly random?: () => number;
+  readonly prepareEnrollment?: (
+    options: GenerateWorkerCertificateEnrollmentOptions,
+  ) => Promise<WorkerCertificateEnrollmentMaterial>;
+}
+
+export type WorkerCertificateRenewalResult =
+  | {
+      readonly status: 'not_due';
+      readonly identity: ActiveWorkerCertificateIdentity;
+      readonly renewAtMs: number;
+    }
+  | {
+      readonly status: 'renewed';
+      readonly identity: ActiveWorkerCertificateIdentity;
+      readonly cleanupPending: boolean;
+    }
+  | {
+      readonly status: 'backing_off';
+      readonly identity: ActiveWorkerCertificateIdentity;
+      readonly nextAttemptAtMs: number;
+    }
+  | {
+      readonly status: 'retry_scheduled';
+      readonly identity?: ActiveWorkerCertificateIdentity;
+      readonly reason: WorkerCertificateRenewalFailureReason;
+      readonly nextAttemptAtMs: number;
+    }
+  | {
+      readonly status: 'unavailable';
+      readonly nextAttemptAtMs: number;
+    };
+
+export type WorkerCertificateRenewalFailureReason =
+  | 'trust_unavailable'
+  | 'enrollment_failed'
+  | 'issuance_failed'
+  | 'installation_failed'
+  | 'timed_out';
+
+export class WorkerCertificateRenewalConfigurationError extends TypeError {
+  constructor(message: string) {
+    super(`Worker certificate renewal is invalid: ${message}`);
+    this.name = 'WorkerCertificateRenewalConfigurationError';
+  }
+}
+
+interface NormalizedPolicy {
+  readonly renewBeforeMs: number;
+  readonly minimumIssuedValidityMs: number;
+  readonly operationTimeoutMs: number;
+  readonly backoffBaseMs: number;
+  readonly backoffMaximumMs: number;
+}
+
+type EnrollmentFactory = NonNullable<
+  WorkerCertificateRenewalCoordinatorOptions['prepareEnrollment']
+>;
+
+function boundedInteger(
+  value: number | undefined,
+  fallback: number,
+  minimum: number,
+  maximum: number,
+  name: string,
+): number {
+  const candidate = value ?? fallback;
+  if (
+    !Number.isSafeInteger(candidate) ||
+    candidate < minimum ||
+    candidate > maximum
+  ) {
+    throw new WorkerCertificateRenewalConfigurationError(`${name} is invalid`);
+  }
+  return candidate;
+}
+
+function normalizePolicy(
+  policy: WorkerCertificateRenewalPolicy | undefined,
+): NormalizedPolicy {
+  if (
+    policy !== undefined &&
+    (!policy || typeof policy !== 'object' || Array.isArray(policy))
+  ) {
+    throw new WorkerCertificateRenewalConfigurationError(
+      'policy must be an object',
+    );
+  }
+  const renewBeforeMs = boundedInteger(
+    policy?.renewBeforeMs,
+    7 * DAY_MS,
+    HOUR_MS,
+    30 * DAY_MS,
+    'renewBeforeMs',
+  );
+  const minimumIssuedValidityMs = boundedInteger(
+    policy?.minimumIssuedValidityMs,
+    8 * DAY_MS,
+    HOUR_MS,
+    365 * DAY_MS,
+    'minimumIssuedValidityMs',
+  );
+  if (minimumIssuedValidityMs <= renewBeforeMs) {
+    throw new WorkerCertificateRenewalConfigurationError(
+      'minimumIssuedValidityMs must exceed renewBeforeMs',
+    );
+  }
+  const backoffBaseMs = boundedInteger(
+    policy?.backoffBaseMs,
+    30_000,
+    1_000,
+    HOUR_MS,
+    'backoffBaseMs',
+  );
+  const backoffMaximumMs = boundedInteger(
+    policy?.backoffMaximumMs,
+    6 * HOUR_MS,
+    backoffBaseMs,
+    6 * HOUR_MS,
+    'backoffMaximumMs',
+  );
+  return Object.freeze({
+    renewBeforeMs,
+    minimumIssuedValidityMs,
+    operationTimeoutMs: boundedInteger(
+      policy?.operationTimeoutMs,
+      30_000,
+      1_000,
+      120_000,
+      'operationTimeoutMs',
+    ),
+    backoffBaseMs,
+    backoffMaximumMs,
+  });
+}
+
+function safeClock(clock: () => number): number {
+  const value = clock();
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new WorkerCertificateRenewalConfigurationError(
+      'clock returned an invalid value',
+    );
+  }
+  return value;
+}
+
+function certificateMaterial(
+  value: string | Buffer,
+): string | Buffer | undefined {
+  if (typeof value !== 'string' && !Buffer.isBuffer(value)) return undefined;
+  const size = Buffer.byteLength(value);
+  if (size < 1 || size > MAX_CERTIFICATE_MATERIAL_BYTES) return undefined;
+  return value;
+}
+
+function enrollmentMaterial(
+  value: WorkerCertificateEnrollmentMaterial,
+  workerId: string,
+): WorkerCertificateEnrollmentMaterial {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    value.algorithm !== 'ECDSA_P256_SHA256' ||
+    value.workerId !== workerId ||
+    !Buffer.isBuffer(value.privateKeyPem) ||
+    value.privateKeyPem.byteLength < 1 ||
+    value.privateKeyPem.byteLength > MAX_ENROLLMENT_MATERIAL_BYTES ||
+    typeof value.certificateSigningRequestPem !== 'string' ||
+    Buffer.byteLength(value.certificateSigningRequestPem) < 1 ||
+    Buffer.byteLength(value.certificateSigningRequestPem) >
+      MAX_ENROLLMENT_MATERIAL_BYTES ||
+    typeof value.publicKeySpkiSha256 !== 'string' ||
+    !/^[a-f0-9]{64}$/.test(value.publicKeySpkiSha256) ||
+    typeof value.dispose !== 'function'
+  ) {
+    if (Buffer.isBuffer(value?.privateKeyPem)) value.privateKeyPem.fill(0);
+    throw new Error('enrollment material is invalid');
+  }
+  return value;
+}
+
+async function defaultEnrollmentFactory(
+  options: GenerateWorkerCertificateEnrollmentOptions,
+): Promise<WorkerCertificateEnrollmentMaterial> {
+  const enrollment = await import('./workerCertificateEnrollment');
+  return enrollment.generateWorkerCertificateEnrollment(options);
+}
+
+function timeoutSignal(
+  external: AbortSignal | undefined,
+  timeoutMs: number,
+): { readonly operation: AbortSignal; readonly timeout: AbortSignal } {
+  const timeout = AbortSignal.timeout(timeoutMs);
+  return {
+    timeout,
+    operation: external ? AbortSignal.any([external, timeout]) : timeout,
+  };
+}
+
+/**
+ * Performs one explicitly triggered renewal check. It owns no timer, watcher or
+ * signal handler; edge and cluster profiles decide when to call it.
+ */
+export class WorkerCertificateRenewalCoordinator {
+  private readonly workerId: string;
+  private readonly store: WorkerCertificateStore;
+  private readonly issuer: WorkerCertificateIssuer;
+  private readonly trustAnchors: WorkerCertificateTrustAnchorProvider;
+  private readonly policy: NormalizedPolicy;
+  private readonly now: () => number;
+  private readonly random: () => number;
+  private readonly prepareEnrollment: EnrollmentFactory;
+  private inFlight?: Promise<WorkerCertificateRenewalResult>;
+
+  constructor(options: WorkerCertificateRenewalCoordinatorOptions) {
+    if (!options || typeof options !== 'object' || Array.isArray(options)) {
+      throw new WorkerCertificateRenewalConfigurationError(
+        'options must be an object',
+      );
+    }
+    if (
+      typeof options.workerId !== 'string' ||
+      !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(options.workerId)
+    ) {
+      throw new WorkerCertificateRenewalConfigurationError(
+        'workerId is invalid',
+      );
+    }
+    if (
+      !options.store ||
+      typeof options.store.readActive !== 'function' ||
+      typeof options.store.install !== 'function' ||
+      typeof options.store.readRenewalState !== 'function' ||
+      typeof options.store.writeRenewalState !== 'function'
+    ) {
+      throw new WorkerCertificateRenewalConfigurationError('store is invalid');
+    }
+    if (!options.issuer || typeof options.issuer.issue !== 'function') {
+      throw new WorkerCertificateRenewalConfigurationError('issuer is invalid');
+    }
+    if (
+      !options.trustAnchors ||
+      typeof options.trustAnchors.load !== 'function'
+    ) {
+      throw new WorkerCertificateRenewalConfigurationError(
+        'trustAnchors is invalid',
+      );
+    }
+    if (options.now !== undefined && typeof options.now !== 'function') {
+      throw new WorkerCertificateRenewalConfigurationError('now is invalid');
+    }
+    if (options.random !== undefined && typeof options.random !== 'function') {
+      throw new WorkerCertificateRenewalConfigurationError('random is invalid');
+    }
+    if (
+      options.prepareEnrollment !== undefined &&
+      typeof options.prepareEnrollment !== 'function'
+    ) {
+      throw new WorkerCertificateRenewalConfigurationError(
+        'prepareEnrollment is invalid',
+      );
+    }
+    this.workerId = options.workerId;
+    this.store = options.store;
+    this.issuer = options.issuer;
+    this.trustAnchors = options.trustAnchors;
+    this.policy = normalizePolicy(options.policy);
+    this.now = options.now ?? Date.now;
+    this.random = options.random ?? Math.random;
+    this.prepareEnrollment =
+      options.prepareEnrollment ?? defaultEnrollmentFactory;
+  }
+
+  run(signal?: AbortSignal): Promise<WorkerCertificateRenewalResult> {
+    if (signal !== undefined && !(signal instanceof AbortSignal)) {
+      return Promise.reject(
+        new WorkerCertificateRenewalConfigurationError('signal is invalid'),
+      );
+    }
+    if (this.inFlight) return this.inFlight;
+    const operation = this.performRun(signal).finally(() => {
+      if (this.inFlight === operation) this.inFlight = undefined;
+    });
+    this.inFlight = operation;
+    return operation;
+  }
+
+  private async performRun(
+    externalSignal: AbortSignal | undefined,
+  ): Promise<WorkerCertificateRenewalResult> {
+    externalSignal?.throwIfAborted();
+    const observedAtMs = safeClock(this.now);
+    const renewalState = await this.store.readRenewalState();
+    const signals = timeoutSignal(
+      externalSignal,
+      this.policy.operationTimeoutMs,
+    );
+    let failureReason: WorkerCertificateRenewalFailureReason =
+      'trust_unavailable';
+    let current: ActiveWorkerCertificateIdentity | undefined;
+    let trustAnchors: readonly (string | Buffer)[] | undefined;
+
+    try {
+      trustAnchors = await this.trustAnchors.load(signals.operation);
+      signals.operation.throwIfAborted();
+      current = await this.store.readActive(trustAnchors, observedAtMs);
+    } catch (error) {
+      if (externalSignal?.aborted) throw externalSignal.reason ?? error;
+      if (signals.timeout.aborted) failureReason = 'timed_out';
+    }
+
+    if (
+      current &&
+      current.notAfterMs - observedAtMs > this.policy.renewBeforeMs
+    ) {
+      return Object.freeze({
+        status: 'not_due',
+        identity: current,
+        renewAtMs: current.notAfterMs - this.policy.renewBeforeMs,
+      });
+    }
+
+    if (
+      renewalState.nextAttemptAtMs !== null &&
+      renewalState.nextAttemptAtMs > observedAtMs
+    ) {
+      if (current) {
+        return Object.freeze({
+          status: 'backing_off',
+          identity: current,
+          nextAttemptAtMs: renewalState.nextAttemptAtMs,
+        });
+      }
+      return Object.freeze({
+        status: 'unavailable',
+        nextAttemptAtMs: renewalState.nextAttemptAtMs,
+      });
+    }
+
+    const attemptedAtMs = safeClock(this.now);
+    let enrollment: WorkerCertificateEnrollmentMaterial | undefined;
+    try {
+      if (!trustAnchors || signals.operation.aborted) {
+        if (signals.timeout.aborted) failureReason = 'timed_out';
+        throw new Error('trust is unavailable');
+      }
+      failureReason = 'enrollment_failed';
+      enrollment = enrollmentMaterial(
+        await this.prepareEnrollment({ workerId: this.workerId }),
+        this.workerId,
+      );
+      signals.operation.throwIfAborted();
+      failureReason = 'issuance_failed';
+      const issued = await this.issuer.issue({
+        workerId: this.workerId,
+        certificateSigningRequestPem: enrollment.certificateSigningRequestPem,
+        ...(current
+          ? { currentCertificateSha256: current.certificateSha256 }
+          : {}),
+        signal: signals.operation,
+      });
+      signals.operation.throwIfAborted();
+      const issuedMaterial = certificateMaterial(issued?.certificateChainPem);
+      if (!issuedMaterial) throw new Error('issued material is invalid');
+      failureReason = 'installation_failed';
+      const installedAtMs = safeClock(this.now);
+      const installed = await this.store.install({
+        privateKeyPem: enrollment.privateKeyPem,
+        certificateChainPem: issuedMaterial,
+        trustAnchors,
+        now: installedAtMs,
+        minimumRemainingValidityMs: this.policy.minimumIssuedValidityMs,
+      });
+      await this.store.writeRenewalState({
+        consecutiveFailures: 0,
+        nextAttemptAtMs: null,
+        lastAttemptAtMs: attemptedAtMs,
+        lastSuccessAtMs: installedAtMs,
+      });
+      return Object.freeze({
+        status: 'renewed',
+        identity: installed,
+        cleanupPending: installed.cleanupPending,
+      });
+    } catch (error) {
+      if (externalSignal?.aborted) throw externalSignal.reason ?? error;
+      if (signals.timeout.aborted) failureReason = 'timed_out';
+      const failedAtMs = safeClock(this.now);
+      const nextState = this.failedState(
+        renewalState,
+        attemptedAtMs,
+        failedAtMs,
+      );
+      await this.store.writeRenewalState(nextState);
+      if (!current || current.notAfterMs <= failedAtMs) {
+        return Object.freeze({
+          status: 'unavailable',
+          nextAttemptAtMs: nextState.nextAttemptAtMs!,
+        });
+      }
+      return Object.freeze({
+        status: 'retry_scheduled',
+        identity: current,
+        reason: failureReason,
+        nextAttemptAtMs: nextState.nextAttemptAtMs!,
+      });
+    } finally {
+      try {
+        enrollment?.dispose();
+      } finally {
+        enrollment?.privateKeyPem.fill(0);
+      }
+    }
+  }
+
+  private failedState(
+    previous: WorkerCertificateRenewalState,
+    attemptedAtMs: number,
+    failedAtMs: number,
+  ): WorkerCertificateRenewalState {
+    const random = this.random();
+    if (!Number.isFinite(random) || random < 0 || random >= 1) {
+      throw new WorkerCertificateRenewalConfigurationError(
+        'random returned an invalid value',
+      );
+    }
+    const consecutiveFailures = Math.min(
+      MAX_CONSECUTIVE_FAILURES,
+      previous.consecutiveFailures + 1,
+    );
+    const exponential = Math.min(
+      this.policy.backoffMaximumMs,
+      this.policy.backoffBaseMs * 2 ** (consecutiveFailures - 1),
+    );
+    const delayMs = Math.max(1, Math.floor(exponential * (0.5 + random / 2)));
+    return Object.freeze({
+      consecutiveFailures,
+      nextAttemptAtMs: failedAtMs + delayMs,
+      lastAttemptAtMs: attemptedAtMs,
+      lastSuccessAtMs: previous.lastSuccessAtMs,
+    });
+  }
+}

packages/ql3-worker-runtime/src/credential/workerCertificateStore.ts

@@ -0,0 +1,611 @@
+// Credential ownership: persist active Worker certificate material atomically.
+import { randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import {
+  chmod,
+  lstat,
+  mkdir,
+  open,
+  readdir,
+  rename,
+  rm,
+} from 'node:fs/promises';
+import { isAbsolute, join } from 'node:path';
+import {
+  assertWorkerCertificateIdentitySummary,
+  validateWorkerCertificateIdentity,
+  type WorkerCertificateIdentitySummary,
+} from './workerCertificateIdentity';
+
+const MAX_IDENTITY_FILE_BYTES = 1024 * 1024;
+const MAX_MANIFEST_BYTES = 4096;
+const GENERATION_PATTERN =
+  /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/;
+const MAX_GENERATIONS = 8;
+
+export interface WorkerCertificateFileStoreOptions {
+  readonly rootDirectory: string;
+  readonly retainedGenerations?: number;
+}
+
+export interface InstallWorkerCertificateIdentityInput {
+  readonly privateKeyPem: string | Buffer;
+  readonly certificateChainPem: string | Buffer;
+  readonly trustAnchors: readonly (string | Buffer)[];
+  readonly now?: number;
+  readonly minimumRemainingValidityMs?: number;
+}
+
+export interface ActiveWorkerCertificateIdentity
+  extends WorkerCertificateIdentitySummary {
+  readonly generationId: string;
+  readonly installedAtMs: number;
+  readonly privateKeyFile: string;
+  readonly certificateChainFile: string;
+}
+
+export interface InstallWorkerCertificateIdentityResult
+  extends ActiveWorkerCertificateIdentity {
+  readonly cleanupPending: boolean;
+}
+
+export interface WorkerCertificateRenewalState {
+  readonly consecutiveFailures: number;
+  readonly nextAttemptAtMs: number | null;
+  readonly lastAttemptAtMs: number | null;
+  readonly lastSuccessAtMs: number | null;
+}
+
+export interface WorkerCertificateStore {
+  readActive(
+    trustAnchors: readonly (string | Buffer)[],
+    now?: number,
+  ): Promise<ActiveWorkerCertificateIdentity | undefined>;
+  install(
+    input: InstallWorkerCertificateIdentityInput,
+  ): Promise<InstallWorkerCertificateIdentityResult>;
+  readRenewalState(): Promise<WorkerCertificateRenewalState>;
+  writeRenewalState(state: WorkerCertificateRenewalState): Promise<void>;
+}
+
+export interface WorkerCertificateIdentityManifest
+  extends WorkerCertificateIdentitySummary {
+  readonly schemaVersion: 1;
+  readonly generationId: string;
+  readonly installedAtMs: number;
+}
+
+interface RenewalStateManifest extends WorkerCertificateRenewalState {
+  readonly schemaVersion: 1;
+}
+
+export class WorkerCertificateStoreError extends Error {
+  constructor(message: string) {
+    super(`Worker certificate store is unavailable: ${message}`);
+    this.name = 'WorkerCertificateStoreError';
+  }
+}
+
+function safeNow(value: number | undefined): number {
+  const now = value ?? Date.now();
+  if (!Number.isSafeInteger(now) || now < 0) {
+    throw new WorkerCertificateStoreError('observation time is invalid');
+  }
+  return now;
+}
+
+async function safeDirectory(path: string): Promise<void> {
+  let existed = true;
+  try {
+    await lstat(path);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+    existed = false;
+    await mkdir(path, { recursive: true, mode: 0o700 });
+  }
+  const stat = await lstat(path);
+  if (
+    !stat.isDirectory() ||
+    stat.isSymbolicLink() ||
+    (stat.mode & 0o077) !== 0
+  ) {
+    throw new WorkerCertificateStoreError('directory metadata is unsafe');
+  }
+  if (!existed) await chmod(path, 0o700);
+}
+
+async function syncDirectory(path: string): Promise<void> {
+  const handle = await open(path, constants.O_RDONLY);
+  try {
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+}
+
+async function writeSyncedFile(
+  path: string,
+  bytes: Buffer,
+  mode: number,
+): Promise<void> {
+  const flags =
+    constants.O_WRONLY |
+    constants.O_CREAT |
+    constants.O_EXCL |
+    (constants.O_NOFOLLOW ?? 0);
+  const handle = await open(path, flags, mode);
+  try {
+    await handle.writeFile(bytes);
+    await handle.sync();
+    await handle.chmod(mode);
+  } finally {
+    await handle.close();
+  }
+}
+
+async function readBoundedFile(
+  path: string,
+  maximumBytes: number,
+): Promise<Buffer> {
+  const flags = constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0);
+  let handle;
+  try {
+    handle = await open(path, flags);
+    const stat = await handle.stat();
+    if (
+      !stat.isFile() ||
+      stat.size < 1 ||
+      stat.size > maximumBytes ||
+      (stat.mode & 0o077) !== 0
+    ) {
+      throw new WorkerCertificateStoreError('file metadata is unsafe');
+    }
+    const bytes = await handle.readFile();
+    if (bytes.byteLength < 1 || bytes.byteLength > maximumBytes) {
+      bytes.fill(0);
+      throw new WorkerCertificateStoreError('file size is unsafe');
+    }
+    return bytes;
+  } catch (error) {
+    if (error instanceof WorkerCertificateStoreError) throw error;
+    throw new WorkerCertificateStoreError('file is unavailable');
+  } finally {
+    await handle?.close().catch(() => undefined);
+  }
+}
+
+async function assertSafeExistingDirectory(path: string): Promise<void> {
+  let stat;
+  try {
+    stat = await lstat(path);
+  } catch {
+    throw new WorkerCertificateStoreError('directory is unavailable');
+  }
+  if (
+    !stat.isDirectory() ||
+    stat.isSymbolicLink() ||
+    (stat.mode & 0o077) !== 0
+  ) {
+    throw new WorkerCertificateStoreError('directory metadata is unsafe');
+  }
+}
+
+function exactKeys(value: object, expected: readonly string[]): boolean {
+  const keys = Object.keys(value).sort();
+  const sorted = [...expected].sort();
+  return (
+    keys.length === sorted.length &&
+    keys.every((key, index) => key === sorted[index])
+  );
+}
+
+function identityManifest(value: unknown): WorkerCertificateIdentityManifest {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    !exactKeys(value, [
+      'certificateSha256',
+      'generationId',
+      'installedAtMs',
+      'notAfterMs',
+      'notBeforeMs',
+      'publicKeySpkiSha256',
+      'schemaVersion',
+      'serialNumber',
+    ])
+  ) {
+    throw new WorkerCertificateStoreError('identity manifest is invalid');
+  }
+  const candidate = value as Partial<WorkerCertificateIdentityManifest>;
+  if (
+    candidate.schemaVersion !== 1 ||
+    typeof candidate.generationId !== 'string' ||
+    !GENERATION_PATTERN.test(candidate.generationId) ||
+    !Number.isSafeInteger(candidate.installedAtMs) ||
+    Number(candidate.installedAtMs) < 0
+  ) {
+    throw new WorkerCertificateStoreError('identity manifest is invalid');
+  }
+  assertWorkerCertificateIdentitySummary(
+    candidate as WorkerCertificateIdentitySummary,
+  );
+  return Object.freeze(candidate as WorkerCertificateIdentityManifest);
+}
+
+function renewalStateManifest(value: unknown): RenewalStateManifest {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    !exactKeys(value, [
+      'consecutiveFailures',
+      'lastAttemptAtMs',
+      'lastSuccessAtMs',
+      'nextAttemptAtMs',
+      'schemaVersion',
+    ])
+  ) {
+    throw new WorkerCertificateStoreError('renewal state is invalid');
+  }
+  const candidate = value as Partial<RenewalStateManifest>;
+  const optionalTime = (time: unknown): boolean =>
+    time === null || (Number.isSafeInteger(time) && Number(time) >= 0);
+  if (
+    candidate.schemaVersion !== 1 ||
+    !Number.isSafeInteger(candidate.consecutiveFailures) ||
+    Number(candidate.consecutiveFailures) < 0 ||
+    Number(candidate.consecutiveFailures) > 16 ||
+    !optionalTime(candidate.nextAttemptAtMs) ||
+    !optionalTime(candidate.lastAttemptAtMs) ||
+    !optionalTime(candidate.lastSuccessAtMs)
+  ) {
+    throw new WorkerCertificateStoreError('renewal state is invalid');
+  }
+  return Object.freeze(candidate as RenewalStateManifest);
+}
+
+async function parseJsonFile<T>(
+  path: string,
+  parser: (value: unknown) => T,
+): Promise<T> {
+  const bytes = await readBoundedFile(path, MAX_MANIFEST_BYTES);
+  try {
+    return parser(JSON.parse(bytes.toString('utf8')) as unknown);
+  } catch (error) {
+    if (error instanceof WorkerCertificateStoreError) throw error;
+    throw new WorkerCertificateStoreError('manifest JSON is invalid');
+  } finally {
+    bytes.fill(0);
+  }
+}
+
+export class WorkerCertificateFileStore implements WorkerCertificateStore {
+  private readonly rootDirectory: string;
+  private readonly generationsDirectory: string;
+  private readonly retainedGenerations: number;
+  private installing = false;
+
+  constructor(options: WorkerCertificateFileStoreOptions) {
+    if (
+      !options ||
+      typeof options !== 'object' ||
+      Array.isArray(options) ||
+      typeof options.rootDirectory !== 'string' ||
+      !isAbsolute(options.rootDirectory) ||
+      options.rootDirectory.length > 4096 ||
+      /[\0\r\n]/.test(options.rootDirectory)
+    ) {
+      throw new WorkerCertificateStoreError('rootDirectory is invalid');
+    }
+    const retainedGenerations = options.retainedGenerations ?? 2;
+    if (
+      !Number.isSafeInteger(retainedGenerations) ||
+      retainedGenerations < 1 ||
+      retainedGenerations > 4
+    ) {
+      throw new WorkerCertificateStoreError(
+        'retainedGenerations must be between 1 and 4',
+      );
+    }
+    this.rootDirectory = options.rootDirectory;
+    this.generationsDirectory = join(options.rootDirectory, 'generations');
+    this.retainedGenerations = retainedGenerations;
+  }
+
+  private async initialize(): Promise<void> {
+    try {
+      await safeDirectory(this.rootDirectory);
+      await safeDirectory(this.generationsDirectory);
+    } catch (error) {
+      if (error instanceof WorkerCertificateStoreError) throw error;
+      throw new WorkerCertificateStoreError('directory is unavailable');
+    }
+  }
+
+  private async writeAtomicManifest(
+    name: string,
+    value: unknown,
+    onCommitted?: () => void,
+  ): Promise<void> {
+    const temporary = join(this.rootDirectory, `.${name}.${randomUUID()}.tmp`);
+    const destination = join(this.rootDirectory, name);
+    const bytes = Buffer.from(`${JSON.stringify(value)}\n`, 'utf8');
+    try {
+      if (bytes.byteLength > MAX_MANIFEST_BYTES) {
+        throw new WorkerCertificateStoreError('manifest exceeds hard limit');
+      }
+      await writeSyncedFile(temporary, bytes, 0o600);
+      await rename(temporary, destination);
+      onCommitted?.();
+      await syncDirectory(this.rootDirectory);
+    } finally {
+      bytes.fill(0);
+      await rm(temporary, { force: true }).catch(() => undefined);
+    }
+  }
+
+  async readActiveSummary(): Promise<
+    WorkerCertificateIdentityManifest | undefined
+  > {
+    await this.initialize();
+    try {
+      return await parseJsonFile(
+        join(this.rootDirectory, 'active.json'),
+        identityManifest,
+      );
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
+      if (
+        error instanceof WorkerCertificateStoreError &&
+        error.message.endsWith('file is unavailable')
+      ) {
+        const stat = await lstat(join(this.rootDirectory, 'active.json')).catch(
+          () => undefined,
+        );
+        if (!stat) return undefined;
+      }
+      throw error;
+    }
+  }
+
+  async readActive(
+    trustAnchors: readonly (string | Buffer)[],
+    now: number = Date.now(),
+  ): Promise<ActiveWorkerCertificateIdentity | undefined> {
+    const manifest = await this.readActiveSummary();
+    if (!manifest) return undefined;
+    const generationDirectory = join(
+      this.generationsDirectory,
+      manifest.generationId,
+    );
+    const privateKeyFile = join(generationDirectory, 'private-key.pem');
+    const certificateChainFile = join(
+      generationDirectory,
+      'certificate-chain.pem',
+    );
+    await assertSafeExistingDirectory(generationDirectory);
+    const privateKeyPem = await readBoundedFile(
+      privateKeyFile,
+      MAX_IDENTITY_FILE_BYTES,
+    );
+    let certificateChainPem: Buffer | undefined;
+    try {
+      certificateChainPem = await readBoundedFile(
+        certificateChainFile,
+        MAX_IDENTITY_FILE_BYTES,
+      );
+      const summary = validateWorkerCertificateIdentity({
+        privateKeyPem,
+        certificateChainPem,
+        trustAnchors,
+        now: safeNow(now),
+      });
+      if (
+        summary.certificateSha256 !== manifest.certificateSha256 ||
+        summary.publicKeySpkiSha256 !== manifest.publicKeySpkiSha256 ||
+        summary.serialNumber !== manifest.serialNumber ||
+        summary.notBeforeMs !== manifest.notBeforeMs ||
+        summary.notAfterMs !== manifest.notAfterMs
+      ) {
+        throw new WorkerCertificateStoreError('active identity was modified');
+      }
+      return Object.freeze({
+        ...summary,
+        generationId: manifest.generationId,
+        installedAtMs: manifest.installedAtMs,
+        privateKeyFile,
+        certificateChainFile,
+      });
+    } finally {
+      privateKeyPem.fill(0);
+      certificateChainPem?.fill(0);
+    }
+  }
+
+  async install(
+    input: InstallWorkerCertificateIdentityInput,
+  ): Promise<InstallWorkerCertificateIdentityResult> {
+    if (this.installing) {
+      throw new WorkerCertificateStoreError('another install is in progress');
+    }
+    this.installing = true;
+    let stagingDirectory: string | undefined;
+    let generationDirectory: string | undefined;
+    let activated = false;
+    try {
+      if (!input || typeof input !== 'object' || Array.isArray(input)) {
+        throw new WorkerCertificateStoreError('install input is invalid');
+      }
+      const now = safeNow(input.now);
+      const summary = validateWorkerCertificateIdentity({ ...input, now });
+      await this.initialize();
+      const entries = await readdir(this.generationsDirectory, {
+        withFileTypes: true,
+      });
+      if (entries.length >= MAX_GENERATIONS) {
+        throw new WorkerCertificateStoreError(
+          'generation capacity requires maintenance',
+        );
+      }
+      const generationId = randomUUID();
+      generationDirectory = join(this.generationsDirectory, generationId);
+      stagingDirectory = join(
+        this.generationsDirectory,
+        `.staging-${generationId}`,
+      );
+      await mkdir(stagingDirectory, { mode: 0o700 });
+      const privateKeyBytes = Buffer.isBuffer(input.privateKeyPem)
+        ? Buffer.from(input.privateKeyPem)
+        : Buffer.from(input.privateKeyPem, 'utf8');
+      const certificateBytes = Buffer.isBuffer(input.certificateChainPem)
+        ? Buffer.from(input.certificateChainPem)
+        : Buffer.from(input.certificateChainPem, 'utf8');
+      const manifest: WorkerCertificateIdentityManifest = Object.freeze({
+        schemaVersion: 1,
+        generationId,
+        installedAtMs: now,
+        ...summary,
+      });
+      const manifestBytes = Buffer.from(`${JSON.stringify(manifest)}\n`);
+      try {
+        await Promise.all([
+          writeSyncedFile(
+            join(stagingDirectory, 'private-key.pem'),
+            privateKeyBytes,
+            0o600,
+          ),
+          writeSyncedFile(
+            join(stagingDirectory, 'certificate-chain.pem'),
+            certificateBytes,
+            0o600,
+          ),
+          writeSyncedFile(
+            join(stagingDirectory, 'metadata.json'),
+            manifestBytes,
+            0o600,
+          ),
+        ]);
+      } finally {
+        privateKeyBytes.fill(0);
+        certificateBytes.fill(0);
+        manifestBytes.fill(0);
+      }
+      await syncDirectory(stagingDirectory);
+      await rename(stagingDirectory, generationDirectory);
+      stagingDirectory = undefined;
+      await syncDirectory(this.generationsDirectory);
+      await this.writeAtomicManifest('active.json', manifest, () => {
+        activated = true;
+      });
+      const cleanupPending = !(await this.pruneRetired(manifest.generationId));
+      return Object.freeze({
+        ...summary,
+        generationId,
+        installedAtMs: now,
+        privateKeyFile: join(generationDirectory, 'private-key.pem'),
+        certificateChainFile: join(
+          generationDirectory,
+          'certificate-chain.pem',
+        ),
+        cleanupPending,
+      });
+    } catch (error) {
+      if (error instanceof WorkerCertificateStoreError) throw error;
+      throw new WorkerCertificateStoreError('install failed');
+    } finally {
+      if (stagingDirectory) {
+        await rm(stagingDirectory, { recursive: true, force: true }).catch(
+          () => undefined,
+        );
+      }
+      if (generationDirectory && !activated) {
+        await rm(generationDirectory, { recursive: true, force: true }).catch(
+          () => undefined,
+        );
+      }
+      this.installing = false;
+    }
+  }
+
+  private async pruneRetired(activeGenerationId: string): Promise<boolean> {
+    try {
+      const entries = await readdir(this.generationsDirectory, {
+        withFileTypes: true,
+      });
+      const generations: Array<{
+        generationId: string;
+        installedAtMs: number;
+      }> = [];
+      for (const entry of entries) {
+        if (!entry.isDirectory() || !GENERATION_PATTERN.test(entry.name)) {
+          continue;
+        }
+        const metadata = await parseJsonFile(
+          join(this.generationsDirectory, entry.name, 'metadata.json'),
+          identityManifest,
+        );
+        if (metadata.generationId !== entry.name) return false;
+        generations.push({
+          generationId: entry.name,
+          installedAtMs: metadata.installedAtMs,
+        });
+      }
+      generations.sort(
+        (left, right) =>
+          right.installedAtMs - left.installedAtMs ||
+          right.generationId.localeCompare(left.generationId),
+      );
+      const keep = new Set(
+        generations
+          .filter((item) => item.generationId !== activeGenerationId)
+          .slice(0, Math.max(0, this.retainedGenerations - 1))
+          .map((item) => item.generationId),
+      );
+      keep.add(activeGenerationId);
+      for (const generation of generations) {
+        if (!keep.has(generation.generationId)) {
+          await rm(join(this.generationsDirectory, generation.generationId), {
+            recursive: true,
+            force: true,
+          });
+        }
+      }
+      await syncDirectory(this.generationsDirectory);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async readRenewalState(): Promise<WorkerCertificateRenewalState> {
+    await this.initialize();
+    try {
+      const manifest = await parseJsonFile(
+        join(this.rootDirectory, 'renewal.json'),
+        renewalStateManifest,
+      );
+      const { schemaVersion: _schemaVersion, ...state } = manifest;
+      return Object.freeze(state);
+    } catch (error) {
+      const stat = await lstat(join(this.rootDirectory, 'renewal.json')).catch(
+        () => undefined,
+      );
+      if (!stat) {
+        return Object.freeze({
+          consecutiveFailures: 0,
+          nextAttemptAtMs: null,
+          lastAttemptAtMs: null,
+          lastSuccessAtMs: null,
+        });
+      }
+      throw error;
+    }
+  }
+
+  async writeRenewalState(state: WorkerCertificateRenewalState): Promise<void> {
+    await this.initialize();
+    const manifest = renewalStateManifest({ schemaVersion: 1, ...state });
+    await this.writeAtomicManifest('renewal.json', manifest);
+  }
+}

packages/ql3-worker-runtime/src/credential/workerProductionCredentialProvider.ts

@@ -0,0 +1,260 @@
+// Credential ownership: load production mTLS identity and bounded Worker token.
+import { constants } from 'node:fs';
+import { lstat, open } from 'node:fs/promises';
+import { dirname, isAbsolute, normalize, parse } from 'node:path';
+import { normalizeWorkerCredentialId } from '@qinglong/runtime-core/worker-credential';
+import {
+  validateWorkerCertificateIdentity,
+  type WorkerCertificateIdentitySummary,
+} from './workerCertificateIdentity';
+import type {
+  WorkerCertificateStore,
+} from './workerCertificateStore';
+import type {
+  WorkerCertificateTrustAnchorProvider,
+} from './workerCertificateRenewal';
+import type {
+  WorkerIngressHttpsCredentialProvider,
+  WorkerIngressHttpsCredentials,
+} from '../remote-execution/transport/workerIngressHttpsClient';
+
+const MAX_CREDENTIAL_TOKEN_BYTES = 256;
+const MAX_TLS_MATERIAL_BYTES = 1024 * 1024;
+const CREDENTIAL_TOKEN =
+  /^ql3w_([A-Za-z0-9][A-Za-z0-9._:-]{0,63})_([A-Za-z0-9_-]{43})$/;
+
+export interface WorkerProductionCredentialProviderOptions {
+  readonly certificateStore: Pick<WorkerCertificateStore, 'readActive'>;
+  readonly trustAnchors: WorkerCertificateTrustAnchorProvider;
+  /** Private, atomically replaceable file containing one ql3w token. */
+  readonly credentialTokenFile: string;
+  readonly expectedCredentialId?: string;
+  readonly now?: () => number;
+}
+
+export class WorkerProductionCredentialProviderError extends Error {
+  constructor(
+    readonly reason: 'invalid_configuration' | 'credentials_unavailable',
+    options?: ErrorOptions,
+  ) {
+    super(`Worker production credentials failed: ${reason}`, options);
+    this.name = 'WorkerProductionCredentialProviderError';
+  }
+}
+
+function privateFilePath(value: string): string {
+  if (
+    typeof value !== 'string' ||
+    !isAbsolute(value) ||
+    parse(value).root === value ||
+    normalize(value) !== value ||
+    value.includes('\0') ||
+    Buffer.byteLength(value, 'utf8') > 4096
+  ) throw new WorkerProductionCredentialProviderError('invalid_configuration');
+  return value;
+}
+
+function now(provider: () => number): number {
+  const value = provider();
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new WorkerProductionCredentialProviderError('credentials_unavailable');
+  }
+  return value;
+}
+
+async function readPrivateFile(
+  path: string,
+  maximumBytes: number,
+): Promise<Buffer> {
+  let handle;
+  try {
+    const parent = await lstat(dirname(path));
+    if (
+      !parent.isDirectory() ||
+      parent.isSymbolicLink() ||
+      (parent.mode & 0o077) !== 0
+    ) throw new Error('unsafe parent');
+    handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+    const stat = await handle.stat();
+    if (
+      !stat.isFile() ||
+      stat.size < 1 ||
+      stat.size > maximumBytes ||
+      (stat.mode & 0o077) !== 0
+    ) throw new Error('unsafe file');
+    const bytes = await handle.readFile();
+    if (bytes.byteLength < 1 || bytes.byteLength > maximumBytes) {
+      bytes.fill(0);
+      throw new Error('unsafe bytes');
+    }
+    return bytes;
+  } catch (error) {
+    throw new WorkerProductionCredentialProviderError(
+      'credentials_unavailable', { cause: error },
+    );
+  } finally {
+    await handle?.close().catch(() => undefined);
+  }
+}
+
+function token(bytes: Buffer, expectedCredentialId?: string): string {
+  try {
+    let length = bytes.byteLength;
+    if (bytes[length - 1] === 0x0a) length -= 1;
+    const value = bytes.subarray(0, length).toString('ascii');
+    const match = CREDENTIAL_TOKEN.exec(value);
+    if (
+      !match ||
+      bytes.subarray(0, length).some((byte) => byte > 0x7f) ||
+      bytes.subarray(0, length).includes(0x0a) ||
+      (expectedCredentialId !== undefined &&
+        match[1] !== expectedCredentialId)
+    ) throw new Error('token is invalid');
+    return value;
+  } catch (error) {
+    throw new WorkerProductionCredentialProviderError(
+      'credentials_unavailable', { cause: error },
+    );
+  } finally {
+    bytes.fill(0);
+  }
+}
+
+function sameIdentity(
+  expected: WorkerCertificateIdentitySummary,
+  actual: WorkerCertificateIdentitySummary,
+): boolean {
+  return actual.certificateSha256 === expected.certificateSha256 &&
+    actual.publicKeySpkiSha256 === expected.publicKeySpkiSha256 &&
+    actual.serialNumber === expected.serialNumber &&
+    actual.notBeforeMs === expected.notBeforeMs &&
+    actual.notAfterMs === expected.notAfterMs;
+}
+
+function copyMaterial(value: string | Buffer): Buffer {
+  const bytes = Buffer.isBuffer(value)
+    ? Buffer.from(value)
+    : Buffer.from(value, 'utf8');
+  if (bytes.byteLength < 1 || bytes.byteLength > MAX_TLS_MATERIAL_BYTES) {
+    bytes.fill(0);
+    throw new WorkerProductionCredentialProviderError(
+      'credentials_unavailable',
+    );
+  }
+  return bytes;
+}
+
+/**
+ * Loads the current certificate generation and ql3w token for every request.
+ * Atomic file replacement therefore rotates credentials without a watcher or
+ * a second Agent. Returned Buffer material is disposable by the HTTPS client.
+ */
+export class WorkerProductionCredentialProvider
+  implements WorkerIngressHttpsCredentialProvider {
+  private readonly certificateStore: Pick<WorkerCertificateStore, 'readActive'>;
+  private readonly trustAnchors: WorkerCertificateTrustAnchorProvider;
+  private readonly credentialTokenFile: string;
+  private readonly expectedCredentialId?: string;
+  private readonly nowProvider: () => number;
+
+  constructor(options: WorkerProductionCredentialProviderOptions) {
+    if (
+      !options ||
+      typeof options.certificateStore?.readActive !== 'function' ||
+      typeof options.trustAnchors?.load !== 'function' ||
+      (options.now !== undefined && typeof options.now !== 'function')
+    ) throw new WorkerProductionCredentialProviderError('invalid_configuration');
+    let expectedCredentialId: string | undefined;
+    try {
+      expectedCredentialId = options.expectedCredentialId === undefined
+        ? undefined
+        : normalizeWorkerCredentialId(options.expectedCredentialId);
+    } catch (error) {
+      throw new WorkerProductionCredentialProviderError(
+        'invalid_configuration', { cause: error },
+      );
+    }
+    this.certificateStore = options.certificateStore;
+    this.trustAnchors = options.trustAnchors;
+    this.credentialTokenFile = privateFilePath(options.credentialTokenFile);
+    this.expectedCredentialId = expectedCredentialId;
+    this.nowProvider = options.now ?? Date.now;
+  }
+
+  async load(signal?: AbortSignal): Promise<WorkerIngressHttpsCredentials> {
+    const operationSignal = signal ?? new AbortController().signal;
+    operationSignal.throwIfAborted();
+    let certificate: Buffer | undefined;
+    let privateKey: Buffer | undefined;
+    const anchors: Buffer[] = [];
+    try {
+      const observedAtMs = now(this.nowProvider);
+      const trustAnchors = await this.trustAnchors.load(operationSignal);
+      operationSignal.throwIfAborted();
+      const active = await this.certificateStore.readActive(
+        trustAnchors,
+        observedAtMs,
+      );
+      if (!active) {
+        throw new WorkerProductionCredentialProviderError(
+          'credentials_unavailable',
+        );
+      }
+      certificate = await readPrivateFile(
+        active.certificateChainFile,
+        MAX_TLS_MATERIAL_BYTES,
+      );
+      privateKey = await readPrivateFile(
+        active.privateKeyFile,
+        MAX_TLS_MATERIAL_BYTES,
+      );
+      operationSignal.throwIfAborted();
+      const summary = validateWorkerCertificateIdentity({
+        certificateChainPem: certificate,
+        privateKeyPem: privateKey,
+        trustAnchors,
+        now: observedAtMs,
+      });
+      if (!sameIdentity(active, summary)) {
+        throw new WorkerProductionCredentialProviderError(
+          'credentials_unavailable',
+        );
+      }
+      const credentialToken = token(
+        await readPrivateFile(
+          this.credentialTokenFile,
+          MAX_CREDENTIAL_TOKEN_BYTES,
+        ),
+        this.expectedCredentialId,
+      );
+      for (const anchor of trustAnchors) anchors.push(copyMaterial(anchor));
+      const disposableCertificate = certificate;
+      const disposablePrivateKey = privateKey;
+      certificate = undefined;
+      privateKey = undefined;
+      let disposed = false;
+      return Object.freeze({
+        authorization: `Worker ${credentialToken}`,
+        certificateChainPem: disposableCertificate,
+        privateKeyPem: disposablePrivateKey,
+        trustAnchors: Object.freeze(anchors),
+        dispose() {
+          if (disposed) return;
+          disposed = true;
+          disposableCertificate.fill(0);
+          disposablePrivateKey.fill(0);
+          anchors.forEach((anchor) => anchor.fill(0));
+        },
+      });
+    } catch (error) {
+      certificate?.fill(0);
+      privateKey?.fill(0);
+      anchors.forEach((anchor) => anchor.fill(0));
+      if (operationSignal.aborted) throw operationSignal.reason ?? error;
+      if (error instanceof WorkerProductionCredentialProviderError) throw error;
+      throw new WorkerProductionCredentialProviderError(
+        'credentials_unavailable', { cause: error },
+      );
+    }
+  }
+}