feat(ql3): establish 3.0 incubation baseline

packages/ql3-runtime-core/src/remote-execution/remoteDispatch.ts

@@ -0,0 +1,156 @@
+import { timingSafeEqual } from 'node:crypto';
+import type { ClusterTaskExecutionRevision } from '../task-definition/clusterExecutionRevision';
+import {
+  CLUSTER_EXECUTOR_TYPE,
+  normalizeClusterTaskExecutionRevision,
+} from '../task-definition/clusterExecutionRevision';
+import type { RunDispatchLeaseRecord } from '../run/runDispatchLease';
+import {
+  assertRunDispatchId,
+  assertRunDispatchLeaseRecord,
+  digestRunDispatchLeaseToken,
+} from '../run/runDispatchLease';
+import {
+  assertWorkerId,
+  assertWorkerSessionId,
+} from '../worker/workerSession';
+
+export * from './remoteWorkerPlacement';
+
+export const MAX_REMOTE_DISPATCH_PAGE_SIZE = 64;
+const SHA256 = /^[a-f0-9]{64}$/;
+
+export interface ClusterDispatchCandidateCursor {
+  readonly priority: number;
+  readonly queuedAtMs: number;
+  readonly attemptCreatedAtMs: number;
+  readonly attemptId: string;
+}
+
+export interface ClusterDispatchCandidate extends ClusterDispatchCandidateCursor {
+  readonly runId: string;
+  readonly projectId: string;
+  readonly taskId: string;
+  readonly taskRevision: string;
+  readonly attemptNumber: number;
+  readonly executorType: typeof CLUSTER_EXECUTOR_TYPE;
+}
+
+export interface ClusterDispatchCandidatePage {
+  readonly observedAtMs: number;
+  readonly candidates: readonly ClusterDispatchCandidate[];
+  readonly truncated: boolean;
+  readonly next?: ClusterDispatchCandidateCursor;
+}
+
+export interface ClusterDispatchRecovery {
+  readonly observedAtMs: number;
+  readonly candidate: ClusterDispatchCandidate;
+  readonly lease: RunDispatchLeaseRecord;
+  readonly workerCurrent: boolean;
+}
+
+export interface ClusterDispatchSource {
+  listClusterDispatchCandidates(options: Readonly<{
+    limit: number;
+    after?: ClusterDispatchCandidateCursor;
+  }>): Promise<ClusterDispatchCandidatePage>;
+  findClusterDispatchRecovery(offerId: string): Promise<ClusterDispatchRecovery | null>;
+}
+
+export interface ClusterRemoteExecutionOffer {
+  readonly offerId: string;
+  readonly deliveryKind: 'new_claim' | 'lease_recovery';
+  readonly executionDigest: string;
+  readonly candidate: ClusterDispatchCandidate;
+  readonly worker: Readonly<{
+    workerId: string;
+    sessionId: string;
+    generation: number;
+  }>;
+  readonly lease: RunDispatchLeaseRecord;
+  /** Ephemeral capability supplied by the Worker; never persist or log it. */
+  readonly leaseToken: string;
+  readonly executionRevision: ClusterTaskExecutionRevision;
+  readonly placementScore: number;
+}
+
+function invalid(message: string): never {
+  throw new TypeError(`Remote dispatch value is invalid: ${message}`);
+}
+
+function safeInteger(value: unknown, label: string, minimum = 0): number {
+  if (!Number.isSafeInteger(value) || (value as number) < minimum) invalid(`${label} is invalid`);
+  return value as number;
+}
+
+export function normalizeClusterDispatchCandidate(value: ClusterDispatchCandidate): ClusterDispatchCandidate {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) invalid('candidate is invalid');
+  for (const [label, id] of [
+    ['runId', value.runId], ['attemptId', value.attemptId], ['projectId', value.projectId],
+    ['taskId', value.taskId], ['taskRevision', value.taskRevision],
+  ] as const) assertRunDispatchId(label, id);
+  if (!Number.isSafeInteger(value.priority)) invalid('candidate priority is invalid');
+  safeInteger(value.queuedAtMs, 'candidate queuedAtMs');
+  safeInteger(value.attemptCreatedAtMs, 'candidate attemptCreatedAtMs');
+  safeInteger(value.attemptNumber, 'candidate attemptNumber', 1);
+  if (value.executorType !== CLUSTER_EXECUTOR_TYPE) invalid('candidate executorType is invalid');
+  return Object.freeze({ ...value });
+}
+
+export function normalizeClusterDispatchCursor(value: ClusterDispatchCandidateCursor): ClusterDispatchCandidateCursor {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) invalid('cursor is invalid');
+  if (!Number.isSafeInteger(value.priority)) invalid('cursor priority is invalid');
+  safeInteger(value.queuedAtMs, 'cursor queuedAtMs');
+  safeInteger(value.attemptCreatedAtMs, 'cursor attemptCreatedAtMs');
+  assertRunDispatchId('attemptId', value.attemptId);
+  return Object.freeze({ ...value });
+}
+
+export function assertRemoteDispatchPageSize(limit: number): void {
+  if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_REMOTE_DISPATCH_PAGE_SIZE) {
+    throw new RangeError(`Remote dispatch page size must be between 1 and ${MAX_REMOTE_DISPATCH_PAGE_SIZE}`);
+  }
+}
+
+export function createClusterRemoteExecutionOffer(value: ClusterRemoteExecutionOffer): ClusterRemoteExecutionOffer {
+  assertRunDispatchId('offerId', value.offerId);
+  const candidate = normalizeClusterDispatchCandidate(value.candidate);
+  assertWorkerId(value.worker.workerId);
+  assertWorkerSessionId(value.worker.sessionId);
+  safeInteger(value.worker.generation, 'offer worker generation', 1);
+  assertRunDispatchLeaseRecord(value.lease);
+  if (
+    value.lease.status !== 'leased' ||
+    value.lease.attemptId !== candidate.attemptId || value.lease.runId !== candidate.runId ||
+    value.lease.workerId !== value.worker.workerId ||
+    value.lease.workerSessionId !== value.worker.sessionId ||
+    value.lease.workerGeneration !== value.worker.generation ||
+    digestRunDispatchLeaseToken(value.leaseToken) !== value.lease.leaseTokenDigest
+  ) invalid('offer authority does not match');
+  const executionRevision = normalizeClusterTaskExecutionRevision(value.executionRevision);
+  if (
+    executionRevision.projectId !== candidate.projectId ||
+    executionRevision.taskId !== candidate.taskId ||
+    executionRevision.taskRevision !== candidate.taskRevision
+  ) invalid('offer execution revision does not match candidate');
+  if (!SHA256.test(value.executionDigest) || value.executionDigest !== executionRevision.contentDigest) invalid('offer execution digest does not match');
+  if (!Number.isSafeInteger(value.placementScore) || value.placementScore < 0) invalid('offer placement score is invalid');
+  return Object.freeze({
+    offerId: value.offerId,
+    deliveryKind: value.deliveryKind,
+    executionDigest: value.executionDigest,
+    candidate,
+    worker: Object.freeze({ ...value.worker }),
+    lease: Object.freeze({ ...value.lease }),
+    leaseToken: value.leaseToken,
+    executionRevision,
+    placementScore: value.placementScore,
+  });
+}
+
+export function leaseTokenMatchesDigest(token: string, digest: string): boolean {
+  const actual = Buffer.from(digestRunDispatchLeaseToken(token), 'hex');
+  const expected = SHA256.test(digest) ? Buffer.from(digest, 'hex') : Buffer.alloc(0);
+  return actual.length === expected.length && timingSafeEqual(actual, expected);
+}

packages/ql3-runtime-core/src/remote-execution/remoteOfferDelivery.ts

@@ -0,0 +1,381 @@
+import type { ClusterRemoteExecutionOffer } from './remoteDispatch';
+import {
+  createClusterRemoteExecutionOffer,
+  normalizeClusterDispatchCandidate,
+} from './remoteDispatch';
+import {
+  assertRunDispatchId,
+  assertRunDispatchLeaseToken,
+  digestRunDispatchLeaseToken,
+} from '../run/runDispatchLease';
+import {
+  assertWorkerId,
+  assertWorkerSessionId,
+} from '../worker/workerSession';
+
+export const REMOTE_EXECUTION_OFFER_DELIVERY_SCHEMA =
+  'qinglong/remote-execution-offer@v1';
+export const MAX_REMOTE_EXECUTION_OFFER_RESPONSE_BYTES = 128 * 1024;
+
+const IDLE_REASONS = [
+  'worker_unavailable',
+  'no_candidates',
+  'no_match',
+  'plans_unavailable',
+  'claim_raced',
+  'claim_budget_exhausted',
+  'scan_budget_exhausted',
+] as const;
+
+export type RemoteExecutionOfferIdleReason = (typeof IDLE_REASONS)[number];
+
+export interface RemoteExecutionOfferClaimAuthority {
+  readonly workerId: string;
+  readonly workerSessionId: string;
+  readonly workerGeneration: number;
+  readonly offerId: string;
+  readonly leaseToken: string;
+}
+
+export interface RemoteExecutionOfferDeliveryStats {
+  readonly pages: number;
+  readonly candidates: number;
+  readonly plansUnavailable: number;
+  readonly placementMismatches: number;
+  readonly claimAttempts: number;
+  readonly claimRaces: number;
+}
+
+export type RemoteExecutionOfferPullResult =
+  | Readonly<{
+      status: 'offered';
+      offer: ClusterRemoteExecutionOffer;
+      stats: RemoteExecutionOfferDeliveryStats;
+      truncated: boolean;
+    }>
+  | Readonly<{
+      status: 'idle';
+      reason: RemoteExecutionOfferIdleReason;
+      stats: RemoteExecutionOfferDeliveryStats;
+      truncated: boolean;
+    }>;
+
+export type RemoteExecutionOfferPullBody =
+  | Readonly<{
+      schema: typeof REMOTE_EXECUTION_OFFER_DELIVERY_SCHEMA;
+      status: 'offered';
+      offer: Readonly<{
+        offerId: string;
+        deliveryKind: ClusterRemoteExecutionOffer['deliveryKind'];
+        executionDigest: string;
+        candidate: ClusterRemoteExecutionOffer['candidate'];
+        worker: ClusterRemoteExecutionOffer['worker'];
+        lease: Readonly<{
+          version: number;
+          leaseGeneration: number;
+          acquiredAtMs: number;
+          renewedAtMs: number;
+          expiresAtMs: number;
+          updatedAtMs: number;
+        }>;
+        executionRevision: ClusterRemoteExecutionOffer['executionRevision'];
+        placementScore: number;
+      }>;
+      stats: RemoteExecutionOfferDeliveryStats;
+      truncated: boolean;
+    }>
+  | Readonly<{
+      schema: typeof REMOTE_EXECUTION_OFFER_DELIVERY_SCHEMA;
+      status: 'idle';
+      reason: RemoteExecutionOfferIdleReason;
+      stats: RemoteExecutionOfferDeliveryStats;
+      truncated: boolean;
+    }>;
+
+export class InvalidRemoteExecutionOfferDeliveryError extends TypeError {
+  readonly code = 'REMOTE_EXECUTION_OFFER_DELIVERY_INVALID';
+
+  constructor(message: string) {
+    super(`Remote execution offer delivery is invalid: ${message}`);
+    this.name = 'InvalidRemoteExecutionOfferDeliveryError';
+  }
+}
+
+function invalid(message: string): never {
+  throw new InvalidRemoteExecutionOfferDeliveryError(message);
+}
+
+function object(value: unknown, label: string): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return invalid(`${label} is not an object`);
+  }
+  return value as Record<string, unknown>;
+}
+
+function exactKeys(
+  value: Record<string, unknown>,
+  expected: readonly string[],
+  label: string,
+): void {
+  const actual = Object.keys(value).sort();
+  const sorted = [...expected].sort();
+  if (
+    actual.length !== sorted.length ||
+    actual.some((key, index) => key !== sorted[index])
+  ) {
+    invalid(`${label} shape is invalid`);
+  }
+}
+
+function safeInteger(value: unknown, label: string, minimum = 0): number {
+  if (!Number.isSafeInteger(value) || (value as number) < minimum) {
+    return invalid(`${label} is invalid`);
+  }
+  return value as number;
+}
+
+function boolean(value: unknown, label: string): boolean {
+  if (typeof value !== 'boolean') return invalid(`${label} is invalid`);
+  return value;
+}
+
+function normalizeOffer(
+  value: ClusterRemoteExecutionOffer,
+): ClusterRemoteExecutionOffer {
+  try {
+    return createClusterRemoteExecutionOffer(value);
+  } catch (error) {
+    if (error instanceof InvalidRemoteExecutionOfferDeliveryError) throw error;
+    return invalid('offered payload is invalid');
+  }
+}
+
+function normalizeStats(value: unknown): RemoteExecutionOfferDeliveryStats {
+  const candidate = object(value, 'stats');
+  const keys = [
+    'pages',
+    'candidates',
+    'plansUnavailable',
+    'placementMismatches',
+    'claimAttempts',
+    'claimRaces',
+  ] as const;
+  exactKeys(candidate, keys, 'stats');
+  const normalized = Object.freeze({
+    pages: safeInteger(candidate.pages, 'stats.pages'),
+    candidates: safeInteger(candidate.candidates, 'stats.candidates'),
+    plansUnavailable: safeInteger(
+      candidate.plansUnavailable,
+      'stats.plansUnavailable',
+    ),
+    placementMismatches: safeInteger(
+      candidate.placementMismatches,
+      'stats.placementMismatches',
+    ),
+    claimAttempts: safeInteger(
+      candidate.claimAttempts,
+      'stats.claimAttempts',
+    ),
+    claimRaces: safeInteger(candidate.claimRaces, 'stats.claimRaces'),
+  });
+  if (
+    normalized.pages > 16 ||
+    normalized.candidates > 1024 ||
+    normalized.plansUnavailable > normalized.candidates ||
+    normalized.placementMismatches > normalized.candidates ||
+    normalized.claimAttempts > 64 ||
+    normalized.claimRaces > normalized.claimAttempts
+  ) {
+    invalid('stats exceed the reviewed budget');
+  }
+  return normalized;
+}
+
+export function normalizeRemoteExecutionOfferClaimAuthority(
+  value: RemoteExecutionOfferClaimAuthority,
+): RemoteExecutionOfferClaimAuthority {
+  const candidate = object(value, 'claim authority');
+  exactKeys(candidate, [
+    'workerId',
+    'workerSessionId',
+    'workerGeneration',
+    'offerId',
+    'leaseToken',
+  ], 'claim authority');
+  assertWorkerId(value.workerId);
+  assertWorkerSessionId(value.workerSessionId);
+  safeInteger(value.workerGeneration, 'workerGeneration', 1);
+  assertRunDispatchId('offerId', value.offerId);
+  assertRunDispatchLeaseToken(value.leaseToken);
+  return Object.freeze({ ...value });
+}
+
+export function createRemoteExecutionOfferPullBody(
+  result: RemoteExecutionOfferPullResult,
+): RemoteExecutionOfferPullBody {
+  const stats = normalizeStats(result.stats);
+  if (result.status === 'idle') {
+    if (!IDLE_REASONS.includes(result.reason)) invalid('idle reason is invalid');
+    return Object.freeze({
+      schema: REMOTE_EXECUTION_OFFER_DELIVERY_SCHEMA,
+      status: 'idle' as const,
+      reason: result.reason,
+      stats,
+      truncated: boolean(result.truncated, 'truncated'),
+    });
+  }
+  const offer = normalizeOffer(result.offer);
+  return Object.freeze({
+    schema: REMOTE_EXECUTION_OFFER_DELIVERY_SCHEMA,
+    status: 'offered' as const,
+    offer: Object.freeze({
+      offerId: offer.offerId,
+      deliveryKind: offer.deliveryKind,
+      executionDigest: offer.executionDigest,
+      candidate: offer.candidate,
+      worker: offer.worker,
+      lease: Object.freeze({
+        version: offer.lease.version,
+        leaseGeneration: offer.lease.leaseGeneration,
+        acquiredAtMs: offer.lease.acquiredAtMs,
+        renewedAtMs: offer.lease.renewedAtMs,
+        expiresAtMs: offer.lease.expiresAtMs,
+        updatedAtMs: offer.lease.updatedAtMs,
+      }),
+      executionRevision: offer.executionRevision,
+      placementScore: offer.placementScore,
+    }),
+    stats,
+    truncated: boolean(result.truncated, 'truncated'),
+  });
+}
+
+export function parseRemoteExecutionOfferPullResponse(
+  serialized: Uint8Array | string,
+  authorityValue: RemoteExecutionOfferClaimAuthority,
+): RemoteExecutionOfferPullResult {
+  const bytes = typeof serialized === 'string'
+    ? Buffer.from(serialized, 'utf8')
+    : Buffer.from(serialized);
+  if (
+    bytes.byteLength < 2 ||
+    bytes.byteLength > MAX_REMOTE_EXECUTION_OFFER_RESPONSE_BYTES
+  ) {
+    invalid('response byte size is outside the allowed range');
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(bytes.toString('utf8')) as unknown;
+  } catch {
+    return invalid('response is not valid JSON');
+  }
+  const authority = normalizeRemoteExecutionOfferClaimAuthority(authorityValue);
+  const response = object(parsed, 'response');
+  if (response.status === 'idle') {
+    exactKeys(response, [
+      'schema', 'status', 'reason', 'stats', 'truncated',
+    ], 'idle response');
+    if (
+      response.schema !== REMOTE_EXECUTION_OFFER_DELIVERY_SCHEMA ||
+      typeof response.reason !== 'string' ||
+      !IDLE_REASONS.includes(response.reason as RemoteExecutionOfferIdleReason)
+    ) {
+      invalid('idle response fence is invalid');
+    }
+    return Object.freeze({
+      status: 'idle' as const,
+      reason: response.reason as RemoteExecutionOfferIdleReason,
+      stats: normalizeStats(response.stats),
+      truncated: boolean(response.truncated, 'truncated'),
+    });
+  }
+  exactKeys(response, [
+    'schema', 'status', 'offer', 'stats', 'truncated',
+  ], 'offered response');
+  if (
+    response.schema !== REMOTE_EXECUTION_OFFER_DELIVERY_SCHEMA ||
+    response.status !== 'offered'
+  ) {
+    invalid('response schema or status is invalid');
+  }
+  const wire = object(response.offer, 'offer');
+  exactKeys(wire, [
+    'offerId', 'deliveryKind', 'executionDigest', 'candidate', 'worker',
+    'lease', 'executionRevision', 'placementScore',
+  ], 'offer');
+  if (wire.offerId !== authority.offerId) invalid('offerId does not match claim');
+  const worker = object(wire.worker, 'offer.worker');
+  exactKeys(worker, ['workerId', 'sessionId', 'generation'], 'offer.worker');
+  if (
+    worker.workerId !== authority.workerId ||
+    worker.sessionId !== authority.workerSessionId ||
+    worker.generation !== authority.workerGeneration
+  ) {
+    invalid('Worker target does not match claim');
+  }
+  let candidate: ClusterRemoteExecutionOffer['candidate'];
+  try {
+    candidate = normalizeClusterDispatchCandidate(
+      wire.candidate as ClusterRemoteExecutionOffer['candidate'],
+    );
+  } catch {
+    return invalid('offer candidate is invalid');
+  }
+  const lease = object(wire.lease, 'offer.lease');
+  exactKeys(lease, [
+    'version', 'leaseGeneration', 'acquiredAtMs', 'renewedAtMs',
+    'expiresAtMs', 'updatedAtMs',
+  ], 'offer.lease');
+  const offer = normalizeOffer({
+    offerId: authority.offerId,
+    deliveryKind: wire.deliveryKind as ClusterRemoteExecutionOffer['deliveryKind'],
+    executionDigest: wire.executionDigest as string,
+    candidate,
+    worker: {
+      workerId: authority.workerId,
+      sessionId: authority.workerSessionId,
+      generation: authority.workerGeneration,
+    },
+    lease: {
+      attemptId: candidate.attemptId,
+      runId: candidate.runId,
+      status: 'leased',
+      version: safeInteger(lease.version, 'offer.lease.version'),
+      leaseGeneration: safeInteger(
+        lease.leaseGeneration,
+        'offer.lease.leaseGeneration',
+        1,
+      ),
+      workerId: authority.workerId,
+      workerSessionId: authority.workerSessionId,
+      workerGeneration: authority.workerGeneration,
+      leaseTokenDigest: digestRunDispatchLeaseToken(authority.leaseToken),
+      acquiredAtMs: safeInteger(
+        lease.acquiredAtMs,
+        'offer.lease.acquiredAtMs',
+      ),
+      renewedAtMs: safeInteger(
+        lease.renewedAtMs,
+        'offer.lease.renewedAtMs',
+      ),
+      expiresAtMs: safeInteger(
+        lease.expiresAtMs,
+        'offer.lease.expiresAtMs',
+      ),
+      updatedAtMs: safeInteger(
+        lease.updatedAtMs,
+        'offer.lease.updatedAtMs',
+      ),
+    },
+    leaseToken: authority.leaseToken,
+    executionRevision:
+      wire.executionRevision as ClusterRemoteExecutionOffer['executionRevision'],
+    placementScore: wire.placementScore as number,
+  });
+  return Object.freeze({
+    status: 'offered' as const,
+    offer,
+    stats: normalizeStats(response.stats),
+    truncated: boolean(response.truncated, 'truncated'),
+  });
+}

packages/ql3-runtime-core/src/remote-execution/remoteRunActivation.ts

@@ -0,0 +1,198 @@
+import {
+  assertRunDispatchId,
+  assertRunDispatchLeaseFence,
+} from '../run/runDispatchLease';
+import type { RunAttemptStatus, RunStatus } from '../run/run';
+
+export interface RemoteRunActivationFence {
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly workerId: string;
+  readonly workerSessionId: string;
+  readonly workerGeneration: number;
+  readonly offerId: string;
+  readonly leaseGeneration: number;
+  /** Bearer capability. Persistence implementations must store only its digest. */
+  readonly leaseToken: string;
+  readonly expectedLeaseVersion: number;
+}
+
+export interface AcknowledgeRemoteRunStartingCommand
+  extends RemoteRunActivationFence {
+  readonly eventId: string;
+}
+
+export interface AcknowledgeRemoteRunRunningCommand
+  extends RemoteRunActivationFence {
+  readonly attemptEventId: string;
+  readonly runEventId: string;
+  readonly executorHandle: string;
+  readonly logArtifactId?: string;
+  /** Sequence bound to the completion callback capability kept by the Worker. */
+  readonly callbackSequence: number;
+  /** Lowercase SHA-256 digest. The raw callback capability never crosses storage. */
+  readonly callbackTokenDigest: string;
+}
+
+export interface FailRemoteRunStartCommand extends RemoteRunActivationFence {
+  readonly attemptEventId: string;
+  readonly runEventId: string;
+}
+
+export type RemoteRunActivationStatus =
+  | 'applied'
+  | 'already_starting'
+  | 'already_running'
+  | 'already_terminal';
+
+export interface RemoteRunActivationSnapshot {
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly runStatus: RunStatus;
+  readonly attemptStatus: RunAttemptStatus;
+  readonly leaseVersion: number;
+  readonly leaseGeneration: number;
+  readonly callbackSequence: number;
+  /** Database-clock deadline durably bound when starting was acknowledged. */
+  readonly deadlineAtMs?: number;
+  readonly startedAtMs?: number;
+  readonly finishedAtMs?: number;
+  readonly executorHandle?: string;
+  readonly logArtifactId?: string;
+  readonly errorCode?: string;
+}
+
+export interface RemoteRunActivationResult {
+  readonly status: RemoteRunActivationStatus;
+  readonly snapshot: Readonly<RemoteRunActivationSnapshot>;
+}
+
+export interface RemoteRunActivationRepository {
+  acknowledgeStarting(
+    command: AcknowledgeRemoteRunStartingCommand,
+  ): Promise<Readonly<RemoteRunActivationResult>>;
+  acknowledgeRunning(
+    command: AcknowledgeRemoteRunRunningCommand,
+  ): Promise<Readonly<RemoteRunActivationResult>>;
+  failStart(
+    command: FailRemoteRunStartCommand,
+  ): Promise<Readonly<RemoteRunActivationResult>>;
+}
+
+export type RemoteRunActivationFenceReason =
+  | 'missing'
+  | 'run_mismatch'
+  | 'execution_owner_mismatch'
+  | 'executor_mismatch'
+  | 'attempt_state_mismatch'
+  | 'run_state_mismatch'
+  | 'worker_mismatch'
+  | 'worker_session_mismatch'
+  | 'worker_generation_mismatch'
+  | 'lease_generation_mismatch'
+  | 'lease_token_mismatch'
+  | 'offer_mismatch'
+  | 'version_mismatch'
+  | 'lease_expired'
+  | 'worker_unavailable'
+  | 'replay_mismatch';
+
+export class RemoteRunActivationFenceRejectedError extends Error {
+  readonly code = 'REMOTE_RUN_ACTIVATION_FENCED';
+
+  constructor(
+    readonly attemptId: string,
+    readonly reason: RemoteRunActivationFenceReason,
+  ) {
+    super(`Remote Run activation for Attempt ${attemptId} was fenced: ${reason}`);
+    this.name = 'RemoteRunActivationFenceRejectedError';
+  }
+}
+
+export class RemoteRunActivationUnavailableError extends Error {
+  readonly code = 'REMOTE_RUN_ACTIVATION_UNAVAILABLE';
+
+  constructor(options?: ErrorOptions) {
+    super('Remote Run activation storage is unavailable', options);
+    this.name = 'RemoteRunActivationUnavailableError';
+  }
+}
+
+function boundedId(name: string, value: string, maximum: number): void {
+  assertRunDispatchId(name, value);
+  if (value.length > maximum) {
+    throw new TypeError(`Remote Run activation ${name} is invalid`);
+  }
+}
+
+export function assertRemoteRunActivationFence(
+  value: RemoteRunActivationFence,
+): void {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new TypeError('Remote Run activation fence is invalid');
+  }
+  boundedId('runId', value.runId, 36);
+  boundedId('attemptId', value.attemptId, 36);
+  boundedId('offerId', value.offerId, 128);
+  assertRunDispatchLeaseFence({
+    workerId: value.workerId,
+    workerSessionId: value.workerSessionId,
+    workerGeneration: value.workerGeneration,
+    leaseGeneration: value.leaseGeneration,
+    leaseToken: value.leaseToken,
+    expectedVersion: value.expectedLeaseVersion,
+  });
+  for (const number of [
+    value.workerGeneration,
+    value.leaseGeneration,
+    value.expectedLeaseVersion,
+  ]) {
+    if (number > 2_147_483_647) {
+      throw new RangeError('Remote Run activation fence version is invalid');
+    }
+  }
+}
+
+export function assertAcknowledgeRemoteRunStartingCommand(
+  command: AcknowledgeRemoteRunStartingCommand,
+): void {
+  assertRemoteRunActivationFence(command);
+  boundedId('eventId', command.eventId, 36);
+}
+
+export function assertAcknowledgeRemoteRunRunningCommand(
+  command: AcknowledgeRemoteRunRunningCommand,
+): void {
+  assertRemoteRunActivationFence(command);
+  boundedId('attemptEventId', command.attemptEventId, 36);
+  boundedId('runEventId', command.runEventId, 36);
+  if (
+    typeof command.executorHandle !== 'string' ||
+    command.executorHandle.length < 1 ||
+    command.executorHandle.length > 512 ||
+    /[\u0000-\u001f\u007f]/.test(command.executorHandle)
+  ) {
+    throw new TypeError('Remote Run activation executorHandle is invalid');
+  }
+  if (command.logArtifactId !== undefined) {
+    boundedId('logArtifactId', command.logArtifactId, 36);
+  }
+  if (
+    !Number.isSafeInteger(command.callbackSequence) ||
+    command.callbackSequence < 1 ||
+    command.callbackSequence > 2_147_483_647
+  ) {
+    throw new RangeError('Remote Run activation callbackSequence is invalid');
+  }
+  if (!/^[0-9a-f]{64}$/.test(command.callbackTokenDigest)) {
+    throw new TypeError('Remote Run activation callbackTokenDigest is invalid');
+  }
+}
+
+export function assertFailRemoteRunStartCommand(
+  command: FailRemoteRunStartCommand,
+): void {
+  assertRemoteRunActivationFence(command);
+  boundedId('attemptEventId', command.attemptEventId, 36);
+  boundedId('runEventId', command.runEventId, 36);
+}

packages/ql3-runtime-core/src/remote-execution/remoteRunActivationDelivery.ts

@@ -0,0 +1,258 @@
+import { RUN_ATTEMPT_STATUSES, RUN_STATUSES } from '../run/run';
+import type {
+  RemoteRunActivationResult,
+  RemoteRunActivationSnapshot,
+  RemoteRunActivationStatus,
+} from './remoteRunActivation';
+import { assertRunDispatchId } from '../run/runDispatchLease';
+
+export const REMOTE_RUN_ACTIVATION_DELIVERY_SCHEMA =
+  'qinglong/remote-run-activation@v1';
+export const MAX_REMOTE_RUN_ACTIVATION_RESPONSE_BYTES = 16 * 1024;
+
+export type RemoteRunActivationResponseBody = Readonly<{
+  schema: typeof REMOTE_RUN_ACTIVATION_DELIVERY_SCHEMA;
+  status: RemoteRunActivationStatus;
+  snapshot: Readonly<RemoteRunActivationSnapshot>;
+}>;
+
+export class InvalidRemoteRunActivationDeliveryError extends TypeError {
+  readonly code = 'REMOTE_RUN_ACTIVATION_DELIVERY_INVALID';
+
+  constructor(message: string) {
+    super(`Remote Run activation delivery is invalid: ${message}`);
+    this.name = 'InvalidRemoteRunActivationDeliveryError';
+  }
+}
+
+const ACTIVATION_STATUSES = new Set<RemoteRunActivationStatus>([
+  'applied',
+  'already_starting',
+  'already_running',
+  'already_terminal',
+]);
+const REQUIRED_SNAPSHOT_KEYS = [
+  'runId',
+  'attemptId',
+  'runStatus',
+  'attemptStatus',
+  'leaseVersion',
+  'leaseGeneration',
+  'callbackSequence',
+] as const;
+const OPTIONAL_SNAPSHOT_KEYS = [
+  'deadlineAtMs',
+  'startedAtMs',
+  'finishedAtMs',
+  'executorHandle',
+  'logArtifactId',
+  'errorCode',
+] as const;
+const SNAPSHOT_KEYS = new Set<string>([
+  ...REQUIRED_SNAPSHOT_KEYS,
+  ...OPTIONAL_SNAPSHOT_KEYS,
+]);
+
+function invalid(message: string): never {
+  throw new InvalidRemoteRunActivationDeliveryError(message);
+}
+
+function object(value: unknown, label: string): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return invalid(`${label} is not an object`);
+  }
+  return value as Record<string, unknown>;
+}
+
+function exactKeys(
+  value: Record<string, unknown>,
+  expected: readonly string[],
+  label: string,
+): void {
+  const actual = Object.keys(value).sort();
+  const sorted = [...expected].sort();
+  if (
+    actual.length !== sorted.length ||
+    actual.some((key, index) => key !== sorted[index])
+  ) {
+    invalid(`${label} shape is invalid`);
+  }
+}
+
+function boundedId(label: string, value: unknown, maximum: number): string {
+  if (typeof value !== 'string') return invalid(`${label} is invalid`);
+  try {
+    assertRunDispatchId(label, value);
+  } catch {
+    return invalid(`${label} is invalid`);
+  }
+  if (value.length > maximum) return invalid(`${label} is invalid`);
+  return value;
+}
+
+function boundedInteger(
+  label: string,
+  value: unknown,
+  maximum = 2_147_483_647,
+): number {
+  if (
+    !Number.isSafeInteger(value) ||
+    (value as number) < 0 ||
+    (value as number) > maximum
+  ) {
+    return invalid(`${label} is invalid`);
+  }
+  return value as number;
+}
+
+function boundedText(label: string, value: unknown, maximum: number): string {
+  if (
+    typeof value !== 'string' ||
+    value.length < 1 ||
+    value.length > maximum ||
+    /[\u0000-\u001f\u007f]/.test(value)
+  ) {
+    return invalid(`${label} is invalid`);
+  }
+  return value;
+}
+
+function normalizeSnapshot(value: unknown): Readonly<RemoteRunActivationSnapshot> {
+  const snapshot = object(value, 'snapshot');
+  const keys = Object.keys(snapshot);
+  if (
+    REQUIRED_SNAPSHOT_KEYS.some((key) => !Object.hasOwn(snapshot, key)) ||
+    keys.some((key) => !SNAPSHOT_KEYS.has(key))
+  ) {
+    return invalid('snapshot shape is invalid');
+  }
+  if (
+    typeof snapshot.runStatus !== 'string' ||
+    !RUN_STATUSES.includes(snapshot.runStatus as never) ||
+    typeof snapshot.attemptStatus !== 'string' ||
+    !RUN_ATTEMPT_STATUSES.includes(snapshot.attemptStatus as never)
+  ) {
+    return invalid('snapshot status is invalid');
+  }
+  const normalized: RemoteRunActivationSnapshot = {
+    runId: boundedId('runId', snapshot.runId, 36),
+    attemptId: boundedId('attemptId', snapshot.attemptId, 36),
+    runStatus: snapshot.runStatus as RemoteRunActivationSnapshot['runStatus'],
+    attemptStatus:
+      snapshot.attemptStatus as RemoteRunActivationSnapshot['attemptStatus'],
+    leaseVersion: boundedInteger('leaseVersion', snapshot.leaseVersion),
+    leaseGeneration: boundedInteger(
+      'leaseGeneration',
+      snapshot.leaseGeneration,
+    ),
+    callbackSequence: boundedInteger(
+      'callbackSequence',
+      snapshot.callbackSequence,
+    ),
+    ...(snapshot.deadlineAtMs === undefined
+      ? {}
+      : { deadlineAtMs: boundedInteger('deadlineAtMs', snapshot.deadlineAtMs, Number.MAX_SAFE_INTEGER) }),
+    ...(snapshot.startedAtMs === undefined
+      ? {}
+      : { startedAtMs: boundedInteger('startedAtMs', snapshot.startedAtMs, Number.MAX_SAFE_INTEGER) }),
+    ...(snapshot.finishedAtMs === undefined
+      ? {}
+      : { finishedAtMs: boundedInteger('finishedAtMs', snapshot.finishedAtMs, Number.MAX_SAFE_INTEGER) }),
+    ...(snapshot.executorHandle === undefined
+      ? {}
+      : { executorHandle: boundedText('executorHandle', snapshot.executorHandle, 512) }),
+    ...(snapshot.logArtifactId === undefined
+      ? {}
+      : { logArtifactId: boundedId('logArtifactId', snapshot.logArtifactId, 36) }),
+    ...(snapshot.errorCode === undefined
+      ? {}
+      : { errorCode: boundedText('errorCode', snapshot.errorCode, 128) }),
+  };
+  return Object.freeze(normalized);
+}
+
+function normalizeResult(value: unknown): Readonly<RemoteRunActivationResult> {
+  const result = object(value, 'result');
+  exactKeys(result, ['status', 'snapshot'], 'result');
+  if (
+    typeof result.status !== 'string' ||
+    !ACTIVATION_STATUSES.has(result.status as RemoteRunActivationStatus)
+  ) {
+    return invalid('status is invalid');
+  }
+  const normalized = Object.freeze({
+    status: result.status as RemoteRunActivationStatus,
+    snapshot: normalizeSnapshot(result.snapshot),
+  });
+  const snapshot = normalized.snapshot;
+  const starting = snapshot.runStatus === 'dispatching' &&
+    snapshot.attemptStatus === 'starting' &&
+    snapshot.executorHandle === undefined &&
+    snapshot.startedAtMs === undefined &&
+    snapshot.finishedAtMs === undefined &&
+    snapshot.errorCode === undefined;
+  const running = snapshot.runStatus === 'running' &&
+    snapshot.attemptStatus === 'running' &&
+    snapshot.executorHandle !== undefined &&
+    snapshot.startedAtMs !== undefined &&
+    snapshot.finishedAtMs === undefined &&
+    snapshot.errorCode === undefined;
+  const terminalStatus = snapshot.runStatus === snapshot.attemptStatus &&
+    ['failed', 'cancelled', 'timed_out'].includes(snapshot.runStatus);
+  const terminal = terminalStatus &&
+    snapshot.executorHandle === undefined &&
+    snapshot.startedAtMs === undefined &&
+    snapshot.finishedAtMs !== undefined &&
+    snapshot.errorCode !== undefined;
+  if (!starting && !running && !terminal) {
+    return invalid('snapshot state is invalid');
+  }
+  if (
+    (normalized.status === 'already_starting' && !starting) ||
+    (normalized.status === 'already_running' && !running) ||
+    (normalized.status === 'already_terminal' && !terminal)
+  ) {
+    return invalid('status and snapshot state disagree');
+  }
+  return normalized;
+}
+
+export function createRemoteRunActivationResponseBody(
+  result: Readonly<RemoteRunActivationResult>,
+): RemoteRunActivationResponseBody {
+  const normalized = normalizeResult(result);
+  return Object.freeze({
+    schema: REMOTE_RUN_ACTIVATION_DELIVERY_SCHEMA,
+    status: normalized.status,
+    snapshot: normalized.snapshot,
+  });
+}
+
+export function parseRemoteRunActivationResponse(
+  serialized: Uint8Array | string,
+): Readonly<RemoteRunActivationResult> {
+  const bytes = typeof serialized === 'string'
+    ? Buffer.from(serialized, 'utf8')
+    : Buffer.from(serialized);
+  if (
+    bytes.byteLength < 2 ||
+    bytes.byteLength > MAX_REMOTE_RUN_ACTIVATION_RESPONSE_BYTES
+  ) {
+    return invalid('response byte size is outside the allowed range');
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(bytes.toString('utf8')) as unknown;
+  } catch {
+    return invalid('response is not valid JSON');
+  }
+  const response = object(parsed, 'response');
+  exactKeys(response, ['schema', 'status', 'snapshot'], 'response');
+  if (response.schema !== REMOTE_RUN_ACTIVATION_DELIVERY_SCHEMA) {
+    return invalid('response schema is invalid');
+  }
+  return normalizeResult({
+    status: response.status,
+    snapshot: response.snapshot,
+  });
+}

packages/ql3-runtime-core/src/remote-execution/remoteSecretDelivery.ts

@@ -0,0 +1,393 @@
+import { digestRunDispatchLeaseToken, assertRunDispatchId } from '../run/runDispatchLease';
+import { parseSecretRef } from '../secret/secretReference';
+import { assertWorkerId, assertWorkerSessionId } from '../worker/workerSession';
+
+export const REMOTE_SECRET_DELIVERY_SCHEMA =
+  'qinglong/remote-secret-delivery@v1';
+export const MAX_REMOTE_SECRET_DELIVERY_REFS = 64;
+export const MAX_REMOTE_SECRET_DELIVERY_REQUEST_BYTES = 64 * 1024;
+export const MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES = 128 * 1024;
+export const MAX_REMOTE_SECRET_VALUE_BYTES = 16 * 1024;
+export const MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES = 64 * 1024;
+
+export interface RemoteWorkerSecretDeliveryCommand {
+  readonly workerId: string;
+  readonly workerSessionId: string;
+  readonly workerGeneration: number;
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly projectId: string;
+  readonly taskId: string;
+  readonly taskRevision: string;
+  readonly executionDigest: string;
+  readonly offerId: string;
+  readonly leaseGeneration: number;
+  readonly leaseToken: string;
+  readonly expectedLeaseVersion: number;
+  readonly secretRefs: readonly string[];
+}
+
+export type RemoteWorkerSecretDeliveryRequestBody = Readonly<
+  Omit<RemoteWorkerSecretDeliveryCommand, 'workerId' | 'workerSessionId'> & {
+    readonly schema: typeof REMOTE_SECRET_DELIVERY_SCHEMA;
+  }
+>;
+
+export interface RemoteWorkerSecretDeliveryAuthority {
+  readonly workerId: string;
+  readonly workerSessionId: string;
+  readonly workerGeneration: number;
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly projectId: string;
+  readonly taskId: string;
+  readonly taskRevision: string;
+  readonly executionDigest: string;
+  readonly offerId: string;
+  readonly leaseGeneration: number;
+  readonly leaseVersion: number;
+  readonly secretRefs: readonly string[];
+}
+
+export interface RemoteWorkerSecretValue {
+  readonly secretRef: string;
+  readonly value: string;
+}
+
+export interface RemoteWorkerSecretResolution {
+  readonly values: readonly RemoteWorkerSecretValue[];
+  readonly dispose?: () => Promise<void> | void;
+}
+
+export interface RemoteWorkerSecretDeliveryAuthorityRepository {
+  authorize(
+    command: RemoteWorkerSecretDeliveryCommand,
+  ): Promise<Readonly<RemoteWorkerSecretDeliveryAuthority>>;
+}
+
+export interface RemoteWorkerSecretValueProvider {
+  resolve(
+    authority: Readonly<RemoteWorkerSecretDeliveryAuthority>,
+  ): Promise<Readonly<RemoteWorkerSecretResolution> | undefined>;
+}
+
+export interface RemoteWorkerSecretDeliveryResult {
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly offerId: string;
+  readonly executionDigest: string;
+  readonly values: readonly RemoteWorkerSecretValue[];
+  readonly dispose?: () => Promise<void> | void;
+}
+
+export type RemoteWorkerSecretDeliveryResponseBody = Readonly<
+  Omit<RemoteWorkerSecretDeliveryResult, 'dispose'> & {
+    readonly schema: typeof REMOTE_SECRET_DELIVERY_SCHEMA;
+  }
+>;
+
+export class InvalidRemoteWorkerSecretDeliveryError extends TypeError {
+  readonly code = 'REMOTE_SECRET_DELIVERY_INVALID';
+
+  constructor(message: string) {
+    super(`Remote Worker Secret delivery is invalid: ${message}`);
+    this.name = 'InvalidRemoteWorkerSecretDeliveryError';
+  }
+}
+
+export class RemoteWorkerSecretDeliveryFenceRejectedError extends Error {
+  readonly code = 'REMOTE_SECRET_DELIVERY_FENCED';
+
+  constructor(readonly reason: 'authority_mismatch' | 'secret_scope_mismatch') {
+    super(`Remote Worker Secret delivery is fenced: ${reason}`);
+    this.name = 'RemoteWorkerSecretDeliveryFenceRejectedError';
+  }
+}
+
+export class RemoteWorkerSecretDeliveryUnavailableError extends Error {
+  readonly code = 'REMOTE_SECRET_DELIVERY_UNAVAILABLE';
+
+  constructor() {
+    super('Remote Worker Secret delivery is unavailable');
+    this.name = 'RemoteWorkerSecretDeliveryUnavailableError';
+  }
+}
+
+function invalid(message: string): never {
+  throw new InvalidRemoteWorkerSecretDeliveryError(message);
+}
+
+function object(value: unknown, label: string): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return invalid(`${label} is not an object`);
+  }
+  return value as Record<string, unknown>;
+}
+
+function exactKeys(
+  value: Record<string, unknown>,
+  expected: readonly string[],
+  label: string,
+): void {
+  const actual = Object.keys(value).sort();
+  const sorted = [...expected].sort();
+  if (
+    actual.length !== sorted.length ||
+    actual.some((key, index) => key !== sorted[index])
+  ) invalid(`${label} shape is invalid`);
+}
+
+function identifier(label: string, value: unknown, maximum = 128): string {
+  if (
+    typeof value !== 'string' ||
+    value.length < 1 ||
+    Buffer.byteLength(value, 'utf8') > maximum ||
+    /[\u0000-\u001f\u007f]/.test(value)
+  ) return invalid(`${label} is invalid`);
+  return value;
+}
+
+function positiveInteger(label: string, value: unknown, minimum = 1): number {
+  if (
+    !Number.isSafeInteger(value) ||
+    (value as number) < minimum ||
+    (value as number) > 2_147_483_647
+  ) return invalid(`${label} is invalid`);
+  return value as number;
+}
+
+function normalizeSecretRefs(
+  value: unknown,
+  projectId: string,
+): readonly string[] {
+  if (
+    !Array.isArray(value) ||
+    value.length < 1 ||
+    value.length > MAX_REMOTE_SECRET_DELIVERY_REFS
+  ) return invalid('secretRefs are invalid');
+  const seen = new Set<string>();
+  const refs = value.map((entry) => {
+    if (typeof entry !== 'string' || seen.has(entry)) {
+      return invalid('secretRefs are invalid');
+    }
+    try {
+      if (parseSecretRef(entry).projectId !== projectId) {
+        return invalid('secretRef project is invalid');
+      }
+    } catch (error) {
+      if (error instanceof InvalidRemoteWorkerSecretDeliveryError) throw error;
+      return invalid('secretRef is invalid');
+    }
+    seen.add(entry);
+    return entry;
+  });
+  return Object.freeze(refs);
+}
+
+export function normalizeRemoteWorkerSecretDeliveryCommand(
+  value: RemoteWorkerSecretDeliveryCommand,
+): Readonly<RemoteWorkerSecretDeliveryCommand> {
+  const command = object(value, 'command');
+  exactKeys(command, [
+    'attemptId', 'executionDigest', 'expectedLeaseVersion', 'leaseGeneration',
+    'leaseToken', 'offerId', 'projectId', 'runId', 'secretRefs', 'taskId',
+    'taskRevision', 'workerGeneration', 'workerId', 'workerSessionId',
+  ], 'command');
+  try {
+    assertWorkerId(command.workerId as string);
+    assertWorkerSessionId(command.workerSessionId as string);
+    assertRunDispatchId('runId', command.runId as string);
+    assertRunDispatchId('attemptId', command.attemptId as string);
+    assertRunDispatchId('offerId', command.offerId as string);
+  } catch {
+    return invalid('authority identifier is invalid');
+  }
+  const projectId = identifier('projectId', command.projectId);
+  const normalized = Object.freeze({
+    workerId: command.workerId as string,
+    workerSessionId: command.workerSessionId as string,
+    workerGeneration: positiveInteger('workerGeneration', command.workerGeneration),
+    runId: command.runId as string,
+    attemptId: command.attemptId as string,
+    projectId,
+    taskId: identifier('taskId', command.taskId),
+    taskRevision: identifier('taskRevision', command.taskRevision),
+    executionDigest: identifier('executionDigest', command.executionDigest, 64),
+    offerId: command.offerId as string,
+    leaseGeneration: positiveInteger('leaseGeneration', command.leaseGeneration),
+    leaseToken: identifier('leaseToken', command.leaseToken, 128),
+    expectedLeaseVersion: positiveInteger(
+      'expectedLeaseVersion', command.expectedLeaseVersion, 0,
+    ),
+    secretRefs: normalizeSecretRefs(command.secretRefs, projectId),
+  });
+  if (!/^[0-9a-f]{64}$/.test(normalized.executionDigest)) {
+    return invalid('executionDigest is invalid');
+  }
+  try {
+    digestRunDispatchLeaseToken(normalized.leaseToken);
+  } catch {
+    return invalid('leaseToken is invalid');
+  }
+  return normalized;
+}
+
+export function normalizeRemoteWorkerSecretDeliveryAuthority(
+  value: RemoteWorkerSecretDeliveryAuthority,
+): Readonly<RemoteWorkerSecretDeliveryAuthority> {
+  const authority = object(value, 'authority');
+  exactKeys(authority, [
+    'attemptId', 'executionDigest', 'leaseGeneration', 'leaseVersion',
+    'offerId', 'projectId', 'runId', 'secretRefs', 'taskId', 'taskRevision',
+    'workerGeneration', 'workerId', 'workerSessionId',
+  ], 'authority');
+  try {
+    assertWorkerId(authority.workerId as string);
+    assertWorkerSessionId(authority.workerSessionId as string);
+    assertRunDispatchId('runId', authority.runId as string);
+    assertRunDispatchId('attemptId', authority.attemptId as string);
+    assertRunDispatchId('offerId', authority.offerId as string);
+  } catch {
+    return invalid('authority identifier is invalid');
+  }
+  const projectId = identifier('projectId', authority.projectId);
+  const normalized = Object.freeze({
+    workerId: authority.workerId as string,
+    workerSessionId: authority.workerSessionId as string,
+    workerGeneration: positiveInteger(
+      'workerGeneration', authority.workerGeneration,
+    ),
+    runId: authority.runId as string,
+    attemptId: authority.attemptId as string,
+    projectId,
+    taskId: identifier('taskId', authority.taskId),
+    taskRevision: identifier('taskRevision', authority.taskRevision),
+    executionDigest: identifier(
+      'executionDigest', authority.executionDigest, 64,
+    ),
+    offerId: authority.offerId as string,
+    leaseGeneration: positiveInteger(
+      'leaseGeneration', authority.leaseGeneration,
+    ),
+    leaseVersion: positiveInteger('leaseVersion', authority.leaseVersion, 0),
+    secretRefs: normalizeSecretRefs(authority.secretRefs, projectId),
+  });
+  if (!/^[0-9a-f]{64}$/.test(normalized.executionDigest)) {
+    return invalid('executionDigest is invalid');
+  }
+  return normalized;
+}
+
+export function createRemoteWorkerSecretDeliveryRequestBody(
+  command: RemoteWorkerSecretDeliveryCommand,
+): RemoteWorkerSecretDeliveryRequestBody {
+  const normalized = normalizeRemoteWorkerSecretDeliveryCommand(command);
+  const { workerId: _workerId, workerSessionId: _sessionId, ...request } = normalized;
+  return Object.freeze({ schema: REMOTE_SECRET_DELIVERY_SCHEMA, ...request });
+}
+
+function normalizeValues(
+  value: unknown,
+  expectedRefs: readonly string[],
+): readonly RemoteWorkerSecretValue[] {
+  if (!Array.isArray(value) || value.length !== expectedRefs.length) {
+    return invalid('Secret values are invalid');
+  }
+  let totalValueBytes = 0;
+  return Object.freeze(value.map((entry, index) => {
+    const item = object(entry, `values[${index}]`);
+    exactKeys(item, ['secretRef', 'value'], `values[${index}]`);
+    if (
+      item.secretRef !== expectedRefs[index] ||
+      typeof item.value !== 'string' ||
+      item.value.includes('\0') ||
+      Buffer.byteLength(item.value, 'utf8') > MAX_REMOTE_SECRET_VALUE_BYTES
+    ) return invalid(`values[${index}] is invalid`);
+    totalValueBytes += Buffer.byteLength(item.value, 'utf8');
+    if (totalValueBytes > MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES) {
+      return invalid('Secret value byte budget exceeded');
+    }
+    return Object.freeze({
+      secretRef: item.secretRef as string,
+      value: item.value,
+    });
+  }));
+}
+
+export function createRemoteWorkerSecretDeliveryResponseBody(
+  result: Readonly<RemoteWorkerSecretDeliveryResult>,
+  expectedRefs: readonly string[],
+): RemoteWorkerSecretDeliveryResponseBody {
+  const value = object(result, 'result');
+  const allowed = ['attemptId', 'dispose', 'executionDigest', 'offerId', 'runId', 'values'];
+  if (Object.keys(value).some((key) => !allowed.includes(key))) {
+    return invalid('result shape is invalid');
+  }
+  const runId = identifier('runId', value.runId, 36);
+  const attemptId = identifier('attemptId', value.attemptId, 36);
+  const offerId = identifier('offerId', value.offerId, 128);
+  const executionDigest = identifier('executionDigest', value.executionDigest, 64);
+  if (!/^[0-9a-f]{64}$/.test(executionDigest)) invalid('executionDigest is invalid');
+  return Object.freeze({
+    schema: REMOTE_SECRET_DELIVERY_SCHEMA,
+    runId,
+    attemptId,
+    offerId,
+    executionDigest,
+    values: normalizeValues(value.values, expectedRefs),
+  });
+}
+
+export function parseRemoteWorkerSecretDeliveryResponse(
+  serialized: Uint8Array | string,
+  expected: Readonly<{
+    runId: string;
+    attemptId: string;
+    offerId: string;
+    executionDigest: string;
+    secretRefs: readonly string[];
+  }>,
+): Readonly<RemoteWorkerSecretDeliveryResult> {
+  const bytes = typeof serialized === 'string'
+    ? Buffer.from(serialized, 'utf8')
+    : Buffer.from(serialized);
+  if (
+    bytes.byteLength < 2 ||
+    bytes.byteLength > MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES
+  ) return invalid('response byte size is outside the allowed range');
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(bytes.toString('utf8')) as unknown;
+  } catch {
+    return invalid('response is not valid JSON');
+  } finally {
+    bytes.fill(0);
+  }
+  const response = object(parsed, 'response');
+  exactKeys(response, [
+    'attemptId', 'executionDigest', 'offerId', 'runId', 'schema', 'values',
+  ], 'response');
+  if (response.schema !== REMOTE_SECRET_DELIVERY_SCHEMA) {
+    return invalid('response schema is invalid');
+  }
+  const result = createRemoteWorkerSecretDeliveryResponseBody({
+    runId: response.runId as string,
+    attemptId: response.attemptId as string,
+    offerId: response.offerId as string,
+    executionDigest: response.executionDigest as string,
+    values: response.values as readonly RemoteWorkerSecretValue[],
+  }, expected.secretRefs);
+  if (
+    result.runId !== expected.runId ||
+    result.attemptId !== expected.attemptId ||
+    result.offerId !== expected.offerId ||
+    result.executionDigest !== expected.executionDigest
+  ) return invalid('response authority does not match request');
+  return Object.freeze({
+    runId: result.runId,
+    attemptId: result.attemptId,
+    offerId: result.offerId,
+    executionDigest: result.executionDigest,
+    values: result.values,
+  });
+}

packages/ql3-runtime-core/src/remote-execution/remoteWorkerCompletion.ts

@@ -0,0 +1,657 @@
+import { assertRunDispatchId, digestRunDispatchLeaseToken } from '../run/runDispatchLease';
+import { assertWorkerId, assertWorkerSessionId } from '../worker/workerSession';
+
+export const REMOTE_WORKER_ARTIFACT_UPLOAD_SCHEMA =
+  'qinglong/remote-worker-artifact-upload@v1';
+export const REMOTE_WORKER_COMPLETION_SCHEMA =
+  'qinglong/remote-worker-completion@v1';
+export const REMOTE_WORKER_ARTIFACT_CONTENT_TYPE =
+  'application/vnd.qinglong.worker-artifact';
+export const MAX_REMOTE_WORKER_ARTIFACT_BYTES = 64 * 1024 * 1024;
+export const MAX_REMOTE_WORKER_ARTIFACT_HEADER_BYTES = 4 * 1024;
+export const MAX_REMOTE_WORKER_ARTIFACT_RESPONSE_BYTES = 4 * 1024;
+export const MAX_REMOTE_WORKER_COMPLETION_REQUEST_BYTES = 16 * 1024;
+export const MAX_REMOTE_WORKER_COMPLETION_RESPONSE_BYTES = 4 * 1024;
+
+const SHA256 = /^[a-f0-9]{64}$/;
+const LOG_ARTIFACT_ID = /^wlog-[a-f0-9]{30}$/;
+const UPLOAD_STATUSES = new Set(['stored', 'already_stored']);
+const COMPLETION_STATUSES = new Set([
+  'applied',
+  'already_completed',
+  'already_terminal',
+]);
+
+export interface RemoteWorkerExecutionFence {
+  readonly workerId: string;
+  readonly workerSessionId: string;
+  readonly workerGeneration: number;
+  readonly projectId: string;
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly offerId: string;
+  readonly leaseGeneration: number;
+  readonly leaseToken: string;
+  readonly expectedLeaseVersion: number;
+}
+
+export interface RemoteWorkerArtifactUploadCommand
+  extends RemoteWorkerExecutionFence {
+  readonly logArtifactId: string;
+  readonly byteLength: number;
+  readonly truncated?: boolean;
+}
+
+export type RemoteWorkerArtifactUploadRequestHeader = Readonly<
+  Omit<
+    RemoteWorkerArtifactUploadCommand,
+    'workerId' | 'workerSessionId' | 'truncated'
+  > & {
+    readonly schema: typeof REMOTE_WORKER_ARTIFACT_UPLOAD_SCHEMA;
+    readonly truncated: boolean | null;
+  }
+>;
+
+export interface RemoteWorkerArtifactReceipt {
+  readonly status: 'stored' | 'already_stored';
+  readonly projectId: string;
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly logArtifactId: string;
+  readonly byteLength: number;
+  readonly sha256: string;
+  readonly truncated?: boolean;
+}
+
+export type RemoteWorkerArtifactUploadResponseBody = Readonly<
+  Omit<RemoteWorkerArtifactReceipt, 'truncated'> & {
+    readonly schema: typeof REMOTE_WORKER_ARTIFACT_UPLOAD_SCHEMA;
+    readonly truncated: boolean | null;
+  }
+>;
+
+export interface RemoteWorkerCompletionCommand
+  extends RemoteWorkerExecutionFence {
+  readonly callbackSequence: number;
+  readonly callbackTokenDigest: string;
+  readonly result: Readonly<{
+    readonly outcome: 'succeeded' | 'failed';
+    readonly startedAtMs: number;
+    readonly finishedAtMs: number;
+    readonly exitCode: number;
+  }>;
+  readonly artifact: Readonly<{
+    readonly logArtifactId: string;
+    readonly byteLength: number;
+    readonly sha256: string;
+    readonly truncated?: boolean;
+  }>;
+}
+
+export type RemoteWorkerCompletionRequestBody = Readonly<
+  Omit<
+    RemoteWorkerCompletionCommand,
+    'workerId' | 'workerSessionId' | 'artifact'
+  > & {
+    readonly schema: typeof REMOTE_WORKER_COMPLETION_SCHEMA;
+    readonly artifact: Readonly<
+      Omit<RemoteWorkerCompletionCommand['artifact'], 'truncated'> & {
+        readonly truncated: boolean | null;
+      }
+    >;
+  }
+>;
+
+export interface RemoteWorkerCompletionResult {
+  readonly status: 'applied' | 'already_completed' | 'already_terminal';
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly callbackSequence: number;
+}
+
+export type RemoteWorkerCompletionResponseBody = Readonly<
+  RemoteWorkerCompletionResult & {
+    readonly schema: typeof REMOTE_WORKER_COMPLETION_SCHEMA;
+  }
+>;
+
+export interface RemoteWorkerArtifactUploadAuthorityRepository {
+  authorizeArtifactUpload(
+    command: RemoteWorkerArtifactUploadCommand,
+  ): Promise<void>;
+}
+
+export interface RemoteWorkerCompletionRepository {
+  complete(
+    command: RemoteWorkerCompletionCommand & Readonly<{
+      attemptEventId: string;
+      runEventId: string;
+    }>,
+  ): Promise<Readonly<RemoteWorkerCompletionResult>>;
+}
+
+export class InvalidRemoteWorkerCompletionError extends TypeError {
+  readonly code = 'REMOTE_WORKER_COMPLETION_INVALID';
+
+  constructor(message: string) {
+    super(`Remote Worker completion is invalid: ${message}`);
+    this.name = 'InvalidRemoteWorkerCompletionError';
+  }
+}
+
+export class RemoteWorkerCompletionFenceRejectedError extends Error {
+  readonly code = 'REMOTE_WORKER_COMPLETION_FENCED';
+
+  constructor(
+    readonly attemptId: string,
+    readonly reason:
+      | 'missing'
+      | 'worker_unavailable'
+      | 'authority_mismatch'
+      | 'lease_expired'
+      | 'state_mismatch'
+      | 'replay_mismatch',
+  ) {
+    super(`Remote Worker completion is fenced: ${reason}`);
+    this.name = 'RemoteWorkerCompletionFenceRejectedError';
+  }
+}
+
+export class RemoteWorkerCompletionUnavailableError extends Error {
+  readonly code = 'REMOTE_WORKER_COMPLETION_UNAVAILABLE';
+
+  constructor(options?: ErrorOptions) {
+    super('Remote Worker completion is unavailable', options);
+    this.name = 'RemoteWorkerCompletionUnavailableError';
+  }
+}
+
+function invalid(message: string): never {
+  throw new InvalidRemoteWorkerCompletionError(message);
+}
+
+function object(value: unknown, label: string): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return invalid(`${label} is not an object`);
+  }
+  return value as Record<string, unknown>;
+}
+
+function exactKeys(
+  value: Record<string, unknown>,
+  expected: readonly string[],
+  label: string,
+): void {
+  const actual = Object.keys(value).sort();
+  const sorted = [...expected].sort();
+  if (
+    actual.length !== sorted.length ||
+    actual.some((key, index) => key !== sorted[index])
+  ) invalid(`${label} shape is invalid`);
+}
+
+function identifier(label: string, value: unknown, maximum = 128): string {
+  if (
+    typeof value !== 'string' ||
+    value.length < 1 ||
+    Buffer.byteLength(value, 'utf8') > maximum ||
+    /[\u0000-\u001f\u007f]/.test(value)
+  ) return invalid(`${label} is invalid`);
+  return value;
+}
+
+function integer(
+  label: string,
+  value: unknown,
+  minimum = 0,
+  maximum = Number.MAX_SAFE_INTEGER,
+): number {
+  if (
+    !Number.isSafeInteger(value) ||
+    (value as number) < minimum ||
+    (value as number) > maximum
+  ) return invalid(`${label} is invalid`);
+  return value as number;
+}
+
+function nullableBoolean(
+  label: string,
+  value: unknown,
+): boolean | undefined {
+  if (value === null) return undefined;
+  if (typeof value !== 'boolean') return invalid(`${label} is invalid`);
+  return value;
+}
+
+function fence(
+  value: Record<string, unknown>,
+): Readonly<RemoteWorkerExecutionFence> {
+  try {
+    assertWorkerId(value.workerId as string);
+    assertWorkerSessionId(value.workerSessionId as string);
+    assertRunDispatchId('runId', value.runId as string);
+    assertRunDispatchId('attemptId', value.attemptId as string);
+    assertRunDispatchId('offerId', value.offerId as string);
+    digestRunDispatchLeaseToken(value.leaseToken as string);
+  } catch {
+    return invalid('execution authority is invalid');
+  }
+  return Object.freeze({
+    workerId: value.workerId as string,
+    workerSessionId: value.workerSessionId as string,
+    workerGeneration: integer(
+      'workerGeneration', value.workerGeneration, 1, 2_147_483_647,
+    ),
+    projectId: identifier('projectId', value.projectId),
+    runId: value.runId as string,
+    attemptId: value.attemptId as string,
+    offerId: value.offerId as string,
+    leaseGeneration: integer(
+      'leaseGeneration', value.leaseGeneration, 1, 2_147_483_647,
+    ),
+    leaseToken: value.leaseToken as string,
+    expectedLeaseVersion: integer(
+      'expectedLeaseVersion', value.expectedLeaseVersion, 0, 2_147_483_647,
+    ),
+  });
+}
+
+const FENCE_KEYS = [
+  'workerId', 'workerSessionId', 'workerGeneration', 'projectId', 'runId',
+  'attemptId', 'offerId', 'leaseGeneration', 'leaseToken',
+  'expectedLeaseVersion',
+] as const;
+
+export function normalizeRemoteWorkerArtifactUploadCommand(
+  value: RemoteWorkerArtifactUploadCommand,
+): Readonly<RemoteWorkerArtifactUploadCommand> {
+  const command = object(value, 'Artifact upload command');
+  const required = [...FENCE_KEYS, 'logArtifactId', 'byteLength'];
+  const keys = Object.keys(command);
+  if (
+    required.some((key) => !Object.hasOwn(command, key)) ||
+    keys.some((key) => ![...required, 'truncated'].includes(key))
+  ) return invalid('Artifact upload command shape is invalid');
+  const authority = fence(command);
+  const logArtifactId = identifier('logArtifactId', command.logArtifactId, 36);
+  if (!LOG_ARTIFACT_ID.test(logArtifactId)) {
+    return invalid('logArtifactId is invalid');
+  }
+  if (
+    command.truncated !== undefined &&
+    typeof command.truncated !== 'boolean'
+  ) return invalid('truncated is invalid');
+  return Object.freeze({
+    ...authority,
+    logArtifactId,
+    byteLength: integer(
+      'byteLength', command.byteLength, 0, MAX_REMOTE_WORKER_ARTIFACT_BYTES,
+    ),
+    ...(command.truncated === undefined
+      ? {}
+      : { truncated: command.truncated as boolean }),
+  });
+}
+
+export function createRemoteWorkerArtifactUploadRequestHeader(
+  command: RemoteWorkerArtifactUploadCommand,
+): RemoteWorkerArtifactUploadRequestHeader {
+  const value = normalizeRemoteWorkerArtifactUploadCommand(command);
+  const { workerId: _workerId, workerSessionId: _sessionId, ...body } = value;
+  return Object.freeze({
+    schema: REMOTE_WORKER_ARTIFACT_UPLOAD_SCHEMA,
+    ...body,
+    truncated: value.truncated ?? null,
+  });
+}
+
+export function createRemoteWorkerArtifactUploadPreamble(
+  command: RemoteWorkerArtifactUploadCommand,
+): Buffer {
+  const serialized = Buffer.from(JSON.stringify(
+    createRemoteWorkerArtifactUploadRequestHeader(command),
+  ), 'utf8');
+  if (
+    serialized.byteLength < 2 ||
+    serialized.byteLength > MAX_REMOTE_WORKER_ARTIFACT_HEADER_BYTES
+  ) {
+    serialized.fill(0);
+    return invalid('Artifact upload header exceeds its byte limit');
+  }
+  const result = Buffer.allocUnsafe(4 + serialized.byteLength);
+  result.writeUInt32BE(serialized.byteLength, 0);
+  serialized.copy(result, 4);
+  serialized.fill(0);
+  return result;
+}
+
+export function parseRemoteWorkerArtifactUploadHeader(
+  serialized: Uint8Array | string,
+  pathAuthority: Readonly<{ workerId: string; workerSessionId: string }>,
+): Readonly<RemoteWorkerArtifactUploadCommand> {
+  const bytes = typeof serialized === 'string'
+    ? Buffer.from(serialized, 'utf8')
+    : Buffer.from(serialized);
+  if (
+    bytes.byteLength < 2 ||
+    bytes.byteLength > MAX_REMOTE_WORKER_ARTIFACT_HEADER_BYTES
+  ) return invalid('Artifact upload header byte size is invalid');
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(bytes.toString('utf8')) as unknown;
+  } catch {
+    return invalid('Artifact upload header is not valid JSON');
+  }
+  const header = object(parsed, 'Artifact upload header');
+  exactKeys(header, [
+    'schema', 'workerGeneration', 'projectId', 'runId', 'attemptId',
+    'offerId', 'leaseGeneration', 'leaseToken', 'expectedLeaseVersion',
+    'logArtifactId', 'byteLength', 'truncated',
+  ], 'Artifact upload header');
+  if (header.schema !== REMOTE_WORKER_ARTIFACT_UPLOAD_SCHEMA) {
+    return invalid('Artifact upload schema is invalid');
+  }
+  return normalizeRemoteWorkerArtifactUploadCommand({
+    workerId: pathAuthority.workerId,
+    workerSessionId: pathAuthority.workerSessionId,
+    workerGeneration: header.workerGeneration as number,
+    projectId: header.projectId as string,
+    runId: header.runId as string,
+    attemptId: header.attemptId as string,
+    offerId: header.offerId as string,
+    leaseGeneration: header.leaseGeneration as number,
+    leaseToken: header.leaseToken as string,
+    expectedLeaseVersion: header.expectedLeaseVersion as number,
+    logArtifactId: header.logArtifactId as string,
+    byteLength: header.byteLength as number,
+    ...(nullableBoolean('truncated', header.truncated) === undefined
+      ? {}
+      : { truncated: header.truncated as boolean }),
+  });
+}
+
+export function normalizeRemoteWorkerArtifactReceipt(
+  value: RemoteWorkerArtifactReceipt,
+): Readonly<RemoteWorkerArtifactReceipt> {
+  const receipt = object(value, 'Artifact receipt');
+  const required = [
+    'status', 'projectId', 'runId', 'attemptId', 'logArtifactId',
+    'byteLength', 'sha256',
+  ];
+  const keys = Object.keys(receipt);
+  if (
+    required.some((key) => !Object.hasOwn(receipt, key)) ||
+    keys.some((key) => ![...required, 'truncated'].includes(key)) ||
+    typeof receipt.status !== 'string' ||
+    !UPLOAD_STATUSES.has(receipt.status)
+  ) return invalid('Artifact receipt shape is invalid');
+  try {
+    assertRunDispatchId('runId', receipt.runId as string);
+    assertRunDispatchId('attemptId', receipt.attemptId as string);
+  } catch {
+    return invalid('Artifact receipt authority is invalid');
+  }
+  const logArtifactId = identifier('logArtifactId', receipt.logArtifactId, 36);
+  if (!LOG_ARTIFACT_ID.test(logArtifactId)) {
+    return invalid('Artifact receipt identity is invalid');
+  }
+  const sha256 = identifier('sha256', receipt.sha256, 64);
+  if (!SHA256.test(sha256)) return invalid('Artifact receipt digest is invalid');
+  if (
+    receipt.truncated !== undefined &&
+    typeof receipt.truncated !== 'boolean'
+  ) return invalid('Artifact receipt truncation is invalid');
+  return Object.freeze({
+    status: receipt.status as RemoteWorkerArtifactReceipt['status'],
+    projectId: identifier('projectId', receipt.projectId),
+    runId: receipt.runId as string,
+    attemptId: receipt.attemptId as string,
+    logArtifactId,
+    byteLength: integer(
+      'byteLength', receipt.byteLength, 0, MAX_REMOTE_WORKER_ARTIFACT_BYTES,
+    ),
+    sha256,
+    ...(receipt.truncated === undefined
+      ? {}
+      : { truncated: receipt.truncated as boolean }),
+  });
+}
+
+export function createRemoteWorkerArtifactUploadResponseBody(
+  receipt: RemoteWorkerArtifactReceipt,
+): RemoteWorkerArtifactUploadResponseBody {
+  const value = normalizeRemoteWorkerArtifactReceipt(receipt);
+  return Object.freeze({
+    schema: REMOTE_WORKER_ARTIFACT_UPLOAD_SCHEMA,
+    ...value,
+    truncated: value.truncated ?? null,
+  });
+}
+
+export function parseRemoteWorkerArtifactUploadResponse(
+  serialized: Uint8Array | string,
+): Readonly<RemoteWorkerArtifactReceipt> {
+  const bytes = typeof serialized === 'string'
+    ? Buffer.from(serialized, 'utf8')
+    : Buffer.from(serialized);
+  if (
+    bytes.byteLength < 2 ||
+    bytes.byteLength > MAX_REMOTE_WORKER_ARTIFACT_RESPONSE_BYTES
+  ) return invalid('Artifact response byte size is invalid');
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(bytes.toString('utf8')) as unknown;
+  } catch {
+    return invalid('Artifact response is not valid JSON');
+  }
+  const response = object(parsed, 'Artifact response');
+  exactKeys(response, [
+    'schema', 'status', 'projectId', 'runId', 'attemptId', 'logArtifactId',
+    'byteLength', 'sha256', 'truncated',
+  ], 'Artifact response');
+  if (response.schema !== REMOTE_WORKER_ARTIFACT_UPLOAD_SCHEMA) {
+    return invalid('Artifact response schema is invalid');
+  }
+  const truncated = nullableBoolean('truncated', response.truncated);
+  return normalizeRemoteWorkerArtifactReceipt({
+    status: response.status as RemoteWorkerArtifactReceipt['status'],
+    projectId: response.projectId as string,
+    runId: response.runId as string,
+    attemptId: response.attemptId as string,
+    logArtifactId: response.logArtifactId as string,
+    byteLength: response.byteLength as number,
+    sha256: response.sha256 as string,
+    ...(truncated === undefined ? {} : { truncated }),
+  });
+}
+
+export function normalizeRemoteWorkerCompletionCommand(
+  value: RemoteWorkerCompletionCommand,
+): Readonly<RemoteWorkerCompletionCommand> {
+  const command = object(value, 'completion command');
+  exactKeys(command, [
+    ...FENCE_KEYS, 'callbackSequence', 'callbackTokenDigest', 'result',
+    'artifact',
+  ], 'completion command');
+  const authority = fence(command);
+  const result = object(command.result, 'completion result');
+  exactKeys(result, [
+    'outcome', 'startedAtMs', 'finishedAtMs', 'exitCode',
+  ], 'completion result');
+  if (
+    result.outcome !== 'succeeded' &&
+    result.outcome !== 'failed'
+  ) return invalid('completion outcome is invalid');
+  const startedAtMs = integer('startedAtMs', result.startedAtMs);
+  const finishedAtMs = integer('finishedAtMs', result.finishedAtMs);
+  const exitCode = integer('exitCode', result.exitCode, 0, 255);
+  if (
+    finishedAtMs < startedAtMs ||
+    (result.outcome === 'succeeded') !== (exitCode === 0)
+  ) return invalid('completion result is inconsistent');
+  const artifact = object(command.artifact, 'completion Artifact');
+  const requiredArtifact = ['logArtifactId', 'byteLength', 'sha256'];
+  const artifactKeys = Object.keys(artifact);
+  if (
+    requiredArtifact.some((key) => !Object.hasOwn(artifact, key)) ||
+    artifactKeys.some((key) => ![...requiredArtifact, 'truncated'].includes(key))
+  ) return invalid('completion Artifact shape is invalid');
+  const logArtifactId = identifier('logArtifactId', artifact.logArtifactId, 36);
+  const sha256 = identifier('sha256', artifact.sha256, 64);
+  const callbackTokenDigest = identifier(
+    'callbackTokenDigest', command.callbackTokenDigest, 64,
+  );
+  if (
+    !LOG_ARTIFACT_ID.test(logArtifactId) ||
+    !SHA256.test(sha256) ||
+    !SHA256.test(callbackTokenDigest) ||
+    (artifact.truncated !== undefined &&
+      typeof artifact.truncated !== 'boolean')
+  ) return invalid('completion evidence is invalid');
+  return Object.freeze({
+    ...authority,
+    callbackSequence: integer(
+      'callbackSequence', command.callbackSequence, 1, 2_147_483_647,
+    ),
+    callbackTokenDigest,
+    result: Object.freeze({
+      outcome: result.outcome as 'succeeded' | 'failed',
+      startedAtMs,
+      finishedAtMs,
+      exitCode,
+    }),
+    artifact: Object.freeze({
+      logArtifactId,
+      byteLength: integer(
+        'byteLength', artifact.byteLength, 0, MAX_REMOTE_WORKER_ARTIFACT_BYTES,
+      ),
+      sha256,
+      ...(artifact.truncated === undefined
+        ? {}
+        : { truncated: artifact.truncated as boolean }),
+    }),
+  });
+}
+
+export function createRemoteWorkerCompletionRequestBody(
+  command: RemoteWorkerCompletionCommand,
+): RemoteWorkerCompletionRequestBody {
+  const value = normalizeRemoteWorkerCompletionCommand(command);
+  const { workerId: _workerId, workerSessionId: _sessionId, ...body } = value;
+  return Object.freeze({
+    schema: REMOTE_WORKER_COMPLETION_SCHEMA,
+    ...body,
+    artifact: Object.freeze({
+      ...value.artifact,
+      truncated: value.artifact.truncated ?? null,
+    }),
+  });
+}
+
+export function parseRemoteWorkerCompletionRequestBody(
+  value: unknown,
+  pathAuthority: Readonly<{ workerId: string; workerSessionId: string }>,
+): Readonly<RemoteWorkerCompletionCommand> {
+  const body = object(value, 'completion request');
+  exactKeys(body, [
+    'schema', 'workerGeneration', 'projectId', 'runId', 'attemptId',
+    'offerId', 'leaseGeneration', 'leaseToken', 'expectedLeaseVersion',
+    'callbackSequence', 'callbackTokenDigest', 'result', 'artifact',
+  ], 'completion request');
+  if (body.schema !== REMOTE_WORKER_COMPLETION_SCHEMA) {
+    return invalid('completion schema is invalid');
+  }
+  const artifact = object(body.artifact, 'completion Artifact');
+  exactKeys(artifact, [
+    'logArtifactId', 'byteLength', 'sha256', 'truncated',
+  ], 'completion Artifact');
+  const truncated = nullableBoolean('truncated', artifact.truncated);
+  return normalizeRemoteWorkerCompletionCommand({
+    workerId: pathAuthority.workerId,
+    workerSessionId: pathAuthority.workerSessionId,
+    workerGeneration: body.workerGeneration as number,
+    projectId: body.projectId as string,
+    runId: body.runId as string,
+    attemptId: body.attemptId as string,
+    offerId: body.offerId as string,
+    leaseGeneration: body.leaseGeneration as number,
+    leaseToken: body.leaseToken as string,
+    expectedLeaseVersion: body.expectedLeaseVersion as number,
+    callbackSequence: body.callbackSequence as number,
+    callbackTokenDigest: body.callbackTokenDigest as string,
+    result: body.result as RemoteWorkerCompletionCommand['result'],
+    artifact: {
+      logArtifactId: artifact.logArtifactId as string,
+      byteLength: artifact.byteLength as number,
+      sha256: artifact.sha256 as string,
+      ...(truncated === undefined ? {} : { truncated }),
+    },
+  });
+}
+
+export function normalizeRemoteWorkerCompletionResult(
+  value: RemoteWorkerCompletionResult,
+): Readonly<RemoteWorkerCompletionResult> {
+  const result = object(value, 'completion response');
+  exactKeys(result, [
+    'status', 'runId', 'attemptId', 'callbackSequence',
+  ], 'completion response');
+  if (
+    typeof result.status !== 'string' ||
+    !COMPLETION_STATUSES.has(result.status)
+  ) return invalid('completion response status is invalid');
+  try {
+    assertRunDispatchId('runId', result.runId as string);
+    assertRunDispatchId('attemptId', result.attemptId as string);
+  } catch {
+    return invalid('completion response authority is invalid');
+  }
+  return Object.freeze({
+    status: result.status as RemoteWorkerCompletionResult['status'],
+    runId: result.runId as string,
+    attemptId: result.attemptId as string,
+    callbackSequence: integer(
+      'callbackSequence', result.callbackSequence, 1, 2_147_483_647,
+    ),
+  });
+}
+
+export function createRemoteWorkerCompletionResponseBody(
+  value: RemoteWorkerCompletionResult,
+): RemoteWorkerCompletionResponseBody {
+  return Object.freeze({
+    schema: REMOTE_WORKER_COMPLETION_SCHEMA,
+    ...normalizeRemoteWorkerCompletionResult(value),
+  });
+}
+
+export function parseRemoteWorkerCompletionResponse(
+  serialized: Uint8Array | string,
+): Readonly<RemoteWorkerCompletionResult> {
+  const bytes = typeof serialized === 'string'
+    ? Buffer.from(serialized, 'utf8')
+    : Buffer.from(serialized);
+  if (
+    bytes.byteLength < 2 ||
+    bytes.byteLength > MAX_REMOTE_WORKER_COMPLETION_RESPONSE_BYTES
+  ) return invalid('completion response byte size is invalid');
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(bytes.toString('utf8')) as unknown;
+  } catch {
+    return invalid('completion response is not valid JSON');
+  }
+  const response = object(parsed, 'completion response envelope');
+  exactKeys(response, [
+    'schema', 'status', 'runId', 'attemptId', 'callbackSequence',
+  ], 'completion response envelope');
+  if (response.schema !== REMOTE_WORKER_COMPLETION_SCHEMA) {
+    return invalid('completion response schema is invalid');
+  }
+  return normalizeRemoteWorkerCompletionResult({
+    status: response.status as RemoteWorkerCompletionResult['status'],
+    runId: response.runId as string,
+    attemptId: response.attemptId as string,
+    callbackSequence: response.callbackSequence as number,
+  });
+}

packages/ql3-runtime-core/src/remote-execution/remoteWorkerLeaseControl.ts

@@ -0,0 +1,421 @@
+import {
+  assertRunDispatchId,
+  assertRunDispatchLeaseDuration,
+  assertRunDispatchLeaseFence,
+  type RunDispatchLeaseFenceReason,
+} from '../run/runDispatchLease';
+import { assertWorkerId, assertWorkerSessionId } from '../worker/workerSession';
+
+export const REMOTE_WORKER_LEASE_CONTROL_SCHEMA =
+  'qinglong/remote-worker-lease-control@v1';
+export const MAX_REMOTE_WORKER_LEASE_CONTROL_REQUEST_BYTES = 8 * 1024;
+export const MAX_REMOTE_WORKER_LEASE_CONTROL_RESPONSE_BYTES = 4 * 1024;
+
+export const REMOTE_WORKER_STOP_REASONS = [
+  'user', 'policy', 'shutdown', 'reconcile', 'timeout',
+] as const;
+export type RemoteWorkerStopReason =
+  (typeof REMOTE_WORKER_STOP_REASONS)[number];
+
+export const REMOTE_WORKER_TERMINAL_STATUSES = [
+  'succeeded', 'failed', 'cancelled', 'timed_out', 'lost',
+] as const;
+export type RemoteWorkerTerminalStatus =
+  (typeof REMOTE_WORKER_TERMINAL_STATUSES)[number];
+
+export interface RemoteWorkerLeaseControlCommand {
+  readonly workerId: string;
+  readonly workerSessionId: string;
+  readonly workerGeneration: number;
+  readonly projectId: string;
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly offerId: string;
+  readonly leaseGeneration: number;
+  readonly leaseToken: string;
+  readonly expectedLeaseVersion: number;
+}
+
+export type RemoteWorkerLeaseControlRequestBody = Readonly<
+  Omit<RemoteWorkerLeaseControlCommand, 'workerId' | 'workerSessionId'> & {
+    readonly schema: typeof REMOTE_WORKER_LEASE_CONTROL_SCHEMA;
+  }
+>;
+
+export type RemoteWorkerLeaseControlResult = Readonly<{
+  readonly status: 'renewed' | 'stop_requested' | 'terminal';
+  readonly projectId: string;
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly offerId: string;
+  readonly leaseGeneration: number;
+  readonly leaseVersion?: number;
+  readonly renewedAtMs?: number;
+  readonly expiresAtMs?: number;
+  readonly stop?: Readonly<{
+    readonly reason: RemoteWorkerStopReason;
+    readonly requestedAtMs: number;
+  }>;
+  readonly terminalStatus?: RemoteWorkerTerminalStatus;
+}>;
+
+export type RemoteWorkerLeaseControlResponseBody = Readonly<{
+  readonly schema: typeof REMOTE_WORKER_LEASE_CONTROL_SCHEMA;
+  readonly status: RemoteWorkerLeaseControlResult['status'];
+  readonly projectId: string;
+  readonly runId: string;
+  readonly attemptId: string;
+  readonly offerId: string;
+  readonly leaseGeneration: number;
+  readonly leaseVersion: number | null;
+  readonly renewedAtMs: number | null;
+  readonly expiresAtMs: number | null;
+  readonly stop: Readonly<{
+    readonly reason: RemoteWorkerStopReason;
+    readonly requestedAtMs: number;
+  }> | null;
+  readonly terminalStatus: RemoteWorkerTerminalStatus | null;
+}>;
+
+export interface RemoteWorkerLeaseControlRepository {
+  control(
+    command: RemoteWorkerLeaseControlCommand & Readonly<{
+      leaseDurationMs: number;
+      timeoutEventId: string;
+    }>,
+  ): Promise<Readonly<RemoteWorkerLeaseControlResult>>;
+}
+
+export class InvalidRemoteWorkerLeaseControlError extends TypeError {
+  readonly code = 'REMOTE_WORKER_LEASE_CONTROL_INVALID';
+
+  constructor(message: string) {
+    super(`Remote Worker lease control is invalid: ${message}`);
+    this.name = 'InvalidRemoteWorkerLeaseControlError';
+  }
+}
+
+export class RemoteWorkerLeaseControlFenceRejectedError extends Error {
+  readonly code = 'REMOTE_WORKER_LEASE_CONTROL_FENCED';
+
+  constructor(
+    readonly attemptId: string,
+    readonly reason:
+      | RunDispatchLeaseFenceReason
+      | 'project_mismatch'
+      | 'execution_owner_mismatch'
+      | 'executor_mismatch'
+      | 'offer_mismatch'
+      | 'state_mismatch',
+  ) {
+    super(`Remote Worker lease control is fenced: ${reason}`);
+    this.name = 'RemoteWorkerLeaseControlFenceRejectedError';
+  }
+}
+
+export class RemoteWorkerLeaseControlUnavailableError extends Error {
+  readonly code = 'REMOTE_WORKER_LEASE_CONTROL_UNAVAILABLE';
+
+  constructor(options?: ErrorOptions) {
+    super('Remote Worker lease control is unavailable', options);
+    this.name = 'RemoteWorkerLeaseControlUnavailableError';
+  }
+}
+
+function invalid(message: string): never {
+  throw new InvalidRemoteWorkerLeaseControlError(message);
+}
+
+function object(value: unknown, label: string): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return invalid(`${label} is not an object`);
+  }
+  return value as Record<string, unknown>;
+}
+
+function exactKeys(
+  value: Record<string, unknown>,
+  expected: readonly string[],
+  label: string,
+): void {
+  const actual = Object.keys(value).sort();
+  const sorted = [...expected].sort();
+  if (
+    actual.length !== sorted.length ||
+    actual.some((key, index) => key !== sorted[index])
+  ) return invalid(`${label} shape is invalid`);
+}
+
+function integer(
+  label: string,
+  value: unknown,
+  minimum = 0,
+  maximum = Number.MAX_SAFE_INTEGER,
+): number {
+  if (
+    !Number.isSafeInteger(value) ||
+    (value as number) < minimum ||
+    (value as number) > maximum
+  ) return invalid(`${label} is invalid`);
+  return value as number;
+}
+
+function identifier(label: string, value: unknown, maximum = 128): string {
+  if (
+    typeof value !== 'string' || value.length < 1 ||
+    Buffer.byteLength(value, 'utf8') > maximum ||
+    /[\u0000-\u001f\u007f]/.test(value)
+  ) return invalid(`${label} is invalid`);
+  return value;
+}
+
+export function normalizeRemoteWorkerLeaseControlCommand(
+  value: RemoteWorkerLeaseControlCommand,
+): Readonly<RemoteWorkerLeaseControlCommand> {
+  const command = object(value, 'lease control command');
+  exactKeys(command, [
+    'workerId', 'workerSessionId', 'workerGeneration', 'projectId', 'runId',
+    'attemptId', 'offerId', 'leaseGeneration', 'leaseToken',
+    'expectedLeaseVersion',
+  ], 'lease control command');
+  try {
+    assertWorkerId(command.workerId as string);
+    assertWorkerSessionId(command.workerSessionId as string);
+    assertRunDispatchId('runId', command.runId as string);
+    assertRunDispatchId('attemptId', command.attemptId as string);
+    assertRunDispatchId('offerId', command.offerId as string);
+    assertRunDispatchLeaseFence({
+      workerId: command.workerId as string,
+      workerSessionId: command.workerSessionId as string,
+      workerGeneration: command.workerGeneration as number,
+      leaseGeneration: command.leaseGeneration as number,
+      leaseToken: command.leaseToken as string,
+      expectedVersion: command.expectedLeaseVersion as number,
+    });
+  } catch {
+    return invalid('lease control authority is invalid');
+  }
+  return Object.freeze({
+    workerId: command.workerId as string,
+    workerSessionId: command.workerSessionId as string,
+    workerGeneration: integer(
+      'workerGeneration', command.workerGeneration, 1, 2_147_483_647,
+    ),
+    projectId: identifier('projectId', command.projectId),
+    runId: command.runId as string,
+    attemptId: command.attemptId as string,
+    offerId: command.offerId as string,
+    leaseGeneration: integer(
+      'leaseGeneration', command.leaseGeneration, 1, 2_147_483_647,
+    ),
+    leaseToken: command.leaseToken as string,
+    expectedLeaseVersion: integer(
+      'expectedLeaseVersion', command.expectedLeaseVersion,
+      0, 2_147_483_647,
+    ),
+  });
+}
+
+export function createRemoteWorkerLeaseControlRequestBody(
+  value: RemoteWorkerLeaseControlCommand,
+): RemoteWorkerLeaseControlRequestBody {
+  const command = normalizeRemoteWorkerLeaseControlCommand(value);
+  const { workerId: _workerId, workerSessionId: _sessionId, ...body } = command;
+  return Object.freeze({ schema: REMOTE_WORKER_LEASE_CONTROL_SCHEMA, ...body });
+}
+
+export function parseRemoteWorkerLeaseControlRequestBody(
+  value: unknown,
+  pathAuthority: Readonly<{ workerId: string; workerSessionId: string }>,
+): Readonly<RemoteWorkerLeaseControlCommand> {
+  const body = object(value, 'lease control request');
+  exactKeys(body, [
+    'schema', 'workerGeneration', 'projectId', 'runId', 'attemptId',
+    'offerId', 'leaseGeneration', 'leaseToken', 'expectedLeaseVersion',
+  ], 'lease control request');
+  if (body.schema !== REMOTE_WORKER_LEASE_CONTROL_SCHEMA) {
+    return invalid('lease control schema is invalid');
+  }
+  return normalizeRemoteWorkerLeaseControlCommand({
+    workerId: pathAuthority.workerId,
+    workerSessionId: pathAuthority.workerSessionId,
+    workerGeneration: body.workerGeneration as number,
+    projectId: body.projectId as string,
+    runId: body.runId as string,
+    attemptId: body.attemptId as string,
+    offerId: body.offerId as string,
+    leaseGeneration: body.leaseGeneration as number,
+    leaseToken: body.leaseToken as string,
+    expectedLeaseVersion: body.expectedLeaseVersion as number,
+  });
+}
+
+export function normalizeRemoteWorkerLeaseControlResult(
+  value: RemoteWorkerLeaseControlResult,
+): Readonly<RemoteWorkerLeaseControlResult> {
+  const result = object(value, 'lease control result');
+  const allowed = [
+    'status', 'projectId', 'runId', 'attemptId', 'offerId', 'leaseGeneration',
+    'leaseVersion', 'renewedAtMs', 'expiresAtMs', 'stop', 'terminalStatus',
+  ];
+  if (Object.keys(result).some((key) => !allowed.includes(key))) {
+    return invalid('lease control result shape is invalid');
+  }
+  for (const key of [
+    'status', 'projectId', 'runId', 'attemptId', 'offerId', 'leaseGeneration',
+  ]) {
+    if (!Object.hasOwn(result, key)) {
+      return invalid('lease control result is incomplete');
+    }
+  }
+  const status = result.status;
+  if (!['renewed', 'stop_requested', 'terminal'].includes(String(status))) {
+    return invalid('lease control result status is invalid');
+  }
+  try {
+    assertRunDispatchId('runId', result.runId as string);
+    assertRunDispatchId('attemptId', result.attemptId as string);
+    assertRunDispatchId('offerId', result.offerId as string);
+  } catch {
+    return invalid('lease control result authority is invalid');
+  }
+  const common = {
+    projectId: identifier('projectId', result.projectId),
+    runId: result.runId as string,
+    attemptId: result.attemptId as string,
+    offerId: result.offerId as string,
+    leaseGeneration: integer(
+      'leaseGeneration', result.leaseGeneration, 1, 2_147_483_647,
+    ),
+  };
+  if (status === 'terminal') {
+    if (
+      result.leaseVersion !== undefined || result.renewedAtMs !== undefined ||
+      result.expiresAtMs !== undefined || result.stop !== undefined ||
+      typeof result.terminalStatus !== 'string' ||
+      !REMOTE_WORKER_TERMINAL_STATUSES.includes(
+        result.terminalStatus as RemoteWorkerTerminalStatus,
+      )
+    ) return invalid('terminal lease control result is invalid');
+    return Object.freeze({
+      status: 'terminal' as const,
+      ...common,
+      terminalStatus: result.terminalStatus as RemoteWorkerTerminalStatus,
+    });
+  }
+  if (
+    result.terminalStatus !== undefined || result.leaseVersion === undefined ||
+    result.renewedAtMs === undefined || result.expiresAtMs === undefined
+  ) return invalid('renewed lease control result is invalid');
+  const leaseVersion = integer(
+    'leaseVersion', result.leaseVersion, 1, 2_147_483_647,
+  );
+  const renewedAtMs = integer('renewedAtMs', result.renewedAtMs);
+  const expiresAtMs = integer('expiresAtMs', result.expiresAtMs);
+  if (expiresAtMs <= renewedAtMs) return invalid('lease expiry is invalid');
+  if (status === 'renewed') {
+    if (result.stop !== undefined) return invalid('renewed control is invalid');
+    return Object.freeze({
+      status: 'renewed' as const,
+      ...common,
+      leaseVersion,
+      renewedAtMs,
+      expiresAtMs,
+    });
+  }
+  const stop = object(result.stop, 'stop control');
+  exactKeys(stop, ['reason', 'requestedAtMs'], 'stop control');
+  if (
+    typeof stop.reason !== 'string' ||
+    !REMOTE_WORKER_STOP_REASONS.includes(stop.reason as RemoteWorkerStopReason)
+  ) return invalid('stop reason is invalid');
+  return Object.freeze({
+    status: 'stop_requested' as const,
+    ...common,
+    leaseVersion,
+    renewedAtMs,
+    expiresAtMs,
+    stop: Object.freeze({
+      reason: stop.reason as RemoteWorkerStopReason,
+      requestedAtMs: integer('requestedAtMs', stop.requestedAtMs),
+    }),
+  });
+}
+
+export function createRemoteWorkerLeaseControlResponseBody(
+  value: RemoteWorkerLeaseControlResult,
+): RemoteWorkerLeaseControlResponseBody {
+  const result = normalizeRemoteWorkerLeaseControlResult(value);
+  return Object.freeze({
+    schema: REMOTE_WORKER_LEASE_CONTROL_SCHEMA,
+    status: result.status,
+    projectId: result.projectId,
+    runId: result.runId,
+    attemptId: result.attemptId,
+    offerId: result.offerId,
+    leaseGeneration: result.leaseGeneration,
+    leaseVersion: result.leaseVersion ?? null,
+    renewedAtMs: result.renewedAtMs ?? null,
+    expiresAtMs: result.expiresAtMs ?? null,
+    stop: result.stop ?? null,
+    terminalStatus: result.terminalStatus ?? null,
+  });
+}
+
+export function parseRemoteWorkerLeaseControlResponse(
+  serialized: Uint8Array | string,
+): Readonly<RemoteWorkerLeaseControlResult> {
+  const bytes = typeof serialized === 'string'
+    ? Buffer.from(serialized, 'utf8')
+    : Buffer.from(serialized);
+  if (
+    bytes.byteLength < 2 ||
+    bytes.byteLength > MAX_REMOTE_WORKER_LEASE_CONTROL_RESPONSE_BYTES
+  ) return invalid('lease control response byte size is invalid');
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(bytes.toString('utf8')) as unknown;
+  } catch {
+    return invalid('lease control response is not valid JSON');
+  }
+  const response = object(parsed, 'lease control response');
+  exactKeys(response, [
+    'schema', 'status', 'projectId', 'runId', 'attemptId', 'offerId',
+    'leaseGeneration', 'leaseVersion', 'renewedAtMs', 'expiresAtMs', 'stop',
+    'terminalStatus',
+  ], 'lease control response');
+  if (response.schema !== REMOTE_WORKER_LEASE_CONTROL_SCHEMA) {
+    return invalid('lease control response schema is invalid');
+  }
+  return normalizeRemoteWorkerLeaseControlResult({
+    status: response.status as RemoteWorkerLeaseControlResult['status'],
+    projectId: response.projectId as string,
+    runId: response.runId as string,
+    attemptId: response.attemptId as string,
+    offerId: response.offerId as string,
+    leaseGeneration: response.leaseGeneration as number,
+    ...(response.leaseVersion === null
+      ? {}
+      : { leaseVersion: response.leaseVersion as number }),
+    ...(response.renewedAtMs === null
+      ? {}
+      : { renewedAtMs: response.renewedAtMs as number }),
+    ...(response.expiresAtMs === null
+      ? {}
+      : { expiresAtMs: response.expiresAtMs as number }),
+    ...(response.stop === null
+      ? {}
+      : { stop: response.stop as NonNullable<RemoteWorkerLeaseControlResult['stop']> }),
+    ...(response.terminalStatus === null
+      ? {}
+      : { terminalStatus: response.terminalStatus as RemoteWorkerTerminalStatus }),
+  });
+}
+
+export function assertRemoteWorkerLeaseControlDuration(value: number): void {
+  try {
+    assertRunDispatchLeaseDuration(value);
+  } catch {
+    invalid('lease duration is invalid');
+  }
+}

packages/ql3-runtime-core/src/remote-execution/remoteWorkerPlacement.ts

@@ -0,0 +1,666 @@
+import { createHash } from 'node:crypto';
+import type { WorkerSessionRecord } from '../worker/workerSession';
+import { assertWorkerSessionRecord } from '../worker/workerSession';
+import { semver } from '../versioning/pinnedSemver';
+
+export const REMOTE_WORKER_EXECUTOR_CAPABILITY = 'remote-worker';
+export const MAX_REMOTE_PLACEMENT_VALUES = 16;
+export const MAX_REMOTE_PLACEMENT_PREFERENCES = 16;
+export const MAX_REMOTE_WORKER_RUNTIMES = 32;
+export const MAX_REMOTE_WORKER_LABELS = 32;
+export const MAX_REMOTE_WORKER_FEATURES = 32;
+export const MAX_REMOTE_WORKER_GPUS = 8;
+
+const CAPABILITY_NAME = /^[a-z0-9][a-z0-9._+-]*$/;
+const LABEL_KEY = /^[A-Za-z0-9][A-Za-z0-9._/-]*$/;
+
+export interface RemoteWorkerRuntimeCapability {
+  readonly name: string;
+  readonly version: string;
+}
+
+export interface RemoteWorkerCapabilities {
+  readonly architecture: string;
+  readonly executors: readonly string[];
+  readonly operatingSystem?: string;
+  readonly runtimes?: readonly RemoteWorkerRuntimeCapability[];
+  readonly labels?: Readonly<Record<string, string>>;
+  readonly capacity?: Readonly<{
+    readonly cpuCores?: number;
+    readonly memoryBytes?: number;
+    readonly diskBytes?: number;
+    readonly gpu?: readonly Readonly<{
+      readonly vendor: string;
+      readonly model?: string;
+      readonly memoryBytes?: number;
+    }>[];
+  }>;
+  readonly features?: readonly string[];
+}
+
+export interface RemoteWorkerRuntimeRequirement {
+  readonly name: string;
+  readonly versionRange?: string;
+}
+
+export interface RemoteWorkerPlacementSpec {
+  readonly required?: Readonly<{
+    readonly architectures?: readonly string[];
+    readonly operatingSystems?: readonly string[];
+    readonly executors?: readonly string[];
+    readonly runtimes?: readonly RemoteWorkerRuntimeRequirement[];
+    readonly labels?: Readonly<Record<string, string>>;
+    readonly minMemoryBytes?: number;
+    readonly minDiskBytes?: number;
+    readonly gpuVendor?: string;
+    readonly features?: readonly string[];
+  }>;
+  readonly preferred?: readonly Readonly<{
+    readonly labels: Readonly<Record<string, string>>;
+    readonly weight: number;
+  }>[];
+}
+
+export type RemoteWorkerPlacementMismatch =
+  | 'worker_unavailable'
+  | 'architecture'
+  | 'operating_system'
+  | 'executor'
+  | 'runtime'
+  | 'label'
+  | 'memory'
+  | 'disk'
+  | 'gpu'
+  | 'feature';
+
+export interface RemoteWorkerPlacementDecision {
+  readonly matches: boolean;
+  readonly score: number;
+  readonly mismatches: readonly RemoteWorkerPlacementMismatch[];
+}
+
+function invalid(message: string): never {
+  throw new TypeError(`Remote Worker placement value is invalid: ${message}`);
+}
+
+function object(value: unknown, label: string): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    invalid(`${label} must be an object`);
+  }
+  return value as Record<string, unknown>;
+}
+
+function exactKeys(
+  value: Record<string, unknown>,
+  required: readonly string[],
+  optional: readonly string[],
+  label: string,
+): void {
+  const allowed = new Set([...required, ...optional]);
+  if (
+    required.some((key) => !Object.hasOwn(value, key)) ||
+    Object.keys(value).some((key) => !allowed.has(key))
+  )
+    invalid(`${label} shape is invalid`);
+}
+
+function boundedString(
+  value: unknown,
+  label: string,
+  maximum: number,
+  pattern?: RegExp,
+): string {
+  if (
+    typeof value !== 'string' ||
+    value.length < 1 ||
+    value.length > maximum ||
+    /[\u0000-\u001f\u007f]/.test(value) ||
+    (pattern !== undefined && !pattern.test(value))
+  )
+    invalid(`${label} is invalid`);
+  return value;
+}
+
+function positiveInteger(
+  value: unknown,
+  label: string,
+  maximum = Number.MAX_SAFE_INTEGER,
+): number {
+  if (
+    !Number.isSafeInteger(value) ||
+    (value as number) < 1 ||
+    (value as number) > maximum
+  ) {
+    invalid(`${label} is invalid`);
+  }
+  return value as number;
+}
+
+function sortedStrings(
+  value: unknown,
+  label: string,
+  maximum: number,
+  allowEmpty = true,
+): readonly string[] {
+  if (
+    !Array.isArray(value) ||
+    value.length > maximum ||
+    (!allowEmpty && value.length === 0)
+  ) {
+    invalid(`${label} is invalid`);
+  }
+  const result = value.map((item, index) =>
+    boundedString(item, `${label}[${index}]`, 64, CAPABILITY_NAME),
+  );
+  if (new Set(result).size !== result.length)
+    invalid(`${label} contains duplicates`);
+  return Object.freeze(result.sort());
+}
+
+function normalizedLabels(
+  value: unknown,
+  label: string,
+  maximum: number,
+): Readonly<Record<string, string>> {
+  const source = object(value, label);
+  const entries = Object.entries(source);
+  if (entries.length > maximum) invalid(`${label} exceeds its item budget`);
+  return Object.freeze(
+    Object.fromEntries(
+      entries
+        .map(
+          ([key, item]) =>
+            [
+              boundedString(key, `${label} key`, 128, LABEL_KEY),
+              boundedString(item, `${label}.${key}`, 256),
+            ] as const,
+        )
+        .sort((left, right) => left[0].localeCompare(right[0])),
+    ),
+  );
+}
+
+export function normalizeRemoteWorkerCapabilities(
+  value: unknown,
+): RemoteWorkerCapabilities {
+  const source = object(value, 'capabilities');
+  exactKeys(
+    source,
+    ['architecture', 'executors'],
+    ['capacity', 'features', 'labels', 'operatingSystem', 'runtimes'],
+    'capabilities',
+  );
+  const architecture = boundedString(
+    source.architecture,
+    'architecture',
+    32,
+    CAPABILITY_NAME,
+  );
+  const executors = sortedStrings(source.executors, 'executors', 16, false);
+  const operatingSystem =
+    source.operatingSystem === undefined
+      ? undefined
+      : boundedString(
+          source.operatingSystem,
+          'operatingSystem',
+          32,
+          CAPABILITY_NAME,
+        );
+  let runtimes: readonly RemoteWorkerRuntimeCapability[] | undefined;
+  if (source.runtimes !== undefined) {
+    if (
+      !Array.isArray(source.runtimes) ||
+      source.runtimes.length > MAX_REMOTE_WORKER_RUNTIMES
+    )
+      invalid('runtimes is invalid');
+    const mapped = source.runtimes.map((item, index) => {
+      const runtime = object(item, `runtimes[${index}]`);
+      exactKeys(runtime, ['name', 'version'], [], `runtimes[${index}]`);
+      const name = boundedString(
+        runtime.name,
+        `runtimes[${index}].name`,
+        64,
+        CAPABILITY_NAME,
+      );
+      const version = boundedString(
+        runtime.version,
+        `runtimes[${index}].version`,
+        64,
+      );
+      if (semver().valid(version) === null)
+        invalid(`runtimes[${index}].version is not semver`);
+      return Object.freeze({ name, version });
+    });
+    if (new Set(mapped.map((item) => item.name)).size !== mapped.length)
+      invalid('runtimes repeats a runtime name');
+    runtimes = Object.freeze(
+      mapped.sort((left, right) => left.name.localeCompare(right.name)),
+    );
+  }
+  const labels =
+    source.labels === undefined
+      ? undefined
+      : normalizedLabels(source.labels, 'labels', MAX_REMOTE_WORKER_LABELS);
+  let capacity: RemoteWorkerCapabilities['capacity'];
+  if (source.capacity !== undefined) {
+    const candidate = object(source.capacity, 'capacity');
+    exactKeys(
+      candidate,
+      [],
+      ['cpuCores', 'diskBytes', 'gpu', 'memoryBytes'],
+      'capacity',
+    );
+    let gpu: NonNullable<RemoteWorkerCapabilities['capacity']>['gpu'];
+    if (candidate.gpu !== undefined) {
+      if (
+        !Array.isArray(candidate.gpu) ||
+        candidate.gpu.length > MAX_REMOTE_WORKER_GPUS
+      )
+        invalid('capacity.gpu is invalid');
+      gpu = Object.freeze(
+        candidate.gpu.map((item, index) => {
+          const device = object(item, `capacity.gpu[${index}]`);
+          exactKeys(
+            device,
+            ['vendor'],
+            ['memoryBytes', 'model'],
+            `capacity.gpu[${index}]`,
+          );
+          return Object.freeze({
+            vendor: boundedString(
+              device.vendor,
+              `capacity.gpu[${index}].vendor`,
+              64,
+              CAPABILITY_NAME,
+            ),
+            ...(device.model === undefined
+              ? {}
+              : {
+                  model: boundedString(
+                    device.model,
+                    `capacity.gpu[${index}].model`,
+                    128,
+                  ),
+                }),
+            ...(device.memoryBytes === undefined
+              ? {}
+              : {
+                  memoryBytes: positiveInteger(
+                    device.memoryBytes,
+                    `capacity.gpu[${index}].memoryBytes`,
+                  ),
+                }),
+          });
+        }),
+      );
+    }
+    capacity = Object.freeze({
+      ...(candidate.cpuCores === undefined
+        ? {}
+        : {
+            cpuCores: positiveInteger(
+              candidate.cpuCores,
+              'capacity.cpuCores',
+              4096,
+            ),
+          }),
+      ...(candidate.memoryBytes === undefined
+        ? {}
+        : {
+            memoryBytes: positiveInteger(
+              candidate.memoryBytes,
+              'capacity.memoryBytes',
+            ),
+          }),
+      ...(candidate.diskBytes === undefined
+        ? {}
+        : {
+            diskBytes: positiveInteger(
+              candidate.diskBytes,
+              'capacity.diskBytes',
+            ),
+          }),
+      ...(gpu === undefined ? {} : { gpu }),
+    });
+  }
+  const features =
+    source.features === undefined
+      ? undefined
+      : sortedStrings(source.features, 'features', MAX_REMOTE_WORKER_FEATURES);
+  return Object.freeze({
+    architecture,
+    executors,
+    ...(operatingSystem === undefined ? {} : { operatingSystem }),
+    ...(runtimes === undefined ? {} : { runtimes }),
+    ...(labels === undefined ? {} : { labels }),
+    ...(capacity === undefined ? {} : { capacity }),
+    ...(features === undefined ? {} : { features }),
+  });
+}
+
+export function canonicalRemoteWorkerCapabilities(value: unknown): Readonly<{
+  capabilities: RemoteWorkerCapabilities;
+  json: string;
+  hash: string;
+}> {
+  const capabilities = normalizeRemoteWorkerCapabilities(value);
+  const json = JSON.stringify(capabilities);
+  return Object.freeze({
+    capabilities,
+    json,
+    hash: createHash('sha256').update(json, 'utf8').digest('hex'),
+  });
+}
+
+export function parseRemoteWorkerCapabilities(
+  record: WorkerSessionRecord,
+): RemoteWorkerCapabilities {
+  assertWorkerSessionRecord(record);
+  const canonical = canonicalRemoteWorkerCapabilities(
+    JSON.parse(record.capabilitiesJson) as unknown,
+  );
+  if (
+    canonical.json !== record.capabilitiesJson ||
+    canonical.hash !== record.capabilitiesHash
+  ) {
+    invalid('capabilities snapshot is not canonical');
+  }
+  return canonical.capabilities;
+}
+
+function optionalStringList(
+  value: unknown,
+  label: string,
+): readonly string[] | undefined {
+  return value === undefined
+    ? undefined
+    : sortedStrings(value, label, MAX_REMOTE_PLACEMENT_VALUES);
+}
+
+export function normalizeRemoteWorkerPlacement(
+  value: unknown,
+): RemoteWorkerPlacementSpec {
+  const source = object(value, 'placement');
+  exactKeys(source, [], ['preferred', 'required'], 'placement');
+  let required: RemoteWorkerPlacementSpec['required'];
+  if (source.required !== undefined) {
+    const candidate = object(source.required, 'placement.required');
+    exactKeys(
+      candidate,
+      [],
+      [
+        'architectures',
+        'executors',
+        'features',
+        'gpuVendor',
+        'labels',
+        'minDiskBytes',
+        'minMemoryBytes',
+        'operatingSystems',
+        'runtimes',
+      ],
+      'placement.required',
+    );
+    const architectures = optionalStringList(
+      candidate.architectures,
+      'placement.required.architectures',
+    );
+    const operatingSystems = optionalStringList(
+      candidate.operatingSystems,
+      'placement.required.operatingSystems',
+    );
+    const executors = optionalStringList(
+      candidate.executors,
+      'placement.required.executors',
+    );
+    const features = optionalStringList(
+      candidate.features,
+      'placement.required.features',
+    );
+    let runtimes: readonly RemoteWorkerRuntimeRequirement[] | undefined;
+    if (candidate.runtimes !== undefined) {
+      if (
+        !Array.isArray(candidate.runtimes) ||
+        candidate.runtimes.length > MAX_REMOTE_PLACEMENT_VALUES
+      )
+        invalid('placement.required.runtimes is invalid');
+      const mapped = candidate.runtimes.map((item, index) => {
+        const runtime = object(item, `placement.required.runtimes[${index}]`);
+        exactKeys(
+          runtime,
+          ['name'],
+          ['versionRange'],
+          `placement.required.runtimes[${index}]`,
+        );
+        const name = boundedString(
+          runtime.name,
+          `placement.required.runtimes[${index}].name`,
+          64,
+          CAPABILITY_NAME,
+        );
+        const versionRange =
+          runtime.versionRange === undefined
+            ? undefined
+            : boundedString(
+                runtime.versionRange,
+                `placement.required.runtimes[${index}].versionRange`,
+                128,
+              );
+        if (
+          versionRange !== undefined &&
+          semver().validRange(versionRange) === null
+        )
+          invalid(
+            `placement.required.runtimes[${index}].versionRange is not semver`,
+          );
+        return Object.freeze({
+          name,
+          ...(versionRange === undefined ? {} : { versionRange }),
+        });
+      });
+      if (new Set(mapped.map((item) => item.name)).size !== mapped.length)
+        invalid('placement.required.runtimes repeats a runtime name');
+      runtimes = Object.freeze(
+        mapped.sort((left, right) => left.name.localeCompare(right.name)),
+      );
+    }
+    required = Object.freeze({
+      ...(architectures === undefined ? {} : { architectures }),
+      ...(operatingSystems === undefined ? {} : { operatingSystems }),
+      ...(executors === undefined ? {} : { executors }),
+      ...(runtimes === undefined ? {} : { runtimes }),
+      ...(candidate.labels === undefined
+        ? {}
+        : {
+            labels: normalizedLabels(
+              candidate.labels,
+              'placement.required.labels',
+              MAX_REMOTE_PLACEMENT_VALUES,
+            ),
+          }),
+      ...(candidate.minMemoryBytes === undefined
+        ? {}
+        : {
+            minMemoryBytes: positiveInteger(
+              candidate.minMemoryBytes,
+              'placement.required.minMemoryBytes',
+            ),
+          }),
+      ...(candidate.minDiskBytes === undefined
+        ? {}
+        : {
+            minDiskBytes: positiveInteger(
+              candidate.minDiskBytes,
+              'placement.required.minDiskBytes',
+            ),
+          }),
+      ...(candidate.gpuVendor === undefined
+        ? {}
+        : {
+            gpuVendor: boundedString(
+              candidate.gpuVendor,
+              'placement.required.gpuVendor',
+              64,
+              CAPABILITY_NAME,
+            ),
+          }),
+      ...(features === undefined ? {} : { features }),
+    });
+  }
+  let preferred: RemoteWorkerPlacementSpec['preferred'];
+  if (source.preferred !== undefined) {
+    if (
+      !Array.isArray(source.preferred) ||
+      source.preferred.length > MAX_REMOTE_PLACEMENT_PREFERENCES
+    )
+      invalid('placement.preferred is invalid');
+    preferred = Object.freeze(
+      source.preferred.map((item, index) => {
+        const preference = object(item, `placement.preferred[${index}]`);
+        exactKeys(
+          preference,
+          ['labels', 'weight'],
+          [],
+          `placement.preferred[${index}]`,
+        );
+        const labels = normalizedLabels(
+          preference.labels,
+          `placement.preferred[${index}].labels`,
+          MAX_REMOTE_PLACEMENT_VALUES,
+        );
+        if (Object.keys(labels).length === 0)
+          invalid(`placement.preferred[${index}].labels is empty`);
+        return Object.freeze({
+          labels,
+          weight: positiveInteger(
+            preference.weight,
+            `placement.preferred[${index}].weight`,
+            100,
+          ),
+        });
+      }),
+    );
+  }
+  return Object.freeze({
+    ...(required === undefined ? {} : { required }),
+    ...(preferred === undefined ? {} : { preferred }),
+  });
+}
+
+export function effectiveRemoteWorkerPlacement(
+  value: unknown,
+): RemoteWorkerPlacementSpec {
+  const placement = normalizeRemoteWorkerPlacement(value ?? {});
+  const executors = placement.required?.executors;
+  if (
+    executors !== undefined &&
+    !executors.includes(REMOTE_WORKER_EXECUTOR_CAPABILITY)
+  ) {
+    invalid(`placement must require ${REMOTE_WORKER_EXECUTOR_CAPABILITY}`);
+  }
+  return normalizeRemoteWorkerPlacement({
+    ...placement,
+    required: {
+      ...placement.required,
+      executors: executors ?? [REMOTE_WORKER_EXECUTOR_CAPABILITY],
+    },
+  });
+}
+
+function containsLabels(
+  actual: Readonly<Record<string, string>> | undefined,
+  expected: Readonly<Record<string, string>>,
+): boolean {
+  return Object.entries(expected).every(
+    ([key, value]) => actual?.[key] === value,
+  );
+}
+
+export function evaluateRemoteWorkerPlacement(
+  worker: WorkerSessionRecord,
+  placementValue: unknown,
+  observedAtMs: number,
+): RemoteWorkerPlacementDecision {
+  assertWorkerSessionRecord(worker);
+  if (!Number.isSafeInteger(observedAtMs) || observedAtMs < 0)
+    invalid('observedAtMs is invalid');
+  const capabilities = parseRemoteWorkerCapabilities(worker);
+  const placement = effectiveRemoteWorkerPlacement(placementValue);
+  const required = placement.required ?? {};
+  const mismatches: RemoteWorkerPlacementMismatch[] = [];
+  if (
+    worker.status !== 'online' ||
+    worker.availableSlots < 1 ||
+    worker.leaseExpiresAtMs <= observedAtMs
+  )
+    mismatches.push('worker_unavailable');
+  if (
+    required.architectures?.length &&
+    !required.architectures.includes(capabilities.architecture)
+  )
+    mismatches.push('architecture');
+  if (
+    required.operatingSystems?.length &&
+    (!capabilities.operatingSystem ||
+      !required.operatingSystems.includes(capabilities.operatingSystem))
+  )
+    mismatches.push('operating_system');
+  if (
+    required.executors?.some(
+      (executor) => !capabilities.executors.includes(executor),
+    )
+  )
+    mismatches.push('executor');
+  if (
+    required.runtimes?.some(
+      (requirement) =>
+        !(capabilities.runtimes ?? []).some(
+          (runtime) =>
+            runtime.name === requirement.name &&
+            (requirement.versionRange === undefined ||
+              semver().satisfies(runtime.version, requirement.versionRange, {
+                includePrerelease: true,
+              })),
+        ),
+    )
+  )
+    mismatches.push('runtime');
+  if (required.labels && !containsLabels(capabilities.labels, required.labels))
+    mismatches.push('label');
+  if (
+    required.minMemoryBytes !== undefined &&
+    (capabilities.capacity?.memoryBytes ?? 0) < required.minMemoryBytes
+  )
+    mismatches.push('memory');
+  if (
+    required.minDiskBytes !== undefined &&
+    (capabilities.capacity?.diskBytes ?? 0) < required.minDiskBytes
+  )
+    mismatches.push('disk');
+  if (
+    required.gpuVendor !== undefined &&
+    !(capabilities.capacity?.gpu ?? []).some(
+      (gpu) => gpu.vendor === required.gpuVendor,
+    )
+  )
+    mismatches.push('gpu');
+  if (
+    required.features?.some(
+      (feature) => !(capabilities.features ?? []).includes(feature),
+    )
+  )
+    mismatches.push('feature');
+  const score = (placement.preferred ?? []).reduce(
+    (total, preference) =>
+      total +
+      (containsLabels(capabilities.labels, preference.labels)
+        ? preference.weight
+        : 0),
+    0,
+  );
+  return Object.freeze({
+    matches: mismatches.length === 0,
+    score,
+    mismatches: Object.freeze(mismatches),
+  });
+}