feat(ql3): add strong cluster run management

packages/ql3-cluster-admin/src/run-management/runManagement.ts

@@ -0,0 +1,273 @@
+import { randomUUID } from 'node:crypto';
+
+import {
+  PostgresProjectPolicyRepository,
+  PostgresRunManualRetryRepository,
+  PostgresSecurityAuditRepository,
+} from '@qinglong/cluster-postgres/run-manager';
+import type { PostgresPool } from '@qinglong/runtime-core';
+import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
+import {
+  InvalidRunManualRetryError,
+  RunManualRetryFenceRejectedError,
+  RunManualRetryNotFoundError,
+  RunManualRetryRateLimitedError,
+  RunManualRetryUnavailableError,
+  type RunManualRetryResult,
+  type RunManualRetrySourceStatus,
+} from '@qinglong/runtime-core/run-manual-retry';
+import {
+  normalizeSecurityPrincipal,
+  type SecurityPolicyFence,
+  type SecurityPrincipal,
+} from '@qinglong/runtime-core/security';
+import { normalizeSecurityAuditRecord } from '@qinglong/runtime-core/security-audit';
+
+const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+const UUID_PATTERN =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+export interface ClusterRunManagementRetryRequest {
+  readonly projectId: string;
+  readonly sourceRunId: string;
+  readonly mutationId: string;
+  readonly expectedRunVersion: number;
+  readonly expectedRunStatus: RunManualRetrySourceStatus;
+  readonly requestId: string;
+  readonly auditEventId: string;
+  readonly failureAuditEventId: string;
+  readonly principal: Readonly<SecurityPrincipal>;
+}
+
+export interface ClusterRunManagementService {
+  retry(
+    request: Readonly<ClusterRunManagementRetryRequest>,
+  ): Promise<Readonly<RunManualRetryResult>>;
+}
+
+export interface ClusterRunManagementOptions {
+  readonly pool: PostgresPool;
+  readonly now?: () => number;
+  readonly randomUuid?: () => string;
+}
+
+export class ClusterRunManagementConfigurationError extends TypeError {
+  readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFIGURATION_INVALID';
+  constructor() {
+    super('Cluster Run management configuration is invalid');
+    this.name = 'ClusterRunManagementConfigurationError';
+  }
+}
+
+export class ClusterRunManagementRequestError extends TypeError {
+  readonly code = 'CLUSTER_RUN_MANAGEMENT_REQUEST_INVALID';
+  constructor() {
+    super('Cluster Run management request is invalid');
+    this.name = 'ClusterRunManagementRequestError';
+  }
+}
+
+export class ClusterRunManagementAuthorizationError extends Error {
+  readonly code = 'CLUSTER_RUN_MANAGEMENT_FORBIDDEN';
+  constructor() {
+    super('Cluster Run management is forbidden');
+    this.name = 'ClusterRunManagementAuthorizationError';
+  }
+}
+
+export class ClusterRunManagementTargetUnavailableError extends Error {
+  readonly code = 'CLUSTER_RUN_MANAGEMENT_TARGET_UNAVAILABLE';
+  constructor() {
+    super('Cluster Run management target is unavailable');
+    this.name = 'ClusterRunManagementTargetUnavailableError';
+  }
+}
+
+export class ClusterRunManagementConflictError extends Error {
+  readonly code = 'CLUSTER_RUN_MANAGEMENT_CONFLICT';
+  constructor() {
+    super('Cluster Run management conflicts with durable state');
+    this.name = 'ClusterRunManagementConflictError';
+  }
+}
+
+export class ClusterRunManagementRateLimitedError extends Error {
+  readonly code = 'CLUSTER_RUN_MANAGEMENT_RATE_LIMITED';
+  constructor(readonly retryAfterMs: number) {
+    super('Cluster Run management rate limit is exhausted');
+    this.name = 'ClusterRunManagementRateLimitedError';
+  }
+}
+
+export class ClusterRunManagementUnavailableError extends Error {
+  readonly code = 'CLUSTER_RUN_MANAGEMENT_UNAVAILABLE';
+  constructor(options?: ErrorOptions) {
+    super('Cluster Run management is unavailable', options);
+    this.name = 'ClusterRunManagementUnavailableError';
+  }
+}
+
+function exactRequest(
+  value: unknown,
+): asserts value is Readonly<ClusterRunManagementRetryRequest> {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    Object.keys(value).sort().join('\0') !==
+      [
+        'auditEventId',
+        'expectedRunStatus',
+        'expectedRunVersion',
+        'failureAuditEventId',
+        'mutationId',
+        'principal',
+        'projectId',
+        'requestId',
+        'sourceRunId',
+      ]
+        .sort()
+        .join('\0')
+  ) {
+    throw new ClusterRunManagementRequestError();
+  }
+}
+
+function validUuid(value: unknown): value is string {
+  return typeof value === 'string' && UUID_PATTERN.test(value);
+}
+
+function failureReason(error: unknown): string {
+  if (error instanceof ClusterRunManagementAuthorizationError) {
+    return 'authorization_rejected';
+  }
+  if (error instanceof RunManualRetryNotFoundError) return 'run_not_found';
+  if (error instanceof RunManualRetryRateLimitedError) return 'rate_limited';
+  if (error instanceof RunManualRetryFenceRejectedError) return error.reason;
+  return 'management_unavailable';
+}
+
+/** Strong OIDC Run management composition over one run-manager Pool. */
+export function createClusterRunManagementService(
+  options: ClusterRunManagementOptions,
+): Readonly<ClusterRunManagementService> {
+  if (
+    !options ||
+    typeof options !== 'object' ||
+    Array.isArray(options) ||
+    Object.keys(options).some(
+      (key) => key !== 'pool' && key !== 'now' && key !== 'randomUuid',
+    ) ||
+    !options.pool ||
+    typeof options.pool.query !== 'function' ||
+    typeof options.pool.connect !== 'function' ||
+    (options.now !== undefined && typeof options.now !== 'function') ||
+    (options.randomUuid !== undefined && typeof options.randomUuid !== 'function')
+  ) {
+    throw new ClusterRunManagementConfigurationError();
+  }
+  const now = options.now ?? Date.now;
+  const createId = options.randomUuid ?? randomUUID;
+  const policy = new ProjectPolicyEngine(
+    new PostgresProjectPolicyRepository(options.pool),
+  );
+  const retries = new PostgresRunManualRetryRepository(options.pool);
+  const audit = new PostgresSecurityAuditRepository(options.pool);
+
+  return Object.freeze({
+    async retry(requestValue: Readonly<ClusterRunManagementRetryRequest>) {
+      exactRequest(requestValue);
+      const observedAtMs = now();
+      let principal: Readonly<SecurityPrincipal>;
+      if (
+        !Number.isSafeInteger(observedAtMs) ||
+        observedAtMs < 0 ||
+        !IDENTIFIER_PATTERN.test(requestValue.projectId) ||
+        !IDENTIFIER_PATTERN.test(requestValue.sourceRunId) ||
+        !IDENTIFIER_PATTERN.test(requestValue.requestId) ||
+        !validUuid(requestValue.mutationId) ||
+        !validUuid(requestValue.auditEventId) ||
+        !validUuid(requestValue.failureAuditEventId) ||
+        requestValue.auditEventId === requestValue.failureAuditEventId
+      ) {
+        throw new ClusterRunManagementRequestError();
+      }
+      try {
+        principal = normalizeSecurityPrincipal(
+          requestValue.principal,
+          observedAtMs,
+        );
+      } catch {
+        throw new ClusterRunManagementRequestError();
+      }
+
+      let fence: Readonly<SecurityPolicyFence> | null = null;
+      try {
+        const decision = await policy.authorize(
+          principal,
+          requestValue.projectId,
+          'run.retry',
+        );
+        fence = decision.fence;
+        if (
+          decision.effect !== 'allow' ||
+          !fence ||
+          fence.bindingVersion === null
+        ) {
+          throw new ClusterRunManagementAuthorizationError();
+        }
+        return await retries.retryRun({
+          projectId: requestValue.projectId,
+          sourceRunId: requestValue.sourceRunId,
+          mutationId: requestValue.mutationId,
+          expectedRunVersion: requestValue.expectedRunVersion,
+          expectedRunStatus: requestValue.expectedRunStatus,
+          runId: createId(),
+          attemptId: createId(),
+          createdEventId: createId(),
+          queuedEventId: createId(),
+          auditEventId: requestValue.auditEventId,
+          requestId: requestValue.requestId,
+          principal,
+          policyFence: fence,
+        });
+      } catch (error) {
+        try {
+          await audit.record(
+            normalizeSecurityAuditRecord({
+              eventId: requestValue.failureAuditEventId,
+              requestId: requestValue.requestId,
+              operationId: 'run.retry',
+              projectId: requestValue.projectId,
+              subject: principal.subject,
+              authenticationId: principal.authenticationId,
+              outcome: 'denied',
+              reasons: [failureReason(error)],
+              fence,
+              occurredAtMs: observedAtMs,
+            }),
+          );
+        } catch (auditError) {
+          throw new ClusterRunManagementUnavailableError({ cause: auditError });
+        }
+        if (error instanceof ClusterRunManagementAuthorizationError) throw error;
+        if (error instanceof InvalidRunManualRetryError) {
+          throw new ClusterRunManagementRequestError();
+        }
+        if (error instanceof RunManualRetryNotFoundError) {
+          throw new ClusterRunManagementTargetUnavailableError();
+        }
+        if (error instanceof RunManualRetryFenceRejectedError) {
+          throw new ClusterRunManagementConflictError();
+        }
+        if (error instanceof RunManualRetryRateLimitedError) {
+          throw new ClusterRunManagementRateLimitedError(error.retryAfterMs);
+        }
+        if (error instanceof RunManualRetryUnavailableError) {
+          throw new ClusterRunManagementUnavailableError({ cause: error });
+        }
+        throw new ClusterRunManagementUnavailableError({ cause: error });
+      }
+    },
+  });
+}

packages/ql3-cluster-admin/src/run-management/runManagementCli.ts

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+
+import {
+  startClusterRunManagementProcess,
+  type ClusterRunManagementProcessRuntime,
+} from './runManagementProcess';
+
+const USAGE = 'Usage: ql3-run-manage';
+
+function fact(error: unknown): Readonly<Record<string, unknown>> {
+  const candidate = error as { readonly name?: unknown; readonly code?: unknown };
+  return Object.freeze({
+    schemaVersion: 1,
+    component: 'qinglong3-run-management',
+    event: 'management_failed',
+    name: typeof candidate?.name === 'string' ? candidate.name : 'Error',
+    ...(typeof candidate?.code === 'string' ? { code: candidate.code } : {}),
+  });
+}
+
+function emit(value: Readonly<Record<string, unknown>>): void {
+  process.stdout.write(`${JSON.stringify(value)}\n`);
+}
+
+async function run(argv: readonly string[]): Promise<void> {
+  if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
+    process.stdout.write(`${USAGE}\n`);
+    return;
+  }
+  if (argv.length !== 0) {
+    process.stderr.write(`${JSON.stringify({ code: 'QL3_RUN_MANAGEMENT_CLI_USAGE_INVALID', message: USAGE })}\n`);
+    process.exitCode = 64;
+    return;
+  }
+  let runtime: Readonly<ClusterRunManagementProcessRuntime>;
+  try {
+    runtime = await startClusterRunManagementProcess({
+      environment: process.env,
+      onError: () => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_unavailable' }),
+    });
+  } catch (error) {
+    process.stderr.write(`${JSON.stringify(fact(error))}\n`);
+    process.exitCode = 1;
+    return;
+  }
+  if (runtime.status === 'disabled') {
+    emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_disabled' });
+    return;
+  }
+  emit({
+    schemaVersion: 1,
+    component: 'qinglong3-run-management',
+    event: 'management_started',
+    address: runtime.address,
+    identityGeneration: runtime.identity.generation,
+    databaseContractVersion: runtime.database.contractVersion,
+    databaseMigrationCount: runtime.database.migrationIds.length,
+  });
+  let stopping: Promise<void> | undefined;
+  const stop = (): Promise<void> => {
+    stopping ??= runtime.close().then(() => emit({ schemaVersion: 1, component: 'qinglong3-run-management', event: 'management_stopped' }));
+    return stopping;
+  };
+  for (const signal of ['SIGINT', 'SIGTERM'] as const) {
+    process.once(signal, () => {
+      void stop().then(
+        () => { process.exitCode = 0; },
+        (error) => { process.stderr.write(`${JSON.stringify(fact(error))}\n`); process.exitCode = 1; },
+      );
+    });
+  }
+}
+
+void run(process.argv.slice(2));

packages/ql3-cluster-admin/src/run-management/runManagementClient.ts

@@ -0,0 +1,106 @@
+import {
+  RUN_MANUAL_RETRY_SCHEMA,
+  normalizeRunManualRetryResult,
+} from '@qinglong/runtime-core/run-manual-retry';
+import {
+  ClusterPluginPackageManagementClientRequestError,
+  executeClusterAuthenticatedManagementClient,
+  type ClusterAuthenticatedManagementClientResult,
+  type ClusterPluginPackageManagementClientConnectionOptions,
+  type ClusterPluginPackageManagementClientPaths,
+} from '../management-support/pluginPackageManagementClient';
+import {
+  normalizeClusterRunManagementCommand,
+  type ClusterRunManagementCommand,
+  type ClusterRunManagementTransportResult,
+} from './runManagementTransport';
+
+const MANAGEMENT_PATH = '/api/v3/runs/management';
+
+export type ClusterRunManagementClientPaths =
+  ClusterPluginPackageManagementClientPaths;
+export type ClusterRunManagementClientConnectionOptions =
+  ClusterPluginPackageManagementClientConnectionOptions;
+export type ClusterRunManagementClientResult =
+  ClusterAuthenticatedManagementClientResult<ClusterRunManagementTransportResult>;
+
+function invalid(): never {
+  throw new ClusterPluginPackageManagementClientRequestError();
+}
+
+function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
+  const actual = Object.keys(value as object).sort();
+  const expected = [...keys].sort();
+  if (
+    actual.length !== expected.length ||
+    actual.some((key, index) => key !== expected[index])
+  ) {
+    invalid();
+  }
+  return value as Record<string, unknown>;
+}
+
+export function validateClusterRunManagementClientResult(
+  value: unknown,
+  command: Readonly<ClusterRunManagementCommand>,
+): Readonly<ClusterRunManagementTransportResult> {
+  const envelope = exact(value, ['schemaVersion', 'operation', 'retry']);
+  if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
+  const retry = exact(envelope.retry, [
+    'schema',
+    'status',
+    'projectId',
+    'sourceRunId',
+    'sourceRunStatus',
+    'sourceRunVersion',
+    'runId',
+    'retryOfRunId',
+    'taskId',
+    'taskRevision',
+    'attemptId',
+    'runStatus',
+    'runVersion',
+    'eventSequence',
+    'executorType',
+    'executionRevisionDigest',
+    'createdAtMs',
+  ]);
+  if (retry.schema !== RUN_MANUAL_RETRY_SCHEMA) invalid();
+  try {
+    const { schema: _schema, ...result } = retry;
+    const normalized = normalizeRunManualRetryResult(result as never);
+    if (
+      normalized.projectId !== command.request.projectId ||
+      normalized.sourceRunId !== command.request.sourceRunId ||
+      normalized.sourceRunVersion !== command.request.body.expectedRunVersion ||
+      normalized.sourceRunStatus !== command.request.body.expectedRunStatus ||
+      normalized.executorType !== 'remote_worker'
+    ) {
+      invalid();
+    }
+  } catch {
+    invalid();
+  }
+  return Object.freeze(
+    envelope as unknown as ClusterRunManagementTransportResult,
+  );
+}
+
+const PROTOCOL = Object.freeze({
+  managementPath: MANAGEMENT_PATH,
+  clientCertificate: 'required' as const,
+  normalizeCommand: normalizeClusterRunManagementCommand,
+  validateResult: validateClusterRunManagementClientResult,
+});
+
+export function executeClusterRunManagementClient(
+  paths: ClusterRunManagementClientPaths,
+  connectionOptions?: ClusterRunManagementClientConnectionOptions,
+): Promise<Readonly<ClusterRunManagementClientResult>> {
+  return executeClusterAuthenticatedManagementClient(
+    paths,
+    PROTOCOL,
+    connectionOptions,
+  );
+}

packages/ql3-cluster-admin/src/run-management/runManagementClientCli.ts

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+
+import { ClusterPluginPackageManagementClientRemoteError } from '../management-support/pluginPackageManagementClient';
+import { executeClusterRunManagementClient } from './runManagementClient';
+
+const USAGE =
+  'Usage: ql3-run-client --config=/absolute/client.json --command=/absolute/command.json --assertion=/absolute/assertion.jwt';
+
+function argumentsFrom(argv: readonly string[]): Readonly<{
+  configFile: string;
+  commandFile: string;
+  assertionFile: string;
+}> | null {
+  if (argv.length !== 3) return null;
+  const values = new Map<string, string>();
+  for (const argument of argv) {
+    const match = /^--(config|command|assertion)=(\/.+)$/.exec(argument);
+    if (!match || values.has(match[1]!)) return null;
+    values.set(match[1]!, match[2]!);
+  }
+  if (!values.has('config') || !values.has('command') || !values.has('assertion')) return null;
+  return Object.freeze({
+    configFile: values.get('config')!,
+    commandFile: values.get('command')!,
+    assertionFile: values.get('assertion')!,
+  });
+}
+
+function failureFact(error: unknown): Readonly<Record<string, unknown>> {
+  const candidate = error as { readonly code?: unknown };
+  return Object.freeze({
+    schemaVersion: 1,
+    component: 'qinglong3-run-management-client',
+    event: 'command_failed',
+    code: typeof candidate?.code === 'string' ? candidate.code : 'QL3_RUN_MANAGEMENT_CLIENT_FAILED',
+    ...(error instanceof ClusterPluginPackageManagementClientRemoteError
+      ? {
+          statusCode: error.statusCode,
+          responseCode: error.responseCode,
+          requestId: error.requestId,
+          ...(error.retryAfterSeconds === null ? {} : { retryAfterSeconds: error.retryAfterSeconds }),
+        }
+      : {}),
+  });
+}
+
+async function run(argv: readonly string[]): Promise<void> {
+  if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
+    process.stdout.write(`${USAGE}\n`);
+    return;
+  }
+  const paths = argumentsFrom(argv);
+  if (!paths) {
+    process.stderr.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'usage_invalid', code: 'QL3_RUN_MANAGEMENT_CLIENT_USAGE_INVALID' })}\n`);
+    process.exitCode = 64;
+    return;
+  }
+  try {
+    const result = await executeClusterRunManagementClient(paths);
+    process.stdout.write(`${JSON.stringify({ schemaVersion: 1, component: 'qinglong3-run-management-client', event: 'command_completed', requestId: result.requestId, result: result.result })}\n`);
+  } catch (error) {
+    process.stderr.write(`${JSON.stringify(failureFact(error))}\n`);
+    process.exitCode = 1;
+  }
+}
+
+void run(process.argv.slice(2));

packages/ql3-cluster-admin/src/run-management/runManagementHttp.ts

@@ -0,0 +1,24 @@
+import {
+  CLUSTER_RUN_MANAGEMENT_PATH,
+  startClusterPluginPackageManagementHttp,
+  type ClusterPluginPackageManagementHttpApplication,
+  type StartClusterPluginPackageManagementHttpOptions,
+} from '../management-support/pluginPackageManagementHttp';
+
+export type ClusterRunManagementHttpApplication =
+  ClusterPluginPackageManagementHttpApplication;
+
+export type StartClusterRunManagementHttpOptions = Omit<
+  StartClusterPluginPackageManagementHttpOptions,
+  'managementPath'
+>;
+
+/** Starts the shared bounded OIDC/mTLS HTTPS adapter on the Run-only path. */
+export function startClusterRunManagementHttp(
+  options: StartClusterRunManagementHttpOptions,
+): Promise<Readonly<ClusterRunManagementHttpApplication>> {
+  return startClusterPluginPackageManagementHttp({
+    ...options,
+    managementPath: CLUSTER_RUN_MANAGEMENT_PATH,
+  });
+}

packages/ql3-cluster-admin/src/run-management/runManagementProcess.ts

@@ -0,0 +1,467 @@
+import type {
+  OpenPostgresDatabase,
+  PostgresDatabaseResource,
+} from '@qinglong/runtime-core';
+import {
+  PostgresRunManagementIdentityKeysetLedgerRepository,
+  assertPostgresRunManagerSchemaReady,
+  createPostgresDatabaseOpener,
+  isPostgresTlsDnsServername,
+  loadPostgresCertificateAuthorityFile,
+  loadPostgresConnectionEnvironment,
+  type PostgresConnectionOptions,
+  type PostgresPoolOptions,
+  type PostgresSchemaReadinessReport,
+} from '@qinglong/cluster-postgres/run-manager';
+import {
+  absoluteManagementEnvironmentFile,
+  booleanManagementEnvironmentValue,
+  boundedManagementEnvironmentValue,
+  integerManagementEnvironmentValue,
+  readManagementTlsFile,
+} from '../management-support/managementProcessSupport';
+import {
+  createClusterRunIdentityKeysetFile,
+  type ClusterPluginPackageIdentityKeysetFile,
+  type ClusterPluginPackageIdentityKeysetSnapshot,
+} from '../management-support/pluginPackageIdentityKeyset';
+import { validateClusterManagementClientTrust } from '../worker-credential/management-server/workerCredentialManagementMutualTls';
+import { createClusterRunManagementService } from './runManagement';
+import {
+  startClusterRunManagementHttp,
+  type ClusterRunManagementHttpApplication,
+  type StartClusterRunManagementHttpOptions,
+} from './runManagementHttp';
+import { createClusterRunManagementTransport } from './runManagementTransport';
+
+const SAFE_HOST = /^[A-Za-z0-9][A-Za-z0-9.:-]{0,254}$/;
+const SAFE_APPLICATION_NAME = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,62}$/;
+
+export type ClusterRunManagementProcessEnvironment = Readonly<
+  Record<string, string | undefined>
+>;
+
+export type ClusterRunManagementProcessConfig =
+  | Readonly<{ enabled: false }>
+  | Readonly<{
+      enabled: true;
+      profile: 'cluster-admin';
+      host: string;
+      port: number;
+      certificateFile: string;
+      privateKeyFile: string;
+      clientCertificateAuthorityFile: string;
+      clientCertificateRevocationListFile: string;
+      identityKeysetFile: string;
+      http: Readonly<{
+        maxBodyBytes: number;
+        maxConnections: number;
+        maxConcurrentRequests: number;
+        requestTimeoutMs: number;
+        drainTimeoutMs: number;
+        rateWindowMs: number;
+        peerRequestLimit: number;
+        globalRequestLimit: number;
+        maxRateLimitPeers: number;
+      }>;
+      database: Readonly<{
+        connection: PostgresConnectionOptions;
+        pool: PostgresPoolOptions;
+      }>;
+    }>;
+
+export type ClusterRunManagementProcessRuntime =
+  | Readonly<{ status: 'disabled'; close(): Promise<void> }>
+  | Readonly<{
+      status: 'active';
+      address: Readonly<{ host: string; port: number }>;
+      database: PostgresSchemaReadinessReport;
+      identity: ClusterPluginPackageIdentityKeysetSnapshot;
+      availabilityStatus(): 'ready' | 'unavailable' | 'stopped';
+      close(): Promise<void>;
+    }>;
+
+export interface StartClusterRunManagementProcessOptions {
+  readonly environment: ClusterRunManagementProcessEnvironment;
+  readonly openDatabase?: OpenPostgresDatabase;
+  readonly identities?: ClusterPluginPackageIdentityKeysetFile;
+  readonly assertReady?: (
+    pool: PostgresDatabaseResource['pool'],
+  ) => Promise<PostgresSchemaReadinessReport>;
+  readonly startHttp?: (
+    options: StartClusterRunManagementHttpOptions,
+  ) => Promise<Readonly<ClusterRunManagementHttpApplication>>;
+  readonly now?: () => number;
+  readonly randomUuid?: () => string;
+  readonly onError?: (error: unknown) => void;
+}
+
+export class ClusterRunManagementProcessConfigError extends TypeError {
+  readonly code = 'QL3_RUN_MANAGEMENT_PROCESS_CONFIG_INVALID';
+  constructor(message: string) {
+    super(`Run management process configuration is invalid: ${message}`);
+    this.name = 'ClusterRunManagementProcessConfigError';
+  }
+}
+
+function failure(message: string): ClusterRunManagementProcessConfigError {
+  return new ClusterRunManagementProcessConfigError(message);
+}
+
+function bounded(
+  environment: ClusterRunManagementProcessEnvironment,
+  name: string,
+  maximumLength: number,
+  required = false,
+): string | undefined {
+  return boundedManagementEnvironmentValue(
+    environment,
+    name,
+    maximumLength,
+    failure,
+    required,
+  );
+}
+
+function bool(
+  environment: ClusterRunManagementProcessEnvironment,
+  name: string,
+): boolean {
+  return booleanManagementEnvironmentValue(environment, name, failure);
+}
+
+function integer(
+  environment: ClusterRunManagementProcessEnvironment,
+  name: string,
+  fallback: number,
+  minimum: number,
+  maximum: number,
+): number {
+  return integerManagementEnvironmentValue(
+    environment,
+    name,
+    fallback,
+    minimum,
+    maximum,
+    failure,
+  );
+}
+
+function absolute(
+  environment: ClusterRunManagementProcessEnvironment,
+  name: string,
+): string {
+  return absoluteManagementEnvironmentFile(environment, name, failure);
+}
+
+function loadDatabase(
+  environment: ClusterRunManagementProcessEnvironment,
+): Readonly<{
+  connection: PostgresConnectionOptions;
+  pool: PostgresPoolOptions;
+}> {
+  let connection: PostgresConnectionOptions;
+  try {
+    connection = loadPostgresConnectionEnvironment(environment, {
+      connectionString: 'QL3_POSTGRES_RUN_MANAGER_URL',
+      host: 'QL3_POSTGRES_RUN_MANAGER_HOST',
+      port: 'QL3_POSTGRES_RUN_MANAGER_PORT',
+      database: 'QL3_POSTGRES_RUN_MANAGER_DATABASE',
+      user: 'QL3_POSTGRES_RUN_MANAGER_USER',
+      password: 'QL3_POSTGRES_RUN_MANAGER_PASSWORD',
+    });
+  } catch (error) {
+    throw failure(
+      error instanceof Error
+        ? error.message
+        : 'PostgreSQL run manager connection is invalid',
+    );
+  }
+  const mode = environment.QL3_POSTGRES_RUN_MANAGER_TLS_MODE ?? 'verify-full';
+  if (mode !== 'verify-full' && mode !== 'disable') {
+    throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_MODE must be verify-full or disable');
+  }
+  if (
+    mode === 'disable' &&
+    !bool(environment, 'QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE')
+  ) {
+    throw failure(
+      'disabling run manager PostgreSQL TLS requires QL3_POSTGRES_RUN_MANAGER_ALLOW_INSECURE=true',
+    );
+  }
+  const servername = bounded(
+    environment,
+    'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME',
+    253,
+  );
+  if (mode === 'verify-full' && !isPostgresTlsDnsServername(servername)) {
+    throw failure(
+      'QL3_POSTGRES_RUN_MANAGER_TLS_SERVERNAME must be an explicit DNS name',
+    );
+  }
+  const caFile = bounded(
+    environment,
+    'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE',
+    4_096,
+  );
+  if (mode === 'disable' && caFile !== undefined) {
+    throw failure(
+      'QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE cannot be used when TLS is disabled',
+    );
+  }
+  let ca: string | undefined;
+  if (caFile !== undefined) {
+    try {
+      ca = loadPostgresCertificateAuthorityFile(caFile);
+    } catch {
+      throw failure('QL3_POSTGRES_RUN_MANAGER_TLS_CA_FILE is invalid');
+    }
+  }
+  const applicationName =
+    bounded(environment, 'QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME', 63) ??
+    'qinglong3-run-manager';
+  if (!SAFE_APPLICATION_NAME.test(applicationName)) {
+    throw failure('QL3_POSTGRES_RUN_MANAGER_APPLICATION_NAME is invalid');
+  }
+  return Object.freeze({
+    connection: Object.freeze({
+      ...connection,
+      tls:
+        mode === 'disable'
+          ? Object.freeze({ mode: 'disable' as const })
+          : Object.freeze({
+              mode: 'verify-full' as const,
+              servername: servername!,
+              ...(ca === undefined ? {} : { ca }),
+            }),
+    }),
+    pool: Object.freeze({
+      applicationName,
+      maxConnections: integer(
+        environment,
+        'QL3_POSTGRES_RUN_MANAGER_POOL_MAX',
+        2,
+        1,
+        4,
+      ),
+      idleTimeoutMs: integer(
+        environment,
+        'QL3_POSTGRES_RUN_MANAGER_IDLE_TIMEOUT_MS',
+        10_000,
+        1_000,
+        60_000,
+      ),
+      connectionTimeoutMs: integer(
+        environment,
+        'QL3_POSTGRES_RUN_MANAGER_CONNECTION_TIMEOUT_MS',
+        5_000,
+        100,
+        60_000,
+      ),
+    }),
+  });
+}
+
+export function loadClusterRunManagementProcessConfig(
+  environment: ClusterRunManagementProcessEnvironment,
+): Readonly<ClusterRunManagementProcessConfig> {
+  if (!environment || typeof environment !== 'object') {
+    throw failure('environment is invalid');
+  }
+  if (!bool(environment, 'QL3_RUN_MANAGEMENT_ENABLED')) {
+    return Object.freeze({ enabled: false as const });
+  }
+  if (environment.QL3_PROFILE !== 'cluster-admin') {
+    throw failure('QL3_PROFILE must be cluster-admin when Run management is enabled');
+  }
+  const host = bounded(environment, 'QL3_RUN_MANAGEMENT_HOST', 255) ?? '0.0.0.0';
+  if (!SAFE_HOST.test(host)) throw failure('QL3_RUN_MANAGEMENT_HOST is invalid');
+  const http = Object.freeze({
+    maxBodyBytes: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_BODY_BYTES', 32 * 1024, 1_024, 256 * 1024),
+    maxConnections: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONNECTIONS', 32, 1, 512),
+    maxConcurrentRequests: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_CONCURRENT_REQUESTS', 16, 1, 256),
+    requestTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_REQUEST_TIMEOUT_MS', 10_000, 1_000, 60_000),
+    drainTimeoutMs: integer(environment, 'QL3_RUN_MANAGEMENT_DRAIN_TIMEOUT_MS', 5_000, 100, 60_000),
+    rateWindowMs: integer(environment, 'QL3_RUN_MANAGEMENT_RATE_WINDOW_MS', 60_000, 1_000, 5 * 60_000),
+    peerRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_PEER_REQUEST_LIMIT', 30, 1, 10_000),
+    globalRequestLimit: integer(environment, 'QL3_RUN_MANAGEMENT_GLOBAL_REQUEST_LIMIT', 300, 1, 100_000),
+    maxRateLimitPeers: integer(environment, 'QL3_RUN_MANAGEMENT_MAX_RATE_LIMIT_PEERS', 1_024, 1, 16_384),
+  });
+  if (http.globalRequestLimit < http.peerRequestLimit) {
+    throw failure('global request limit cannot be below peer request limit');
+  }
+  return Object.freeze({
+    enabled: true as const,
+    profile: 'cluster-admin' as const,
+    host,
+    port: integer(environment, 'QL3_RUN_MANAGEMENT_PORT', 8_448, 1, 65_535),
+    certificateFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_CERT_FILE'),
+    privateKeyFile: absolute(environment, 'QL3_RUN_MANAGEMENT_TLS_KEY_FILE'),
+    clientCertificateAuthorityFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CA_FILE'),
+    clientCertificateRevocationListFile: absolute(environment, 'QL3_RUN_MANAGEMENT_CLIENT_CRL_FILE'),
+    identityKeysetFile: absolute(environment, 'QL3_RUN_MANAGEMENT_IDENTITY_KEYSET_FILE'),
+    http,
+    database: loadDatabase(environment),
+  });
+}
+
+export async function startClusterRunManagementProcess(
+  options: StartClusterRunManagementProcessOptions,
+): Promise<Readonly<ClusterRunManagementProcessRuntime>> {
+  if (
+    !options ||
+    typeof options !== 'object' ||
+    Array.isArray(options) ||
+    Object.keys(options).some(
+      (key) =>
+        ![
+          'environment',
+          'openDatabase',
+          'identities',
+          'assertReady',
+          'startHttp',
+          'now',
+          'randomUuid',
+          'onError',
+        ].includes(key),
+    ) ||
+    !options.environment ||
+    typeof options.environment !== 'object' ||
+    (options.openDatabase !== undefined && typeof options.openDatabase !== 'function') ||
+    (options.identities !== undefined &&
+      (typeof options.identities.reload !== 'function' ||
+        typeof options.identities.bind !== 'function')) ||
+    (options.assertReady !== undefined && typeof options.assertReady !== 'function') ||
+    (options.startHttp !== undefined && typeof options.startHttp !== 'function') ||
+    (options.now !== undefined && typeof options.now !== 'function') ||
+    (options.randomUuid !== undefined && typeof options.randomUuid !== 'function') ||
+    (options.onError !== undefined && typeof options.onError !== 'function')
+  ) {
+    throw failure('options are invalid');
+  }
+  const config = loadClusterRunManagementProcessConfig(options.environment);
+  if (!config.enabled) {
+    return Object.freeze({ status: 'disabled' as const, close: () => Promise.resolve() });
+  }
+  const now = options.now ?? Date.now;
+  let http: Readonly<ClusterRunManagementHttpApplication> | undefined;
+  let database: PostgresDatabaseResource | undefined;
+  let unavailableError: unknown;
+  let closePromise: Promise<void> | undefined;
+  const report = (error: unknown): void => {
+    try {
+      options.onError?.(error);
+    } catch {
+      // Diagnostics never own availability or cleanup.
+    }
+  };
+  const openDatabase =
+    options.openDatabase ??
+    createPostgresDatabaseOpener({
+      role: 'run-manager',
+      connection: config.database.connection,
+      pool: config.database.pool,
+      onPoolError(error) {
+        const first = unavailableError === undefined;
+        unavailableError ??= error;
+        http?.withdraw(error);
+        if (first) report(error);
+      },
+    });
+  try {
+    database = await openDatabase();
+    const evidence = await (options.assertReady ?? assertPostgresRunManagerSchemaReady)(database.pool);
+    if (unavailableError !== undefined) throw unavailableError;
+    const identities =
+      options.identities ??
+      createClusterRunIdentityKeysetFile({
+        filePath: config.identityKeysetFile,
+        now,
+        ledger: new PostgresRunManagementIdentityKeysetLedgerRepository(
+          database.pool,
+          'run-management',
+        ),
+      });
+    const identity = await identities.reload();
+    const service = createClusterRunManagementService({
+      pool: database.pool,
+      now,
+      ...(options.randomUuid === undefined ? {} : { randomUuid: options.randomUuid }),
+    });
+    const transport = createClusterRunManagementTransport({ service, now });
+    const privateKey = readManagementTlsFile(config.privateKeyFile, true, failure);
+    try {
+      const certificate = readManagementTlsFile(config.certificateFile, false, failure);
+      const clientCertificateAuthority = readManagementTlsFile(
+        config.clientCertificateAuthorityFile,
+        false,
+        failure,
+      );
+      const clientCertificateRevocationList = readManagementTlsFile(
+        config.clientCertificateRevocationListFile,
+        false,
+        failure,
+      );
+      validateClusterManagementClientTrust(
+        clientCertificateAuthority,
+        clientCertificateRevocationList,
+        now(),
+        failure,
+      );
+      http = await (options.startHttp ?? startClusterRunManagementHttp)({
+        host: config.host,
+        port: config.port,
+        tls: {
+          privateKey,
+          certificate,
+          clientCertificateAuthority,
+          clientCertificateRevocationList,
+        },
+        transport,
+        identities,
+        limits: config.http,
+        now,
+        onError: report,
+      });
+    } finally {
+      privateKey.fill(0);
+    }
+    if (unavailableError !== undefined) http.withdraw(unavailableError);
+    return Object.freeze({
+      status: 'active' as const,
+      address: http.address,
+      database: evidence,
+      identity,
+      availabilityStatus: () => http?.availabilityStatus() ?? 'stopped',
+      close(): Promise<void> {
+        closePromise ??= (async () => {
+          let primaryError: unknown;
+          try {
+            await http?.close();
+          } catch (error) {
+            primaryError = error;
+          }
+          try {
+            await database?.close();
+          } catch (error) {
+            primaryError ??= error;
+          }
+          if (primaryError) throw primaryError;
+        })();
+        return closePromise;
+      },
+    });
+  } catch (error) {
+    try {
+      await http?.close();
+    } catch {
+      // Preserve startup failure.
+    }
+    try {
+      await database?.close();
+    } catch {
+      // Preserve startup failure.
+    }
+    throw error;
+  }
+}

packages/ql3-cluster-admin/src/run-management/runManagementTransport.ts

@@ -0,0 +1,214 @@
+import {
+  createRunManualRetryResponseBody,
+  parseRunManualRetryRequestBody,
+  type RunManualRetryResponseBody,
+} from '@qinglong/runtime-core/run-manual-retry';
+import {
+  normalizeSecurityPrincipal,
+  type SecurityPrincipal,
+} from '@qinglong/runtime-core/security';
+import type { ClusterRunManagementService } from './runManagement';
+
+const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+const UUID_PATTERN =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
+
+export type ClusterRunManagementCommand = Readonly<{
+  schemaVersion: 1;
+  operation: 'run.retry';
+  request: Readonly<{
+    projectId: string;
+    sourceRunId: string;
+    requestId: string;
+    auditEventId: string;
+    failureAuditEventId: string;
+    body: Readonly<{
+      schema: 'qinglong/run-manual-retry@v1';
+      mutationId: string;
+      expectedRunVersion: number;
+      expectedRunStatus: 'failed' | 'cancelled' | 'timed_out';
+    }>;
+  }>;
+}>;
+
+export type ClusterRunManagementTransportResult = Readonly<{
+  schemaVersion: 1;
+  operation: 'run.retry';
+  retry: Readonly<RunManualRetryResponseBody>;
+}>;
+
+export interface ClusterRunManagementAuthentication {
+  authenticate(): Promise<Readonly<SecurityPrincipal> | null>;
+}
+
+export interface ClusterRunManagementTransport {
+  execute(
+    command: unknown,
+    authentication: ClusterRunManagementAuthentication,
+  ): Promise<Readonly<ClusterRunManagementTransportResult>>;
+}
+
+export class ClusterRunManagementTransportConfigurationError extends TypeError {
+  readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_CONFIGURATION_INVALID';
+  constructor() {
+    super('Cluster Run management transport configuration is invalid');
+    this.name = 'ClusterRunManagementTransportConfigurationError';
+  }
+}
+
+export class ClusterRunManagementTransportRequestError extends TypeError {
+  readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_REQUEST_INVALID';
+  constructor() {
+    super('Cluster Run management transport request is invalid');
+    this.name = 'ClusterRunManagementTransportRequestError';
+  }
+}
+
+export class ClusterRunManagementTransportAuthenticationError extends Error {
+  readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_AUTHENTICATION_REQUIRED';
+  constructor() {
+    super('Cluster Run management transport requires a strong User principal');
+    this.name = 'ClusterRunManagementTransportAuthenticationError';
+  }
+}
+
+export class ClusterRunManagementTransportUnavailableError extends Error {
+  readonly code = 'CLUSTER_RUN_MANAGEMENT_TRANSPORT_UNAVAILABLE';
+  constructor() {
+    super('Cluster Run management transport is unavailable');
+    this.name = 'ClusterRunManagementTransportUnavailableError';
+  }
+}
+
+function invalid(): never {
+  throw new ClusterRunManagementTransportRequestError();
+}
+
+function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
+  const actual = Object.keys(value as object).sort();
+  const expected = [...keys].sort();
+  if (
+    actual.length !== expected.length ||
+    actual.some((key, index) => key !== expected[index])
+  ) {
+    invalid();
+  }
+  return value as Record<string, unknown>;
+}
+
+function identifier(value: unknown): string {
+  if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) invalid();
+  return value;
+}
+
+function uuid(value: unknown): string {
+  if (typeof value !== 'string' || !UUID_PATTERN.test(value)) invalid();
+  return value;
+}
+
+export function normalizeClusterRunManagementCommand(
+  value: unknown,
+): Readonly<ClusterRunManagementCommand> {
+  const envelope = exact(value, ['schemaVersion', 'operation', 'request']);
+  if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
+  const request = exact(envelope.request, [
+    'projectId',
+    'sourceRunId',
+    'requestId',
+    'auditEventId',
+    'failureAuditEventId',
+    'body',
+  ]);
+  let body: ReturnType<typeof parseRunManualRetryRequestBody>;
+  try {
+    body = parseRunManualRetryRequestBody(request.body);
+  } catch {
+    invalid();
+  }
+  const auditEventId = uuid(request.auditEventId);
+  const failureAuditEventId = uuid(request.failureAuditEventId);
+  if (auditEventId === failureAuditEventId) invalid();
+  return Object.freeze({
+    schemaVersion: 1,
+    operation: 'run.retry',
+    request: Object.freeze({
+      projectId: identifier(request.projectId),
+      sourceRunId: identifier(request.sourceRunId),
+      requestId: identifier(request.requestId),
+      auditEventId,
+      failureAuditEventId,
+      body,
+    }),
+  });
+}
+
+export function createClusterRunManagementTransport(options: Readonly<{
+  service: ClusterRunManagementService;
+  now?: () => number;
+}>): Readonly<ClusterRunManagementTransport> {
+  if (
+    !options ||
+    typeof options !== 'object' ||
+    Array.isArray(options) ||
+    Object.keys(options).some((key) => key !== 'service' && key !== 'now') ||
+    !options.service ||
+    typeof options.service.retry !== 'function' ||
+    (options.now !== undefined && typeof options.now !== 'function')
+  ) {
+    throw new ClusterRunManagementTransportConfigurationError();
+  }
+  const now = options.now ?? Date.now;
+  return Object.freeze({
+    async execute(
+      commandValue: unknown,
+      authentication: ClusterRunManagementAuthentication,
+    ) {
+      const command = normalizeClusterRunManagementCommand(commandValue);
+      if (
+        !authentication ||
+        typeof authentication !== 'object' ||
+        Array.isArray(authentication) ||
+        Object.keys(authentication).length !== 1 ||
+        typeof authentication.authenticate !== 'function'
+      ) {
+        throw new ClusterRunManagementTransportConfigurationError();
+      }
+      let candidate: Readonly<SecurityPrincipal> | null;
+      try {
+        candidate = await authentication.authenticate();
+      } catch {
+        throw new ClusterRunManagementTransportUnavailableError();
+      }
+      let principal: Readonly<SecurityPrincipal>;
+      try {
+        principal = normalizeSecurityPrincipal(candidate as SecurityPrincipal, now());
+      } catch {
+        throw new ClusterRunManagementTransportAuthenticationError();
+      }
+      if (
+        principal.subject.type !== 'user' ||
+        !STRONG_ASSURANCES.has(principal.assurance)
+      ) {
+        throw new ClusterRunManagementTransportAuthenticationError();
+      }
+      const result = await options.service.retry({
+        projectId: command.request.projectId,
+        sourceRunId: command.request.sourceRunId,
+        mutationId: command.request.body.mutationId,
+        expectedRunVersion: command.request.body.expectedRunVersion,
+        expectedRunStatus: command.request.body.expectedRunStatus,
+        requestId: command.request.requestId,
+        auditEventId: command.request.auditEventId,
+        failureAuditEventId: command.request.failureAuditEventId,
+        principal,
+      });
+      return Object.freeze({
+        schemaVersion: 1,
+        operation: 'run.retry',
+        retry: createRunManualRetryResponseBody(result),
+      });
+    },
+  });
+}