feat(ql3): add bounded copilot product client

packages/ql3-cluster-admin/src/copilot-client/cli.ts

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+
+import {
+  ClusterCopilotClientRemoteError,
+  executeClusterCopilotClient,
+} from './client';
+
+const USAGE =
+  'Usage: ql3-copilot-client --config=/absolute/client.json --command=/absolute/command.json --credential=/absolute/credential';
+
+function argumentsFrom(argv: readonly string[]): Readonly<{
+  configFile: string;
+  commandFile: string;
+  credentialFile: string;
+}> | null {
+  if (argv.length !== 3) return null;
+  const values = new Map<string, string>();
+  for (const argument of argv) {
+    const match = /^--(config|command|credential)=(\/.+)$/.exec(argument);
+    if (!match || values.has(match[1]!)) return null;
+    values.set(match[1]!, match[2]!);
+  }
+  if (
+    !values.has('config') ||
+    !values.has('command') ||
+    !values.has('credential')
+  ) {
+    return null;
+  }
+  return Object.freeze({
+    configFile: values.get('config')!,
+    commandFile: values.get('command')!,
+    credentialFile: values.get('credential')!,
+  });
+}
+
+function failureFact(error: unknown): Readonly<Record<string, unknown>> {
+  const candidate = error as { readonly code?: unknown };
+  return Object.freeze({
+    schemaVersion: 1,
+    component: 'qinglong3-cluster-copilot-client',
+    event: 'command_failed',
+    code:
+      typeof candidate?.code === 'string'
+        ? candidate.code
+        : 'QL3_CLUSTER_COPILOT_CLIENT_FAILED',
+    ...(error instanceof ClusterCopilotClientRemoteError
+      ? {
+          statusCode: error.statusCode,
+          responseCode: error.responseCode,
+          requestId: error.requestId,
+          ...(error.retryAfterSeconds === null
+            ? {}
+            : { retryAfterSeconds: error.retryAfterSeconds }),
+        }
+      : {}),
+  });
+}
+
+async function run(argv: readonly string[]): Promise<void> {
+  if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
+    process.stdout.write(`${USAGE}\n`);
+    return;
+  }
+  const paths = argumentsFrom(argv);
+  if (!paths) {
+    process.stderr.write(
+      `${JSON.stringify({
+        schemaVersion: 1,
+        component: 'qinglong3-cluster-copilot-client',
+        event: 'usage_invalid',
+        code: 'QL3_CLUSTER_COPILOT_CLIENT_USAGE_INVALID',
+      })}\n`,
+    );
+    process.exitCode = 64;
+    return;
+  }
+  try {
+    const result = await executeClusterCopilotClient(paths);
+    process.stdout.write(
+      `${JSON.stringify({
+        schemaVersion: 1,
+        component: 'qinglong3-cluster-copilot-client',
+        event: 'command_completed',
+        operation: result.operation,
+        requestId: result.requestId,
+        result: result.result,
+      })}\n`,
+    );
+  } catch (error) {
+    process.stderr.write(`${JSON.stringify(failureFact(error))}\n`);
+    process.exitCode = 1;
+  }
+}
+
+void run(process.argv.slice(2));

packages/ql3-cluster-admin/src/copilot-client/client.ts

@@ -0,0 +1,636 @@
+import { randomUUID, X509Certificate } from 'node:crypto';
+import { request as httpsRequest } from 'node:https';
+import { isIP, type LookupFunction } from 'node:net';
+import { TextDecoder } from 'node:util';
+
+import {
+  ClusterPluginPackageManagementClientConfigurationError,
+  readCanonicalFile,
+} from '../management-support/managementClientConfiguration';
+import {
+  InvalidClusterCopilotClientCommandError,
+  InvalidClusterCopilotClientResponseError,
+  normalizeClusterCopilotClientCommand,
+  prepareClusterCopilotClientRequest,
+  validateClusterCopilotClientResponse,
+  type ClusterCopilotClientCommand,
+  type ClusterCopilotClientOperation,
+} from './contracts';
+
+export {
+  CLUSTER_COPILOT_CLIENT_COMMAND_SCHEMA,
+  CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA,
+  CLUSTER_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESPONSE_SCHEMA,
+  CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESPONSE_SCHEMA,
+  CLUSTER_COPILOT_FAILURE_DIAGNOSIS_REQUEST_SCHEMA,
+  CLUSTER_COPILOT_FAILURE_DIAGNOSIS_RESPONSE_SCHEMA,
+  InvalidClusterCopilotClientCommandError,
+  InvalidClusterCopilotClientResponseError,
+  normalizeClusterCopilotClientCommand,
+  prepareClusterCopilotClientRequest,
+  validateClusterCopilotClientResponse,
+} from './contracts';
+export type {
+  ClusterCopilotClientCommand,
+  ClusterCopilotClientOperation,
+  ClusterCopilotClientPreparedRequest,
+} from './contracts';
+
+export const CLUSTER_COPILOT_CLIENT_CONFIG_SCHEMA =
+  'qinglong/cluster-copilot-client-config@v1' as const;
+
+export interface ClusterCopilotClientPaths {
+  readonly configFile: string;
+  readonly commandFile: string;
+  readonly credentialFile: string;
+}
+
+export interface ClusterCopilotClientOptions {
+  readonly createRequestId?: () => string;
+  readonly lookup?: LookupFunction;
+}
+
+export interface ClusterCopilotClientResult {
+  readonly schemaVersion: 1;
+  readonly operation: ClusterCopilotClientOperation;
+  readonly requestId: string;
+  readonly result: Readonly<Record<string, unknown>>;
+}
+
+export interface ClusterCopilotClientConfigurationSummary {
+  readonly schemaVersion: 1;
+  readonly transport: 'https';
+  readonly clientCertificate: 'forbidden';
+}
+
+export interface ClusterCopilotClientReadiness {
+  readonly schemaVersion: 1;
+  readonly transport: 'https';
+  readonly ready: boolean;
+}
+
+interface PreparedClusterCopilotClientConfiguration {
+  readonly endpoint: URL;
+  readonly servername: string;
+  readonly port: number;
+  readonly requestTimeoutMs: number;
+  readonly caBytes: Buffer;
+  dispose(): void;
+}
+
+interface JsonResponse {
+  readonly statusCode: number;
+  readonly headers: Readonly<Record<string, string | string[] | undefined>>;
+  readonly rawHeaders: readonly string[];
+  readonly body: unknown;
+}
+
+export class ClusterCopilotClientConfigurationError extends TypeError {
+  readonly code = 'QL3_CLUSTER_COPILOT_CLIENT_CONFIG_INVALID';
+
+  constructor() {
+    super('Cluster Copilot client configuration is invalid');
+    this.name = 'ClusterCopilotClientConfigurationError';
+  }
+}
+
+export class ClusterCopilotClientRequestError extends Error {
+  readonly code = 'QL3_CLUSTER_COPILOT_CLIENT_REQUEST_FAILED';
+
+  constructor(options?: ErrorOptions) {
+    super('Cluster Copilot client request failed', options);
+    this.name = 'ClusterCopilotClientRequestError';
+  }
+}
+
+export class ClusterCopilotClientRemoteError extends Error {
+  readonly code = 'QL3_CLUSTER_COPILOT_CLIENT_REMOTE_REJECTED';
+
+  constructor(
+    readonly statusCode: number,
+    readonly responseCode: string,
+    readonly requestId: string,
+    readonly retryAfterSeconds: number | null,
+  ) {
+    super('Cluster Copilot server rejected the request');
+    this.name = 'ClusterCopilotClientRemoteError';
+  }
+}
+
+const MAXIMUM_CONFIG_BYTES = 16 * 1024;
+const MAXIMUM_CA_BYTES = 256 * 1024;
+const MAXIMUM_COMMAND_BYTES = 16 * 1024;
+const MAXIMUM_CREDENTIAL_BYTES = 256;
+const MAXIMUM_RESPONSE_BYTES = 2 * 1024 * 1024;
+const MAXIMUM_READINESS_RESPONSE_BYTES = 1_024;
+const DNS_NAME =
+  /^(?=.{1,253}$)[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*$/;
+const API_CREDENTIAL =
+  /^ql3c_[A-Za-z0-9][A-Za-z0-9._:-]{0,63}_[A-Za-z0-9_-]{43}$/;
+const RESPONSE_CODE = /^[a-z][a-z0-9_]{0,127}$/;
+
+function configurationFailure(): never {
+  throw new ClusterCopilotClientConfigurationError();
+}
+
+function exact(
+  value: unknown,
+  keys: readonly string[],
+): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return configurationFailure();
+  }
+  const actual = Object.keys(value).sort();
+  const expected = [...keys].sort();
+  if (
+    actual.length !== expected.length ||
+    actual.some((key, index) => key !== expected[index])
+  ) {
+    return configurationFailure();
+  }
+  return value as Record<string, unknown>;
+}
+
+function decodeJson(bytes: Buffer, kind: 'config' | 'command'): unknown {
+  try {
+    return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes));
+  } catch {
+    if (kind === 'command') {
+      throw new InvalidClusterCopilotClientCommandError();
+    }
+    return configurationFailure();
+  }
+}
+
+function prepareConfiguration(
+  configFile: string,
+): PreparedClusterCopilotClientConfiguration {
+  let configBytes: Buffer | undefined;
+  let caBytes: Buffer | undefined;
+  try {
+    configBytes = readCanonicalFile(
+      configFile,
+      MAXIMUM_CONFIG_BYTES,
+      'private',
+    );
+    const config = exact(decodeJson(configBytes, 'config'), [
+      'caFile',
+      'endpoint',
+      'requestTimeoutMs',
+      'schema',
+      'servername',
+    ]);
+    if (
+      config.schema !== CLUSTER_COPILOT_CLIENT_CONFIG_SCHEMA ||
+      typeof config.endpoint !== 'string' ||
+      typeof config.servername !== 'string' ||
+      !DNS_NAME.test(config.servername) ||
+      isIP(config.servername) !== 0 ||
+      typeof config.caFile !== 'string' ||
+      !Number.isSafeInteger(config.requestTimeoutMs) ||
+      (config.requestTimeoutMs as number) < 1_000 ||
+      (config.requestTimeoutMs as number) > 120_000
+    ) {
+      return configurationFailure();
+    }
+    let endpoint: URL;
+    try {
+      endpoint = new URL(config.endpoint);
+    } catch {
+      return configurationFailure();
+    }
+    if (
+      endpoint.protocol !== 'https:' ||
+      endpoint.username !== '' ||
+      endpoint.password !== '' ||
+      endpoint.search !== '' ||
+      endpoint.hash !== '' ||
+      endpoint.pathname !== '/' ||
+      endpoint.hostname !== config.servername ||
+      isIP(endpoint.hostname) !== 0
+    ) {
+      return configurationFailure();
+    }
+    const port = endpoint.port === '' ? 443 : Number(endpoint.port);
+    if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) {
+      return configurationFailure();
+    }
+    caBytes = readCanonicalFile(
+      config.caFile,
+      MAXIMUM_CA_BYTES,
+      'public-integrity',
+    );
+    try {
+      new X509Certificate(caBytes);
+    } catch {
+      return configurationFailure();
+    }
+    let disposed = false;
+    return Object.freeze({
+      endpoint,
+      servername: config.servername,
+      port,
+      requestTimeoutMs: config.requestTimeoutMs as number,
+      caBytes,
+      dispose() {
+        if (disposed) return;
+        disposed = true;
+        caBytes?.fill(0);
+      },
+    });
+  } catch (error) {
+    caBytes?.fill(0);
+    if (error instanceof ClusterCopilotClientConfigurationError) throw error;
+    throw new ClusterCopilotClientConfigurationError();
+  } finally {
+    configBytes?.fill(0);
+  }
+}
+
+function validateOptions(
+  options: ClusterCopilotClientOptions | undefined,
+): Readonly<ClusterCopilotClientOptions> {
+  if (options === undefined) return Object.freeze({});
+  if (
+    !options ||
+    typeof options !== 'object' ||
+    Array.isArray(options) ||
+    Object.keys(options).some(
+      (key) => key !== 'createRequestId' && key !== 'lookup',
+    ) ||
+    (options.createRequestId !== undefined &&
+      typeof options.createRequestId !== 'function') ||
+    (options.lookup !== undefined && typeof options.lookup !== 'function')
+  ) {
+    return configurationFailure();
+  }
+  return Object.freeze({ ...options });
+}
+
+function rawHeaderCount(rawHeaders: readonly string[], name: string): number {
+  let count = 0;
+  for (let index = 0; index < rawHeaders.length; index += 2) {
+    if (rawHeaders[index]?.toLowerCase() === name) count += 1;
+  }
+  return count;
+}
+
+function responseHeadersValid(
+  response: JsonResponse,
+  byteLength: number,
+): boolean {
+  const contentLength = response.headers['content-length'];
+  return (
+    rawHeaderCount(response.rawHeaders, 'content-type') === 1 &&
+    response.headers['content-type'] === 'application/json; charset=utf-8' &&
+    response.headers['content-encoding'] === undefined &&
+    rawHeaderCount(response.rawHeaders, 'content-length') <= 1 &&
+    (contentLength === undefined ||
+      (typeof contentLength === 'string' &&
+        /^(?:0|[1-9][0-9]*)$/.test(contentLength) &&
+        Number(contentLength) === byteLength))
+  );
+}
+
+function readinessStatus(value: unknown): 'ready' | 'not_ready' {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new ClusterCopilotClientRequestError();
+  }
+  const record = value as Record<string, unknown>;
+  if (
+    Object.keys(record).length !== 1 ||
+    !Object.hasOwn(record, 'status') ||
+    (record.status !== 'ready' && record.status !== 'not_ready')
+  ) {
+    throw new ClusterCopilotClientRequestError();
+  }
+  return record.status;
+}
+
+function requestJson(
+  prepared: PreparedClusterCopilotClientConfiguration,
+  request: Readonly<{
+    method: 'GET' | 'POST';
+    path: string;
+    requestId?: string;
+    authorization?: string;
+    body?: Buffer;
+  }>,
+  maximumResponseBytes: number,
+  options: Readonly<ClusterCopilotClientOptions>,
+): Promise<Readonly<JsonResponse>> {
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    const chunks: Buffer[] = [];
+    let length = 0;
+    const clearChunks = (): void => {
+      for (const chunk of chunks) chunk.fill(0);
+    };
+    const finish = (error: unknown, result?: Readonly<JsonResponse>): void => {
+      if (settled) return;
+      settled = true;
+      if (error) reject(error);
+      else resolve(result!);
+    };
+    const outgoing = httpsRequest(
+      {
+        protocol: 'https:',
+        hostname: prepared.endpoint.hostname,
+        port: prepared.port,
+        path: request.path,
+        method: request.method,
+        servername: prepared.servername,
+        ca: prepared.caBytes,
+        minVersion: 'TLSv1.3',
+        maxVersion: 'TLSv1.3',
+        rejectUnauthorized: true,
+        agent: false,
+        ...(options.lookup === undefined ? {} : { lookup: options.lookup }),
+        headers: {
+          accept: 'application/json',
+          'accept-encoding': 'identity',
+          connection: 'close',
+          ...(request.authorization === undefined
+            ? {}
+            : { authorization: request.authorization }),
+          ...(request.requestId === undefined
+            ? {}
+            : { 'x-request-id': request.requestId }),
+          ...(request.body === undefined
+            ? {}
+            : {
+                'content-type': 'application/json; charset=utf-8',
+                'content-length': String(request.body.length),
+              }),
+        },
+      },
+      (incoming) => {
+        incoming.once('aborted', () => {
+          clearChunks();
+          finish(new ClusterCopilotClientRequestError());
+        });
+        incoming.once('error', (cause) => {
+          clearChunks();
+          finish(new ClusterCopilotClientRequestError({ cause }));
+        });
+        incoming.on('data', (chunk: Buffer | string) => {
+          const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+          length += bytes.length;
+          if (length > maximumResponseBytes) {
+            bytes.fill(0);
+            clearChunks();
+            const error = new ClusterCopilotClientRequestError();
+            incoming.destroy();
+            outgoing.destroy(error);
+            finish(error);
+            return;
+          }
+          chunks.push(bytes);
+        });
+        incoming.once('end', () => {
+          const bytes = Buffer.concat(chunks, length);
+          try {
+            const provisional: JsonResponse = Object.freeze({
+              statusCode: incoming.statusCode ?? 0,
+              headers: incoming.headers,
+              rawHeaders: Object.freeze([...incoming.rawHeaders]),
+              body: null,
+            });
+            if (!responseHeadersValid(provisional, bytes.length)) {
+              throw new ClusterCopilotClientRequestError();
+            }
+            let body: unknown;
+            try {
+              body = JSON.parse(
+                new TextDecoder('utf-8', { fatal: true }).decode(bytes),
+              );
+            } catch (cause) {
+              throw new ClusterCopilotClientRequestError({ cause });
+            }
+            finish(
+              undefined,
+              Object.freeze({ ...provisional, body }),
+            );
+          } catch (error) {
+            finish(
+              error instanceof ClusterCopilotClientRequestError
+                ? error
+                : new ClusterCopilotClientRequestError({
+                    cause: error instanceof Error ? error : undefined,
+                  }),
+            );
+          } finally {
+            bytes.fill(0);
+            clearChunks();
+          }
+        });
+      },
+    );
+    outgoing.setTimeout(prepared.requestTimeoutMs, () => {
+      outgoing.destroy(new ClusterCopilotClientRequestError());
+    });
+    outgoing.once('error', (cause) => {
+      clearChunks();
+      finish(
+        cause instanceof ClusterCopilotClientRequestError
+          ? cause
+          : new ClusterCopilotClientRequestError({ cause }),
+      );
+    });
+    outgoing.end(request.body);
+  });
+}
+
+function responseRequestId(
+  response: JsonResponse,
+  expected: string,
+): string {
+  const value = response.headers['x-request-id'];
+  if (
+    rawHeaderCount(response.rawHeaders, 'x-request-id') !== 1 ||
+    typeof value !== 'string' ||
+    value !== expected
+  ) {
+    throw new ClusterCopilotClientRequestError();
+  }
+  return value;
+}
+
+function retryAfterSeconds(value: string | string[] | undefined): number | null {
+  if (typeof value !== 'string' || !/^[1-9][0-9]{0,3}$/.test(value)) {
+    return null;
+  }
+  const seconds = Number(value);
+  return seconds <= 3_600 ? seconds : null;
+}
+
+function remoteCode(value: unknown): string {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new ClusterCopilotClientRequestError();
+  }
+  const record = value as Record<string, unknown>;
+  const keys = Object.keys(record).sort();
+  if (
+    keys.length < 1 ||
+    keys.length > 3 ||
+    keys[0] !== 'code' ||
+    keys.some((key) => key !== 'code' && key !== 'reason' && key !== 'schema') ||
+    typeof record.code !== 'string' ||
+    !RESPONSE_CODE.test(record.code)
+  ) {
+    throw new ClusterCopilotClientRequestError();
+  }
+  return record.code;
+}
+
+export function validateClusterCopilotClientConfiguration(
+  configFile: string,
+): Readonly<ClusterCopilotClientConfigurationSummary> {
+  const prepared = prepareConfiguration(configFile);
+  try {
+    return Object.freeze({
+      schemaVersion: 1,
+      transport: 'https',
+      clientCertificate: 'forbidden',
+    });
+  } finally {
+    prepared.dispose();
+  }
+}
+
+export async function probeClusterCopilotClientReadiness(
+  configFile: string,
+  options?: ClusterCopilotClientOptions,
+): Promise<Readonly<ClusterCopilotClientReadiness>> {
+  const normalizedOptions = validateOptions(options);
+  const prepared = prepareConfiguration(configFile);
+  try {
+    const response = await requestJson(
+      prepared,
+      Object.freeze({ method: 'GET', path: '/readyz' }),
+      MAXIMUM_READINESS_RESPONSE_BYTES,
+      normalizedOptions,
+    );
+    const status = readinessStatus(response.body);
+    const ready = response.statusCode === 200 && status === 'ready';
+    const notReady =
+      response.statusCode === 503 && status === 'not_ready';
+    if (!ready && !notReady) throw new ClusterCopilotClientRequestError();
+    return Object.freeze({ schemaVersion: 1, transport: 'https', ready });
+  } catch (error) {
+    if (
+      error instanceof ClusterCopilotClientConfigurationError ||
+      error instanceof ClusterCopilotClientRequestError
+    ) {
+      throw error;
+    }
+    throw new ClusterCopilotClientRequestError({
+      cause: error instanceof Error ? error : undefined,
+    });
+  } finally {
+    prepared.dispose();
+  }
+}
+
+export async function executeClusterCopilotClient(
+  paths: ClusterCopilotClientPaths,
+  options?: ClusterCopilotClientOptions,
+): Promise<Readonly<ClusterCopilotClientResult>> {
+  const normalizedOptions = validateOptions(options);
+  exact(paths, ['commandFile', 'configFile', 'credentialFile']);
+  let commandBytes: Buffer | undefined;
+  let credentialBytes: Buffer | undefined;
+  let bodyBytes: Buffer | undefined;
+  let prepared: PreparedClusterCopilotClientConfiguration | undefined;
+  try {
+    prepared = prepareConfiguration(paths.configFile);
+    commandBytes = readCanonicalFile(
+      paths.commandFile,
+      MAXIMUM_COMMAND_BYTES,
+      'private',
+    );
+    credentialBytes = readCanonicalFile(
+      paths.credentialFile,
+      MAXIMUM_CREDENTIAL_BYTES,
+      'private',
+    );
+    const command = normalizeClusterCopilotClientCommand(
+      decodeJson(commandBytes, 'command'),
+    );
+    if (credentialBytes.some((byte) => byte > 0x7f)) {
+      return configurationFailure();
+    }
+    const credential = credentialBytes.toString('ascii');
+    if (!API_CREDENTIAL.test(credential)) return configurationFailure();
+    const transportRequestId =
+      command.operation === 'inspect' || command.operation === 'output'
+        ? (normalizedOptions.createRequestId ?? randomUUID)()
+        : undefined;
+    const request = prepareClusterCopilotClientRequest(
+      command,
+      transportRequestId,
+    );
+    if (request.body !== null) {
+      bodyBytes = Buffer.from(JSON.stringify(request.body), 'utf8');
+      if (
+        bodyBytes.length < 2 ||
+        bodyBytes.length > MAXIMUM_COMMAND_BYTES
+      ) {
+        return configurationFailure();
+      }
+    }
+    const response = await requestJson(
+      prepared,
+      Object.freeze({
+        method: request.method,
+        path: request.path,
+        requestId: request.requestId,
+        authorization: `Bearer ${credential}`,
+        ...(bodyBytes === undefined ? {} : { body: bodyBytes }),
+      }),
+      MAXIMUM_RESPONSE_BYTES,
+      normalizedOptions,
+    );
+    const requestId = responseRequestId(response, request.requestId);
+    if (request.acceptedStatusCodes.includes(response.statusCode)) {
+      return Object.freeze({
+        schemaVersion: 1,
+        operation: command.operation,
+        requestId,
+        result: validateClusterCopilotClientResponse(response.body, command),
+      });
+    }
+    if (response.statusCode >= 400 && response.statusCode <= 599) {
+      throw new ClusterCopilotClientRemoteError(
+        response.statusCode,
+        remoteCode(response.body),
+        requestId,
+        retryAfterSeconds(response.headers['retry-after']),
+      );
+    }
+    throw new ClusterCopilotClientRequestError();
+  } catch (error) {
+    if (
+      error instanceof ClusterPluginPackageManagementClientConfigurationError
+    ) {
+      throw new ClusterCopilotClientConfigurationError();
+    }
+    if (
+      error instanceof ClusterCopilotClientConfigurationError ||
+      error instanceof InvalidClusterCopilotClientCommandError ||
+      error instanceof InvalidClusterCopilotClientResponseError ||
+      error instanceof ClusterCopilotClientRequestError ||
+      error instanceof ClusterCopilotClientRemoteError
+    ) {
+      throw error;
+    }
+    throw new ClusterCopilotClientRequestError({
+      cause: error instanceof Error ? error : undefined,
+    });
+  } finally {
+    bodyBytes?.fill(0);
+    commandBytes?.fill(0);
+    credentialBytes?.fill(0);
+    prepared?.dispose();
+  }
+}

packages/ql3-cluster-admin/src/copilot-client/contracts.ts

@@ -0,0 +1,550 @@
+import { CLUSTER_RUN_CANCELLATION_SCHEMA } from '@qinglong/runtime-core/cluster-run-cancellation';
+
+export const CLUSTER_COPILOT_CLIENT_COMMAND_SCHEMA =
+  'qinglong/cluster-copilot-client-command@v1' as const;
+export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_REQUEST_SCHEMA =
+  'qinglong/cluster-copilot-failure-diagnosis-request@v1' as const;
+export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_RESPONSE_SCHEMA =
+  'qinglong/cluster-copilot-failure-diagnosis-response@v1' as const;
+export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESPONSE_SCHEMA =
+  'qinglong/cluster-copilot-failure-diagnosis-inspection-response@v1' as const;
+export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESPONSE_SCHEMA =
+  'qinglong/cluster-copilot-failure-diagnosis-output-read-response@v1' as const;
+export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA =
+  'qinglong/cluster-copilot-failure-diagnosis-cancellation-response@v1' as const;
+
+export type ClusterCopilotClientOperation =
+  | 'diagnose'
+  | 'inspect'
+  | 'output'
+  | 'cancel';
+
+interface ClusterCopilotClientCommandBase {
+  readonly schema: typeof CLUSTER_COPILOT_CLIENT_COMMAND_SCHEMA;
+  readonly operation: ClusterCopilotClientOperation;
+  readonly projectId: string;
+  readonly sourceRunId: string;
+  readonly requestId: string;
+}
+
+export type ClusterCopilotClientCommand =
+  | (ClusterCopilotClientCommandBase &
+      Readonly<{ readonly operation: 'diagnose'; readonly traceId: string }>)
+  | (ClusterCopilotClientCommandBase &
+      Readonly<{ readonly operation: 'inspect' | 'output' }>)
+  | (ClusterCopilotClientCommandBase &
+      Readonly<{ readonly operation: 'cancel'; readonly mutationId: string }>);
+
+export interface ClusterCopilotClientPreparedRequest {
+  readonly method: 'GET' | 'POST';
+  readonly path: string;
+  readonly requestId: string;
+  readonly body: Readonly<Record<string, unknown>> | null;
+  readonly acceptedStatusCodes: readonly number[];
+}
+
+export class InvalidClusterCopilotClientCommandError extends TypeError {
+  readonly code = 'QL3_CLUSTER_COPILOT_CLIENT_COMMAND_INVALID';
+
+  constructor() {
+    super('Cluster Copilot client command is invalid');
+    this.name = 'InvalidClusterCopilotClientCommandError';
+  }
+}
+
+export class InvalidClusterCopilotClientResponseError extends Error {
+  readonly code = 'QL3_CLUSTER_COPILOT_CLIENT_RESPONSE_INVALID';
+
+  constructor() {
+    super('Cluster Copilot client response is invalid');
+    this.name = 'InvalidClusterCopilotClientResponseError';
+  }
+}
+
+const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
+const DIGEST = /^[0-9a-f]{64}$/;
+const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']);
+const STAGES = new Set(['model', 'tool', 'log', 'deadline', 'cancellation']);
+const REASONS = new Set([
+  'tool_failed',
+  'tool_timed_out',
+  'log_not_found',
+  'log_pending',
+  'log_missing',
+  'log_retired',
+  'tool_budget_exhausted',
+  'deadline_exceeded',
+  'cancellation_requested',
+]);
+const FINISH_REASONS = new Set([
+  'stop',
+  'length',
+  'content_filter',
+  'tool_call',
+  'unknown',
+]);
+const CANCELLATION_STATUSES = new Set([
+  'accepted',
+  'already_requested',
+  'already_terminal',
+]);
+const CANCELLATION_REASONS = new Set([
+  'user',
+  'policy',
+  'shutdown',
+  'reconcile',
+  'timeout',
+]);
+
+function invalidCommand(): never {
+  throw new InvalidClusterCopilotClientCommandError();
+}
+
+function invalidResponse(): never {
+  throw new InvalidClusterCopilotClientResponseError();
+}
+
+function exact(
+  value: unknown,
+  keys: readonly string[],
+): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return invalidResponse();
+  }
+  const actual = Object.keys(value).sort();
+  const expected = [...keys].sort();
+  if (
+    actual.length !== expected.length ||
+    actual.some((key, index) => key !== expected[index])
+  ) {
+    return invalidResponse();
+  }
+  return value as Record<string, unknown>;
+}
+
+function exactCommand(
+  value: unknown,
+  keys: readonly string[],
+): Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return invalidCommand();
+  }
+  const actual = Object.keys(value).sort();
+  const expected = [...keys].sort();
+  if (
+    actual.length !== expected.length ||
+    actual.some((key, index) => key !== expected[index])
+  ) {
+    return invalidCommand();
+  }
+  return value as Record<string, unknown>;
+}
+
+function nonNegativeInteger(value: unknown): value is number {
+  return Number.isSafeInteger(value) && (value as number) >= 0;
+}
+
+function identity(value: unknown): value is string {
+  return typeof value === 'string' && IDENTITY.test(value);
+}
+
+function runId(value: unknown): value is string {
+  return typeof value === 'string' && RUN_ID.test(value);
+}
+
+export function normalizeClusterCopilotClientCommand(
+  value: unknown,
+): Readonly<ClusterCopilotClientCommand> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return invalidCommand();
+  }
+  const candidate = value as Record<string, unknown>;
+  const operation = candidate.operation;
+  if (
+    operation !== 'diagnose' &&
+    operation !== 'inspect' &&
+    operation !== 'output' &&
+    operation !== 'cancel'
+  ) {
+    return invalidCommand();
+  }
+  const record = exactCommand(value, [
+    'operation',
+    'projectId',
+    'requestId',
+    'schema',
+    'sourceRunId',
+    ...(operation === 'diagnose' ? ['traceId'] : []),
+    ...(operation === 'cancel' ? ['mutationId'] : []),
+  ]);
+  if (
+    record.schema !== CLUSTER_COPILOT_CLIENT_COMMAND_SCHEMA ||
+    !identity(record.projectId) ||
+    !runId(record.sourceRunId) ||
+    !identity(record.requestId) ||
+    (operation === 'diagnose' && !identity(record.traceId)) ||
+    (operation === 'cancel' && !identity(record.mutationId))
+  ) {
+    return invalidCommand();
+  }
+  return Object.freeze({ ...record }) as Readonly<ClusterCopilotClientCommand>;
+}
+
+function targetPath(command: Readonly<ClusterCopilotClientCommand>): string {
+  const target = `/api/v3/projects/${encodeURIComponent(
+    command.projectId,
+  )}/runs/${encodeURIComponent(
+    command.sourceRunId,
+  )}/copilot/failure-diagnoses`;
+  return command.operation === 'diagnose'
+    ? target
+    : `${target}/${encodeURIComponent(command.requestId)}`;
+}
+
+export function prepareClusterCopilotClientRequest(
+  command: Readonly<ClusterCopilotClientCommand>,
+  readRequestId?: string,
+): Readonly<ClusterCopilotClientPreparedRequest> {
+  const normalized = normalizeClusterCopilotClientCommand(command);
+  if (
+    (normalized.operation === 'inspect' || normalized.operation === 'output') &&
+    !identity(readRequestId)
+  ) {
+    return invalidCommand();
+  }
+  const target = targetPath(normalized);
+  if (normalized.operation === 'diagnose') {
+    return Object.freeze({
+      method: 'POST',
+      path: target,
+      requestId: normalized.requestId,
+      body: Object.freeze({
+        schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_REQUEST_SCHEMA,
+        traceId: normalized.traceId,
+      }),
+      acceptedStatusCodes: Object.freeze([200, 201]),
+    });
+  }
+  if (normalized.operation === 'cancel') {
+    return Object.freeze({
+      method: 'POST',
+      path: `${target}/cancellation`,
+      requestId: normalized.mutationId,
+      body: Object.freeze({
+        schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
+        mutationId: normalized.mutationId,
+      }),
+      acceptedStatusCodes: Object.freeze([200, 202]),
+    });
+  }
+  return Object.freeze({
+    method: 'GET',
+    path: normalized.operation === 'output' ? `${target}/output` : target,
+    requestId: readRequestId!,
+    body: null,
+    acceptedStatusCodes: Object.freeze([200]),
+  });
+}
+
+function exactTarget(
+  value: Record<string, unknown>,
+  command: Readonly<ClusterCopilotClientCommand>,
+): boolean {
+  return (
+    value.projectId === command.projectId &&
+    value.sourceRunId === command.sourceRunId &&
+    value.requestId === command.requestId
+  );
+}
+
+function usage(
+  value: unknown,
+  settled: boolean,
+): Readonly<Record<string, unknown>> {
+  const hasCostMicros =
+    value !== null &&
+    typeof value === 'object' &&
+    !Array.isArray(value) &&
+    Object.hasOwn(value, 'costMicros');
+  const record = exact(
+    value,
+    settled
+      ? ['costMicros', 'currency', 'inputTokens', 'outputTokens', 'totalTokens']
+      : [
+          'inputTokens',
+          'outputTokens',
+          'totalTokens',
+          ...(hasCostMicros ? ['costMicros'] : []),
+        ],
+  );
+  if (
+    !nonNegativeInteger(record.inputTokens) ||
+    !nonNegativeInteger(record.outputTokens) ||
+    !nonNegativeInteger(record.totalTokens) ||
+    record.totalTokens !== record.inputTokens + record.outputTokens ||
+    (settled &&
+      !(
+        (record.currency === null && record.costMicros === null) ||
+        (record.currency === 'USD' && nonNegativeInteger(record.costMicros))
+      )) ||
+    (!settled &&
+      record.costMicros !== undefined &&
+      !nonNegativeInteger(record.costMicros))
+  ) {
+    return invalidResponse();
+  }
+  return Object.freeze({ ...record });
+}
+
+function diagnosisResponse(
+  value: unknown,
+  command: Extract<ClusterCopilotClientCommand, { readonly operation: 'diagnose' }>,
+): Readonly<Record<string, unknown>> {
+  const record = exact(value, [
+    'diagnosisRunId',
+    'outcome',
+    'outputArtifact',
+    'reason',
+    'replayed',
+    'requestId',
+    'schema',
+    'sourceRunId',
+    'stage',
+    'status',
+  ]);
+  if (
+    record.schema !== CLUSTER_COPILOT_FAILURE_DIAGNOSIS_RESPONSE_SCHEMA ||
+    record.requestId !== command.requestId ||
+    record.sourceRunId !== command.sourceRunId ||
+    !runId(record.diagnosisRunId) ||
+    (record.status !== 'created' && record.status !== 'existing') ||
+    record.replayed !== (record.status === 'existing') ||
+    typeof record.outcome !== 'string' ||
+    !OUTCOMES.has(record.outcome) ||
+    typeof record.stage !== 'string' ||
+    !STAGES.has(record.stage) ||
+    (record.stage === 'model') !== (record.reason === null) ||
+    (record.reason !== null &&
+      (typeof record.reason !== 'string' || !REASONS.has(record.reason)))
+  ) {
+    return invalidResponse();
+  }
+  let outputArtifact: Readonly<Record<string, unknown>> | null = null;
+  if (record.outputArtifact !== null) {
+    const artifact = exact(record.outputArtifact, [
+      'artifactDigest',
+      'artifactId',
+    ]);
+    if (
+      !identity(artifact.artifactId) ||
+      typeof artifact.artifactDigest !== 'string' ||
+      !DIGEST.test(artifact.artifactDigest) ||
+      record.stage !== 'model' ||
+      record.outcome !== 'succeeded'
+    ) {
+      return invalidResponse();
+    }
+    outputArtifact = Object.freeze({ ...artifact });
+  } else if (record.stage === 'model' && record.outcome === 'succeeded') {
+    return invalidResponse();
+  }
+  return Object.freeze({ ...record, outputArtifact });
+}
+
+function inspectionResponse(
+  value: unknown,
+  command: Readonly<ClusterCopilotClientCommand>,
+): Readonly<Record<string, unknown>> {
+  const record = exact(value, [
+    'admittedAtMs',
+    'diagnosisRunId',
+    'finalizedAtMs',
+    'outcome',
+    'outputAvailable',
+    'projectId',
+    'reason',
+    'requestId',
+    'schema',
+    'sourceRunId',
+    'stage',
+    'status',
+    'usage',
+  ]);
+  if (
+    record.schema !==
+      CLUSTER_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESPONSE_SCHEMA ||
+    !exactTarget(record, command) ||
+    !runId(record.diagnosisRunId) ||
+    !nonNegativeInteger(record.admittedAtMs) ||
+    typeof record.outputAvailable !== 'boolean'
+  ) {
+    return invalidResponse();
+  }
+  if (record.status === 'running') {
+    if (
+      record.outcome !== null ||
+      record.stage !== null ||
+      record.reason !== null ||
+      record.outputAvailable !== false ||
+      record.finalizedAtMs !== null ||
+      record.usage !== null
+    ) {
+      return invalidResponse();
+    }
+    return Object.freeze({ ...record });
+  }
+  if (
+    record.status !== 'terminal' ||
+    typeof record.outcome !== 'string' ||
+    !OUTCOMES.has(record.outcome) ||
+    typeof record.stage !== 'string' ||
+    !STAGES.has(record.stage) ||
+    !nonNegativeInteger(record.finalizedAtMs) ||
+    record.finalizedAtMs < record.admittedAtMs ||
+    (record.stage === 'model') !== (record.reason === null) ||
+    (record.reason !== null &&
+      (typeof record.reason !== 'string' || !REASONS.has(record.reason))) ||
+    record.outputAvailable !==
+      (record.stage === 'model' && record.outcome === 'succeeded')
+  ) {
+    return invalidResponse();
+  }
+  return Object.freeze({
+    ...record,
+    usage: record.usage === null ? null : usage(record.usage, true),
+  });
+}
+
+function outputResponse(
+  value: unknown,
+  command: Readonly<ClusterCopilotClientCommand>,
+): Readonly<Record<string, unknown>> {
+  const record = exact(value, [
+    'diagnosisRunId',
+    'projectId',
+    'reference',
+    'requestId',
+    'result',
+    'schema',
+    'sourceRunId',
+    'status',
+  ]);
+  const reference = exact(record.reference, [
+    'artifactDigest',
+    'artifactId',
+    'contentDigest',
+    'outputBytes',
+    'sealedAtMs',
+  ]);
+  const result = exact(record.result, ['finishReason', 'text', 'usage']);
+  if (
+    record.schema !==
+      CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESPONSE_SCHEMA ||
+    record.status !== 'available' ||
+    !exactTarget(record, command) ||
+    !runId(record.diagnosisRunId) ||
+    !identity(reference.artifactId) ||
+    typeof reference.artifactDigest !== 'string' ||
+    !DIGEST.test(reference.artifactDigest) ||
+    typeof reference.contentDigest !== 'string' ||
+    !DIGEST.test(reference.contentDigest) ||
+    !nonNegativeInteger(reference.outputBytes) ||
+    reference.outputBytes > 1024 * 1024 ||
+    !nonNegativeInteger(reference.sealedAtMs) ||
+    typeof result.text !== 'string' ||
+    Buffer.byteLength(result.text, 'utf8') !== reference.outputBytes ||
+    typeof result.finishReason !== 'string' ||
+    !FINISH_REASONS.has(result.finishReason)
+  ) {
+    return invalidResponse();
+  }
+  return Object.freeze({
+    ...record,
+    reference: Object.freeze({ ...reference }),
+    result: Object.freeze({
+      ...result,
+      usage: usage(result.usage, false),
+    }),
+  });
+}
+
+function cancellationResponse(
+  value: unknown,
+  command: Readonly<ClusterCopilotClientCommand>,
+): Readonly<Record<string, unknown>> {
+  const record = exact(value, [
+    'cancelReason',
+    'cancelRequestedAtMs',
+    'convergence',
+    'diagnosisRunId',
+    'eventSequence',
+    'outcome',
+    'projectId',
+    'requestId',
+    'runStatus',
+    'runVersion',
+    'schema',
+    'sourceRunId',
+    'status',
+  ]);
+  if (
+    record.schema !==
+      CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA ||
+    !exactTarget(record, command) ||
+    !runId(record.diagnosisRunId) ||
+    typeof record.status !== 'string' ||
+    !CANCELLATION_STATUSES.has(record.status) ||
+    !nonNegativeInteger(record.runVersion) ||
+    !nonNegativeInteger(record.eventSequence) ||
+    record.runVersion !== record.eventSequence ||
+    !(
+      (record.cancelRequestedAtMs === null && record.cancelReason === null) ||
+      (nonNegativeInteger(record.cancelRequestedAtMs) &&
+        typeof record.cancelReason === 'string' &&
+        CANCELLATION_REASONS.has(record.cancelReason))
+    )
+  ) {
+    return invalidResponse();
+  }
+  if (
+    record.convergence === 'model_in_flight' &&
+    (record.runStatus !== 'running' ||
+      record.outcome !== null ||
+      record.cancelRequestedAtMs === null)
+  ) {
+    return invalidResponse();
+  }
+  if (
+    record.convergence === 'terminal' &&
+    (typeof record.runStatus !== 'string' ||
+      !OUTCOMES.has(record.runStatus) ||
+      record.outcome !== record.runStatus)
+  ) {
+    return invalidResponse();
+  }
+  if (
+    record.convergence !== 'model_in_flight' &&
+    record.convergence !== 'terminal'
+  ) {
+    return invalidResponse();
+  }
+  return Object.freeze({ ...record });
+}
+
+export function validateClusterCopilotClientResponse(
+  value: unknown,
+  command: Readonly<ClusterCopilotClientCommand>,
+): Readonly<Record<string, unknown>> {
+  const normalized = normalizeClusterCopilotClientCommand(command);
+  if (normalized.operation === 'diagnose') {
+    return diagnosisResponse(value, normalized);
+  }
+  if (normalized.operation === 'inspect') {
+    return inspectionResponse(value, normalized);
+  }
+  if (normalized.operation === 'output') {
+    return outputResponse(value, normalized);
+  }
+  return cancellationResponse(value, normalized);
+}

packages/ql3-cluster-admin/src/product-cli/productCommand.ts

@@ -34,6 +34,12 @@ const SEMVER_PATTERN =
 
 export const QINGLONG3_CLUSTER_PRODUCT_COMMANDS: readonly QingLong3ClusterProductCommandDefinition[] =
   Object.freeze([
+    Object.freeze({
+      name: 'copilot',
+      binary: 'ql3-copilot-client',
+      target: 'copilot-client/cli.js',
+      description: 'diagnose, inspect, read or cancel Runs through the API',
+    }),
     Object.freeze({
       name: 'package',
       binary: 'ql3-plugin-package-client',

packages/ql3-cluster-admin/src/product-cli/productContext.ts

@@ -10,6 +10,10 @@ import {
 import { isAbsolute } from 'node:path';
 import { TextDecoder } from 'node:util';
 
+import {
+  probeClusterCopilotClientReadiness,
+  validateClusterCopilotClientConfiguration,
+} from '../copilot-client/client';
 import {
   validateClusterAuthenticatedManagementClientConfiguration,
 } from '../management-support/pluginPackageManagementClient';
@@ -24,6 +28,7 @@ const MAXIMUM_CONTEXT_BYTES = 64 * 1024;
 const MAXIMUM_PATH_BYTES = 4_096;
 const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/u;
 const CONTEXT_COMMANDS = Object.freeze([
+  'copilot',
   'package',
   'package-kubernetes',
   'worker-credential',
@@ -80,7 +85,10 @@ export interface QingLong3ClusterProductContextProbe {
 }
 
 const CONTEXT_COMMAND_CLIENT_KINDS: Readonly<
-  Record<ContextCommandName, ClusterAuthenticatedManagementClientKind>
+  Record<
+    Exclude<ContextCommandName, 'copilot'>,
+    ClusterAuthenticatedManagementClientKind
+  >
 > = Object.freeze({
   package: 'package',
   'package-kubernetes': 'package',
@@ -317,12 +325,23 @@ export async function validateQingLong3ClusterProductContext(
     for (const name of CONTEXT_COMMANDS) {
       const command = context.commands[name];
       if (command === undefined) continue;
-      const clientKind = CONTEXT_COMMAND_CLIENT_KINDS[name];
-      const https = validateClusterAuthenticatedManagementClientConfiguration(
-        command.configFile,
-        clientKind,
-      );
-      if (name === 'package-kubernetes') {
+      if (name === 'copilot') {
+        const https = validateClusterCopilotClientConfiguration(
+          command.configFile,
+        );
+        commands.push(
+          Object.freeze({
+            name,
+            transport: https.transport,
+            clientCertificate: https.clientCertificate,
+          }),
+        );
+      } else if (name === 'package-kubernetes') {
+        const https =
+          validateClusterAuthenticatedManagementClientConfiguration(
+            command.configFile,
+            CONTEXT_COMMAND_CLIENT_KINDS[name],
+          );
         const kubernetes =
           await validateClusterPluginPackageManagementKubernetesConfiguration(
             command.kubernetesFile!,
@@ -336,6 +355,11 @@ export async function validateQingLong3ClusterProductContext(
           }),
         );
       } else {
+        const https =
+          validateClusterAuthenticatedManagementClientConfiguration(
+            command.configFile,
+            CONTEXT_COMMAND_CLIENT_KINDS[name],
+          );
         commands.push(
           Object.freeze({
             name,
@@ -378,6 +402,8 @@ export async function probeQingLong3ClusterProductContext(
               command.configFile,
               command.kubernetesFile!,
             )
+          : name === 'copilot'
+            ? await probeClusterCopilotClientReadiness(command.configFile)
           : await probeClusterAuthenticatedManagementClientReadiness(
               command.configFile,
               CONTEXT_COMMAND_CLIENT_KINDS[name],