feat(ql3): add fenced copilot diagnosis cancellation

2026-09-21 18:08:20 +08:00 · 2026-08-16 01:34:55 +08:00
parent f77b19e615
commit e5a3d83a8d
17 changed files with 1685 additions and 8 deletions
@@ -2,6 +2,7 @@ import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile';
 import type { DurableModelInvocationCoordinator } from '@qinglong/ai/durable-model-invocation';
 import { CopilotFailureDiagnosisModelCompletionCoordinator } from '@qinglong/ai/failure-diagnosis-model-execution';
 import type { CopilotFailureDiagnosisApplicationService } from '@qinglong/ai/failure-diagnosis-application';
+import type { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation';
 import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential';
 import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage';
 import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority';
@@ -35,6 +36,10 @@ import {
  createProductionClusterCopilotFailureDiagnosisReadService,
  type CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions,
 } from './copilot/failureDiagnosisReadComposition';
+import {
+  createProductionClusterCopilotFailureDiagnosisCancellation,
+  type CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
+} from './copilot/failureDiagnosisCancellationComposition';

 export interface EnabledProductionClusterAiConfig {
  readonly enabled: true;
@@ -63,6 +68,9 @@ export interface ProductionClusterAiControlApplicationOptions {
  ) => ReturnType<
    typeof createProductionClusterCopilotFailureDiagnosisReadService
  >;
+  readonly createCopilotCancellation?: (
+    options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
+  ) => Readonly<CopilotFailureDiagnosisCancellationService>;
  readonly openAiDatabase?: ReturnType<typeof createPostgresDatabaseOpener>;
 }

@@ -263,11 +271,15 @@ export async function startProductionClusterAiControlApplication(
  const createCopilotRead =
    options.createCopilotRead ??
    createProductionClusterCopilotFailureDiagnosisReadService;
+  const createCopilotCancellation =
+    options.createCopilotCancellation ??
+    createProductionClusterCopilotFailureDiagnosisCancellation;
  if (
    typeof startControl !== 'function' ||
    typeof bootstrapPrompt !== 'function' ||
    typeof createCopilot !== 'function' ||
    typeof createCopilotRead !== 'function' ||
+    typeof createCopilotCancellation !== 'function' ||
    (options.openAiDatabase !== undefined &&
      typeof options.openAiDatabase !== 'function')
  ) {
@@ -325,6 +337,9 @@ export async function startProductionClusterAiControlApplication(
        typeof createProductionClusterCopilotFailureDiagnosisReadService
      >
    | undefined;
+  let copilotCancellationApplication:
+    | Readonly<CopilotFailureDiagnosisCancellationService>
+    | undefined;
  let copilotSuccessfulCompletion:
    | CopilotFailureDiagnosisModelCompletionCoordinator
    | undefined;
@@ -455,6 +470,9 @@ export async function startProductionClusterAiControlApplication(
        pool: aiDatabase.pool,
        prepared: preparedCopilot,
      });
+      copilotCancellationApplication = createCopilotCancellation({
+        pool: aiDatabase.pool,
+      });
    }
    controlApplication = await startControl({
      ...options.control,
@@ -482,12 +500,14 @@ export async function startProductionClusterAiControlApplication(
            },
          }),
      ...(copilotApplication === undefined ||
-      copilotReadApplication === undefined
+      copilotReadApplication === undefined ||
+      copilotCancellationApplication === undefined
        ? {}
        : {
            copilotFailureDiagnosis: {
              capability: copilotApplication,
              readCapability: copilotReadApplication,
+              cancellationCapability: copilotCancellationApplication,
            },
          }),
    });
@@ -0,0 +1,47 @@
+import { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation';
+import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage';
+import {
+  PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository,
+  terminalizeCopilotFailureDiagnosisBeforeModel,
+} from '@qinglong/ai/failure-diagnosis-pre-model-terminalization';
+import {
+  PostgresClusterRunCancellationRepository,
+  type QingLongPostgresPool,
+} from '@qinglong/cluster-postgres/runtime';
+
+export interface CreateProductionClusterCopilotFailureDiagnosisCancellationOptions {
+  readonly pool: QingLongPostgresPool;
+}
+
+/**
+ * Reuses the AI Pool and existing Run/admission ledgers. The cancellation
+ * capability owns no connection, timer, listener or background lifecycle.
+ */
+export function createProductionClusterCopilotFailureDiagnosisCancellation(
+  options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
+): Readonly<CopilotFailureDiagnosisCancellationService> {
+  if (
+    !options ||
+    typeof options !== 'object' ||
+    Array.isArray(options) ||
+    typeof options.pool?.query !== 'function' ||
+    typeof options.pool?.connect !== 'function'
+  ) {
+    throw new TypeError(
+      'Production Cluster Copilot failure diagnosis cancellation dependencies are invalid',
+    );
+  }
+  const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository(
+    options.pool,
+  );
+  const terminalizations =
+    new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository(
+      options.pool,
+    );
+  return new CopilotFailureDiagnosisCancellationService({
+    admissions,
+    cancellations: new PostgresClusterRunCancellationRepository(options.pool),
+    terminalizations: Object.freeze({ repository: terminalizations }),
+    terminalizeBeforeModel: terminalizeCopilotFailureDiagnosisBeforeModel,
+  });
+}
@@ -71,6 +71,10 @@ import {
  type ClusterCopilotFailureDiagnosisInspectionCapability,
  type ClusterCopilotFailureDiagnosisOutputReadCapability,
 } from '../copilot/failure-diagnosis/failureDiagnosisReadRoutes';
+import {
+  createClusterControlCopilotFailureDiagnosisCancellationRoute,
+  type ClusterCopilotFailureDiagnosisCancellationCapability,
+} from '../copilot/failure-diagnosis/failureDiagnosisCancellationRoute';

 export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
  'task.get',
@@ -100,6 +104,7 @@ export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS =
    'copilot.failure_diagnosis.execute',
    'copilot.failure_diagnosis.read',
    'copilot.failure_diagnosis.output.read',
+    'copilot.failure_diagnosis.cancel',
  ] as const);

 export interface ClusterCopilotFailureDiagnosisReadCapability
@@ -129,6 +134,7 @@ export interface ProductionClusterControlAssemblyOptions {
  readonly copilotFailureDiagnosis?: Readonly<{
    readonly capability: ClusterCopilotFailureDiagnosisCapability;
    readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
+    readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability;
  }>;
  readonly workerIngress?: Readonly<{
    readonly config: EnabledClusterWorkerIngressConfig;
@@ -178,6 +184,7 @@ export interface ProductionClusterControlApplicationOptions
  readonly copilotFailureDiagnosis?: Readonly<{
    readonly capability: ClusterCopilotFailureDiagnosisCapability;
    readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
+    readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability;
  }>;
  readonly workerIngress?: ProductionClusterWorkerIngressOptions;
 }
@@ -303,6 +310,14 @@ export function createProductionClusterControlApplicationStack(
            options.copilotFailureDiagnosis.readCapability,
          ),
        ]),
+    ...(options.copilotFailureDiagnosis?.cancellationCapability === undefined
+      ? []
+      : [
+          createClusterControlCopilotFailureDiagnosisCancellationRoute(
+            options.copilotFailureDiagnosis.cancellationCapability,
+            createEventId,
+          ),
+        ]),
  ];
  const routes = createClusterControlRouteRegistry(routeDefinitions);
  const expectedRouteCount =
@@ -313,7 +328,10 @@ export function createProductionClusterControlApplicationStack(
    (options.promptOutputRead === undefined ? 0 : 1) +
    (options.promptExecutionOutputRead === undefined ? 0 : 1) +
    (options.copilotFailureDiagnosis === undefined ? 0 : 1) +
-    (options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2);
+    (options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2) +
+    (options.copilotFailureDiagnosis?.cancellationCapability === undefined
+      ? 0
+      : 1);
  if (routes.size !== expectedRouteCount) {
    throw new Error('Production cluster-control route allowlist is incomplete');
  }
@@ -0,0 +1,261 @@
+// Cluster Copilot resolves an external request key before cancelling its Run.
+import {
+  CopilotFailureDiagnosisCancellationNotFoundError,
+  CopilotFailureDiagnosisCancellationUnavailableError,
+  InvalidCopilotFailureDiagnosisCancellationError,
+  type CopilotFailureDiagnosisCancellationCommand,
+} from '@qinglong/ai/failure-diagnosis-cancellation';
+import {
+  CLUSTER_RUN_CANCELLATION_SCHEMA,
+  ClusterRunCancellationFenceRejectedError,
+  ClusterRunCancellationNotFoundError,
+  ClusterRunCancellationUnavailableError,
+  InvalidClusterRunCancellationError,
+  parseClusterRunCancellationRequestBody,
+} from '@qinglong/runtime-core/cluster-run-cancellation';
+
+import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
+import type {
+  ClusterControlAuthorizedOperationRequest,
+  ClusterControlRouteDefinition,
+  ClusterControlRouteParameters,
+} from '../../transport/routeRegistry';
+
+export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA =
+  'qinglong/cluster-copilot-failure-diagnosis-cancellation-response@v1' as const;
+
+export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE =
+  Object.freeze({
+    method: 'POST' as const,
+    path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/cancellation',
+    operationId: 'copilot.failure_diagnosis.cancel',
+    permission: 'run.stop',
+    projectParameter: 'projectId',
+  });
+
+export interface ClusterCopilotFailureDiagnosisCancellationCapability {
+  cancel(
+    command: Readonly<CopilotFailureDiagnosisCancellationCommand>,
+  ): Promise<unknown>;
+}
+
+export type ClusterCopilotFailureDiagnosisCancellationEventIdFactory =
+  () => string;
+
+const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
+const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']);
+const STATUSES = new Set(['accepted', 'already_requested', 'already_terminal']);
+const CANCEL_REASONS = new Set([
+  'user',
+  'policy',
+  'shutdown',
+  'reconcile',
+  'timeout',
+]);
+
+function response(
+  statusCode: number,
+  body: Readonly<Record<string, unknown>>,
+): ClusterControlAdmissionResponse {
+  return Object.freeze({ statusCode, body: Object.freeze(body) });
+}
+
+function exactRecord(
+  value: unknown,
+  expected: readonly string[],
+): Record<string, unknown> | null {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
+  const record = value as Record<string, unknown>;
+  const actual = Object.keys(record).sort();
+  const canonical = [...expected].sort();
+  return actual.length === canonical.length &&
+    actual.every((key, index) => key === canonical[index])
+    ? record
+    : null;
+}
+
+function safeInteger(value: unknown): value is number {
+  return Number.isSafeInteger(value) && (value as number) >= 0;
+}
+
+function projectResult(
+  value: unknown,
+  target: Readonly<{
+    projectId: string;
+    sourceRunId: string;
+    requestId: string;
+  }>,
+): Readonly<Record<string, unknown>> | null {
+  const candidate = exactRecord(value, [
+    'cancelReason',
+    'cancelRequestedAtMs',
+    'convergence',
+    'diagnosisRunId',
+    'eventSequence',
+    'outcome',
+    'projectId',
+    'requestId',
+    'runStatus',
+    'runVersion',
+    'schema',
+    'sourceRunId',
+    'status',
+  ]);
+  if (
+    !candidate ||
+    candidate.schema !==
+      'qinglong/copilot-failure-diagnosis-cancellation-result@v1' ||
+    candidate.projectId !== target.projectId ||
+    candidate.sourceRunId !== target.sourceRunId ||
+    candidate.requestId !== target.requestId ||
+    typeof candidate.diagnosisRunId !== 'string' ||
+    !RUN_ID.test(candidate.diagnosisRunId) ||
+    typeof candidate.status !== 'string' ||
+    !STATUSES.has(candidate.status) ||
+    !safeInteger(candidate.runVersion) ||
+    !safeInteger(candidate.eventSequence) ||
+    candidate.runVersion !== candidate.eventSequence ||
+    !(
+      (candidate.cancelRequestedAtMs === null &&
+        candidate.cancelReason === null) ||
+      (safeInteger(candidate.cancelRequestedAtMs) &&
+        typeof candidate.cancelReason === 'string' &&
+        CANCEL_REASONS.has(candidate.cancelReason))
+    )
+  ) {
+    return null;
+  }
+  if (candidate.convergence === 'model_in_flight') {
+    if (
+      candidate.runStatus !== 'running' ||
+      candidate.outcome !== null ||
+      candidate.cancelRequestedAtMs === null
+    ) {
+      return null;
+    }
+  } else if (candidate.convergence === 'terminal') {
+    if (
+      typeof candidate.runStatus !== 'string' ||
+      !OUTCOMES.has(candidate.runStatus) ||
+      candidate.outcome !== candidate.runStatus
+    ) {
+      return null;
+    }
+  } else {
+    return null;
+  }
+  return Object.freeze({
+    schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA,
+    status: candidate.status,
+    convergence: candidate.convergence,
+    projectId: target.projectId,
+    sourceRunId: target.sourceRunId,
+    requestId: target.requestId,
+    diagnosisRunId: candidate.diagnosisRunId,
+    runStatus: candidate.runStatus,
+    outcome: candidate.outcome,
+    runVersion: candidate.runVersion,
+    eventSequence: candidate.eventSequence,
+    cancelRequestedAtMs: candidate.cancelRequestedAtMs,
+    cancelReason: candidate.cancelReason,
+  });
+}
+
+export function createClusterControlCopilotFailureDiagnosisCancellationRoute(
+  capability: ClusterCopilotFailureDiagnosisCancellationCapability,
+  createEventId: ClusterCopilotFailureDiagnosisCancellationEventIdFactory,
+): Readonly<ClusterControlRouteDefinition> {
+  if (
+    !capability ||
+    typeof capability.cancel !== 'function' ||
+    typeof createEventId !== 'function'
+  ) {
+    throw new TypeError(
+      'Cluster-control Copilot failure diagnosis cancellation route is invalid',
+    );
+  }
+  return Object.freeze({
+    ...CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE,
+    async handle(
+      authorized: ClusterControlAuthorizedOperationRequest,
+      parameters: ClusterControlRouteParameters,
+    ) {
+      let body;
+      try {
+        body = parseClusterRunCancellationRequestBody(authorized.request.body);
+      } catch (error) {
+        return error instanceof InvalidClusterRunCancellationError
+          ? response(400, {
+              code: 'invalid_copilot_failure_diagnosis_cancellation_request',
+              schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
+            })
+          : response(503, {
+              code: 'copilot_failure_diagnosis_cancellation_unavailable',
+            });
+      }
+      const projectId = authorized.projectId;
+      const sourceRunId = parameters.runId;
+      const requestId = parameters.requestId;
+      if (
+        projectId === null ||
+        typeof sourceRunId !== 'string' ||
+        !RUN_ID.test(sourceRunId) ||
+        typeof requestId !== 'string' ||
+        !IDENTITY.test(requestId) ||
+        !authorized.policyFence ||
+        authorized.policyFence.bindingVersion === null
+      ) {
+        return response(503, {
+          code: 'copilot_failure_diagnosis_cancellation_unavailable',
+        });
+      }
+      const target = Object.freeze({ projectId, sourceRunId, requestId });
+      try {
+        const result = await capability.cancel({
+          ...target,
+          mutationId: body.mutationId,
+          eventId: createEventId(),
+          subject: authorized.principal.subject,
+          policyFence: authorized.policyFence,
+        });
+        const view = projectResult(result, target);
+        if (!view) {
+          return response(503, {
+            code: 'copilot_failure_diagnosis_cancellation_unavailable',
+          });
+        }
+        return response(view.status === 'accepted' ? 202 : 200, view);
+      } catch (error) {
+        if (
+          error instanceof CopilotFailureDiagnosisCancellationNotFoundError ||
+          error instanceof ClusterRunCancellationNotFoundError
+        ) {
+          return response(404, {
+            code: 'copilot_failure_diagnosis_not_found',
+          });
+        }
+        if (error instanceof ClusterRunCancellationFenceRejectedError) {
+          return response(409, {
+            code: 'copilot_failure_diagnosis_cancellation_fence_rejected',
+            reason: error.reason,
+          });
+        }
+        if (
+          error instanceof InvalidCopilotFailureDiagnosisCancellationError ||
+          error instanceof
+            CopilotFailureDiagnosisCancellationUnavailableError ||
+          error instanceof InvalidClusterRunCancellationError ||
+          error instanceof ClusterRunCancellationUnavailableError
+        ) {
+          return response(503, {
+            code: 'copilot_failure_diagnosis_cancellation_unavailable',
+          });
+        }
+        return response(503, {
+          code: 'copilot_failure_diagnosis_cancellation_unavailable',
+        });
+      }
+    },
+  });
+}