feat(ql3): add fenced copilot diagnosis cancellation

This commit is contained in:
whyour
2026-08-16 01:34:55 +08:00
parent f77b19e615
commit e5a3d83a8d
17 changed files with 1685 additions and 8 deletions
@@ -2,6 +2,7 @@ import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile';
import type { DurableModelInvocationCoordinator } from '@qinglong/ai/durable-model-invocation';
import { CopilotFailureDiagnosisModelCompletionCoordinator } from '@qinglong/ai/failure-diagnosis-model-execution';
import type { CopilotFailureDiagnosisApplicationService } from '@qinglong/ai/failure-diagnosis-application';
import type { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation';
import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential';
import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage';
import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority';
@@ -35,6 +36,10 @@ import {
createProductionClusterCopilotFailureDiagnosisReadService,
type CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions,
} from './copilot/failureDiagnosisReadComposition';
import {
createProductionClusterCopilotFailureDiagnosisCancellation,
type CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
} from './copilot/failureDiagnosisCancellationComposition';
export interface EnabledProductionClusterAiConfig {
readonly enabled: true;
@@ -63,6 +68,9 @@ export interface ProductionClusterAiControlApplicationOptions {
) => ReturnType<
typeof createProductionClusterCopilotFailureDiagnosisReadService
>;
readonly createCopilotCancellation?: (
options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
) => Readonly<CopilotFailureDiagnosisCancellationService>;
readonly openAiDatabase?: ReturnType<typeof createPostgresDatabaseOpener>;
}
@@ -263,11 +271,15 @@ export async function startProductionClusterAiControlApplication(
const createCopilotRead =
options.createCopilotRead ??
createProductionClusterCopilotFailureDiagnosisReadService;
const createCopilotCancellation =
options.createCopilotCancellation ??
createProductionClusterCopilotFailureDiagnosisCancellation;
if (
typeof startControl !== 'function' ||
typeof bootstrapPrompt !== 'function' ||
typeof createCopilot !== 'function' ||
typeof createCopilotRead !== 'function' ||
typeof createCopilotCancellation !== 'function' ||
(options.openAiDatabase !== undefined &&
typeof options.openAiDatabase !== 'function')
) {
@@ -325,6 +337,9 @@ export async function startProductionClusterAiControlApplication(
typeof createProductionClusterCopilotFailureDiagnosisReadService
>
| undefined;
let copilotCancellationApplication:
| Readonly<CopilotFailureDiagnosisCancellationService>
| undefined;
let copilotSuccessfulCompletion:
| CopilotFailureDiagnosisModelCompletionCoordinator
| undefined;
@@ -455,6 +470,9 @@ export async function startProductionClusterAiControlApplication(
pool: aiDatabase.pool,
prepared: preparedCopilot,
});
copilotCancellationApplication = createCopilotCancellation({
pool: aiDatabase.pool,
});
}
controlApplication = await startControl({
...options.control,
@@ -482,12 +500,14 @@ export async function startProductionClusterAiControlApplication(
},
}),
...(copilotApplication === undefined ||
copilotReadApplication === undefined
copilotReadApplication === undefined ||
copilotCancellationApplication === undefined
? {}
: {
copilotFailureDiagnosis: {
capability: copilotApplication,
readCapability: copilotReadApplication,
cancellationCapability: copilotCancellationApplication,
},
}),
});
@@ -0,0 +1,47 @@
import { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation';
import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage';
import {
PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository,
terminalizeCopilotFailureDiagnosisBeforeModel,
} from '@qinglong/ai/failure-diagnosis-pre-model-terminalization';
import {
PostgresClusterRunCancellationRepository,
type QingLongPostgresPool,
} from '@qinglong/cluster-postgres/runtime';
export interface CreateProductionClusterCopilotFailureDiagnosisCancellationOptions {
readonly pool: QingLongPostgresPool;
}
/**
* Reuses the AI Pool and existing Run/admission ledgers. The cancellation
* capability owns no connection, timer, listener or background lifecycle.
*/
export function createProductionClusterCopilotFailureDiagnosisCancellation(
options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
): Readonly<CopilotFailureDiagnosisCancellationService> {
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
typeof options.pool?.query !== 'function' ||
typeof options.pool?.connect !== 'function'
) {
throw new TypeError(
'Production Cluster Copilot failure diagnosis cancellation dependencies are invalid',
);
}
const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository(
options.pool,
);
const terminalizations =
new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository(
options.pool,
);
return new CopilotFailureDiagnosisCancellationService({
admissions,
cancellations: new PostgresClusterRunCancellationRepository(options.pool),
terminalizations: Object.freeze({ repository: terminalizations }),
terminalizeBeforeModel: terminalizeCopilotFailureDiagnosisBeforeModel,
});
}
@@ -71,6 +71,10 @@ import {
type ClusterCopilotFailureDiagnosisInspectionCapability,
type ClusterCopilotFailureDiagnosisOutputReadCapability,
} from '../copilot/failure-diagnosis/failureDiagnosisReadRoutes';
import {
createClusterControlCopilotFailureDiagnosisCancellationRoute,
type ClusterCopilotFailureDiagnosisCancellationCapability,
} from '../copilot/failure-diagnosis/failureDiagnosisCancellationRoute';
export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
'task.get',
@@ -100,6 +104,7 @@ export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS =
'copilot.failure_diagnosis.execute',
'copilot.failure_diagnosis.read',
'copilot.failure_diagnosis.output.read',
'copilot.failure_diagnosis.cancel',
] as const);
export interface ClusterCopilotFailureDiagnosisReadCapability
@@ -129,6 +134,7 @@ export interface ProductionClusterControlAssemblyOptions {
readonly copilotFailureDiagnosis?: Readonly<{
readonly capability: ClusterCopilotFailureDiagnosisCapability;
readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability;
}>;
readonly workerIngress?: Readonly<{
readonly config: EnabledClusterWorkerIngressConfig;
@@ -178,6 +184,7 @@ export interface ProductionClusterControlApplicationOptions
readonly copilotFailureDiagnosis?: Readonly<{
readonly capability: ClusterCopilotFailureDiagnosisCapability;
readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability;
}>;
readonly workerIngress?: ProductionClusterWorkerIngressOptions;
}
@@ -303,6 +310,14 @@ export function createProductionClusterControlApplicationStack(
options.copilotFailureDiagnosis.readCapability,
),
]),
...(options.copilotFailureDiagnosis?.cancellationCapability === undefined
? []
: [
createClusterControlCopilotFailureDiagnosisCancellationRoute(
options.copilotFailureDiagnosis.cancellationCapability,
createEventId,
),
]),
];
const routes = createClusterControlRouteRegistry(routeDefinitions);
const expectedRouteCount =
@@ -313,7 +328,10 @@ export function createProductionClusterControlApplicationStack(
(options.promptOutputRead === undefined ? 0 : 1) +
(options.promptExecutionOutputRead === undefined ? 0 : 1) +
(options.copilotFailureDiagnosis === undefined ? 0 : 1) +
(options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2);
(options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2) +
(options.copilotFailureDiagnosis?.cancellationCapability === undefined
? 0
: 1);
if (routes.size !== expectedRouteCount) {
throw new Error('Production cluster-control route allowlist is incomplete');
}