open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-23 03:18:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add fenced copilot diagnosis cancellation

Browse Source
This commit is contained in:
whyour
2026-08-16 01:34:55 +08:00
parent f77b19e615
commit e5a3d83a8d
17 changed files with 1685 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-cluster-control/package.json
+10
View File
@@ -40,6 +40,11 @@
"require": "./dist/application-runtime/copilot/failureDiagnosisComposition.js",
"default": "./dist/application-runtime/copilot/failureDiagnosisComposition.js"
},
"./copilot-cancellation-production": {
"types": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.d.ts",
"require": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.js",
"default": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.js"
},
"./failure-diagnosis-output-keyring": {
"types": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.d.ts",
"require": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js",
@@ -55,6 +60,11 @@
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js",
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js"
},
"./copilot-cancellation-route": {
"types": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.d.ts",
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.js",
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.js"
},
"./http": {
"types": "./dist/transport/httpSurface.d.ts",
"require": "./dist/transport/httpSurface.js",
packages/ql3-cluster-control/src/application-runtime/aiProductionApplication.ts
+21 -1
View File
@@ -2,6 +2,7 @@ import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile';
import type { DurableModelInvocationCoordinator } from '@qinglong/ai/durable-model-invocation';
import { CopilotFailureDiagnosisModelCompletionCoordinator } from '@qinglong/ai/failure-diagnosis-model-execution';
import type { CopilotFailureDiagnosisApplicationService } from '@qinglong/ai/failure-diagnosis-application';
import type { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation';
import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential';
import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage';
import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority';
@@ -35,6 +36,10 @@ import {
createProductionClusterCopilotFailureDiagnosisReadService,
type CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions,
} from './copilot/failureDiagnosisReadComposition';
import {
createProductionClusterCopilotFailureDiagnosisCancellation,
type CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
} from './copilot/failureDiagnosisCancellationComposition';
export interface EnabledProductionClusterAiConfig {
readonly enabled: true;
@@ -63,6 +68,9 @@ export interface ProductionClusterAiControlApplicationOptions {
) => ReturnType<
typeof createProductionClusterCopilotFailureDiagnosisReadService
>;
readonly createCopilotCancellation?: (
options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
) => Readonly<CopilotFailureDiagnosisCancellationService>;
readonly openAiDatabase?: ReturnType<typeof createPostgresDatabaseOpener>;
}
@@ -263,11 +271,15 @@ export async function startProductionClusterAiControlApplication(
const createCopilotRead =
options.createCopilotRead ??
createProductionClusterCopilotFailureDiagnosisReadService;
const createCopilotCancellation =
options.createCopilotCancellation ??
createProductionClusterCopilotFailureDiagnosisCancellation;
if (
typeof startControl !== 'function' ||
typeof bootstrapPrompt !== 'function' ||
typeof createCopilot !== 'function' ||
typeof createCopilotRead !== 'function' ||
typeof createCopilotCancellation !== 'function' ||
(options.openAiDatabase !== undefined &&
typeof options.openAiDatabase !== 'function')
) {
@@ -325,6 +337,9 @@ export async function startProductionClusterAiControlApplication(
typeof createProductionClusterCopilotFailureDiagnosisReadService
>
| undefined;
let copilotCancellationApplication:
| Readonly<CopilotFailureDiagnosisCancellationService>
| undefined;
let copilotSuccessfulCompletion:
| CopilotFailureDiagnosisModelCompletionCoordinator
| undefined;
@@ -455,6 +470,9 @@ export async function startProductionClusterAiControlApplication(
pool: aiDatabase.pool,
prepared: preparedCopilot,
});
copilotCancellationApplication = createCopilotCancellation({
pool: aiDatabase.pool,
});
}
controlApplication = await startControl({
...options.control,
@@ -482,12 +500,14 @@ export async function startProductionClusterAiControlApplication(
},
}),
...(copilotApplication === undefined ||
copilotReadApplication === undefined
copilotReadApplication === undefined ||
copilotCancellationApplication === undefined
? {}
: {
copilotFailureDiagnosis: {
capability: copilotApplication,
readCapability: copilotReadApplication,
cancellationCapability: copilotCancellationApplication,
},
}),
});
packages/ql3-cluster-control/src/application-runtime/copilot/failureDiagnosisCancellationComposition.ts
+47
View File
@@ -0,0 +1,47 @@
import { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation';
import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage';
import {
PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository,
terminalizeCopilotFailureDiagnosisBeforeModel,
} from '@qinglong/ai/failure-diagnosis-pre-model-terminalization';
import {
PostgresClusterRunCancellationRepository,
type QingLongPostgresPool,
} from '@qinglong/cluster-postgres/runtime';
export interface CreateProductionClusterCopilotFailureDiagnosisCancellationOptions {
readonly pool: QingLongPostgresPool;
}
/**
* Reuses the AI Pool and existing Run/admission ledgers. The cancellation
* capability owns no connection, timer, listener or background lifecycle.
*/
export function createProductionClusterCopilotFailureDiagnosisCancellation(
options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
): Readonly<CopilotFailureDiagnosisCancellationService> {
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
typeof options.pool?.query !== 'function' ||
typeof options.pool?.connect !== 'function'
) {
throw new TypeError(
'Production Cluster Copilot failure diagnosis cancellation dependencies are invalid',
);
}
const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository(
options.pool,
);
const terminalizations =
new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository(
options.pool,
);
return new CopilotFailureDiagnosisCancellationService({
admissions,
cancellations: new PostgresClusterRunCancellationRepository(options.pool),
terminalizations: Object.freeze({ repository: terminalizations }),
terminalizeBeforeModel: terminalizeCopilotFailureDiagnosisBeforeModel,
});
}
packages/ql3-cluster-control/src/application-runtime/productionApplication.ts
+19 -1
View File
@@ -71,6 +71,10 @@ import {
type ClusterCopilotFailureDiagnosisInspectionCapability,
type ClusterCopilotFailureDiagnosisOutputReadCapability,
} from '../copilot/failure-diagnosis/failureDiagnosisReadRoutes';
import {
createClusterControlCopilotFailureDiagnosisCancellationRoute,
type ClusterCopilotFailureDiagnosisCancellationCapability,
} from '../copilot/failure-diagnosis/failureDiagnosisCancellationRoute';
export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
'task.get',
@@ -100,6 +104,7 @@ export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS =
'copilot.failure_diagnosis.execute',
'copilot.failure_diagnosis.read',
'copilot.failure_diagnosis.output.read',
'copilot.failure_diagnosis.cancel',
] as const);
export interface ClusterCopilotFailureDiagnosisReadCapability
@@ -129,6 +134,7 @@ export interface ProductionClusterControlAssemblyOptions {
readonly copilotFailureDiagnosis?: Readonly<{
readonly capability: ClusterCopilotFailureDiagnosisCapability;
readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability;
}>;
readonly workerIngress?: Readonly<{
readonly config: EnabledClusterWorkerIngressConfig;
@@ -178,6 +184,7 @@ export interface ProductionClusterControlApplicationOptions
readonly copilotFailureDiagnosis?: Readonly<{
readonly capability: ClusterCopilotFailureDiagnosisCapability;
readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability;
}>;
readonly workerIngress?: ProductionClusterWorkerIngressOptions;
}
@@ -303,6 +310,14 @@ export function createProductionClusterControlApplicationStack(
options.copilotFailureDiagnosis.readCapability,
),
]),
...(options.copilotFailureDiagnosis?.cancellationCapability === undefined
? []
: [
createClusterControlCopilotFailureDiagnosisCancellationRoute(
options.copilotFailureDiagnosis.cancellationCapability,
createEventId,
),
]),
];
const routes = createClusterControlRouteRegistry(routeDefinitions);
const expectedRouteCount =
@@ -313,7 +328,10 @@ export function createProductionClusterControlApplicationStack(
(options.promptOutputRead === undefined ? 0 : 1) +
(options.promptExecutionOutputRead === undefined ? 0 : 1) +
(options.copilotFailureDiagnosis === undefined ? 0 : 1) +
(options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2);
(options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2) +
(options.copilotFailureDiagnosis?.cancellationCapability === undefined
? 0
: 1);
if (routes.size !== expectedRouteCount) {
throw new Error('Production cluster-control route allowlist is incomplete');
}
packages/ql3-cluster-control/src/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.ts
+261
View File
@@ -0,0 +1,261 @@
// Cluster Copilot resolves an external request key before cancelling its Run.
import {
CopilotFailureDiagnosisCancellationNotFoundError,
CopilotFailureDiagnosisCancellationUnavailableError,
InvalidCopilotFailureDiagnosisCancellationError,
type CopilotFailureDiagnosisCancellationCommand,
} from '@qinglong/ai/failure-diagnosis-cancellation';
import {
CLUSTER_RUN_CANCELLATION_SCHEMA,
ClusterRunCancellationFenceRejectedError,
ClusterRunCancellationNotFoundError,
ClusterRunCancellationUnavailableError,
InvalidClusterRunCancellationError,
parseClusterRunCancellationRequestBody,
} from '@qinglong/runtime-core/cluster-run-cancellation';
import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
import type {
ClusterControlAuthorizedOperationRequest,
ClusterControlRouteDefinition,
ClusterControlRouteParameters,
} from '../../transport/routeRegistry';
export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA =
'qinglong/cluster-copilot-failure-diagnosis-cancellation-response@v1' as const;
export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE =
Object.freeze({
method: 'POST' as const,
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/cancellation',
operationId: 'copilot.failure_diagnosis.cancel',
permission: 'run.stop',
projectParameter: 'projectId',
});
export interface ClusterCopilotFailureDiagnosisCancellationCapability {
cancel(
command: Readonly<CopilotFailureDiagnosisCancellationCommand>,
): Promise<unknown>;
}
export type ClusterCopilotFailureDiagnosisCancellationEventIdFactory =
() => string;
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']);
const STATUSES = new Set(['accepted', 'already_requested', 'already_terminal']);
const CANCEL_REASONS = new Set([
'user',
'policy',
'shutdown',
'reconcile',
'timeout',
]);
function response(
statusCode: number,
body: Readonly<Record<string, unknown>>,
): ClusterControlAdmissionResponse {
return Object.freeze({ statusCode, body: Object.freeze(body) });
}
function exactRecord(
value: unknown,
expected: readonly string[],
): Record<string, unknown> | null {
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
const record = value as Record<string, unknown>;
const actual = Object.keys(record).sort();
const canonical = [...expected].sort();
return actual.length === canonical.length &&
actual.every((key, index) => key === canonical[index])
? record
: null;
}
function safeInteger(value: unknown): value is number {
return Number.isSafeInteger(value) && (value as number) >= 0;
}
function projectResult(
value: unknown,
target: Readonly<{
projectId: string;
sourceRunId: string;
requestId: string;
}>,
): Readonly<Record<string, unknown>> | null {
const candidate = exactRecord(value, [
'cancelReason',
'cancelRequestedAtMs',
'convergence',
'diagnosisRunId',
'eventSequence',
'outcome',
'projectId',
'requestId',
'runStatus',
'runVersion',
'schema',
'sourceRunId',
'status',
]);
if (
!candidate ||
candidate.schema !==
'qinglong/copilot-failure-diagnosis-cancellation-result@v1' ||
candidate.projectId !== target.projectId ||
candidate.sourceRunId !== target.sourceRunId ||
candidate.requestId !== target.requestId ||
typeof candidate.diagnosisRunId !== 'string' ||
!RUN_ID.test(candidate.diagnosisRunId) ||
typeof candidate.status !== 'string' ||
!STATUSES.has(candidate.status) ||
!safeInteger(candidate.runVersion) ||
!safeInteger(candidate.eventSequence) ||
candidate.runVersion !== candidate.eventSequence ||
!(
(candidate.cancelRequestedAtMs === null &&
candidate.cancelReason === null) ||
(safeInteger(candidate.cancelRequestedAtMs) &&
typeof candidate.cancelReason === 'string' &&
CANCEL_REASONS.has(candidate.cancelReason))
)
) {
return null;
}
if (candidate.convergence === 'model_in_flight') {
if (
candidate.runStatus !== 'running' ||
candidate.outcome !== null ||
candidate.cancelRequestedAtMs === null
) {
return null;
}
} else if (candidate.convergence === 'terminal') {
if (
typeof candidate.runStatus !== 'string' ||
!OUTCOMES.has(candidate.runStatus) ||
candidate.outcome !== candidate.runStatus
) {
return null;
}
} else {
return null;
}
return Object.freeze({
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA,
status: candidate.status,
convergence: candidate.convergence,
projectId: target.projectId,
sourceRunId: target.sourceRunId,
requestId: target.requestId,
diagnosisRunId: candidate.diagnosisRunId,
runStatus: candidate.runStatus,
outcome: candidate.outcome,
runVersion: candidate.runVersion,
eventSequence: candidate.eventSequence,
cancelRequestedAtMs: candidate.cancelRequestedAtMs,
cancelReason: candidate.cancelReason,
});
}
export function createClusterControlCopilotFailureDiagnosisCancellationRoute(
capability: ClusterCopilotFailureDiagnosisCancellationCapability,
createEventId: ClusterCopilotFailureDiagnosisCancellationEventIdFactory,
): Readonly<ClusterControlRouteDefinition> {
if (
!capability ||
typeof capability.cancel !== 'function' ||
typeof createEventId !== 'function'
) {
throw new TypeError(
'Cluster-control Copilot failure diagnosis cancellation route is invalid',
);
}
return Object.freeze({
...CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE,
async handle(
authorized: ClusterControlAuthorizedOperationRequest,
parameters: ClusterControlRouteParameters,
) {
let body;
try {
body = parseClusterRunCancellationRequestBody(authorized.request.body);
} catch (error) {
return error instanceof InvalidClusterRunCancellationError
? response(400, {
code: 'invalid_copilot_failure_diagnosis_cancellation_request',
schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
})
: response(503, {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
});
}
const projectId = authorized.projectId;
const sourceRunId = parameters.runId;
const requestId = parameters.requestId;
if (
projectId === null ||
typeof sourceRunId !== 'string' ||
!RUN_ID.test(sourceRunId) ||
typeof requestId !== 'string' ||
!IDENTITY.test(requestId) ||
!authorized.policyFence ||
authorized.policyFence.bindingVersion === null
) {
return response(503, {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
});
}
const target = Object.freeze({ projectId, sourceRunId, requestId });
try {
const result = await capability.cancel({
...target,
mutationId: body.mutationId,
eventId: createEventId(),
subject: authorized.principal.subject,
policyFence: authorized.policyFence,
});
const view = projectResult(result, target);
if (!view) {
return response(503, {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
});
}
return response(view.status === 'accepted' ? 202 : 200, view);
} catch (error) {
if (
error instanceof CopilotFailureDiagnosisCancellationNotFoundError ||
error instanceof ClusterRunCancellationNotFoundError
) {
return response(404, {
code: 'copilot_failure_diagnosis_not_found',
});
}
if (error instanceof ClusterRunCancellationFenceRejectedError) {
return response(409, {
code: 'copilot_failure_diagnosis_cancellation_fence_rejected',
reason: error.reason,
});
}
if (
error instanceof InvalidCopilotFailureDiagnosisCancellationError ||
error instanceof
CopilotFailureDiagnosisCancellationUnavailableError ||
error instanceof InvalidClusterRunCancellationError ||
error instanceof ClusterRunCancellationUnavailableError
) {
return response(503, {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
});
}
return response(503, {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
});
}
},
});
}
packages/ql3-cluster-control/test/aiProductionApplication.test.cjs
+11
View File
@@ -169,9 +169,11 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
const artifactStore = { put() {}, inspect() {}, readLogRange() {} };
const copilot = Object.freeze({ execute() {} });
const copilotRead = Object.freeze({ inspect() {}, readOutput() {} });
const copilotCancellation = Object.freeze({ cancel() {} });
let registeredSink;
let created;
let createdRead;
let createdCancellation;
let controlOptions;
try {
await Promise.all([
@@ -233,6 +235,10 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
createdRead = options;
return copilotRead;
},
createCopilotCancellation(options) {
createdCancellation = options;
return copilotCancellation;
},
async startControl(options) {
controlOptions = options;
return {
@@ -257,11 +263,16 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
assert.equal(created.artifactStore, artifactStore);
assert.equal(createdRead.pool, fakePool);
assert.equal(typeof createdRead.prepared.outputKeys.resolve, 'function');
assert.equal(createdCancellation.pool, fakePool);
assert.equal(controlOptions.copilotFailureDiagnosis.capability, copilot);
assert.equal(
controlOptions.copilotFailureDiagnosis.readCapability,
copilotRead,
);
assert.equal(
controlOptions.copilotFailureDiagnosis.cancellationCapability,
copilotCancellation,
);
assert.equal(await application.stop(), 'stopped');
} finally {
config.fill(0);
packages/ql3-cluster-control/test/copilotFailureDiagnosisCancellationRoute.test.cjs
+232
View File
@@ -0,0 +1,232 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
CopilotFailureDiagnosisCancellationNotFoundError,
} = require('@qinglong/ai/failure-diagnosis-cancellation');
const {
CLUSTER_RUN_CANCELLATION_SCHEMA,
ClusterRunCancellationFenceRejectedError,
} = require('@qinglong/runtime-core/cluster-run-cancellation');
const {
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA,
CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE,
createClusterControlCopilotFailureDiagnosisCancellationRoute,
} = require('@qinglong/cluster-control/copilot-cancellation-route');
function authorized(body, overrides = {}) {
return {
request: {
requestId: 'transport-request-1',
method: 'POST',
path: '/api/v3/projects/project-1/runs/source-run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation',
query: {},
headers: {},
signal: new AbortController().signal,
body,
},
principal: {
subject: { type: 'user', id: 'owner-1' },
authenticationId: 'credential-1',
authenticatedAtMs: 1,
expiresAtMs: 10_000,
assurance: 'multi_factor',
},
operationId: 'copilot.failure_diagnosis.cancel',
permission: 'run.stop',
projectId: 'project-1',
policyFence: { projectVersion: 3, bindingVersion: 7 },
...overrides,
};
}
const parameters = {
projectId: 'project-1',
runId: 'source-run-1',
requestId: 'diagnosis-request-1',
};
function body(overrides = {}) {
return {
schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
mutationId: '11111111-1111-4111-8111-111111111111',
...overrides,
};
}
function result(overrides = {}) {
return {
schema: 'qinglong/copilot-failure-diagnosis-cancellation-result@v1',
status: 'accepted',
convergence: 'terminal',
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
diagnosisRunId: 'diagnosis-run-1',
runStatus: 'cancelled',
outcome: 'cancelled',
runVersion: 7,
eventSequence: 7,
cancelRequestedAtMs: 500,
cancelReason: 'user',
...overrides,
};
}
test('defines an exact run.stop route and passes only fenced target facts', async () => {
let command;
const request = authorized(body());
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
{
async cancel(value) {
command = value;
return result();
},
},
() => '22222222-2222-4222-8222-222222222222',
);
const response = await route.handle(request, parameters);
assert.deepEqual(
CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE,
{
method: 'POST',
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/cancellation',
operationId: 'copilot.failure_diagnosis.cancel',
permission: 'run.stop',
projectParameter: 'projectId',
},
);
assert.deepEqual(command, {
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
mutationId: '11111111-1111-4111-8111-111111111111',
eventId: '22222222-2222-4222-8222-222222222222',
subject: request.principal.subject,
policyFence: request.policyFence,
});
assert.deepEqual(response, {
statusCode: 202,
body: {
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA,
status: 'accepted',
convergence: 'terminal',
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
diagnosisRunId: 'diagnosis-run-1',
runStatus: 'cancelled',
outcome: 'cancelled',
runVersion: 7,
eventSequence: 7,
cancelRequestedAtMs: 500,
cancelReason: 'user',
},
});
});
test('projects an in-flight durable intent without claiming Provider abort', async () => {
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
{
async cancel() {
return result({
status: 'already_requested',
convergence: 'model_in_flight',
runStatus: 'running',
outcome: null,
runVersion: 6,
eventSequence: 6,
});
},
},
() => '22222222-2222-4222-8222-222222222222',
);
const response = await route.handle(authorized(body()), parameters);
assert.equal(response.statusCode, 200);
assert.equal(response.body.convergence, 'model_in_flight');
assert.equal(response.body.runStatus, 'running');
assert.equal(response.body.outcome, null);
assert.equal('providerAborted' in response.body, false);
});
test('rejects non-exact bodies before invoking the capability', async () => {
let calls = 0;
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
{
async cancel() {
calls += 1;
return result();
},
},
() => '22222222-2222-4222-8222-222222222222',
);
for (const value of [
null,
{},
body({ runId: 'caller-selected' }),
body({ reason: 'timeout' }),
body({ mutationId: '' }),
]) {
const response = await route.handle(authorized(value), parameters);
assert.equal(response.statusCode, 400);
}
assert.equal(calls, 0);
});
test('fails closed on widened or identity-drifted capability results', async () => {
for (const value of [
result({ sourceRunId: 'other' }),
result({ privateProvider: 'must-not-cross' }),
result({ runVersion: 8 }),
result({ convergence: 'model_in_flight' }),
]) {
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
{
async cancel() {
return value;
},
},
() => '22222222-2222-4222-8222-222222222222',
);
const response = await route.handle(authorized(body()), parameters);
assert.deepEqual(response, {
statusCode: 503,
body: {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
},
});
}
});
test('maps hidden targets, Policy races and storage failures to stable codes', async () => {
for (const [error, statusCode, code] of [
[
new CopilotFailureDiagnosisCancellationNotFoundError(),
404,
'copilot_failure_diagnosis_not_found',
],
[
new ClusterRunCancellationFenceRejectedError('authorization_changed'),
409,
'copilot_failure_diagnosis_cancellation_fence_rejected',
],
[
new Error('private storage detail'),
503,
'copilot_failure_diagnosis_cancellation_unavailable',
],
]) {
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
{
async cancel() {
throw error;
},
},
() => '22222222-2222-4222-8222-222222222222',
);
const response = await route.handle(authorized(body()), parameters);
assert.equal(response.statusCode, statusCode);
assert.equal(response.body.code, code);
assert.equal(JSON.stringify(response).includes('private'), false);
}
});
packages/ql3-cluster-control/test/productionApplication.test.cjs
+45
View File
@@ -732,6 +732,7 @@ test('optionally exposes Prompt execution behind shared admission and policy', a
'copilot.failure_diagnosis.execute',
'copilot.failure_diagnosis.read',
'copilot.failure_diagnosis.output.read',
'copilot.failure_diagnosis.cancel',
]);
const response = await invoke(
stack,
@@ -762,6 +763,7 @@ test('optionally exposes Prompt execution behind shared admission and policy', a
test('optionally exposes Copilot diagnosis behind shared authentication, Policy and audit', async () => {
const { events, input } = fixture();
let command;
let cancellationCommand;
const capability = {
async execute(value) {
command = value;
@@ -803,11 +805,30 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy
requestId: value.requestId,
};
},
async cancel(value) {
cancellationCommand = value;
return {
schema: 'qinglong/copilot-failure-diagnosis-cancellation-result@v1',
status: 'accepted',
convergence: 'terminal',
projectId: value.projectId,
sourceRunId: value.sourceRunId,
requestId: value.requestId,
diagnosisRunId: 'diagnosis-run-1',
runStatus: 'cancelled',
outcome: 'cancelled',
runVersion: 7,
eventSequence: 7,
cancelRequestedAtMs: 2_000,
cancelReason: 'user',
};
},
};
const stack = createProductionClusterControlApplicationStack(input, {
copilotFailureDiagnosis: {
capability,
readCapability: capability,
cancellationCapability: capability,
},
});
const result = await invoke(
@@ -849,6 +870,25 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy
);
assert.equal(inspection.statusCode, 404);
assert.equal(output.statusCode, 404);
const cancellation = await invoke(
stack,
metadata(
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation',
'POST',
{
schema: 'qinglong/run-cancellation@v1',
mutationId: '00000000-0000-4000-8000-000000000099',
},
),
);
assert.equal(cancellation.statusCode, 202);
assert.equal(cancellationCommand.projectId, 'project-1');
assert.equal(cancellationCommand.sourceRunId, 'run-1');
assert.equal(cancellationCommand.requestId, 'diagnosis-request-1');
assert.deepEqual(cancellationCommand.policyFence, {
projectVersion: 3,
bindingVersion: 7,
});
assert.equal(
events.includes('audit:copilot.failure_diagnosis.read:allowed'),
true,
@@ -857,6 +897,10 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy
events.includes('audit:copilot.failure_diagnosis.output.read:allowed'),
true,
);
assert.equal(
events.includes('audit:copilot.failure_diagnosis.cancel:allowed'),
true,
);
});
test('keeps the Copilot route absent by default and never invokes it after Policy denial', async () => {
@@ -879,6 +923,7 @@ test('keeps the Copilot route absent by default and never invokes it after Polic
for (const path of [
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1',
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/output',
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation',
]) {
await assert.rejects(
defaultStack.admission.prepare(metadata(path)),