feat(ql3): add fenced copilot diagnosis cancellation

This commit is contained in:
whyour
2026-08-16 01:34:55 +08:00
parent f77b19e615
commit e5a3d83a8d
17 changed files with 1685 additions and 8 deletions
+5
View File
@@ -65,6 +65,11 @@
"require": "./dist/copilot/failure-diagnosis/read-model/service.js",
"default": "./dist/copilot/failure-diagnosis/read-model/service.js"
},
"./failure-diagnosis-cancellation": {
"types": "./dist/copilot/failure-diagnosis/cancellation/service.d.ts",
"require": "./dist/copilot/failure-diagnosis/cancellation/service.js",
"default": "./dist/copilot/failure-diagnosis/cancellation/service.js"
},
"./failure-diagnosis-pre-model-terminalization": {
"types": "./dist/copilot/failure-diagnosis/preModelTerminalization.d.ts",
"require": "./dist/copilot/failure-diagnosis/preModelTerminalization.js",
@@ -0,0 +1,415 @@
import { RUN_STATUSES, type RunStatus } from '@qinglong/runtime-core';
import type {
ClusterRunCancellationRepository,
ClusterRunCancellationResult,
} from '@qinglong/runtime-core/cluster-run-cancellation';
import type {
SecurityPolicyFence,
SecuritySubject,
} from '@qinglong/runtime-core/security';
import type {
CopilotFailureDiagnosisAdmissionRepository,
CopilotFailureDiagnosisAdmissionReceipt,
CopilotFailureDiagnosisExecutionPlan,
} from '../admission/contracts';
import { normalizeCopilotFailureDiagnosisAdmissionReceipt } from '../admission/durableEvidence';
import { normalizeCopilotFailureDiagnosisExecutionPlan } from '../admission/plan';
import {
terminalizeCopilotFailureDiagnosisBeforeModel,
type CopilotFailureDiagnosisPreModelTerminalizationDependencies,
} from '../terminalization/coordinator';
import {
CopilotFailureDiagnosisPreModelTerminalizationConflictError,
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError,
CopilotFailureDiagnosisPreModelTerminalizationUnavailableError,
} from '../terminalization/contracts';
export const COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA =
'qinglong/copilot-failure-diagnosis-cancellation-result@v1' as const;
export interface CopilotFailureDiagnosisCancellationCommand {
readonly projectId: string;
readonly sourceRunId: string;
readonly requestId: string;
readonly mutationId: string;
readonly eventId: string;
readonly subject: Readonly<SecuritySubject>;
readonly policyFence: Readonly<SecurityPolicyFence>;
}
export interface CopilotFailureDiagnosisCancellationResult {
readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA;
readonly status: ClusterRunCancellationResult['status'];
readonly convergence: 'terminal' | 'model_in_flight';
readonly projectId: string;
readonly sourceRunId: string;
readonly requestId: string;
readonly diagnosisRunId: string;
readonly runStatus: RunStatus;
readonly outcome: 'succeeded' | 'failed' | 'timed_out' | 'cancelled' | null;
readonly runVersion: number;
readonly eventSequence: number;
readonly cancelRequestedAtMs: number | null;
readonly cancelReason:
| 'user'
| 'policy'
| 'shutdown'
| 'reconcile'
| 'timeout'
| null;
}
export interface CopilotFailureDiagnosisCancellationDependencies {
readonly admissions: Pick<
CopilotFailureDiagnosisAdmissionRepository,
'findByRequestId' | 'findPlanByRequestId'
>;
readonly cancellations: ClusterRunCancellationRepository;
readonly terminalizations: CopilotFailureDiagnosisPreModelTerminalizationDependencies;
readonly terminalizeBeforeModel?: typeof terminalizeCopilotFailureDiagnosisBeforeModel;
}
export class InvalidCopilotFailureDiagnosisCancellationError extends TypeError {
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_INVALID';
constructor() {
super('Copilot failure diagnosis cancellation is invalid');
this.name = 'InvalidCopilotFailureDiagnosisCancellationError';
}
}
export class CopilotFailureDiagnosisCancellationNotFoundError extends Error {
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_NOT_FOUND';
constructor() {
super('Copilot failure diagnosis cancellation target does not exist');
this.name = 'CopilotFailureDiagnosisCancellationNotFoundError';
}
}
export class CopilotFailureDiagnosisCancellationUnavailableError extends Error {
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_UNAVAILABLE';
constructor(options?: ErrorOptions) {
super('Copilot failure diagnosis cancellation is unavailable', options);
this.name = 'CopilotFailureDiagnosisCancellationUnavailableError';
}
}
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
const TERMINAL = new Set<RunStatus>([
'succeeded',
'failed',
'cancelled',
'timed_out',
]);
function unavailable(cause?: unknown): never {
throw new CopilotFailureDiagnosisCancellationUnavailableError({
cause: cause instanceof Error ? cause : undefined,
});
}
function exactKeys(value: object, expected: readonly string[]): boolean {
const actual = Object.keys(value).sort();
const canonical = [...expected].sort();
return (
actual.length === canonical.length &&
actual.every((key, index) => key === canonical[index])
);
}
function command(
value: CopilotFailureDiagnosisCancellationCommand,
): Readonly<CopilotFailureDiagnosisCancellationCommand> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!exactKeys(value, [
'eventId',
'mutationId',
'policyFence',
'projectId',
'requestId',
'sourceRunId',
'subject',
]) ||
!IDENTITY.test(value.projectId) ||
!RUN_ID.test(value.sourceRunId) ||
!IDENTITY.test(value.requestId) ||
!IDENTITY.test(value.mutationId) ||
!IDENTITY.test(value.eventId) ||
!value.subject ||
typeof value.subject !== 'object' ||
!value.policyFence ||
typeof value.policyFence !== 'object'
) {
throw new InvalidCopilotFailureDiagnosisCancellationError();
}
return Object.freeze({ ...value });
}
function evidenceMatches(
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
receipt: Readonly<CopilotFailureDiagnosisAdmissionReceipt>,
): boolean {
return (
receipt.requestId === plan.requestId &&
receipt.planDigest === plan.planDigest &&
receipt.runId === plan.runId &&
receipt.sourceRunId === plan.source.runId &&
receipt.sourceRunVersion === plan.source.runVersion &&
receipt.sourceAttemptId === plan.source.attemptId &&
receipt.toolStepRunId === plan.toolStepRunId &&
receipt.modelStepRunId === plan.modelStepRunId
);
}
function targetMatches(
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
value: Readonly<CopilotFailureDiagnosisCancellationCommand>,
): boolean {
return (
plan.projectId === value.projectId &&
plan.source.runId === value.sourceRunId &&
plan.requestId === value.requestId
);
}
function result(
target: Readonly<CopilotFailureDiagnosisCancellationCommand>,
cancellation: Readonly<ClusterRunCancellationResult>,
state: Readonly<{
convergence: CopilotFailureDiagnosisCancellationResult['convergence'];
runStatus: RunStatus;
runVersion: number;
eventSequence: number;
cancelRequestedAtMs?: number;
cancelReason?: NonNullable<
CopilotFailureDiagnosisCancellationResult['cancelReason']
>;
}>,
): Readonly<CopilotFailureDiagnosisCancellationResult> {
if (
cancellation.projectId !== target.projectId ||
!RUN_STATUSES.includes(state.runStatus) ||
!Number.isSafeInteger(state.runVersion) ||
state.runVersion < 0 ||
!Number.isSafeInteger(state.eventSequence) ||
state.eventSequence < 0 ||
(state.convergence === 'terminal') !== TERMINAL.has(state.runStatus) ||
(state.convergence === 'model_in_flight' && state.runStatus !== 'running')
) {
return unavailable();
}
return Object.freeze({
schema: COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA,
status: cancellation.status,
convergence: state.convergence,
projectId: target.projectId,
sourceRunId: target.sourceRunId,
requestId: target.requestId,
diagnosisRunId: cancellation.runId,
runStatus: state.runStatus,
outcome: TERMINAL.has(state.runStatus)
? (state.runStatus as 'succeeded' | 'failed' | 'timed_out' | 'cancelled')
: null,
runVersion: state.runVersion,
eventSequence: state.eventSequence,
cancelRequestedAtMs: state.cancelRequestedAtMs ?? null,
cancelReason: state.cancelReason ?? null,
});
}
/**
* Resolves an external request key to its server-owned diagnosis Run, writes
* one ordinary Run cancellation intent, then converges only while no Model
* invocation start exists.
*/
export class CopilotFailureDiagnosisCancellationService {
readonly #dependencies: Readonly<CopilotFailureDiagnosisCancellationDependencies>;
readonly #terminalize: typeof terminalizeCopilotFailureDiagnosisBeforeModel;
constructor(dependencies: CopilotFailureDiagnosisCancellationDependencies) {
if (
!dependencies ||
typeof dependencies !== 'object' ||
Array.isArray(dependencies) ||
typeof dependencies.admissions?.findByRequestId !== 'function' ||
typeof dependencies.admissions?.findPlanByRequestId !== 'function' ||
typeof dependencies.cancellations?.requestUserCancellation !==
'function' ||
typeof dependencies.terminalizations?.repository?.findByRequestId !==
'function' ||
typeof dependencies.terminalizations?.repository?.readAuthority !==
'function' ||
typeof dependencies.terminalizations?.repository?.commit !== 'function' ||
(dependencies.terminalizeBeforeModel !== undefined &&
typeof dependencies.terminalizeBeforeModel !== 'function')
) {
throw new InvalidCopilotFailureDiagnosisCancellationError();
}
this.#dependencies = Object.freeze({ ...dependencies });
this.#terminalize =
dependencies.terminalizeBeforeModel ??
terminalizeCopilotFailureDiagnosisBeforeModel;
}
async cancel(
value: CopilotFailureDiagnosisCancellationCommand,
): Promise<Readonly<CopilotFailureDiagnosisCancellationResult>> {
const target = command(value);
let plan: Readonly<CopilotFailureDiagnosisExecutionPlan> | null;
let receipt: Readonly<CopilotFailureDiagnosisAdmissionReceipt> | null;
try {
const located = await Promise.all([
this.#dependencies.admissions.findPlanByRequestId(target.requestId),
this.#dependencies.admissions.findByRequestId(target.requestId),
]);
plan = located[0]
? normalizeCopilotFailureDiagnosisExecutionPlan(located[0])
: null;
receipt = located[1]
? normalizeCopilotFailureDiagnosisAdmissionReceipt(located[1])
: null;
} catch (cause) {
return unavailable(cause);
}
if (!plan || !receipt || !targetMatches(plan, target)) {
throw new CopilotFailureDiagnosisCancellationNotFoundError();
}
if (!evidenceMatches(plan, receipt)) return unavailable();
const cancellation =
await this.#dependencies.cancellations.requestUserCancellation({
projectId: target.projectId,
runId: plan.runId,
mutationId: target.mutationId,
eventId: target.eventId,
subject: target.subject,
policyFence: target.policyFence,
});
if (
cancellation.projectId !== target.projectId ||
cancellation.runId !== plan.runId
) {
return unavailable();
}
if (TERMINAL.has(cancellation.runStatus)) {
return result(target, cancellation, {
convergence: 'terminal',
runStatus: cancellation.runStatus,
runVersion: cancellation.runVersion,
eventSequence: cancellation.eventSequence,
...(cancellation.cancelRequestedAtMs === undefined
? {}
: { cancelRequestedAtMs: cancellation.cancelRequestedAtMs }),
...(cancellation.cancelReason === undefined
? {}
: { cancelReason: cancellation.cancelReason }),
});
}
for (let attempt = 0; attempt < 2; attempt += 1) {
try {
const terminalized = await this.#terminalize(
target.requestId,
{ kind: 'boundary' },
this.#dependencies.terminalizations,
);
return result(target, cancellation, {
convergence: 'terminal',
runStatus: terminalized.receipt.outcome,
runVersion: terminalized.receipt.finalRunVersion,
eventSequence: terminalized.receipt.finalRunEventSequence,
...(cancellation.cancelRequestedAtMs === undefined
? {}
: { cancelRequestedAtMs: cancellation.cancelRequestedAtMs }),
...(cancellation.cancelReason === undefined
? {}
: { cancelReason: cancellation.cancelReason }),
});
} catch (cause) {
if (
!(
cause instanceof
CopilotFailureDiagnosisPreModelTerminalizationConflictError
) &&
!(
cause instanceof
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError
)
) {
if (
cause instanceof
CopilotFailureDiagnosisPreModelTerminalizationUnavailableError
) {
return unavailable(cause);
}
return unavailable(cause);
}
}
try {
const authority =
await this.#dependencies.terminalizations.repository.readAuthority(
target.requestId,
);
if (
!targetMatches(authority.plan, target) ||
authority.plan.planDigest !== plan.planDigest ||
authority.run.id !== plan.runId ||
authority.run.projectId !== target.projectId ||
authority.run.version !== authority.run.eventSequence
) {
return unavailable();
}
if (TERMINAL.has(authority.run.status)) {
return result(target, cancellation, {
convergence: 'terminal',
runStatus: authority.run.status,
runVersion: authority.run.version,
eventSequence: authority.run.eventSequence,
...(authority.run.cancelRequestedAtMs === undefined
? {}
: { cancelRequestedAtMs: authority.run.cancelRequestedAtMs }),
...(authority.run.cancelReason === undefined
? {}
: { cancelReason: authority.run.cancelReason }),
});
}
if (
authority.run.status === 'running' &&
authority.modelStartExists &&
authority.run.cancelRequestedAtMs !== undefined &&
authority.run.cancelReason !== undefined
) {
return result(target, cancellation, {
convergence: 'model_in_flight',
runStatus: 'running',
runVersion: authority.run.version,
eventSequence: authority.run.eventSequence,
cancelRequestedAtMs: authority.run.cancelRequestedAtMs,
cancelReason: authority.run.cancelReason,
});
}
if (
attempt === 0 &&
authority.run.status === 'running' &&
!authority.modelStartExists &&
authority.run.cancelRequestedAtMs !== undefined &&
authority.run.cancelReason !== undefined
) {
continue;
}
return unavailable();
} catch (cause) {
return unavailable(cause);
}
}
return unavailable();
}
}
@@ -0,0 +1,341 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
BUILTIN_RUN_LOG_EXCERPT_TOOL,
BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION,
createBuiltInRunLogExcerptToolHandlerBinding,
} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool');
const {
createPluginPackageResourceGenerationFromReferences,
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
const {
createProjectToolDefinitionSnapshot,
projectToolDefinitionRegistry,
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
const {
prepareToolInvocation,
} = require('@qinglong/runtime-core/tool-registry');
const {
TrustedToolHandlerBindingRegistry,
createTrustedToolInvocationPlan,
} = require('@qinglong/runtime-core/trusted-tool-invocation');
const {
CopilotFailureDiagnosisCancellationNotFoundError,
CopilotFailureDiagnosisCancellationService,
CopilotFailureDiagnosisCancellationUnavailableError,
} = require('@qinglong/ai/failure-diagnosis-cancellation');
const {
createCopilotFailureDiagnosisAdmissionBundle,
prepareCopilotFailureDiagnosisExecution,
} = require('@qinglong/ai/failure-diagnosis-execution-admission');
const {
CopilotFailureDiagnosisPreModelTerminalizationConflictError,
} = require('@qinglong/ai/failure-diagnosis-pre-model-terminalization');
async function durablePlan() {
const generation = createPluginPackageResourceGenerationFromReferences({
installationId: 'install-cancel-test',
projectId: 'project-cancel',
packageName: 'qinglong',
lockDigest: 'a'.repeat(64),
generation: 1,
previousActiveLockDigest: null,
contentDigest: 'b'.repeat(64),
resources: [],
});
const snapshot = createProjectToolDefinitionSnapshot({
projectId: 'project-cancel',
contributions: [
{
generation,
revisionDigest: 'c'.repeat(64),
definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION],
},
],
});
const binding = createBuiltInRunLogExcerptToolHandlerBinding(snapshot, [
'cluster-control',
]);
const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]);
const principal = {
subject: { type: 'user', id: 'owner-cancel' },
authenticationId: 'auth-cancel',
authenticatedAtMs: 100,
expiresAtMs: 10_000,
assurance: 'multi_factor',
};
const invocation = await prepareToolInvocation(
projectToolDefinitionRegistry(snapshot),
{
projectId: 'project-cancel',
principal,
nowMs: 200,
tool: BUILTIN_RUN_LOG_EXCERPT_TOOL,
input: { runId: 'source-run-cancel', attemptId: 'source-attempt-cancel' },
},
{
async authorize() {
return {
effect: 'allow',
reasons: ['role_grant'],
fence: { projectVersion: 2, bindingVersion: 3 },
};
},
},
);
const tool = createTrustedToolInvocationPlan(bindings, invocation, {
actionRef: 'cancel-log-tool',
inputArtifactId: 'cancel-input',
previewArtifactId: 'cancel-preview',
artifactKeyId: 'cancel-key',
artifactKey: Buffer.alloc(32, 0x11),
artifactNonce: Buffer.alloc(12, 0x22),
profile: 'cluster-control',
preview: {
title: 'Read failed Run log',
summary: 'Read bounded evidence',
fields: [
{ kind: 'identifier', label: 'Run', value: 'source-run-cancel' },
{
kind: 'identifier',
label: 'Attempt',
value: 'source-attempt-cancel',
},
],
warnings: ['potentially_sensitive_output'],
},
sealedAtMs: 300,
});
return prepareCopilotFailureDiagnosisExecution({
requestId: 'diagnosis-request-cancel',
traceId: 'diagnosis-trace-cancel',
source: {
runId: 'source-run-cancel',
runVersion: 4,
runStatus: 'failed',
attemptId: 'source-attempt-cancel',
attemptStatus: 'failed',
attemptFinishedAtMs: 250,
logArtifactId: 'source-log-cancel',
},
toolPlan: tool.plan,
bindings,
model: {
provider: 'provider-primary',
model: 'model-diagnosis',
modelBoundary: 'external',
responseLanguage: 'zh-CN',
maxOutputTokens: 256,
egressPolicy: {
schema: 'qinglong/copilot-model-egress-policy@v1',
revision: 'cancel-policy-v1',
potentiallySensitiveDataBoundaries: ['external'],
maxInputBytes: 64 * 1024,
maxOutputTokens: 512,
},
},
plannedAtMs: 400,
deadlineAtMs: 8_000,
});
}
function command(overrides = {}) {
return {
projectId: 'project-cancel',
sourceRunId: 'source-run-cancel',
requestId: 'diagnosis-request-cancel',
mutationId: '11111111-1111-4111-8111-111111111111',
eventId: '22222222-2222-4222-8222-222222222222',
subject: { type: 'user', id: 'owner-cancel' },
policyFence: { projectVersion: 2, bindingVersion: 3 },
...overrides,
};
}
function fixture(plan, overrides = {}) {
const receipt = createCopilotFailureDiagnosisAdmissionBundle(plan).receipt;
const state = {
cancellationCalls: [],
terminalizeCalls: 0,
cancellation: {
status: 'accepted',
projectId: plan.projectId,
runId: plan.runId,
runStatus: 'running',
runVersion: 4,
eventSequence: 4,
cancelRequestedAtMs: 500,
cancelReason: 'user',
},
authority: {
plan,
run: {
id: plan.runId,
projectId: plan.projectId,
status: 'running',
version: 4,
eventSequence: 4,
cancelRequestedAtMs: 500,
cancelReason: 'user',
},
toolStep: {},
modelStep: {},
modelStartExists: false,
observedAtMs: 501,
},
...overrides,
};
const terminalizations = {
repository: {
async findByRequestId() {
return null;
},
async readAuthority() {
return state.authority;
},
async commit() {
throw new Error('not called by injected terminalizer');
},
},
};
const service = new CopilotFailureDiagnosisCancellationService({
admissions: {
async findPlanByRequestId() {
return state.plan === undefined ? plan : state.plan;
},
async findByRequestId() {
return state.receipt === undefined ? receipt : state.receipt;
},
},
cancellations: {
async requestUserCancellation(value) {
state.cancellationCalls.push(value);
return state.cancellation;
},
},
terminalizations,
async terminalizeBeforeModel(requestId, trigger) {
state.terminalizeCalls += 1;
if (state.terminalizeError) throw state.terminalizeError;
assert.equal(requestId, plan.requestId);
assert.deepEqual(trigger, { kind: 'boundary' });
return {
status: 'created',
receipt: {
outcome: 'cancelled',
finalRunVersion: 7,
finalRunEventSequence: 7,
},
};
},
});
return { service, state, receipt };
}
test('resolves the request key to a server-owned Run and terminalizes pre-Model', async () => {
const plan = await durablePlan();
const { service, state } = fixture(plan);
const result = await service.cancel(command());
assert.deepEqual(state.cancellationCalls, [
{
projectId: 'project-cancel',
runId: plan.runId,
mutationId: '11111111-1111-4111-8111-111111111111',
eventId: '22222222-2222-4222-8222-222222222222',
subject: { type: 'user', id: 'owner-cancel' },
policyFence: { projectVersion: 2, bindingVersion: 3 },
},
]);
assert.equal(state.terminalizeCalls, 1);
assert.equal(result.status, 'accepted');
assert.equal(result.convergence, 'terminal');
assert.equal(result.runStatus, 'cancelled');
assert.equal(result.outcome, 'cancelled');
assert.equal(result.diagnosisRunId, plan.runId);
assert.equal(result.cancelRequestedAtMs, 500);
});
test('masks cross-target requests before writing a cancellation intent', async () => {
const plan = await durablePlan();
const { service, state } = fixture(plan);
await assert.rejects(
service.cancel(command({ sourceRunId: 'other-source-run' })),
CopilotFailureDiagnosisCancellationNotFoundError,
);
assert.equal(state.cancellationCalls.length, 0);
});
test('keeps a durable intent pending when Model start wins the race', async () => {
const plan = await durablePlan();
const conflict =
new CopilotFailureDiagnosisPreModelTerminalizationConflictError(
'Model started',
);
const { service, state } = fixture(plan, {
terminalizeError: conflict,
authority: {
plan,
run: {
id: plan.runId,
projectId: plan.projectId,
status: 'running',
version: 5,
eventSequence: 5,
cancelRequestedAtMs: 500,
cancelReason: 'user',
},
toolStep: {},
modelStep: {},
modelStartExists: true,
observedAtMs: 501,
},
});
const result = await service.cancel(command());
assert.equal(state.terminalizeCalls, 1);
assert.equal(result.convergence, 'model_in_flight');
assert.equal(result.runStatus, 'running');
assert.equal(result.outcome, null);
assert.equal(result.runVersion, 5);
});
test('reports the real terminal winner instead of forging cancellation', async () => {
const plan = await durablePlan();
const { service } = fixture(plan, {
terminalizeError:
new CopilotFailureDiagnosisPreModelTerminalizationConflictError(),
authority: {
plan,
run: {
id: plan.runId,
projectId: plan.projectId,
status: 'succeeded',
version: 7,
eventSequence: 7,
cancelRequestedAtMs: 500,
cancelReason: 'user',
},
toolStep: {},
modelStep: {},
modelStartExists: true,
observedAtMs: 600,
},
});
const result = await service.cancel(command());
assert.equal(result.convergence, 'terminal');
assert.equal(result.runStatus, 'succeeded');
assert.equal(result.outcome, 'succeeded');
});
test('fails closed when plan and admission receipt drift', async () => {
const plan = await durablePlan();
const base = fixture(plan);
const { service, state } = fixture(plan, {
receipt: { ...base.receipt, runId: 'diagnosis-run-drift' },
});
await assert.rejects(
service.cancel(command()),
CopilotFailureDiagnosisCancellationUnavailableError,
);
assert.equal(state.cancellationCalls.length, 0);
});
+10
View File
@@ -40,6 +40,11 @@
"require": "./dist/application-runtime/copilot/failureDiagnosisComposition.js",
"default": "./dist/application-runtime/copilot/failureDiagnosisComposition.js"
},
"./copilot-cancellation-production": {
"types": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.d.ts",
"require": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.js",
"default": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.js"
},
"./failure-diagnosis-output-keyring": {
"types": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.d.ts",
"require": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js",
@@ -55,6 +60,11 @@
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js",
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js"
},
"./copilot-cancellation-route": {
"types": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.d.ts",
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.js",
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.js"
},
"./http": {
"types": "./dist/transport/httpSurface.d.ts",
"require": "./dist/transport/httpSurface.js",
@@ -2,6 +2,7 @@ import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile';
import type { DurableModelInvocationCoordinator } from '@qinglong/ai/durable-model-invocation';
import { CopilotFailureDiagnosisModelCompletionCoordinator } from '@qinglong/ai/failure-diagnosis-model-execution';
import type { CopilotFailureDiagnosisApplicationService } from '@qinglong/ai/failure-diagnosis-application';
import type { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation';
import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential';
import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage';
import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority';
@@ -35,6 +36,10 @@ import {
createProductionClusterCopilotFailureDiagnosisReadService,
type CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions,
} from './copilot/failureDiagnosisReadComposition';
import {
createProductionClusterCopilotFailureDiagnosisCancellation,
type CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
} from './copilot/failureDiagnosisCancellationComposition';
export interface EnabledProductionClusterAiConfig {
readonly enabled: true;
@@ -63,6 +68,9 @@ export interface ProductionClusterAiControlApplicationOptions {
) => ReturnType<
typeof createProductionClusterCopilotFailureDiagnosisReadService
>;
readonly createCopilotCancellation?: (
options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
) => Readonly<CopilotFailureDiagnosisCancellationService>;
readonly openAiDatabase?: ReturnType<typeof createPostgresDatabaseOpener>;
}
@@ -263,11 +271,15 @@ export async function startProductionClusterAiControlApplication(
const createCopilotRead =
options.createCopilotRead ??
createProductionClusterCopilotFailureDiagnosisReadService;
const createCopilotCancellation =
options.createCopilotCancellation ??
createProductionClusterCopilotFailureDiagnosisCancellation;
if (
typeof startControl !== 'function' ||
typeof bootstrapPrompt !== 'function' ||
typeof createCopilot !== 'function' ||
typeof createCopilotRead !== 'function' ||
typeof createCopilotCancellation !== 'function' ||
(options.openAiDatabase !== undefined &&
typeof options.openAiDatabase !== 'function')
) {
@@ -325,6 +337,9 @@ export async function startProductionClusterAiControlApplication(
typeof createProductionClusterCopilotFailureDiagnosisReadService
>
| undefined;
let copilotCancellationApplication:
| Readonly<CopilotFailureDiagnosisCancellationService>
| undefined;
let copilotSuccessfulCompletion:
| CopilotFailureDiagnosisModelCompletionCoordinator
| undefined;
@@ -455,6 +470,9 @@ export async function startProductionClusterAiControlApplication(
pool: aiDatabase.pool,
prepared: preparedCopilot,
});
copilotCancellationApplication = createCopilotCancellation({
pool: aiDatabase.pool,
});
}
controlApplication = await startControl({
...options.control,
@@ -482,12 +500,14 @@ export async function startProductionClusterAiControlApplication(
},
}),
...(copilotApplication === undefined ||
copilotReadApplication === undefined
copilotReadApplication === undefined ||
copilotCancellationApplication === undefined
? {}
: {
copilotFailureDiagnosis: {
capability: copilotApplication,
readCapability: copilotReadApplication,
cancellationCapability: copilotCancellationApplication,
},
}),
});
@@ -0,0 +1,47 @@
import { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation';
import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage';
import {
PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository,
terminalizeCopilotFailureDiagnosisBeforeModel,
} from '@qinglong/ai/failure-diagnosis-pre-model-terminalization';
import {
PostgresClusterRunCancellationRepository,
type QingLongPostgresPool,
} from '@qinglong/cluster-postgres/runtime';
export interface CreateProductionClusterCopilotFailureDiagnosisCancellationOptions {
readonly pool: QingLongPostgresPool;
}
/**
* Reuses the AI Pool and existing Run/admission ledgers. The cancellation
* capability owns no connection, timer, listener or background lifecycle.
*/
export function createProductionClusterCopilotFailureDiagnosisCancellation(
options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
): Readonly<CopilotFailureDiagnosisCancellationService> {
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
typeof options.pool?.query !== 'function' ||
typeof options.pool?.connect !== 'function'
) {
throw new TypeError(
'Production Cluster Copilot failure diagnosis cancellation dependencies are invalid',
);
}
const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository(
options.pool,
);
const terminalizations =
new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository(
options.pool,
);
return new CopilotFailureDiagnosisCancellationService({
admissions,
cancellations: new PostgresClusterRunCancellationRepository(options.pool),
terminalizations: Object.freeze({ repository: terminalizations }),
terminalizeBeforeModel: terminalizeCopilotFailureDiagnosisBeforeModel,
});
}
@@ -71,6 +71,10 @@ import {
type ClusterCopilotFailureDiagnosisInspectionCapability,
type ClusterCopilotFailureDiagnosisOutputReadCapability,
} from '../copilot/failure-diagnosis/failureDiagnosisReadRoutes';
import {
createClusterControlCopilotFailureDiagnosisCancellationRoute,
type ClusterCopilotFailureDiagnosisCancellationCapability,
} from '../copilot/failure-diagnosis/failureDiagnosisCancellationRoute';
export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
'task.get',
@@ -100,6 +104,7 @@ export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS =
'copilot.failure_diagnosis.execute',
'copilot.failure_diagnosis.read',
'copilot.failure_diagnosis.output.read',
'copilot.failure_diagnosis.cancel',
] as const);
export interface ClusterCopilotFailureDiagnosisReadCapability
@@ -129,6 +134,7 @@ export interface ProductionClusterControlAssemblyOptions {
readonly copilotFailureDiagnosis?: Readonly<{
readonly capability: ClusterCopilotFailureDiagnosisCapability;
readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability;
}>;
readonly workerIngress?: Readonly<{
readonly config: EnabledClusterWorkerIngressConfig;
@@ -178,6 +184,7 @@ export interface ProductionClusterControlApplicationOptions
readonly copilotFailureDiagnosis?: Readonly<{
readonly capability: ClusterCopilotFailureDiagnosisCapability;
readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability;
}>;
readonly workerIngress?: ProductionClusterWorkerIngressOptions;
}
@@ -303,6 +310,14 @@ export function createProductionClusterControlApplicationStack(
options.copilotFailureDiagnosis.readCapability,
),
]),
...(options.copilotFailureDiagnosis?.cancellationCapability === undefined
? []
: [
createClusterControlCopilotFailureDiagnosisCancellationRoute(
options.copilotFailureDiagnosis.cancellationCapability,
createEventId,
),
]),
];
const routes = createClusterControlRouteRegistry(routeDefinitions);
const expectedRouteCount =
@@ -313,7 +328,10 @@ export function createProductionClusterControlApplicationStack(
(options.promptOutputRead === undefined ? 0 : 1) +
(options.promptExecutionOutputRead === undefined ? 0 : 1) +
(options.copilotFailureDiagnosis === undefined ? 0 : 1) +
(options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2);
(options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2) +
(options.copilotFailureDiagnosis?.cancellationCapability === undefined
? 0
: 1);
if (routes.size !== expectedRouteCount) {
throw new Error('Production cluster-control route allowlist is incomplete');
}
@@ -0,0 +1,261 @@
// Cluster Copilot resolves an external request key before cancelling its Run.
import {
CopilotFailureDiagnosisCancellationNotFoundError,
CopilotFailureDiagnosisCancellationUnavailableError,
InvalidCopilotFailureDiagnosisCancellationError,
type CopilotFailureDiagnosisCancellationCommand,
} from '@qinglong/ai/failure-diagnosis-cancellation';
import {
CLUSTER_RUN_CANCELLATION_SCHEMA,
ClusterRunCancellationFenceRejectedError,
ClusterRunCancellationNotFoundError,
ClusterRunCancellationUnavailableError,
InvalidClusterRunCancellationError,
parseClusterRunCancellationRequestBody,
} from '@qinglong/runtime-core/cluster-run-cancellation';
import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
import type {
ClusterControlAuthorizedOperationRequest,
ClusterControlRouteDefinition,
ClusterControlRouteParameters,
} from '../../transport/routeRegistry';
export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA =
'qinglong/cluster-copilot-failure-diagnosis-cancellation-response@v1' as const;
export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE =
Object.freeze({
method: 'POST' as const,
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/cancellation',
operationId: 'copilot.failure_diagnosis.cancel',
permission: 'run.stop',
projectParameter: 'projectId',
});
export interface ClusterCopilotFailureDiagnosisCancellationCapability {
cancel(
command: Readonly<CopilotFailureDiagnosisCancellationCommand>,
): Promise<unknown>;
}
export type ClusterCopilotFailureDiagnosisCancellationEventIdFactory =
() => string;
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']);
const STATUSES = new Set(['accepted', 'already_requested', 'already_terminal']);
const CANCEL_REASONS = new Set([
'user',
'policy',
'shutdown',
'reconcile',
'timeout',
]);
function response(
statusCode: number,
body: Readonly<Record<string, unknown>>,
): ClusterControlAdmissionResponse {
return Object.freeze({ statusCode, body: Object.freeze(body) });
}
function exactRecord(
value: unknown,
expected: readonly string[],
): Record<string, unknown> | null {
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
const record = value as Record<string, unknown>;
const actual = Object.keys(record).sort();
const canonical = [...expected].sort();
return actual.length === canonical.length &&
actual.every((key, index) => key === canonical[index])
? record
: null;
}
function safeInteger(value: unknown): value is number {
return Number.isSafeInteger(value) && (value as number) >= 0;
}
function projectResult(
value: unknown,
target: Readonly<{
projectId: string;
sourceRunId: string;
requestId: string;
}>,
): Readonly<Record<string, unknown>> | null {
const candidate = exactRecord(value, [
'cancelReason',
'cancelRequestedAtMs',
'convergence',
'diagnosisRunId',
'eventSequence',
'outcome',
'projectId',
'requestId',
'runStatus',
'runVersion',
'schema',
'sourceRunId',
'status',
]);
if (
!candidate ||
candidate.schema !==
'qinglong/copilot-failure-diagnosis-cancellation-result@v1' ||
candidate.projectId !== target.projectId ||
candidate.sourceRunId !== target.sourceRunId ||
candidate.requestId !== target.requestId ||
typeof candidate.diagnosisRunId !== 'string' ||
!RUN_ID.test(candidate.diagnosisRunId) ||
typeof candidate.status !== 'string' ||
!STATUSES.has(candidate.status) ||
!safeInteger(candidate.runVersion) ||
!safeInteger(candidate.eventSequence) ||
candidate.runVersion !== candidate.eventSequence ||
!(
(candidate.cancelRequestedAtMs === null &&
candidate.cancelReason === null) ||
(safeInteger(candidate.cancelRequestedAtMs) &&
typeof candidate.cancelReason === 'string' &&
CANCEL_REASONS.has(candidate.cancelReason))
)
) {
return null;
}
if (candidate.convergence === 'model_in_flight') {
if (
candidate.runStatus !== 'running' ||
candidate.outcome !== null ||
candidate.cancelRequestedAtMs === null
) {
return null;
}
} else if (candidate.convergence === 'terminal') {
if (
typeof candidate.runStatus !== 'string' ||
!OUTCOMES.has(candidate.runStatus) ||
candidate.outcome !== candidate.runStatus
) {
return null;
}
} else {
return null;
}
return Object.freeze({
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA,
status: candidate.status,
convergence: candidate.convergence,
projectId: target.projectId,
sourceRunId: target.sourceRunId,
requestId: target.requestId,
diagnosisRunId: candidate.diagnosisRunId,
runStatus: candidate.runStatus,
outcome: candidate.outcome,
runVersion: candidate.runVersion,
eventSequence: candidate.eventSequence,
cancelRequestedAtMs: candidate.cancelRequestedAtMs,
cancelReason: candidate.cancelReason,
});
}
export function createClusterControlCopilotFailureDiagnosisCancellationRoute(
capability: ClusterCopilotFailureDiagnosisCancellationCapability,
createEventId: ClusterCopilotFailureDiagnosisCancellationEventIdFactory,
): Readonly<ClusterControlRouteDefinition> {
if (
!capability ||
typeof capability.cancel !== 'function' ||
typeof createEventId !== 'function'
) {
throw new TypeError(
'Cluster-control Copilot failure diagnosis cancellation route is invalid',
);
}
return Object.freeze({
...CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE,
async handle(
authorized: ClusterControlAuthorizedOperationRequest,
parameters: ClusterControlRouteParameters,
) {
let body;
try {
body = parseClusterRunCancellationRequestBody(authorized.request.body);
} catch (error) {
return error instanceof InvalidClusterRunCancellationError
? response(400, {
code: 'invalid_copilot_failure_diagnosis_cancellation_request',
schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
})
: response(503, {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
});
}
const projectId = authorized.projectId;
const sourceRunId = parameters.runId;
const requestId = parameters.requestId;
if (
projectId === null ||
typeof sourceRunId !== 'string' ||
!RUN_ID.test(sourceRunId) ||
typeof requestId !== 'string' ||
!IDENTITY.test(requestId) ||
!authorized.policyFence ||
authorized.policyFence.bindingVersion === null
) {
return response(503, {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
});
}
const target = Object.freeze({ projectId, sourceRunId, requestId });
try {
const result = await capability.cancel({
...target,
mutationId: body.mutationId,
eventId: createEventId(),
subject: authorized.principal.subject,
policyFence: authorized.policyFence,
});
const view = projectResult(result, target);
if (!view) {
return response(503, {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
});
}
return response(view.status === 'accepted' ? 202 : 200, view);
} catch (error) {
if (
error instanceof CopilotFailureDiagnosisCancellationNotFoundError ||
error instanceof ClusterRunCancellationNotFoundError
) {
return response(404, {
code: 'copilot_failure_diagnosis_not_found',
});
}
if (error instanceof ClusterRunCancellationFenceRejectedError) {
return response(409, {
code: 'copilot_failure_diagnosis_cancellation_fence_rejected',
reason: error.reason,
});
}
if (
error instanceof InvalidCopilotFailureDiagnosisCancellationError ||
error instanceof
CopilotFailureDiagnosisCancellationUnavailableError ||
error instanceof InvalidClusterRunCancellationError ||
error instanceof ClusterRunCancellationUnavailableError
) {
return response(503, {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
});
}
return response(503, {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
});
}
},
});
}
@@ -169,9 +169,11 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
const artifactStore = { put() {}, inspect() {}, readLogRange() {} };
const copilot = Object.freeze({ execute() {} });
const copilotRead = Object.freeze({ inspect() {}, readOutput() {} });
const copilotCancellation = Object.freeze({ cancel() {} });
let registeredSink;
let created;
let createdRead;
let createdCancellation;
let controlOptions;
try {
await Promise.all([
@@ -233,6 +235,10 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
createdRead = options;
return copilotRead;
},
createCopilotCancellation(options) {
createdCancellation = options;
return copilotCancellation;
},
async startControl(options) {
controlOptions = options;
return {
@@ -257,11 +263,16 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
assert.equal(created.artifactStore, artifactStore);
assert.equal(createdRead.pool, fakePool);
assert.equal(typeof createdRead.prepared.outputKeys.resolve, 'function');
assert.equal(createdCancellation.pool, fakePool);
assert.equal(controlOptions.copilotFailureDiagnosis.capability, copilot);
assert.equal(
controlOptions.copilotFailureDiagnosis.readCapability,
copilotRead,
);
assert.equal(
controlOptions.copilotFailureDiagnosis.cancellationCapability,
copilotCancellation,
);
assert.equal(await application.stop(), 'stopped');
} finally {
config.fill(0);
@@ -0,0 +1,232 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
CopilotFailureDiagnosisCancellationNotFoundError,
} = require('@qinglong/ai/failure-diagnosis-cancellation');
const {
CLUSTER_RUN_CANCELLATION_SCHEMA,
ClusterRunCancellationFenceRejectedError,
} = require('@qinglong/runtime-core/cluster-run-cancellation');
const {
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA,
CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE,
createClusterControlCopilotFailureDiagnosisCancellationRoute,
} = require('@qinglong/cluster-control/copilot-cancellation-route');
function authorized(body, overrides = {}) {
return {
request: {
requestId: 'transport-request-1',
method: 'POST',
path: '/api/v3/projects/project-1/runs/source-run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation',
query: {},
headers: {},
signal: new AbortController().signal,
body,
},
principal: {
subject: { type: 'user', id: 'owner-1' },
authenticationId: 'credential-1',
authenticatedAtMs: 1,
expiresAtMs: 10_000,
assurance: 'multi_factor',
},
operationId: 'copilot.failure_diagnosis.cancel',
permission: 'run.stop',
projectId: 'project-1',
policyFence: { projectVersion: 3, bindingVersion: 7 },
...overrides,
};
}
const parameters = {
projectId: 'project-1',
runId: 'source-run-1',
requestId: 'diagnosis-request-1',
};
function body(overrides = {}) {
return {
schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
mutationId: '11111111-1111-4111-8111-111111111111',
...overrides,
};
}
function result(overrides = {}) {
return {
schema: 'qinglong/copilot-failure-diagnosis-cancellation-result@v1',
status: 'accepted',
convergence: 'terminal',
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
diagnosisRunId: 'diagnosis-run-1',
runStatus: 'cancelled',
outcome: 'cancelled',
runVersion: 7,
eventSequence: 7,
cancelRequestedAtMs: 500,
cancelReason: 'user',
...overrides,
};
}
test('defines an exact run.stop route and passes only fenced target facts', async () => {
let command;
const request = authorized(body());
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
{
async cancel(value) {
command = value;
return result();
},
},
() => '22222222-2222-4222-8222-222222222222',
);
const response = await route.handle(request, parameters);
assert.deepEqual(
CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE,
{
method: 'POST',
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/cancellation',
operationId: 'copilot.failure_diagnosis.cancel',
permission: 'run.stop',
projectParameter: 'projectId',
},
);
assert.deepEqual(command, {
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
mutationId: '11111111-1111-4111-8111-111111111111',
eventId: '22222222-2222-4222-8222-222222222222',
subject: request.principal.subject,
policyFence: request.policyFence,
});
assert.deepEqual(response, {
statusCode: 202,
body: {
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA,
status: 'accepted',
convergence: 'terminal',
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
diagnosisRunId: 'diagnosis-run-1',
runStatus: 'cancelled',
outcome: 'cancelled',
runVersion: 7,
eventSequence: 7,
cancelRequestedAtMs: 500,
cancelReason: 'user',
},
});
});
test('projects an in-flight durable intent without claiming Provider abort', async () => {
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
{
async cancel() {
return result({
status: 'already_requested',
convergence: 'model_in_flight',
runStatus: 'running',
outcome: null,
runVersion: 6,
eventSequence: 6,
});
},
},
() => '22222222-2222-4222-8222-222222222222',
);
const response = await route.handle(authorized(body()), parameters);
assert.equal(response.statusCode, 200);
assert.equal(response.body.convergence, 'model_in_flight');
assert.equal(response.body.runStatus, 'running');
assert.equal(response.body.outcome, null);
assert.equal('providerAborted' in response.body, false);
});
test('rejects non-exact bodies before invoking the capability', async () => {
let calls = 0;
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
{
async cancel() {
calls += 1;
return result();
},
},
() => '22222222-2222-4222-8222-222222222222',
);
for (const value of [
null,
{},
body({ runId: 'caller-selected' }),
body({ reason: 'timeout' }),
body({ mutationId: '' }),
]) {
const response = await route.handle(authorized(value), parameters);
assert.equal(response.statusCode, 400);
}
assert.equal(calls, 0);
});
test('fails closed on widened or identity-drifted capability results', async () => {
for (const value of [
result({ sourceRunId: 'other' }),
result({ privateProvider: 'must-not-cross' }),
result({ runVersion: 8 }),
result({ convergence: 'model_in_flight' }),
]) {
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
{
async cancel() {
return value;
},
},
() => '22222222-2222-4222-8222-222222222222',
);
const response = await route.handle(authorized(body()), parameters);
assert.deepEqual(response, {
statusCode: 503,
body: {
code: 'copilot_failure_diagnosis_cancellation_unavailable',
},
});
}
});
test('maps hidden targets, Policy races and storage failures to stable codes', async () => {
for (const [error, statusCode, code] of [
[
new CopilotFailureDiagnosisCancellationNotFoundError(),
404,
'copilot_failure_diagnosis_not_found',
],
[
new ClusterRunCancellationFenceRejectedError('authorization_changed'),
409,
'copilot_failure_diagnosis_cancellation_fence_rejected',
],
[
new Error('private storage detail'),
503,
'copilot_failure_diagnosis_cancellation_unavailable',
],
]) {
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
{
async cancel() {
throw error;
},
},
() => '22222222-2222-4222-8222-222222222222',
);
const response = await route.handle(authorized(body()), parameters);
assert.equal(response.statusCode, statusCode);
assert.equal(response.body.code, code);
assert.equal(JSON.stringify(response).includes('private'), false);
}
});
@@ -732,6 +732,7 @@ test('optionally exposes Prompt execution behind shared admission and policy', a
'copilot.failure_diagnosis.execute',
'copilot.failure_diagnosis.read',
'copilot.failure_diagnosis.output.read',
'copilot.failure_diagnosis.cancel',
]);
const response = await invoke(
stack,
@@ -762,6 +763,7 @@ test('optionally exposes Prompt execution behind shared admission and policy', a
test('optionally exposes Copilot diagnosis behind shared authentication, Policy and audit', async () => {
const { events, input } = fixture();
let command;
let cancellationCommand;
const capability = {
async execute(value) {
command = value;
@@ -803,11 +805,30 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy
requestId: value.requestId,
};
},
async cancel(value) {
cancellationCommand = value;
return {
schema: 'qinglong/copilot-failure-diagnosis-cancellation-result@v1',
status: 'accepted',
convergence: 'terminal',
projectId: value.projectId,
sourceRunId: value.sourceRunId,
requestId: value.requestId,
diagnosisRunId: 'diagnosis-run-1',
runStatus: 'cancelled',
outcome: 'cancelled',
runVersion: 7,
eventSequence: 7,
cancelRequestedAtMs: 2_000,
cancelReason: 'user',
};
},
};
const stack = createProductionClusterControlApplicationStack(input, {
copilotFailureDiagnosis: {
capability,
readCapability: capability,
cancellationCapability: capability,
},
});
const result = await invoke(
@@ -849,6 +870,25 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy
);
assert.equal(inspection.statusCode, 404);
assert.equal(output.statusCode, 404);
const cancellation = await invoke(
stack,
metadata(
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation',
'POST',
{
schema: 'qinglong/run-cancellation@v1',
mutationId: '00000000-0000-4000-8000-000000000099',
},
),
);
assert.equal(cancellation.statusCode, 202);
assert.equal(cancellationCommand.projectId, 'project-1');
assert.equal(cancellationCommand.sourceRunId, 'run-1');
assert.equal(cancellationCommand.requestId, 'diagnosis-request-1');
assert.deepEqual(cancellationCommand.policyFence, {
projectVersion: 3,
bindingVersion: 7,
});
assert.equal(
events.includes('audit:copilot.failure_diagnosis.read:allowed'),
true,
@@ -857,6 +897,10 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy
events.includes('audit:copilot.failure_diagnosis.output.read:allowed'),
true,
);
assert.equal(
events.includes('audit:copilot.failure_diagnosis.cancel:allowed'),
true,
);
});
test('keeps the Copilot route absent by default and never invokes it after Policy denial', async () => {
@@ -879,6 +923,7 @@ test('keeps the Copilot route absent by default and never invokes it after Polic
for (const path of [
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1',
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/output',
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation',
]) {
await assert.rejects(
defaultStack.admission.prepare(metadata(path)),