mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add fenced copilot diagnosis cancellation
This commit is contained in:
@@ -65,6 +65,11 @@
|
||||
"require": "./dist/copilot/failure-diagnosis/read-model/service.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/read-model/service.js"
|
||||
},
|
||||
"./failure-diagnosis-cancellation": {
|
||||
"types": "./dist/copilot/failure-diagnosis/cancellation/service.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/cancellation/service.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/cancellation/service.js"
|
||||
},
|
||||
"./failure-diagnosis-pre-model-terminalization": {
|
||||
"types": "./dist/copilot/failure-diagnosis/preModelTerminalization.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/preModelTerminalization.js",
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
import { RUN_STATUSES, type RunStatus } from '@qinglong/runtime-core';
|
||||
import type {
|
||||
ClusterRunCancellationRepository,
|
||||
ClusterRunCancellationResult,
|
||||
} from '@qinglong/runtime-core/cluster-run-cancellation';
|
||||
import type {
|
||||
SecurityPolicyFence,
|
||||
SecuritySubject,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
|
||||
import type {
|
||||
CopilotFailureDiagnosisAdmissionRepository,
|
||||
CopilotFailureDiagnosisAdmissionReceipt,
|
||||
CopilotFailureDiagnosisExecutionPlan,
|
||||
} from '../admission/contracts';
|
||||
import { normalizeCopilotFailureDiagnosisAdmissionReceipt } from '../admission/durableEvidence';
|
||||
import { normalizeCopilotFailureDiagnosisExecutionPlan } from '../admission/plan';
|
||||
import {
|
||||
terminalizeCopilotFailureDiagnosisBeforeModel,
|
||||
type CopilotFailureDiagnosisPreModelTerminalizationDependencies,
|
||||
} from '../terminalization/coordinator';
|
||||
import {
|
||||
CopilotFailureDiagnosisPreModelTerminalizationConflictError,
|
||||
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError,
|
||||
CopilotFailureDiagnosisPreModelTerminalizationUnavailableError,
|
||||
} from '../terminalization/contracts';
|
||||
|
||||
export const COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA =
|
||||
'qinglong/copilot-failure-diagnosis-cancellation-result@v1' as const;
|
||||
|
||||
export interface CopilotFailureDiagnosisCancellationCommand {
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly requestId: string;
|
||||
readonly mutationId: string;
|
||||
readonly eventId: string;
|
||||
readonly subject: Readonly<SecuritySubject>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisCancellationResult {
|
||||
readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA;
|
||||
readonly status: ClusterRunCancellationResult['status'];
|
||||
readonly convergence: 'terminal' | 'model_in_flight';
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly requestId: string;
|
||||
readonly diagnosisRunId: string;
|
||||
readonly runStatus: RunStatus;
|
||||
readonly outcome: 'succeeded' | 'failed' | 'timed_out' | 'cancelled' | null;
|
||||
readonly runVersion: number;
|
||||
readonly eventSequence: number;
|
||||
readonly cancelRequestedAtMs: number | null;
|
||||
readonly cancelReason:
|
||||
| 'user'
|
||||
| 'policy'
|
||||
| 'shutdown'
|
||||
| 'reconcile'
|
||||
| 'timeout'
|
||||
| null;
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisCancellationDependencies {
|
||||
readonly admissions: Pick<
|
||||
CopilotFailureDiagnosisAdmissionRepository,
|
||||
'findByRequestId' | 'findPlanByRequestId'
|
||||
>;
|
||||
readonly cancellations: ClusterRunCancellationRepository;
|
||||
readonly terminalizations: CopilotFailureDiagnosisPreModelTerminalizationDependencies;
|
||||
readonly terminalizeBeforeModel?: typeof terminalizeCopilotFailureDiagnosisBeforeModel;
|
||||
}
|
||||
|
||||
export class InvalidCopilotFailureDiagnosisCancellationError extends TypeError {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_INVALID';
|
||||
|
||||
constructor() {
|
||||
super('Copilot failure diagnosis cancellation is invalid');
|
||||
this.name = 'InvalidCopilotFailureDiagnosisCancellationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisCancellationNotFoundError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_NOT_FOUND';
|
||||
|
||||
constructor() {
|
||||
super('Copilot failure diagnosis cancellation target does not exist');
|
||||
this.name = 'CopilotFailureDiagnosisCancellationNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisCancellationUnavailableError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Copilot failure diagnosis cancellation is unavailable', options);
|
||||
this.name = 'CopilotFailureDiagnosisCancellationUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
|
||||
const TERMINAL = new Set<RunStatus>([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
]);
|
||||
|
||||
function unavailable(cause?: unknown): never {
|
||||
throw new CopilotFailureDiagnosisCancellationUnavailableError({
|
||||
cause: cause instanceof Error ? cause : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const actual = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return (
|
||||
actual.length === canonical.length &&
|
||||
actual.every((key, index) => key === canonical[index])
|
||||
);
|
||||
}
|
||||
|
||||
function command(
|
||||
value: CopilotFailureDiagnosisCancellationCommand,
|
||||
): Readonly<CopilotFailureDiagnosisCancellationCommand> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!exactKeys(value, [
|
||||
'eventId',
|
||||
'mutationId',
|
||||
'policyFence',
|
||||
'projectId',
|
||||
'requestId',
|
||||
'sourceRunId',
|
||||
'subject',
|
||||
]) ||
|
||||
!IDENTITY.test(value.projectId) ||
|
||||
!RUN_ID.test(value.sourceRunId) ||
|
||||
!IDENTITY.test(value.requestId) ||
|
||||
!IDENTITY.test(value.mutationId) ||
|
||||
!IDENTITY.test(value.eventId) ||
|
||||
!value.subject ||
|
||||
typeof value.subject !== 'object' ||
|
||||
!value.policyFence ||
|
||||
typeof value.policyFence !== 'object'
|
||||
) {
|
||||
throw new InvalidCopilotFailureDiagnosisCancellationError();
|
||||
}
|
||||
return Object.freeze({ ...value });
|
||||
}
|
||||
|
||||
function evidenceMatches(
|
||||
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
|
||||
receipt: Readonly<CopilotFailureDiagnosisAdmissionReceipt>,
|
||||
): boolean {
|
||||
return (
|
||||
receipt.requestId === plan.requestId &&
|
||||
receipt.planDigest === plan.planDigest &&
|
||||
receipt.runId === plan.runId &&
|
||||
receipt.sourceRunId === plan.source.runId &&
|
||||
receipt.sourceRunVersion === plan.source.runVersion &&
|
||||
receipt.sourceAttemptId === plan.source.attemptId &&
|
||||
receipt.toolStepRunId === plan.toolStepRunId &&
|
||||
receipt.modelStepRunId === plan.modelStepRunId
|
||||
);
|
||||
}
|
||||
|
||||
function targetMatches(
|
||||
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
|
||||
value: Readonly<CopilotFailureDiagnosisCancellationCommand>,
|
||||
): boolean {
|
||||
return (
|
||||
plan.projectId === value.projectId &&
|
||||
plan.source.runId === value.sourceRunId &&
|
||||
plan.requestId === value.requestId
|
||||
);
|
||||
}
|
||||
|
||||
function result(
|
||||
target: Readonly<CopilotFailureDiagnosisCancellationCommand>,
|
||||
cancellation: Readonly<ClusterRunCancellationResult>,
|
||||
state: Readonly<{
|
||||
convergence: CopilotFailureDiagnosisCancellationResult['convergence'];
|
||||
runStatus: RunStatus;
|
||||
runVersion: number;
|
||||
eventSequence: number;
|
||||
cancelRequestedAtMs?: number;
|
||||
cancelReason?: NonNullable<
|
||||
CopilotFailureDiagnosisCancellationResult['cancelReason']
|
||||
>;
|
||||
}>,
|
||||
): Readonly<CopilotFailureDiagnosisCancellationResult> {
|
||||
if (
|
||||
cancellation.projectId !== target.projectId ||
|
||||
!RUN_STATUSES.includes(state.runStatus) ||
|
||||
!Number.isSafeInteger(state.runVersion) ||
|
||||
state.runVersion < 0 ||
|
||||
!Number.isSafeInteger(state.eventSequence) ||
|
||||
state.eventSequence < 0 ||
|
||||
(state.convergence === 'terminal') !== TERMINAL.has(state.runStatus) ||
|
||||
(state.convergence === 'model_in_flight' && state.runStatus !== 'running')
|
||||
) {
|
||||
return unavailable();
|
||||
}
|
||||
return Object.freeze({
|
||||
schema: COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA,
|
||||
status: cancellation.status,
|
||||
convergence: state.convergence,
|
||||
projectId: target.projectId,
|
||||
sourceRunId: target.sourceRunId,
|
||||
requestId: target.requestId,
|
||||
diagnosisRunId: cancellation.runId,
|
||||
runStatus: state.runStatus,
|
||||
outcome: TERMINAL.has(state.runStatus)
|
||||
? (state.runStatus as 'succeeded' | 'failed' | 'timed_out' | 'cancelled')
|
||||
: null,
|
||||
runVersion: state.runVersion,
|
||||
eventSequence: state.eventSequence,
|
||||
cancelRequestedAtMs: state.cancelRequestedAtMs ?? null,
|
||||
cancelReason: state.cancelReason ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves an external request key to its server-owned diagnosis Run, writes
|
||||
* one ordinary Run cancellation intent, then converges only while no Model
|
||||
* invocation start exists.
|
||||
*/
|
||||
export class CopilotFailureDiagnosisCancellationService {
|
||||
readonly #dependencies: Readonly<CopilotFailureDiagnosisCancellationDependencies>;
|
||||
readonly #terminalize: typeof terminalizeCopilotFailureDiagnosisBeforeModel;
|
||||
|
||||
constructor(dependencies: CopilotFailureDiagnosisCancellationDependencies) {
|
||||
if (
|
||||
!dependencies ||
|
||||
typeof dependencies !== 'object' ||
|
||||
Array.isArray(dependencies) ||
|
||||
typeof dependencies.admissions?.findByRequestId !== 'function' ||
|
||||
typeof dependencies.admissions?.findPlanByRequestId !== 'function' ||
|
||||
typeof dependencies.cancellations?.requestUserCancellation !==
|
||||
'function' ||
|
||||
typeof dependencies.terminalizations?.repository?.findByRequestId !==
|
||||
'function' ||
|
||||
typeof dependencies.terminalizations?.repository?.readAuthority !==
|
||||
'function' ||
|
||||
typeof dependencies.terminalizations?.repository?.commit !== 'function' ||
|
||||
(dependencies.terminalizeBeforeModel !== undefined &&
|
||||
typeof dependencies.terminalizeBeforeModel !== 'function')
|
||||
) {
|
||||
throw new InvalidCopilotFailureDiagnosisCancellationError();
|
||||
}
|
||||
this.#dependencies = Object.freeze({ ...dependencies });
|
||||
this.#terminalize =
|
||||
dependencies.terminalizeBeforeModel ??
|
||||
terminalizeCopilotFailureDiagnosisBeforeModel;
|
||||
}
|
||||
|
||||
async cancel(
|
||||
value: CopilotFailureDiagnosisCancellationCommand,
|
||||
): Promise<Readonly<CopilotFailureDiagnosisCancellationResult>> {
|
||||
const target = command(value);
|
||||
let plan: Readonly<CopilotFailureDiagnosisExecutionPlan> | null;
|
||||
let receipt: Readonly<CopilotFailureDiagnosisAdmissionReceipt> | null;
|
||||
try {
|
||||
const located = await Promise.all([
|
||||
this.#dependencies.admissions.findPlanByRequestId(target.requestId),
|
||||
this.#dependencies.admissions.findByRequestId(target.requestId),
|
||||
]);
|
||||
plan = located[0]
|
||||
? normalizeCopilotFailureDiagnosisExecutionPlan(located[0])
|
||||
: null;
|
||||
receipt = located[1]
|
||||
? normalizeCopilotFailureDiagnosisAdmissionReceipt(located[1])
|
||||
: null;
|
||||
} catch (cause) {
|
||||
return unavailable(cause);
|
||||
}
|
||||
if (!plan || !receipt || !targetMatches(plan, target)) {
|
||||
throw new CopilotFailureDiagnosisCancellationNotFoundError();
|
||||
}
|
||||
if (!evidenceMatches(plan, receipt)) return unavailable();
|
||||
|
||||
const cancellation =
|
||||
await this.#dependencies.cancellations.requestUserCancellation({
|
||||
projectId: target.projectId,
|
||||
runId: plan.runId,
|
||||
mutationId: target.mutationId,
|
||||
eventId: target.eventId,
|
||||
subject: target.subject,
|
||||
policyFence: target.policyFence,
|
||||
});
|
||||
if (
|
||||
cancellation.projectId !== target.projectId ||
|
||||
cancellation.runId !== plan.runId
|
||||
) {
|
||||
return unavailable();
|
||||
}
|
||||
if (TERMINAL.has(cancellation.runStatus)) {
|
||||
return result(target, cancellation, {
|
||||
convergence: 'terminal',
|
||||
runStatus: cancellation.runStatus,
|
||||
runVersion: cancellation.runVersion,
|
||||
eventSequence: cancellation.eventSequence,
|
||||
...(cancellation.cancelRequestedAtMs === undefined
|
||||
? {}
|
||||
: { cancelRequestedAtMs: cancellation.cancelRequestedAtMs }),
|
||||
...(cancellation.cancelReason === undefined
|
||||
? {}
|
||||
: { cancelReason: cancellation.cancelReason }),
|
||||
});
|
||||
}
|
||||
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
try {
|
||||
const terminalized = await this.#terminalize(
|
||||
target.requestId,
|
||||
{ kind: 'boundary' },
|
||||
this.#dependencies.terminalizations,
|
||||
);
|
||||
return result(target, cancellation, {
|
||||
convergence: 'terminal',
|
||||
runStatus: terminalized.receipt.outcome,
|
||||
runVersion: terminalized.receipt.finalRunVersion,
|
||||
eventSequence: terminalized.receipt.finalRunEventSequence,
|
||||
...(cancellation.cancelRequestedAtMs === undefined
|
||||
? {}
|
||||
: { cancelRequestedAtMs: cancellation.cancelRequestedAtMs }),
|
||||
...(cancellation.cancelReason === undefined
|
||||
? {}
|
||||
: { cancelReason: cancellation.cancelReason }),
|
||||
});
|
||||
} catch (cause) {
|
||||
if (
|
||||
!(
|
||||
cause instanceof
|
||||
CopilotFailureDiagnosisPreModelTerminalizationConflictError
|
||||
) &&
|
||||
!(
|
||||
cause instanceof
|
||||
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError
|
||||
)
|
||||
) {
|
||||
if (
|
||||
cause instanceof
|
||||
CopilotFailureDiagnosisPreModelTerminalizationUnavailableError
|
||||
) {
|
||||
return unavailable(cause);
|
||||
}
|
||||
return unavailable(cause);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const authority =
|
||||
await this.#dependencies.terminalizations.repository.readAuthority(
|
||||
target.requestId,
|
||||
);
|
||||
if (
|
||||
!targetMatches(authority.plan, target) ||
|
||||
authority.plan.planDigest !== plan.planDigest ||
|
||||
authority.run.id !== plan.runId ||
|
||||
authority.run.projectId !== target.projectId ||
|
||||
authority.run.version !== authority.run.eventSequence
|
||||
) {
|
||||
return unavailable();
|
||||
}
|
||||
if (TERMINAL.has(authority.run.status)) {
|
||||
return result(target, cancellation, {
|
||||
convergence: 'terminal',
|
||||
runStatus: authority.run.status,
|
||||
runVersion: authority.run.version,
|
||||
eventSequence: authority.run.eventSequence,
|
||||
...(authority.run.cancelRequestedAtMs === undefined
|
||||
? {}
|
||||
: { cancelRequestedAtMs: authority.run.cancelRequestedAtMs }),
|
||||
...(authority.run.cancelReason === undefined
|
||||
? {}
|
||||
: { cancelReason: authority.run.cancelReason }),
|
||||
});
|
||||
}
|
||||
if (
|
||||
authority.run.status === 'running' &&
|
||||
authority.modelStartExists &&
|
||||
authority.run.cancelRequestedAtMs !== undefined &&
|
||||
authority.run.cancelReason !== undefined
|
||||
) {
|
||||
return result(target, cancellation, {
|
||||
convergence: 'model_in_flight',
|
||||
runStatus: 'running',
|
||||
runVersion: authority.run.version,
|
||||
eventSequence: authority.run.eventSequence,
|
||||
cancelRequestedAtMs: authority.run.cancelRequestedAtMs,
|
||||
cancelReason: authority.run.cancelReason,
|
||||
});
|
||||
}
|
||||
if (
|
||||
attempt === 0 &&
|
||||
authority.run.status === 'running' &&
|
||||
!authority.modelStartExists &&
|
||||
authority.run.cancelRequestedAtMs !== undefined &&
|
||||
authority.run.cancelReason !== undefined
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
return unavailable();
|
||||
} catch (cause) {
|
||||
return unavailable(cause);
|
||||
}
|
||||
}
|
||||
return unavailable();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||
BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION,
|
||||
createBuiltInRunLogExcerptToolHandlerBinding,
|
||||
} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool');
|
||||
const {
|
||||
createPluginPackageResourceGenerationFromReferences,
|
||||
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
|
||||
const {
|
||||
createProjectToolDefinitionSnapshot,
|
||||
projectToolDefinitionRegistry,
|
||||
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
|
||||
const {
|
||||
prepareToolInvocation,
|
||||
} = require('@qinglong/runtime-core/tool-registry');
|
||||
const {
|
||||
TrustedToolHandlerBindingRegistry,
|
||||
createTrustedToolInvocationPlan,
|
||||
} = require('@qinglong/runtime-core/trusted-tool-invocation');
|
||||
const {
|
||||
CopilotFailureDiagnosisCancellationNotFoundError,
|
||||
CopilotFailureDiagnosisCancellationService,
|
||||
CopilotFailureDiagnosisCancellationUnavailableError,
|
||||
} = require('@qinglong/ai/failure-diagnosis-cancellation');
|
||||
const {
|
||||
createCopilotFailureDiagnosisAdmissionBundle,
|
||||
prepareCopilotFailureDiagnosisExecution,
|
||||
} = require('@qinglong/ai/failure-diagnosis-execution-admission');
|
||||
const {
|
||||
CopilotFailureDiagnosisPreModelTerminalizationConflictError,
|
||||
} = require('@qinglong/ai/failure-diagnosis-pre-model-terminalization');
|
||||
|
||||
async function durablePlan() {
|
||||
const generation = createPluginPackageResourceGenerationFromReferences({
|
||||
installationId: 'install-cancel-test',
|
||||
projectId: 'project-cancel',
|
||||
packageName: 'qinglong',
|
||||
lockDigest: 'a'.repeat(64),
|
||||
generation: 1,
|
||||
previousActiveLockDigest: null,
|
||||
contentDigest: 'b'.repeat(64),
|
||||
resources: [],
|
||||
});
|
||||
const snapshot = createProjectToolDefinitionSnapshot({
|
||||
projectId: 'project-cancel',
|
||||
contributions: [
|
||||
{
|
||||
generation,
|
||||
revisionDigest: 'c'.repeat(64),
|
||||
definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION],
|
||||
},
|
||||
],
|
||||
});
|
||||
const binding = createBuiltInRunLogExcerptToolHandlerBinding(snapshot, [
|
||||
'cluster-control',
|
||||
]);
|
||||
const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]);
|
||||
const principal = {
|
||||
subject: { type: 'user', id: 'owner-cancel' },
|
||||
authenticationId: 'auth-cancel',
|
||||
authenticatedAtMs: 100,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'multi_factor',
|
||||
};
|
||||
const invocation = await prepareToolInvocation(
|
||||
projectToolDefinitionRegistry(snapshot),
|
||||
{
|
||||
projectId: 'project-cancel',
|
||||
principal,
|
||||
nowMs: 200,
|
||||
tool: BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||
input: { runId: 'source-run-cancel', attemptId: 'source-attempt-cancel' },
|
||||
},
|
||||
{
|
||||
async authorize() {
|
||||
return {
|
||||
effect: 'allow',
|
||||
reasons: ['role_grant'],
|
||||
fence: { projectVersion: 2, bindingVersion: 3 },
|
||||
};
|
||||
},
|
||||
},
|
||||
);
|
||||
const tool = createTrustedToolInvocationPlan(bindings, invocation, {
|
||||
actionRef: 'cancel-log-tool',
|
||||
inputArtifactId: 'cancel-input',
|
||||
previewArtifactId: 'cancel-preview',
|
||||
artifactKeyId: 'cancel-key',
|
||||
artifactKey: Buffer.alloc(32, 0x11),
|
||||
artifactNonce: Buffer.alloc(12, 0x22),
|
||||
profile: 'cluster-control',
|
||||
preview: {
|
||||
title: 'Read failed Run log',
|
||||
summary: 'Read bounded evidence',
|
||||
fields: [
|
||||
{ kind: 'identifier', label: 'Run', value: 'source-run-cancel' },
|
||||
{
|
||||
kind: 'identifier',
|
||||
label: 'Attempt',
|
||||
value: 'source-attempt-cancel',
|
||||
},
|
||||
],
|
||||
warnings: ['potentially_sensitive_output'],
|
||||
},
|
||||
sealedAtMs: 300,
|
||||
});
|
||||
return prepareCopilotFailureDiagnosisExecution({
|
||||
requestId: 'diagnosis-request-cancel',
|
||||
traceId: 'diagnosis-trace-cancel',
|
||||
source: {
|
||||
runId: 'source-run-cancel',
|
||||
runVersion: 4,
|
||||
runStatus: 'failed',
|
||||
attemptId: 'source-attempt-cancel',
|
||||
attemptStatus: 'failed',
|
||||
attemptFinishedAtMs: 250,
|
||||
logArtifactId: 'source-log-cancel',
|
||||
},
|
||||
toolPlan: tool.plan,
|
||||
bindings,
|
||||
model: {
|
||||
provider: 'provider-primary',
|
||||
model: 'model-diagnosis',
|
||||
modelBoundary: 'external',
|
||||
responseLanguage: 'zh-CN',
|
||||
maxOutputTokens: 256,
|
||||
egressPolicy: {
|
||||
schema: 'qinglong/copilot-model-egress-policy@v1',
|
||||
revision: 'cancel-policy-v1',
|
||||
potentiallySensitiveDataBoundaries: ['external'],
|
||||
maxInputBytes: 64 * 1024,
|
||||
maxOutputTokens: 512,
|
||||
},
|
||||
},
|
||||
plannedAtMs: 400,
|
||||
deadlineAtMs: 8_000,
|
||||
});
|
||||
}
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-cancel',
|
||||
sourceRunId: 'source-run-cancel',
|
||||
requestId: 'diagnosis-request-cancel',
|
||||
mutationId: '11111111-1111-4111-8111-111111111111',
|
||||
eventId: '22222222-2222-4222-8222-222222222222',
|
||||
subject: { type: 'user', id: 'owner-cancel' },
|
||||
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(plan, overrides = {}) {
|
||||
const receipt = createCopilotFailureDiagnosisAdmissionBundle(plan).receipt;
|
||||
const state = {
|
||||
cancellationCalls: [],
|
||||
terminalizeCalls: 0,
|
||||
cancellation: {
|
||||
status: 'accepted',
|
||||
projectId: plan.projectId,
|
||||
runId: plan.runId,
|
||||
runStatus: 'running',
|
||||
runVersion: 4,
|
||||
eventSequence: 4,
|
||||
cancelRequestedAtMs: 500,
|
||||
cancelReason: 'user',
|
||||
},
|
||||
authority: {
|
||||
plan,
|
||||
run: {
|
||||
id: plan.runId,
|
||||
projectId: plan.projectId,
|
||||
status: 'running',
|
||||
version: 4,
|
||||
eventSequence: 4,
|
||||
cancelRequestedAtMs: 500,
|
||||
cancelReason: 'user',
|
||||
},
|
||||
toolStep: {},
|
||||
modelStep: {},
|
||||
modelStartExists: false,
|
||||
observedAtMs: 501,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
const terminalizations = {
|
||||
repository: {
|
||||
async findByRequestId() {
|
||||
return null;
|
||||
},
|
||||
async readAuthority() {
|
||||
return state.authority;
|
||||
},
|
||||
async commit() {
|
||||
throw new Error('not called by injected terminalizer');
|
||||
},
|
||||
},
|
||||
};
|
||||
const service = new CopilotFailureDiagnosisCancellationService({
|
||||
admissions: {
|
||||
async findPlanByRequestId() {
|
||||
return state.plan === undefined ? plan : state.plan;
|
||||
},
|
||||
async findByRequestId() {
|
||||
return state.receipt === undefined ? receipt : state.receipt;
|
||||
},
|
||||
},
|
||||
cancellations: {
|
||||
async requestUserCancellation(value) {
|
||||
state.cancellationCalls.push(value);
|
||||
return state.cancellation;
|
||||
},
|
||||
},
|
||||
terminalizations,
|
||||
async terminalizeBeforeModel(requestId, trigger) {
|
||||
state.terminalizeCalls += 1;
|
||||
if (state.terminalizeError) throw state.terminalizeError;
|
||||
assert.equal(requestId, plan.requestId);
|
||||
assert.deepEqual(trigger, { kind: 'boundary' });
|
||||
return {
|
||||
status: 'created',
|
||||
receipt: {
|
||||
outcome: 'cancelled',
|
||||
finalRunVersion: 7,
|
||||
finalRunEventSequence: 7,
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
return { service, state, receipt };
|
||||
}
|
||||
|
||||
test('resolves the request key to a server-owned Run and terminalizes pre-Model', async () => {
|
||||
const plan = await durablePlan();
|
||||
const { service, state } = fixture(plan);
|
||||
const result = await service.cancel(command());
|
||||
assert.deepEqual(state.cancellationCalls, [
|
||||
{
|
||||
projectId: 'project-cancel',
|
||||
runId: plan.runId,
|
||||
mutationId: '11111111-1111-4111-8111-111111111111',
|
||||
eventId: '22222222-2222-4222-8222-222222222222',
|
||||
subject: { type: 'user', id: 'owner-cancel' },
|
||||
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
||||
},
|
||||
]);
|
||||
assert.equal(state.terminalizeCalls, 1);
|
||||
assert.equal(result.status, 'accepted');
|
||||
assert.equal(result.convergence, 'terminal');
|
||||
assert.equal(result.runStatus, 'cancelled');
|
||||
assert.equal(result.outcome, 'cancelled');
|
||||
assert.equal(result.diagnosisRunId, plan.runId);
|
||||
assert.equal(result.cancelRequestedAtMs, 500);
|
||||
});
|
||||
|
||||
test('masks cross-target requests before writing a cancellation intent', async () => {
|
||||
const plan = await durablePlan();
|
||||
const { service, state } = fixture(plan);
|
||||
await assert.rejects(
|
||||
service.cancel(command({ sourceRunId: 'other-source-run' })),
|
||||
CopilotFailureDiagnosisCancellationNotFoundError,
|
||||
);
|
||||
assert.equal(state.cancellationCalls.length, 0);
|
||||
});
|
||||
|
||||
test('keeps a durable intent pending when Model start wins the race', async () => {
|
||||
const plan = await durablePlan();
|
||||
const conflict =
|
||||
new CopilotFailureDiagnosisPreModelTerminalizationConflictError(
|
||||
'Model started',
|
||||
);
|
||||
const { service, state } = fixture(plan, {
|
||||
terminalizeError: conflict,
|
||||
authority: {
|
||||
plan,
|
||||
run: {
|
||||
id: plan.runId,
|
||||
projectId: plan.projectId,
|
||||
status: 'running',
|
||||
version: 5,
|
||||
eventSequence: 5,
|
||||
cancelRequestedAtMs: 500,
|
||||
cancelReason: 'user',
|
||||
},
|
||||
toolStep: {},
|
||||
modelStep: {},
|
||||
modelStartExists: true,
|
||||
observedAtMs: 501,
|
||||
},
|
||||
});
|
||||
const result = await service.cancel(command());
|
||||
assert.equal(state.terminalizeCalls, 1);
|
||||
assert.equal(result.convergence, 'model_in_flight');
|
||||
assert.equal(result.runStatus, 'running');
|
||||
assert.equal(result.outcome, null);
|
||||
assert.equal(result.runVersion, 5);
|
||||
});
|
||||
|
||||
test('reports the real terminal winner instead of forging cancellation', async () => {
|
||||
const plan = await durablePlan();
|
||||
const { service } = fixture(plan, {
|
||||
terminalizeError:
|
||||
new CopilotFailureDiagnosisPreModelTerminalizationConflictError(),
|
||||
authority: {
|
||||
plan,
|
||||
run: {
|
||||
id: plan.runId,
|
||||
projectId: plan.projectId,
|
||||
status: 'succeeded',
|
||||
version: 7,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: 500,
|
||||
cancelReason: 'user',
|
||||
},
|
||||
toolStep: {},
|
||||
modelStep: {},
|
||||
modelStartExists: true,
|
||||
observedAtMs: 600,
|
||||
},
|
||||
});
|
||||
const result = await service.cancel(command());
|
||||
assert.equal(result.convergence, 'terminal');
|
||||
assert.equal(result.runStatus, 'succeeded');
|
||||
assert.equal(result.outcome, 'succeeded');
|
||||
});
|
||||
|
||||
test('fails closed when plan and admission receipt drift', async () => {
|
||||
const plan = await durablePlan();
|
||||
const base = fixture(plan);
|
||||
const { service, state } = fixture(plan, {
|
||||
receipt: { ...base.receipt, runId: 'diagnosis-run-drift' },
|
||||
});
|
||||
await assert.rejects(
|
||||
service.cancel(command()),
|
||||
CopilotFailureDiagnosisCancellationUnavailableError,
|
||||
);
|
||||
assert.equal(state.cancellationCalls.length, 0);
|
||||
});
|
||||
@@ -40,6 +40,11 @@
|
||||
"require": "./dist/application-runtime/copilot/failureDiagnosisComposition.js",
|
||||
"default": "./dist/application-runtime/copilot/failureDiagnosisComposition.js"
|
||||
},
|
||||
"./copilot-cancellation-production": {
|
||||
"types": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.d.ts",
|
||||
"require": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.js",
|
||||
"default": "./dist/application-runtime/copilot/failureDiagnosisCancellationComposition.js"
|
||||
},
|
||||
"./failure-diagnosis-output-keyring": {
|
||||
"types": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js",
|
||||
@@ -55,6 +60,11 @@
|
||||
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js"
|
||||
},
|
||||
"./copilot-cancellation-route": {
|
||||
"types": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisCancellationRoute.js"
|
||||
},
|
||||
"./http": {
|
||||
"types": "./dist/transport/httpSurface.d.ts",
|
||||
"require": "./dist/transport/httpSurface.js",
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile';
|
||||
import type { DurableModelInvocationCoordinator } from '@qinglong/ai/durable-model-invocation';
|
||||
import { CopilotFailureDiagnosisModelCompletionCoordinator } from '@qinglong/ai/failure-diagnosis-model-execution';
|
||||
import type { CopilotFailureDiagnosisApplicationService } from '@qinglong/ai/failure-diagnosis-application';
|
||||
import type { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation';
|
||||
import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential';
|
||||
import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage';
|
||||
import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority';
|
||||
@@ -35,6 +36,10 @@ import {
|
||||
createProductionClusterCopilotFailureDiagnosisReadService,
|
||||
type CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions,
|
||||
} from './copilot/failureDiagnosisReadComposition';
|
||||
import {
|
||||
createProductionClusterCopilotFailureDiagnosisCancellation,
|
||||
type CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
|
||||
} from './copilot/failureDiagnosisCancellationComposition';
|
||||
|
||||
export interface EnabledProductionClusterAiConfig {
|
||||
readonly enabled: true;
|
||||
@@ -63,6 +68,9 @@ export interface ProductionClusterAiControlApplicationOptions {
|
||||
) => ReturnType<
|
||||
typeof createProductionClusterCopilotFailureDiagnosisReadService
|
||||
>;
|
||||
readonly createCopilotCancellation?: (
|
||||
options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
|
||||
) => Readonly<CopilotFailureDiagnosisCancellationService>;
|
||||
readonly openAiDatabase?: ReturnType<typeof createPostgresDatabaseOpener>;
|
||||
}
|
||||
|
||||
@@ -263,11 +271,15 @@ export async function startProductionClusterAiControlApplication(
|
||||
const createCopilotRead =
|
||||
options.createCopilotRead ??
|
||||
createProductionClusterCopilotFailureDiagnosisReadService;
|
||||
const createCopilotCancellation =
|
||||
options.createCopilotCancellation ??
|
||||
createProductionClusterCopilotFailureDiagnosisCancellation;
|
||||
if (
|
||||
typeof startControl !== 'function' ||
|
||||
typeof bootstrapPrompt !== 'function' ||
|
||||
typeof createCopilot !== 'function' ||
|
||||
typeof createCopilotRead !== 'function' ||
|
||||
typeof createCopilotCancellation !== 'function' ||
|
||||
(options.openAiDatabase !== undefined &&
|
||||
typeof options.openAiDatabase !== 'function')
|
||||
) {
|
||||
@@ -325,6 +337,9 @@ export async function startProductionClusterAiControlApplication(
|
||||
typeof createProductionClusterCopilotFailureDiagnosisReadService
|
||||
>
|
||||
| undefined;
|
||||
let copilotCancellationApplication:
|
||||
| Readonly<CopilotFailureDiagnosisCancellationService>
|
||||
| undefined;
|
||||
let copilotSuccessfulCompletion:
|
||||
| CopilotFailureDiagnosisModelCompletionCoordinator
|
||||
| undefined;
|
||||
@@ -455,6 +470,9 @@ export async function startProductionClusterAiControlApplication(
|
||||
pool: aiDatabase.pool,
|
||||
prepared: preparedCopilot,
|
||||
});
|
||||
copilotCancellationApplication = createCopilotCancellation({
|
||||
pool: aiDatabase.pool,
|
||||
});
|
||||
}
|
||||
controlApplication = await startControl({
|
||||
...options.control,
|
||||
@@ -482,12 +500,14 @@ export async function startProductionClusterAiControlApplication(
|
||||
},
|
||||
}),
|
||||
...(copilotApplication === undefined ||
|
||||
copilotReadApplication === undefined
|
||||
copilotReadApplication === undefined ||
|
||||
copilotCancellationApplication === undefined
|
||||
? {}
|
||||
: {
|
||||
copilotFailureDiagnosis: {
|
||||
capability: copilotApplication,
|
||||
readCapability: copilotReadApplication,
|
||||
cancellationCapability: copilotCancellationApplication,
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
import { CopilotFailureDiagnosisCancellationService } from '@qinglong/ai/failure-diagnosis-cancellation';
|
||||
import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage';
|
||||
import {
|
||||
PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository,
|
||||
terminalizeCopilotFailureDiagnosisBeforeModel,
|
||||
} from '@qinglong/ai/failure-diagnosis-pre-model-terminalization';
|
||||
import {
|
||||
PostgresClusterRunCancellationRepository,
|
||||
type QingLongPostgresPool,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
|
||||
export interface CreateProductionClusterCopilotFailureDiagnosisCancellationOptions {
|
||||
readonly pool: QingLongPostgresPool;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reuses the AI Pool and existing Run/admission ledgers. The cancellation
|
||||
* capability owns no connection, timer, listener or background lifecycle.
|
||||
*/
|
||||
export function createProductionClusterCopilotFailureDiagnosisCancellation(
|
||||
options: CreateProductionClusterCopilotFailureDiagnosisCancellationOptions,
|
||||
): Readonly<CopilotFailureDiagnosisCancellationService> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
typeof options.pool?.query !== 'function' ||
|
||||
typeof options.pool?.connect !== 'function'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Production Cluster Copilot failure diagnosis cancellation dependencies are invalid',
|
||||
);
|
||||
}
|
||||
const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository(
|
||||
options.pool,
|
||||
);
|
||||
const terminalizations =
|
||||
new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository(
|
||||
options.pool,
|
||||
);
|
||||
return new CopilotFailureDiagnosisCancellationService({
|
||||
admissions,
|
||||
cancellations: new PostgresClusterRunCancellationRepository(options.pool),
|
||||
terminalizations: Object.freeze({ repository: terminalizations }),
|
||||
terminalizeBeforeModel: terminalizeCopilotFailureDiagnosisBeforeModel,
|
||||
});
|
||||
}
|
||||
@@ -71,6 +71,10 @@ import {
|
||||
type ClusterCopilotFailureDiagnosisInspectionCapability,
|
||||
type ClusterCopilotFailureDiagnosisOutputReadCapability,
|
||||
} from '../copilot/failure-diagnosis/failureDiagnosisReadRoutes';
|
||||
import {
|
||||
createClusterControlCopilotFailureDiagnosisCancellationRoute,
|
||||
type ClusterCopilotFailureDiagnosisCancellationCapability,
|
||||
} from '../copilot/failure-diagnosis/failureDiagnosisCancellationRoute';
|
||||
|
||||
export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
|
||||
'task.get',
|
||||
@@ -100,6 +104,7 @@ export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS =
|
||||
'copilot.failure_diagnosis.execute',
|
||||
'copilot.failure_diagnosis.read',
|
||||
'copilot.failure_diagnosis.output.read',
|
||||
'copilot.failure_diagnosis.cancel',
|
||||
] as const);
|
||||
|
||||
export interface ClusterCopilotFailureDiagnosisReadCapability
|
||||
@@ -129,6 +134,7 @@ export interface ProductionClusterControlAssemblyOptions {
|
||||
readonly copilotFailureDiagnosis?: Readonly<{
|
||||
readonly capability: ClusterCopilotFailureDiagnosisCapability;
|
||||
readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
|
||||
readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability;
|
||||
}>;
|
||||
readonly workerIngress?: Readonly<{
|
||||
readonly config: EnabledClusterWorkerIngressConfig;
|
||||
@@ -178,6 +184,7 @@ export interface ProductionClusterControlApplicationOptions
|
||||
readonly copilotFailureDiagnosis?: Readonly<{
|
||||
readonly capability: ClusterCopilotFailureDiagnosisCapability;
|
||||
readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
|
||||
readonly cancellationCapability?: ClusterCopilotFailureDiagnosisCancellationCapability;
|
||||
}>;
|
||||
readonly workerIngress?: ProductionClusterWorkerIngressOptions;
|
||||
}
|
||||
@@ -303,6 +310,14 @@ export function createProductionClusterControlApplicationStack(
|
||||
options.copilotFailureDiagnosis.readCapability,
|
||||
),
|
||||
]),
|
||||
...(options.copilotFailureDiagnosis?.cancellationCapability === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlCopilotFailureDiagnosisCancellationRoute(
|
||||
options.copilotFailureDiagnosis.cancellationCapability,
|
||||
createEventId,
|
||||
),
|
||||
]),
|
||||
];
|
||||
const routes = createClusterControlRouteRegistry(routeDefinitions);
|
||||
const expectedRouteCount =
|
||||
@@ -313,7 +328,10 @@ export function createProductionClusterControlApplicationStack(
|
||||
(options.promptOutputRead === undefined ? 0 : 1) +
|
||||
(options.promptExecutionOutputRead === undefined ? 0 : 1) +
|
||||
(options.copilotFailureDiagnosis === undefined ? 0 : 1) +
|
||||
(options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2);
|
||||
(options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2) +
|
||||
(options.copilotFailureDiagnosis?.cancellationCapability === undefined
|
||||
? 0
|
||||
: 1);
|
||||
if (routes.size !== expectedRouteCount) {
|
||||
throw new Error('Production cluster-control route allowlist is incomplete');
|
||||
}
|
||||
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
// Cluster Copilot resolves an external request key before cancelling its Run.
|
||||
import {
|
||||
CopilotFailureDiagnosisCancellationNotFoundError,
|
||||
CopilotFailureDiagnosisCancellationUnavailableError,
|
||||
InvalidCopilotFailureDiagnosisCancellationError,
|
||||
type CopilotFailureDiagnosisCancellationCommand,
|
||||
} from '@qinglong/ai/failure-diagnosis-cancellation';
|
||||
import {
|
||||
CLUSTER_RUN_CANCELLATION_SCHEMA,
|
||||
ClusterRunCancellationFenceRejectedError,
|
||||
ClusterRunCancellationNotFoundError,
|
||||
ClusterRunCancellationUnavailableError,
|
||||
InvalidClusterRunCancellationError,
|
||||
parseClusterRunCancellationRequestBody,
|
||||
} from '@qinglong/runtime-core/cluster-run-cancellation';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA =
|
||||
'qinglong/cluster-copilot-failure-diagnosis-cancellation-response@v1' as const;
|
||||
|
||||
export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE =
|
||||
Object.freeze({
|
||||
method: 'POST' as const,
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/cancellation',
|
||||
operationId: 'copilot.failure_diagnosis.cancel',
|
||||
permission: 'run.stop',
|
||||
projectParameter: 'projectId',
|
||||
});
|
||||
|
||||
export interface ClusterCopilotFailureDiagnosisCancellationCapability {
|
||||
cancel(
|
||||
command: Readonly<CopilotFailureDiagnosisCancellationCommand>,
|
||||
): Promise<unknown>;
|
||||
}
|
||||
|
||||
export type ClusterCopilotFailureDiagnosisCancellationEventIdFactory =
|
||||
() => string;
|
||||
|
||||
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
|
||||
const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']);
|
||||
const STATUSES = new Set(['accepted', 'already_requested', 'already_terminal']);
|
||||
const CANCEL_REASONS = new Set([
|
||||
'user',
|
||||
'policy',
|
||||
'shutdown',
|
||||
'reconcile',
|
||||
'timeout',
|
||||
]);
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function exactRecord(
|
||||
value: unknown,
|
||||
expected: readonly string[],
|
||||
): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
|
||||
const record = value as Record<string, unknown>;
|
||||
const actual = Object.keys(record).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return actual.length === canonical.length &&
|
||||
actual.every((key, index) => key === canonical[index])
|
||||
? record
|
||||
: null;
|
||||
}
|
||||
|
||||
function safeInteger(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && (value as number) >= 0;
|
||||
}
|
||||
|
||||
function projectResult(
|
||||
value: unknown,
|
||||
target: Readonly<{
|
||||
projectId: string;
|
||||
sourceRunId: string;
|
||||
requestId: string;
|
||||
}>,
|
||||
): Readonly<Record<string, unknown>> | null {
|
||||
const candidate = exactRecord(value, [
|
||||
'cancelReason',
|
||||
'cancelRequestedAtMs',
|
||||
'convergence',
|
||||
'diagnosisRunId',
|
||||
'eventSequence',
|
||||
'outcome',
|
||||
'projectId',
|
||||
'requestId',
|
||||
'runStatus',
|
||||
'runVersion',
|
||||
'schema',
|
||||
'sourceRunId',
|
||||
'status',
|
||||
]);
|
||||
if (
|
||||
!candidate ||
|
||||
candidate.schema !==
|
||||
'qinglong/copilot-failure-diagnosis-cancellation-result@v1' ||
|
||||
candidate.projectId !== target.projectId ||
|
||||
candidate.sourceRunId !== target.sourceRunId ||
|
||||
candidate.requestId !== target.requestId ||
|
||||
typeof candidate.diagnosisRunId !== 'string' ||
|
||||
!RUN_ID.test(candidate.diagnosisRunId) ||
|
||||
typeof candidate.status !== 'string' ||
|
||||
!STATUSES.has(candidate.status) ||
|
||||
!safeInteger(candidate.runVersion) ||
|
||||
!safeInteger(candidate.eventSequence) ||
|
||||
candidate.runVersion !== candidate.eventSequence ||
|
||||
!(
|
||||
(candidate.cancelRequestedAtMs === null &&
|
||||
candidate.cancelReason === null) ||
|
||||
(safeInteger(candidate.cancelRequestedAtMs) &&
|
||||
typeof candidate.cancelReason === 'string' &&
|
||||
CANCEL_REASONS.has(candidate.cancelReason))
|
||||
)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
if (candidate.convergence === 'model_in_flight') {
|
||||
if (
|
||||
candidate.runStatus !== 'running' ||
|
||||
candidate.outcome !== null ||
|
||||
candidate.cancelRequestedAtMs === null
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
} else if (candidate.convergence === 'terminal') {
|
||||
if (
|
||||
typeof candidate.runStatus !== 'string' ||
|
||||
!OUTCOMES.has(candidate.runStatus) ||
|
||||
candidate.outcome !== candidate.runStatus
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
return Object.freeze({
|
||||
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA,
|
||||
status: candidate.status,
|
||||
convergence: candidate.convergence,
|
||||
projectId: target.projectId,
|
||||
sourceRunId: target.sourceRunId,
|
||||
requestId: target.requestId,
|
||||
diagnosisRunId: candidate.diagnosisRunId,
|
||||
runStatus: candidate.runStatus,
|
||||
outcome: candidate.outcome,
|
||||
runVersion: candidate.runVersion,
|
||||
eventSequence: candidate.eventSequence,
|
||||
cancelRequestedAtMs: candidate.cancelRequestedAtMs,
|
||||
cancelReason: candidate.cancelReason,
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterControlCopilotFailureDiagnosisCancellationRoute(
|
||||
capability: ClusterCopilotFailureDiagnosisCancellationCapability,
|
||||
createEventId: ClusterCopilotFailureDiagnosisCancellationEventIdFactory,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (
|
||||
!capability ||
|
||||
typeof capability.cancel !== 'function' ||
|
||||
typeof createEventId !== 'function'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster-control Copilot failure diagnosis cancellation route is invalid',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
let body;
|
||||
try {
|
||||
body = parseClusterRunCancellationRequestBody(authorized.request.body);
|
||||
} catch (error) {
|
||||
return error instanceof InvalidClusterRunCancellationError
|
||||
? response(400, {
|
||||
code: 'invalid_copilot_failure_diagnosis_cancellation_request',
|
||||
schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
|
||||
})
|
||||
: response(503, {
|
||||
code: 'copilot_failure_diagnosis_cancellation_unavailable',
|
||||
});
|
||||
}
|
||||
const projectId = authorized.projectId;
|
||||
const sourceRunId = parameters.runId;
|
||||
const requestId = parameters.requestId;
|
||||
if (
|
||||
projectId === null ||
|
||||
typeof sourceRunId !== 'string' ||
|
||||
!RUN_ID.test(sourceRunId) ||
|
||||
typeof requestId !== 'string' ||
|
||||
!IDENTITY.test(requestId) ||
|
||||
!authorized.policyFence ||
|
||||
authorized.policyFence.bindingVersion === null
|
||||
) {
|
||||
return response(503, {
|
||||
code: 'copilot_failure_diagnosis_cancellation_unavailable',
|
||||
});
|
||||
}
|
||||
const target = Object.freeze({ projectId, sourceRunId, requestId });
|
||||
try {
|
||||
const result = await capability.cancel({
|
||||
...target,
|
||||
mutationId: body.mutationId,
|
||||
eventId: createEventId(),
|
||||
subject: authorized.principal.subject,
|
||||
policyFence: authorized.policyFence,
|
||||
});
|
||||
const view = projectResult(result, target);
|
||||
if (!view) {
|
||||
return response(503, {
|
||||
code: 'copilot_failure_diagnosis_cancellation_unavailable',
|
||||
});
|
||||
}
|
||||
return response(view.status === 'accepted' ? 202 : 200, view);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof CopilotFailureDiagnosisCancellationNotFoundError ||
|
||||
error instanceof ClusterRunCancellationNotFoundError
|
||||
) {
|
||||
return response(404, {
|
||||
code: 'copilot_failure_diagnosis_not_found',
|
||||
});
|
||||
}
|
||||
if (error instanceof ClusterRunCancellationFenceRejectedError) {
|
||||
return response(409, {
|
||||
code: 'copilot_failure_diagnosis_cancellation_fence_rejected',
|
||||
reason: error.reason,
|
||||
});
|
||||
}
|
||||
if (
|
||||
error instanceof InvalidCopilotFailureDiagnosisCancellationError ||
|
||||
error instanceof
|
||||
CopilotFailureDiagnosisCancellationUnavailableError ||
|
||||
error instanceof InvalidClusterRunCancellationError ||
|
||||
error instanceof ClusterRunCancellationUnavailableError
|
||||
) {
|
||||
return response(503, {
|
||||
code: 'copilot_failure_diagnosis_cancellation_unavailable',
|
||||
});
|
||||
}
|
||||
return response(503, {
|
||||
code: 'copilot_failure_diagnosis_cancellation_unavailable',
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -169,9 +169,11 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
|
||||
const artifactStore = { put() {}, inspect() {}, readLogRange() {} };
|
||||
const copilot = Object.freeze({ execute() {} });
|
||||
const copilotRead = Object.freeze({ inspect() {}, readOutput() {} });
|
||||
const copilotCancellation = Object.freeze({ cancel() {} });
|
||||
let registeredSink;
|
||||
let created;
|
||||
let createdRead;
|
||||
let createdCancellation;
|
||||
let controlOptions;
|
||||
try {
|
||||
await Promise.all([
|
||||
@@ -233,6 +235,10 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
|
||||
createdRead = options;
|
||||
return copilotRead;
|
||||
},
|
||||
createCopilotCancellation(options) {
|
||||
createdCancellation = options;
|
||||
return copilotCancellation;
|
||||
},
|
||||
async startControl(options) {
|
||||
controlOptions = options;
|
||||
return {
|
||||
@@ -257,11 +263,16 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
|
||||
assert.equal(created.artifactStore, artifactStore);
|
||||
assert.equal(createdRead.pool, fakePool);
|
||||
assert.equal(typeof createdRead.prepared.outputKeys.resolve, 'function');
|
||||
assert.equal(createdCancellation.pool, fakePool);
|
||||
assert.equal(controlOptions.copilotFailureDiagnosis.capability, copilot);
|
||||
assert.equal(
|
||||
controlOptions.copilotFailureDiagnosis.readCapability,
|
||||
copilotRead,
|
||||
);
|
||||
assert.equal(
|
||||
controlOptions.copilotFailureDiagnosis.cancellationCapability,
|
||||
copilotCancellation,
|
||||
);
|
||||
assert.equal(await application.stop(), 'stopped');
|
||||
} finally {
|
||||
config.fill(0);
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
CopilotFailureDiagnosisCancellationNotFoundError,
|
||||
} = require('@qinglong/ai/failure-diagnosis-cancellation');
|
||||
const {
|
||||
CLUSTER_RUN_CANCELLATION_SCHEMA,
|
||||
ClusterRunCancellationFenceRejectedError,
|
||||
} = require('@qinglong/runtime-core/cluster-run-cancellation');
|
||||
const {
|
||||
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA,
|
||||
CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE,
|
||||
createClusterControlCopilotFailureDiagnosisCancellationRoute,
|
||||
} = require('@qinglong/cluster-control/copilot-cancellation-route');
|
||||
|
||||
function authorized(body, overrides = {}) {
|
||||
return {
|
||||
request: {
|
||||
requestId: 'transport-request-1',
|
||||
method: 'POST',
|
||||
path: '/api/v3/projects/project-1/runs/source-run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation',
|
||||
query: {},
|
||||
headers: {},
|
||||
signal: new AbortController().signal,
|
||||
body,
|
||||
},
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'owner-1' },
|
||||
authenticationId: 'credential-1',
|
||||
authenticatedAtMs: 1,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
operationId: 'copilot.failure_diagnosis.cancel',
|
||||
permission: 'run.stop',
|
||||
projectId: 'project-1',
|
||||
policyFence: { projectVersion: 3, bindingVersion: 7 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const parameters = {
|
||||
projectId: 'project-1',
|
||||
runId: 'source-run-1',
|
||||
requestId: 'diagnosis-request-1',
|
||||
};
|
||||
|
||||
function body(overrides = {}) {
|
||||
return {
|
||||
schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
|
||||
mutationId: '11111111-1111-4111-8111-111111111111',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function result(overrides = {}) {
|
||||
return {
|
||||
schema: 'qinglong/copilot-failure-diagnosis-cancellation-result@v1',
|
||||
status: 'accepted',
|
||||
convergence: 'terminal',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'diagnosis-request-1',
|
||||
diagnosisRunId: 'diagnosis-run-1',
|
||||
runStatus: 'cancelled',
|
||||
outcome: 'cancelled',
|
||||
runVersion: 7,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: 500,
|
||||
cancelReason: 'user',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('defines an exact run.stop route and passes only fenced target facts', async () => {
|
||||
let command;
|
||||
const request = authorized(body());
|
||||
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
|
||||
{
|
||||
async cancel(value) {
|
||||
command = value;
|
||||
return result();
|
||||
},
|
||||
},
|
||||
() => '22222222-2222-4222-8222-222222222222',
|
||||
);
|
||||
const response = await route.handle(request, parameters);
|
||||
assert.deepEqual(
|
||||
CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_ROUTE,
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/cancellation',
|
||||
operationId: 'copilot.failure_diagnosis.cancel',
|
||||
permission: 'run.stop',
|
||||
projectParameter: 'projectId',
|
||||
},
|
||||
);
|
||||
assert.deepEqual(command, {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'diagnosis-request-1',
|
||||
mutationId: '11111111-1111-4111-8111-111111111111',
|
||||
eventId: '22222222-2222-4222-8222-222222222222',
|
||||
subject: request.principal.subject,
|
||||
policyFence: request.policyFence,
|
||||
});
|
||||
assert.deepEqual(response, {
|
||||
statusCode: 202,
|
||||
body: {
|
||||
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESPONSE_SCHEMA,
|
||||
status: 'accepted',
|
||||
convergence: 'terminal',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
requestId: 'diagnosis-request-1',
|
||||
diagnosisRunId: 'diagnosis-run-1',
|
||||
runStatus: 'cancelled',
|
||||
outcome: 'cancelled',
|
||||
runVersion: 7,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: 500,
|
||||
cancelReason: 'user',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('projects an in-flight durable intent without claiming Provider abort', async () => {
|
||||
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
|
||||
{
|
||||
async cancel() {
|
||||
return result({
|
||||
status: 'already_requested',
|
||||
convergence: 'model_in_flight',
|
||||
runStatus: 'running',
|
||||
outcome: null,
|
||||
runVersion: 6,
|
||||
eventSequence: 6,
|
||||
});
|
||||
},
|
||||
},
|
||||
() => '22222222-2222-4222-8222-222222222222',
|
||||
);
|
||||
const response = await route.handle(authorized(body()), parameters);
|
||||
assert.equal(response.statusCode, 200);
|
||||
assert.equal(response.body.convergence, 'model_in_flight');
|
||||
assert.equal(response.body.runStatus, 'running');
|
||||
assert.equal(response.body.outcome, null);
|
||||
assert.equal('providerAborted' in response.body, false);
|
||||
});
|
||||
|
||||
test('rejects non-exact bodies before invoking the capability', async () => {
|
||||
let calls = 0;
|
||||
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
|
||||
{
|
||||
async cancel() {
|
||||
calls += 1;
|
||||
return result();
|
||||
},
|
||||
},
|
||||
() => '22222222-2222-4222-8222-222222222222',
|
||||
);
|
||||
for (const value of [
|
||||
null,
|
||||
{},
|
||||
body({ runId: 'caller-selected' }),
|
||||
body({ reason: 'timeout' }),
|
||||
body({ mutationId: '' }),
|
||||
]) {
|
||||
const response = await route.handle(authorized(value), parameters);
|
||||
assert.equal(response.statusCode, 400);
|
||||
}
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
|
||||
test('fails closed on widened or identity-drifted capability results', async () => {
|
||||
for (const value of [
|
||||
result({ sourceRunId: 'other' }),
|
||||
result({ privateProvider: 'must-not-cross' }),
|
||||
result({ runVersion: 8 }),
|
||||
result({ convergence: 'model_in_flight' }),
|
||||
]) {
|
||||
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
|
||||
{
|
||||
async cancel() {
|
||||
return value;
|
||||
},
|
||||
},
|
||||
() => '22222222-2222-4222-8222-222222222222',
|
||||
);
|
||||
const response = await route.handle(authorized(body()), parameters);
|
||||
assert.deepEqual(response, {
|
||||
statusCode: 503,
|
||||
body: {
|
||||
code: 'copilot_failure_diagnosis_cancellation_unavailable',
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('maps hidden targets, Policy races and storage failures to stable codes', async () => {
|
||||
for (const [error, statusCode, code] of [
|
||||
[
|
||||
new CopilotFailureDiagnosisCancellationNotFoundError(),
|
||||
404,
|
||||
'copilot_failure_diagnosis_not_found',
|
||||
],
|
||||
[
|
||||
new ClusterRunCancellationFenceRejectedError('authorization_changed'),
|
||||
409,
|
||||
'copilot_failure_diagnosis_cancellation_fence_rejected',
|
||||
],
|
||||
[
|
||||
new Error('private storage detail'),
|
||||
503,
|
||||
'copilot_failure_diagnosis_cancellation_unavailable',
|
||||
],
|
||||
]) {
|
||||
const route = createClusterControlCopilotFailureDiagnosisCancellationRoute(
|
||||
{
|
||||
async cancel() {
|
||||
throw error;
|
||||
},
|
||||
},
|
||||
() => '22222222-2222-4222-8222-222222222222',
|
||||
);
|
||||
const response = await route.handle(authorized(body()), parameters);
|
||||
assert.equal(response.statusCode, statusCode);
|
||||
assert.equal(response.body.code, code);
|
||||
assert.equal(JSON.stringify(response).includes('private'), false);
|
||||
}
|
||||
});
|
||||
@@ -732,6 +732,7 @@ test('optionally exposes Prompt execution behind shared admission and policy', a
|
||||
'copilot.failure_diagnosis.execute',
|
||||
'copilot.failure_diagnosis.read',
|
||||
'copilot.failure_diagnosis.output.read',
|
||||
'copilot.failure_diagnosis.cancel',
|
||||
]);
|
||||
const response = await invoke(
|
||||
stack,
|
||||
@@ -762,6 +763,7 @@ test('optionally exposes Prompt execution behind shared admission and policy', a
|
||||
test('optionally exposes Copilot diagnosis behind shared authentication, Policy and audit', async () => {
|
||||
const { events, input } = fixture();
|
||||
let command;
|
||||
let cancellationCommand;
|
||||
const capability = {
|
||||
async execute(value) {
|
||||
command = value;
|
||||
@@ -803,11 +805,30 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy
|
||||
requestId: value.requestId,
|
||||
};
|
||||
},
|
||||
async cancel(value) {
|
||||
cancellationCommand = value;
|
||||
return {
|
||||
schema: 'qinglong/copilot-failure-diagnosis-cancellation-result@v1',
|
||||
status: 'accepted',
|
||||
convergence: 'terminal',
|
||||
projectId: value.projectId,
|
||||
sourceRunId: value.sourceRunId,
|
||||
requestId: value.requestId,
|
||||
diagnosisRunId: 'diagnosis-run-1',
|
||||
runStatus: 'cancelled',
|
||||
outcome: 'cancelled',
|
||||
runVersion: 7,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: 2_000,
|
||||
cancelReason: 'user',
|
||||
};
|
||||
},
|
||||
};
|
||||
const stack = createProductionClusterControlApplicationStack(input, {
|
||||
copilotFailureDiagnosis: {
|
||||
capability,
|
||||
readCapability: capability,
|
||||
cancellationCapability: capability,
|
||||
},
|
||||
});
|
||||
const result = await invoke(
|
||||
@@ -849,6 +870,25 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy
|
||||
);
|
||||
assert.equal(inspection.statusCode, 404);
|
||||
assert.equal(output.statusCode, 404);
|
||||
const cancellation = await invoke(
|
||||
stack,
|
||||
metadata(
|
||||
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation',
|
||||
'POST',
|
||||
{
|
||||
schema: 'qinglong/run-cancellation@v1',
|
||||
mutationId: '00000000-0000-4000-8000-000000000099',
|
||||
},
|
||||
),
|
||||
);
|
||||
assert.equal(cancellation.statusCode, 202);
|
||||
assert.equal(cancellationCommand.projectId, 'project-1');
|
||||
assert.equal(cancellationCommand.sourceRunId, 'run-1');
|
||||
assert.equal(cancellationCommand.requestId, 'diagnosis-request-1');
|
||||
assert.deepEqual(cancellationCommand.policyFence, {
|
||||
projectVersion: 3,
|
||||
bindingVersion: 7,
|
||||
});
|
||||
assert.equal(
|
||||
events.includes('audit:copilot.failure_diagnosis.read:allowed'),
|
||||
true,
|
||||
@@ -857,6 +897,10 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy
|
||||
events.includes('audit:copilot.failure_diagnosis.output.read:allowed'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
events.includes('audit:copilot.failure_diagnosis.cancel:allowed'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('keeps the Copilot route absent by default and never invokes it after Policy denial', async () => {
|
||||
@@ -879,6 +923,7 @@ test('keeps the Copilot route absent by default and never invokes it after Polic
|
||||
for (const path of [
|
||||
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1',
|
||||
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/output',
|
||||
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/cancellation',
|
||||
]) {
|
||||
await assert.rejects(
|
||||
defaultStack.admission.prepare(metadata(path)),
|
||||
|
||||
Reference in New Issue
Block a user