mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 19:29:13 +08:00
feat(ql3): add fenced copilot diagnosis cancellation
This commit is contained in:
whyour
@@ -65,6 +65,11 @@
|
||||
"require": "./dist/copilot/failure-diagnosis/read-model/service.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/read-model/service.js"
|
||||
},
|
||||
"./failure-diagnosis-cancellation": {
|
||||
"types": "./dist/copilot/failure-diagnosis/cancellation/service.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/cancellation/service.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/cancellation/service.js"
|
||||
},
|
||||
"./failure-diagnosis-pre-model-terminalization": {
|
||||
"types": "./dist/copilot/failure-diagnosis/preModelTerminalization.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/preModelTerminalization.js",
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
import { RUN_STATUSES, type RunStatus } from '@qinglong/runtime-core';
|
||||
import type {
|
||||
ClusterRunCancellationRepository,
|
||||
ClusterRunCancellationResult,
|
||||
} from '@qinglong/runtime-core/cluster-run-cancellation';
|
||||
import type {
|
||||
SecurityPolicyFence,
|
||||
SecuritySubject,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
|
||||
import type {
|
||||
CopilotFailureDiagnosisAdmissionRepository,
|
||||
CopilotFailureDiagnosisAdmissionReceipt,
|
||||
CopilotFailureDiagnosisExecutionPlan,
|
||||
} from '../admission/contracts';
|
||||
import { normalizeCopilotFailureDiagnosisAdmissionReceipt } from '../admission/durableEvidence';
|
||||
import { normalizeCopilotFailureDiagnosisExecutionPlan } from '../admission/plan';
|
||||
import {
|
||||
terminalizeCopilotFailureDiagnosisBeforeModel,
|
||||
type CopilotFailureDiagnosisPreModelTerminalizationDependencies,
|
||||
} from '../terminalization/coordinator';
|
||||
import {
|
||||
CopilotFailureDiagnosisPreModelTerminalizationConflictError,
|
||||
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError,
|
||||
CopilotFailureDiagnosisPreModelTerminalizationUnavailableError,
|
||||
} from '../terminalization/contracts';
|
||||
|
||||
export const COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA =
|
||||
'qinglong/copilot-failure-diagnosis-cancellation-result@v1' as const;
|
||||
|
||||
export interface CopilotFailureDiagnosisCancellationCommand {
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly requestId: string;
|
||||
readonly mutationId: string;
|
||||
readonly eventId: string;
|
||||
readonly subject: Readonly<SecuritySubject>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisCancellationResult {
|
||||
readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA;
|
||||
readonly status: ClusterRunCancellationResult['status'];
|
||||
readonly convergence: 'terminal' | 'model_in_flight';
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly requestId: string;
|
||||
readonly diagnosisRunId: string;
|
||||
readonly runStatus: RunStatus;
|
||||
readonly outcome: 'succeeded' | 'failed' | 'timed_out' | 'cancelled' | null;
|
||||
readonly runVersion: number;
|
||||
readonly eventSequence: number;
|
||||
readonly cancelRequestedAtMs: number | null;
|
||||
readonly cancelReason:
|
||||
| 'user'
|
||||
| 'policy'
|
||||
| 'shutdown'
|
||||
| 'reconcile'
|
||||
| 'timeout'
|
||||
| null;
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisCancellationDependencies {
|
||||
readonly admissions: Pick<
|
||||
CopilotFailureDiagnosisAdmissionRepository,
|
||||
'findByRequestId' | 'findPlanByRequestId'
|
||||
>;
|
||||
readonly cancellations: ClusterRunCancellationRepository;
|
||||
readonly terminalizations: CopilotFailureDiagnosisPreModelTerminalizationDependencies;
|
||||
readonly terminalizeBeforeModel?: typeof terminalizeCopilotFailureDiagnosisBeforeModel;
|
||||
}
|
||||
|
||||
export class InvalidCopilotFailureDiagnosisCancellationError extends TypeError {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_INVALID';
|
||||
|
||||
constructor() {
|
||||
super('Copilot failure diagnosis cancellation is invalid');
|
||||
this.name = 'InvalidCopilotFailureDiagnosisCancellationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisCancellationNotFoundError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_NOT_FOUND';
|
||||
|
||||
constructor() {
|
||||
super('Copilot failure diagnosis cancellation target does not exist');
|
||||
this.name = 'CopilotFailureDiagnosisCancellationNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisCancellationUnavailableError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Copilot failure diagnosis cancellation is unavailable', options);
|
||||
this.name = 'CopilotFailureDiagnosisCancellationUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
|
||||
const TERMINAL = new Set<RunStatus>([
|
||||
'succeeded',
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
]);
|
||||
|
||||
function unavailable(cause?: unknown): never {
|
||||
throw new CopilotFailureDiagnosisCancellationUnavailableError({
|
||||
cause: cause instanceof Error ? cause : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): boolean {
|
||||
const actual = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
return (
|
||||
actual.length === canonical.length &&
|
||||
actual.every((key, index) => key === canonical[index])
|
||||
);
|
||||
}
|
||||
|
||||
function command(
|
||||
value: CopilotFailureDiagnosisCancellationCommand,
|
||||
): Readonly<CopilotFailureDiagnosisCancellationCommand> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!exactKeys(value, [
|
||||
'eventId',
|
||||
'mutationId',
|
||||
'policyFence',
|
||||
'projectId',
|
||||
'requestId',
|
||||
'sourceRunId',
|
||||
'subject',
|
||||
]) ||
|
||||
!IDENTITY.test(value.projectId) ||
|
||||
!RUN_ID.test(value.sourceRunId) ||
|
||||
!IDENTITY.test(value.requestId) ||
|
||||
!IDENTITY.test(value.mutationId) ||
|
||||
!IDENTITY.test(value.eventId) ||
|
||||
!value.subject ||
|
||||
typeof value.subject !== 'object' ||
|
||||
!value.policyFence ||
|
||||
typeof value.policyFence !== 'object'
|
||||
) {
|
||||
throw new InvalidCopilotFailureDiagnosisCancellationError();
|
||||
}
|
||||
return Object.freeze({ ...value });
|
||||
}
|
||||
|
||||
function evidenceMatches(
|
||||
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
|
||||
receipt: Readonly<CopilotFailureDiagnosisAdmissionReceipt>,
|
||||
): boolean {
|
||||
return (
|
||||
receipt.requestId === plan.requestId &&
|
||||
receipt.planDigest === plan.planDigest &&
|
||||
receipt.runId === plan.runId &&
|
||||
receipt.sourceRunId === plan.source.runId &&
|
||||
receipt.sourceRunVersion === plan.source.runVersion &&
|
||||
receipt.sourceAttemptId === plan.source.attemptId &&
|
||||
receipt.toolStepRunId === plan.toolStepRunId &&
|
||||
receipt.modelStepRunId === plan.modelStepRunId
|
||||
);
|
||||
}
|
||||
|
||||
function targetMatches(
|
||||
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
|
||||
value: Readonly<CopilotFailureDiagnosisCancellationCommand>,
|
||||
): boolean {
|
||||
return (
|
||||
plan.projectId === value.projectId &&
|
||||
plan.source.runId === value.sourceRunId &&
|
||||
plan.requestId === value.requestId
|
||||
);
|
||||
}
|
||||
|
||||
function result(
|
||||
target: Readonly<CopilotFailureDiagnosisCancellationCommand>,
|
||||
cancellation: Readonly<ClusterRunCancellationResult>,
|
||||
state: Readonly<{
|
||||
convergence: CopilotFailureDiagnosisCancellationResult['convergence'];
|
||||
runStatus: RunStatus;
|
||||
runVersion: number;
|
||||
eventSequence: number;
|
||||
cancelRequestedAtMs?: number;
|
||||
cancelReason?: NonNullable<
|
||||
CopilotFailureDiagnosisCancellationResult['cancelReason']
|
||||
>;
|
||||
}>,
|
||||
): Readonly<CopilotFailureDiagnosisCancellationResult> {
|
||||
if (
|
||||
cancellation.projectId !== target.projectId ||
|
||||
!RUN_STATUSES.includes(state.runStatus) ||
|
||||
!Number.isSafeInteger(state.runVersion) ||
|
||||
state.runVersion < 0 ||
|
||||
!Number.isSafeInteger(state.eventSequence) ||
|
||||
state.eventSequence < 0 ||
|
||||
(state.convergence === 'terminal') !== TERMINAL.has(state.runStatus) ||
|
||||
(state.convergence === 'model_in_flight' && state.runStatus !== 'running')
|
||||
) {
|
||||
return unavailable();
|
||||
}
|
||||
return Object.freeze({
|
||||
schema: COPILOT_FAILURE_DIAGNOSIS_CANCELLATION_RESULT_SCHEMA,
|
||||
status: cancellation.status,
|
||||
convergence: state.convergence,
|
||||
projectId: target.projectId,
|
||||
sourceRunId: target.sourceRunId,
|
||||
requestId: target.requestId,
|
||||
diagnosisRunId: cancellation.runId,
|
||||
runStatus: state.runStatus,
|
||||
outcome: TERMINAL.has(state.runStatus)
|
||||
? (state.runStatus as 'succeeded' | 'failed' | 'timed_out' | 'cancelled')
|
||||
: null,
|
||||
runVersion: state.runVersion,
|
||||
eventSequence: state.eventSequence,
|
||||
cancelRequestedAtMs: state.cancelRequestedAtMs ?? null,
|
||||
cancelReason: state.cancelReason ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves an external request key to its server-owned diagnosis Run, writes
|
||||
* one ordinary Run cancellation intent, then converges only while no Model
|
||||
* invocation start exists.
|
||||
*/
|
||||
export class CopilotFailureDiagnosisCancellationService {
|
||||
readonly #dependencies: Readonly<CopilotFailureDiagnosisCancellationDependencies>;
|
||||
readonly #terminalize: typeof terminalizeCopilotFailureDiagnosisBeforeModel;
|
||||
|
||||
constructor(dependencies: CopilotFailureDiagnosisCancellationDependencies) {
|
||||
if (
|
||||
!dependencies ||
|
||||
typeof dependencies !== 'object' ||
|
||||
Array.isArray(dependencies) ||
|
||||
typeof dependencies.admissions?.findByRequestId !== 'function' ||
|
||||
typeof dependencies.admissions?.findPlanByRequestId !== 'function' ||
|
||||
typeof dependencies.cancellations?.requestUserCancellation !==
|
||||
'function' ||
|
||||
typeof dependencies.terminalizations?.repository?.findByRequestId !==
|
||||
'function' ||
|
||||
typeof dependencies.terminalizations?.repository?.readAuthority !==
|
||||
'function' ||
|
||||
typeof dependencies.terminalizations?.repository?.commit !== 'function' ||
|
||||
(dependencies.terminalizeBeforeModel !== undefined &&
|
||||
typeof dependencies.terminalizeBeforeModel !== 'function')
|
||||
) {
|
||||
throw new InvalidCopilotFailureDiagnosisCancellationError();
|
||||
}
|
||||
this.#dependencies = Object.freeze({ ...dependencies });
|
||||
this.#terminalize =
|
||||
dependencies.terminalizeBeforeModel ??
|
||||
terminalizeCopilotFailureDiagnosisBeforeModel;
|
||||
}
|
||||
|
||||
async cancel(
|
||||
value: CopilotFailureDiagnosisCancellationCommand,
|
||||
): Promise<Readonly<CopilotFailureDiagnosisCancellationResult>> {
|
||||
const target = command(value);
|
||||
let plan: Readonly<CopilotFailureDiagnosisExecutionPlan> | null;
|
||||
let receipt: Readonly<CopilotFailureDiagnosisAdmissionReceipt> | null;
|
||||
try {
|
||||
const located = await Promise.all([
|
||||
this.#dependencies.admissions.findPlanByRequestId(target.requestId),
|
||||
this.#dependencies.admissions.findByRequestId(target.requestId),
|
||||
]);
|
||||
plan = located[0]
|
||||
? normalizeCopilotFailureDiagnosisExecutionPlan(located[0])
|
||||
: null;
|
||||
receipt = located[1]
|
||||
? normalizeCopilotFailureDiagnosisAdmissionReceipt(located[1])
|
||||
: null;
|
||||
} catch (cause) {
|
||||
return unavailable(cause);
|
||||
}
|
||||
if (!plan || !receipt || !targetMatches(plan, target)) {
|
||||
throw new CopilotFailureDiagnosisCancellationNotFoundError();
|
||||
}
|
||||
if (!evidenceMatches(plan, receipt)) return unavailable();
|
||||
|
||||
const cancellation =
|
||||
await this.#dependencies.cancellations.requestUserCancellation({
|
||||
projectId: target.projectId,
|
||||
runId: plan.runId,
|
||||
mutationId: target.mutationId,
|
||||
eventId: target.eventId,
|
||||
subject: target.subject,
|
||||
policyFence: target.policyFence,
|
||||
});
|
||||
if (
|
||||
cancellation.projectId !== target.projectId ||
|
||||
cancellation.runId !== plan.runId
|
||||
) {
|
||||
return unavailable();
|
||||
}
|
||||
if (TERMINAL.has(cancellation.runStatus)) {
|
||||
return result(target, cancellation, {
|
||||
convergence: 'terminal',
|
||||
runStatus: cancellation.runStatus,
|
||||
runVersion: cancellation.runVersion,
|
||||
eventSequence: cancellation.eventSequence,
|
||||
...(cancellation.cancelRequestedAtMs === undefined
|
||||
? {}
|
||||
: { cancelRequestedAtMs: cancellation.cancelRequestedAtMs }),
|
||||
...(cancellation.cancelReason === undefined
|
||||
? {}
|
||||
: { cancelReason: cancellation.cancelReason }),
|
||||
});
|
||||
}
|
||||
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
try {
|
||||
const terminalized = await this.#terminalize(
|
||||
target.requestId,
|
||||
{ kind: 'boundary' },
|
||||
this.#dependencies.terminalizations,
|
||||
);
|
||||
return result(target, cancellation, {
|
||||
convergence: 'terminal',
|
||||
runStatus: terminalized.receipt.outcome,
|
||||
runVersion: terminalized.receipt.finalRunVersion,
|
||||
eventSequence: terminalized.receipt.finalRunEventSequence,
|
||||
...(cancellation.cancelRequestedAtMs === undefined
|
||||
? {}
|
||||
: { cancelRequestedAtMs: cancellation.cancelRequestedAtMs }),
|
||||
...(cancellation.cancelReason === undefined
|
||||
? {}
|
||||
: { cancelReason: cancellation.cancelReason }),
|
||||
});
|
||||
} catch (cause) {
|
||||
if (
|
||||
!(
|
||||
cause instanceof
|
||||
CopilotFailureDiagnosisPreModelTerminalizationConflictError
|
||||
) &&
|
||||
!(
|
||||
cause instanceof
|
||||
CopilotFailureDiagnosisPreModelTerminalizationNotReadyError
|
||||
)
|
||||
) {
|
||||
if (
|
||||
cause instanceof
|
||||
CopilotFailureDiagnosisPreModelTerminalizationUnavailableError
|
||||
) {
|
||||
return unavailable(cause);
|
||||
}
|
||||
return unavailable(cause);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const authority =
|
||||
await this.#dependencies.terminalizations.repository.readAuthority(
|
||||
target.requestId,
|
||||
);
|
||||
if (
|
||||
!targetMatches(authority.plan, target) ||
|
||||
authority.plan.planDigest !== plan.planDigest ||
|
||||
authority.run.id !== plan.runId ||
|
||||
authority.run.projectId !== target.projectId ||
|
||||
authority.run.version !== authority.run.eventSequence
|
||||
) {
|
||||
return unavailable();
|
||||
}
|
||||
if (TERMINAL.has(authority.run.status)) {
|
||||
return result(target, cancellation, {
|
||||
convergence: 'terminal',
|
||||
runStatus: authority.run.status,
|
||||
runVersion: authority.run.version,
|
||||
eventSequence: authority.run.eventSequence,
|
||||
...(authority.run.cancelRequestedAtMs === undefined
|
||||
? {}
|
||||
: { cancelRequestedAtMs: authority.run.cancelRequestedAtMs }),
|
||||
...(authority.run.cancelReason === undefined
|
||||
? {}
|
||||
: { cancelReason: authority.run.cancelReason }),
|
||||
});
|
||||
}
|
||||
if (
|
||||
authority.run.status === 'running' &&
|
||||
authority.modelStartExists &&
|
||||
authority.run.cancelRequestedAtMs !== undefined &&
|
||||
authority.run.cancelReason !== undefined
|
||||
) {
|
||||
return result(target, cancellation, {
|
||||
convergence: 'model_in_flight',
|
||||
runStatus: 'running',
|
||||
runVersion: authority.run.version,
|
||||
eventSequence: authority.run.eventSequence,
|
||||
cancelRequestedAtMs: authority.run.cancelRequestedAtMs,
|
||||
cancelReason: authority.run.cancelReason,
|
||||
});
|
||||
}
|
||||
if (
|
||||
attempt === 0 &&
|
||||
authority.run.status === 'running' &&
|
||||
!authority.modelStartExists &&
|
||||
authority.run.cancelRequestedAtMs !== undefined &&
|
||||
authority.run.cancelReason !== undefined
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
return unavailable();
|
||||
} catch (cause) {
|
||||
return unavailable(cause);
|
||||
}
|
||||
}
|
||||
return unavailable();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||
BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION,
|
||||
createBuiltInRunLogExcerptToolHandlerBinding,
|
||||
} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool');
|
||||
const {
|
||||
createPluginPackageResourceGenerationFromReferences,
|
||||
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
|
||||
const {
|
||||
createProjectToolDefinitionSnapshot,
|
||||
projectToolDefinitionRegistry,
|
||||
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
|
||||
const {
|
||||
prepareToolInvocation,
|
||||
} = require('@qinglong/runtime-core/tool-registry');
|
||||
const {
|
||||
TrustedToolHandlerBindingRegistry,
|
||||
createTrustedToolInvocationPlan,
|
||||
} = require('@qinglong/runtime-core/trusted-tool-invocation');
|
||||
const {
|
||||
CopilotFailureDiagnosisCancellationNotFoundError,
|
||||
CopilotFailureDiagnosisCancellationService,
|
||||
CopilotFailureDiagnosisCancellationUnavailableError,
|
||||
} = require('@qinglong/ai/failure-diagnosis-cancellation');
|
||||
const {
|
||||
createCopilotFailureDiagnosisAdmissionBundle,
|
||||
prepareCopilotFailureDiagnosisExecution,
|
||||
} = require('@qinglong/ai/failure-diagnosis-execution-admission');
|
||||
const {
|
||||
CopilotFailureDiagnosisPreModelTerminalizationConflictError,
|
||||
} = require('@qinglong/ai/failure-diagnosis-pre-model-terminalization');
|
||||
|
||||
async function durablePlan() {
|
||||
const generation = createPluginPackageResourceGenerationFromReferences({
|
||||
installationId: 'install-cancel-test',
|
||||
projectId: 'project-cancel',
|
||||
packageName: 'qinglong',
|
||||
lockDigest: 'a'.repeat(64),
|
||||
generation: 1,
|
||||
previousActiveLockDigest: null,
|
||||
contentDigest: 'b'.repeat(64),
|
||||
resources: [],
|
||||
});
|
||||
const snapshot = createProjectToolDefinitionSnapshot({
|
||||
projectId: 'project-cancel',
|
||||
contributions: [
|
||||
{
|
||||
generation,
|
||||
revisionDigest: 'c'.repeat(64),
|
||||
definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION],
|
||||
},
|
||||
],
|
||||
});
|
||||
const binding = createBuiltInRunLogExcerptToolHandlerBinding(snapshot, [
|
||||
'cluster-control',
|
||||
]);
|
||||
const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]);
|
||||
const principal = {
|
||||
subject: { type: 'user', id: 'owner-cancel' },
|
||||
authenticationId: 'auth-cancel',
|
||||
authenticatedAtMs: 100,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'multi_factor',
|
||||
};
|
||||
const invocation = await prepareToolInvocation(
|
||||
projectToolDefinitionRegistry(snapshot),
|
||||
{
|
||||
projectId: 'project-cancel',
|
||||
principal,
|
||||
nowMs: 200,
|
||||
tool: BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||
input: { runId: 'source-run-cancel', attemptId: 'source-attempt-cancel' },
|
||||
},
|
||||
{
|
||||
async authorize() {
|
||||
return {
|
||||
effect: 'allow',
|
||||
reasons: ['role_grant'],
|
||||
fence: { projectVersion: 2, bindingVersion: 3 },
|
||||
};
|
||||
},
|
||||
},
|
||||
);
|
||||
const tool = createTrustedToolInvocationPlan(bindings, invocation, {
|
||||
actionRef: 'cancel-log-tool',
|
||||
inputArtifactId: 'cancel-input',
|
||||
previewArtifactId: 'cancel-preview',
|
||||
artifactKeyId: 'cancel-key',
|
||||
artifactKey: Buffer.alloc(32, 0x11),
|
||||
artifactNonce: Buffer.alloc(12, 0x22),
|
||||
profile: 'cluster-control',
|
||||
preview: {
|
||||
title: 'Read failed Run log',
|
||||
summary: 'Read bounded evidence',
|
||||
fields: [
|
||||
{ kind: 'identifier', label: 'Run', value: 'source-run-cancel' },
|
||||
{
|
||||
kind: 'identifier',
|
||||
label: 'Attempt',
|
||||
value: 'source-attempt-cancel',
|
||||
},
|
||||
],
|
||||
warnings: ['potentially_sensitive_output'],
|
||||
},
|
||||
sealedAtMs: 300,
|
||||
});
|
||||
return prepareCopilotFailureDiagnosisExecution({
|
||||
requestId: 'diagnosis-request-cancel',
|
||||
traceId: 'diagnosis-trace-cancel',
|
||||
source: {
|
||||
runId: 'source-run-cancel',
|
||||
runVersion: 4,
|
||||
runStatus: 'failed',
|
||||
attemptId: 'source-attempt-cancel',
|
||||
attemptStatus: 'failed',
|
||||
attemptFinishedAtMs: 250,
|
||||
logArtifactId: 'source-log-cancel',
|
||||
},
|
||||
toolPlan: tool.plan,
|
||||
bindings,
|
||||
model: {
|
||||
provider: 'provider-primary',
|
||||
model: 'model-diagnosis',
|
||||
modelBoundary: 'external',
|
||||
responseLanguage: 'zh-CN',
|
||||
maxOutputTokens: 256,
|
||||
egressPolicy: {
|
||||
schema: 'qinglong/copilot-model-egress-policy@v1',
|
||||
revision: 'cancel-policy-v1',
|
||||
potentiallySensitiveDataBoundaries: ['external'],
|
||||
maxInputBytes: 64 * 1024,
|
||||
maxOutputTokens: 512,
|
||||
},
|
||||
},
|
||||
plannedAtMs: 400,
|
||||
deadlineAtMs: 8_000,
|
||||
});
|
||||
}
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-cancel',
|
||||
sourceRunId: 'source-run-cancel',
|
||||
requestId: 'diagnosis-request-cancel',
|
||||
mutationId: '11111111-1111-4111-8111-111111111111',
|
||||
eventId: '22222222-2222-4222-8222-222222222222',
|
||||
subject: { type: 'user', id: 'owner-cancel' },
|
||||
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(plan, overrides = {}) {
|
||||
const receipt = createCopilotFailureDiagnosisAdmissionBundle(plan).receipt;
|
||||
const state = {
|
||||
cancellationCalls: [],
|
||||
terminalizeCalls: 0,
|
||||
cancellation: {
|
||||
status: 'accepted',
|
||||
projectId: plan.projectId,
|
||||
runId: plan.runId,
|
||||
runStatus: 'running',
|
||||
runVersion: 4,
|
||||
eventSequence: 4,
|
||||
cancelRequestedAtMs: 500,
|
||||
cancelReason: 'user',
|
||||
},
|
||||
authority: {
|
||||
plan,
|
||||
run: {
|
||||
id: plan.runId,
|
||||
projectId: plan.projectId,
|
||||
status: 'running',
|
||||
version: 4,
|
||||
eventSequence: 4,
|
||||
cancelRequestedAtMs: 500,
|
||||
cancelReason: 'user',
|
||||
},
|
||||
toolStep: {},
|
||||
modelStep: {},
|
||||
modelStartExists: false,
|
||||
observedAtMs: 501,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
const terminalizations = {
|
||||
repository: {
|
||||
async findByRequestId() {
|
||||
return null;
|
||||
},
|
||||
async readAuthority() {
|
||||
return state.authority;
|
||||
},
|
||||
async commit() {
|
||||
throw new Error('not called by injected terminalizer');
|
||||
},
|
||||
},
|
||||
};
|
||||
const service = new CopilotFailureDiagnosisCancellationService({
|
||||
admissions: {
|
||||
async findPlanByRequestId() {
|
||||
return state.plan === undefined ? plan : state.plan;
|
||||
},
|
||||
async findByRequestId() {
|
||||
return state.receipt === undefined ? receipt : state.receipt;
|
||||
},
|
||||
},
|
||||
cancellations: {
|
||||
async requestUserCancellation(value) {
|
||||
state.cancellationCalls.push(value);
|
||||
return state.cancellation;
|
||||
},
|
||||
},
|
||||
terminalizations,
|
||||
async terminalizeBeforeModel(requestId, trigger) {
|
||||
state.terminalizeCalls += 1;
|
||||
if (state.terminalizeError) throw state.terminalizeError;
|
||||
assert.equal(requestId, plan.requestId);
|
||||
assert.deepEqual(trigger, { kind: 'boundary' });
|
||||
return {
|
||||
status: 'created',
|
||||
receipt: {
|
||||
outcome: 'cancelled',
|
||||
finalRunVersion: 7,
|
||||
finalRunEventSequence: 7,
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
return { service, state, receipt };
|
||||
}
|
||||
|
||||
test('resolves the request key to a server-owned Run and terminalizes pre-Model', async () => {
|
||||
const plan = await durablePlan();
|
||||
const { service, state } = fixture(plan);
|
||||
const result = await service.cancel(command());
|
||||
assert.deepEqual(state.cancellationCalls, [
|
||||
{
|
||||
projectId: 'project-cancel',
|
||||
runId: plan.runId,
|
||||
mutationId: '11111111-1111-4111-8111-111111111111',
|
||||
eventId: '22222222-2222-4222-8222-222222222222',
|
||||
subject: { type: 'user', id: 'owner-cancel' },
|
||||
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
||||
},
|
||||
]);
|
||||
assert.equal(state.terminalizeCalls, 1);
|
||||
assert.equal(result.status, 'accepted');
|
||||
assert.equal(result.convergence, 'terminal');
|
||||
assert.equal(result.runStatus, 'cancelled');
|
||||
assert.equal(result.outcome, 'cancelled');
|
||||
assert.equal(result.diagnosisRunId, plan.runId);
|
||||
assert.equal(result.cancelRequestedAtMs, 500);
|
||||
});
|
||||
|
||||
test('masks cross-target requests before writing a cancellation intent', async () => {
|
||||
const plan = await durablePlan();
|
||||
const { service, state } = fixture(plan);
|
||||
await assert.rejects(
|
||||
service.cancel(command({ sourceRunId: 'other-source-run' })),
|
||||
CopilotFailureDiagnosisCancellationNotFoundError,
|
||||
);
|
||||
assert.equal(state.cancellationCalls.length, 0);
|
||||
});
|
||||
|
||||
test('keeps a durable intent pending when Model start wins the race', async () => {
|
||||
const plan = await durablePlan();
|
||||
const conflict =
|
||||
new CopilotFailureDiagnosisPreModelTerminalizationConflictError(
|
||||
'Model started',
|
||||
);
|
||||
const { service, state } = fixture(plan, {
|
||||
terminalizeError: conflict,
|
||||
authority: {
|
||||
plan,
|
||||
run: {
|
||||
id: plan.runId,
|
||||
projectId: plan.projectId,
|
||||
status: 'running',
|
||||
version: 5,
|
||||
eventSequence: 5,
|
||||
cancelRequestedAtMs: 500,
|
||||
cancelReason: 'user',
|
||||
},
|
||||
toolStep: {},
|
||||
modelStep: {},
|
||||
modelStartExists: true,
|
||||
observedAtMs: 501,
|
||||
},
|
||||
});
|
||||
const result = await service.cancel(command());
|
||||
assert.equal(state.terminalizeCalls, 1);
|
||||
assert.equal(result.convergence, 'model_in_flight');
|
||||
assert.equal(result.runStatus, 'running');
|
||||
assert.equal(result.outcome, null);
|
||||
assert.equal(result.runVersion, 5);
|
||||
});
|
||||
|
||||
test('reports the real terminal winner instead of forging cancellation', async () => {
|
||||
const plan = await durablePlan();
|
||||
const { service } = fixture(plan, {
|
||||
terminalizeError:
|
||||
new CopilotFailureDiagnosisPreModelTerminalizationConflictError(),
|
||||
authority: {
|
||||
plan,
|
||||
run: {
|
||||
id: plan.runId,
|
||||
projectId: plan.projectId,
|
||||
status: 'succeeded',
|
||||
version: 7,
|
||||
eventSequence: 7,
|
||||
cancelRequestedAtMs: 500,
|
||||
cancelReason: 'user',
|
||||
},
|
||||
toolStep: {},
|
||||
modelStep: {},
|
||||
modelStartExists: true,
|
||||
observedAtMs: 600,
|
||||
},
|
||||
});
|
||||
const result = await service.cancel(command());
|
||||
assert.equal(result.convergence, 'terminal');
|
||||
assert.equal(result.runStatus, 'succeeded');
|
||||
assert.equal(result.outcome, 'succeeded');
|
||||
});
|
||||
|
||||
test('fails closed when plan and admission receipt drift', async () => {
|
||||
const plan = await durablePlan();
|
||||
const base = fixture(plan);
|
||||
const { service, state } = fixture(plan, {
|
||||
receipt: { ...base.receipt, runId: 'diagnosis-run-drift' },
|
||||
});
|
||||
await assert.rejects(
|
||||
service.cancel(command()),
|
||||
CopilotFailureDiagnosisCancellationUnavailableError,
|
||||
);
|
||||
assert.equal(state.cancellationCalls.length, 0);
|
||||
});
|
||||
Reference in New Issue
Block a user