mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): expose fenced copilot diagnosis API
This commit is contained in:
whyour
@@ -45,6 +45,11 @@
|
||||
"require": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js"
|
||||
},
|
||||
"./copilot-routes": {
|
||||
"types": "./dist/copilot/failure-diagnosis/failureDiagnosisRoute.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisRoute.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisRoute.js"
|
||||
},
|
||||
"./http": {
|
||||
"types": "./dist/transport/httpSurface.d.ts",
|
||||
"require": "./dist/transport/httpSurface.js",
|
||||
|
||||
@@ -445,6 +445,13 @@ export async function startProductionClusterAiControlApplication(
|
||||
capability: promptApplication.promptExecutionOutputs,
|
||||
},
|
||||
}),
|
||||
...(copilotApplication === undefined
|
||||
? {}
|
||||
: {
|
||||
copilotFailureDiagnosis: {
|
||||
capability: copilotApplication,
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (controlApplication.status !== 'active') {
|
||||
throw new Error('AI-enabled cluster-control did not activate');
|
||||
|
||||
@@ -61,6 +61,10 @@ import {
|
||||
} from '../worker-ingress/productionWorkerIngress';
|
||||
import type { ClusterWorkerIngressApplicationResult } from '../worker-ingress/workerIngressApplication';
|
||||
import { createClusterControlPluginPackageWorkflowRoutes } from '../plugin-package/workflow/pluginPackageWorkflowRoute';
|
||||
import {
|
||||
createClusterControlCopilotFailureDiagnosisRoute,
|
||||
type ClusterCopilotFailureDiagnosisCapability,
|
||||
} from '../copilot/failure-diagnosis/failureDiagnosisRoute';
|
||||
|
||||
export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
|
||||
'task.get',
|
||||
@@ -87,6 +91,7 @@ export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS =
|
||||
'prompt.execution.read',
|
||||
'prompt.execution.output.read',
|
||||
'prompt.output.read',
|
||||
'copilot.failure_diagnosis.execute',
|
||||
] as const);
|
||||
|
||||
export interface ProductionClusterControlAssemblyOptions {
|
||||
@@ -109,6 +114,9 @@ export interface ProductionClusterControlAssemblyOptions {
|
||||
readonly promptExecutionOutputRead?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionOutputReadCapability;
|
||||
}>;
|
||||
readonly copilotFailureDiagnosis?: Readonly<{
|
||||
readonly capability: ClusterCopilotFailureDiagnosisCapability;
|
||||
}>;
|
||||
readonly workerIngress?: Readonly<{
|
||||
readonly config: EnabledClusterWorkerIngressConfig;
|
||||
readonly onDiagnostic?: (error: unknown) => void | Promise<void>;
|
||||
@@ -154,6 +162,9 @@ export interface ProductionClusterControlApplicationOptions
|
||||
readonly promptExecutionOutputRead?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionOutputReadCapability;
|
||||
}>;
|
||||
readonly copilotFailureDiagnosis?: Readonly<{
|
||||
readonly capability: ClusterCopilotFailureDiagnosisCapability;
|
||||
}>;
|
||||
readonly workerIngress?: ProductionClusterWorkerIngressOptions;
|
||||
}
|
||||
|
||||
@@ -261,6 +272,13 @@ export function createProductionClusterControlApplicationStack(
|
||||
options.promptExecutionOutputRead.capability,
|
||||
),
|
||||
]),
|
||||
...(options.copilotFailureDiagnosis === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlCopilotFailureDiagnosisRoute(
|
||||
options.copilotFailureDiagnosis.capability,
|
||||
),
|
||||
]),
|
||||
];
|
||||
const routes = createClusterControlRouteRegistry(routeDefinitions);
|
||||
const expectedRouteCount =
|
||||
@@ -269,7 +287,8 @@ export function createProductionClusterControlApplicationStack(
|
||||
(options.promptExecution === undefined ? 0 : 1) +
|
||||
(options.promptExecutionInspection === undefined ? 0 : 1) +
|
||||
(options.promptOutputRead === undefined ? 0 : 1) +
|
||||
(options.promptExecutionOutputRead === undefined ? 0 : 1);
|
||||
(options.promptExecutionOutputRead === undefined ? 0 : 1) +
|
||||
(options.copilotFailureDiagnosis === undefined ? 0 : 1);
|
||||
if (routes.size !== expectedRouteCount) {
|
||||
throw new Error('Production cluster-control route allowlist is incomplete');
|
||||
}
|
||||
@@ -359,6 +378,7 @@ export function startProductionClusterControlApplication(
|
||||
promptExecutionInspection,
|
||||
promptOutputRead,
|
||||
promptExecutionOutputRead,
|
||||
copilotFailureDiagnosis,
|
||||
...applicationOptions
|
||||
} = options;
|
||||
const database = createClusterControlDatabaseBinding(config);
|
||||
@@ -402,6 +422,9 @@ export function startProductionClusterControlApplication(
|
||||
...(promptExecutionOutputRead === undefined
|
||||
? {}
|
||||
: { promptExecutionOutputRead }),
|
||||
...(copilotFailureDiagnosis === undefined
|
||||
? {}
|
||||
: { copilotFailureDiagnosis }),
|
||||
...(workerIngress === undefined
|
||||
? {}
|
||||
: {
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
// Cluster Copilot owns one bounded, Policy-fenced diagnosis admission route.
|
||||
import type { SecurityPrincipal } from '@qinglong/runtime-core/security';
|
||||
import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_REQUEST_SCHEMA =
|
||||
'qinglong/cluster-copilot-failure-diagnosis-request@v1' as const;
|
||||
export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_RESPONSE_SCHEMA =
|
||||
'qinglong/cluster-copilot-failure-diagnosis-response@v1' as const;
|
||||
export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_ROUTE = Object.freeze({
|
||||
method: 'POST' as const,
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses',
|
||||
operationId: 'copilot.failure_diagnosis.execute',
|
||||
permission: 'model.invoke',
|
||||
projectParameter: 'projectId',
|
||||
});
|
||||
|
||||
export interface ClusterCopilotFailureDiagnosisCommand {
|
||||
readonly requestId: string;
|
||||
readonly traceId: string;
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
}
|
||||
|
||||
interface ClusterCopilotFailureDiagnosisOutputReference {
|
||||
readonly artifactId: string;
|
||||
readonly artifactDigest: string;
|
||||
}
|
||||
|
||||
export interface ClusterCopilotFailureDiagnosisCapability {
|
||||
execute(command: Readonly<ClusterCopilotFailureDiagnosisCommand>): Promise<
|
||||
Readonly<{
|
||||
readonly admissionStatus: 'created' | 'existing';
|
||||
readonly admission: Readonly<{
|
||||
readonly requestId: string;
|
||||
readonly runId: string;
|
||||
readonly sourceRunId: string;
|
||||
}>;
|
||||
readonly tool: Readonly<{ readonly outcome: string }> | null;
|
||||
readonly model: Readonly<{
|
||||
readonly outcome: string;
|
||||
readonly output: Readonly<ClusterCopilotFailureDiagnosisOutputReference> | null;
|
||||
}> | null;
|
||||
readonly terminalization: Readonly<{
|
||||
readonly stage: string;
|
||||
readonly reason: string;
|
||||
readonly outcome: string;
|
||||
}> | null;
|
||||
readonly terminalizationRequired: boolean;
|
||||
}>
|
||||
>;
|
||||
}
|
||||
|
||||
class InvalidClusterCopilotFailureDiagnosisRequestError extends TypeError {}
|
||||
|
||||
const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
|
||||
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
|
||||
const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']);
|
||||
const TERMINAL_STAGES = new Set(['tool', 'log', 'deadline', 'cancellation']);
|
||||
const TERMINAL_REASONS = new Set([
|
||||
'tool_failed',
|
||||
'tool_timed_out',
|
||||
'log_not_found',
|
||||
'log_pending',
|
||||
'log_missing',
|
||||
'log_retired',
|
||||
'tool_budget_exhausted',
|
||||
'deadline_exceeded',
|
||||
'cancellation_requested',
|
||||
]);
|
||||
|
||||
function invalid(): never {
|
||||
throw new InvalidClusterCopilotFailureDiagnosisRequestError();
|
||||
}
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function parseBody(value: unknown): Readonly<{ traceId: string }> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
Object.getPrototypeOf(value) !== Object.prototype ||
|
||||
Object.keys(value).sort().join('\0') !== ['schema', 'traceId'].join('\0')
|
||||
) {
|
||||
return invalid();
|
||||
}
|
||||
const body = value as Record<string, unknown>;
|
||||
if (
|
||||
body.schema !== CLUSTER_COPILOT_FAILURE_DIAGNOSIS_REQUEST_SCHEMA ||
|
||||
typeof body.traceId !== 'string' ||
|
||||
!ID_PATTERN.test(body.traceId)
|
||||
) {
|
||||
return invalid();
|
||||
}
|
||||
return Object.freeze({ traceId: body.traceId });
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | null {
|
||||
if (
|
||||
!error ||
|
||||
typeof error !== 'object' ||
|
||||
!('code' in error) ||
|
||||
typeof error.code !== 'string'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return error.code;
|
||||
}
|
||||
|
||||
function executionError(error: unknown): ClusterControlAdmissionResponse {
|
||||
const code = errorCode(error);
|
||||
if (
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_CONFLICT' ||
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_ADMISSION_CONFLICT' ||
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_ADMISSION_NOT_ALLOWED' ||
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_TOOL_EXECUTION_CONFLICT' ||
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_CONFLICT' ||
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_CONFLICT' ||
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_IN_PROGRESS' ||
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_MODEL_RESOLUTION_REQUIRED' ||
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_PRE_MODEL_TERMINALIZATION_CONFLICT'
|
||||
) {
|
||||
return response(409, { code: 'copilot_failure_diagnosis_conflict' });
|
||||
}
|
||||
if (
|
||||
code === 'TRUSTED_TOOL_EXECUTION_POLICY_DENIED' ||
|
||||
code === 'TRUSTED_TOOL_EXECUTION_APPROVAL_REQUIRED'
|
||||
) {
|
||||
return response(403, { code: 'copilot_failure_diagnosis_forbidden' });
|
||||
}
|
||||
if (
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_BUSY' ||
|
||||
code === 'MODEL_GATEWAY_BUSY'
|
||||
) {
|
||||
return response(429, {
|
||||
code: 'copilot_failure_diagnosis_capacity_exceeded',
|
||||
});
|
||||
}
|
||||
if (
|
||||
code === 'MODEL_POLICY_DENIED' ||
|
||||
code === 'MODEL_BUDGET_EXCEEDED' ||
|
||||
code === 'COPILOT_MODEL_EGRESS_DENIED' ||
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_BUDGET_EXCEEDED'
|
||||
) {
|
||||
return response(422, { code: 'copilot_failure_diagnosis_policy_rejected' });
|
||||
}
|
||||
if (
|
||||
code === 'MODEL_INVOCATION_DEADLINE_EXCEEDED' ||
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_TOOL_EXECUTION_DEADLINE_EXCEEDED'
|
||||
) {
|
||||
return response(504, {
|
||||
code: 'copilot_failure_diagnosis_deadline_exceeded',
|
||||
});
|
||||
}
|
||||
if (code === 'MODEL_INVOCATION_ABORTED') {
|
||||
return response(408, { code: 'copilot_failure_diagnosis_aborted' });
|
||||
}
|
||||
if (
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_INVALID' ||
|
||||
code === 'COPILOT_FAILURE_DIAGNOSIS_EXECUTION_PLAN_INVALID'
|
||||
) {
|
||||
return response(400, { code: 'invalid_copilot_failure_diagnosis_request' });
|
||||
}
|
||||
return response(503, { code: 'copilot_failure_diagnosis_unavailable' });
|
||||
}
|
||||
|
||||
function projectResult(
|
||||
result: Awaited<
|
||||
ReturnType<ClusterCopilotFailureDiagnosisCapability['execute']>
|
||||
>,
|
||||
requestId: string,
|
||||
sourceRunId: string,
|
||||
): ClusterControlAdmissionResponse {
|
||||
const model = result?.model;
|
||||
const terminalization = result?.terminalization;
|
||||
const outcome = model?.outcome ?? terminalization?.outcome;
|
||||
if (
|
||||
!result ||
|
||||
(result.admissionStatus !== 'created' &&
|
||||
result.admissionStatus !== 'existing') ||
|
||||
!result.admission ||
|
||||
result.admission.requestId !== requestId ||
|
||||
result.admission.sourceRunId !== sourceRunId ||
|
||||
!RUN_ID_PATTERN.test(result.admission.runId) ||
|
||||
result.terminalizationRequired !== false ||
|
||||
(model === null) === (terminalization === null) ||
|
||||
typeof outcome !== 'string' ||
|
||||
!OUTCOMES.has(outcome)
|
||||
) {
|
||||
return response(503, { code: 'copilot_failure_diagnosis_unavailable' });
|
||||
}
|
||||
let stage: string;
|
||||
let reason: string | null;
|
||||
let outputArtifact: Readonly<ClusterCopilotFailureDiagnosisOutputReference> | null;
|
||||
if (model) {
|
||||
if (
|
||||
!OUTCOMES.has(model.outcome) ||
|
||||
(model.outcome === 'succeeded') !== (model.output !== null) ||
|
||||
(model.output !== null &&
|
||||
(!ID_PATTERN.test(model.output.artifactId) ||
|
||||
!DIGEST_PATTERN.test(model.output.artifactDigest)))
|
||||
) {
|
||||
return response(503, { code: 'copilot_failure_diagnosis_unavailable' });
|
||||
}
|
||||
stage = 'model';
|
||||
reason = null;
|
||||
outputArtifact = model.output;
|
||||
} else {
|
||||
if (
|
||||
!terminalization ||
|
||||
!TERMINAL_STAGES.has(terminalization.stage) ||
|
||||
!TERMINAL_REASONS.has(terminalization.reason)
|
||||
) {
|
||||
return response(503, { code: 'copilot_failure_diagnosis_unavailable' });
|
||||
}
|
||||
stage = terminalization.stage;
|
||||
reason = terminalization.reason;
|
||||
outputArtifact = null;
|
||||
}
|
||||
return response(result.admissionStatus === 'created' ? 201 : 200, {
|
||||
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_RESPONSE_SCHEMA,
|
||||
requestId,
|
||||
status: result.admissionStatus,
|
||||
replayed: result.admissionStatus === 'existing',
|
||||
sourceRunId,
|
||||
diagnosisRunId: result.admission.runId,
|
||||
outcome,
|
||||
stage,
|
||||
reason,
|
||||
outputArtifact:
|
||||
outputArtifact === null
|
||||
? null
|
||||
: Object.freeze({
|
||||
artifactId: outputArtifact.artifactId,
|
||||
artifactDigest: outputArtifact.artifactDigest,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterControlCopilotFailureDiagnosisRoute(
|
||||
capability: ClusterCopilotFailureDiagnosisCapability,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (!capability || typeof capability.execute !== 'function') {
|
||||
throw new TypeError(
|
||||
'Cluster-control Copilot failure diagnosis capability is invalid',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_ROUTE,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
routeParameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
let body: Readonly<{ traceId: string }>;
|
||||
try {
|
||||
body = parseBody(authorized.request.body);
|
||||
} catch {
|
||||
return response(400, {
|
||||
code: 'invalid_copilot_failure_diagnosis_request',
|
||||
});
|
||||
}
|
||||
const projectId = authorized.projectId;
|
||||
const sourceRunId = routeParameters.runId;
|
||||
if (
|
||||
projectId === null ||
|
||||
typeof sourceRunId !== 'string' ||
|
||||
!RUN_ID_PATTERN.test(sourceRunId)
|
||||
) {
|
||||
return response(503, { code: 'copilot_failure_diagnosis_unavailable' });
|
||||
}
|
||||
try {
|
||||
const result = await capability.execute({
|
||||
requestId: authorized.request.requestId,
|
||||
traceId: body.traceId,
|
||||
projectId,
|
||||
sourceRunId,
|
||||
principal: authorized.principal,
|
||||
});
|
||||
return projectResult(result, authorized.request.requestId, sourceRunId);
|
||||
} catch (error) {
|
||||
return executionError(error);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -117,7 +117,7 @@ async function projectedFile(root, name, bytes) {
|
||||
await chmod(join(root, name), 0o440);
|
||||
}
|
||||
|
||||
test('Copilot composition is explicit, shares the Prompt gateway and exposes no route', async () => {
|
||||
test('Copilot composition is explicit, shares the Prompt gateway and injects one route capability', async () => {
|
||||
const secretRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-secret-'));
|
||||
const configRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-config-'));
|
||||
const invocationRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-invocation-'));
|
||||
@@ -229,7 +229,7 @@ test('Copilot composition is explicit, shares the Prompt gateway and exposes no
|
||||
assert.equal(created.gateway, gateway);
|
||||
assert.equal(created.successfulCompletion, registeredSink);
|
||||
assert.equal(created.artifactStore, artifactStore);
|
||||
assert.equal('copilot' in controlOptions, false);
|
||||
assert.equal(controlOptions.copilotFailureDiagnosis.capability, copilot);
|
||||
assert.equal(await application.stop(), 'stopped');
|
||||
} finally {
|
||||
config.fill(0); invocation.fill(0); result.fill(0); output.fill(0);
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_REQUEST_SCHEMA,
|
||||
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_RESPONSE_SCHEMA,
|
||||
CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_ROUTE,
|
||||
createClusterControlCopilotFailureDiagnosisRoute,
|
||||
} = require('@qinglong/cluster-control/copilot-routes');
|
||||
|
||||
function authorized(body, overrides = {}) {
|
||||
return {
|
||||
request: {
|
||||
requestId: 'diagnosis-request-1',
|
||||
method: 'POST',
|
||||
path: '/api/v3/projects/project-1/runs/source-run-1/copilot/failure-diagnoses',
|
||||
query: {},
|
||||
headers: {},
|
||||
signal: new AbortController().signal,
|
||||
body,
|
||||
},
|
||||
principal: {
|
||||
subject: { type: 'api_app', id: 'app-1' },
|
||||
authenticationId: 'credential-1',
|
||||
authenticatedAtMs: 1,
|
||||
expiresAtMs: 10_000,
|
||||
assurance: 'service',
|
||||
},
|
||||
operationId: 'copilot.failure_diagnosis.execute',
|
||||
permission: 'model.invoke',
|
||||
projectId: 'project-1',
|
||||
policyFence: { projectVersion: 3, bindingVersion: 7 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function body(overrides = {}) {
|
||||
return {
|
||||
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_REQUEST_SCHEMA,
|
||||
traceId: 'trace-1',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function succeeded(admissionStatus = 'created') {
|
||||
return {
|
||||
admissionStatus,
|
||||
admission: {
|
||||
requestId: 'diagnosis-request-1',
|
||||
runId: 'diagnosis-run-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
},
|
||||
tool: { outcome: 'succeeded', output: { private: 'must not cross' } },
|
||||
model: {
|
||||
outcome: 'succeeded',
|
||||
output: {
|
||||
artifactId: 'cdo:artifact-1',
|
||||
artifactDigest: 'a'.repeat(64),
|
||||
provider: 'private-provider',
|
||||
},
|
||||
plaintext: 'private diagnosis',
|
||||
},
|
||||
terminalization: null,
|
||||
terminalizationRequired: false,
|
||||
};
|
||||
}
|
||||
|
||||
test('defines one exact model.invoke route and binds HTTP request identity', async () => {
|
||||
let command;
|
||||
const route = createClusterControlCopilotFailureDiagnosisRoute({
|
||||
async execute(value) {
|
||||
command = value;
|
||||
return succeeded();
|
||||
},
|
||||
});
|
||||
const request = authorized(body());
|
||||
const result = await route.handle(request, {
|
||||
projectId: 'project-1',
|
||||
runId: 'source-run-1',
|
||||
});
|
||||
|
||||
assert.deepEqual(CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_ROUTE, {
|
||||
method: 'POST',
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses',
|
||||
operationId: 'copilot.failure_diagnosis.execute',
|
||||
permission: 'model.invoke',
|
||||
projectParameter: 'projectId',
|
||||
});
|
||||
assert.deepEqual(command, {
|
||||
requestId: 'diagnosis-request-1',
|
||||
traceId: 'trace-1',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
principal: request.principal,
|
||||
});
|
||||
assert.equal('policyFence' in command, false);
|
||||
assert.equal('model' in command, false);
|
||||
assert.equal('attemptId' in command, false);
|
||||
assert.equal(result.statusCode, 201);
|
||||
assert.deepEqual(result.body, {
|
||||
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_RESPONSE_SCHEMA,
|
||||
requestId: 'diagnosis-request-1',
|
||||
status: 'created',
|
||||
replayed: false,
|
||||
sourceRunId: 'source-run-1',
|
||||
diagnosisRunId: 'diagnosis-run-1',
|
||||
outcome: 'succeeded',
|
||||
stage: 'model',
|
||||
reason: null,
|
||||
outputArtifact: {
|
||||
artifactId: 'cdo:artifact-1',
|
||||
artifactDigest: 'a'.repeat(64),
|
||||
},
|
||||
});
|
||||
assert.equal(JSON.stringify(result).includes('private'), false);
|
||||
});
|
||||
|
||||
test('returns a content-free existing receipt for exact replay', async () => {
|
||||
const route = createClusterControlCopilotFailureDiagnosisRoute({
|
||||
async execute() {
|
||||
return succeeded('existing');
|
||||
},
|
||||
});
|
||||
const result = await route.handle(authorized(body()), {
|
||||
projectId: 'project-1',
|
||||
runId: 'source-run-1',
|
||||
});
|
||||
assert.equal(result.statusCode, 200);
|
||||
assert.equal(result.body.status, 'existing');
|
||||
assert.equal(result.body.replayed, true);
|
||||
});
|
||||
|
||||
test('projects pre-Model terminalization without Tool or log content', async () => {
|
||||
const route = createClusterControlCopilotFailureDiagnosisRoute({
|
||||
async execute() {
|
||||
return {
|
||||
admissionStatus: 'created',
|
||||
admission: {
|
||||
requestId: 'diagnosis-request-1',
|
||||
runId: 'diagnosis-run-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
},
|
||||
tool: { outcome: 'failed', privateLog: 'must not cross' },
|
||||
model: null,
|
||||
terminalization: {
|
||||
stage: 'log',
|
||||
reason: 'log_retired',
|
||||
outcome: 'failed',
|
||||
privateEvidence: 'must not cross',
|
||||
},
|
||||
terminalizationRequired: false,
|
||||
};
|
||||
},
|
||||
});
|
||||
const result = await route.handle(authorized(body()), {
|
||||
projectId: 'project-1',
|
||||
runId: 'source-run-1',
|
||||
});
|
||||
assert.equal(result.statusCode, 201);
|
||||
assert.equal(result.body.stage, 'log');
|
||||
assert.equal(result.body.reason, 'log_retired');
|
||||
assert.equal(result.body.outcome, 'failed');
|
||||
assert.equal(result.body.outputArtifact, null);
|
||||
assert.equal(JSON.stringify(result).includes('private'), false);
|
||||
});
|
||||
|
||||
test('rejects non-exact bodies before invoking the capability', async () => {
|
||||
let calls = 0;
|
||||
const route = createClusterControlCopilotFailureDiagnosisRoute({
|
||||
async execute() {
|
||||
calls += 1;
|
||||
return succeeded();
|
||||
},
|
||||
});
|
||||
for (const invalid of [
|
||||
null,
|
||||
{},
|
||||
body({ requestId: 'body-request-must-not-exist' }),
|
||||
body({ provider: 'caller-selected' }),
|
||||
body({ traceId: '' }),
|
||||
Object.assign(Object.create(null), body()),
|
||||
]) {
|
||||
const result = await route.handle(authorized(invalid), {
|
||||
projectId: 'project-1',
|
||||
runId: 'source-run-1',
|
||||
});
|
||||
assert.equal(result.statusCode, 400);
|
||||
assert.deepEqual(result.body, {
|
||||
code: 'invalid_copilot_failure_diagnosis_request',
|
||||
});
|
||||
}
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
|
||||
test('fails closed on capability responses that do not bind the durable identity', async () => {
|
||||
for (const mutation of [
|
||||
(value) => ({
|
||||
...value,
|
||||
admission: { ...value.admission, requestId: 'other' },
|
||||
}),
|
||||
(value) => ({
|
||||
...value,
|
||||
admission: { ...value.admission, sourceRunId: 'other' },
|
||||
}),
|
||||
(value) => ({ ...value, terminalizationRequired: true }),
|
||||
(value) => ({ ...value, model: null }),
|
||||
(value) => ({ ...value, model: { ...value.model, output: null } }),
|
||||
]) {
|
||||
const route = createClusterControlCopilotFailureDiagnosisRoute({
|
||||
async execute() {
|
||||
return mutation(succeeded());
|
||||
},
|
||||
});
|
||||
const result = await route.handle(authorized(body()), {
|
||||
projectId: 'project-1',
|
||||
runId: 'source-run-1',
|
||||
});
|
||||
assert.deepEqual(result, {
|
||||
statusCode: 503,
|
||||
body: { code: 'copilot_failure_diagnosis_unavailable' },
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('maps internal failures to stable low-sensitive transport codes', async () => {
|
||||
for (const [internal, statusCode, external] of [
|
||||
[
|
||||
'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_CONFLICT',
|
||||
409,
|
||||
'copilot_failure_diagnosis_conflict',
|
||||
],
|
||||
[
|
||||
'TRUSTED_TOOL_EXECUTION_POLICY_DENIED',
|
||||
403,
|
||||
'copilot_failure_diagnosis_forbidden',
|
||||
],
|
||||
[
|
||||
'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_BUSY',
|
||||
429,
|
||||
'copilot_failure_diagnosis_capacity_exceeded',
|
||||
],
|
||||
[
|
||||
'COPILOT_MODEL_EGRESS_DENIED',
|
||||
422,
|
||||
'copilot_failure_diagnosis_policy_rejected',
|
||||
],
|
||||
[
|
||||
'MODEL_INVOCATION_DEADLINE_EXCEEDED',
|
||||
504,
|
||||
'copilot_failure_diagnosis_deadline_exceeded',
|
||||
],
|
||||
['MODEL_INVOCATION_ABORTED', 408, 'copilot_failure_diagnosis_aborted'],
|
||||
['PRIVATE_STORAGE_FAILURE', 503, 'copilot_failure_diagnosis_unavailable'],
|
||||
]) {
|
||||
const route = createClusterControlCopilotFailureDiagnosisRoute({
|
||||
async execute() {
|
||||
throw Object.assign(new Error('private internal detail'), {
|
||||
code: internal,
|
||||
});
|
||||
},
|
||||
});
|
||||
const result = await route.handle(authorized(body()), {
|
||||
projectId: 'project-1',
|
||||
runId: 'source-run-1',
|
||||
});
|
||||
assert.equal(result.statusCode, statusCode, internal);
|
||||
assert.deepEqual(result.body, { code: external });
|
||||
assert.equal(JSON.stringify(result).includes('private'), false);
|
||||
}
|
||||
});
|
||||
@@ -729,6 +729,7 @@ test('optionally exposes Prompt execution behind shared admission and policy', a
|
||||
'prompt.execution.read',
|
||||
'prompt.execution.output.read',
|
||||
'prompt.output.read',
|
||||
'copilot.failure_diagnosis.execute',
|
||||
]);
|
||||
const response = await invoke(
|
||||
stack,
|
||||
@@ -756,6 +757,134 @@ test('optionally exposes Prompt execution behind shared admission and policy', a
|
||||
assert.equal(events.includes('audit:prompt.execute:allowed'), true);
|
||||
});
|
||||
|
||||
test('optionally exposes Copilot diagnosis behind shared authentication, Policy and audit', async () => {
|
||||
const { events, input } = fixture();
|
||||
let command;
|
||||
const capability = {
|
||||
async execute(value) {
|
||||
command = value;
|
||||
events.push(`diagnose:${value.sourceRunId}`);
|
||||
return {
|
||||
admissionStatus: 'created',
|
||||
admission: {
|
||||
requestId: value.requestId,
|
||||
runId: 'diagnosis-run-1',
|
||||
sourceRunId: value.sourceRunId,
|
||||
},
|
||||
tool: { outcome: 'succeeded' },
|
||||
model: {
|
||||
outcome: 'succeeded',
|
||||
output: {
|
||||
artifactId: 'cdo:artifact-1',
|
||||
artifactDigest: 'a'.repeat(64),
|
||||
},
|
||||
},
|
||||
terminalization: null,
|
||||
terminalizationRequired: false,
|
||||
};
|
||||
},
|
||||
};
|
||||
const stack = createProductionClusterControlApplicationStack(input, {
|
||||
copilotFailureDiagnosis: { capability },
|
||||
});
|
||||
const result = await invoke(
|
||||
stack,
|
||||
metadata(
|
||||
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses',
|
||||
'POST',
|
||||
{
|
||||
schema: 'qinglong/cluster-copilot-failure-diagnosis-request@v1',
|
||||
traceId: 'trace-production-1',
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
assert.equal(result.statusCode, 201);
|
||||
assert.equal(command.requestId, 'request-production-1');
|
||||
assert.equal(command.projectId, 'project-1');
|
||||
assert.equal(command.sourceRunId, 'run-1');
|
||||
assert.equal(command.principal.subject.id, 'app-production');
|
||||
assert.equal('policyFence' in command, false);
|
||||
assert.deepEqual(events.slice(-4), [
|
||||
'authenticate',
|
||||
'authorize',
|
||||
'audit:copilot.failure_diagnosis.execute:allowed',
|
||||
'diagnose:run-1',
|
||||
]);
|
||||
});
|
||||
|
||||
test('keeps the Copilot route absent by default and never invokes it after Policy denial', async () => {
|
||||
const defaultFixture = fixture();
|
||||
const defaultStack = createProductionClusterControlApplicationStack(
|
||||
defaultFixture.input,
|
||||
);
|
||||
const request = metadata(
|
||||
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses',
|
||||
'POST',
|
||||
{
|
||||
schema: 'qinglong/cluster-copilot-failure-diagnosis-request@v1',
|
||||
traceId: 'trace-production-1',
|
||||
},
|
||||
);
|
||||
await assert.rejects(
|
||||
defaultStack.admission.prepare(request),
|
||||
(error) => error?.statusCode === 404 && error?.code === 'route_not_found',
|
||||
);
|
||||
|
||||
let calls = 0;
|
||||
const deniedFixture = fixture({
|
||||
policies: {
|
||||
async resolve() {
|
||||
deniedFixture.events.push('authorize');
|
||||
return {
|
||||
project: {
|
||||
id: 'project-1',
|
||||
name: 'Denied Project',
|
||||
slug: 'denied-project',
|
||||
status: 'active',
|
||||
version: 3,
|
||||
createdAtMs: 1,
|
||||
updatedAtMs: 2,
|
||||
},
|
||||
binding: {
|
||||
projectId: 'project-1',
|
||||
subject: { type: 'api_app', id: 'app-production' },
|
||||
state: 'active',
|
||||
role: 'viewer',
|
||||
version: 7,
|
||||
mutationId: 'binding-denied-1',
|
||||
changedBy: { type: 'system', id: 'bootstrap' },
|
||||
createdAtMs: 1,
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
const deniedStack = createProductionClusterControlApplicationStack(
|
||||
deniedFixture.input,
|
||||
{
|
||||
copilotFailureDiagnosis: {
|
||||
capability: {
|
||||
async execute() {
|
||||
calls += 1;
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
await assert.rejects(
|
||||
deniedStack.admission.prepare(request),
|
||||
(error) => error?.statusCode === 403 && error?.code === 'forbidden',
|
||||
);
|
||||
assert.equal(calls, 0);
|
||||
assert.equal(
|
||||
deniedFixture.events.includes(
|
||||
'audit:copilot.failure_diagnosis.execute:denied',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('optionally exposes the redacted Prompt catalog behind shared admission and policy', async () => {
|
||||
const { events, input } = fixture();
|
||||
const stack = createProductionClusterControlApplicationStack(input, {
|
||||
|
||||
Reference in New Issue
Block a user