feat(ql3): expose copilot diagnosis read model

This commit is contained in:
whyour
2026-08-16 00:41:34 +08:00
parent 774af4a4c9
commit f77b19e615
16 changed files with 2526 additions and 48 deletions
+5
View File
@@ -60,6 +60,11 @@
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js",
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js"
},
"./failure-diagnosis-read-model": {
"types": "./dist/copilot/failure-diagnosis/read-model/service.d.ts",
"require": "./dist/copilot/failure-diagnosis/read-model/service.js",
"default": "./dist/copilot/failure-diagnosis/read-model/service.js"
},
"./failure-diagnosis-pre-model-terminalization": {
"types": "./dist/copilot/failure-diagnosis/preModelTerminalization.d.ts",
"require": "./dist/copilot/failure-diagnosis/preModelTerminalization.js",
@@ -0,0 +1,666 @@
import {
normalizeSecurityPrincipal,
type SecurityPolicyDecision,
type SecurityPrincipal,
} from '@qinglong/runtime-core/security';
import type { ProjectPermission } from '@qinglong/runtime-core/project-policy';
import type { ModelInvocationPriceSettlement } from '../../../pricing/pricing';
import type { ModelInvocationUsageLedgerRecord } from '../../../usage/usageLedger';
import type { GenerateResult } from '../../../model-gateway/model';
import type {
CopilotFailureDiagnosisAdmissionRepository,
CopilotFailureDiagnosisAdmissionReceipt,
CopilotFailureDiagnosisExecutionPlan,
} from '../admission/contracts';
import type {
CopilotFailureDiagnosisFinalizationReceipt,
CopilotFailureDiagnosisFinalizationRepository,
} from '../model-execution/finalization';
import {
openCopilotFailureDiagnosisOutputArtifact,
type CopilotFailureDiagnosisOutputArtifact,
type CopilotFailureDiagnosisOutputKeyProvider,
} from '../model-execution/outputArtifact';
import type { CopilotFailureDiagnosisOutputCompletionRepository } from '../model-execution/completion';
import type {
CopilotFailureDiagnosisPreModelTerminalizationReceipt,
CopilotFailureDiagnosisPreModelTerminalizationRepository,
} from '../terminalization/contracts';
export const COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA =
'qinglong/copilot-failure-diagnosis-inspection-result@v1' as const;
export const COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA =
'qinglong/copilot-failure-diagnosis-output-read-result@v1' as const;
export interface CopilotFailureDiagnosisReadTarget {
readonly principal: Readonly<SecurityPrincipal>;
readonly projectId: string;
readonly sourceRunId: string;
readonly requestId: string;
}
export interface CopilotFailureDiagnosisReadAuthorizer {
authorize(
principal: Readonly<SecurityPrincipal>,
projectId: string,
permission: ProjectPermission,
): Promise<Readonly<SecurityPolicyDecision>>;
}
export interface CopilotFailureDiagnosisReadModelRepository
extends Pick<
CopilotFailureDiagnosisOutputCompletionRepository,
'findCopilotFailureDiagnosisOutput'
> {
findUsage(
invocationId: string,
): Promise<Readonly<ModelInvocationUsageLedgerRecord> | null>;
findPriceSettlement(
invocationId: string,
): Promise<Readonly<ModelInvocationPriceSettlement> | null>;
}
export interface CopilotFailureDiagnosisUsageView {
readonly inputTokens: number;
readonly outputTokens: number;
readonly totalTokens: number;
readonly currency: 'USD' | null;
readonly costMicros: number | null;
}
export type CopilotFailureDiagnosisInspectionResult = Readonly<
| {
schema: typeof COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA;
status: 'not_found';
projectId: string;
sourceRunId: string;
requestId: string;
}
| {
schema: typeof COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA;
status: 'running' | 'terminal';
projectId: string;
sourceRunId: string;
requestId: string;
diagnosisRunId: string;
outcome: 'succeeded' | 'failed' | 'timed_out' | 'cancelled' | null;
stage: 'model' | 'tool' | 'log' | 'deadline' | 'cancellation' | null;
reason: string | null;
outputAvailable: boolean;
admittedAtMs: number;
finalizedAtMs: number | null;
usage: Readonly<CopilotFailureDiagnosisUsageView> | null;
}
>;
export type CopilotFailureDiagnosisOutputReadResult = Readonly<
| {
schema: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA;
status: 'not_found';
projectId: string;
sourceRunId: string;
requestId: string;
}
| {
schema: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA;
status: 'available';
projectId: string;
sourceRunId: string;
requestId: string;
diagnosisRunId: string;
reference: Readonly<{
artifactId: string;
artifactDigest: string;
contentDigest: string;
outputBytes: number;
sealedAtMs: number;
}>;
result: Readonly<Pick<GenerateResult, 'text' | 'finishReason' | 'usage'>>;
}
>;
export class InvalidCopilotFailureDiagnosisReadRequestError extends TypeError {
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_READ_INVALID';
constructor() {
super('Copilot failure diagnosis read request is invalid');
this.name = 'InvalidCopilotFailureDiagnosisReadRequestError';
}
}
export class CopilotFailureDiagnosisReadUnavailableError extends Error {
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_READ_UNAVAILABLE';
constructor(options?: ErrorOptions) {
super('Copilot failure diagnosis read is unavailable', options);
this.name = 'CopilotFailureDiagnosisReadUnavailableError';
}
}
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/;
const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
const DIGEST = /^[0-9a-f]{64}$/;
const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']);
const STAGES = new Set(['tool', 'log', 'deadline', 'cancellation']);
const REASONS = new Set([
'tool_failed',
'tool_timed_out',
'log_not_found',
'log_pending',
'log_missing',
'log_retired',
'tool_budget_exhausted',
'deadline_exceeded',
'cancellation_requested',
]);
function unavailable(
cause?: unknown,
): CopilotFailureDiagnosisReadUnavailableError {
return new CopilotFailureDiagnosisReadUnavailableError({
cause: cause instanceof Error ? cause : undefined,
});
}
function exactKeys(value: object, expected: readonly string[]): boolean {
const actual = Object.keys(value).sort();
const canonical = [...expected].sort();
return (
actual.length === canonical.length &&
actual.every((key, index) => key === canonical[index])
);
}
function safeInteger(value: unknown): value is number {
return Number.isSafeInteger(value) && (value as number) >= 0;
}
function normalizeTarget(
value: CopilotFailureDiagnosisReadTarget,
nowMs: number,
): Readonly<CopilotFailureDiagnosisReadTarget> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
!exactKeys(value, ['principal', 'projectId', 'requestId', 'sourceRunId']) ||
typeof value.projectId !== 'string' ||
!IDENTITY.test(value.projectId) ||
typeof value.sourceRunId !== 'string' ||
!RUN_ID.test(value.sourceRunId) ||
typeof value.requestId !== 'string' ||
!IDENTITY.test(value.requestId)
) {
throw new InvalidCopilotFailureDiagnosisReadRequestError();
}
try {
return Object.freeze({
principal: normalizeSecurityPrincipal(value.principal, nowMs),
projectId: value.projectId,
sourceRunId: value.sourceRunId,
requestId: value.requestId,
});
} catch {
throw new InvalidCopilotFailureDiagnosisReadRequestError();
}
}
function inspectionNotFound(
target: Omit<CopilotFailureDiagnosisReadTarget, 'principal'>,
): CopilotFailureDiagnosisInspectionResult {
return Object.freeze({
schema: COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA,
status: 'not_found' as const,
...target,
});
}
function outputNotFound(
target: Omit<CopilotFailureDiagnosisReadTarget, 'principal'>,
): CopilotFailureDiagnosisOutputReadResult {
return Object.freeze({
schema: COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA,
status: 'not_found' as const,
...target,
});
}
function targetView(
command: Readonly<CopilotFailureDiagnosisReadTarget>,
): Readonly<Omit<CopilotFailureDiagnosisReadTarget, 'principal'>> {
return Object.freeze({
projectId: command.projectId,
sourceRunId: command.sourceRunId,
requestId: command.requestId,
});
}
function planMatchesTarget(
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
command: Readonly<CopilotFailureDiagnosisReadTarget>,
): boolean {
return (
plan.requestId === command.requestId &&
plan.projectId === command.projectId &&
plan.source?.runId === command.sourceRunId
);
}
function validPlan(
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
): boolean {
return (
!!plan &&
typeof plan === 'object' &&
IDENTITY.test(plan.requestId) &&
IDENTITY.test(plan.projectId) &&
RUN_ID.test(plan.runId) &&
RUN_ID.test(plan.source?.runId) &&
IDENTITY.test(plan.modelStepRunId) &&
IDENTITY.test(plan.modelInvocationId) &&
DIGEST.test(plan.planDigest) &&
safeInteger(plan.plannedAtMs)
);
}
function validTerminalization(
value: Readonly<CopilotFailureDiagnosisPreModelTerminalizationReceipt>,
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
): boolean {
return (
!!value &&
value.requestId === plan.requestId &&
value.planDigest === plan.planDigest &&
value.runId === plan.runId &&
STAGES.has(value.stage) &&
REASONS.has(value.reason) &&
OUTCOMES.has(value.outcome) &&
safeInteger(value.finalizedAtMs)
);
}
function validFinalization(
value: Readonly<CopilotFailureDiagnosisFinalizationReceipt>,
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
): boolean {
return (
!!value &&
value.requestId === plan.requestId &&
value.planDigest === plan.planDigest &&
value.runId === plan.runId &&
value.modelStepRunId === plan.modelStepRunId &&
value.invocationId === plan.modelInvocationId &&
DIGEST.test(value.completionDigest) &&
OUTCOMES.has(value.outcome) &&
(value.outputArtifactId === null ||
IDENTITY.test(value.outputArtifactId)) &&
(value.outcome === 'succeeded') === (value.outputArtifactId !== null) &&
safeInteger(value.finalizedAtMs)
);
}
function validUsage(
usage: Readonly<ModelInvocationUsageLedgerRecord>,
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
finalization: Readonly<CopilotFailureDiagnosisFinalizationReceipt>,
): boolean {
return (
usage.invocationId === plan.modelInvocationId &&
usage.projectId === plan.projectId &&
usage.runId === plan.runId &&
usage.stepRunId === plan.modelStepRunId &&
usage.traceId === plan.traceId &&
usage.completionDigest === finalization.completionDigest &&
usage.outcome === finalization.outcome &&
safeInteger(usage.inputTokens) &&
safeInteger(usage.outputTokens) &&
safeInteger(usage.totalTokens) &&
usage.totalTokens === usage.inputTokens + usage.outputTokens &&
(usage.costMicros === null || safeInteger(usage.costMicros))
);
}
function settlementMatches(
settlement: Readonly<ModelInvocationPriceSettlement>,
usage: Readonly<ModelInvocationUsageLedgerRecord>,
finalization: Readonly<CopilotFailureDiagnosisFinalizationReceipt>,
): boolean {
return (
settlement.invocationId === usage.invocationId &&
settlement.projectId === usage.projectId &&
settlement.completionDigest === finalization.completionDigest &&
settlement.currency === 'USD' &&
settlement.inputTokens === usage.inputTokens &&
settlement.outputTokens === usage.outputTokens &&
safeInteger(settlement.costMicros) &&
usage.costMicros === settlement.costMicros
);
}
function validArtifact(
artifact: Readonly<CopilotFailureDiagnosisOutputArtifact>,
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
finalization: Readonly<CopilotFailureDiagnosisFinalizationReceipt>,
): boolean {
return (
artifact.artifactId === finalization.outputArtifactId &&
artifact.requestId === plan.requestId &&
artifact.planDigest === plan.planDigest &&
artifact.projectId === plan.projectId &&
artifact.runId === plan.runId &&
artifact.stepRunId === plan.modelStepRunId &&
artifact.invocationId === plan.modelInvocationId &&
DIGEST.test(artifact.artifactDigest) &&
DIGEST.test(artifact.contentDigest) &&
safeInteger(artifact.outputBytes) &&
safeInteger(artifact.sealedAtMs)
);
}
interface LocatedDiagnosis {
readonly plan: Readonly<CopilotFailureDiagnosisExecutionPlan>;
readonly admission: Readonly<CopilotFailureDiagnosisAdmissionReceipt>;
readonly terminalization: Readonly<CopilotFailureDiagnosisPreModelTerminalizationReceipt> | null;
readonly finalization: Readonly<CopilotFailureDiagnosisFinalizationReceipt> | null;
}
/**
* Request-keyed product read boundary. It resolves durable authority before
* current Policy, never accepts storage/model identities from the caller, and
* only resolves output key material after every binding is proven.
*/
export class CopilotFailureDiagnosisReadService {
readonly #admissions: Pick<
CopilotFailureDiagnosisAdmissionRepository,
'findByRequestId' | 'findPlanByRequestId'
>;
readonly #terminalizations: Pick<
CopilotFailureDiagnosisPreModelTerminalizationRepository,
'findByRequestId'
>;
readonly #finalizations: Pick<
CopilotFailureDiagnosisFinalizationRepository,
'findFinalization'
>;
readonly #models: CopilotFailureDiagnosisReadModelRepository;
readonly #authorizer: CopilotFailureDiagnosisReadAuthorizer;
readonly #keys: CopilotFailureDiagnosisOutputKeyProvider;
readonly #now: () => number;
constructor(
options: Readonly<{
admissions: Pick<
CopilotFailureDiagnosisAdmissionRepository,
'findByRequestId' | 'findPlanByRequestId'
>;
terminalizations: Pick<
CopilotFailureDiagnosisPreModelTerminalizationRepository,
'findByRequestId'
>;
finalizations: Pick<
CopilotFailureDiagnosisFinalizationRepository,
'findFinalization'
>;
models: CopilotFailureDiagnosisReadModelRepository;
authorizer: CopilotFailureDiagnosisReadAuthorizer;
keys: CopilotFailureDiagnosisOutputKeyProvider;
now?: () => number;
}>,
) {
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
typeof options.admissions?.findPlanByRequestId !== 'function' ||
typeof options.admissions?.findByRequestId !== 'function' ||
typeof options.terminalizations?.findByRequestId !== 'function' ||
typeof options.finalizations?.findFinalization !== 'function' ||
typeof options.models?.findCopilotFailureDiagnosisOutput !== 'function' ||
typeof options.models?.findUsage !== 'function' ||
typeof options.models?.findPriceSettlement !== 'function' ||
typeof options.authorizer?.authorize !== 'function' ||
typeof options.keys?.resolve !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function')
) {
throw unavailable();
}
this.#admissions = options.admissions;
this.#terminalizations = options.terminalizations;
this.#finalizations = options.finalizations;
this.#models = options.models;
this.#authorizer = options.authorizer;
this.#keys = options.keys;
this.#now = options.now ?? Date.now;
}
async #locate(
command: Readonly<CopilotFailureDiagnosisReadTarget>,
permission: ProjectPermission,
): Promise<Readonly<LocatedDiagnosis> | null> {
let plan;
let admission;
try {
[plan, admission] = await Promise.all([
this.#admissions.findPlanByRequestId(command.requestId),
this.#admissions.findByRequestId(command.requestId),
]);
} catch (cause) {
throw unavailable(cause);
}
if (plan === null && admission === null) return null;
if (plan === null || admission === null) throw unavailable();
if (!validPlan(plan)) throw unavailable();
if (!planMatchesTarget(plan, command)) return null;
if (
admission.requestId !== plan.requestId ||
admission.planDigest !== plan.planDigest ||
admission.runId !== plan.runId ||
admission.sourceRunId !== plan.source.runId ||
!safeInteger(admission.admittedAtMs)
) {
throw unavailable();
}
let decision;
try {
decision = await this.#authorizer.authorize(
command.principal,
command.projectId,
permission,
);
} catch (cause) {
throw unavailable(cause);
}
if (
!decision ||
typeof decision !== 'object' ||
!['allow', 'deny', 'require_approval'].includes(decision.effect)
) {
throw unavailable();
}
if (decision.effect !== 'allow') return null;
let terminalization;
let finalization;
try {
[terminalization, finalization] = await Promise.all([
this.#terminalizations.findByRequestId(command.requestId),
this.#finalizations.findFinalization(command.requestId),
]);
} catch (cause) {
throw unavailable(cause);
}
if (terminalization && finalization) throw unavailable();
if (terminalization && !validTerminalization(terminalization, plan)) {
throw unavailable();
}
if (finalization && !validFinalization(finalization, plan)) {
throw unavailable();
}
return Object.freeze({ plan, admission, terminalization, finalization });
}
async inspect(
value: CopilotFailureDiagnosisReadTarget,
): Promise<CopilotFailureDiagnosisInspectionResult> {
const nowMs = this.#now();
if (!safeInteger(nowMs)) throw unavailable();
const command = normalizeTarget(value, nowMs);
const target = targetView(command);
const located = await this.#locate(command, 'run.read');
if (!located) return inspectionNotFound(target);
const { plan, admission, terminalization, finalization } = located;
if (!terminalization && !finalization) {
return Object.freeze({
schema: COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA,
status: 'running' as const,
...target,
diagnosisRunId: plan.runId,
outcome: null,
stage: null,
reason: null,
outputAvailable: false,
admittedAtMs: admission.admittedAtMs,
finalizedAtMs: null,
usage: null,
});
}
if (terminalization) {
return Object.freeze({
schema: COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA,
status: 'terminal' as const,
...target,
diagnosisRunId: plan.runId,
outcome: terminalization.outcome,
stage: terminalization.stage,
reason: terminalization.reason,
outputAvailable: false,
admittedAtMs: admission.admittedAtMs,
finalizedAtMs: terminalization.finalizedAtMs,
usage: Object.freeze({
inputTokens: 0,
outputTokens: 0,
totalTokens: 0,
currency: 'USD' as const,
costMicros: 0,
}),
});
}
const finalized = finalization!;
let usage;
let settlement;
try {
[usage, settlement] = await Promise.all([
this.#models.findUsage(plan.modelInvocationId),
this.#models.findPriceSettlement(plan.modelInvocationId),
]);
} catch (cause) {
throw unavailable(cause);
}
if (usage && !validUsage(usage, plan, finalized)) throw unavailable();
if (
settlement &&
(!usage || !settlementMatches(settlement, usage, finalized))
) {
throw unavailable();
}
return Object.freeze({
schema: COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA,
status: 'terminal' as const,
...target,
diagnosisRunId: plan.runId,
outcome: finalized.outcome,
stage: 'model' as const,
reason: null,
outputAvailable: finalized.outputArtifactId !== null,
admittedAtMs: admission.admittedAtMs,
finalizedAtMs: finalized.finalizedAtMs,
usage:
usage === null
? null
: Object.freeze({
inputTokens: usage.inputTokens,
outputTokens: usage.outputTokens,
totalTokens: usage.totalTokens,
currency: settlement ? ('USD' as const) : null,
costMicros: settlement?.costMicros ?? null,
}),
});
}
async readOutput(
value: CopilotFailureDiagnosisReadTarget,
): Promise<CopilotFailureDiagnosisOutputReadResult> {
const nowMs = this.#now();
if (!safeInteger(nowMs)) throw unavailable();
const command = normalizeTarget(value, nowMs);
const target = targetView(command);
const located = await this.#locate(command, 'artifact.read');
if (!located || !located.finalization) return outputNotFound(target);
const { plan, finalization } = located;
const outputArtifactId = finalization.outputArtifactId;
if (outputArtifactId === null) return outputNotFound(target);
let artifact;
try {
artifact = await this.#models.findCopilotFailureDiagnosisOutput(
outputArtifactId,
);
} catch (cause) {
throw unavailable(cause);
}
if (!artifact) return outputNotFound(target);
if (!validArtifact(artifact, plan, finalization)) throw unavailable();
let material;
try {
material = await this.#keys.resolve(artifact.keyId);
} catch (cause) {
throw unavailable(cause);
}
if (
!material ||
typeof material !== 'object' ||
Array.isArray(material) ||
material.keyId !== artifact.keyId ||
!(material.key instanceof Uint8Array) ||
material.key.byteLength !== 32
) {
try {
material?.key?.fill(0);
} catch {
// Invalid key material remains unavailable and must not escape.
}
throw unavailable();
}
try {
const opened = openCopilotFailureDiagnosisOutputArtifact(
artifact,
material.key,
);
return Object.freeze({
schema: COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA,
status: 'available' as const,
...target,
diagnosisRunId: plan.runId,
reference: Object.freeze({
artifactId: artifact.artifactId,
artifactDigest: artifact.artifactDigest,
contentDigest: artifact.contentDigest,
outputBytes: artifact.outputBytes,
sealedAtMs: artifact.sealedAtMs,
}),
result: Object.freeze({
text: opened.text,
finishReason: opened.finishReason,
usage: Object.freeze({ ...opened.usage }),
}),
});
} catch (cause) {
throw unavailable(cause);
} finally {
try {
material.key.fill(0);
} catch (cause) {
throw unavailable(cause);
}
}
}
}
@@ -0,0 +1,404 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA,
COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA,
CopilotFailureDiagnosisReadService,
CopilotFailureDiagnosisReadUnavailableError,
} = require('@qinglong/ai/failure-diagnosis-read-model');
const {
createCopilotFailureDiagnosisOutputArtifact,
} = require('@qinglong/ai/failure-diagnosis-model-execution');
const DIGEST = 'a'.repeat(64);
const COMPLETION_DIGEST = 'b'.repeat(64);
function principal() {
return {
subject: { type: 'api_app', id: 'app-1' },
authenticationId: 'credential-1',
authenticatedAtMs: 10,
expiresAtMs: 10_000,
assurance: 'service',
};
}
function target(overrides = {}) {
return {
principal: principal(),
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
...overrides,
};
}
function plan(overrides = {}) {
return {
requestId: 'diagnosis-request-1',
projectId: 'project-1',
source: { runId: 'source-run-1' },
runId: 'diagnosis-run-1',
modelStepRunId: 'model-step-1',
modelInvocationId: 'model-invocation-1',
traceId: 'trace-1',
planDigest: DIGEST,
plannedAtMs: 100,
...overrides,
};
}
function admission(overrides = {}) {
return {
requestId: 'diagnosis-request-1',
planDigest: DIGEST,
runId: 'diagnosis-run-1',
sourceRunId: 'source-run-1',
admittedAtMs: 110,
...overrides,
};
}
function finalization(overrides = {}) {
return {
requestId: 'diagnosis-request-1',
planDigest: DIGEST,
runId: 'diagnosis-run-1',
modelStepRunId: 'model-step-1',
invocationId: 'model-invocation-1',
completionDigest: COMPLETION_DIGEST,
outcome: 'succeeded',
outputArtifactId: 'cdo:artifact-1',
finalizedAtMs: 300,
...overrides,
};
}
function terminalization(overrides = {}) {
return {
requestId: 'diagnosis-request-1',
planDigest: DIGEST,
runId: 'diagnosis-run-1',
stage: 'cancellation',
reason: 'cancellation_requested',
outcome: 'cancelled',
finalizedAtMs: 250,
...overrides,
};
}
function usage(overrides = {}) {
return {
invocationId: 'model-invocation-1',
projectId: 'project-1',
runId: 'diagnosis-run-1',
stepRunId: 'model-step-1',
traceId: 'trace-1',
completionDigest: COMPLETION_DIGEST,
outcome: 'succeeded',
inputTokens: 12,
outputTokens: 8,
totalTokens: 20,
costMicros: 42,
...overrides,
};
}
function settlement(overrides = {}) {
return {
invocationId: 'model-invocation-1',
projectId: 'project-1',
completionDigest: COMPLETION_DIGEST,
currency: 'USD',
inputTokens: 12,
outputTokens: 8,
costMicros: 42,
...overrides,
};
}
function fixture(overrides = {}) {
const state = {
plan: plan(),
admission: admission(),
terminalization: null,
finalization: null,
usage: null,
settlement: null,
artifact: null,
decision: {
effect: 'allow',
reasons: ['role_grant'],
fence: {
projectVersion: 1,
bindingVersion: 1,
},
},
keyMaterial: null,
keyResolves: 0,
permissions: [],
...overrides,
};
const service = new CopilotFailureDiagnosisReadService({
admissions: {
async findPlanByRequestId() {
return state.plan;
},
async findByRequestId() {
return state.admission;
},
},
terminalizations: {
async findByRequestId() {
return state.terminalization;
},
},
finalizations: {
async findFinalization() {
return state.finalization;
},
},
models: {
async findCopilotFailureDiagnosisOutput() {
return state.artifact;
},
async findUsage() {
return state.usage;
},
async findPriceSettlement() {
return state.settlement;
},
},
authorizer: {
async authorize(_principal, _projectId, permission) {
state.permissions.push(permission);
return state.decision;
},
},
keys: {
async active() {
throw new Error('read must not request the active key');
},
async resolve() {
state.keyResolves += 1;
return state.keyMaterial;
},
},
now: () => 500,
});
return { service, state };
}
test('inspects running and pre-Model cancellation without content or model metadata', async () => {
const running = fixture();
assert.deepEqual(await running.service.inspect(target()), {
schema: COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESULT_SCHEMA,
status: 'running',
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
diagnosisRunId: 'diagnosis-run-1',
outcome: null,
stage: null,
reason: null,
outputAvailable: false,
admittedAtMs: 110,
finalizedAtMs: null,
usage: null,
});
assert.deepEqual(running.state.permissions, ['run.read']);
const cancelled = fixture({ terminalization: terminalization() });
const result = await cancelled.service.inspect(target());
assert.equal(result.status, 'terminal');
assert.equal(result.stage, 'cancellation');
assert.equal(result.reason, 'cancellation_requested');
assert.deepEqual(result.usage, {
inputTokens: 0,
outputTokens: 0,
totalTokens: 0,
currency: 'USD',
costMicros: 0,
});
assert.equal(JSON.stringify(result).includes('provider'), false);
});
test('projects only exact durable usage and price settlement', async () => {
const settled = fixture({
finalization: finalization(),
usage: usage(),
settlement: settlement(),
});
const result = await settled.service.inspect(target());
assert.equal(result.status, 'terminal');
assert.equal(result.outcome, 'succeeded');
assert.equal(result.outputAvailable, true);
assert.deepEqual(result.usage, {
inputTokens: 12,
outputTokens: 8,
totalTokens: 20,
currency: 'USD',
costMicros: 42,
});
const unpriced = fixture({
finalization: finalization(),
usage: usage({ costMicros: null }),
});
assert.deepEqual((await unpriced.service.inspect(target())).usage, {
inputTokens: 12,
outputTokens: 8,
totalTokens: 20,
currency: null,
costMicros: null,
});
const inconsistent = fixture({
finalization: finalization(),
usage: usage(),
settlement: settlement({ costMicros: 43 }),
});
await assert.rejects(
inconsistent.service.inspect(target()),
CopilotFailureDiagnosisReadUnavailableError,
);
});
test('masks absence, cross-target and current Policy denial and rejects dual terminal facts', async () => {
const absent = fixture({ plan: null, admission: null });
assert.equal((await absent.service.inspect(target())).status, 'not_found');
const incompleteAdmission = fixture({ plan: null });
await assert.rejects(
incompleteAdmission.service.inspect(target()),
CopilotFailureDiagnosisReadUnavailableError,
);
const crossProject = fixture({ plan: plan({ projectId: 'project-2' }) });
assert.equal(
(await crossProject.service.inspect(target())).status,
'not_found',
);
assert.deepEqual(crossProject.state.permissions, []);
const denied = fixture({
decision: { effect: 'deny', reasons: ['permission_missing'], fence: null },
});
assert.equal((await denied.service.inspect(target())).status, 'not_found');
const conflict = fixture({
terminalization: terminalization(),
finalization: finalization(),
});
await assert.rejects(
conflict.service.inspect(target()),
CopilotFailureDiagnosisReadUnavailableError,
);
});
test('decrypts an exact success Artifact, omits provider/model and wipes resolved key', async () => {
const encryptionKey = Buffer.alloc(32, 0x44);
const artifact = createCopilotFailureDiagnosisOutputArtifact(
{
requestId: 'diagnosis-request-1',
planDigest: DIGEST,
toolCompletionDigest: 'c'.repeat(64),
projectId: 'project-1',
runId: 'diagnosis-run-1',
stepRunId: 'model-step-1',
invocationId: 'model-invocation-1',
result: {
provider: 'private-provider',
model: 'private-model',
text: 'bounded diagnosis',
finishReason: 'stop',
usage: { inputTokens: 12, outputTokens: 8, totalTokens: 20 },
},
egressEvidence: { policyRevision: 'private-policy' },
keyId: 'output-key-1',
key: encryptionKey,
sealedAtMs: 200,
},
() => Buffer.alloc(12, 0x22),
);
const resolvedKey = Buffer.from(encryptionKey);
const exactFinalization = finalization({
outputArtifactId: artifact.artifactId,
});
const output = fixture({
finalization: exactFinalization,
artifact,
keyMaterial: { keyId: 'output-key-1', key: resolvedKey },
});
const result = await output.service.readOutput(target());
assert.equal(
result.schema,
COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESULT_SCHEMA,
);
assert.equal(result.status, 'available');
assert.equal(result.result.text, 'bounded diagnosis');
assert.equal(result.reference.artifactId, artifact.artifactId);
assert.equal(JSON.stringify(result).includes('private-provider'), false);
assert.equal(JSON.stringify(result).includes('private-model'), false);
assert.equal(
resolvedKey.every((byte) => byte === 0),
true,
);
assert.deepEqual(output.state.permissions, ['artifact.read']);
});
test('never resolves keys for denial/non-success and wipes keys after tamper failure', async () => {
const denied = fixture({
finalization: finalization(),
decision: { effect: 'deny', reasons: ['permission_missing'], fence: null },
});
assert.equal((await denied.service.readOutput(target())).status, 'not_found');
assert.equal(denied.state.keyResolves, 0);
const failed = fixture({
finalization: finalization({
outcome: 'failed',
outputArtifactId: null,
}),
});
assert.equal((await failed.service.readOutput(target())).status, 'not_found');
assert.equal(failed.state.keyResolves, 0);
const encryptionKey = Buffer.alloc(32, 0x55);
const artifact = createCopilotFailureDiagnosisOutputArtifact({
requestId: 'diagnosis-request-1',
planDigest: DIGEST,
toolCompletionDigest: 'c'.repeat(64),
projectId: 'project-1',
runId: 'diagnosis-run-1',
stepRunId: 'model-step-1',
invocationId: 'model-invocation-1',
result: {
provider: 'provider',
model: 'model',
text: 'diagnosis',
finishReason: 'stop',
usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
},
egressEvidence: { policyRevision: 'private-policy' },
keyId: 'output-key-1',
key: encryptionKey,
sealedAtMs: 200,
});
const resolvedKey = Buffer.from(encryptionKey);
const tampered = { ...artifact, ciphertext: `${artifact.ciphertext}A` };
const corrupted = fixture({
finalization: finalization({ outputArtifactId: artifact.artifactId }),
artifact: tampered,
keyMaterial: { keyId: 'output-key-1', key: resolvedKey },
});
await assert.rejects(
corrupted.service.readOutput(target()),
CopilotFailureDiagnosisReadUnavailableError,
);
assert.equal(
resolvedKey.every((byte) => byte === 0),
true,
);
});
@@ -50,6 +50,11 @@
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisRoute.js",
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisRoute.js"
},
"./copilot-read-routes": {
"types": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.d.ts",
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js",
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisReadRoutes.js"
},
"./http": {
"types": "./dist/transport/httpSurface.d.ts",
"require": "./dist/transport/httpSurface.js",
@@ -31,6 +31,10 @@ import {
type ClusterCopilotFailureDiagnosisProjection,
type CreateProductionClusterCopilotFailureDiagnosisOptions,
} from './copilot/failureDiagnosisComposition';
import {
createProductionClusterCopilotFailureDiagnosisReadService,
type CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions,
} from './copilot/failureDiagnosisReadComposition';
export interface EnabledProductionClusterAiConfig {
readonly enabled: true;
@@ -54,6 +58,11 @@ export interface ProductionClusterAiControlApplicationOptions {
readonly createCopilot?: (
options: CreateProductionClusterCopilotFailureDiagnosisOptions,
) => Promise<Readonly<CopilotFailureDiagnosisApplicationService>>;
readonly createCopilotRead?: (
options: CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions,
) => ReturnType<
typeof createProductionClusterCopilotFailureDiagnosisReadService
>;
readonly openAiDatabase?: ReturnType<typeof createPostgresDatabaseOpener>;
}
@@ -241,7 +250,9 @@ export async function startProductionClusterAiControlApplication(
Array.isArray(options) ||
typeof options.audit !== 'function'
) {
throw new TypeError('Production Cluster AI application options are invalid');
throw new TypeError(
'Production Cluster AI application options are invalid',
);
}
const startControl =
options.startControl ?? startProductionClusterControlApplication;
@@ -249,14 +260,20 @@ export async function startProductionClusterAiControlApplication(
options.bootstrapPrompt ?? bootstrapPostgresPluginPackagePromptApplication;
const createCopilot =
options.createCopilot ?? createProductionClusterCopilotFailureDiagnosis;
const createCopilotRead =
options.createCopilotRead ??
createProductionClusterCopilotFailureDiagnosisReadService;
if (
typeof startControl !== 'function' ||
typeof bootstrapPrompt !== 'function' ||
typeof createCopilot !== 'function' ||
typeof createCopilotRead !== 'function' ||
(options.openAiDatabase !== undefined &&
typeof options.openAiDatabase !== 'function')
) {
throw new TypeError('Production Cluster AI application factories are invalid');
throw new TypeError(
'Production Cluster AI application factories are invalid',
);
}
const copilotArtifactStore = options.control.workerIngress?.artifactStore;
if (
@@ -303,18 +320,27 @@ export async function startProductionClusterAiControlApplication(
let copilotApplication:
| Readonly<CopilotFailureDiagnosisApplicationService>
| undefined;
let copilotReadApplication:
| ReturnType<
typeof createProductionClusterCopilotFailureDiagnosisReadService
>
| undefined;
let copilotSuccessfulCompletion:
| CopilotFailureDiagnosisModelCompletionCoordinator
| undefined;
let stopPromise: Promise<ClusterControlStopResult> | undefined;
let promptOutputPolicy: ProjectPolicyEngine | undefined;
const promptOutputReadAuthorizer = Object.freeze({
async authorize(request: Readonly<{
principal: Parameters<ProjectPolicyEngine['authorize']>[0];
projectId: string;
}>) {
async authorize(
request: Readonly<{
principal: Parameters<ProjectPolicyEngine['authorize']>[0];
projectId: string;
}>,
) {
if (!aiDatabase) {
throw new Error('Cluster AI database is unavailable during output read');
throw new Error(
'Cluster AI database is unavailable during output read',
);
}
promptOutputPolicy ??= new ProjectPolicyEngine(
new PostgresProjectPolicyRepository(aiDatabase.pool),
@@ -361,7 +387,9 @@ export async function startProductionClusterAiControlApplication(
},
async loadProviders() {
if (!aiDatabase) {
throw new Error('Cluster AI database is unavailable during provider load');
throw new Error(
'Cluster AI database is unavailable during provider load',
);
}
const credentialStorage = new PostgresModelProviderCredentialReader(
aiDatabase.pool,
@@ -409,7 +437,11 @@ export async function startProductionClusterAiControlApplication(
throw new Error('Cluster AI Prompt application did not activate');
}
if (preparedCopilot !== undefined) {
if (!copilotSuccessfulCompletion || !aiDatabase || !copilotArtifactStore) {
if (
!copilotSuccessfulCompletion ||
!aiDatabase ||
!copilotArtifactStore
) {
throw new Error('Cluster Copilot shared authorities did not activate');
}
copilotApplication = await createCopilot({
@@ -419,6 +451,10 @@ export async function startProductionClusterAiControlApplication(
successfulCompletion: copilotSuccessfulCompletion,
artifactStore: copilotArtifactStore,
});
copilotReadApplication = createCopilotRead({
pool: aiDatabase.pool,
prepared: preparedCopilot,
});
}
controlApplication = await startControl({
...options.control,
@@ -445,11 +481,13 @@ export async function startProductionClusterAiControlApplication(
capability: promptApplication.promptExecutionOutputs,
},
}),
...(copilotApplication === undefined
...(copilotApplication === undefined ||
copilotReadApplication === undefined
? {}
: {
copilotFailureDiagnosis: {
capability: copilotApplication,
readCapability: copilotReadApplication,
},
}),
});
@@ -0,0 +1,57 @@
import { CopilotFailureDiagnosisReadService } from '@qinglong/ai/failure-diagnosis-read-model';
import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage';
import { PostgresCopilotFailureDiagnosisModelRepository } from '@qinglong/ai/postgres-failure-diagnosis-model-execution-storage';
import { PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository } from '@qinglong/ai/failure-diagnosis-pre-model-terminalization';
import {
PostgresProjectPolicyRepository,
type QingLongPostgresPool,
} from '@qinglong/cluster-postgres/runtime';
import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
import type { PreparedClusterCopilotFailureDiagnosisProjection } from './failureDiagnosisComposition';
export interface CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions {
readonly pool: QingLongPostgresPool;
readonly prepared: PreparedClusterCopilotFailureDiagnosisProjection;
}
/**
* Reuses the execution Pool, durable repositories and projected output keys;
* creating this service owns no connection, listener or background lifecycle.
*/
export function createProductionClusterCopilotFailureDiagnosisReadService(
options: CreateProductionClusterCopilotFailureDiagnosisReadServiceOptions,
): Readonly<CopilotFailureDiagnosisReadService> {
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
typeof options.pool?.query !== 'function' ||
typeof options.pool?.connect !== 'function' ||
typeof options.prepared?.outputKeys?.resolve !== 'function'
) {
throw new TypeError(
'Production Cluster Copilot failure diagnosis read dependencies are invalid',
);
}
const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository(
options.pool,
);
const models = new PostgresCopilotFailureDiagnosisModelRepository(
options.pool,
);
const terminalizations =
new PostgresCopilotFailureDiagnosisPreModelTerminalizationRepository(
options.pool,
);
return new CopilotFailureDiagnosisReadService({
admissions,
terminalizations,
finalizations: models,
models,
authorizer: new ProjectPolicyEngine(
new PostgresProjectPolicyRepository(options.pool),
),
keys: options.prepared.outputKeys,
});
}
@@ -65,6 +65,12 @@ import {
createClusterControlCopilotFailureDiagnosisRoute,
type ClusterCopilotFailureDiagnosisCapability,
} from '../copilot/failure-diagnosis/failureDiagnosisRoute';
import {
createClusterControlCopilotFailureDiagnosisInspectionRoute,
createClusterControlCopilotFailureDiagnosisOutputReadRoute,
type ClusterCopilotFailureDiagnosisInspectionCapability,
type ClusterCopilotFailureDiagnosisOutputReadCapability,
} from '../copilot/failure-diagnosis/failureDiagnosisReadRoutes';
export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
'task.get',
@@ -92,8 +98,14 @@ export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS =
'prompt.execution.output.read',
'prompt.output.read',
'copilot.failure_diagnosis.execute',
'copilot.failure_diagnosis.read',
'copilot.failure_diagnosis.output.read',
] as const);
export interface ClusterCopilotFailureDiagnosisReadCapability
extends ClusterCopilotFailureDiagnosisInspectionCapability,
ClusterCopilotFailureDiagnosisOutputReadCapability {}
export interface ProductionClusterControlAssemblyOptions {
readonly createEventId?: ClusterRunCancellationEventIdFactory;
readonly promptCatalog?: Readonly<{
@@ -116,6 +128,7 @@ export interface ProductionClusterControlAssemblyOptions {
}>;
readonly copilotFailureDiagnosis?: Readonly<{
readonly capability: ClusterCopilotFailureDiagnosisCapability;
readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
}>;
readonly workerIngress?: Readonly<{
readonly config: EnabledClusterWorkerIngressConfig;
@@ -164,6 +177,7 @@ export interface ProductionClusterControlApplicationOptions
}>;
readonly copilotFailureDiagnosis?: Readonly<{
readonly capability: ClusterCopilotFailureDiagnosisCapability;
readonly readCapability?: ClusterCopilotFailureDiagnosisReadCapability;
}>;
readonly workerIngress?: ProductionClusterWorkerIngressOptions;
}
@@ -279,6 +293,16 @@ export function createProductionClusterControlApplicationStack(
options.copilotFailureDiagnosis.capability,
),
]),
...(options.copilotFailureDiagnosis?.readCapability === undefined
? []
: [
createClusterControlCopilotFailureDiagnosisInspectionRoute(
options.copilotFailureDiagnosis.readCapability,
),
createClusterControlCopilotFailureDiagnosisOutputReadRoute(
options.copilotFailureDiagnosis.readCapability,
),
]),
];
const routes = createClusterControlRouteRegistry(routeDefinitions);
const expectedRouteCount =
@@ -288,7 +312,8 @@ export function createProductionClusterControlApplicationStack(
(options.promptExecutionInspection === undefined ? 0 : 1) +
(options.promptOutputRead === undefined ? 0 : 1) +
(options.promptExecutionOutputRead === undefined ? 0 : 1) +
(options.copilotFailureDiagnosis === undefined ? 0 : 1);
(options.copilotFailureDiagnosis === undefined ? 0 : 1) +
(options.copilotFailureDiagnosis?.readCapability === undefined ? 0 : 2);
if (routes.size !== expectedRouteCount) {
throw new Error('Production cluster-control route allowlist is incomplete');
}
@@ -0,0 +1,436 @@
// Cluster Copilot exposes separate low-sensitive status and protected output reads.
import type { SecurityPrincipal } from '@qinglong/runtime-core/security';
import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
import type {
ClusterControlAuthorizedOperationRequest,
ClusterControlRouteDefinition,
ClusterControlRouteParameters,
} from '../../transport/routeRegistry';
export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESPONSE_SCHEMA =
'qinglong/cluster-copilot-failure-diagnosis-inspection-response@v1' as const;
export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESPONSE_SCHEMA =
'qinglong/cluster-copilot-failure-diagnosis-output-read-response@v1' as const;
export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_ROUTE =
Object.freeze({
method: 'GET' as const,
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}',
operationId: 'copilot.failure_diagnosis.read',
permission: 'run.read',
projectParameter: 'projectId',
});
export const CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_ROUTE =
Object.freeze({
method: 'GET' as const,
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/output',
operationId: 'copilot.failure_diagnosis.output.read',
permission: 'artifact.read',
projectParameter: 'projectId',
});
interface ClusterCopilotFailureDiagnosisReadCommand {
readonly principal: Readonly<SecurityPrincipal>;
readonly projectId: string;
readonly sourceRunId: string;
readonly requestId: string;
}
export interface ClusterCopilotFailureDiagnosisInspectionCapability {
inspect(
command: Readonly<ClusterCopilotFailureDiagnosisReadCommand>,
): Promise<unknown>;
}
export interface ClusterCopilotFailureDiagnosisOutputReadCapability {
readOutput(
command: Readonly<ClusterCopilotFailureDiagnosisReadCommand>,
): Promise<unknown>;
}
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/;
const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
const DIGEST = /^[0-9a-f]{64}$/;
const FINISH_REASONS = new Set([
'stop',
'length',
'content_filter',
'tool_call',
'unknown',
]);
const OUTCOMES = new Set(['succeeded', 'failed', 'timed_out', 'cancelled']);
const STAGES = new Set(['model', 'tool', 'log', 'deadline', 'cancellation']);
const REASONS = new Set([
'tool_failed',
'tool_timed_out',
'log_not_found',
'log_pending',
'log_missing',
'log_retired',
'tool_budget_exhausted',
'deadline_exceeded',
'cancellation_requested',
]);
function response(
statusCode: number,
body: Readonly<Record<string, unknown>>,
): ClusterControlAdmissionResponse {
return Object.freeze({ statusCode, body: Object.freeze(body) });
}
function exactRecord(
value: unknown,
required: readonly string[],
optional: readonly string[] = [],
): Record<string, unknown> | null {
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
const record = value as Record<string, unknown>;
const actual = Object.keys(record).sort();
const expected = [
...required,
...optional.filter((key) => key in record),
].sort();
return actual.length === expected.length &&
actual.every((key, index) => key === expected[index])
? record
: null;
}
function nonNegativeInteger(value: unknown): value is number {
return Number.isSafeInteger(value) && (value as number) >= 0;
}
function target(
authorized: ClusterControlAuthorizedOperationRequest,
parameters: ClusterControlRouteParameters,
): Readonly<ClusterCopilotFailureDiagnosisReadCommand> | null {
if (
authorized.request.body !== null ||
authorized.projectId === null ||
typeof parameters.runId !== 'string' ||
!RUN_ID.test(parameters.runId) ||
typeof parameters.requestId !== 'string' ||
!IDENTITY.test(parameters.requestId)
) {
return null;
}
return Object.freeze({
principal: authorized.principal,
projectId: authorized.projectId,
sourceRunId: parameters.runId,
requestId: parameters.requestId,
});
}
function exactTarget(
value: Record<string, unknown>,
expected: Readonly<ClusterCopilotFailureDiagnosisReadCommand>,
): boolean {
return (
value.projectId === expected.projectId &&
value.sourceRunId === expected.sourceRunId &&
value.requestId === expected.requestId
);
}
function notFound(
value: unknown,
expected: Readonly<ClusterCopilotFailureDiagnosisReadCommand>,
schema: string,
): boolean {
const candidate = exactRecord(value, [
'projectId',
'requestId',
'schema',
'sourceRunId',
'status',
]);
return (
!!candidate &&
candidate.schema === schema &&
candidate.status === 'not_found' &&
exactTarget(candidate, expected)
);
}
function usageView(value: unknown): Readonly<Record<string, unknown>> | null {
const usage = exactRecord(value, [
'costMicros',
'currency',
'inputTokens',
'outputTokens',
'totalTokens',
]);
if (
!usage ||
!nonNegativeInteger(usage.inputTokens) ||
!nonNegativeInteger(usage.outputTokens) ||
!nonNegativeInteger(usage.totalTokens) ||
usage.totalTokens !== usage.inputTokens + usage.outputTokens ||
!(
(usage.currency === null && usage.costMicros === null) ||
(usage.currency === 'USD' && nonNegativeInteger(usage.costMicros))
)
) {
return null;
}
return Object.freeze({ ...usage });
}
function inspectionView(
value: unknown,
expected: Readonly<ClusterCopilotFailureDiagnosisReadCommand>,
): Readonly<Record<string, unknown>> | null {
const candidate = exactRecord(value, [
'admittedAtMs',
'diagnosisRunId',
'finalizedAtMs',
'outcome',
'outputAvailable',
'projectId',
'reason',
'requestId',
'schema',
'sourceRunId',
'stage',
'status',
'usage',
]);
if (
!candidate ||
candidate.schema !==
'qinglong/copilot-failure-diagnosis-inspection-result@v1' ||
!exactTarget(candidate, expected) ||
typeof candidate.diagnosisRunId !== 'string' ||
!RUN_ID.test(candidate.diagnosisRunId) ||
!nonNegativeInteger(candidate.admittedAtMs) ||
typeof candidate.outputAvailable !== 'boolean'
) {
return null;
}
if (candidate.status === 'running') {
if (
candidate.outcome !== null ||
candidate.stage !== null ||
candidate.reason !== null ||
candidate.outputAvailable !== false ||
candidate.finalizedAtMs !== null ||
candidate.usage !== null
) {
return null;
}
} else if (candidate.status === 'terminal') {
if (
typeof candidate.outcome !== 'string' ||
!OUTCOMES.has(candidate.outcome) ||
typeof candidate.stage !== 'string' ||
!STAGES.has(candidate.stage) ||
!nonNegativeInteger(candidate.finalizedAtMs) ||
candidate.finalizedAtMs < candidate.admittedAtMs ||
(candidate.stage === 'model') !== (candidate.reason === null) ||
(candidate.reason !== null &&
(typeof candidate.reason !== 'string' ||
!REASONS.has(candidate.reason))) ||
candidate.outputAvailable !==
(candidate.stage === 'model' && candidate.outcome === 'succeeded')
) {
return null;
}
if (candidate.usage !== null && !usageView(candidate.usage)) return null;
} else {
return null;
}
return Object.freeze({
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESPONSE_SCHEMA,
status: candidate.status,
projectId: expected.projectId,
sourceRunId: expected.sourceRunId,
requestId: expected.requestId,
diagnosisRunId: candidate.diagnosisRunId,
outcome: candidate.outcome,
stage: candidate.stage,
reason: candidate.reason,
outputAvailable: candidate.outputAvailable,
admittedAtMs: candidate.admittedAtMs,
finalizedAtMs: candidate.finalizedAtMs,
usage: candidate.usage === null ? null : usageView(candidate.usage),
});
}
function outputView(
value: unknown,
expected: Readonly<ClusterCopilotFailureDiagnosisReadCommand>,
): Readonly<Record<string, unknown>> | null {
const candidate = exactRecord(value, [
'diagnosisRunId',
'projectId',
'reference',
'requestId',
'result',
'schema',
'sourceRunId',
'status',
]);
const reference = candidate
? exactRecord(candidate.reference, [
'artifactDigest',
'artifactId',
'contentDigest',
'outputBytes',
'sealedAtMs',
])
: null;
const result = candidate
? exactRecord(candidate.result, ['finishReason', 'text', 'usage'])
: null;
const usage = result
? exactRecord(
result.usage,
['inputTokens', 'outputTokens', 'totalTokens'],
['costMicros'],
)
: null;
if (
!candidate ||
!reference ||
!result ||
!usage ||
candidate.schema !==
'qinglong/copilot-failure-diagnosis-output-read-result@v1' ||
candidate.status !== 'available' ||
!exactTarget(candidate, expected) ||
typeof candidate.diagnosisRunId !== 'string' ||
!RUN_ID.test(candidate.diagnosisRunId) ||
typeof reference.artifactId !== 'string' ||
!IDENTITY.test(reference.artifactId) ||
typeof reference.artifactDigest !== 'string' ||
!DIGEST.test(reference.artifactDigest) ||
typeof reference.contentDigest !== 'string' ||
!DIGEST.test(reference.contentDigest) ||
!nonNegativeInteger(reference.outputBytes) ||
reference.outputBytes > 1024 * 1024 ||
!nonNegativeInteger(reference.sealedAtMs) ||
typeof result.text !== 'string' ||
Buffer.byteLength(result.text, 'utf8') !== reference.outputBytes ||
typeof result.finishReason !== 'string' ||
!FINISH_REASONS.has(result.finishReason) ||
!nonNegativeInteger(usage.inputTokens) ||
!nonNegativeInteger(usage.outputTokens) ||
!nonNegativeInteger(usage.totalTokens) ||
usage.totalTokens !== usage.inputTokens + usage.outputTokens ||
(usage.costMicros !== undefined && !nonNegativeInteger(usage.costMicros))
) {
return null;
}
return Object.freeze({
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESPONSE_SCHEMA,
status: 'available',
projectId: expected.projectId,
sourceRunId: expected.sourceRunId,
requestId: expected.requestId,
diagnosisRunId: candidate.diagnosisRunId,
reference: Object.freeze({ ...reference }),
result: Object.freeze({
text: result.text,
finishReason: result.finishReason,
usage: Object.freeze({ ...usage }),
}),
});
}
export function createClusterControlCopilotFailureDiagnosisInspectionRoute(
capability: ClusterCopilotFailureDiagnosisInspectionCapability,
): Readonly<ClusterControlRouteDefinition> {
if (!capability || typeof capability.inspect !== 'function') {
throw new TypeError(
'Cluster-control Copilot diagnosis inspection capability is invalid',
);
}
return Object.freeze({
...CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_ROUTE,
async handle(
authorized: ClusterControlAuthorizedOperationRequest,
parameters: ClusterControlRouteParameters,
) {
const command = target(authorized, parameters);
if (!command) {
return response(400, {
code: 'invalid_copilot_failure_diagnosis_read_request',
});
}
try {
const result = await capability.inspect(command);
if (
notFound(
result,
command,
'qinglong/copilot-failure-diagnosis-inspection-result@v1',
)
) {
return response(404, { code: 'copilot_failure_diagnosis_not_found' });
}
const view = inspectionView(result, command);
return view
? response(200, view)
: response(503, {
code: 'copilot_failure_diagnosis_read_unavailable',
});
} catch {
return response(503, {
code: 'copilot_failure_diagnosis_read_unavailable',
});
}
},
});
}
export function createClusterControlCopilotFailureDiagnosisOutputReadRoute(
capability: ClusterCopilotFailureDiagnosisOutputReadCapability,
): Readonly<ClusterControlRouteDefinition> {
if (!capability || typeof capability.readOutput !== 'function') {
throw new TypeError(
'Cluster-control Copilot diagnosis output read capability is invalid',
);
}
return Object.freeze({
...CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_ROUTE,
async handle(
authorized: ClusterControlAuthorizedOperationRequest,
parameters: ClusterControlRouteParameters,
) {
const command = target(authorized, parameters);
if (!command) {
return response(400, {
code: 'invalid_copilot_failure_diagnosis_output_read_request',
});
}
try {
const result = await capability.readOutput(command);
if (
notFound(
result,
command,
'qinglong/copilot-failure-diagnosis-output-read-result@v1',
)
) {
return response(404, {
code: 'copilot_failure_diagnosis_output_not_found',
});
}
const view = outputView(result, command);
return view
? response(200, view)
: response(503, {
code: 'copilot_failure_diagnosis_output_read_unavailable',
});
} catch {
return response(503, {
code: 'copilot_failure_diagnosis_output_read_unavailable',
});
}
},
});
}
@@ -29,7 +29,8 @@ const {
function enabledEnvironment(overrides = {}) {
return {
QL3_CLUSTER_AI_ENABLED: 'true',
QL3_CLUSTER_AI_PROVIDER_AUTHORITY_FILE: '/var/run/qinglong/ai/providers.json',
QL3_CLUSTER_AI_PROVIDER_AUTHORITY_FILE:
'/var/run/qinglong/ai/providers.json',
QL3_CLUSTER_AI_SECRET_ROOT: '/var/run/qinglong/ai/provider-secrets',
...overrides,
};
@@ -120,26 +121,30 @@ async function projectedFile(root, name, bytes) {
test('Copilot composition is explicit, shares the Prompt gateway and injects one route capability', async () => {
const secretRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-secret-'));
const configRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-config-'));
const invocationRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-invocation-'));
const invocationRoot = await mkdtemp(
join(tmpdir(), 'ql3-copilot-invocation-'),
);
const resultRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-result-'));
const outputRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-output-'));
const key = Buffer.alloc(32, 0x55).toString('base64url');
const config = Buffer.from(`${JSON.stringify({
schema: 'qinglong/cluster-copilot-failure-diagnosis-config@v1',
provider: 'provider-primary',
model: 'model-diagnosis',
modelBoundary: 'external',
responseLanguage: 'zh-CN',
maxOutputTokens: 512,
executionTimeoutMs: 60_000,
egressPolicy: {
schema: 'qinglong/copilot-model-egress-policy@v1',
revision: 'cluster-copilot-v1',
potentiallySensitiveDataBoundaries: ['external'],
maxInputBytes: 64 * 1024,
maxOutputTokens: 1024,
},
})}\n`);
const config = Buffer.from(
`${JSON.stringify({
schema: 'qinglong/cluster-copilot-failure-diagnosis-config@v1',
provider: 'provider-primary',
model: 'model-diagnosis',
modelBoundary: 'external',
responseLanguage: 'zh-CN',
maxOutputTokens: 512,
executionTimeoutMs: 60_000,
egressPolicy: {
schema: 'qinglong/copilot-model-egress-policy@v1',
revision: 'cluster-copilot-v1',
potentiallySensitiveDataBoundaries: ['external'],
maxInputBytes: 64 * 1024,
maxOutputTokens: 1024,
},
})}\n`,
);
const invocation = canonicalClusterToolInvocationKeyringManifest({
schema: CLUSTER_TOOL_INVOCATION_KEYRING_MANIFEST_SCHEMA,
activeKeyId: 'invocation-key-1',
@@ -163,8 +168,10 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
const fakePool = { query() {}, connect() {} };
const artifactStore = { put() {}, inspect() {}, readLogRange() {} };
const copilot = Object.freeze({ execute() {} });
const copilotRead = Object.freeze({ inspect() {}, readOutput() {} });
let registeredSink;
let created;
let createdRead;
let controlOptions;
try {
await Promise.all([
@@ -205,22 +212,41 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
async recordWithAtomicSuccess() {},
});
return {
status: 'active', profile: 'cluster', readiness: {}, capability: gateway,
prompts: {}, promptCatalog: {}, promptExecutions: {},
promptExecutionInspections: {}, async stop() { return 'stopped'; },
status: 'active',
profile: 'cluster',
readiness: {},
capability: gateway,
prompts: {},
promptCatalog: {},
promptExecutions: {},
promptExecutionInspections: {},
async stop() {
return 'stopped';
},
};
},
async createCopilot(options) {
created = options;
return copilot;
},
createCopilotRead(options) {
createdRead = options;
return copilotRead;
},
async startControl(options) {
controlOptions = options;
return {
status: 'active', address: { host: '127.0.0.1', port: 5800 },
evidence: {}, recovery: { safe: true, remaining: 0, failed: 0 },
unavailable: new Promise(() => {}), availabilityStatus() { return 'ready'; },
async stop() { return 'stopped'; },
status: 'active',
address: { host: '127.0.0.1', port: 5800 },
evidence: {},
recovery: { safe: true, remaining: 0, failed: 0 },
unavailable: new Promise(() => {}),
availabilityStatus() {
return 'ready';
},
async stop() {
return 'stopped';
},
};
},
});
@@ -229,10 +255,19 @@ test('Copilot composition is explicit, shares the Prompt gateway and injects one
assert.equal(created.gateway, gateway);
assert.equal(created.successfulCompletion, registeredSink);
assert.equal(created.artifactStore, artifactStore);
assert.equal(createdRead.pool, fakePool);
assert.equal(typeof createdRead.prepared.outputKeys.resolve, 'function');
assert.equal(controlOptions.copilotFailureDiagnosis.capability, copilot);
assert.equal(
controlOptions.copilotFailureDiagnosis.readCapability,
copilotRead,
);
assert.equal(await application.stop(), 'stopped');
} finally {
config.fill(0); invocation.fill(0); result.fill(0); output.fill(0);
config.fill(0);
invocation.fill(0);
result.fill(0);
output.fill(0);
await Promise.all([
rm(secretRoot, { recursive: true, force: true }),
rm(configRoot, { recursive: true, force: true }),
@@ -362,7 +397,9 @@ test('output-enabled AI composition wires exact and request-keyed protected read
promptExecutionInspections: { inspectAuthorized() {} },
promptOutputs,
promptExecutionOutputs,
async stop() { return 'stopped'; },
async stop() {
return 'stopped';
},
};
},
async startControl(options) {
@@ -373,8 +410,12 @@ test('output-enabled AI composition wires exact and request-keyed protected read
evidence: {},
recovery: { safe: true, remaining: 0, failed: 0 },
unavailable: new Promise(() => {}),
availabilityStatus() { return 'ready'; },
async stop() { return 'stopped'; },
availabilityStatus() {
return 'ready';
},
async stop() {
return 'stopped';
},
};
},
});
@@ -0,0 +1,288 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESPONSE_SCHEMA,
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESPONSE_SCHEMA,
CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_ROUTE,
CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_ROUTE,
createClusterControlCopilotFailureDiagnosisInspectionRoute,
createClusterControlCopilotFailureDiagnosisOutputReadRoute,
} = require('@qinglong/cluster-control/copilot-read-routes');
function authorized(path, body = null) {
return {
request: {
requestId: 'transport-request-1',
method: 'GET',
path,
query: {},
headers: {},
signal: new AbortController().signal,
body,
},
principal: {
subject: { type: 'api_app', id: 'app-1' },
authenticationId: 'credential-1',
authenticatedAtMs: 1,
expiresAtMs: 10_000,
assurance: 'service',
},
operationId: 'copilot.failure_diagnosis.read',
permission: 'run.read',
projectId: 'project-1',
policyFence: { projectVersion: 3, bindingVersion: 7 },
};
}
const parameters = {
projectId: 'project-1',
runId: 'source-run-1',
requestId: 'diagnosis-request-1',
};
function running(overrides = {}) {
return {
schema: 'qinglong/copilot-failure-diagnosis-inspection-result@v1',
status: 'running',
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
diagnosisRunId: 'diagnosis-run-1',
outcome: null,
stage: null,
reason: null,
outputAvailable: false,
admittedAtMs: 100,
finalizedAtMs: null,
usage: null,
...overrides,
};
}
test('defines separate run.read inspection and artifact.read output routes', () => {
assert.deepEqual(CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_ROUTE, {
method: 'GET',
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}',
operationId: 'copilot.failure_diagnosis.read',
permission: 'run.read',
projectParameter: 'projectId',
});
assert.deepEqual(
CLUSTER_CONTROL_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_ROUTE,
{
method: 'GET',
path: '/api/v3/projects/{projectId}/runs/{runId}/copilot/failure-diagnoses/{requestId}/output',
operationId: 'copilot.failure_diagnosis.output.read',
permission: 'artifact.read',
projectParameter: 'projectId',
},
);
});
test('projects a request-keyed running inspection and passes only trusted target facts', async () => {
let command;
const route = createClusterControlCopilotFailureDiagnosisInspectionRoute({
async inspect(value) {
command = value;
return running();
},
});
const request = authorized(
'/api/v3/projects/project-1/runs/source-run-1/copilot/failure-diagnoses/diagnosis-request-1',
);
const result = await route.handle(request, parameters);
assert.deepEqual(command, {
principal: request.principal,
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
});
assert.deepEqual(result, {
statusCode: 200,
body: {
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_INSPECTION_RESPONSE_SCHEMA,
status: 'running',
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
diagnosisRunId: 'diagnosis-run-1',
outcome: null,
stage: null,
reason: null,
outputAvailable: false,
admittedAtMs: 100,
finalizedAtMs: null,
usage: null,
},
});
});
test('projects terminal cancellation and settled Model usage without private fields', async () => {
for (const [value, expected] of [
[
running({
status: 'terminal',
outcome: 'cancelled',
stage: 'cancellation',
reason: 'cancellation_requested',
finalizedAtMs: 200,
usage: {
inputTokens: 0,
outputTokens: 0,
totalTokens: 0,
currency: 'USD',
costMicros: 0,
},
}),
'cancellation',
],
[
running({
status: 'terminal',
outcome: 'succeeded',
stage: 'model',
reason: null,
outputAvailable: true,
finalizedAtMs: 200,
usage: {
inputTokens: 11,
outputTokens: 7,
totalTokens: 18,
currency: 'USD',
costMicros: 29,
},
}),
'model',
],
]) {
const route = createClusterControlCopilotFailureDiagnosisInspectionRoute({
async inspect() {
return value;
},
});
const result = await route.handle(authorized('/read'), parameters);
assert.equal(result.statusCode, 200);
assert.equal(result.body.stage, expected);
assert.equal(JSON.stringify(result).includes('provider'), false);
assert.equal(JSON.stringify(result).includes('modelId'), false);
}
});
test('masks absent reads and fails closed on invalid input or widened results', async () => {
let calls = 0;
const route = createClusterControlCopilotFailureDiagnosisInspectionRoute({
async inspect() {
calls += 1;
return {
schema: 'qinglong/copilot-failure-diagnosis-inspection-result@v1',
status: 'not_found',
projectId: 'project-1',
sourceRunId: 'source-run-1',
requestId: 'diagnosis-request-1',
};
},
});
assert.equal(
(await route.handle(authorized('/read'), parameters)).statusCode,
404,
);
assert.equal(
(await route.handle(authorized('/read', {}), parameters)).statusCode,
400,
);
assert.equal(
(
await route.handle(authorized('/read'), {
...parameters,
requestId: '../private',
})
).statusCode,
400,
);
assert.equal(calls, 1);
const widened = createClusterControlCopilotFailureDiagnosisInspectionRoute({
async inspect() {
return running({ privateModel: 'must not cross' });
},
});
assert.equal(
(await widened.handle(authorized('/read'), parameters)).statusCode,
503,
);
});
test('returns only decrypted diagnosis content and low-sensitive Artifact metadata', async () => {
let command;
const route = createClusterControlCopilotFailureDiagnosisOutputReadRoute({
async readOutput(value) {
command = value;
return {
schema: 'qinglong/copilot-failure-diagnosis-output-read-result@v1',
status: 'available',
projectId: value.projectId,
sourceRunId: value.sourceRunId,
requestId: value.requestId,
diagnosisRunId: 'diagnosis-run-1',
reference: {
artifactId: 'cdo:artifact-1',
artifactDigest: 'a'.repeat(64),
contentDigest: 'b'.repeat(64),
outputBytes: Buffer.byteLength('diagnosis'),
sealedAtMs: 200,
},
result: {
text: 'diagnosis',
finishReason: 'stop',
usage: { inputTokens: 3, outputTokens: 2, totalTokens: 5 },
},
};
},
});
const request = authorized('/output');
request.operationId = 'copilot.failure_diagnosis.output.read';
request.permission = 'artifact.read';
const result = await route.handle(request, parameters);
assert.equal(result.statusCode, 200);
assert.equal(
result.body.schema,
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_READ_RESPONSE_SCHEMA,
);
assert.equal(result.body.result.text, 'diagnosis');
assert.equal('provider' in result.body.result, false);
assert.equal('model' in result.body.result, false);
assert.equal(command.principal, request.principal);
});
test('masks absent output and maps dependency/cipher failures to one 503 code', async () => {
const absent = createClusterControlCopilotFailureDiagnosisOutputReadRoute({
async readOutput(value) {
return {
schema: 'qinglong/copilot-failure-diagnosis-output-read-result@v1',
status: 'not_found',
projectId: value.projectId,
sourceRunId: value.sourceRunId,
requestId: value.requestId,
};
},
});
assert.equal(
(await absent.handle(authorized('/output'), parameters)).statusCode,
404,
);
const unavailable =
createClusterControlCopilotFailureDiagnosisOutputReadRoute({
async readOutput() {
throw new Error('private key failure');
},
});
assert.deepEqual(
await unavailable.handle(authorized('/output'), parameters),
{
statusCode: 503,
body: { code: 'copilot_failure_diagnosis_output_read_unavailable' },
},
);
});
@@ -730,6 +730,8 @@ test('optionally exposes Prompt execution behind shared admission and policy', a
'prompt.execution.output.read',
'prompt.output.read',
'copilot.failure_diagnosis.execute',
'copilot.failure_diagnosis.read',
'copilot.failure_diagnosis.output.read',
]);
const response = await invoke(
stack,
@@ -783,9 +785,30 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy
terminalizationRequired: false,
};
},
async inspect(value) {
return {
schema: 'qinglong/copilot-failure-diagnosis-inspection-result@v1',
status: 'not_found',
projectId: value.projectId,
sourceRunId: value.sourceRunId,
requestId: value.requestId,
};
},
async readOutput(value) {
return {
schema: 'qinglong/copilot-failure-diagnosis-output-read-result@v1',
status: 'not_found',
projectId: value.projectId,
sourceRunId: value.sourceRunId,
requestId: value.requestId,
};
},
};
const stack = createProductionClusterControlApplicationStack(input, {
copilotFailureDiagnosis: { capability },
copilotFailureDiagnosis: {
capability,
readCapability: capability,
},
});
const result = await invoke(
stack,
@@ -811,6 +834,29 @@ test('optionally exposes Copilot diagnosis behind shared authentication, Policy
'audit:copilot.failure_diagnosis.execute:allowed',
'diagnose:run-1',
]);
const inspection = await invoke(
stack,
metadata(
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1',
),
);
const output = await invoke(
stack,
metadata(
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/output',
),
);
assert.equal(inspection.statusCode, 404);
assert.equal(output.statusCode, 404);
assert.equal(
events.includes('audit:copilot.failure_diagnosis.read:allowed'),
true,
);
assert.equal(
events.includes('audit:copilot.failure_diagnosis.output.read:allowed'),
true,
);
});
test('keeps the Copilot route absent by default and never invokes it after Policy denial', async () => {
@@ -830,6 +876,15 @@ test('keeps the Copilot route absent by default and never invokes it after Polic
defaultStack.admission.prepare(request),
(error) => error?.statusCode === 404 && error?.code === 'route_not_found',
);
for (const path of [
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1',
'/api/v3/projects/project-1/runs/run-1/copilot/failure-diagnoses/diagnosis-request-1/output',
]) {
await assert.rejects(
defaultStack.admission.prepare(metadata(path)),
(error) => error?.statusCode === 404 && error?.code === 'route_not_found',
);
}
let calls = 0;
const deniedFixture = fixture({