mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-23 12:05:27 +08:00
feat(ql3): compose cluster copilot diagnosis
This commit is contained in:
whyour
@@ -55,6 +55,11 @@
|
||||
"require": "./dist/copilot/failure-diagnosis/modelExecution.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/modelExecution.js"
|
||||
},
|
||||
"./failure-diagnosis-application": {
|
||||
"types": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/failureDiagnosisApplication.js"
|
||||
},
|
||||
"./postgres-failure-diagnosis-model-execution-storage": {
|
||||
"types": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.js",
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import type { SecurityPrincipal } from '@qinglong/runtime-core/security';
|
||||
|
||||
import type { CopilotFailureDiagnosisAdmissionReceipt } from '../admission/contracts';
|
||||
import type { CopilotFailureDiagnosisModelExecutionResult } from '../model-execution/coordinator';
|
||||
import type { CopilotFailureDiagnosisToolExecutionResult } from '../tool-execution/contracts';
|
||||
|
||||
export const MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS = 64;
|
||||
|
||||
export interface ExecuteCopilotFailureDiagnosisApplicationCommand {
|
||||
readonly requestId: string;
|
||||
readonly traceId: string;
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
}
|
||||
|
||||
export interface ExecuteCopilotFailureDiagnosisApplicationResult {
|
||||
readonly admissionStatus: 'created' | 'existing';
|
||||
readonly admission: Readonly<CopilotFailureDiagnosisAdmissionReceipt>;
|
||||
readonly tool: Readonly<CopilotFailureDiagnosisToolExecutionResult>;
|
||||
readonly model: Readonly<CopilotFailureDiagnosisModelExecutionResult> | null;
|
||||
readonly terminalizationRequired: boolean;
|
||||
}
|
||||
|
||||
export class InvalidCopilotFailureDiagnosisApplicationError extends TypeError {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_INVALID';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Copilot failure diagnosis application is invalid: ${message}`);
|
||||
this.name = 'InvalidCopilotFailureDiagnosisApplicationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisApplicationConflictError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_CONFLICT';
|
||||
|
||||
constructor(message = 'the durable diagnosis request changed') {
|
||||
super(`Copilot failure diagnosis application conflicts: ${message}`);
|
||||
this.name = 'CopilotFailureDiagnosisApplicationConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisApplicationUnavailableError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Copilot failure diagnosis application is unavailable', options);
|
||||
this.name = 'CopilotFailureDiagnosisApplicationUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisApplicationBusyError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_BUSY';
|
||||
|
||||
constructor() {
|
||||
super('Copilot failure diagnosis application request budget is exhausted');
|
||||
this.name = 'CopilotFailureDiagnosisApplicationBusyError';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,572 @@
|
||||
import { Buffer } from 'node:buffer';
|
||||
import { createHash, createHmac } from 'node:crypto';
|
||||
|
||||
import {
|
||||
BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||
createBuiltInRunLogExcerptToolHandlerBinding,
|
||||
} from '@qinglong/runtime-core/builtin-run-log-excerpt-tool';
|
||||
import type { RunRepositoryReader } from '@qinglong/runtime-core/run-repository';
|
||||
import type { ToolPolicyAuthorizer } from '@qinglong/runtime-core/tool-registry';
|
||||
import {
|
||||
prepareToolInvocation,
|
||||
} from '@qinglong/runtime-core/tool-registry';
|
||||
import type { ProjectToolDefinitionSnapshotRepository } from '@qinglong/runtime-core/project-tool-definition-snapshot';
|
||||
import { projectToolDefinitionRegistry } from '@qinglong/runtime-core/project-tool-definition-snapshot';
|
||||
import {
|
||||
createToolInvocationInputArtifact,
|
||||
createToolInvocationPreviewArtifact,
|
||||
type ToolInvocationArtifactKeyProvider,
|
||||
type ToolInvocationArtifactRepository,
|
||||
type ToolInvocationPreviewDocument,
|
||||
} from '@qinglong/runtime-core/tool-invocation-artifact';
|
||||
import {
|
||||
TrustedToolHandlerBindingRegistry,
|
||||
createTrustedToolInvocationPlan,
|
||||
} from '@qinglong/runtime-core/trusted-tool-invocation';
|
||||
|
||||
import { MAX_MODEL_INVOCATION_MS } from '../../../model-gateway/model';
|
||||
import {
|
||||
prepareCopilotFailureDiagnosisExecution,
|
||||
} from '../admission/plan';
|
||||
import type {
|
||||
CopilotFailureDiagnosisAdmissionRepository,
|
||||
CopilotFailureDiagnosisExecutionPlan,
|
||||
PrepareCopilotFailureDiagnosisModelIntent,
|
||||
} from '../admission/contracts';
|
||||
import {
|
||||
executeCopilotFailureDiagnosisTool,
|
||||
type CopilotFailureDiagnosisToolExecutionDependencies,
|
||||
} from '../tool-execution/coordinator';
|
||||
import {
|
||||
executeCopilotFailureDiagnosisModel,
|
||||
type CopilotFailureDiagnosisModelExecutionDependencies,
|
||||
} from '../model-execution/coordinator';
|
||||
import {
|
||||
CopilotFailureDiagnosisApplicationBusyError,
|
||||
CopilotFailureDiagnosisApplicationConflictError,
|
||||
CopilotFailureDiagnosisApplicationUnavailableError,
|
||||
InvalidCopilotFailureDiagnosisApplicationError,
|
||||
MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS,
|
||||
type ExecuteCopilotFailureDiagnosisApplicationCommand,
|
||||
type ExecuteCopilotFailureDiagnosisApplicationResult,
|
||||
} from './contracts';
|
||||
|
||||
const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
|
||||
const NONCE_DOMAIN = Buffer.from(
|
||||
'qinglong/copilot-failure-diagnosis-tool-invocation-nonce@v1\0',
|
||||
'utf8',
|
||||
);
|
||||
const IDENTITY_DOMAIN = Buffer.from(
|
||||
'qinglong/copilot-failure-diagnosis-application-identity@v1\0',
|
||||
'utf8',
|
||||
);
|
||||
|
||||
export interface CopilotFailureDiagnosisApplicationDependencies {
|
||||
readonly admissions: CopilotFailureDiagnosisAdmissionRepository;
|
||||
readonly snapshots: Pick<ProjectToolDefinitionSnapshotRepository, 'findCurrent'>;
|
||||
readonly runs: Pick<
|
||||
RunRepositoryReader,
|
||||
'findRunById' | 'findLatestAttemptByRunId'
|
||||
>;
|
||||
readonly artifacts: ToolInvocationArtifactRepository;
|
||||
readonly invocationKeys: Pick<
|
||||
ToolInvocationArtifactKeyProvider,
|
||||
'active' | 'resolve'
|
||||
>;
|
||||
readonly authorizer: ToolPolicyAuthorizer;
|
||||
readonly tool: CopilotFailureDiagnosisToolExecutionDependencies;
|
||||
readonly model: CopilotFailureDiagnosisModelExecutionDependencies;
|
||||
readonly executeTool: typeof executeCopilotFailureDiagnosisTool;
|
||||
readonly executeModel: typeof executeCopilotFailureDiagnosisModel;
|
||||
readonly modelIntent: Readonly<PrepareCopilotFailureDiagnosisModelIntent>;
|
||||
readonly executionTimeoutMs: number;
|
||||
readonly now?: () => number;
|
||||
readonly nonceFactory?: (input: Readonly<{
|
||||
key: Uint8Array;
|
||||
keyId: string;
|
||||
requestId: string;
|
||||
projectId: string;
|
||||
sourceRunId: string;
|
||||
invocationActionDigest: string;
|
||||
}>) => Uint8Array;
|
||||
}
|
||||
|
||||
interface ActiveRequest {
|
||||
readonly digest: string;
|
||||
readonly promise: Promise<
|
||||
Readonly<ExecuteCopilotFailureDiagnosisApplicationResult>
|
||||
>;
|
||||
}
|
||||
|
||||
function invalid(message: string): never {
|
||||
throw new InvalidCopilotFailureDiagnosisApplicationError(message);
|
||||
}
|
||||
|
||||
function unavailable(cause?: unknown): never {
|
||||
throw new CopilotFailureDiagnosisApplicationUnavailableError({
|
||||
cause: cause instanceof Error ? cause : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function digest(value: unknown): string {
|
||||
return createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
||||
}
|
||||
|
||||
function identity(prefix: string, requestId: string): string {
|
||||
return `${prefix}:${createHash('sha256')
|
||||
.update(IDENTITY_DOMAIN)
|
||||
.update(prefix)
|
||||
.update('\0')
|
||||
.update(requestId)
|
||||
.digest('hex')
|
||||
.slice(0, 32)}`;
|
||||
}
|
||||
|
||||
function preview(
|
||||
sourceRunId: string,
|
||||
sourceAttemptId: string,
|
||||
): Readonly<ToolInvocationPreviewDocument> {
|
||||
return Object.freeze({
|
||||
title: 'Diagnose failed Run',
|
||||
summary: 'Read one bounded, redacted and untrusted execution log excerpt',
|
||||
fields: Object.freeze([
|
||||
Object.freeze({
|
||||
kind: 'identifier' as const,
|
||||
label: 'Run',
|
||||
value: sourceRunId,
|
||||
}),
|
||||
Object.freeze({
|
||||
kind: 'identifier' as const,
|
||||
label: 'Attempt',
|
||||
value: sourceAttemptId,
|
||||
}),
|
||||
]),
|
||||
warnings: Object.freeze(['potentially_sensitive_output']),
|
||||
});
|
||||
}
|
||||
|
||||
function defaultNonce(input: Readonly<{
|
||||
key: Uint8Array;
|
||||
keyId: string;
|
||||
requestId: string;
|
||||
projectId: string;
|
||||
sourceRunId: string;
|
||||
invocationActionDigest: string;
|
||||
}>): Uint8Array {
|
||||
const derived = createHmac('sha256', Buffer.from(input.key))
|
||||
.update(NONCE_DOMAIN)
|
||||
.update(
|
||||
JSON.stringify({
|
||||
keyId: input.keyId,
|
||||
requestId: input.requestId,
|
||||
projectId: input.projectId,
|
||||
sourceRunId: input.sourceRunId,
|
||||
invocationActionDigest: input.invocationActionDigest,
|
||||
}),
|
||||
)
|
||||
.digest();
|
||||
try {
|
||||
return Buffer.from(derived.subarray(0, 12));
|
||||
} finally {
|
||||
derived.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeCommand(
|
||||
value: ExecuteCopilotFailureDiagnosisApplicationCommand,
|
||||
): Readonly<ExecuteCopilotFailureDiagnosisApplicationCommand> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).sort().join('\0') !==
|
||||
['principal', 'projectId', 'requestId', 'sourceRunId', 'traceId'].join(
|
||||
'\0',
|
||||
) ||
|
||||
!ID_PATTERN.test(value.requestId) ||
|
||||
!ID_PATTERN.test(value.traceId) ||
|
||||
!ID_PATTERN.test(value.projectId) ||
|
||||
!RUN_ID_PATTERN.test(value.sourceRunId) ||
|
||||
!value.principal ||
|
||||
typeof value.principal !== 'object' ||
|
||||
Array.isArray(value.principal)
|
||||
) {
|
||||
return invalid('command is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
requestId: value.requestId,
|
||||
traceId: value.traceId,
|
||||
projectId: value.projectId,
|
||||
sourceRunId: value.sourceRunId,
|
||||
principal: value.principal,
|
||||
});
|
||||
}
|
||||
|
||||
function assertDependencies(
|
||||
value: CopilotFailureDiagnosisApplicationDependencies,
|
||||
): void {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
typeof value.admissions?.findByRequestId !== 'function' ||
|
||||
typeof value.admissions?.findPlanByRequestId !== 'function' ||
|
||||
typeof value.admissions?.admit !== 'function' ||
|
||||
typeof value.snapshots?.findCurrent !== 'function' ||
|
||||
typeof value.runs?.findRunById !== 'function' ||
|
||||
typeof value.runs?.findLatestAttemptByRunId !== 'function' ||
|
||||
typeof value.artifacts?.put !== 'function' ||
|
||||
typeof value.invocationKeys?.active !== 'function' ||
|
||||
typeof value.invocationKeys?.resolve !== 'function' ||
|
||||
typeof value.authorizer?.authorize !== 'function' ||
|
||||
typeof value.executeTool !== 'function' ||
|
||||
typeof value.executeModel !== 'function' ||
|
||||
!value.tool ||
|
||||
!value.model ||
|
||||
!value.modelIntent ||
|
||||
typeof value.modelIntent !== 'object' ||
|
||||
!Number.isSafeInteger(value.executionTimeoutMs) ||
|
||||
value.executionTimeoutMs < 1 ||
|
||||
value.executionTimeoutMs > MAX_MODEL_INVOCATION_MS ||
|
||||
(value.now !== undefined && typeof value.now !== 'function') ||
|
||||
(value.nonceFactory !== undefined &&
|
||||
typeof value.nonceFactory !== 'function')
|
||||
) {
|
||||
return invalid('dependencies are invalid');
|
||||
}
|
||||
if (
|
||||
value.tool.admissions !== value.admissions ||
|
||||
value.tool.snapshots !== value.snapshots ||
|
||||
value.tool.runs !== value.runs ||
|
||||
value.tool.artifacts !== value.artifacts ||
|
||||
value.tool.invocationKeys !== value.invocationKeys ||
|
||||
value.model.admissions !== value.admissions ||
|
||||
value.model.unlocks !== value.tool.unlocks
|
||||
) {
|
||||
return invalid('dependency authorities are not shared');
|
||||
}
|
||||
}
|
||||
|
||||
function sameSubject(
|
||||
left: Readonly<{ type: string; id: string }>,
|
||||
right: Readonly<{ type: string; id: string }>,
|
||||
): boolean {
|
||||
return left.type === right.type && left.id === right.id;
|
||||
}
|
||||
|
||||
function sameModelIntent(
|
||||
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
|
||||
configured: Readonly<PrepareCopilotFailureDiagnosisModelIntent>,
|
||||
): boolean {
|
||||
return (
|
||||
plan.model.provider === configured.provider &&
|
||||
plan.model.model === configured.model &&
|
||||
plan.model.modelBoundary === configured.modelBoundary &&
|
||||
plan.model.responseLanguage === configured.responseLanguage &&
|
||||
plan.model.maxOutputTokens === configured.maxOutputTokens &&
|
||||
JSON.stringify(plan.model.egressPolicy) ===
|
||||
JSON.stringify(configured.egressPolicy)
|
||||
);
|
||||
}
|
||||
|
||||
function nonceInput(
|
||||
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
|
||||
key: Uint8Array,
|
||||
) {
|
||||
return Object.freeze({
|
||||
key,
|
||||
keyId: plan.tool.invocationArtifact.keyId,
|
||||
requestId: plan.requestId,
|
||||
projectId: plan.projectId,
|
||||
sourceRunId: plan.source.runId,
|
||||
invocationActionDigest: plan.tool.invocationActionDigest,
|
||||
});
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisApplicationService {
|
||||
readonly #dependencies: CopilotFailureDiagnosisApplicationDependencies;
|
||||
readonly #active = new Map<string, ActiveRequest>();
|
||||
|
||||
constructor(dependencies: CopilotFailureDiagnosisApplicationDependencies) {
|
||||
assertDependencies(dependencies);
|
||||
this.#dependencies = dependencies;
|
||||
}
|
||||
|
||||
execute(
|
||||
commandValue: ExecuteCopilotFailureDiagnosisApplicationCommand,
|
||||
): Promise<Readonly<ExecuteCopilotFailureDiagnosisApplicationResult>> {
|
||||
const command = normalizeCommand(commandValue);
|
||||
const commandDigest = digest(command);
|
||||
const active = this.#active.get(command.requestId);
|
||||
if (active) {
|
||||
if (active.digest !== commandDigest) {
|
||||
throw new CopilotFailureDiagnosisApplicationConflictError();
|
||||
}
|
||||
return active.promise;
|
||||
}
|
||||
if (
|
||||
this.#active.size >=
|
||||
MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisApplicationBusyError();
|
||||
}
|
||||
const promise = this.#execute(command).finally(() => {
|
||||
this.#active.delete(command.requestId);
|
||||
});
|
||||
this.#active.set(command.requestId, { digest: commandDigest, promise });
|
||||
return promise;
|
||||
}
|
||||
|
||||
async #execute(
|
||||
command: Readonly<ExecuteCopilotFailureDiagnosisApplicationCommand>,
|
||||
): Promise<Readonly<ExecuteCopilotFailureDiagnosisApplicationResult>> {
|
||||
const existing = await this.#dependencies.admissions.findPlanByRequestId(
|
||||
command.requestId,
|
||||
);
|
||||
let plan: Readonly<CopilotFailureDiagnosisExecutionPlan>;
|
||||
let admissionStatus: 'created' | 'existing';
|
||||
let admission;
|
||||
if (existing) {
|
||||
if (
|
||||
existing.projectId !== command.projectId ||
|
||||
existing.source.runId !== command.sourceRunId ||
|
||||
existing.traceId !== command.traceId ||
|
||||
!sameSubject(existing.requestedBySubject, command.principal.subject) ||
|
||||
!sameModelIntent(existing, this.#dependencies.modelIntent)
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisApplicationConflictError();
|
||||
}
|
||||
plan = existing;
|
||||
const admitted = await this.#dependencies.admissions.admit(plan);
|
||||
admissionStatus = admitted.status;
|
||||
admission = admitted.receipt;
|
||||
await this.#materializeArtifacts(plan);
|
||||
} else {
|
||||
const prepared = await this.#prepare(command);
|
||||
plan = prepared.plan;
|
||||
const admitted = await this.#dependencies.admissions.admit(plan);
|
||||
admissionStatus = admitted.status;
|
||||
admission = admitted.receipt;
|
||||
try {
|
||||
await this.#dependencies.artifacts.put(
|
||||
prepared.inputArtifact,
|
||||
prepared.previewArtifact,
|
||||
);
|
||||
} catch (cause) {
|
||||
throw new CopilotFailureDiagnosisApplicationUnavailableError({
|
||||
cause: cause instanceof Error ? cause : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
const tool = await this.#dependencies.executeTool(
|
||||
{
|
||||
requestId: plan.requestId,
|
||||
principal: command.principal,
|
||||
authorizer: this.#dependencies.authorizer,
|
||||
},
|
||||
this.#dependencies.tool,
|
||||
);
|
||||
if (tool.outcome !== 'succeeded') {
|
||||
return Object.freeze({
|
||||
admissionStatus,
|
||||
admission,
|
||||
tool,
|
||||
model: null,
|
||||
terminalizationRequired: true,
|
||||
});
|
||||
}
|
||||
const model = await this.#dependencies.executeModel(
|
||||
plan.requestId,
|
||||
this.#dependencies.model,
|
||||
);
|
||||
return Object.freeze({
|
||||
admissionStatus,
|
||||
admission,
|
||||
tool,
|
||||
model,
|
||||
terminalizationRequired: false,
|
||||
});
|
||||
}
|
||||
|
||||
async #prepare(
|
||||
command: Readonly<ExecuteCopilotFailureDiagnosisApplicationCommand>,
|
||||
) {
|
||||
const now = this.#clock();
|
||||
let run;
|
||||
let attempt;
|
||||
let snapshotRecord;
|
||||
try {
|
||||
[run, attempt, snapshotRecord] = await Promise.all([
|
||||
this.#dependencies.runs.findRunById(command.sourceRunId),
|
||||
this.#dependencies.runs.findLatestAttemptByRunId(command.sourceRunId),
|
||||
this.#dependencies.snapshots.findCurrent(command.projectId),
|
||||
]);
|
||||
} catch (cause) {
|
||||
return unavailable(cause);
|
||||
}
|
||||
if (
|
||||
!run ||
|
||||
!attempt ||
|
||||
!snapshotRecord ||
|
||||
run.projectId !== command.projectId ||
|
||||
attempt.runId !== run.id ||
|
||||
!['failed', 'timed_out'].includes(run.status) ||
|
||||
!['failed', 'timed_out', 'lost'].includes(attempt.status) ||
|
||||
(run.status === 'failed' &&
|
||||
!['failed', 'lost'].includes(attempt.status)) ||
|
||||
(run.status === 'timed_out' && attempt.status !== 'timed_out') ||
|
||||
!Number.isSafeInteger(attempt.finishedAtMs) ||
|
||||
typeof attempt.logArtifactId !== 'string'
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisApplicationConflictError(
|
||||
'source Run is not an exact diagnosable terminal fence',
|
||||
);
|
||||
}
|
||||
const snapshot = snapshotRecord.snapshot;
|
||||
const binding = createBuiltInRunLogExcerptToolHandlerBinding(snapshot, [
|
||||
'cluster-control',
|
||||
]);
|
||||
const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]);
|
||||
const invocation = await prepareToolInvocation(
|
||||
projectToolDefinitionRegistry(snapshot),
|
||||
{
|
||||
projectId: command.projectId,
|
||||
principal: command.principal,
|
||||
nowMs: now,
|
||||
tool: BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||
input: { runId: run.id, attemptId: attempt.id },
|
||||
},
|
||||
this.#dependencies.authorizer,
|
||||
);
|
||||
if (invocation.status !== 'ready') {
|
||||
throw new CopilotFailureDiagnosisApplicationConflictError(
|
||||
'Tool admission is not ready',
|
||||
);
|
||||
}
|
||||
const key = await this.#dependencies.invocationKeys.active();
|
||||
try {
|
||||
const baseIdentity = identity('cda', command.requestId);
|
||||
const nonce = (
|
||||
this.#dependencies.nonceFactory ?? defaultNonce
|
||||
)({
|
||||
key: key.key,
|
||||
keyId: key.keyId,
|
||||
requestId: command.requestId,
|
||||
projectId: command.projectId,
|
||||
sourceRunId: run.id,
|
||||
invocationActionDigest: invocation.actionDigest,
|
||||
});
|
||||
const tool = createTrustedToolInvocationPlan(bindings, invocation, {
|
||||
actionRef: baseIdentity,
|
||||
profile: 'cluster-control',
|
||||
preview: preview(run.id, attempt.id),
|
||||
inputArtifactId: identity('cdia', command.requestId),
|
||||
previewArtifactId: identity('cdpa', command.requestId),
|
||||
artifactKeyId: key.keyId,
|
||||
artifactKey: key.key,
|
||||
artifactNonce: nonce,
|
||||
sealedAtMs: now,
|
||||
});
|
||||
nonce.fill(0);
|
||||
const plan = prepareCopilotFailureDiagnosisExecution({
|
||||
requestId: command.requestId,
|
||||
traceId: command.traceId,
|
||||
source: {
|
||||
runId: run.id,
|
||||
runVersion: run.version,
|
||||
runStatus: run.status as 'failed' | 'timed_out',
|
||||
attemptId: attempt.id,
|
||||
attemptStatus: attempt.status as 'failed' | 'timed_out' | 'lost',
|
||||
attemptFinishedAtMs: attempt.finishedAtMs!,
|
||||
logArtifactId: attempt.logArtifactId,
|
||||
},
|
||||
toolPlan: tool.plan,
|
||||
bindings,
|
||||
model: this.#dependencies.modelIntent,
|
||||
deadlineAtMs: now + this.#dependencies.executionTimeoutMs,
|
||||
plannedAtMs: now,
|
||||
});
|
||||
return Object.freeze({
|
||||
plan,
|
||||
inputArtifact: tool.inputArtifact,
|
||||
previewArtifact: tool.previewArtifact,
|
||||
});
|
||||
} finally {
|
||||
key.key.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
async #materializeArtifacts(
|
||||
plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
|
||||
): Promise<void> {
|
||||
const material = await this.#dependencies.invocationKeys.resolve(
|
||||
plan.tool.invocationArtifact.keyId,
|
||||
);
|
||||
if (!material || material.keyId !== plan.tool.invocationArtifact.keyId) {
|
||||
return unavailable();
|
||||
}
|
||||
try {
|
||||
const nonce = (
|
||||
this.#dependencies.nonceFactory ?? defaultNonce
|
||||
)(nonceInput(plan, material.key));
|
||||
const inputArtifact = createToolInvocationInputArtifact(
|
||||
{
|
||||
artifactId: plan.tool.invocationArtifact.artifactId,
|
||||
projectId: plan.projectId,
|
||||
actionRef: plan.tool.actionRef,
|
||||
requestedBy: plan.requestedBySubject,
|
||||
tool: BUILTIN_RUN_LOG_EXCERPT_TOOL,
|
||||
input: {
|
||||
attemptId: plan.source.attemptId,
|
||||
runId: plan.source.runId,
|
||||
},
|
||||
inputDigest: plan.tool.invocationArtifact.inputDigest,
|
||||
invocationActionDigest: plan.tool.invocationActionDigest,
|
||||
keyId: material.keyId,
|
||||
key: material.key,
|
||||
sealedAtMs: plan.tool.sealedAtMs,
|
||||
},
|
||||
() => nonce,
|
||||
);
|
||||
nonce.fill(0);
|
||||
const previewArtifact = createToolInvocationPreviewArtifact({
|
||||
artifactId: plan.tool.previewArtifact.artifactId,
|
||||
projectId: plan.projectId,
|
||||
actionRef: plan.tool.actionRef,
|
||||
actionDigest: plan.tool.actionDigest,
|
||||
redactionContractDigest:
|
||||
plan.tool.previewArtifact.redactionContractDigest,
|
||||
preview: preview(plan.source.runId, plan.source.attemptId),
|
||||
sealedAtMs: plan.tool.sealedAtMs,
|
||||
});
|
||||
if (
|
||||
inputArtifact.artifactDigest !==
|
||||
plan.tool.invocationArtifact.artifactDigest ||
|
||||
previewArtifact.artifactDigest !==
|
||||
plan.tool.previewArtifact.artifactDigest
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisApplicationConflictError(
|
||||
'durable Tool Artifact references cannot be reconstructed',
|
||||
);
|
||||
}
|
||||
await this.#dependencies.artifacts.put(inputArtifact, previewArtifact);
|
||||
} finally {
|
||||
material.key.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
#clock(): number {
|
||||
let value: number;
|
||||
try {
|
||||
value = (this.#dependencies.now ?? Date.now)();
|
||||
} catch (cause) {
|
||||
return unavailable(cause);
|
||||
}
|
||||
if (!Number.isSafeInteger(value) || value < 0) return invalid('clock');
|
||||
if (value + this.#dependencies.executionTimeoutMs > Number.MAX_SAFE_INTEGER) {
|
||||
return invalid('deadline overflows');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from './application/contracts';
|
||||
export * from './application/service';
|
||||
@@ -5,7 +5,10 @@ import {
|
||||
type TrustedToolSuccessCompletionReadDependencies,
|
||||
} from '@qinglong/runtime-core/trusted-tool-completion';
|
||||
|
||||
import { BoundedModelGateway } from '../../../model-gateway/gateway';
|
||||
import type {
|
||||
BoundedModelGateway,
|
||||
ModelInvocationSuccessfulCompletionSink,
|
||||
} from '../../../model-gateway/gateway';
|
||||
import type { ModelInvocationRepository } from '../../../model-invocation/modelInvocation';
|
||||
import type { CopilotFailureDiagnosisToolExecutionAdmissionReader } from '../tool-execution/contracts';
|
||||
import type { CopilotFailureDiagnosisToolUnlockRepository } from '../tool-execution/contracts';
|
||||
@@ -36,20 +39,26 @@ export interface CopilotFailureDiagnosisModelExecutionDependencies {
|
||||
CopilotFailureDiagnosisOutputCompletionRepository,
|
||||
'findCopilotFailureDiagnosisOutput'
|
||||
>;
|
||||
readonly gateway: BoundedModelGateway;
|
||||
readonly gateway: Pick<
|
||||
BoundedModelGateway,
|
||||
'generate' | 'supportsSuccessfulCompletionSink'
|
||||
>;
|
||||
readonly successfulCompletion: CopilotFailureDiagnosisModelCompletionCoordinator;
|
||||
readonly finalizations: CopilotFailureDiagnosisFinalizationRepository;
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisToolResultReader {
|
||||
open(startId: string): Promise<Readonly<TrustedToolSuccessCompletionResult>>;
|
||||
open(
|
||||
requestId: string,
|
||||
startId: string,
|
||||
): Promise<Readonly<TrustedToolSuccessCompletionResult>>;
|
||||
}
|
||||
|
||||
export function createCopilotFailureDiagnosisToolResultReader(
|
||||
dependencies: TrustedToolSuccessCompletionReadDependencies,
|
||||
): CopilotFailureDiagnosisToolResultReader {
|
||||
return Object.freeze({
|
||||
open: (startId: string) =>
|
||||
open: (_requestId: string, startId: string) =>
|
||||
openTrustedToolSuccessCompletion(startId, dependencies),
|
||||
});
|
||||
}
|
||||
@@ -97,7 +106,8 @@ function assertDependencies(
|
||||
typeof value.modelInvocations?.findStart !== 'function' ||
|
||||
typeof value.modelInvocations?.findCompletion !== 'function' ||
|
||||
typeof value.outputs?.findCopilotFailureDiagnosisOutput !== 'function' ||
|
||||
!(value.gateway instanceof BoundedModelGateway) ||
|
||||
typeof value.gateway?.generate !== 'function' ||
|
||||
typeof value.gateway?.supportsSuccessfulCompletionSink !== 'function' ||
|
||||
typeof value.successfulCompletion?.begin !== 'function' ||
|
||||
typeof value.successfulCompletion?.reference !== 'function' ||
|
||||
typeof value.successfulCompletion?.end !== 'function' ||
|
||||
@@ -215,7 +225,7 @@ export async function executeCopilotFailureDiagnosisModel(
|
||||
);
|
||||
}
|
||||
|
||||
const tool = await dependencies.toolResults.open(unlock.startId);
|
||||
const tool = await dependencies.toolResults.open(requestId, unlock.startId);
|
||||
if (
|
||||
tool.completion.completionDigest !== unlock.toolCompletionDigest ||
|
||||
tool.completion.runId !== plan.runId ||
|
||||
@@ -241,7 +251,7 @@ export async function executeCopilotFailureDiagnosisModel(
|
||||
});
|
||||
if (
|
||||
!dependencies.gateway.supportsSuccessfulCompletionSink(
|
||||
dependencies.successfulCompletion,
|
||||
dependencies.successfulCompletion as ModelInvocationSuccessfulCompletionSink,
|
||||
)
|
||||
) {
|
||||
throw new InvalidCopilotFailureDiagnosisModelExecutionError(
|
||||
|
||||
@@ -39,6 +39,12 @@ import {
|
||||
type ModelPriceCatalogResolver,
|
||||
} from '../pricing/pricing';
|
||||
|
||||
export {
|
||||
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||
MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS,
|
||||
ModelInvocationSuccessfulCompletionRouter,
|
||||
} from './successfulCompletionRouter';
|
||||
|
||||
export const MAX_MODEL_GATEWAY_CONCURRENCY = 64;
|
||||
|
||||
export class ModelProviderUnavailableError extends Error {
|
||||
@@ -126,6 +132,9 @@ export interface BoundedModelGatewayOptions {
|
||||
}
|
||||
|
||||
export interface ModelInvocationSuccessfulCompletionSink {
|
||||
supportsSuccessfulCompletionSink?(
|
||||
sink: ModelInvocationSuccessfulCompletionSink,
|
||||
): boolean;
|
||||
record(
|
||||
audit: Readonly<ModelInvocationAuditRecord>,
|
||||
result: Readonly<GenerateResult>,
|
||||
@@ -372,7 +381,11 @@ export class BoundedModelGateway {
|
||||
supportsSuccessfulCompletionSink(
|
||||
sink: ModelInvocationSuccessfulCompletionSink,
|
||||
): boolean {
|
||||
return this.#successfulCompletion === sink;
|
||||
const configured = this.#successfulCompletion;
|
||||
return (
|
||||
configured === sink ||
|
||||
configured?.supportsSuccessfulCompletionSink?.(sink) === true
|
||||
);
|
||||
}
|
||||
|
||||
async #prepare(
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import type { ModelInvocationSuccessfulCompletionSink } from './gateway';
|
||||
import type {
|
||||
GenerateResult,
|
||||
ModelInvocationAuditRecord,
|
||||
ModelInvocationAuditResult,
|
||||
} from './model';
|
||||
|
||||
export const MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS = 8;
|
||||
|
||||
export class InvalidModelInvocationSuccessfulCompletionRouterError extends TypeError {
|
||||
readonly code = 'MODEL_INVOCATION_SUCCESSFUL_COMPLETION_ROUTER_INVALID';
|
||||
|
||||
constructor() {
|
||||
super('Model invocation successful completion router is invalid');
|
||||
this.name = 'InvalidModelInvocationSuccessfulCompletionRouterError';
|
||||
}
|
||||
}
|
||||
|
||||
/** Bounded dispatch for mutually exclusive durable output domains. */
|
||||
export class ModelInvocationSuccessfulCompletionRouter
|
||||
implements ModelInvocationSuccessfulCompletionSink
|
||||
{
|
||||
readonly #sinks: readonly ModelInvocationSuccessfulCompletionSink[];
|
||||
|
||||
constructor(sinks: readonly ModelInvocationSuccessfulCompletionSink[]) {
|
||||
if (
|
||||
!Array.isArray(sinks) ||
|
||||
sinks.length < 2 ||
|
||||
sinks.length > MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS ||
|
||||
new Set(sinks).size !== sinks.length ||
|
||||
sinks.some(
|
||||
(sink) =>
|
||||
!sink ||
|
||||
typeof sink !== 'object' ||
|
||||
typeof sink.record !== 'function',
|
||||
)
|
||||
) {
|
||||
throw new InvalidModelInvocationSuccessfulCompletionRouterError();
|
||||
}
|
||||
this.#sinks = Object.freeze([...sinks]);
|
||||
Object.freeze(this);
|
||||
}
|
||||
|
||||
supportsSuccessfulCompletionSink(
|
||||
sink: ModelInvocationSuccessfulCompletionSink,
|
||||
): boolean {
|
||||
return this.#sinks.some(
|
||||
(candidate) =>
|
||||
candidate === sink ||
|
||||
candidate.supportsSuccessfulCompletionSink?.(sink) === true,
|
||||
);
|
||||
}
|
||||
|
||||
async record(
|
||||
audit: Readonly<ModelInvocationAuditRecord>,
|
||||
result: Readonly<GenerateResult>,
|
||||
): Promise<
|
||||
Readonly<
|
||||
| { handled: false }
|
||||
| { handled: true; disposition: ModelInvocationAuditResult }
|
||||
>
|
||||
> {
|
||||
for (const sink of this.#sinks) {
|
||||
const routed = await sink.record(audit, result);
|
||||
if (
|
||||
!routed ||
|
||||
typeof routed !== 'object' ||
|
||||
Array.isArray(routed) ||
|
||||
(routed.handled !== true && routed.handled !== false)
|
||||
) {
|
||||
throw new InvalidModelInvocationSuccessfulCompletionRouterError();
|
||||
}
|
||||
if (routed.handled) return routed;
|
||||
if (Object.keys(routed).length !== 1) {
|
||||
throw new InvalidModelInvocationSuccessfulCompletionRouterError();
|
||||
}
|
||||
}
|
||||
return Object.freeze({ handled: false as const });
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,10 @@ import {
|
||||
import type { PluginPackagePromptOutputReadService } from '../../prompt-output/pluginPackagePromptOutputRead';
|
||||
import type { PluginPackagePromptExecutionOutputReadService } from '../../prompt-output/pluginPackagePromptExecutionOutputRead';
|
||||
import { bootstrapModelGatewayProfile } from '../../profile/profileComposition';
|
||||
import {
|
||||
ModelInvocationSuccessfulCompletionRouter,
|
||||
type ModelInvocationSuccessfulCompletionSink,
|
||||
} from '../../model-gateway/gateway';
|
||||
import {
|
||||
PostgresPluginPackagePromptApplicationUnavailableError,
|
||||
unavailable,
|
||||
@@ -49,6 +53,8 @@ function assertEnabledOptions(
|
||||
(options.confirmActive !== undefined &&
|
||||
typeof options.confirmActive !== 'function') ||
|
||||
(options.now !== undefined && typeof options.now !== 'function') ||
|
||||
(options.createAdditionalSuccessfulCompletion !== undefined &&
|
||||
typeof options.createAdditionalSuccessfulCompletion !== 'function') ||
|
||||
(options.promptOutputKeys !== undefined &&
|
||||
(!options.promptOutputKeys ||
|
||||
typeof options.promptOutputKeys !== 'object' ||
|
||||
@@ -118,17 +124,36 @@ export async function bootstrapPostgresPluginPackagePromptApplication(
|
||||
});
|
||||
},
|
||||
loadProviders: options.loadProviders,
|
||||
...(options.promptOutputKeys === undefined
|
||||
...(options.promptOutputKeys === undefined &&
|
||||
options.createAdditionalSuccessfulCompletion === undefined
|
||||
? {}
|
||||
: {
|
||||
createSuccessfulCompletion: (coordinator) => {
|
||||
durableOutput =
|
||||
new PluginPackagePromptOutputCompletionCoordinator({
|
||||
const sinks: ModelInvocationSuccessfulCompletionSink[] = [];
|
||||
if (options.promptOutputKeys !== undefined) {
|
||||
durableOutput =
|
||||
new PluginPackagePromptOutputCompletionCoordinator({
|
||||
coordinator,
|
||||
keys: options.promptOutputKeys!,
|
||||
...(options.now === undefined ? {} : { now: options.now }),
|
||||
});
|
||||
return durableOutput;
|
||||
sinks.push(durableOutput);
|
||||
}
|
||||
if (options.createAdditionalSuccessfulCompletion !== undefined) {
|
||||
const additionalSuccessfulCompletion =
|
||||
options.createAdditionalSuccessfulCompletion(coordinator);
|
||||
if (
|
||||
!additionalSuccessfulCompletion ||
|
||||
typeof additionalSuccessfulCompletion.record !== 'function'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL Package Prompt additional completion is invalid',
|
||||
);
|
||||
}
|
||||
sinks.push(additionalSuccessfulCompletion);
|
||||
}
|
||||
if (sinks.length === 1) return sinks[0]!;
|
||||
return new ModelInvocationSuccessfulCompletionRouter(sinks);
|
||||
},
|
||||
}),
|
||||
audit: options.audit,
|
||||
|
||||
@@ -21,6 +21,8 @@ import type {
|
||||
ModelGatewayProfileAudit,
|
||||
ModelGatewayProviderAuthority,
|
||||
} from '../../profile/profileComposition';
|
||||
import type { DurableModelInvocationCoordinator } from '../../model-invocation/durableModelInvocationCoordinator';
|
||||
import type { ModelInvocationSuccessfulCompletionSink } from '../../model-gateway/gateway';
|
||||
import type { PluginPackagePromptCatalogCapability } from '../pluginPackagePromptCatalog';
|
||||
import type { PluginPackagePromptExecutionInspectionRepository } from '../pluginPackagePromptExecutionInspection';
|
||||
|
||||
@@ -80,6 +82,9 @@ export type BootstrapPostgresPluginPackagePromptApplicationOptions =
|
||||
maxConcurrent?: number;
|
||||
recoveryLimit?: number;
|
||||
now?: () => number;
|
||||
createAdditionalSuccessfulCompletion?: (
|
||||
coordinator: DurableModelInvocationCoordinator,
|
||||
) => ModelInvocationSuccessfulCompletionSink;
|
||||
promptOutputKeys?: PluginPackagePromptOutputArtifactKeyProvider;
|
||||
promptOutputRead?: Readonly<{
|
||||
authorizer: PluginPackagePromptOutputArtifactReadAuthorizer;
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
CopilotFailureDiagnosisApplicationService,
|
||||
CopilotFailureDiagnosisApplicationUnavailableError,
|
||||
} = require('@qinglong/ai/failure-diagnosis-application');
|
||||
const {
|
||||
BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION,
|
||||
} = require('@qinglong/runtime-core/builtin-run-log-excerpt-tool');
|
||||
const {
|
||||
createPluginPackageResourceGenerationFromReferences,
|
||||
} = require('@qinglong/runtime-core/plugin-package-resource-generation');
|
||||
const {
|
||||
createProjectToolDefinitionSnapshot,
|
||||
} = require('@qinglong/runtime-core/project-tool-definition-snapshot');
|
||||
|
||||
const NOW = 1_800_000_000_000;
|
||||
const KEY = Buffer.alloc(32, 0x42);
|
||||
const MODEL = Object.freeze({
|
||||
provider: 'provider-primary',
|
||||
model: 'model-diagnosis',
|
||||
modelBoundary: 'external',
|
||||
responseLanguage: 'zh-CN',
|
||||
maxOutputTokens: 512,
|
||||
egressPolicy: Object.freeze({
|
||||
schema: 'qinglong/copilot-model-egress-policy@v1',
|
||||
revision: 'application-test-v1',
|
||||
potentiallySensitiveDataBoundaries: Object.freeze(['external']),
|
||||
maxInputBytes: 64 * 1024,
|
||||
maxOutputTokens: 1024,
|
||||
}),
|
||||
});
|
||||
|
||||
function snapshot() {
|
||||
const generation = createPluginPackageResourceGenerationFromReferences({
|
||||
installationId: 'installation-copilot-test',
|
||||
projectId: 'project-1',
|
||||
packageName: 'qinglong',
|
||||
lockDigest: 'a'.repeat(64),
|
||||
generation: 1,
|
||||
previousActiveLockDigest: null,
|
||||
contentDigest: 'b'.repeat(64),
|
||||
resources: [],
|
||||
});
|
||||
return createProjectToolDefinitionSnapshot({
|
||||
projectId: 'project-1',
|
||||
contributions: [
|
||||
{
|
||||
generation,
|
||||
revisionDigest: 'c'.repeat(64),
|
||||
definitions: [BUILTIN_RUN_LOG_EXCERPT_TOOL_DEFINITION],
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
requestId: 'diagnosis-request-1',
|
||||
traceId: 'diagnosis-trace-1',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'owner-1' },
|
||||
authenticationId: 'auth-1',
|
||||
authenticatedAtMs: NOW - 1000,
|
||||
expiresAtMs: NOW + 60_000,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
let plan = null;
|
||||
let artifacts = null;
|
||||
let failArtifactOnce = options.failArtifactOnce === true;
|
||||
let activeKeyCopies = [];
|
||||
let resolvedKeyCopies = [];
|
||||
let toolCalls = 0;
|
||||
let modelCalls = 0;
|
||||
let releaseTool;
|
||||
const toolGate = options.blockTool
|
||||
? new Promise((resolve) => { releaseTool = resolve; })
|
||||
: Promise.resolve();
|
||||
const admissions = {
|
||||
async findByRequestId(requestId) {
|
||||
return plan?.requestId === requestId
|
||||
? { requestId, planDigest: plan.planDigest }
|
||||
: null;
|
||||
},
|
||||
async findPlanByRequestId(requestId) {
|
||||
return plan?.requestId === requestId ? plan : null;
|
||||
},
|
||||
async admit(value) {
|
||||
const status = plan ? 'existing' : 'created';
|
||||
plan ??= value;
|
||||
assert.equal(plan.planDigest, value.planDigest);
|
||||
return {
|
||||
status,
|
||||
receipt: {
|
||||
requestId: plan.requestId,
|
||||
planDigest: plan.planDigest,
|
||||
runId: plan.runId,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
const snapshots = {
|
||||
async findCurrent() { return { snapshot: snapshot(), committedAtMs: NOW }; },
|
||||
};
|
||||
const runs = {
|
||||
async findRunById() {
|
||||
return {
|
||||
id: 'source-run-1', projectId: 'project-1', status: 'failed',
|
||||
version: 8, eventSequence: 8,
|
||||
};
|
||||
},
|
||||
async findLatestAttemptByRunId() {
|
||||
return {
|
||||
id: 'source-attempt-1', runId: 'source-run-1', attempt: 1,
|
||||
status: 'failed', executorType: 'remote_worker', callbackSequence: 0,
|
||||
createdAtMs: NOW - 3000, finishedAtMs: NOW - 1000,
|
||||
logArtifactId: `wlog-${'d'.repeat(30)}`,
|
||||
};
|
||||
},
|
||||
};
|
||||
const artifactRepository = {
|
||||
async put(input, preview) {
|
||||
if (failArtifactOnce) {
|
||||
failArtifactOnce = false;
|
||||
throw new Error('simulated admission-to-artifact crash');
|
||||
}
|
||||
if (artifacts) {
|
||||
assert.deepEqual(input, artifacts.input);
|
||||
assert.deepEqual(preview, artifacts.preview);
|
||||
return { status: 'existing' };
|
||||
}
|
||||
artifacts = { input, preview };
|
||||
return { status: 'inserted' };
|
||||
},
|
||||
async findInput() { return artifacts?.input ?? null; },
|
||||
async findPreview() { return artifacts?.preview ?? null; },
|
||||
};
|
||||
const invocationKeys = {
|
||||
async active() {
|
||||
const copy = Buffer.from(KEY);
|
||||
activeKeyCopies.push(copy);
|
||||
return { keyId: 'invocation-key-1', key: copy };
|
||||
},
|
||||
async resolve(keyId) {
|
||||
assert.equal(keyId, 'invocation-key-1');
|
||||
const copy = Buffer.from(KEY);
|
||||
resolvedKeyCopies.push(copy);
|
||||
return { keyId, key: copy };
|
||||
},
|
||||
};
|
||||
const unlocks = { async findByRequestId() { return null; }, async commit() {} };
|
||||
const tool = {
|
||||
admissions,
|
||||
snapshots,
|
||||
runs,
|
||||
artifacts: artifactRepository,
|
||||
invocationKeys,
|
||||
resultKeys: { async resolve() { return null; } },
|
||||
stepRuns: { async findById() { return null; } },
|
||||
barriers: {}, completions: {}, failureCompletions: {},
|
||||
resultKeyCatalog: {}, resultRekeys: {}, logs: {}, unlocks,
|
||||
};
|
||||
const model = {
|
||||
admissions,
|
||||
unlocks,
|
||||
toolResults: {}, modelInvocations: {}, outputs: {}, gateway: {},
|
||||
successfulCompletion: {}, finalizations: {},
|
||||
};
|
||||
const service = new CopilotFailureDiagnosisApplicationService({
|
||||
admissions,
|
||||
snapshots,
|
||||
runs,
|
||||
artifacts: artifactRepository,
|
||||
invocationKeys,
|
||||
authorizer: {
|
||||
async authorize() {
|
||||
return {
|
||||
effect: 'allow', reasons: ['role_grant'],
|
||||
fence: { projectVersion: 1, bindingVersion: 1 },
|
||||
};
|
||||
},
|
||||
},
|
||||
tool,
|
||||
model,
|
||||
modelIntent: MODEL,
|
||||
executionTimeoutMs: 60_000,
|
||||
now: () => NOW,
|
||||
async executeTool() {
|
||||
toolCalls += 1;
|
||||
await toolGate;
|
||||
return options.toolFailure
|
||||
? { outcome: 'failed', completionStatus: 'created', unlockStatus: null }
|
||||
: {
|
||||
outcome: 'succeeded', completionStatus: 'created',
|
||||
unlockStatus: 'created', completion: {}, unlock: {},
|
||||
};
|
||||
},
|
||||
async executeModel() {
|
||||
modelCalls += 1;
|
||||
return {
|
||||
outcome: 'succeeded',
|
||||
output: { artifactId: 'output-1' },
|
||||
finalization: { requestId: 'diagnosis-request-1' },
|
||||
};
|
||||
},
|
||||
});
|
||||
return {
|
||||
service,
|
||||
releaseTool: () => releaseTool?.(),
|
||||
state: () => ({
|
||||
plan, artifacts, toolCalls, modelCalls,
|
||||
activeKeyCopies, resolvedKeyCopies,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
test('application derives, admits and executes one server-owned diagnosis with exact replay', async () => {
|
||||
const testFixture = fixture();
|
||||
const first = await testFixture.service.execute(command());
|
||||
assert.equal(first.admissionStatus, 'created');
|
||||
assert.equal(first.tool.outcome, 'succeeded');
|
||||
assert.equal(first.model.outcome, 'succeeded');
|
||||
assert.equal(first.terminalizationRequired, false);
|
||||
const second = await testFixture.service.execute(command());
|
||||
assert.equal(second.admissionStatus, 'existing');
|
||||
const state = testFixture.state();
|
||||
assert.equal(state.toolCalls, 2);
|
||||
assert.equal(state.modelCalls, 2);
|
||||
assert.equal(state.plan.source.attemptId, 'source-attempt-1');
|
||||
assert.equal(state.plan.tool.invocationArtifact.artifactId.startsWith('cdia:'), true);
|
||||
assert.equal(state.activeKeyCopies[0].every((value) => value === 0), true);
|
||||
assert.equal(state.resolvedKeyCopies[0].every((value) => value === 0), true);
|
||||
});
|
||||
|
||||
test('application repairs the durable admission-to-Artifact crash window', async () => {
|
||||
const testFixture = fixture({ failArtifactOnce: true });
|
||||
await assert.rejects(
|
||||
testFixture.service.execute(command()),
|
||||
CopilotFailureDiagnosisApplicationUnavailableError,
|
||||
);
|
||||
assert.ok(testFixture.state().plan);
|
||||
assert.equal(testFixture.state().artifacts, null);
|
||||
const replay = await testFixture.service.execute(command());
|
||||
assert.equal(replay.admissionStatus, 'existing');
|
||||
assert.ok(testFixture.state().artifacts);
|
||||
});
|
||||
|
||||
test('application coalesces exact callers and exposes Tool terminalization debt', async () => {
|
||||
const concurrent = fixture({ blockTool: true });
|
||||
const first = concurrent.service.execute(command());
|
||||
const second = concurrent.service.execute(command());
|
||||
assert.equal(first, second);
|
||||
concurrent.releaseTool();
|
||||
await first;
|
||||
assert.equal(concurrent.state().toolCalls, 1);
|
||||
|
||||
const failed = fixture({ toolFailure: true });
|
||||
const result = await failed.service.execute(command());
|
||||
assert.equal(result.tool.outcome, 'failed');
|
||||
assert.equal(result.model, null);
|
||||
assert.equal(result.terminalizationRequired, true);
|
||||
assert.equal(failed.state().modelCalls, 0);
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
BoundedModelGateway,
|
||||
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||
ModelInvocationSuccessfulCompletionRouter,
|
||||
} = require('@qinglong/ai/gateway');
|
||||
|
||||
function sink(name, handled = false) {
|
||||
return {
|
||||
async record(audit) {
|
||||
audit.order.push(name);
|
||||
return handled
|
||||
? { handled: true, disposition: { status: 'created' } }
|
||||
: { handled: false };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('successful completion router exposes exact children and stops at the owning sink', async () => {
|
||||
const first = sink('prompt');
|
||||
const second = sink('copilot', true);
|
||||
const third = sink('unreachable', true);
|
||||
const nested = new ModelInvocationSuccessfulCompletionRouter([
|
||||
second,
|
||||
third,
|
||||
]);
|
||||
const router = new ModelInvocationSuccessfulCompletionRouter([first, nested]);
|
||||
const order = [];
|
||||
assert.equal(router.supportsSuccessfulCompletionSink(first), true);
|
||||
assert.equal(router.supportsSuccessfulCompletionSink(second), true);
|
||||
assert.equal(router.supportsSuccessfulCompletionSink({ record() {} }), false);
|
||||
assert.deepEqual(await router.record({ order }, {}), {
|
||||
handled: true,
|
||||
disposition: { status: 'created' },
|
||||
});
|
||||
assert.deepEqual(order, ['prompt', 'copilot']);
|
||||
|
||||
const gateway = new BoundedModelGateway({
|
||||
providers: [
|
||||
{
|
||||
type: 'test',
|
||||
async generate() { throw new Error('unused'); },
|
||||
async *stream() { throw new Error('unused'); },
|
||||
async listModels() { return []; },
|
||||
},
|
||||
],
|
||||
policies: { async resolve() { throw new Error('unused'); } },
|
||||
pricing: { async resolve() { return null; } },
|
||||
audit: { async record() {} },
|
||||
successfulCompletion: router,
|
||||
maxConcurrent: 1,
|
||||
});
|
||||
assert.equal(gateway.supportsSuccessfulCompletionSink(router), true);
|
||||
assert.equal(gateway.supportsSuccessfulCompletionSink(second), true);
|
||||
});
|
||||
|
||||
test('successful completion router rejects unbounded, duplicate and malformed sinks', async () => {
|
||||
const valid = sink('valid');
|
||||
assert.throws(
|
||||
() => new ModelInvocationSuccessfulCompletionRouter([valid]),
|
||||
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||
);
|
||||
assert.throws(
|
||||
() => new ModelInvocationSuccessfulCompletionRouter([valid, valid]),
|
||||
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||
);
|
||||
assert.throws(
|
||||
() => new ModelInvocationSuccessfulCompletionRouter([valid, {}]),
|
||||
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||
);
|
||||
const malformed = new ModelInvocationSuccessfulCompletionRouter([
|
||||
valid,
|
||||
{ async record() { return { handled: false, widened: true }; } },
|
||||
]);
|
||||
await assert.rejects(
|
||||
malformed.record({ order: [] }, {}),
|
||||
InvalidModelInvocationSuccessfulCompletionRouterError,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user