mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): execute copilot diagnosis models
This commit is contained in:
whyour
@@ -50,6 +50,16 @@
|
||||
"require": "./dist/copilot/failure-diagnosis/postgresToolExecutionRepository.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/postgresToolExecutionRepository.js"
|
||||
},
|
||||
"./failure-diagnosis-model-execution": {
|
||||
"types": "./dist/copilot/failure-diagnosis/modelExecution.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/modelExecution.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/modelExecution.js"
|
||||
},
|
||||
"./postgres-failure-diagnosis-model-execution-storage": {
|
||||
"types": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.js",
|
||||
"default": "./dist/copilot/failure-diagnosis/postgresModelExecutionRepository.js"
|
||||
},
|
||||
"./model-invocation": {
|
||||
"types": "./dist/model-invocation/modelInvocation.d.ts",
|
||||
"require": "./dist/model-invocation/modelInvocation.js",
|
||||
|
||||
@@ -0,0 +1,283 @@
|
||||
import type {
|
||||
ModelInvocationCompletionCommand,
|
||||
ModelInvocationRepository,
|
||||
} from '../../../model-invocation/modelInvocation';
|
||||
import type { DurableModelInvocationCoordinator } from '../../../model-invocation/durableModelInvocationCoordinator';
|
||||
import type { ModelInvocationAtomicSuccess } from '../../../model-invocation/modelInvocationAtomicSuccess';
|
||||
import type {
|
||||
GenerateResult,
|
||||
ModelInvocationAuditRecord,
|
||||
} from '../../../model-gateway/model';
|
||||
import type { ModelInvocationSuccessfulCompletionSink } from '../../../model-gateway/gateway';
|
||||
import type { CopilotFailureDiagnosisExecutionPlan } from '../admission/contracts';
|
||||
import { normalizeCopilotFailureDiagnosisExecutionPlan } from '../admission/plan';
|
||||
import type { FailureDiagnosisPromptPlan } from '../contracts';
|
||||
import {
|
||||
CopilotFailureDiagnosisOutputArtifactConflictError,
|
||||
CopilotFailureDiagnosisOutputArtifactUnavailableError,
|
||||
copilotFailureDiagnosisOutputReference,
|
||||
createCopilotFailureDiagnosisOutputArtifact,
|
||||
normalizeCopilotFailureDiagnosisOutputArtifact,
|
||||
type CopilotFailureDiagnosisOutputArtifact,
|
||||
type CopilotFailureDiagnosisOutputKeyProvider,
|
||||
type CopilotFailureDiagnosisOutputReference,
|
||||
} from './outputArtifact';
|
||||
|
||||
export const MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_MODEL_COMPLETIONS = 64;
|
||||
|
||||
export interface CommitCopilotFailureDiagnosisOutputResult {
|
||||
readonly status: 'created' | 'existing';
|
||||
readonly reference: Readonly<CopilotFailureDiagnosisOutputReference>;
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisOutputCompletionRepository {
|
||||
findCopilotFailureDiagnosisOutput(
|
||||
artifactId: string,
|
||||
): Promise<Readonly<CopilotFailureDiagnosisOutputArtifact> | null>;
|
||||
completeWithCopilotFailureDiagnosisOutput(
|
||||
command: Readonly<ModelInvocationCompletionCommand>,
|
||||
artifact: Readonly<CopilotFailureDiagnosisOutputArtifact>,
|
||||
): Promise<Readonly<CommitCopilotFailureDiagnosisOutputResult>>;
|
||||
}
|
||||
|
||||
export function isCopilotFailureDiagnosisOutputCompletionRepository(
|
||||
value: ModelInvocationRepository,
|
||||
): value is ModelInvocationRepository &
|
||||
CopilotFailureDiagnosisOutputCompletionRepository {
|
||||
return (
|
||||
typeof (value as Partial<CopilotFailureDiagnosisOutputCompletionRepository>)
|
||||
.findCopilotFailureDiagnosisOutput === 'function' &&
|
||||
typeof (value as Partial<CopilotFailureDiagnosisOutputCompletionRepository>)
|
||||
.completeWithCopilotFailureDiagnosisOutput === 'function'
|
||||
);
|
||||
}
|
||||
|
||||
export function assertCopilotFailureDiagnosisOutputCompletionBinding(
|
||||
command: Readonly<ModelInvocationCompletionCommand>,
|
||||
artifactValue: CopilotFailureDiagnosisOutputArtifact,
|
||||
): Readonly<{
|
||||
artifact: Readonly<CopilotFailureDiagnosisOutputArtifact>;
|
||||
reference: Readonly<CopilotFailureDiagnosisOutputReference>;
|
||||
}> {
|
||||
const artifact = normalizeCopilotFailureDiagnosisOutputArtifact(artifactValue);
|
||||
if (
|
||||
command.completion.outcome !== 'succeeded' ||
|
||||
command.completion.errorCode !== null ||
|
||||
command.completion.invocationId !== artifact.invocationId ||
|
||||
command.completion.projectId !== artifact.projectId ||
|
||||
command.completion.runId !== artifact.runId ||
|
||||
command.completion.stepRunId !== artifact.stepRunId ||
|
||||
command.completion.outputBytes !== artifact.outputBytes ||
|
||||
command.start.provider !== artifact.provider ||
|
||||
command.start.model !== artifact.model ||
|
||||
command.stepRunMutation.stepRun.outputRef !== artifact.artifactId
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactConflictError();
|
||||
}
|
||||
return Object.freeze({
|
||||
artifact,
|
||||
reference: copilotFailureDiagnosisOutputReference(artifact),
|
||||
});
|
||||
}
|
||||
|
||||
function atomicSuccess(
|
||||
artifactValue: CopilotFailureDiagnosisOutputArtifact,
|
||||
): ModelInvocationAtomicSuccess<CopilotFailureDiagnosisOutputReference> {
|
||||
const artifact = normalizeCopilotFailureDiagnosisOutputArtifact(artifactValue);
|
||||
const reference = copilotFailureDiagnosisOutputReference(artifact);
|
||||
const conflict = (): Error =>
|
||||
new CopilotFailureDiagnosisOutputArtifactConflictError();
|
||||
const extension: ModelInvocationAtomicSuccess<CopilotFailureDiagnosisOutputReference> = {
|
||||
outputRef: artifact.artifactId,
|
||||
assertAudit(audit): void {
|
||||
if (
|
||||
audit.phase !== 'completed' ||
|
||||
audit.requestId !== artifact.invocationId ||
|
||||
audit.projectId !== artifact.projectId ||
|
||||
audit.runId !== artifact.runId ||
|
||||
audit.stepRunId !== artifact.stepRunId ||
|
||||
audit.provider !== artifact.provider ||
|
||||
audit.model !== artifact.model ||
|
||||
audit.outputBytes !== artifact.outputBytes
|
||||
) {
|
||||
throw conflict();
|
||||
}
|
||||
},
|
||||
async find(repository) {
|
||||
if (!isCopilotFailureDiagnosisOutputCompletionRepository(repository)) {
|
||||
throw conflict();
|
||||
}
|
||||
const stored = await repository.findCopilotFailureDiagnosisOutput(
|
||||
artifact.artifactId,
|
||||
);
|
||||
if (!stored) return null;
|
||||
if (JSON.stringify(stored) !== JSON.stringify(artifact)) throw conflict();
|
||||
return copilotFailureDiagnosisOutputReference(stored);
|
||||
},
|
||||
matches(stored): boolean {
|
||||
return JSON.stringify(stored) === JSON.stringify(reference);
|
||||
},
|
||||
async commit(repository, command) {
|
||||
if (!isCopilotFailureDiagnosisOutputCompletionRepository(repository)) {
|
||||
throw conflict();
|
||||
}
|
||||
return repository.completeWithCopilotFailureDiagnosisOutput(
|
||||
command,
|
||||
artifact,
|
||||
);
|
||||
},
|
||||
conflict,
|
||||
};
|
||||
return Object.freeze(extension);
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisModelCompletionLease {
|
||||
readonly invocationId: string;
|
||||
}
|
||||
|
||||
interface ActiveCompletion {
|
||||
readonly lease: Readonly<CopilotFailureDiagnosisModelCompletionLease>;
|
||||
readonly plan: Readonly<CopilotFailureDiagnosisExecutionPlan>;
|
||||
readonly prompt: Readonly<FailureDiagnosisPromptPlan>;
|
||||
readonly toolCompletionDigest: string;
|
||||
reference: Readonly<CopilotFailureDiagnosisOutputReference> | null;
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisModelCompletionCoordinator
|
||||
implements ModelInvocationSuccessfulCompletionSink
|
||||
{
|
||||
readonly #coordinator: DurableModelInvocationCoordinator;
|
||||
readonly #keys: CopilotFailureDiagnosisOutputKeyProvider;
|
||||
readonly #now: () => number;
|
||||
readonly #nonceFactory: (() => Uint8Array) | undefined;
|
||||
readonly #active = new Map<string, ActiveCompletion>();
|
||||
|
||||
constructor(options: Readonly<{
|
||||
coordinator: DurableModelInvocationCoordinator;
|
||||
keys: CopilotFailureDiagnosisOutputKeyProvider;
|
||||
now?: () => number;
|
||||
nonceFactory?: () => Uint8Array;
|
||||
}>) {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
typeof options.coordinator?.recordWithAtomicSuccess !== 'function' ||
|
||||
typeof options.keys?.active !== 'function' ||
|
||||
typeof options.keys?.resolve !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function') ||
|
||||
(options.nonceFactory !== undefined &&
|
||||
typeof options.nonceFactory !== 'function')
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactUnavailableError();
|
||||
}
|
||||
this.#coordinator = options.coordinator;
|
||||
this.#keys = options.keys;
|
||||
this.#now = options.now ?? Date.now;
|
||||
this.#nonceFactory = options.nonceFactory;
|
||||
}
|
||||
|
||||
begin(input: Readonly<{
|
||||
plan: CopilotFailureDiagnosisExecutionPlan;
|
||||
prompt: FailureDiagnosisPromptPlan;
|
||||
toolCompletionDigest: string;
|
||||
}>): Readonly<CopilotFailureDiagnosisModelCompletionLease> {
|
||||
const plan = normalizeCopilotFailureDiagnosisExecutionPlan(input.plan);
|
||||
if (
|
||||
input.prompt.request.provider !== plan.model.provider ||
|
||||
input.prompt.request.model !== plan.model.model ||
|
||||
input.prompt.request.maxOutputTokens !== plan.model.maxOutputTokens ||
|
||||
input.prompt.egressEvidence.modelBoundary !== plan.model.modelBoundary ||
|
||||
input.prompt.egressEvidence.policyRevision !==
|
||||
plan.model.egressPolicy.revision ||
|
||||
input.prompt.egressEvidence.maxOutputTokens !==
|
||||
plan.model.maxOutputTokens ||
|
||||
input.prompt.completionRequirements.persistence !== 'encrypted_only' ||
|
||||
input.prompt.completionRequirements.plaintextAudit !== 'forbidden' ||
|
||||
!/^[0-9a-f]{64}$/.test(input.toolCompletionDigest) ||
|
||||
this.#active.has(plan.modelInvocationId) ||
|
||||
this.#active.size >=
|
||||
MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_MODEL_COMPLETIONS
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactConflictError();
|
||||
}
|
||||
const lease = Object.freeze({ invocationId: plan.modelInvocationId });
|
||||
this.#active.set(plan.modelInvocationId, {
|
||||
lease,
|
||||
plan,
|
||||
prompt: input.prompt,
|
||||
toolCompletionDigest: input.toolCompletionDigest,
|
||||
reference: null,
|
||||
});
|
||||
return lease;
|
||||
}
|
||||
|
||||
reference(
|
||||
lease: Readonly<CopilotFailureDiagnosisModelCompletionLease>,
|
||||
): Readonly<CopilotFailureDiagnosisOutputReference> | null {
|
||||
const active = this.#active.get(lease.invocationId);
|
||||
if (!active || active.lease !== lease) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactUnavailableError();
|
||||
}
|
||||
return active.reference;
|
||||
}
|
||||
|
||||
end(lease: Readonly<CopilotFailureDiagnosisModelCompletionLease>): void {
|
||||
const active = this.#active.get(lease.invocationId);
|
||||
if (!active || active.lease !== lease) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactUnavailableError();
|
||||
}
|
||||
this.#active.delete(lease.invocationId);
|
||||
}
|
||||
|
||||
async record(
|
||||
audit: Readonly<ModelInvocationAuditRecord>,
|
||||
result: Readonly<GenerateResult>,
|
||||
) {
|
||||
const active = this.#active.get(audit.requestId);
|
||||
if (!active) return Object.freeze({ handled: false as const });
|
||||
const { plan, prompt } = active;
|
||||
if (
|
||||
audit.phase !== 'completed' ||
|
||||
audit.projectId !== plan.projectId ||
|
||||
audit.runId !== plan.runId ||
|
||||
audit.stepRunId !== plan.modelStepRunId ||
|
||||
audit.traceId !== plan.traceId ||
|
||||
audit.requestId !== plan.modelInvocationId ||
|
||||
result.provider !== plan.model.provider ||
|
||||
result.model !== plan.model.model
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactConflictError();
|
||||
}
|
||||
const material = await this.#keys.active();
|
||||
try {
|
||||
const artifact = createCopilotFailureDiagnosisOutputArtifact(
|
||||
{
|
||||
requestId: plan.requestId,
|
||||
planDigest: plan.planDigest,
|
||||
toolCompletionDigest: active.toolCompletionDigest,
|
||||
projectId: plan.projectId,
|
||||
runId: plan.runId,
|
||||
stepRunId: plan.modelStepRunId,
|
||||
invocationId: plan.modelInvocationId,
|
||||
result,
|
||||
egressEvidence: prompt.egressEvidence,
|
||||
keyId: material.keyId,
|
||||
key: material.key,
|
||||
sealedAtMs: this.#now(),
|
||||
},
|
||||
this.#nonceFactory,
|
||||
);
|
||||
const disposition = await this.#coordinator.recordWithAtomicSuccess(
|
||||
audit,
|
||||
atomicSuccess(artifact),
|
||||
);
|
||||
active.reference = disposition.reference;
|
||||
return Object.freeze({
|
||||
handled: true as const,
|
||||
disposition: Object.freeze({ status: disposition.status }),
|
||||
});
|
||||
} finally {
|
||||
material.key.fill(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
import type { ToolJsonValue } from '@qinglong/runtime-core/tool-registry';
|
||||
import {
|
||||
openTrustedToolSuccessCompletion,
|
||||
type TrustedToolSuccessCompletionResult,
|
||||
type TrustedToolSuccessCompletionReadDependencies,
|
||||
} from '@qinglong/runtime-core/trusted-tool-completion';
|
||||
|
||||
import { BoundedModelGateway } from '../../../model-gateway/gateway';
|
||||
import type { ModelInvocationRepository } from '../../../model-invocation/modelInvocation';
|
||||
import type { CopilotFailureDiagnosisToolExecutionAdmissionReader } from '../tool-execution/contracts';
|
||||
import type { CopilotFailureDiagnosisToolUnlockRepository } from '../tool-execution/contracts';
|
||||
import { buildFailureDiagnosisPromptPlan } from '../prompt';
|
||||
import { normalizeFailureDiagnosisProjection } from '../validation';
|
||||
import type {
|
||||
CopilotFailureDiagnosisModelCompletionCoordinator,
|
||||
CopilotFailureDiagnosisOutputCompletionRepository,
|
||||
} from './completion';
|
||||
import {
|
||||
copilotFailureDiagnosisOutputReference,
|
||||
type CopilotFailureDiagnosisOutputReference,
|
||||
} from './outputArtifact';
|
||||
import type {
|
||||
CopilotFailureDiagnosisFinalizationReceipt,
|
||||
CopilotFailureDiagnosisFinalizationRepository,
|
||||
} from './finalization';
|
||||
|
||||
export interface CopilotFailureDiagnosisModelExecutionDependencies {
|
||||
readonly admissions: CopilotFailureDiagnosisToolExecutionAdmissionReader;
|
||||
readonly unlocks: Pick<CopilotFailureDiagnosisToolUnlockRepository, 'findByRequestId'>;
|
||||
readonly toolResults: CopilotFailureDiagnosisToolResultReader;
|
||||
readonly modelInvocations: Pick<
|
||||
ModelInvocationRepository,
|
||||
'findStart' | 'findCompletion'
|
||||
>;
|
||||
readonly outputs: Pick<
|
||||
CopilotFailureDiagnosisOutputCompletionRepository,
|
||||
'findCopilotFailureDiagnosisOutput'
|
||||
>;
|
||||
readonly gateway: BoundedModelGateway;
|
||||
readonly successfulCompletion: CopilotFailureDiagnosisModelCompletionCoordinator;
|
||||
readonly finalizations: CopilotFailureDiagnosisFinalizationRepository;
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisToolResultReader {
|
||||
open(startId: string): Promise<Readonly<TrustedToolSuccessCompletionResult>>;
|
||||
}
|
||||
|
||||
export function createCopilotFailureDiagnosisToolResultReader(
|
||||
dependencies: TrustedToolSuccessCompletionReadDependencies,
|
||||
): CopilotFailureDiagnosisToolResultReader {
|
||||
return Object.freeze({
|
||||
open: (startId: string) =>
|
||||
openTrustedToolSuccessCompletion(startId, dependencies),
|
||||
});
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisModelExecutionResult {
|
||||
readonly outcome: 'succeeded' | 'failed' | 'timed_out' | 'cancelled';
|
||||
readonly output: Readonly<CopilotFailureDiagnosisOutputReference> | null;
|
||||
readonly finalization: Readonly<CopilotFailureDiagnosisFinalizationReceipt>;
|
||||
}
|
||||
|
||||
export class InvalidCopilotFailureDiagnosisModelExecutionError extends TypeError {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_INVALID';
|
||||
constructor(message: string) {
|
||||
super(`Copilot failure diagnosis Model execution is invalid: ${message}`);
|
||||
this.name = 'InvalidCopilotFailureDiagnosisModelExecutionError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisModelExecutionConflictError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_CONFLICT';
|
||||
constructor(message = 'durable Model execution facts changed') {
|
||||
super(`Copilot failure diagnosis Model execution conflicts: ${message}`);
|
||||
this.name = 'CopilotFailureDiagnosisModelExecutionConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisModelExecutionUnavailableError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_UNAVAILABLE';
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Copilot failure diagnosis Model execution is unavailable', options);
|
||||
this.name = 'CopilotFailureDiagnosisModelExecutionUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function assertDependencies(
|
||||
value: CopilotFailureDiagnosisModelExecutionDependencies,
|
||||
): void {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
typeof value.admissions?.findPlanByRequestId !== 'function' ||
|
||||
typeof value.admissions?.findByRequestId !== 'function' ||
|
||||
typeof value.unlocks?.findByRequestId !== 'function' ||
|
||||
typeof value.toolResults?.open !== 'function' ||
|
||||
typeof value.modelInvocations?.findStart !== 'function' ||
|
||||
typeof value.modelInvocations?.findCompletion !== 'function' ||
|
||||
typeof value.outputs?.findCopilotFailureDiagnosisOutput !== 'function' ||
|
||||
!(value.gateway instanceof BoundedModelGateway) ||
|
||||
typeof value.successfulCompletion?.begin !== 'function' ||
|
||||
typeof value.successfulCompletion?.reference !== 'function' ||
|
||||
typeof value.successfulCompletion?.end !== 'function' ||
|
||||
typeof value.finalizations?.findFinalization !== 'function' ||
|
||||
typeof value.finalizations?.finalize !== 'function'
|
||||
) {
|
||||
throw new InvalidCopilotFailureDiagnosisModelExecutionError(
|
||||
'dependencies are invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function projection(
|
||||
output: ToolJsonValue,
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
) {
|
||||
if (!output || typeof output !== 'object' || Array.isArray(output)) {
|
||||
throw new CopilotFailureDiagnosisModelExecutionConflictError(
|
||||
'Tool output is not an object',
|
||||
);
|
||||
}
|
||||
const record = output as Readonly<Record<string, ToolJsonValue>>;
|
||||
if (
|
||||
record.status !== 'available' ||
|
||||
record.runId !== runId ||
|
||||
record.attemptId !== attemptId ||
|
||||
record.profile !== 'cluster-control'
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisModelExecutionConflictError(
|
||||
'the admitted source log projection is unavailable',
|
||||
);
|
||||
}
|
||||
return normalizeFailureDiagnosisProjection(
|
||||
{
|
||||
content: record.content,
|
||||
sourceBytes: record.sourceBytes,
|
||||
modelTextBytes: record.modelTextBytes,
|
||||
redaction: record.redaction,
|
||||
normalization: record.normalization,
|
||||
trust: record.trust,
|
||||
},
|
||||
'cluster-control',
|
||||
);
|
||||
}
|
||||
|
||||
async function finalize(
|
||||
requestId: string,
|
||||
dependencies: CopilotFailureDiagnosisModelExecutionDependencies,
|
||||
): Promise<Readonly<CopilotFailureDiagnosisModelExecutionResult>> {
|
||||
const result = await dependencies.finalizations.finalize(requestId);
|
||||
let output: Readonly<CopilotFailureDiagnosisOutputReference> | null = null;
|
||||
if (result.receipt.outputArtifactId !== null) {
|
||||
const artifact =
|
||||
await dependencies.outputs.findCopilotFailureDiagnosisOutput(
|
||||
result.receipt.outputArtifactId,
|
||||
);
|
||||
if (artifact) output = copilotFailureDiagnosisOutputReference(artifact);
|
||||
if (!output) {
|
||||
throw new CopilotFailureDiagnosisModelExecutionConflictError(
|
||||
'terminal output reference is unavailable',
|
||||
);
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
outcome: result.receipt.outcome,
|
||||
output,
|
||||
finalization: result.receipt,
|
||||
});
|
||||
}
|
||||
|
||||
export async function executeCopilotFailureDiagnosisModel(
|
||||
requestId: string,
|
||||
dependencies: CopilotFailureDiagnosisModelExecutionDependencies,
|
||||
): Promise<Readonly<CopilotFailureDiagnosisModelExecutionResult>> {
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(requestId)) {
|
||||
throw new InvalidCopilotFailureDiagnosisModelExecutionError(
|
||||
'request id is invalid',
|
||||
);
|
||||
}
|
||||
assertDependencies(dependencies);
|
||||
const existingFinalization = await dependencies.finalizations.findFinalization(
|
||||
requestId,
|
||||
);
|
||||
if (existingFinalization) return finalize(requestId, dependencies);
|
||||
|
||||
const [plan, admission, unlock] = await Promise.all([
|
||||
dependencies.admissions.findPlanByRequestId(requestId),
|
||||
dependencies.admissions.findByRequestId(requestId),
|
||||
dependencies.unlocks.findByRequestId(requestId),
|
||||
]);
|
||||
if (
|
||||
!plan ||
|
||||
!admission ||
|
||||
!unlock ||
|
||||
admission.planDigest !== plan.planDigest ||
|
||||
unlock.planDigest !== plan.planDigest ||
|
||||
unlock.runId !== plan.runId ||
|
||||
unlock.modelStepRunId !== plan.modelStepRunId
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisModelExecutionConflictError(
|
||||
'admission or Tool unlock evidence is incomplete',
|
||||
);
|
||||
}
|
||||
const existingCompletion = await dependencies.modelInvocations.findCompletion(
|
||||
plan.modelInvocationId,
|
||||
);
|
||||
if (existingCompletion) return finalize(requestId, dependencies);
|
||||
const existingStart = await dependencies.modelInvocations.findStart(
|
||||
plan.modelInvocationId,
|
||||
);
|
||||
if (existingStart) {
|
||||
throw new CopilotFailureDiagnosisModelExecutionConflictError(
|
||||
'an incomplete Model invocation cannot be executed again automatically',
|
||||
);
|
||||
}
|
||||
|
||||
const tool = await dependencies.toolResults.open(unlock.startId);
|
||||
if (
|
||||
tool.completion.completionDigest !== unlock.toolCompletionDigest ||
|
||||
tool.completion.runId !== plan.runId ||
|
||||
tool.completion.stepRunId !== plan.toolStepRunId
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisModelExecutionConflictError(
|
||||
'Tool completion evidence changed',
|
||||
);
|
||||
}
|
||||
const prompt = buildFailureDiagnosisPromptPlan({
|
||||
provider: plan.model.provider,
|
||||
model: plan.model.model,
|
||||
modelBoundary: plan.model.modelBoundary,
|
||||
profile: 'cluster-control',
|
||||
responseLanguage: plan.model.responseLanguage,
|
||||
projection: projection(
|
||||
tool.output,
|
||||
plan.source.runId,
|
||||
plan.source.attemptId,
|
||||
),
|
||||
maxOutputTokens: plan.model.maxOutputTokens,
|
||||
egressPolicy: plan.model.egressPolicy,
|
||||
});
|
||||
if (
|
||||
!dependencies.gateway.supportsSuccessfulCompletionSink(
|
||||
dependencies.successfulCompletion,
|
||||
)
|
||||
) {
|
||||
throw new InvalidCopilotFailureDiagnosisModelExecutionError(
|
||||
'Gateway successful completion sink is not the Copilot sink',
|
||||
);
|
||||
}
|
||||
const lease = dependencies.successfulCompletion.begin({
|
||||
plan,
|
||||
prompt,
|
||||
toolCompletionDigest: unlock.toolCompletionDigest,
|
||||
});
|
||||
try {
|
||||
try {
|
||||
await dependencies.gateway.generate(prompt.request, {
|
||||
projectId: plan.projectId,
|
||||
runId: plan.runId,
|
||||
stepRunId: plan.modelStepRunId,
|
||||
traceId: plan.traceId,
|
||||
requestId: plan.modelInvocationId,
|
||||
deadlineAtMs: plan.deadlineAtMs,
|
||||
});
|
||||
} catch (cause) {
|
||||
const completion = await dependencies.modelInvocations.findCompletion(
|
||||
plan.modelInvocationId,
|
||||
);
|
||||
if (!completion) {
|
||||
throw new CopilotFailureDiagnosisModelExecutionUnavailableError({
|
||||
cause: cause instanceof Error ? cause : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
return finalize(requestId, dependencies);
|
||||
} finally {
|
||||
dependencies.successfulCompletion.end(lease);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
export const COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_RECEIPT_SCHEMA =
|
||||
'qinglong/copilot-failure-diagnosis-finalization-receipt@v1' as const;
|
||||
export const COPILOT_FAILURE_DIAGNOSIS_FINAL_OUTCOMES = [
|
||||
'succeeded',
|
||||
'failed',
|
||||
'timed_out',
|
||||
'cancelled',
|
||||
] as const;
|
||||
|
||||
export type CopilotFailureDiagnosisFinalOutcome =
|
||||
(typeof COPILOT_FAILURE_DIAGNOSIS_FINAL_OUTCOMES)[number];
|
||||
|
||||
export interface CopilotFailureDiagnosisFinalizationReceipt {
|
||||
readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_RECEIPT_SCHEMA;
|
||||
readonly requestId: string;
|
||||
readonly planDigest: string;
|
||||
readonly runId: string;
|
||||
readonly modelStepRunId: string;
|
||||
readonly invocationId: string;
|
||||
readonly completionDigest: string;
|
||||
readonly outcome: CopilotFailureDiagnosisFinalOutcome;
|
||||
readonly outputArtifactId: string | null;
|
||||
readonly finalRunVersion: number;
|
||||
readonly finalRunEventSequence: number;
|
||||
readonly runEventId: string;
|
||||
readonly finalizedAtMs: number;
|
||||
readonly receiptDigest: string;
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisFinalizationRepository {
|
||||
findFinalization(
|
||||
requestId: string,
|
||||
): Promise<Readonly<CopilotFailureDiagnosisFinalizationReceipt> | null>;
|
||||
finalize(requestId: string): Promise<Readonly<{
|
||||
status: 'created' | 'existing';
|
||||
receipt: Readonly<CopilotFailureDiagnosisFinalizationReceipt>;
|
||||
}>>;
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisModelExecutionInProgressError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_IN_PROGRESS';
|
||||
constructor() {
|
||||
super('Copilot failure diagnosis Model execution is in progress');
|
||||
this.name = 'CopilotFailureDiagnosisModelExecutionInProgressError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisModelResolutionRequiredError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_MODEL_RESOLUTION_REQUIRED';
|
||||
constructor() {
|
||||
super('Copilot failure diagnosis Model execution requires resolution');
|
||||
this.name = 'CopilotFailureDiagnosisModelResolutionRequiredError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisFinalizationConflictError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_CONFLICT';
|
||||
constructor() {
|
||||
super('Copilot failure diagnosis finalization conflicts');
|
||||
this.name = 'CopilotFailureDiagnosisFinalizationConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisFinalizationUnavailableError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_UNAVAILABLE';
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Copilot failure diagnosis finalization is unavailable', options);
|
||||
this.name = 'CopilotFailureDiagnosisFinalizationUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/;
|
||||
const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
|
||||
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
|
||||
const DIGEST_DOMAIN =
|
||||
'qinglong/copilot-failure-diagnosis-finalization-receipt-digest@v1\0';
|
||||
const EVENT_DOMAIN =
|
||||
'qinglong/copilot-failure-diagnosis-finalization-event-id@v1\0';
|
||||
|
||||
function invalid(): never {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
|
||||
function text(value: unknown, pattern: RegExp): string {
|
||||
if (typeof value !== 'string' || !pattern.test(value)) return invalid();
|
||||
return value;
|
||||
}
|
||||
|
||||
function integer(value: unknown): number {
|
||||
if (!Number.isSafeInteger(value) || (value as number) < 0) return invalid();
|
||||
return value as number;
|
||||
}
|
||||
|
||||
function unsigned(
|
||||
value: Omit<CopilotFailureDiagnosisFinalizationReceipt, 'receiptDigest'>,
|
||||
): object {
|
||||
return {
|
||||
schema: value.schema,
|
||||
requestId: value.requestId,
|
||||
planDigest: value.planDigest,
|
||||
runId: value.runId,
|
||||
modelStepRunId: value.modelStepRunId,
|
||||
invocationId: value.invocationId,
|
||||
completionDigest: value.completionDigest,
|
||||
outcome: value.outcome,
|
||||
outputArtifactId: value.outputArtifactId,
|
||||
finalRunVersion: value.finalRunVersion,
|
||||
finalRunEventSequence: value.finalRunEventSequence,
|
||||
runEventId: value.runEventId,
|
||||
finalizedAtMs: value.finalizedAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
export function copilotFailureDiagnosisFinalizationReceiptDigest(
|
||||
value: Omit<CopilotFailureDiagnosisFinalizationReceipt, 'receiptDigest'>,
|
||||
): string {
|
||||
return createHash('sha256')
|
||||
.update(DIGEST_DOMAIN)
|
||||
.update(JSON.stringify(unsigned(value)))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
export function copilotFailureDiagnosisFinalizationEventIdentity(
|
||||
invocationId: string,
|
||||
completionDigest: string,
|
||||
): string {
|
||||
const hex = createHash('sha256')
|
||||
.update(EVENT_DOMAIN)
|
||||
.update(text(invocationId, ID_PATTERN))
|
||||
.update(text(completionDigest, DIGEST_PATTERN))
|
||||
.digest('hex')
|
||||
.slice(0, 32)
|
||||
.split('');
|
||||
hex[12] = '4';
|
||||
hex[16] = '8';
|
||||
const value = hex.join('');
|
||||
return `${value.slice(0, 8)}-${value.slice(8, 12)}-${value.slice(
|
||||
12,
|
||||
16,
|
||||
)}-${value.slice(16, 20)}-${value.slice(20)}`;
|
||||
}
|
||||
|
||||
export function normalizeCopilotFailureDiagnosisFinalizationReceipt(
|
||||
value: CopilotFailureDiagnosisFinalizationReceipt,
|
||||
): Readonly<CopilotFailureDiagnosisFinalizationReceipt> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
return invalid();
|
||||
}
|
||||
const expected = [
|
||||
'completionDigest', 'finalRunEventSequence', 'finalRunVersion',
|
||||
'finalizedAtMs', 'invocationId', 'outcome', 'outputArtifactId',
|
||||
'modelStepRunId', 'planDigest', 'receiptDigest', 'requestId', 'runEventId',
|
||||
'runId', 'schema',
|
||||
];
|
||||
const keys = Reflect.ownKeys(value);
|
||||
if (
|
||||
keys.length !== expected.length ||
|
||||
keys.some((item) => typeof item !== 'string' || !expected.includes(item)) ||
|
||||
value.schema !== COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_RECEIPT_SCHEMA ||
|
||||
!COPILOT_FAILURE_DIAGNOSIS_FINAL_OUTCOMES.includes(value.outcome)
|
||||
) {
|
||||
return invalid();
|
||||
}
|
||||
const outputArtifactId =
|
||||
value.outputArtifactId === null
|
||||
? null
|
||||
: text(value.outputArtifactId, ID_PATTERN);
|
||||
if ((value.outcome === 'succeeded') !== (outputArtifactId !== null)) {
|
||||
return invalid();
|
||||
}
|
||||
const normalized = Object.freeze({
|
||||
schema: value.schema,
|
||||
requestId: text(value.requestId, ID_PATTERN),
|
||||
planDigest: text(value.planDigest, DIGEST_PATTERN),
|
||||
runId: text(value.runId, RUN_ID_PATTERN),
|
||||
modelStepRunId: text(value.modelStepRunId, ID_PATTERN),
|
||||
invocationId: text(value.invocationId, ID_PATTERN),
|
||||
completionDigest: text(value.completionDigest, DIGEST_PATTERN),
|
||||
outcome: value.outcome,
|
||||
outputArtifactId,
|
||||
finalRunVersion: integer(value.finalRunVersion),
|
||||
finalRunEventSequence: integer(value.finalRunEventSequence),
|
||||
runEventId: text(value.runEventId, RUN_ID_PATTERN),
|
||||
finalizedAtMs: integer(value.finalizedAtMs),
|
||||
} satisfies Omit<
|
||||
CopilotFailureDiagnosisFinalizationReceipt,
|
||||
'receiptDigest'
|
||||
>);
|
||||
if (
|
||||
normalized.finalRunVersion < 1 ||
|
||||
normalized.finalRunEventSequence !== normalized.finalRunVersion ||
|
||||
normalized.runEventId !==
|
||||
copilotFailureDiagnosisFinalizationEventIdentity(
|
||||
normalized.invocationId,
|
||||
normalized.completionDigest,
|
||||
) ||
|
||||
text(value.receiptDigest, DIGEST_PATTERN) !==
|
||||
copilotFailureDiagnosisFinalizationReceiptDigest(normalized)
|
||||
) {
|
||||
return invalid();
|
||||
}
|
||||
return Object.freeze({ ...normalized, receiptDigest: value.receiptDigest });
|
||||
}
|
||||
|
||||
export function createCopilotFailureDiagnosisFinalizationReceipt(
|
||||
value: Omit<
|
||||
CopilotFailureDiagnosisFinalizationReceipt,
|
||||
'schema' | 'runEventId' | 'receiptDigest'
|
||||
>,
|
||||
): Readonly<CopilotFailureDiagnosisFinalizationReceipt> {
|
||||
const unsignedReceipt = Object.freeze({
|
||||
schema: COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_RECEIPT_SCHEMA,
|
||||
...value,
|
||||
runEventId: copilotFailureDiagnosisFinalizationEventIdentity(
|
||||
value.invocationId,
|
||||
value.completionDigest,
|
||||
),
|
||||
});
|
||||
return normalizeCopilotFailureDiagnosisFinalizationReceipt({
|
||||
...unsignedReceipt,
|
||||
receiptDigest:
|
||||
copilotFailureDiagnosisFinalizationReceiptDigest(unsignedReceipt),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,491 @@
|
||||
import { Buffer } from 'node:buffer';
|
||||
import {
|
||||
createCipheriv,
|
||||
createDecipheriv,
|
||||
createHash,
|
||||
randomBytes,
|
||||
} from 'node:crypto';
|
||||
|
||||
import type { GenerateResult } from '../../../model-gateway/model';
|
||||
import { normalizeGenerateResult } from '../../../model-gateway/validation';
|
||||
import type { FailureDiagnosisModelEgressEvidence } from '../contracts';
|
||||
|
||||
export const COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_SCHEMA =
|
||||
'qinglong/copilot-failure-diagnosis-output-artifact@v1' as const;
|
||||
export const COPILOT_FAILURE_DIAGNOSIS_OUTPUT_REFERENCE_SCHEMA =
|
||||
'qinglong/copilot-failure-diagnosis-output-reference@v1' as const;
|
||||
export const COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM =
|
||||
'aes-256-gcm' as const;
|
||||
export const MAX_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_BYTES =
|
||||
1536 * 1024;
|
||||
|
||||
export interface CopilotFailureDiagnosisOutputArtifact {
|
||||
readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_SCHEMA;
|
||||
readonly artifactId: string;
|
||||
readonly requestId: string;
|
||||
readonly planDigest: string;
|
||||
readonly toolCompletionDigest: string;
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly stepRunId: string;
|
||||
readonly invocationId: string;
|
||||
readonly provider: string;
|
||||
readonly model: string;
|
||||
readonly egressEvidenceDigest: string;
|
||||
readonly contentDigest: string;
|
||||
readonly outputBytes: number;
|
||||
readonly keyId: string;
|
||||
readonly algorithm: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM;
|
||||
readonly nonce: string;
|
||||
readonly ciphertext: string;
|
||||
readonly authTag: string;
|
||||
readonly plaintextBytes: number;
|
||||
readonly sealedAtMs: number;
|
||||
readonly artifactDigest: string;
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisOutputReference {
|
||||
readonly schema: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_REFERENCE_SCHEMA;
|
||||
readonly artifactId: string;
|
||||
readonly requestId: string;
|
||||
readonly planDigest: string;
|
||||
readonly toolCompletionDigest: string;
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly stepRunId: string;
|
||||
readonly invocationId: string;
|
||||
readonly provider: string;
|
||||
readonly model: string;
|
||||
readonly egressEvidenceDigest: string;
|
||||
readonly contentDigest: string;
|
||||
readonly outputBytes: number;
|
||||
readonly keyId: string;
|
||||
readonly algorithm: typeof COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM;
|
||||
readonly sealedAtMs: number;
|
||||
readonly artifactDigest: string;
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisOutputKeyMaterial {
|
||||
readonly keyId: string;
|
||||
readonly key: Uint8Array;
|
||||
}
|
||||
|
||||
export interface CopilotFailureDiagnosisOutputKeyProvider {
|
||||
active(): Promise<CopilotFailureDiagnosisOutputKeyMaterial>;
|
||||
resolve(keyId: string): Promise<CopilotFailureDiagnosisOutputKeyMaterial | null>;
|
||||
}
|
||||
|
||||
export class InvalidCopilotFailureDiagnosisOutputArtifactError extends TypeError {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_INVALID';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Copilot failure diagnosis output Artifact is invalid: ${message}`);
|
||||
this.name = 'InvalidCopilotFailureDiagnosisOutputArtifactError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisOutputArtifactConflictError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_CONFLICT';
|
||||
|
||||
constructor() {
|
||||
super('Copilot failure diagnosis output Artifact conflicts');
|
||||
this.name = 'CopilotFailureDiagnosisOutputArtifactConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class CopilotFailureDiagnosisOutputArtifactUnavailableError extends Error {
|
||||
readonly code = 'COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Copilot failure diagnosis output Artifact is unavailable', options);
|
||||
this.name = 'CopilotFailureDiagnosisOutputArtifactUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/;
|
||||
const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
|
||||
const MODEL_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/;
|
||||
const KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
||||
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
|
||||
const BASE64URL_PATTERN = /^[A-Za-z0-9_-]+$/;
|
||||
const ARTIFACT_ID_DOMAIN = Buffer.from(
|
||||
'qinglong/copilot-failure-diagnosis-output-artifact-id@v1\0',
|
||||
);
|
||||
const CONTENT_DIGEST_DOMAIN = Buffer.from(
|
||||
'qinglong/copilot-failure-diagnosis-output-content-digest@v1\0',
|
||||
);
|
||||
const EGRESS_DIGEST_DOMAIN = Buffer.from(
|
||||
'qinglong/copilot-failure-diagnosis-output-egress-digest@v1\0',
|
||||
);
|
||||
const ARTIFACT_DIGEST_DOMAIN = Buffer.from(
|
||||
'qinglong/copilot-failure-diagnosis-output-artifact-digest@v1\0',
|
||||
);
|
||||
const AAD_DOMAIN = Buffer.from(
|
||||
'qinglong/copilot-failure-diagnosis-output-artifact-aad@v1\0',
|
||||
);
|
||||
|
||||
function invalid(message: string): never {
|
||||
throw new InvalidCopilotFailureDiagnosisOutputArtifactError(message);
|
||||
}
|
||||
|
||||
function hash(domain: Buffer, value: unknown): string {
|
||||
return createHash('sha256')
|
||||
.update(domain)
|
||||
.update(JSON.stringify(value))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
function text(value: unknown, pattern: RegExp, label: string): string {
|
||||
if (typeof value !== 'string' || !pattern.test(value)) {
|
||||
return invalid(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function integer(value: unknown, maximum: number, label: string): number {
|
||||
if (
|
||||
!Number.isSafeInteger(value) ||
|
||||
(value as number) < 0 ||
|
||||
(value as number) > maximum
|
||||
) {
|
||||
return invalid(`${label} is invalid`);
|
||||
}
|
||||
return value as number;
|
||||
}
|
||||
|
||||
function bytes(value: unknown, expected?: number): Buffer {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
!BASE64URL_PATTERN.test(value)
|
||||
) {
|
||||
return invalid('encoded bytes are invalid');
|
||||
}
|
||||
const decoded = Buffer.from(value, 'base64url');
|
||||
if (
|
||||
decoded.toString('base64url') !== value ||
|
||||
(expected !== undefined && decoded.length !== expected)
|
||||
) {
|
||||
decoded.fill(0);
|
||||
return invalid('encoded bytes are invalid');
|
||||
}
|
||||
return decoded;
|
||||
}
|
||||
|
||||
function key(value: Uint8Array): Buffer {
|
||||
if (!(value instanceof Uint8Array) || value.byteLength !== 32) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactUnavailableError();
|
||||
}
|
||||
return Buffer.from(value);
|
||||
}
|
||||
|
||||
function unsigned(
|
||||
value: CopilotFailureDiagnosisOutputArtifact,
|
||||
): Omit<CopilotFailureDiagnosisOutputArtifact, 'artifactDigest'> {
|
||||
return {
|
||||
schema: value.schema,
|
||||
artifactId: value.artifactId,
|
||||
requestId: value.requestId,
|
||||
planDigest: value.planDigest,
|
||||
toolCompletionDigest: value.toolCompletionDigest,
|
||||
projectId: value.projectId,
|
||||
runId: value.runId,
|
||||
stepRunId: value.stepRunId,
|
||||
invocationId: value.invocationId,
|
||||
provider: value.provider,
|
||||
model: value.model,
|
||||
egressEvidenceDigest: value.egressEvidenceDigest,
|
||||
contentDigest: value.contentDigest,
|
||||
outputBytes: value.outputBytes,
|
||||
keyId: value.keyId,
|
||||
algorithm: value.algorithm,
|
||||
nonce: value.nonce,
|
||||
ciphertext: value.ciphertext,
|
||||
authTag: value.authTag,
|
||||
plaintextBytes: value.plaintextBytes,
|
||||
sealedAtMs: value.sealedAtMs,
|
||||
};
|
||||
}
|
||||
|
||||
function metadata(
|
||||
value: CopilotFailureDiagnosisOutputArtifact,
|
||||
): Omit<
|
||||
CopilotFailureDiagnosisOutputArtifact,
|
||||
'artifactDigest' | 'nonce' | 'ciphertext' | 'authTag'
|
||||
> {
|
||||
const {
|
||||
artifactDigest: _artifactDigest,
|
||||
nonce: _nonce,
|
||||
ciphertext: _ciphertext,
|
||||
authTag: _authTag,
|
||||
...record
|
||||
} = value;
|
||||
return record;
|
||||
}
|
||||
|
||||
function aad(value: ReturnType<typeof metadata>): Buffer {
|
||||
return Buffer.concat([AAD_DOMAIN, Buffer.from(JSON.stringify(value))]);
|
||||
}
|
||||
|
||||
export function copilotFailureDiagnosisOutputArtifactIdentity(
|
||||
invocationIdValue: string,
|
||||
): string {
|
||||
const invocationId = text(invocationIdValue, ID_PATTERN, 'invocation id');
|
||||
return `cdo:${hash(ARTIFACT_ID_DOMAIN, invocationId).slice(0, 32)}`;
|
||||
}
|
||||
|
||||
export function copilotFailureDiagnosisEgressEvidenceDigest(
|
||||
value: Readonly<FailureDiagnosisModelEgressEvidence>,
|
||||
): string {
|
||||
return hash(EGRESS_DIGEST_DOMAIN, value);
|
||||
}
|
||||
|
||||
export function normalizeCopilotFailureDiagnosisOutputArtifact(
|
||||
value: CopilotFailureDiagnosisOutputArtifact,
|
||||
): Readonly<CopilotFailureDiagnosisOutputArtifact> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
return invalid('Artifact must be an object');
|
||||
}
|
||||
const expected = [
|
||||
'algorithm', 'artifactDigest', 'artifactId', 'authTag', 'ciphertext',
|
||||
'contentDigest', 'egressEvidenceDigest', 'invocationId', 'keyId', 'model',
|
||||
'nonce', 'outputBytes', 'plaintextBytes', 'planDigest', 'projectId',
|
||||
'provider', 'requestId', 'runId', 'schema', 'sealedAtMs', 'stepRunId',
|
||||
'toolCompletionDigest',
|
||||
];
|
||||
const keys = Reflect.ownKeys(value);
|
||||
if (
|
||||
keys.length !== expected.length ||
|
||||
keys.some((item) => typeof item !== 'string' || !expected.includes(item))
|
||||
) {
|
||||
return invalid('Artifact shape is invalid');
|
||||
}
|
||||
if (
|
||||
value.schema !== COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_SCHEMA ||
|
||||
value.algorithm !== COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM
|
||||
) {
|
||||
return invalid('Artifact protocol is unsupported');
|
||||
}
|
||||
const normalized = Object.freeze({
|
||||
schema: value.schema,
|
||||
artifactId: text(value.artifactId, ID_PATTERN, 'Artifact id'),
|
||||
requestId: text(value.requestId, ID_PATTERN, 'request id'),
|
||||
planDigest: text(value.planDigest, DIGEST_PATTERN, 'plan digest'),
|
||||
toolCompletionDigest: text(
|
||||
value.toolCompletionDigest,
|
||||
DIGEST_PATTERN,
|
||||
'Tool completion digest',
|
||||
),
|
||||
projectId: text(value.projectId, ID_PATTERN, 'Project id'),
|
||||
runId: text(value.runId, RUN_ID_PATTERN, 'Run id'),
|
||||
stepRunId: text(value.stepRunId, ID_PATTERN, 'StepRun id'),
|
||||
invocationId: text(value.invocationId, ID_PATTERN, 'invocation id'),
|
||||
provider: text(value.provider, ID_PATTERN, 'provider'),
|
||||
model: text(value.model, MODEL_PATTERN, 'model'),
|
||||
egressEvidenceDigest: text(
|
||||
value.egressEvidenceDigest,
|
||||
DIGEST_PATTERN,
|
||||
'egress evidence digest',
|
||||
),
|
||||
contentDigest: text(value.contentDigest, DIGEST_PATTERN, 'content digest'),
|
||||
outputBytes: integer(value.outputBytes, 1024 * 1024, 'output bytes'),
|
||||
keyId: text(value.keyId, KEY_PATTERN, 'key id'),
|
||||
algorithm: value.algorithm,
|
||||
nonce: bytes(value.nonce, 12).toString('base64url'),
|
||||
ciphertext: bytes(value.ciphertext).toString('base64url'),
|
||||
authTag: bytes(value.authTag, 16).toString('base64url'),
|
||||
plaintextBytes: integer(
|
||||
value.plaintextBytes,
|
||||
1024 * 1024 + 4096,
|
||||
'plaintext bytes',
|
||||
),
|
||||
sealedAtMs: integer(value.sealedAtMs, Number.MAX_SAFE_INTEGER, 'seal time'),
|
||||
artifactDigest: text(
|
||||
value.artifactDigest,
|
||||
DIGEST_PATTERN,
|
||||
'Artifact digest',
|
||||
),
|
||||
} satisfies CopilotFailureDiagnosisOutputArtifact);
|
||||
if (
|
||||
normalized.artifactId !==
|
||||
copilotFailureDiagnosisOutputArtifactIdentity(normalized.invocationId) ||
|
||||
Buffer.byteLength(JSON.stringify(normalized)) >
|
||||
MAX_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_BYTES ||
|
||||
hash(ARTIFACT_DIGEST_DOMAIN, unsigned(normalized)) !==
|
||||
normalized.artifactDigest
|
||||
) {
|
||||
return invalid('Artifact binding is invalid');
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function copilotFailureDiagnosisOutputReference(
|
||||
value: CopilotFailureDiagnosisOutputArtifact,
|
||||
): Readonly<CopilotFailureDiagnosisOutputReference> {
|
||||
const artifact = normalizeCopilotFailureDiagnosisOutputArtifact(value);
|
||||
return Object.freeze({
|
||||
schema: COPILOT_FAILURE_DIAGNOSIS_OUTPUT_REFERENCE_SCHEMA,
|
||||
artifactId: artifact.artifactId,
|
||||
requestId: artifact.requestId,
|
||||
planDigest: artifact.planDigest,
|
||||
toolCompletionDigest: artifact.toolCompletionDigest,
|
||||
projectId: artifact.projectId,
|
||||
runId: artifact.runId,
|
||||
stepRunId: artifact.stepRunId,
|
||||
invocationId: artifact.invocationId,
|
||||
provider: artifact.provider,
|
||||
model: artifact.model,
|
||||
egressEvidenceDigest: artifact.egressEvidenceDigest,
|
||||
contentDigest: artifact.contentDigest,
|
||||
outputBytes: artifact.outputBytes,
|
||||
keyId: artifact.keyId,
|
||||
algorithm: artifact.algorithm,
|
||||
sealedAtMs: artifact.sealedAtMs,
|
||||
artifactDigest: artifact.artifactDigest,
|
||||
});
|
||||
}
|
||||
|
||||
export function createCopilotFailureDiagnosisOutputArtifact(
|
||||
input: Readonly<{
|
||||
requestId: string;
|
||||
planDigest: string;
|
||||
toolCompletionDigest: string;
|
||||
projectId: string;
|
||||
runId: string;
|
||||
stepRunId: string;
|
||||
invocationId: string;
|
||||
result: Readonly<GenerateResult>;
|
||||
egressEvidence: Readonly<FailureDiagnosisModelEgressEvidence>;
|
||||
keyId: string;
|
||||
key: Uint8Array;
|
||||
sealedAtMs: number;
|
||||
}>,
|
||||
nonceFactory: () => Uint8Array = () => randomBytes(12),
|
||||
): Readonly<CopilotFailureDiagnosisOutputArtifact> {
|
||||
const result = normalizeGenerateResult(input.result);
|
||||
const plaintext = Buffer.from(JSON.stringify(result));
|
||||
const ownedKey = key(input.key);
|
||||
let nonce: Buffer | undefined;
|
||||
try {
|
||||
nonce = Buffer.from(nonceFactory());
|
||||
if (nonce.length !== 12) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactUnavailableError();
|
||||
}
|
||||
const base = {
|
||||
schema: COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ARTIFACT_SCHEMA,
|
||||
artifactId: copilotFailureDiagnosisOutputArtifactIdentity(
|
||||
input.invocationId,
|
||||
),
|
||||
requestId: text(input.requestId, ID_PATTERN, 'request id'),
|
||||
planDigest: text(input.planDigest, DIGEST_PATTERN, 'plan digest'),
|
||||
toolCompletionDigest: text(
|
||||
input.toolCompletionDigest,
|
||||
DIGEST_PATTERN,
|
||||
'Tool completion digest',
|
||||
),
|
||||
projectId: text(input.projectId, ID_PATTERN, 'Project id'),
|
||||
runId: text(input.runId, RUN_ID_PATTERN, 'Run id'),
|
||||
stepRunId: text(input.stepRunId, ID_PATTERN, 'StepRun id'),
|
||||
invocationId: text(input.invocationId, ID_PATTERN, 'invocation id'),
|
||||
provider: result.provider,
|
||||
model: result.model,
|
||||
egressEvidenceDigest: copilotFailureDiagnosisEgressEvidenceDigest(
|
||||
input.egressEvidence,
|
||||
),
|
||||
contentDigest: hash(CONTENT_DIGEST_DOMAIN, result),
|
||||
outputBytes: Buffer.byteLength(result.text),
|
||||
keyId: text(input.keyId, KEY_PATTERN, 'key id'),
|
||||
algorithm: COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM,
|
||||
plaintextBytes: plaintext.length,
|
||||
sealedAtMs: integer(
|
||||
input.sealedAtMs,
|
||||
Number.MAX_SAFE_INTEGER,
|
||||
'seal time',
|
||||
),
|
||||
} as const;
|
||||
const associated = aad(base as ReturnType<typeof metadata>);
|
||||
const cipher = createCipheriv(
|
||||
COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM,
|
||||
ownedKey,
|
||||
nonce,
|
||||
{ authTagLength: 16 },
|
||||
);
|
||||
cipher.setAAD(associated);
|
||||
const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
|
||||
try {
|
||||
const unsignedArtifact = {
|
||||
...base,
|
||||
nonce: nonce.toString('base64url'),
|
||||
ciphertext: ciphertext.toString('base64url'),
|
||||
authTag: cipher.getAuthTag().toString('base64url'),
|
||||
};
|
||||
const candidate = {
|
||||
...unsignedArtifact,
|
||||
artifactDigest: '0'.repeat(64),
|
||||
};
|
||||
return normalizeCopilotFailureDiagnosisOutputArtifact({
|
||||
...candidate,
|
||||
artifactDigest: hash(ARTIFACT_DIGEST_DOMAIN, unsigned(candidate)),
|
||||
});
|
||||
} finally {
|
||||
associated.fill(0);
|
||||
ciphertext.fill(0);
|
||||
}
|
||||
} catch (cause) {
|
||||
if (cause instanceof InvalidCopilotFailureDiagnosisOutputArtifactError) {
|
||||
throw cause;
|
||||
}
|
||||
throw new CopilotFailureDiagnosisOutputArtifactUnavailableError({ cause });
|
||||
} finally {
|
||||
plaintext.fill(0);
|
||||
ownedKey.fill(0);
|
||||
nonce?.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
export function openCopilotFailureDiagnosisOutputArtifact(
|
||||
value: CopilotFailureDiagnosisOutputArtifact,
|
||||
keyValue: Uint8Array,
|
||||
): Readonly<GenerateResult> {
|
||||
const artifact = normalizeCopilotFailureDiagnosisOutputArtifact(value);
|
||||
const ownedKey = key(keyValue);
|
||||
const nonce = bytes(artifact.nonce, 12);
|
||||
const ciphertext = bytes(artifact.ciphertext);
|
||||
const authTag = bytes(artifact.authTag, 16);
|
||||
const associated = aad(metadata(artifact));
|
||||
let plaintext: Buffer | undefined;
|
||||
try {
|
||||
const decipher = createDecipheriv(
|
||||
COPILOT_FAILURE_DIAGNOSIS_OUTPUT_ALGORITHM,
|
||||
ownedKey,
|
||||
nonce,
|
||||
{ authTagLength: 16 },
|
||||
);
|
||||
decipher.setAAD(associated);
|
||||
decipher.setAuthTag(authTag);
|
||||
plaintext = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
|
||||
const result = normalizeGenerateResult(JSON.parse(plaintext.toString()));
|
||||
if (
|
||||
plaintext.length !== artifact.plaintextBytes ||
|
||||
result.provider !== artifact.provider ||
|
||||
result.model !== artifact.model ||
|
||||
Buffer.byteLength(result.text) !== artifact.outputBytes ||
|
||||
hash(CONTENT_DIGEST_DOMAIN, result) !== artifact.contentDigest
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactUnavailableError();
|
||||
}
|
||||
return result;
|
||||
} catch (cause) {
|
||||
if (cause instanceof CopilotFailureDiagnosisOutputArtifactUnavailableError) {
|
||||
throw cause;
|
||||
}
|
||||
throw new CopilotFailureDiagnosisOutputArtifactUnavailableError({ cause });
|
||||
} finally {
|
||||
ownedKey.fill(0);
|
||||
nonce.fill(0);
|
||||
ciphertext.fill(0);
|
||||
authTag.fill(0);
|
||||
associated.fill(0);
|
||||
plaintext?.fill(0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,577 @@
|
||||
import type { PostgresClient, PostgresPool } from '@qinglong/runtime-core';
|
||||
|
||||
import { PostgresModelInvocationRepository } from '../../../model-invocation/postgres-model-invocation-repository/repository';
|
||||
import { completeWithAtomicOutputOperation } from '../../../model-invocation/postgres-model-invocation-repository/completionOperations';
|
||||
import {
|
||||
normalizeModelInvocationCompletionRecord,
|
||||
type ModelInvocationCompletionCommand,
|
||||
type ModelInvocationCompletionRecord,
|
||||
} from '../../../model-invocation/modelInvocation';
|
||||
import { normalizeCopilotFailureDiagnosisExecutionPlan } from '../admission/plan';
|
||||
import type { CopilotFailureDiagnosisExecutionPlan } from '../admission/contracts';
|
||||
import {
|
||||
assertCopilotFailureDiagnosisOutputCompletionBinding,
|
||||
type CommitCopilotFailureDiagnosisOutputResult,
|
||||
type CopilotFailureDiagnosisOutputCompletionRepository,
|
||||
} from './completion';
|
||||
import {
|
||||
CopilotFailureDiagnosisOutputArtifactConflictError,
|
||||
CopilotFailureDiagnosisOutputArtifactUnavailableError,
|
||||
copilotFailureDiagnosisOutputArtifactIdentity,
|
||||
normalizeCopilotFailureDiagnosisOutputArtifact,
|
||||
type CopilotFailureDiagnosisOutputArtifact,
|
||||
} from './outputArtifact';
|
||||
import {
|
||||
CopilotFailureDiagnosisFinalizationConflictError,
|
||||
CopilotFailureDiagnosisFinalizationUnavailableError,
|
||||
CopilotFailureDiagnosisModelExecutionInProgressError,
|
||||
CopilotFailureDiagnosisModelResolutionRequiredError,
|
||||
createCopilotFailureDiagnosisFinalizationReceipt,
|
||||
normalizeCopilotFailureDiagnosisFinalizationReceipt,
|
||||
type CopilotFailureDiagnosisFinalizationReceipt,
|
||||
type CopilotFailureDiagnosisFinalizationRepository,
|
||||
type CopilotFailureDiagnosisFinalOutcome,
|
||||
} from './finalization';
|
||||
|
||||
const TABLE = '"ql3_ai"."copilot_failure_diagnosis_model_outputs"';
|
||||
const FINALIZATION_TABLE =
|
||||
'"ql3_ai"."copilot_failure_diagnosis_finalizations"';
|
||||
|
||||
interface OutputRow extends Record<string, unknown> {
|
||||
readonly artifactJson: unknown;
|
||||
}
|
||||
|
||||
function integer(value: unknown): number {
|
||||
if (typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isSafeInteger(parsed)) return parsed;
|
||||
}
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
|
||||
function object(value: unknown): Record<string, unknown> {
|
||||
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
try {
|
||||
return JSON.parse(value) as Record<string, unknown>;
|
||||
} catch {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
}
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
|
||||
function string(value: unknown): string {
|
||||
if (typeof value !== 'string') {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function unavailable(cause?: unknown): never {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactUnavailableError({
|
||||
cause: cause instanceof Error ? cause : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function parse(row: OutputRow): Readonly<CopilotFailureDiagnosisOutputArtifact> {
|
||||
try {
|
||||
return normalizeCopilotFailureDiagnosisOutputArtifact(
|
||||
row.artifactJson as CopilotFailureDiagnosisOutputArtifact,
|
||||
);
|
||||
} catch (cause) {
|
||||
return unavailable(cause);
|
||||
}
|
||||
}
|
||||
|
||||
async function read(
|
||||
queryable: Pick<PostgresPool, 'query'> | Pick<PostgresClient, 'query'>,
|
||||
artifactId: string,
|
||||
): Promise<Readonly<CopilotFailureDiagnosisOutputArtifact> | null> {
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(artifactId)) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactConflictError();
|
||||
}
|
||||
try {
|
||||
const result = await queryable.query<OutputRow>(
|
||||
`SELECT artifact_json AS "artifactJson"
|
||||
FROM ${TABLE}
|
||||
WHERE artifact_id = $1`,
|
||||
[artifactId],
|
||||
);
|
||||
if (result.rows.length > 1) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactConflictError();
|
||||
}
|
||||
return result.rows[0] ? parse(result.rows[0]) : null;
|
||||
} catch (cause) {
|
||||
if (cause instanceof CopilotFailureDiagnosisOutputArtifactConflictError) {
|
||||
throw cause;
|
||||
}
|
||||
return unavailable(cause);
|
||||
}
|
||||
}
|
||||
|
||||
async function put(
|
||||
client: PostgresClient,
|
||||
artifactValue: CopilotFailureDiagnosisOutputArtifact,
|
||||
): Promise<Readonly<CopilotFailureDiagnosisOutputArtifact>> {
|
||||
const artifact = normalizeCopilotFailureDiagnosisOutputArtifact(artifactValue);
|
||||
const existing = await read(client, artifact.artifactId);
|
||||
if (existing) {
|
||||
if (JSON.stringify(existing) !== JSON.stringify(artifact)) {
|
||||
throw new CopilotFailureDiagnosisOutputArtifactConflictError();
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
try {
|
||||
await client.query(
|
||||
`INSERT INTO ${TABLE} (
|
||||
artifact_id, request_id, plan_digest, tool_completion_digest,
|
||||
project_id, run_id, step_run_id, invocation_id, provider, model,
|
||||
egress_evidence_digest, content_digest, output_bytes, key_id,
|
||||
algorithm, sealed_at_ms, artifact_digest, artifact_json
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
|
||||
$11, $12, $13, $14, $15, $16, $17, $18::jsonb
|
||||
)`,
|
||||
[
|
||||
artifact.artifactId,
|
||||
artifact.requestId,
|
||||
artifact.planDigest,
|
||||
artifact.toolCompletionDigest,
|
||||
artifact.projectId,
|
||||
artifact.runId,
|
||||
artifact.stepRunId,
|
||||
artifact.invocationId,
|
||||
artifact.provider,
|
||||
artifact.model,
|
||||
artifact.egressEvidenceDigest,
|
||||
artifact.contentDigest,
|
||||
artifact.outputBytes,
|
||||
artifact.keyId,
|
||||
artifact.algorithm,
|
||||
artifact.sealedAtMs,
|
||||
artifact.artifactDigest,
|
||||
JSON.stringify(artifact),
|
||||
],
|
||||
);
|
||||
return artifact;
|
||||
} catch (cause) {
|
||||
return unavailable(cause);
|
||||
}
|
||||
}
|
||||
|
||||
export class PostgresCopilotFailureDiagnosisModelRepository
|
||||
extends PostgresModelInvocationRepository
|
||||
implements
|
||||
CopilotFailureDiagnosisOutputCompletionRepository,
|
||||
CopilotFailureDiagnosisFinalizationRepository
|
||||
{
|
||||
readonly #pool: PostgresPool;
|
||||
|
||||
constructor(pool: PostgresPool) {
|
||||
super(pool);
|
||||
this.#pool = pool;
|
||||
}
|
||||
|
||||
findCopilotFailureDiagnosisOutput(
|
||||
artifactId: string,
|
||||
): Promise<Readonly<CopilotFailureDiagnosisOutputArtifact> | null> {
|
||||
return read(this.#pool, artifactId);
|
||||
}
|
||||
|
||||
async completeWithCopilotFailureDiagnosisOutput(
|
||||
commandValue: Readonly<ModelInvocationCompletionCommand>,
|
||||
artifactValue: Readonly<CopilotFailureDiagnosisOutputArtifact>,
|
||||
): Promise<Readonly<CommitCopilotFailureDiagnosisOutputResult>> {
|
||||
const binding = assertCopilotFailureDiagnosisOutputCompletionBinding(
|
||||
commandValue,
|
||||
artifactValue,
|
||||
);
|
||||
const result = await completeWithAtomicOutputOperation(
|
||||
this.#pool,
|
||||
commandValue,
|
||||
{
|
||||
artifact: binding.artifact,
|
||||
reference: binding.reference,
|
||||
read: (client) => read(client, binding.artifact.artifactId),
|
||||
put: (client) => put(client, binding.artifact),
|
||||
matches: (stored) =>
|
||||
JSON.stringify(stored) === JSON.stringify(binding.artifact),
|
||||
},
|
||||
);
|
||||
return Object.freeze({
|
||||
status: result.status,
|
||||
reference: result.reference,
|
||||
});
|
||||
}
|
||||
|
||||
async findFinalization(
|
||||
requestId: string,
|
||||
): Promise<Readonly<CopilotFailureDiagnosisFinalizationReceipt> | null> {
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(requestId)) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
try {
|
||||
const result = await this.#pool.query<Record<string, unknown>>(
|
||||
`SELECT request_id AS "requestId", plan_digest AS "planDigest",
|
||||
run_id AS "runId", model_step_run_id AS "modelStepRunId",
|
||||
invocation_id AS "invocationId",
|
||||
completion_digest AS "completionDigest", outcome,
|
||||
output_artifact_id AS "outputArtifactId",
|
||||
final_run_version AS "finalRunVersion",
|
||||
final_run_event_sequence AS "finalRunEventSequence",
|
||||
run_event_id AS "runEventId",
|
||||
finalized_at_ms AS "finalizedAtMs",
|
||||
receipt_digest AS "receiptDigest",
|
||||
receipt_json AS "receiptJson"
|
||||
FROM ${FINALIZATION_TABLE}
|
||||
WHERE request_id = $1`,
|
||||
[requestId],
|
||||
);
|
||||
if (result.rows.length > 1) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
if (!result.rows[0]) return null;
|
||||
const row = result.rows[0];
|
||||
const receipt = normalizeCopilotFailureDiagnosisFinalizationReceipt(
|
||||
object(row.receiptJson) as unknown as CopilotFailureDiagnosisFinalizationReceipt,
|
||||
);
|
||||
if (
|
||||
receipt.requestId !== string(row.requestId) ||
|
||||
receipt.planDigest !== string(row.planDigest) ||
|
||||
receipt.runId !== string(row.runId) ||
|
||||
receipt.modelStepRunId !== string(row.modelStepRunId) ||
|
||||
receipt.invocationId !== string(row.invocationId) ||
|
||||
receipt.completionDigest !== string(row.completionDigest) ||
|
||||
receipt.outcome !== string(row.outcome) ||
|
||||
receipt.outputArtifactId !== row.outputArtifactId ||
|
||||
receipt.finalRunVersion !== integer(row.finalRunVersion) ||
|
||||
receipt.finalRunEventSequence !== integer(row.finalRunEventSequence) ||
|
||||
receipt.runEventId !== string(row.runEventId) ||
|
||||
receipt.finalizedAtMs !== integer(row.finalizedAtMs) ||
|
||||
receipt.receiptDigest !== string(row.receiptDigest)
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
const durable = await this.#pool.query<Record<string, unknown>>(
|
||||
`SELECT run.status AS "runStatus", run.version AS "runVersion",
|
||||
run.event_sequence AS "runEventSequence",
|
||||
run.finished_at_ms AS "finishedAtMs",
|
||||
run.output_ref AS "outputRef",
|
||||
step.status AS "stepStatus",
|
||||
event.type AS "eventType", event.dedupe_key AS "dedupeKey",
|
||||
event.step_run_id AS "eventStepRunId",
|
||||
event.payload, event.created_at_ms AS "eventCreatedAtMs",
|
||||
completion.completion_digest AS "completionDigest",
|
||||
completion.outcome AS "completionOutcome"
|
||||
FROM "ql3"."runs" AS run
|
||||
JOIN "ql3"."step_runs" AS step
|
||||
ON step.run_id = run.id AND step.id = $1
|
||||
JOIN "ql3"."run_events" AS event
|
||||
ON event.run_id = run.id AND event.id = $2
|
||||
JOIN "ql3_ai"."model_invocation_completions" AS completion
|
||||
ON completion.invocation_id = $3
|
||||
WHERE run.id = $4`,
|
||||
[
|
||||
receipt.modelStepRunId,
|
||||
receipt.runEventId,
|
||||
receipt.invocationId,
|
||||
receipt.runId,
|
||||
],
|
||||
);
|
||||
if (durable.rows.length !== 1) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
const proof = durable.rows[0]!;
|
||||
const payload = object(proof.payload);
|
||||
if (
|
||||
string(proof.runStatus) !== receipt.outcome ||
|
||||
integer(proof.runVersion) !== receipt.finalRunVersion ||
|
||||
integer(proof.runEventSequence) !== receipt.finalRunEventSequence ||
|
||||
integer(proof.finishedAtMs) !== receipt.finalizedAtMs ||
|
||||
proof.outputRef !== receipt.outputArtifactId ||
|
||||
string(proof.stepStatus) !== receipt.outcome ||
|
||||
string(proof.eventType) !== `copilot.diagnosis.${receipt.outcome}` ||
|
||||
string(proof.dedupeKey) !== receipt.runEventId ||
|
||||
string(proof.eventStepRunId) !== receipt.modelStepRunId ||
|
||||
integer(proof.eventCreatedAtMs) !== receipt.finalizedAtMs ||
|
||||
string(proof.completionDigest) !== receipt.completionDigest ||
|
||||
string(proof.completionOutcome) !== receipt.outcome ||
|
||||
payload.requestId !== receipt.requestId ||
|
||||
payload.planDigest !== receipt.planDigest ||
|
||||
payload.invocationId !== receipt.invocationId ||
|
||||
payload.completionDigest !== receipt.completionDigest ||
|
||||
payload.outcome !== receipt.outcome ||
|
||||
payload.outputArtifactId !== receipt.outputArtifactId
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
return receipt;
|
||||
} catch (cause) {
|
||||
if (cause instanceof CopilotFailureDiagnosisFinalizationConflictError) {
|
||||
throw cause;
|
||||
}
|
||||
throw new CopilotFailureDiagnosisFinalizationUnavailableError({ cause });
|
||||
}
|
||||
}
|
||||
|
||||
async finalize(requestId: string): Promise<Readonly<{
|
||||
status: 'created' | 'existing';
|
||||
receipt: Readonly<CopilotFailureDiagnosisFinalizationReceipt>;
|
||||
}>> {
|
||||
const existing = await this.findFinalization(requestId);
|
||||
if (existing) {
|
||||
return Object.freeze({ status: 'existing' as const, receipt: existing });
|
||||
}
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
let client: PostgresClient;
|
||||
try {
|
||||
client = await this.#pool.connect();
|
||||
} catch (cause) {
|
||||
throw new CopilotFailureDiagnosisFinalizationUnavailableError({ cause });
|
||||
}
|
||||
let began = false;
|
||||
try {
|
||||
await client.query('BEGIN ISOLATION LEVEL SERIALIZABLE');
|
||||
began = true;
|
||||
await client.query(`SELECT set_config('statement_timeout', $1, true)`, [
|
||||
'5s',
|
||||
]);
|
||||
await client.query(`SELECT set_config('lock_timeout', $1, true)`, [
|
||||
'2s',
|
||||
]);
|
||||
const result = await this.#finalizeInTransaction(client, requestId);
|
||||
await client.query('COMMIT');
|
||||
began = false;
|
||||
return result;
|
||||
} catch (cause) {
|
||||
if (began) {
|
||||
try {
|
||||
await client.query('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the original failure.
|
||||
}
|
||||
}
|
||||
const state =
|
||||
cause && typeof cause === 'object' && 'code' in cause
|
||||
? String(cause.code)
|
||||
: '';
|
||||
if ((state === '40001' || state === '40P01') && attempt < 2) {
|
||||
continue;
|
||||
}
|
||||
const recovered = await this.findFinalization(requestId);
|
||||
if (recovered) {
|
||||
return Object.freeze({
|
||||
status: 'existing' as const,
|
||||
receipt: recovered,
|
||||
});
|
||||
}
|
||||
if (
|
||||
cause instanceof CopilotFailureDiagnosisFinalizationConflictError ||
|
||||
cause instanceof CopilotFailureDiagnosisModelExecutionInProgressError ||
|
||||
cause instanceof CopilotFailureDiagnosisModelResolutionRequiredError ||
|
||||
cause instanceof CopilotFailureDiagnosisFinalizationUnavailableError
|
||||
) {
|
||||
throw cause;
|
||||
}
|
||||
throw new CopilotFailureDiagnosisFinalizationUnavailableError({ cause });
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
throw new CopilotFailureDiagnosisFinalizationUnavailableError();
|
||||
}
|
||||
|
||||
async #finalizeInTransaction(
|
||||
client: PostgresClient,
|
||||
requestId: string,
|
||||
): Promise<Readonly<{
|
||||
status: 'created' | 'existing';
|
||||
receipt: Readonly<CopilotFailureDiagnosisFinalizationReceipt>;
|
||||
}>> {
|
||||
const admission = await client.query<Record<string, unknown>>(
|
||||
`SELECT plan_json AS "planJson"
|
||||
FROM "ql3_ai"."copilot_failure_diagnosis_admissions"
|
||||
WHERE request_id = $1`,
|
||||
[requestId],
|
||||
);
|
||||
if (admission.rows.length !== 1) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
const plan = normalizeCopilotFailureDiagnosisExecutionPlan(
|
||||
object(admission.rows[0]!.planJson) as unknown as CopilotFailureDiagnosisExecutionPlan,
|
||||
);
|
||||
const durable = await client.query<Record<string, unknown>>(
|
||||
`SELECT run.status AS "runStatus", run.version AS "runVersion",
|
||||
run.event_sequence AS "runEventSequence",
|
||||
step.status AS "stepStatus",
|
||||
step.step_run_digest AS "stepRunDigest",
|
||||
completion.record_json AS "completionJson"
|
||||
FROM "ql3"."runs" AS run
|
||||
JOIN "ql3"."step_runs" AS step
|
||||
ON step.run_id = run.id AND step.id = $1
|
||||
LEFT JOIN "ql3_ai"."model_invocation_completions" AS completion
|
||||
ON completion.invocation_id = $2
|
||||
WHERE run.id = $3
|
||||
FOR UPDATE OF run, step`,
|
||||
[plan.modelStepRunId, plan.modelInvocationId, plan.runId],
|
||||
);
|
||||
if (durable.rows.length !== 1) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
const row = durable.rows[0]!;
|
||||
if (row.completionJson === null || row.completionJson === undefined) {
|
||||
throw new CopilotFailureDiagnosisModelExecutionInProgressError();
|
||||
}
|
||||
const completion = normalizeModelInvocationCompletionRecord(
|
||||
object(row.completionJson) as unknown as ModelInvocationCompletionRecord,
|
||||
);
|
||||
if (
|
||||
completion.invocationId !== plan.modelInvocationId ||
|
||||
completion.projectId !== plan.projectId ||
|
||||
completion.runId !== plan.runId ||
|
||||
completion.stepRunId !== plan.modelStepRunId ||
|
||||
completion.traceId !== plan.traceId ||
|
||||
string(row.runStatus) !== 'running' ||
|
||||
string(row.stepRunDigest) !== completion.completedStepRunDigest
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
if (completion.outcome === 'outcome_unknown') {
|
||||
throw new CopilotFailureDiagnosisModelResolutionRequiredError();
|
||||
}
|
||||
const outcome: CopilotFailureDiagnosisFinalOutcome = completion.outcome;
|
||||
if (string(row.stepStatus) !== outcome) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
const output =
|
||||
outcome === 'succeeded'
|
||||
? await read(
|
||||
client,
|
||||
copilotFailureDiagnosisOutputArtifactIdentity(
|
||||
plan.modelInvocationId,
|
||||
),
|
||||
)
|
||||
: null;
|
||||
if (
|
||||
(outcome === 'succeeded' &&
|
||||
(!output ||
|
||||
output.requestId !== plan.requestId ||
|
||||
output.planDigest !== plan.planDigest ||
|
||||
output.invocationId !== completion.invocationId ||
|
||||
output.outputBytes !== completion.outputBytes)) ||
|
||||
(outcome !== 'succeeded' && output !== null)
|
||||
) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
const runVersion = integer(row.runVersion);
|
||||
const eventSequence = integer(row.runEventSequence);
|
||||
if (runVersion !== eventSequence) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
const receipt = createCopilotFailureDiagnosisFinalizationReceipt({
|
||||
requestId: plan.requestId,
|
||||
planDigest: plan.planDigest,
|
||||
runId: plan.runId,
|
||||
modelStepRunId: plan.modelStepRunId,
|
||||
invocationId: plan.modelInvocationId,
|
||||
completionDigest: completion.completionDigest,
|
||||
outcome,
|
||||
outputArtifactId: output?.artifactId ?? null,
|
||||
finalRunVersion: runVersion + 1,
|
||||
finalRunEventSequence: eventSequence + 1,
|
||||
finalizedAtMs: completion.completedAtMs,
|
||||
});
|
||||
const failure = outcome === 'succeeded'
|
||||
? { code: null, summary: null }
|
||||
: outcome === 'timed_out'
|
||||
? {
|
||||
code: 'COPILOT_FAILURE_DIAGNOSIS_TIMED_OUT',
|
||||
summary: 'Copilot failure diagnosis timed out',
|
||||
}
|
||||
: {
|
||||
code: 'COPILOT_FAILURE_DIAGNOSIS_FAILED',
|
||||
summary: 'Copilot failure diagnosis failed',
|
||||
};
|
||||
const updated = await client.query(
|
||||
`UPDATE "ql3"."runs"
|
||||
SET status = $1, version = $2, event_sequence = $3,
|
||||
output_ref = $4, finished_at_ms = $5,
|
||||
error_code = $6, error_summary = $7
|
||||
WHERE id = $8 AND status = 'running'
|
||||
AND version = $9 AND event_sequence = $10`,
|
||||
[
|
||||
outcome,
|
||||
receipt.finalRunVersion,
|
||||
receipt.finalRunEventSequence,
|
||||
receipt.outputArtifactId,
|
||||
receipt.finalizedAtMs,
|
||||
failure.code,
|
||||
failure.summary,
|
||||
receipt.runId,
|
||||
runVersion,
|
||||
eventSequence,
|
||||
],
|
||||
);
|
||||
if ((updated.rowCount ?? updated.rows.length) !== 1) {
|
||||
throw new CopilotFailureDiagnosisFinalizationConflictError();
|
||||
}
|
||||
const payload = JSON.stringify({
|
||||
requestId: receipt.requestId,
|
||||
planDigest: receipt.planDigest,
|
||||
invocationId: receipt.invocationId,
|
||||
completionDigest: receipt.completionDigest,
|
||||
outcome: receipt.outcome,
|
||||
outputArtifactId: receipt.outputArtifactId,
|
||||
});
|
||||
await client.query(
|
||||
`INSERT INTO "ql3"."run_events" (
|
||||
id, run_id, sequence, type, dedupe_key, actor_type, actor_id,
|
||||
attempt_id, step_run_id, payload, created_at_ms
|
||||
) VALUES ($1, $2, $3, $4, $1, 'system', NULL, NULL, $5, $6::jsonb, $7)`,
|
||||
[
|
||||
receipt.runEventId,
|
||||
receipt.runId,
|
||||
receipt.finalRunEventSequence,
|
||||
`copilot.diagnosis.${receipt.outcome}`,
|
||||
plan.modelStepRunId,
|
||||
payload,
|
||||
receipt.finalizedAtMs,
|
||||
],
|
||||
);
|
||||
await client.query(
|
||||
`INSERT INTO ${FINALIZATION_TABLE} (
|
||||
request_id, plan_digest, run_id, model_step_run_id, invocation_id,
|
||||
completion_digest,
|
||||
outcome, output_artifact_id, final_run_version,
|
||||
final_run_event_sequence, run_event_id, finalized_at_ms,
|
||||
receipt_digest, receipt_json
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,
|
||||
$14::jsonb
|
||||
)`,
|
||||
[
|
||||
receipt.requestId,
|
||||
receipt.planDigest,
|
||||
receipt.runId,
|
||||
receipt.modelStepRunId,
|
||||
receipt.invocationId,
|
||||
receipt.completionDigest,
|
||||
receipt.outcome,
|
||||
receipt.outputArtifactId,
|
||||
receipt.finalRunVersion,
|
||||
receipt.finalRunEventSequence,
|
||||
receipt.runEventId,
|
||||
receipt.finalizedAtMs,
|
||||
receipt.receiptDigest,
|
||||
JSON.stringify(receipt),
|
||||
],
|
||||
);
|
||||
return Object.freeze({ status: 'created' as const, receipt });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export * from './model-execution/completion';
|
||||
export * from './model-execution/coordinator';
|
||||
export * from './model-execution/finalization';
|
||||
export * from './model-execution/outputArtifact';
|
||||
@@ -0,0 +1 @@
|
||||
export { PostgresCopilotFailureDiagnosisModelRepository } from './model-execution/postgresRepository';
|
||||
@@ -65,6 +65,8 @@ export const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID =
|
||||
'pg-9018-ai-copilot-failure-diagnosis-admissions';
|
||||
export const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID =
|
||||
'pg-9019-ai-copilot-failure-diagnosis-tool-unlocks';
|
||||
export const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID =
|
||||
'pg-9020-ai-copilot-failure-diagnosis-model-executions';
|
||||
export const POSTGRES_MODEL_INVOCATION_SCHEMA = 'ql3_ai';
|
||||
export const LOCAL_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE =
|
||||
'QingLong3AiSchemaMigrations';
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_SCHEMA,
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE,
|
||||
} from './identities';
|
||||
@@ -65,6 +66,7 @@ const POSTGRES_HISTORY_IDENTITY = Object.freeze({
|
||||
POSTGRES_PLUGIN_PACKAGE_PROMPT_PRODUCT_AUTHORIZATION_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID,
|
||||
]),
|
||||
streamId: POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID,
|
||||
dialect: 'postgresql' as const,
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { PostgresQueryable } from '@qinglong/runtime-core';
|
||||
import {
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_SCHEMA,
|
||||
} from '../identities';
|
||||
import { defineSqlMigration } from '../shared';
|
||||
@@ -11,6 +12,8 @@ const ADMISSION_TABLE = 'copilot_failure_diagnosis_admissions';
|
||||
const SOURCE_SNAPSHOT_FUNCTION =
|
||||
'copilot_failure_diagnosis_admission_source_snapshot';
|
||||
const TOOL_UNLOCK_TABLE = 'copilot_failure_diagnosis_tool_unlocks';
|
||||
const MODEL_OUTPUT_TABLE = 'copilot_failure_diagnosis_model_outputs';
|
||||
const FINALIZATION_TABLE = 'copilot_failure_diagnosis_finalizations';
|
||||
|
||||
const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}" (
|
||||
@@ -319,7 +322,166 @@ const postgresCopilotFailureDiagnosisToolUnlockMigration =
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_OUTPUT_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${MODEL_OUTPUT_TABLE}" (
|
||||
artifact_id varchar(128) PRIMARY KEY,
|
||||
request_id varchar(128) NOT NULL UNIQUE,
|
||||
plan_digest char(64) NOT NULL UNIQUE,
|
||||
tool_completion_digest char(64) NOT NULL,
|
||||
project_id varchar(128) NOT NULL,
|
||||
run_id varchar(36) NOT NULL,
|
||||
step_run_id varchar(128) NOT NULL UNIQUE,
|
||||
invocation_id varchar(128) NOT NULL UNIQUE,
|
||||
provider varchar(128) NOT NULL,
|
||||
model varchar(256) NOT NULL,
|
||||
egress_evidence_digest char(64) NOT NULL,
|
||||
content_digest char(64) NOT NULL,
|
||||
output_bytes integer NOT NULL,
|
||||
key_id varchar(128) NOT NULL,
|
||||
algorithm varchar(32) NOT NULL,
|
||||
sealed_at_ms bigint NOT NULL,
|
||||
artifact_digest char(64) NOT NULL UNIQUE,
|
||||
artifact_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_model_output_admission_fk
|
||||
FOREIGN KEY (request_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}"
|
||||
(request_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_model_output_unlock_fk
|
||||
FOREIGN KEY (request_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${TOOL_UNLOCK_TABLE}"
|
||||
(request_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_model_output_completion_fk
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions"
|
||||
(invocation_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_model_output_step_fk
|
||||
FOREIGN KEY (run_id, step_run_id)
|
||||
REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_model_output_identity_check CHECK (
|
||||
artifact_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
|
||||
request_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$' AND
|
||||
project_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$' AND
|
||||
step_run_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
invocation_id ~ '^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$' AND
|
||||
output_bytes BETWEEN 0 AND 1048576 AND sealed_at_ms >= 0
|
||||
),
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_model_output_digest_check CHECK (
|
||||
plan_digest ~ '^[0-9a-f]{64}$' AND
|
||||
tool_completion_digest ~ '^[0-9a-f]{64}$' AND
|
||||
egress_evidence_digest ~ '^[0-9a-f]{64}$' AND
|
||||
content_digest ~ '^[0-9a-f]{64}$' AND
|
||||
artifact_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_model_output_json_check CHECK (
|
||||
jsonb_typeof(artifact_json) = 'object' AND
|
||||
octet_length(artifact_json::text) BETWEEN 2 AND 1572864 AND
|
||||
artifact_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/copilot-failure-diagnosis-output-artifact@v1',
|
||||
'artifactId', artifact_id, 'requestId', request_id,
|
||||
'planDigest', plan_digest,
|
||||
'toolCompletionDigest', tool_completion_digest,
|
||||
'projectId', project_id, 'runId', run_id,
|
||||
'stepRunId', step_run_id, 'invocationId', invocation_id,
|
||||
'provider', provider, 'model', model,
|
||||
'egressEvidenceDigest', egress_evidence_digest,
|
||||
'contentDigest', content_digest, 'outputBytes', output_bytes,
|
||||
'keyId', key_id, 'algorithm', algorithm,
|
||||
'sealedAtMs', sealed_at_ms, 'artifactDigest', artifact_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const POSTGRES_COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_TABLE_SQL = `
|
||||
CREATE TABLE "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${FINALIZATION_TABLE}" (
|
||||
request_id varchar(128) PRIMARY KEY,
|
||||
plan_digest char(64) NOT NULL UNIQUE,
|
||||
run_id varchar(36) NOT NULL UNIQUE,
|
||||
model_step_run_id varchar(128) NOT NULL UNIQUE,
|
||||
invocation_id varchar(128) NOT NULL UNIQUE,
|
||||
completion_digest char(64) NOT NULL UNIQUE,
|
||||
outcome varchar(32) NOT NULL,
|
||||
output_artifact_id varchar(128),
|
||||
final_run_version integer NOT NULL,
|
||||
final_run_event_sequence integer NOT NULL,
|
||||
run_event_id varchar(36) NOT NULL UNIQUE,
|
||||
finalized_at_ms bigint NOT NULL,
|
||||
receipt_digest char(64) NOT NULL UNIQUE,
|
||||
receipt_json jsonb NOT NULL,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_finalization_admission_fk
|
||||
FOREIGN KEY (request_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${ADMISSION_TABLE}"
|
||||
(request_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_finalization_completion_fk
|
||||
FOREIGN KEY (invocation_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."model_invocation_completions"
|
||||
(invocation_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_finalization_output_fk
|
||||
FOREIGN KEY (output_artifact_id)
|
||||
REFERENCES "${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${MODEL_OUTPUT_TABLE}"
|
||||
(artifact_id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_finalization_run_fk
|
||||
FOREIGN KEY (run_id) REFERENCES "ql3"."runs" (id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_finalization_step_fk
|
||||
FOREIGN KEY (run_id, model_step_run_id)
|
||||
REFERENCES "ql3"."step_runs" (run_id, id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_finalization_event_fk
|
||||
FOREIGN KEY (run_event_id)
|
||||
REFERENCES "ql3"."run_events" (id) ON DELETE RESTRICT,
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_finalization_state_check CHECK (
|
||||
outcome IN ('succeeded', 'failed', 'timed_out', 'cancelled') AND
|
||||
((outcome = 'succeeded' AND output_artifact_id IS NOT NULL) OR
|
||||
(outcome <> 'succeeded' AND output_artifact_id IS NULL)) AND
|
||||
final_run_version >= 1 AND
|
||||
final_run_event_sequence = final_run_version AND
|
||||
finalized_at_ms >= 0
|
||||
),
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_finalization_digest_check CHECK (
|
||||
plan_digest ~ '^[0-9a-f]{64}$' AND
|
||||
completion_digest ~ '^[0-9a-f]{64}$' AND
|
||||
receipt_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_ai_copilot_diagnosis_finalization_json_check CHECK (
|
||||
jsonb_typeof(receipt_json) = 'object' AND
|
||||
octet_length(receipt_json::text) BETWEEN 2 AND 16384 AND
|
||||
receipt_json @> jsonb_build_object(
|
||||
'schema', 'qinglong/copilot-failure-diagnosis-finalization-receipt@v1',
|
||||
'requestId', request_id, 'planDigest', plan_digest,
|
||||
'runId', run_id, 'modelStepRunId', model_step_run_id,
|
||||
'invocationId', invocation_id,
|
||||
'completionDigest', completion_digest, 'outcome', outcome,
|
||||
'finalRunVersion', final_run_version,
|
||||
'finalRunEventSequence', final_run_event_sequence,
|
||||
'runEventId', run_event_id, 'finalizedAtMs', finalized_at_ms,
|
||||
'receiptDigest', receipt_digest
|
||||
)
|
||||
)
|
||||
)`;
|
||||
|
||||
const postgresCopilotFailureDiagnosisModelExecutionMigration =
|
||||
defineSqlMigration<PostgresQueryable>(
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID,
|
||||
[
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_OUTPUT_TABLE_SQL,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_FINALIZATION_TABLE_SQL,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${MODEL_OUTPUT_TABLE}"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${MODEL_OUTPUT_TABLE}"
|
||||
TO ql3_runtime`,
|
||||
`REVOKE ALL ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${FINALIZATION_TABLE}"
|
||||
FROM PUBLIC`,
|
||||
`GRANT SELECT, INSERT ON TABLE
|
||||
"${POSTGRES_MODEL_INVOCATION_SCHEMA}"."${FINALIZATION_TABLE}"
|
||||
TO ql3_runtime`,
|
||||
],
|
||||
(context, statement) => context.query(statement).then(() => undefined),
|
||||
);
|
||||
|
||||
export const postgresCopilotMigrations = Object.freeze([
|
||||
postgresCopilotFailureDiagnosisAdmissionMigration,
|
||||
postgresCopilotFailureDiagnosisToolUnlockMigration,
|
||||
postgresCopilotFailureDiagnosisModelExecutionMigration,
|
||||
]);
|
||||
|
||||
@@ -33,16 +33,9 @@ import {
|
||||
type ModelInvocationPriceQuote,
|
||||
} from '../pricing/pricing';
|
||||
import {
|
||||
isPluginPackagePromptOutputCompletionRepository,
|
||||
type PluginPackagePromptOutputCompletionRepository,
|
||||
} from '../prompt-output/pluginPackagePromptOutputCompletion';
|
||||
import {
|
||||
PluginPackagePromptOutputArtifactConflictError,
|
||||
normalizePluginPackagePromptOutputArtifact,
|
||||
pluginPackagePromptOutputArtifactReference,
|
||||
type PluginPackagePromptOutputArtifact,
|
||||
type PluginPackagePromptOutputArtifactReference,
|
||||
} from '../prompt-output/pluginPackagePromptOutputArtifact';
|
||||
normalizeModelInvocationAtomicSuccess,
|
||||
type ModelInvocationAtomicSuccess,
|
||||
} from './modelInvocationAtomicSuccess';
|
||||
|
||||
const MAX_COORDINATOR_ATTEMPTS = 3;
|
||||
|
||||
@@ -242,32 +235,21 @@ export class DurableModelInvocationCoordinator
|
||||
return this.#admit(record, admission, quote);
|
||||
}
|
||||
|
||||
async recordWithPromptOutputArtifact(
|
||||
async recordWithAtomicSuccess<TReference>(
|
||||
record: Readonly<ModelInvocationAuditRecord>,
|
||||
artifactValue: PluginPackagePromptOutputArtifact,
|
||||
extensionValue: ModelInvocationAtomicSuccess<TReference>,
|
||||
): Promise<
|
||||
Readonly<{
|
||||
status: 'created' | 'existing';
|
||||
reference: Readonly<PluginPackagePromptOutputArtifactReference>;
|
||||
reference: Readonly<TReference>;
|
||||
}>
|
||||
> {
|
||||
const artifact = normalizePluginPackagePromptOutputArtifact(artifactValue);
|
||||
if (
|
||||
record.phase !== 'completed' ||
|
||||
record.requestId !== artifact.invocationId ||
|
||||
record.projectId !== artifact.projectId ||
|
||||
record.runId !== artifact.runId ||
|
||||
record.stepRunId !== artifact.stepRunId ||
|
||||
record.provider !== artifact.provider ||
|
||||
record.model !== artifact.model ||
|
||||
record.outputBytes !== artifact.outputBytes ||
|
||||
!isPluginPackagePromptOutputCompletionRepository(this.repository)
|
||||
) {
|
||||
throw new PluginPackagePromptOutputArtifactConflictError();
|
||||
}
|
||||
const result = await this.#complete(record, artifact);
|
||||
const extension = normalizeModelInvocationAtomicSuccess(extensionValue);
|
||||
if (record.phase !== 'completed') throw extension.conflict();
|
||||
extension.assertAudit(record);
|
||||
const result = await this.#complete(record, extension);
|
||||
if (!result.reference) {
|
||||
throw new PluginPackagePromptOutputArtifactConflictError();
|
||||
throw extension.conflict();
|
||||
}
|
||||
return Object.freeze({
|
||||
status: result.status,
|
||||
@@ -383,22 +365,18 @@ export class DurableModelInvocationCoordinator
|
||||
throw new ModelInvocationConflictError();
|
||||
}
|
||||
|
||||
async #complete(
|
||||
async #complete<TReference = never>(
|
||||
audit: Readonly<ModelInvocationAuditRecord>,
|
||||
artifactValue?: Readonly<PluginPackagePromptOutputArtifact>,
|
||||
extensionValue?: ModelInvocationAtomicSuccess<TReference>,
|
||||
): Promise<
|
||||
Readonly<
|
||||
ModelInvocationAuditDisposition & {
|
||||
reference?: Readonly<PluginPackagePromptOutputArtifactReference>;
|
||||
reference?: Readonly<TReference>;
|
||||
}
|
||||
>
|
||||
> {
|
||||
const artifact = artifactValue
|
||||
? normalizePluginPackagePromptOutputArtifact(artifactValue)
|
||||
: undefined;
|
||||
const artifactRepository = artifact
|
||||
? (this.repository as ModelInvocationRepository &
|
||||
PluginPackagePromptOutputCompletionRepository)
|
||||
const extension = extensionValue
|
||||
? normalizeModelInvocationAtomicSuccess(extensionValue)
|
||||
: undefined;
|
||||
const startValue = await this.repository.findStart(audit.requestId);
|
||||
if (!startValue) throw new ModelInvocationConflictError();
|
||||
@@ -406,16 +384,14 @@ export class DurableModelInvocationCoordinator
|
||||
const existing = await this.repository.findCompletion(audit.requestId);
|
||||
if (existing) {
|
||||
assertCompletionMatchesAudit(existing, start, audit);
|
||||
if (artifact && artifactRepository) {
|
||||
const stored = await artifactRepository.findPromptOutputArtifact(
|
||||
artifact.artifactId,
|
||||
);
|
||||
if (!stored || JSON.stringify(stored) !== JSON.stringify(artifact)) {
|
||||
throw new PluginPackagePromptOutputArtifactConflictError();
|
||||
if (extension) {
|
||||
const stored = await extension.find(this.repository);
|
||||
if (!stored || !extension.matches(stored)) {
|
||||
throw extension.conflict();
|
||||
}
|
||||
return Object.freeze({
|
||||
status: 'existing' as const,
|
||||
reference: pluginPackagePromptOutputArtifactReference(stored),
|
||||
reference: stored,
|
||||
});
|
||||
}
|
||||
return Object.freeze({ status: 'existing' as const });
|
||||
@@ -430,7 +406,7 @@ export class DurableModelInvocationCoordinator
|
||||
) {
|
||||
throw new ModelInvocationConflictError();
|
||||
}
|
||||
const transition = completionTransition(audit, artifact?.artifactId);
|
||||
const transition = completionTransition(audit, extension?.outputRef);
|
||||
const mutationIdentity = createModelInvocationMutationIdentity(
|
||||
audit.requestId,
|
||||
'completion',
|
||||
@@ -464,7 +440,7 @@ export class DurableModelInvocationCoordinator
|
||||
actor: { type: 'executor', id: 'model-gateway' },
|
||||
},
|
||||
),
|
||||
artifact?.artifactId,
|
||||
extension?.outputRef,
|
||||
);
|
||||
try {
|
||||
const pricingAware = isPricingAwareModelInvocationRepository(
|
||||
@@ -479,42 +455,32 @@ export class DurableModelInvocationCoordinator
|
||||
const reservation = quotaAware
|
||||
? await this.repository.findQuotaReservation(audit.requestId)
|
||||
: null;
|
||||
const result =
|
||||
artifactRepository && artifact
|
||||
? await artifactRepository.completeWithPromptOutputArtifact(
|
||||
command,
|
||||
artifact,
|
||||
)
|
||||
: pricingAware && quote
|
||||
if (extension) {
|
||||
const result = await extension.commit(this.repository, command);
|
||||
return Object.freeze({
|
||||
status: result.status,
|
||||
reference: result.reference,
|
||||
});
|
||||
}
|
||||
const result = pricingAware && quote
|
||||
? await this.repository.completeWithPricing(command)
|
||||
: quotaAware && reservation
|
||||
? await this.repository.completeWithQuota(command)
|
||||
: await this.repository.complete(command);
|
||||
return Object.freeze({
|
||||
status: result.status,
|
||||
...(artifact && artifactRepository
|
||||
? {
|
||||
reference: pluginPackagePromptOutputArtifactReference(artifact),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
} catch (error) {
|
||||
const stored = await this.#completionAfterFailure(
|
||||
start,
|
||||
audit,
|
||||
error,
|
||||
artifact,
|
||||
artifactRepository,
|
||||
extension,
|
||||
);
|
||||
if (stored) {
|
||||
return Object.freeze({
|
||||
status: 'existing' as const,
|
||||
...(artifact
|
||||
? {
|
||||
reference:
|
||||
pluginPackagePromptOutputArtifactReference(artifact),
|
||||
}
|
||||
: {}),
|
||||
...(extension ? { reference: stored.reference } : {}),
|
||||
});
|
||||
}
|
||||
if (
|
||||
@@ -569,31 +535,29 @@ export class DurableModelInvocationCoordinator
|
||||
}
|
||||
}
|
||||
|
||||
async #completionAfterFailure(
|
||||
async #completionAfterFailure<TReference = never>(
|
||||
start: Readonly<ModelInvocationStartRecord>,
|
||||
audit: Readonly<ModelInvocationAuditRecord>,
|
||||
original: unknown,
|
||||
artifact?: Readonly<PluginPackagePromptOutputArtifact>,
|
||||
artifactRepository?: PluginPackagePromptOutputCompletionRepository,
|
||||
): Promise<Readonly<ModelInvocationCompletionRecord> | null> {
|
||||
extension?: ModelInvocationAtomicSuccess<TReference>,
|
||||
): Promise<
|
||||
Readonly<{
|
||||
completion: Readonly<ModelInvocationCompletionRecord>;
|
||||
reference?: Readonly<TReference>;
|
||||
}> | null
|
||||
> {
|
||||
try {
|
||||
const stored = await this.repository.findCompletion(audit.requestId);
|
||||
if (!stored) return null;
|
||||
const completion = assertCompletionMatchesAudit(stored, start, audit);
|
||||
if (artifact) {
|
||||
if (!artifactRepository) throw original;
|
||||
const storedArtifact =
|
||||
await artifactRepository.findPromptOutputArtifact(
|
||||
artifact.artifactId,
|
||||
);
|
||||
if (
|
||||
!storedArtifact ||
|
||||
JSON.stringify(storedArtifact) !== JSON.stringify(artifact)
|
||||
) {
|
||||
if (extension) {
|
||||
const reference = await extension.find(this.repository);
|
||||
if (!reference || !extension.matches(reference)) {
|
||||
throw original;
|
||||
}
|
||||
return Object.freeze({ completion, reference });
|
||||
}
|
||||
return completion;
|
||||
return Object.freeze({ completion });
|
||||
} catch {
|
||||
throw original;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import type {
|
||||
ModelInvocationAuditRecord,
|
||||
ModelInvocationAuditDisposition,
|
||||
} from '../model-gateway/model';
|
||||
import type {
|
||||
ModelInvocationCompletionCommand,
|
||||
ModelInvocationRepository,
|
||||
} from './modelInvocation';
|
||||
|
||||
export interface ModelInvocationAtomicSuccessCommit<TReference> {
|
||||
readonly status: ModelInvocationAuditDisposition['status'];
|
||||
readonly reference: Readonly<TReference>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Domain-owned extension for one encrypted successful Model output.
|
||||
*
|
||||
* The generic Model coordinator owns the StepRun/usage/pricing protocol while
|
||||
* this extension owns the output identity, exact replay check and dialect
|
||||
* transaction that persists the encrypted output beside that protocol.
|
||||
*/
|
||||
export interface ModelInvocationAtomicSuccess<TReference> {
|
||||
readonly outputRef: string;
|
||||
assertAudit(record: Readonly<ModelInvocationAuditRecord>): void;
|
||||
find(
|
||||
repository: ModelInvocationRepository,
|
||||
): Promise<Readonly<TReference> | null>;
|
||||
matches(reference: Readonly<TReference>): boolean;
|
||||
commit(
|
||||
repository: ModelInvocationRepository,
|
||||
command: Readonly<ModelInvocationCompletionCommand>,
|
||||
): Promise<Readonly<ModelInvocationAtomicSuccessCommit<TReference>>>;
|
||||
conflict(): Error;
|
||||
}
|
||||
|
||||
export function normalizeModelInvocationAtomicSuccess<TReference>(
|
||||
value: ModelInvocationAtomicSuccess<TReference>,
|
||||
): ModelInvocationAtomicSuccess<TReference> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
typeof value.outputRef !== 'string' ||
|
||||
value.outputRef.length < 1 ||
|
||||
value.outputRef.length > 512 ||
|
||||
typeof value.assertAudit !== 'function' ||
|
||||
typeof value.find !== 'function' ||
|
||||
typeof value.matches !== 'function' ||
|
||||
typeof value.commit !== 'function' ||
|
||||
typeof value.conflict !== 'function'
|
||||
) {
|
||||
throw new TypeError('Model invocation atomic success is invalid');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
+60
-19
@@ -1,4 +1,4 @@
|
||||
import type { PostgresPool } from '@qinglong/runtime-core';
|
||||
import type { PostgresClient, PostgresPool } from '@qinglong/runtime-core';
|
||||
|
||||
import { createModelInvocationPriceSettlement } from '../../pricing/pricing';
|
||||
import { type PluginPackagePromptOutputArtifact } from '../../prompt-output/pluginPackagePromptOutputArtifact';
|
||||
@@ -315,17 +315,36 @@ export async function completeWithPricingOperation(
|
||||
});
|
||||
}
|
||||
|
||||
export async function completeWithPromptOutputArtifactOperation(
|
||||
export interface PostgresModelInvocationAtomicOutputBinding<
|
||||
TArtifact,
|
||||
TReference,
|
||||
> {
|
||||
readonly artifact: Readonly<TArtifact>;
|
||||
readonly reference: Readonly<TReference>;
|
||||
read(client: PostgresClient): Promise<Readonly<TArtifact> | null>;
|
||||
put(client: PostgresClient): Promise<Readonly<TArtifact>>;
|
||||
matches(stored: Readonly<TArtifact>): boolean;
|
||||
}
|
||||
|
||||
export interface CommitPostgresModelInvocationAtomicOutputResult<
|
||||
TArtifact,
|
||||
TReference,
|
||||
> extends CommitModelInvocationResult<ModelInvocationCompletionRecord> {
|
||||
readonly artifact: Readonly<TArtifact>;
|
||||
readonly reference: Readonly<TReference>;
|
||||
}
|
||||
|
||||
export async function completeWithAtomicOutputOperation<TArtifact, TReference>(
|
||||
pool: PostgresPool,
|
||||
commandValue: ModelInvocationCompletionCommand,
|
||||
artifactValue: PluginPackagePromptOutputArtifact,
|
||||
): Promise<Readonly<CommitPluginPackagePromptOutputResult>> {
|
||||
binding: PostgresModelInvocationAtomicOutputBinding<TArtifact, TReference>,
|
||||
): Promise<
|
||||
Readonly<
|
||||
CommitPostgresModelInvocationAtomicOutputResult<TArtifact, TReference>
|
||||
>
|
||||
> {
|
||||
const command = normalizeModelInvocationCompletionCommand(commandValue);
|
||||
const completion = command.completion;
|
||||
const binding = assertPluginPackagePromptOutputCompletionBinding(
|
||||
command,
|
||||
artifactValue,
|
||||
);
|
||||
const expectedUsage = createModelInvocationUsageLedgerRecord(
|
||||
command.start,
|
||||
completion,
|
||||
@@ -377,10 +396,7 @@ export async function completeWithPromptOutputArtifactOperation(
|
||||
if (existing[0]) {
|
||||
const [storedArtifact, usage, priceSettlements, quotaSettlements] =
|
||||
await Promise.all([
|
||||
readPostgresPluginPackagePromptOutputArtifactInTransaction(
|
||||
client,
|
||||
binding.artifact.artifactId,
|
||||
),
|
||||
binding.read(client),
|
||||
usageRows(client, 'usage.invocation_id = $1', [
|
||||
completion.invocationId,
|
||||
]),
|
||||
@@ -395,7 +411,7 @@ export async function completeWithPromptOutputArtifactOperation(
|
||||
if (
|
||||
JSON.stringify(stored) !== JSON.stringify(completion) ||
|
||||
!storedArtifact ||
|
||||
JSON.stringify(storedArtifact) !== JSON.stringify(binding.artifact) ||
|
||||
!binding.matches(storedArtifact) ||
|
||||
usage.length !== (expectedUsage ? 1 : 0) ||
|
||||
(expectedUsage &&
|
||||
JSON.stringify(parseUsage(usage[0]!)) !==
|
||||
@@ -434,12 +450,6 @@ export async function completeWithPromptOutputArtifactOperation(
|
||||
throw new ModelInvocationConflictError();
|
||||
}
|
||||
await assertCurrent(client, command.stepRunMutation, completion.projectId);
|
||||
const artifact = (
|
||||
await putPostgresPluginPackagePromptOutputArtifactInTransaction(
|
||||
client,
|
||||
binding.artifact,
|
||||
)
|
||||
).artifact;
|
||||
await applyMutation(client, command.stepRunMutation);
|
||||
await insertCompletion(client, completion);
|
||||
if (expectedUsage) await insertUsage(client, expectedUsage);
|
||||
@@ -449,6 +459,7 @@ export async function completeWithPromptOutputArtifactOperation(
|
||||
if (expectedQuotaSettlement) {
|
||||
await insertQuotaSettlement(client, expectedQuotaSettlement);
|
||||
}
|
||||
const artifact = await binding.put(client);
|
||||
return Object.freeze({
|
||||
status: 'created' as const,
|
||||
record: completion,
|
||||
@@ -457,3 +468,33 @@ export async function completeWithPromptOutputArtifactOperation(
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function completeWithPromptOutputArtifactOperation(
|
||||
pool: PostgresPool,
|
||||
commandValue: ModelInvocationCompletionCommand,
|
||||
artifactValue: PluginPackagePromptOutputArtifact,
|
||||
): Promise<Readonly<CommitPluginPackagePromptOutputResult>> {
|
||||
const command = normalizeModelInvocationCompletionCommand(commandValue);
|
||||
const binding = assertPluginPackagePromptOutputCompletionBinding(
|
||||
command,
|
||||
artifactValue,
|
||||
);
|
||||
return completeWithAtomicOutputOperation(pool, command, {
|
||||
artifact: binding.artifact,
|
||||
reference: binding.reference,
|
||||
read: (client) =>
|
||||
readPostgresPluginPackagePromptOutputArtifactInTransaction(
|
||||
client,
|
||||
binding.artifact.artifactId,
|
||||
),
|
||||
put: async (client) =>
|
||||
(
|
||||
await putPostgresPluginPackagePromptOutputArtifactInTransaction(
|
||||
client,
|
||||
binding.artifact,
|
||||
)
|
||||
).artifact,
|
||||
matches: (stored) =>
|
||||
JSON.stringify(stored) === JSON.stringify(binding.artifact),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import type {
|
||||
} from '../model-gateway/model';
|
||||
import type { ModelInvocationSuccessfulCompletionSink } from '../model-gateway/gateway';
|
||||
import type { DurableModelInvocationCoordinator } from '../model-invocation/durableModelInvocationCoordinator';
|
||||
import type { ModelInvocationAtomicSuccess } from '../model-invocation/modelInvocationAtomicSuccess';
|
||||
import {
|
||||
normalizePluginPackagePromptExecutionPlan,
|
||||
type PluginPackagePromptExecutionPlan,
|
||||
@@ -113,6 +114,61 @@ export function assertPluginPackagePromptOutputCompletionBinding(
|
||||
});
|
||||
}
|
||||
|
||||
function pluginPackagePromptOutputAtomicSuccess(
|
||||
artifactValue: PluginPackagePromptOutputArtifact,
|
||||
): ModelInvocationAtomicSuccess<PluginPackagePromptOutputArtifactReference> {
|
||||
const artifact = normalizePluginPackagePromptOutputArtifact(artifactValue);
|
||||
const reference = pluginPackagePromptOutputArtifactReference(artifact);
|
||||
const conflict = (): Error =>
|
||||
new PluginPackagePromptOutputArtifactConflictError();
|
||||
const extension: ModelInvocationAtomicSuccess<PluginPackagePromptOutputArtifactReference> = {
|
||||
outputRef: artifact.artifactId,
|
||||
assertAudit(audit: Readonly<ModelInvocationAuditRecord>): void {
|
||||
if (
|
||||
audit.phase !== 'completed' ||
|
||||
audit.requestId !== artifact.invocationId ||
|
||||
audit.projectId !== artifact.projectId ||
|
||||
audit.runId !== artifact.runId ||
|
||||
audit.stepRunId !== artifact.stepRunId ||
|
||||
audit.provider !== artifact.provider ||
|
||||
audit.model !== artifact.model ||
|
||||
audit.outputBytes !== artifact.outputBytes
|
||||
) {
|
||||
throw conflict();
|
||||
}
|
||||
},
|
||||
async find(repository) {
|
||||
if (!isPluginPackagePromptOutputCompletionRepository(repository)) {
|
||||
throw conflict();
|
||||
}
|
||||
const stored = await repository.findPromptOutputArtifact(
|
||||
artifact.artifactId,
|
||||
);
|
||||
if (!stored) return null;
|
||||
if (JSON.stringify(stored) !== JSON.stringify(artifact)) throw conflict();
|
||||
return pluginPackagePromptOutputArtifactReference(stored);
|
||||
},
|
||||
matches(stored): boolean {
|
||||
return JSON.stringify(stored) === JSON.stringify(reference);
|
||||
},
|
||||
async commit(repository, command) {
|
||||
if (!isPluginPackagePromptOutputCompletionRepository(repository)) {
|
||||
throw conflict();
|
||||
}
|
||||
const result = await repository.completeWithPromptOutputArtifact(
|
||||
command,
|
||||
artifact,
|
||||
);
|
||||
return Object.freeze({
|
||||
status: result.status,
|
||||
reference: result.reference,
|
||||
});
|
||||
},
|
||||
conflict,
|
||||
};
|
||||
return Object.freeze(extension);
|
||||
}
|
||||
|
||||
interface ActiveCompletion {
|
||||
readonly lease: Readonly<PluginPackagePromptOutputCompletionLease>;
|
||||
readonly plan: Readonly<PluginPackagePromptExecutionPlan>;
|
||||
@@ -149,7 +205,7 @@ export class PluginPackagePromptOutputCompletionCoordinator
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
!options.coordinator ||
|
||||
typeof options.coordinator.recordWithPromptOutputArtifact !==
|
||||
typeof options.coordinator.recordWithAtomicSuccess !==
|
||||
'function' ||
|
||||
!options.keys ||
|
||||
typeof options.keys.active !== 'function' ||
|
||||
@@ -252,7 +308,10 @@ export class PluginPackagePromptOutputCompletionCoordinator
|
||||
this.#nonceFactory,
|
||||
);
|
||||
const disposition =
|
||||
await this.#coordinator.recordWithPromptOutputArtifact(audit, artifact);
|
||||
await this.#coordinator.recordWithAtomicSuccess(
|
||||
audit,
|
||||
pluginPackagePromptOutputAtomicSuccess(artifact),
|
||||
);
|
||||
active.reference = disposition.reference;
|
||||
return Object.freeze({
|
||||
handled: true as const,
|
||||
|
||||
@@ -40,6 +40,13 @@ const {
|
||||
normalizeCopilotFailureDiagnosisToolUnlockCommand,
|
||||
restoreCopilotFailureDiagnosisTrustedToolAuthority,
|
||||
} = require('../dist/copilot/failure-diagnosis/toolExecution.js');
|
||||
const {
|
||||
CopilotFailureDiagnosisOutputArtifactUnavailableError,
|
||||
copilotFailureDiagnosisOutputReference,
|
||||
createCopilotFailureDiagnosisFinalizationReceipt,
|
||||
createCopilotFailureDiagnosisOutputArtifact,
|
||||
openCopilotFailureDiagnosisOutputArtifact,
|
||||
} = require('../dist/copilot/failure-diagnosis/modelExecution.js');
|
||||
|
||||
const DIGEST_A = 'a'.repeat(64);
|
||||
const DIGEST_B = 'b'.repeat(64);
|
||||
@@ -425,3 +432,373 @@ test('fails closed on widened or digest-drifted durable plans', async () => {
|
||||
InvalidCopilotFailureDiagnosisExecutionPlanError,
|
||||
);
|
||||
});
|
||||
|
||||
test('encrypts one Copilot diagnosis output and exposes only a content-free reference', async () => {
|
||||
const current = await plan();
|
||||
const prompt = require('../dist/copilot/failure-diagnosis/prompt.js')
|
||||
.buildFailureDiagnosisPromptPlan({
|
||||
provider: current.model.provider,
|
||||
model: current.model.model,
|
||||
modelBoundary: current.model.modelBoundary,
|
||||
profile: 'cluster-control',
|
||||
responseLanguage: current.model.responseLanguage,
|
||||
projection: {
|
||||
content: 'failure: connection refused',
|
||||
sourceBytes: 27,
|
||||
modelTextBytes: 27,
|
||||
redaction: {
|
||||
contract: 'recognized_credentials_v1',
|
||||
residualSensitivity: 'potentially_sensitive',
|
||||
replacements: 0,
|
||||
categories: [],
|
||||
},
|
||||
normalization: {
|
||||
invalidUtf8: false,
|
||||
unsafeCodePointsReplaced: 0,
|
||||
},
|
||||
trust: {
|
||||
classification: 'untrusted_execution_output',
|
||||
instructionPolicy: 'data_only_never_execute',
|
||||
actionAuthority: 'none',
|
||||
suspectedPromptInjection: false,
|
||||
signals: [],
|
||||
},
|
||||
},
|
||||
maxOutputTokens: current.model.maxOutputTokens,
|
||||
egressPolicy: current.model.egressPolicy,
|
||||
});
|
||||
const result = {
|
||||
provider: current.model.provider,
|
||||
model: current.model.model,
|
||||
text: 'Likely a refused upstream connection.',
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 120, outputTokens: 8, totalTokens: 128 },
|
||||
};
|
||||
const artifact = createCopilotFailureDiagnosisOutputArtifact(
|
||||
{
|
||||
requestId: current.requestId,
|
||||
planDigest: current.planDigest,
|
||||
toolCompletionDigest: DIGEST_A,
|
||||
projectId: current.projectId,
|
||||
runId: current.runId,
|
||||
stepRunId: current.modelStepRunId,
|
||||
invocationId: current.modelInvocationId,
|
||||
result,
|
||||
egressEvidence: prompt.egressEvidence,
|
||||
keyId: 'copilot-output-key-1',
|
||||
key: Buffer.alloc(32, 0x61),
|
||||
sealedAtMs: 4_000,
|
||||
},
|
||||
() => Buffer.alloc(12, 0x62),
|
||||
);
|
||||
assert.equal(JSON.stringify(artifact).includes(result.text), false);
|
||||
assert.deepEqual(
|
||||
openCopilotFailureDiagnosisOutputArtifact(
|
||||
artifact,
|
||||
Buffer.alloc(32, 0x61),
|
||||
),
|
||||
result,
|
||||
);
|
||||
const reference = copilotFailureDiagnosisOutputReference(artifact);
|
||||
assert.equal(reference.artifactId, artifact.artifactId);
|
||||
assert.equal(JSON.stringify(reference).includes('ciphertext'), false);
|
||||
assert.equal(JSON.stringify(reference).includes(result.text), false);
|
||||
assert.throws(
|
||||
() =>
|
||||
openCopilotFailureDiagnosisOutputArtifact(
|
||||
{ ...artifact, ciphertext: `${artifact.ciphertext.slice(0, -1)}A` },
|
||||
Buffer.alloc(32, 0x61),
|
||||
),
|
||||
TypeError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
openCopilotFailureDiagnosisOutputArtifact(
|
||||
artifact,
|
||||
Buffer.alloc(32, 0x63),
|
||||
),
|
||||
CopilotFailureDiagnosisOutputArtifactUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('binds a content-free diagnosis Run finalization receipt', async () => {
|
||||
const current = await plan();
|
||||
const receipt = createCopilotFailureDiagnosisFinalizationReceipt({
|
||||
requestId: current.requestId,
|
||||
planDigest: current.planDigest,
|
||||
runId: current.runId,
|
||||
modelStepRunId: current.modelStepRunId,
|
||||
invocationId: current.modelInvocationId,
|
||||
completionDigest: DIGEST_B,
|
||||
outcome: 'succeeded',
|
||||
outputArtifactId: 'cdo:diagnosis-output',
|
||||
finalRunVersion: 9,
|
||||
finalRunEventSequence: 9,
|
||||
finalizedAtMs: 4_100,
|
||||
});
|
||||
assert.equal(receipt.runEventId.length, 36);
|
||||
assert.equal(receipt.receiptDigest.length, 64);
|
||||
assert.equal(JSON.stringify(receipt).includes('model output'), false);
|
||||
assert.throws(
|
||||
() =>
|
||||
createCopilotFailureDiagnosisFinalizationReceipt({
|
||||
...receipt,
|
||||
outcome: 'failed',
|
||||
}),
|
||||
/finalization conflicts/,
|
||||
);
|
||||
});
|
||||
|
||||
test('publishes Model execution through explicit AI subpaths only', () => {
|
||||
const root = require('../dist');
|
||||
const execution = require('@qinglong/ai/failure-diagnosis-model-execution');
|
||||
const storage = require('@qinglong/ai/postgres-failure-diagnosis-model-execution-storage');
|
||||
assert.equal(root.executeCopilotFailureDiagnosisModel, undefined);
|
||||
assert.equal(
|
||||
typeof execution.executeCopilotFailureDiagnosisModel,
|
||||
'function',
|
||||
);
|
||||
assert.equal(
|
||||
typeof storage.PostgresCopilotFailureDiagnosisModelRepository,
|
||||
'function',
|
||||
);
|
||||
});
|
||||
|
||||
test('executes the unlocked Model once, commits ciphertext, and terminalizes replay', async () => {
|
||||
const current = await plan();
|
||||
const admission = createCopilotFailureDiagnosisAdmissionBundle(current);
|
||||
const unlockCommand = createCopilotFailureDiagnosisToolUnlockCommand({
|
||||
plan: current,
|
||||
completion: successfulToolCompletion(current),
|
||||
modelStepRun: admission.modelStepMutation.stepRun,
|
||||
run: {
|
||||
id: current.runId,
|
||||
projectId: current.projectId,
|
||||
status: 'running',
|
||||
version: 5,
|
||||
eventSequence: 5,
|
||||
},
|
||||
});
|
||||
const {
|
||||
DurableModelInvocationCoordinator,
|
||||
} = require('../dist/model-invocation/durableModelInvocationCoordinator.js');
|
||||
const { BoundedModelGateway } = require('../dist/model-gateway/gateway.js');
|
||||
const {
|
||||
CopilotFailureDiagnosisModelCompletionCoordinator,
|
||||
assertCopilotFailureDiagnosisOutputCompletionBinding,
|
||||
copilotFailureDiagnosisOutputReference,
|
||||
executeCopilotFailureDiagnosisModel,
|
||||
} = require('../dist/copilot/failure-diagnosis/modelExecution.js');
|
||||
|
||||
let stepRun = unlockCommand.modelStepRunMutation.stepRun;
|
||||
let runVersion = unlockCommand.receipt.finalRunVersion;
|
||||
let runEventSequence = unlockCommand.receipt.finalRunEventSequence;
|
||||
let start = null;
|
||||
let completion = null;
|
||||
let outputArtifact = null;
|
||||
let finalization = null;
|
||||
const repository = {
|
||||
async findStart() {
|
||||
return start;
|
||||
},
|
||||
async findCompletion() {
|
||||
return completion;
|
||||
},
|
||||
async readAuthority() {
|
||||
return {
|
||||
projectId: current.projectId,
|
||||
runId: current.runId,
|
||||
runVersion,
|
||||
runEventSequence,
|
||||
stepRun,
|
||||
};
|
||||
},
|
||||
async listIncomplete() {
|
||||
return { observedAtMs: 3_000, candidates: [], hasMore: false };
|
||||
},
|
||||
async admit(command) {
|
||||
start = command.start;
|
||||
stepRun = command.stepRunMutation.stepRun;
|
||||
runVersion += 1;
|
||||
runEventSequence += 1;
|
||||
return { status: 'created', record: start };
|
||||
},
|
||||
async complete(command) {
|
||||
completion = command.completion;
|
||||
stepRun = command.stepRunMutation.stepRun;
|
||||
runVersion += 1;
|
||||
runEventSequence += 1;
|
||||
return { status: 'created', record: completion };
|
||||
},
|
||||
async findCopilotFailureDiagnosisOutput() {
|
||||
return outputArtifact;
|
||||
},
|
||||
async completeWithCopilotFailureDiagnosisOutput(command, artifact) {
|
||||
const binding = assertCopilotFailureDiagnosisOutputCompletionBinding(
|
||||
command,
|
||||
artifact,
|
||||
);
|
||||
completion = command.completion;
|
||||
outputArtifact = binding.artifact;
|
||||
stepRun = command.stepRunMutation.stepRun;
|
||||
runVersion += 1;
|
||||
runEventSequence += 1;
|
||||
return { status: 'created', reference: binding.reference };
|
||||
},
|
||||
};
|
||||
const durable = new DurableModelInvocationCoordinator(repository);
|
||||
const successfulCompletion =
|
||||
new CopilotFailureDiagnosisModelCompletionCoordinator({
|
||||
coordinator: durable,
|
||||
keys: {
|
||||
async active() {
|
||||
return { keyId: 'copilot-output-key-1', key: Buffer.alloc(32, 7) };
|
||||
},
|
||||
async resolve(keyId) {
|
||||
return keyId === 'copilot-output-key-1'
|
||||
? { keyId, key: Buffer.alloc(32, 7) }
|
||||
: null;
|
||||
},
|
||||
},
|
||||
now: () => 3_500,
|
||||
nonceFactory: () => Buffer.alloc(12, 8),
|
||||
});
|
||||
let providerCalls = 0;
|
||||
const gateway = new BoundedModelGateway({
|
||||
providers: [
|
||||
{
|
||||
type: current.model.provider,
|
||||
async listModels() {
|
||||
return [{ id: current.model.model }];
|
||||
},
|
||||
async generate() {
|
||||
providerCalls += 1;
|
||||
return {
|
||||
provider: current.model.provider,
|
||||
model: current.model.model,
|
||||
text: 'The upstream service refused the connection.',
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 100, outputTokens: 9, totalTokens: 109 },
|
||||
};
|
||||
},
|
||||
async *stream() {},
|
||||
},
|
||||
],
|
||||
policies: {
|
||||
async resolve() {
|
||||
return {
|
||||
revision: 'model-policy-1',
|
||||
allowedProviders: [current.model.provider],
|
||||
allowedModels: [current.model.model],
|
||||
maxInputBytes: 64 * 1024,
|
||||
maxOutputBytes: 64 * 1024,
|
||||
maxOutputTokens: 1_024,
|
||||
maxTotalTokens: 2_048,
|
||||
maxCostMicros: null,
|
||||
priceRevision: null,
|
||||
};
|
||||
},
|
||||
},
|
||||
pricing: { async resolve() { return null; } },
|
||||
audit: durable,
|
||||
successfulCompletion,
|
||||
maxConcurrent: 1,
|
||||
now: () => 3_100,
|
||||
});
|
||||
const finalizations = {
|
||||
async findFinalization() {
|
||||
return finalization;
|
||||
},
|
||||
async finalize() {
|
||||
if (!finalization) {
|
||||
finalization = createCopilotFailureDiagnosisFinalizationReceipt({
|
||||
requestId: current.requestId,
|
||||
planDigest: current.planDigest,
|
||||
runId: current.runId,
|
||||
modelStepRunId: current.modelStepRunId,
|
||||
invocationId: current.modelInvocationId,
|
||||
completionDigest: completion.completionDigest,
|
||||
outcome: completion.outcome,
|
||||
outputArtifactId: outputArtifact.artifactId,
|
||||
finalRunVersion: runVersion + 1,
|
||||
finalRunEventSequence: runEventSequence + 1,
|
||||
finalizedAtMs: completion.completedAtMs,
|
||||
});
|
||||
return { status: 'created', receipt: finalization };
|
||||
}
|
||||
return { status: 'existing', receipt: finalization };
|
||||
},
|
||||
};
|
||||
const dependencies = {
|
||||
admissions: {
|
||||
async findPlanByRequestId() {
|
||||
return current;
|
||||
},
|
||||
async findByRequestId() {
|
||||
return admission.receipt;
|
||||
},
|
||||
},
|
||||
unlocks: {
|
||||
async findByRequestId() {
|
||||
return unlockCommand.receipt;
|
||||
},
|
||||
},
|
||||
toolResults: {
|
||||
async open() {
|
||||
return {
|
||||
status: 'existing',
|
||||
completion: successfulToolCompletion(current),
|
||||
output: {
|
||||
status: 'available',
|
||||
runId: current.source.runId,
|
||||
attemptId: current.source.attemptId,
|
||||
profile: 'cluster-control',
|
||||
sourceWindowBytes: 16 * 1024,
|
||||
content: 'connect ECONNREFUSED 127.0.0.1:5432',
|
||||
sourceBytes: 35,
|
||||
modelTextBytes: 35,
|
||||
redaction: {
|
||||
contract: 'recognized_credentials_v1',
|
||||
residualSensitivity: 'potentially_sensitive',
|
||||
replacements: 0,
|
||||
categories: [],
|
||||
},
|
||||
normalization: {
|
||||
invalidUtf8: false,
|
||||
unsafeCodePointsReplaced: 0,
|
||||
},
|
||||
trust: {
|
||||
classification: 'untrusted_execution_output',
|
||||
instructionPolicy: 'data_only_never_execute',
|
||||
actionAuthority: 'none',
|
||||
suspectedPromptInjection: false,
|
||||
signals: [],
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
modelInvocations: repository,
|
||||
outputs: repository,
|
||||
gateway,
|
||||
successfulCompletion,
|
||||
finalizations,
|
||||
};
|
||||
const created = await executeCopilotFailureDiagnosisModel(
|
||||
current.requestId,
|
||||
dependencies,
|
||||
);
|
||||
assert.equal(created.outcome, 'succeeded');
|
||||
assert.equal(providerCalls, 1);
|
||||
assert.equal(JSON.stringify(outputArtifact).includes('refused'), false);
|
||||
assert.deepEqual(
|
||||
created.output,
|
||||
copilotFailureDiagnosisOutputReference(outputArtifact),
|
||||
);
|
||||
const replay = await executeCopilotFailureDiagnosisModel(
|
||||
current.requestId,
|
||||
dependencies,
|
||||
);
|
||||
assert.equal(replay.finalization.receiptDigest, created.finalization.receiptDigest);
|
||||
assert.equal(providerCalls, 1);
|
||||
});
|
||||
|
||||
@@ -21,6 +21,7 @@ const {
|
||||
LOCAL_MODEL_PRICE_CATALOG_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_ADMISSION_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID,
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_ID,
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE,
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_STREAM_ID,
|
||||
@@ -431,6 +432,10 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async ()
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_TOOL_UNLOCK_MIGRATION_ID,
|
||||
'pg-9019-ai-copilot-failure-diagnosis-tool-unlocks',
|
||||
);
|
||||
assert.equal(
|
||||
POSTGRES_COPILOT_FAILURE_DIAGNOSIS_MODEL_EXECUTION_MIGRATION_ID,
|
||||
'pg-9020-ai-copilot-failure-diagnosis-model-executions',
|
||||
);
|
||||
assert.equal(
|
||||
POSTGRES_MODEL_INVOCATION_MIGRATION_HISTORY_TABLE,
|
||||
'ai_schema_migrations',
|
||||
@@ -518,7 +523,7 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async ()
|
||||
);
|
||||
assert.equal(
|
||||
postgresModelInvocationMigrationDefinition.migrations.length,
|
||||
19,
|
||||
20,
|
||||
);
|
||||
|
||||
const diagnosisAdmissionStatements = [];
|
||||
@@ -565,6 +570,33 @@ test('PostgreSQL AI schema is an independent reviewed feature stream', async ()
|
||||
assert.match(diagnosisToolUnlockSql, /TO ql3_runtime/);
|
||||
assert.doesNotMatch(diagnosisToolUnlockSql, /GRANT[^;]*(?:UPDATE|DELETE)/);
|
||||
|
||||
const diagnosisModelExecutionStatements = [];
|
||||
await postgresModelInvocationMigrationDefinition.migrations[19].up({
|
||||
async query(statement) {
|
||||
diagnosisModelExecutionStatements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
const diagnosisModelExecutionSql =
|
||||
diagnosisModelExecutionStatements.join('\n');
|
||||
assert.match(
|
||||
diagnosisModelExecutionSql,
|
||||
/CREATE TABLE "ql3_ai"\."copilot_failure_diagnosis_model_outputs"/,
|
||||
);
|
||||
assert.match(
|
||||
diagnosisModelExecutionSql,
|
||||
/CREATE TABLE "ql3_ai"\."copilot_failure_diagnosis_finalizations"/,
|
||||
);
|
||||
assert.match(
|
||||
diagnosisModelExecutionSql,
|
||||
/FOREIGN KEY \(invocation_id\)[\s\S]*model_invocation_completions/,
|
||||
);
|
||||
assert.match(diagnosisModelExecutionSql, /TO ql3_runtime/);
|
||||
assert.doesNotMatch(
|
||||
diagnosisModelExecutionSql,
|
||||
/GRANT[^;]*(?:UPDATE|DELETE)/,
|
||||
);
|
||||
|
||||
const retirementStatements = [];
|
||||
await postgresModelInvocationMigrationDefinition.migrations[10].up({
|
||||
async query(statement) {
|
||||
|
||||
+1
-1
@@ -175,7 +175,7 @@ test('readiness binds exact migration history and least-privilege primary author
|
||||
assert.equal(report.ready, true);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-9019-ai-copilot-failure-diagnosis-tool-unlocks',
|
||||
'pg-9020-ai-copilot-failure-diagnosis-model-executions',
|
||||
);
|
||||
assert.match(
|
||||
queries[1],
|
||||
|
||||
@@ -277,7 +277,7 @@ test('tester readiness freezes migration history and least privilege', async ()
|
||||
assert.equal(ready.ready, true);
|
||||
assert.equal(
|
||||
ready.migrationIds.at(-1),
|
||||
'pg-9019-ai-copilot-failure-diagnosis-tool-unlocks',
|
||||
'pg-9020-ai-copilot-failure-diagnosis-model-executions',
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
|
||||
@@ -37,6 +37,12 @@ import {
|
||||
} from './trustedToolExecution';
|
||||
import type { ToolJsonValue } from './tool-registry/toolRegistry';
|
||||
|
||||
export {
|
||||
openTrustedToolSuccessCompletion,
|
||||
type TrustedToolSuccessCompletionReadDependencies,
|
||||
type TrustedToolSuccessCompletionResult,
|
||||
} from './trustedToolSuccessCompletion';
|
||||
|
||||
export interface TrustedToolFailureCompletionIdentities {
|
||||
readonly mutationId: string;
|
||||
readonly eventId: string;
|
||||
|
||||
@@ -60,15 +60,24 @@ export interface TrustedToolSuccessCompletionIdentityFactory {
|
||||
}
|
||||
|
||||
export interface TrustedToolSuccessCompletionDependencies
|
||||
extends TrustedToolExecutionDependencies {
|
||||
readonly completions: ToolExecutionCompletionRepository;
|
||||
extends TrustedToolExecutionDependencies,
|
||||
TrustedToolSuccessCompletionReadDependencies {
|
||||
readonly stepRuns: Pick<StepRunRepository, 'findById'>;
|
||||
readonly runs: Pick<RunRepositoryReader, 'findRunById'>;
|
||||
readonly identities: TrustedToolSuccessCompletionIdentityFactory;
|
||||
readonly nonceFactory?: () => Uint8Array;
|
||||
}
|
||||
|
||||
export interface TrustedToolSuccessCompletionReadDependencies {
|
||||
readonly completions: ToolExecutionCompletionRepository;
|
||||
readonly barriers: Pick<
|
||||
TrustedToolExecutionDependencies['barriers'],
|
||||
'findByStartId'
|
||||
>;
|
||||
readonly resultKeyCatalog: ToolResultKeyCatalogReader;
|
||||
readonly resultRekeys: ToolExecutionResultRekeyReader;
|
||||
readonly resultKeys: Pick<ToolInvocationArtifactKeyProvider, 'resolve'>;
|
||||
readonly identities: TrustedToolSuccessCompletionIdentityFactory;
|
||||
readonly nonceFactory?: () => Uint8Array;
|
||||
readonly adapters: TrustedToolExecutionAdapterRegistry;
|
||||
}
|
||||
|
||||
export interface TrustedToolSuccessCompletionResult {
|
||||
@@ -101,26 +110,15 @@ function sameValue(left: unknown, right: unknown): boolean {
|
||||
function validateDependencies(
|
||||
dependencies: TrustedToolSuccessCompletionDependencies,
|
||||
): void {
|
||||
validateReadDependencies(dependencies);
|
||||
if (
|
||||
!dependencies ||
|
||||
typeof dependencies !== 'object' ||
|
||||
!dependencies.completions ||
|
||||
typeof dependencies.completions.findByStartId !== 'function' ||
|
||||
typeof dependencies.completions.findResultArtifact !== 'function' ||
|
||||
typeof dependencies.completions.commit !== 'function' ||
|
||||
!dependencies.stepRuns ||
|
||||
typeof dependencies.stepRuns.findById !== 'function' ||
|
||||
!dependencies.runs ||
|
||||
typeof dependencies.runs.findRunById !== 'function' ||
|
||||
!dependencies.resultKeys ||
|
||||
typeof dependencies.resultKeys.resolve !== 'function' ||
|
||||
!dependencies.resultKeyCatalog ||
|
||||
typeof dependencies.resultKeyCatalog.findCurrent !== 'function' ||
|
||||
!dependencies.resultRekeys ||
|
||||
typeof dependencies.resultRekeys.findHeadByArtifactId !== 'function' ||
|
||||
!dependencies.identities ||
|
||||
typeof dependencies.identities.create !== 'function' ||
|
||||
!(dependencies.adapters instanceof TrustedToolExecutionAdapterRegistry) ||
|
||||
(dependencies.nonceFactory !== undefined &&
|
||||
typeof dependencies.nonceFactory !== 'function')
|
||||
) {
|
||||
@@ -128,9 +126,32 @@ function validateDependencies(
|
||||
}
|
||||
}
|
||||
|
||||
function validateReadDependencies(
|
||||
dependencies: TrustedToolSuccessCompletionReadDependencies,
|
||||
): void {
|
||||
if (
|
||||
!dependencies ||
|
||||
typeof dependencies !== 'object' ||
|
||||
!dependencies.completions ||
|
||||
typeof dependencies.completions.findByStartId !== 'function' ||
|
||||
typeof dependencies.completions.findResultArtifact !== 'function' ||
|
||||
!dependencies.barriers ||
|
||||
typeof dependencies.barriers.findByStartId !== 'function' ||
|
||||
!dependencies.resultKeys ||
|
||||
typeof dependencies.resultKeys.resolve !== 'function' ||
|
||||
!dependencies.resultKeyCatalog ||
|
||||
typeof dependencies.resultKeyCatalog.findCurrent !== 'function' ||
|
||||
!dependencies.resultRekeys ||
|
||||
typeof dependencies.resultRekeys.findHeadByArtifactId !== 'function' ||
|
||||
!(dependencies.adapters instanceof TrustedToolExecutionAdapterRegistry)
|
||||
) {
|
||||
unavailable();
|
||||
}
|
||||
}
|
||||
|
||||
async function findResultRekeyHead(
|
||||
artifactId: string,
|
||||
dependencies: TrustedToolSuccessCompletionDependencies,
|
||||
dependencies: TrustedToolSuccessCompletionReadDependencies,
|
||||
): Promise<Readonly<ToolExecutionResultRekeyOverlay> | null> {
|
||||
try {
|
||||
const value = await dependencies.resultRekeys.findHeadByArtifactId(
|
||||
@@ -145,7 +166,7 @@ async function findResultRekeyHead(
|
||||
}
|
||||
|
||||
async function findResultKeyCatalog(
|
||||
dependencies: TrustedToolSuccessCompletionDependencies,
|
||||
dependencies: TrustedToolSuccessCompletionReadDependencies,
|
||||
): Promise<Readonly<ToolResultKeyCatalogRecord>> {
|
||||
try {
|
||||
const value = await dependencies.resultKeyCatalog.findCurrent();
|
||||
@@ -169,7 +190,7 @@ function validCatalogMaterial(
|
||||
|
||||
async function findCompletion(
|
||||
startId: string,
|
||||
dependencies: TrustedToolSuccessCompletionDependencies,
|
||||
dependencies: TrustedToolSuccessCompletionReadDependencies,
|
||||
): Promise<Readonly<ToolExecutionCompletionRecord> | null> {
|
||||
try {
|
||||
const value = await dependencies.completions.findByStartId(startId);
|
||||
@@ -183,7 +204,7 @@ async function findCompletion(
|
||||
|
||||
async function findBarrier(
|
||||
startId: string,
|
||||
dependencies: TrustedToolSuccessCompletionDependencies,
|
||||
dependencies: TrustedToolSuccessCompletionReadDependencies,
|
||||
): Promise<Readonly<ToolExecutionStartBarrierRecord>> {
|
||||
try {
|
||||
const value = await dependencies.barriers.findByStartId(startId);
|
||||
@@ -245,7 +266,7 @@ function completionMatches(
|
||||
|
||||
async function openDurableCompletion(
|
||||
completion: Readonly<ToolExecutionCompletionRecord>,
|
||||
dependencies: TrustedToolSuccessCompletionDependencies,
|
||||
dependencies: TrustedToolSuccessCompletionReadDependencies,
|
||||
): Promise<Readonly<TrustedToolSuccessCompletionResult>> {
|
||||
const barrier = await findBarrier(completion.startId, dependencies);
|
||||
let artifact: Readonly<ToolExecutionResultArtifact>;
|
||||
@@ -307,6 +328,17 @@ async function openDurableCompletion(
|
||||
}
|
||||
}
|
||||
|
||||
/** Reopens one exact encrypted Tool success without executing the Tool. */
|
||||
export async function openTrustedToolSuccessCompletion(
|
||||
startId: string,
|
||||
dependencies: TrustedToolSuccessCompletionReadDependencies,
|
||||
): Promise<Readonly<TrustedToolSuccessCompletionResult>> {
|
||||
validateReadDependencies(dependencies);
|
||||
const completion = await findCompletion(startId, dependencies);
|
||||
if (!completion) return unavailable();
|
||||
return openDurableCompletion(completion, dependencies);
|
||||
}
|
||||
|
||||
async function findStepRun(
|
||||
barrier: Readonly<ToolExecutionStartBarrierRecord>,
|
||||
dependencies: TrustedToolSuccessCompletionDependencies,
|
||||
@@ -351,7 +383,7 @@ export async function executeAndCompleteTrustedToolSuccess(
|
||||
validateDependencies(dependencies);
|
||||
|
||||
const existing = await findCompletion(startId, dependencies);
|
||||
if (existing) return openDurableCompletion(existing, dependencies);
|
||||
if (existing) return openTrustedToolSuccessCompletion(startId, dependencies);
|
||||
|
||||
const executionResult = await executeTrustedToolAfterStart(
|
||||
startId,
|
||||
@@ -359,7 +391,9 @@ export async function executeAndCompleteTrustedToolSuccess(
|
||||
);
|
||||
|
||||
const concurrent = await findCompletion(startId, dependencies);
|
||||
if (concurrent) return openDurableCompletion(concurrent, dependencies);
|
||||
if (concurrent) {
|
||||
return openTrustedToolSuccessCompletion(startId, dependencies);
|
||||
}
|
||||
|
||||
const barrier = await findBarrier(startId, dependencies);
|
||||
const adapter = dependencies.adapters.resolve(barrier);
|
||||
@@ -469,7 +503,9 @@ export async function executeAndCompleteTrustedToolSuccess(
|
||||
});
|
||||
} catch (cause) {
|
||||
const recovered = await findCompletion(startId, dependencies);
|
||||
if (recovered) return openDurableCompletion(recovered, dependencies);
|
||||
if (recovered) {
|
||||
return openTrustedToolSuccessCompletion(startId, dependencies);
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user