feat(ql3): compose cluster copilot diagnosis

packages/ql3-ai/src/copilot/failure-diagnosis/application/contracts.ts

@@ -0,0 +1,59 @@
+import type { SecurityPrincipal } from '@qinglong/runtime-core/security';
+
+import type { CopilotFailureDiagnosisAdmissionReceipt } from '../admission/contracts';
+import type { CopilotFailureDiagnosisModelExecutionResult } from '../model-execution/coordinator';
+import type { CopilotFailureDiagnosisToolExecutionResult } from '../tool-execution/contracts';
+
+export const MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS = 64;
+
+export interface ExecuteCopilotFailureDiagnosisApplicationCommand {
+  readonly requestId: string;
+  readonly traceId: string;
+  readonly projectId: string;
+  readonly sourceRunId: string;
+  readonly principal: Readonly<SecurityPrincipal>;
+}
+
+export interface ExecuteCopilotFailureDiagnosisApplicationResult {
+  readonly admissionStatus: 'created' | 'existing';
+  readonly admission: Readonly<CopilotFailureDiagnosisAdmissionReceipt>;
+  readonly tool: Readonly<CopilotFailureDiagnosisToolExecutionResult>;
+  readonly model: Readonly<CopilotFailureDiagnosisModelExecutionResult> | null;
+  readonly terminalizationRequired: boolean;
+}
+
+export class InvalidCopilotFailureDiagnosisApplicationError extends TypeError {
+  readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_INVALID';
+
+  constructor(message: string) {
+    super(`Copilot failure diagnosis application is invalid: ${message}`);
+    this.name = 'InvalidCopilotFailureDiagnosisApplicationError';
+  }
+}
+
+export class CopilotFailureDiagnosisApplicationConflictError extends Error {
+  readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_CONFLICT';
+
+  constructor(message = 'the durable diagnosis request changed') {
+    super(`Copilot failure diagnosis application conflicts: ${message}`);
+    this.name = 'CopilotFailureDiagnosisApplicationConflictError';
+  }
+}
+
+export class CopilotFailureDiagnosisApplicationUnavailableError extends Error {
+  readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_UNAVAILABLE';
+
+  constructor(options?: ErrorOptions) {
+    super('Copilot failure diagnosis application is unavailable', options);
+    this.name = 'CopilotFailureDiagnosisApplicationUnavailableError';
+  }
+}
+
+export class CopilotFailureDiagnosisApplicationBusyError extends Error {
+  readonly code = 'COPILOT_FAILURE_DIAGNOSIS_APPLICATION_BUSY';
+
+  constructor() {
+    super('Copilot failure diagnosis application request budget is exhausted');
+    this.name = 'CopilotFailureDiagnosisApplicationBusyError';
+  }
+}

packages/ql3-ai/src/copilot/failure-diagnosis/application/service.ts

@@ -0,0 +1,572 @@
+import { Buffer } from 'node:buffer';
+import { createHash, createHmac } from 'node:crypto';
+
+import {
+  BUILTIN_RUN_LOG_EXCERPT_TOOL,
+  createBuiltInRunLogExcerptToolHandlerBinding,
+} from '@qinglong/runtime-core/builtin-run-log-excerpt-tool';
+import type { RunRepositoryReader } from '@qinglong/runtime-core/run-repository';
+import type { ToolPolicyAuthorizer } from '@qinglong/runtime-core/tool-registry';
+import {
+  prepareToolInvocation,
+} from '@qinglong/runtime-core/tool-registry';
+import type { ProjectToolDefinitionSnapshotRepository } from '@qinglong/runtime-core/project-tool-definition-snapshot';
+import { projectToolDefinitionRegistry } from '@qinglong/runtime-core/project-tool-definition-snapshot';
+import {
+  createToolInvocationInputArtifact,
+  createToolInvocationPreviewArtifact,
+  type ToolInvocationArtifactKeyProvider,
+  type ToolInvocationArtifactRepository,
+  type ToolInvocationPreviewDocument,
+} from '@qinglong/runtime-core/tool-invocation-artifact';
+import {
+  TrustedToolHandlerBindingRegistry,
+  createTrustedToolInvocationPlan,
+} from '@qinglong/runtime-core/trusted-tool-invocation';
+
+import { MAX_MODEL_INVOCATION_MS } from '../../../model-gateway/model';
+import {
+  prepareCopilotFailureDiagnosisExecution,
+} from '../admission/plan';
+import type {
+  CopilotFailureDiagnosisAdmissionRepository,
+  CopilotFailureDiagnosisExecutionPlan,
+  PrepareCopilotFailureDiagnosisModelIntent,
+} from '../admission/contracts';
+import {
+  executeCopilotFailureDiagnosisTool,
+  type CopilotFailureDiagnosisToolExecutionDependencies,
+} from '../tool-execution/coordinator';
+import {
+  executeCopilotFailureDiagnosisModel,
+  type CopilotFailureDiagnosisModelExecutionDependencies,
+} from '../model-execution/coordinator';
+import {
+  CopilotFailureDiagnosisApplicationBusyError,
+  CopilotFailureDiagnosisApplicationConflictError,
+  CopilotFailureDiagnosisApplicationUnavailableError,
+  InvalidCopilotFailureDiagnosisApplicationError,
+  MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS,
+  type ExecuteCopilotFailureDiagnosisApplicationCommand,
+  type ExecuteCopilotFailureDiagnosisApplicationResult,
+} from './contracts';
+
+const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
+const NONCE_DOMAIN = Buffer.from(
+  'qinglong/copilot-failure-diagnosis-tool-invocation-nonce@v1\0',
+  'utf8',
+);
+const IDENTITY_DOMAIN = Buffer.from(
+  'qinglong/copilot-failure-diagnosis-application-identity@v1\0',
+  'utf8',
+);
+
+export interface CopilotFailureDiagnosisApplicationDependencies {
+  readonly admissions: CopilotFailureDiagnosisAdmissionRepository;
+  readonly snapshots: Pick<ProjectToolDefinitionSnapshotRepository, 'findCurrent'>;
+  readonly runs: Pick<
+    RunRepositoryReader,
+    'findRunById' | 'findLatestAttemptByRunId'
+  >;
+  readonly artifacts: ToolInvocationArtifactRepository;
+  readonly invocationKeys: Pick<
+    ToolInvocationArtifactKeyProvider,
+    'active' | 'resolve'
+  >;
+  readonly authorizer: ToolPolicyAuthorizer;
+  readonly tool: CopilotFailureDiagnosisToolExecutionDependencies;
+  readonly model: CopilotFailureDiagnosisModelExecutionDependencies;
+  readonly executeTool: typeof executeCopilotFailureDiagnosisTool;
+  readonly executeModel: typeof executeCopilotFailureDiagnosisModel;
+  readonly modelIntent: Readonly<PrepareCopilotFailureDiagnosisModelIntent>;
+  readonly executionTimeoutMs: number;
+  readonly now?: () => number;
+  readonly nonceFactory?: (input: Readonly<{
+    key: Uint8Array;
+    keyId: string;
+    requestId: string;
+    projectId: string;
+    sourceRunId: string;
+    invocationActionDigest: string;
+  }>) => Uint8Array;
+}
+
+interface ActiveRequest {
+  readonly digest: string;
+  readonly promise: Promise<
+    Readonly<ExecuteCopilotFailureDiagnosisApplicationResult>
+  >;
+}
+
+function invalid(message: string): never {
+  throw new InvalidCopilotFailureDiagnosisApplicationError(message);
+}
+
+function unavailable(cause?: unknown): never {
+  throw new CopilotFailureDiagnosisApplicationUnavailableError({
+    cause: cause instanceof Error ? cause : undefined,
+  });
+}
+
+function digest(value: unknown): string {
+  return createHash('sha256').update(JSON.stringify(value)).digest('hex');
+}
+
+function identity(prefix: string, requestId: string): string {
+  return `${prefix}:${createHash('sha256')
+    .update(IDENTITY_DOMAIN)
+    .update(prefix)
+    .update('\0')
+    .update(requestId)
+    .digest('hex')
+    .slice(0, 32)}`;
+}
+
+function preview(
+  sourceRunId: string,
+  sourceAttemptId: string,
+): Readonly<ToolInvocationPreviewDocument> {
+  return Object.freeze({
+    title: 'Diagnose failed Run',
+    summary: 'Read one bounded, redacted and untrusted execution log excerpt',
+    fields: Object.freeze([
+      Object.freeze({
+        kind: 'identifier' as const,
+        label: 'Run',
+        value: sourceRunId,
+      }),
+      Object.freeze({
+        kind: 'identifier' as const,
+        label: 'Attempt',
+        value: sourceAttemptId,
+      }),
+    ]),
+    warnings: Object.freeze(['potentially_sensitive_output']),
+  });
+}
+
+function defaultNonce(input: Readonly<{
+  key: Uint8Array;
+  keyId: string;
+  requestId: string;
+  projectId: string;
+  sourceRunId: string;
+  invocationActionDigest: string;
+}>): Uint8Array {
+  const derived = createHmac('sha256', Buffer.from(input.key))
+    .update(NONCE_DOMAIN)
+    .update(
+      JSON.stringify({
+        keyId: input.keyId,
+        requestId: input.requestId,
+        projectId: input.projectId,
+        sourceRunId: input.sourceRunId,
+        invocationActionDigest: input.invocationActionDigest,
+      }),
+    )
+    .digest();
+  try {
+    return Buffer.from(derived.subarray(0, 12));
+  } finally {
+    derived.fill(0);
+  }
+}
+
+function normalizeCommand(
+  value: ExecuteCopilotFailureDiagnosisApplicationCommand,
+): Readonly<ExecuteCopilotFailureDiagnosisApplicationCommand> {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    Object.keys(value).sort().join('\0') !==
+      ['principal', 'projectId', 'requestId', 'sourceRunId', 'traceId'].join(
+        '\0',
+      ) ||
+    !ID_PATTERN.test(value.requestId) ||
+    !ID_PATTERN.test(value.traceId) ||
+    !ID_PATTERN.test(value.projectId) ||
+    !RUN_ID_PATTERN.test(value.sourceRunId) ||
+    !value.principal ||
+    typeof value.principal !== 'object' ||
+    Array.isArray(value.principal)
+  ) {
+    return invalid('command is invalid');
+  }
+  return Object.freeze({
+    requestId: value.requestId,
+    traceId: value.traceId,
+    projectId: value.projectId,
+    sourceRunId: value.sourceRunId,
+    principal: value.principal,
+  });
+}
+
+function assertDependencies(
+  value: CopilotFailureDiagnosisApplicationDependencies,
+): void {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    typeof value.admissions?.findByRequestId !== 'function' ||
+    typeof value.admissions?.findPlanByRequestId !== 'function' ||
+    typeof value.admissions?.admit !== 'function' ||
+    typeof value.snapshots?.findCurrent !== 'function' ||
+    typeof value.runs?.findRunById !== 'function' ||
+    typeof value.runs?.findLatestAttemptByRunId !== 'function' ||
+    typeof value.artifacts?.put !== 'function' ||
+    typeof value.invocationKeys?.active !== 'function' ||
+    typeof value.invocationKeys?.resolve !== 'function' ||
+    typeof value.authorizer?.authorize !== 'function' ||
+    typeof value.executeTool !== 'function' ||
+    typeof value.executeModel !== 'function' ||
+    !value.tool ||
+    !value.model ||
+    !value.modelIntent ||
+    typeof value.modelIntent !== 'object' ||
+    !Number.isSafeInteger(value.executionTimeoutMs) ||
+    value.executionTimeoutMs < 1 ||
+    value.executionTimeoutMs > MAX_MODEL_INVOCATION_MS ||
+    (value.now !== undefined && typeof value.now !== 'function') ||
+    (value.nonceFactory !== undefined &&
+      typeof value.nonceFactory !== 'function')
+  ) {
+    return invalid('dependencies are invalid');
+  }
+  if (
+    value.tool.admissions !== value.admissions ||
+    value.tool.snapshots !== value.snapshots ||
+    value.tool.runs !== value.runs ||
+    value.tool.artifacts !== value.artifacts ||
+    value.tool.invocationKeys !== value.invocationKeys ||
+    value.model.admissions !== value.admissions ||
+    value.model.unlocks !== value.tool.unlocks
+  ) {
+    return invalid('dependency authorities are not shared');
+  }
+}
+
+function sameSubject(
+  left: Readonly<{ type: string; id: string }>,
+  right: Readonly<{ type: string; id: string }>,
+): boolean {
+  return left.type === right.type && left.id === right.id;
+}
+
+function sameModelIntent(
+  plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
+  configured: Readonly<PrepareCopilotFailureDiagnosisModelIntent>,
+): boolean {
+  return (
+    plan.model.provider === configured.provider &&
+    plan.model.model === configured.model &&
+    plan.model.modelBoundary === configured.modelBoundary &&
+    plan.model.responseLanguage === configured.responseLanguage &&
+    plan.model.maxOutputTokens === configured.maxOutputTokens &&
+    JSON.stringify(plan.model.egressPolicy) ===
+      JSON.stringify(configured.egressPolicy)
+  );
+}
+
+function nonceInput(
+  plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
+  key: Uint8Array,
+) {
+  return Object.freeze({
+    key,
+    keyId: plan.tool.invocationArtifact.keyId,
+    requestId: plan.requestId,
+    projectId: plan.projectId,
+    sourceRunId: plan.source.runId,
+    invocationActionDigest: plan.tool.invocationActionDigest,
+  });
+}
+
+export class CopilotFailureDiagnosisApplicationService {
+  readonly #dependencies: CopilotFailureDiagnosisApplicationDependencies;
+  readonly #active = new Map<string, ActiveRequest>();
+
+  constructor(dependencies: CopilotFailureDiagnosisApplicationDependencies) {
+    assertDependencies(dependencies);
+    this.#dependencies = dependencies;
+  }
+
+  execute(
+    commandValue: ExecuteCopilotFailureDiagnosisApplicationCommand,
+  ): Promise<Readonly<ExecuteCopilotFailureDiagnosisApplicationResult>> {
+    const command = normalizeCommand(commandValue);
+    const commandDigest = digest(command);
+    const active = this.#active.get(command.requestId);
+    if (active) {
+      if (active.digest !== commandDigest) {
+        throw new CopilotFailureDiagnosisApplicationConflictError();
+      }
+      return active.promise;
+    }
+    if (
+      this.#active.size >=
+      MAX_ACTIVE_COPILOT_FAILURE_DIAGNOSIS_APPLICATION_REQUESTS
+    ) {
+      throw new CopilotFailureDiagnosisApplicationBusyError();
+    }
+    const promise = this.#execute(command).finally(() => {
+      this.#active.delete(command.requestId);
+    });
+    this.#active.set(command.requestId, { digest: commandDigest, promise });
+    return promise;
+  }
+
+  async #execute(
+    command: Readonly<ExecuteCopilotFailureDiagnosisApplicationCommand>,
+  ): Promise<Readonly<ExecuteCopilotFailureDiagnosisApplicationResult>> {
+    const existing = await this.#dependencies.admissions.findPlanByRequestId(
+      command.requestId,
+    );
+    let plan: Readonly<CopilotFailureDiagnosisExecutionPlan>;
+    let admissionStatus: 'created' | 'existing';
+    let admission;
+    if (existing) {
+      if (
+        existing.projectId !== command.projectId ||
+        existing.source.runId !== command.sourceRunId ||
+        existing.traceId !== command.traceId ||
+        !sameSubject(existing.requestedBySubject, command.principal.subject) ||
+        !sameModelIntent(existing, this.#dependencies.modelIntent)
+      ) {
+        throw new CopilotFailureDiagnosisApplicationConflictError();
+      }
+      plan = existing;
+      const admitted = await this.#dependencies.admissions.admit(plan);
+      admissionStatus = admitted.status;
+      admission = admitted.receipt;
+      await this.#materializeArtifacts(plan);
+    } else {
+      const prepared = await this.#prepare(command);
+      plan = prepared.plan;
+      const admitted = await this.#dependencies.admissions.admit(plan);
+      admissionStatus = admitted.status;
+      admission = admitted.receipt;
+      try {
+        await this.#dependencies.artifacts.put(
+          prepared.inputArtifact,
+          prepared.previewArtifact,
+        );
+      } catch (cause) {
+        throw new CopilotFailureDiagnosisApplicationUnavailableError({
+          cause: cause instanceof Error ? cause : undefined,
+        });
+      }
+    }
+    const tool = await this.#dependencies.executeTool(
+      {
+        requestId: plan.requestId,
+        principal: command.principal,
+        authorizer: this.#dependencies.authorizer,
+      },
+      this.#dependencies.tool,
+    );
+    if (tool.outcome !== 'succeeded') {
+      return Object.freeze({
+        admissionStatus,
+        admission,
+        tool,
+        model: null,
+        terminalizationRequired: true,
+      });
+    }
+    const model = await this.#dependencies.executeModel(
+      plan.requestId,
+      this.#dependencies.model,
+    );
+    return Object.freeze({
+      admissionStatus,
+      admission,
+      tool,
+      model,
+      terminalizationRequired: false,
+    });
+  }
+
+  async #prepare(
+    command: Readonly<ExecuteCopilotFailureDiagnosisApplicationCommand>,
+  ) {
+    const now = this.#clock();
+    let run;
+    let attempt;
+    let snapshotRecord;
+    try {
+      [run, attempt, snapshotRecord] = await Promise.all([
+        this.#dependencies.runs.findRunById(command.sourceRunId),
+        this.#dependencies.runs.findLatestAttemptByRunId(command.sourceRunId),
+        this.#dependencies.snapshots.findCurrent(command.projectId),
+      ]);
+    } catch (cause) {
+      return unavailable(cause);
+    }
+    if (
+      !run ||
+      !attempt ||
+      !snapshotRecord ||
+      run.projectId !== command.projectId ||
+      attempt.runId !== run.id ||
+      !['failed', 'timed_out'].includes(run.status) ||
+      !['failed', 'timed_out', 'lost'].includes(attempt.status) ||
+      (run.status === 'failed' &&
+        !['failed', 'lost'].includes(attempt.status)) ||
+      (run.status === 'timed_out' && attempt.status !== 'timed_out') ||
+      !Number.isSafeInteger(attempt.finishedAtMs) ||
+      typeof attempt.logArtifactId !== 'string'
+    ) {
+      throw new CopilotFailureDiagnosisApplicationConflictError(
+        'source Run is not an exact diagnosable terminal fence',
+      );
+    }
+    const snapshot = snapshotRecord.snapshot;
+    const binding = createBuiltInRunLogExcerptToolHandlerBinding(snapshot, [
+      'cluster-control',
+    ]);
+    const bindings = new TrustedToolHandlerBindingRegistry(snapshot, [binding]);
+    const invocation = await prepareToolInvocation(
+      projectToolDefinitionRegistry(snapshot),
+      {
+        projectId: command.projectId,
+        principal: command.principal,
+        nowMs: now,
+        tool: BUILTIN_RUN_LOG_EXCERPT_TOOL,
+        input: { runId: run.id, attemptId: attempt.id },
+      },
+      this.#dependencies.authorizer,
+    );
+    if (invocation.status !== 'ready') {
+      throw new CopilotFailureDiagnosisApplicationConflictError(
+        'Tool admission is not ready',
+      );
+    }
+    const key = await this.#dependencies.invocationKeys.active();
+    try {
+      const baseIdentity = identity('cda', command.requestId);
+      const nonce = (
+        this.#dependencies.nonceFactory ?? defaultNonce
+      )({
+        key: key.key,
+        keyId: key.keyId,
+        requestId: command.requestId,
+        projectId: command.projectId,
+        sourceRunId: run.id,
+        invocationActionDigest: invocation.actionDigest,
+      });
+      const tool = createTrustedToolInvocationPlan(bindings, invocation, {
+        actionRef: baseIdentity,
+        profile: 'cluster-control',
+        preview: preview(run.id, attempt.id),
+        inputArtifactId: identity('cdia', command.requestId),
+        previewArtifactId: identity('cdpa', command.requestId),
+        artifactKeyId: key.keyId,
+        artifactKey: key.key,
+        artifactNonce: nonce,
+        sealedAtMs: now,
+      });
+      nonce.fill(0);
+      const plan = prepareCopilotFailureDiagnosisExecution({
+        requestId: command.requestId,
+        traceId: command.traceId,
+        source: {
+          runId: run.id,
+          runVersion: run.version,
+          runStatus: run.status as 'failed' | 'timed_out',
+          attemptId: attempt.id,
+          attemptStatus: attempt.status as 'failed' | 'timed_out' | 'lost',
+          attemptFinishedAtMs: attempt.finishedAtMs!,
+          logArtifactId: attempt.logArtifactId,
+        },
+        toolPlan: tool.plan,
+        bindings,
+        model: this.#dependencies.modelIntent,
+        deadlineAtMs: now + this.#dependencies.executionTimeoutMs,
+        plannedAtMs: now,
+      });
+      return Object.freeze({
+        plan,
+        inputArtifact: tool.inputArtifact,
+        previewArtifact: tool.previewArtifact,
+      });
+    } finally {
+      key.key.fill(0);
+    }
+  }
+
+  async #materializeArtifacts(
+    plan: Readonly<CopilotFailureDiagnosisExecutionPlan>,
+  ): Promise<void> {
+    const material = await this.#dependencies.invocationKeys.resolve(
+      plan.tool.invocationArtifact.keyId,
+    );
+    if (!material || material.keyId !== plan.tool.invocationArtifact.keyId) {
+      return unavailable();
+    }
+    try {
+      const nonce = (
+        this.#dependencies.nonceFactory ?? defaultNonce
+      )(nonceInput(plan, material.key));
+      const inputArtifact = createToolInvocationInputArtifact(
+        {
+          artifactId: plan.tool.invocationArtifact.artifactId,
+          projectId: plan.projectId,
+          actionRef: plan.tool.actionRef,
+          requestedBy: plan.requestedBySubject,
+          tool: BUILTIN_RUN_LOG_EXCERPT_TOOL,
+          input: {
+            attemptId: plan.source.attemptId,
+            runId: plan.source.runId,
+          },
+          inputDigest: plan.tool.invocationArtifact.inputDigest,
+          invocationActionDigest: plan.tool.invocationActionDigest,
+          keyId: material.keyId,
+          key: material.key,
+          sealedAtMs: plan.tool.sealedAtMs,
+        },
+        () => nonce,
+      );
+      nonce.fill(0);
+      const previewArtifact = createToolInvocationPreviewArtifact({
+        artifactId: plan.tool.previewArtifact.artifactId,
+        projectId: plan.projectId,
+        actionRef: plan.tool.actionRef,
+        actionDigest: plan.tool.actionDigest,
+        redactionContractDigest:
+          plan.tool.previewArtifact.redactionContractDigest,
+        preview: preview(plan.source.runId, plan.source.attemptId),
+        sealedAtMs: plan.tool.sealedAtMs,
+      });
+      if (
+        inputArtifact.artifactDigest !==
+          plan.tool.invocationArtifact.artifactDigest ||
+        previewArtifact.artifactDigest !==
+          plan.tool.previewArtifact.artifactDigest
+      ) {
+        throw new CopilotFailureDiagnosisApplicationConflictError(
+          'durable Tool Artifact references cannot be reconstructed',
+        );
+      }
+      await this.#dependencies.artifacts.put(inputArtifact, previewArtifact);
+    } finally {
+      material.key.fill(0);
+    }
+  }
+
+  #clock(): number {
+    let value: number;
+    try {
+      value = (this.#dependencies.now ?? Date.now)();
+    } catch (cause) {
+      return unavailable(cause);
+    }
+    if (!Number.isSafeInteger(value) || value < 0) return invalid('clock');
+    if (value + this.#dependencies.executionTimeoutMs > Number.MAX_SAFE_INTEGER) {
+      return invalid('deadline overflows');
+    }
+    return value;
+  }
+}

packages/ql3-ai/src/copilot/failure-diagnosis/failureDiagnosisApplication.ts

@@ -0,0 +1,2 @@
+export * from './application/contracts';
+export * from './application/service';

packages/ql3-ai/src/copilot/failure-diagnosis/model-execution/coordinator.ts

@@ -5,7 +5,10 @@ import {
   type TrustedToolSuccessCompletionReadDependencies,
 } from '@qinglong/runtime-core/trusted-tool-completion';
 
-import { BoundedModelGateway } from '../../../model-gateway/gateway';
+import type {
+  BoundedModelGateway,
+  ModelInvocationSuccessfulCompletionSink,
+} from '../../../model-gateway/gateway';
 import type { ModelInvocationRepository } from '../../../model-invocation/modelInvocation';
 import type { CopilotFailureDiagnosisToolExecutionAdmissionReader } from '../tool-execution/contracts';
 import type { CopilotFailureDiagnosisToolUnlockRepository } from '../tool-execution/contracts';
@@ -36,20 +39,26 @@ export interface CopilotFailureDiagnosisModelExecutionDependencies {
     CopilotFailureDiagnosisOutputCompletionRepository,
     'findCopilotFailureDiagnosisOutput'
   >;
-  readonly gateway: BoundedModelGateway;
+  readonly gateway: Pick<
+    BoundedModelGateway,
+    'generate' | 'supportsSuccessfulCompletionSink'
+  >;
   readonly successfulCompletion: CopilotFailureDiagnosisModelCompletionCoordinator;
   readonly finalizations: CopilotFailureDiagnosisFinalizationRepository;
 }
 
 export interface CopilotFailureDiagnosisToolResultReader {
-  open(startId: string): Promise<Readonly<TrustedToolSuccessCompletionResult>>;
+  open(
+    requestId: string,
+    startId: string,
+  ): Promise<Readonly<TrustedToolSuccessCompletionResult>>;
 }
 
 export function createCopilotFailureDiagnosisToolResultReader(
   dependencies: TrustedToolSuccessCompletionReadDependencies,
 ): CopilotFailureDiagnosisToolResultReader {
   return Object.freeze({
-    open: (startId: string) =>
+    open: (_requestId: string, startId: string) =>
       openTrustedToolSuccessCompletion(startId, dependencies),
   });
 }
@@ -97,7 +106,8 @@ function assertDependencies(
     typeof value.modelInvocations?.findStart !== 'function' ||
     typeof value.modelInvocations?.findCompletion !== 'function' ||
     typeof value.outputs?.findCopilotFailureDiagnosisOutput !== 'function' ||
-    !(value.gateway instanceof BoundedModelGateway) ||
+    typeof value.gateway?.generate !== 'function' ||
+    typeof value.gateway?.supportsSuccessfulCompletionSink !== 'function' ||
     typeof value.successfulCompletion?.begin !== 'function' ||
     typeof value.successfulCompletion?.reference !== 'function' ||
     typeof value.successfulCompletion?.end !== 'function' ||
@@ -215,7 +225,7 @@ export async function executeCopilotFailureDiagnosisModel(
     );
   }
 
-  const tool = await dependencies.toolResults.open(unlock.startId);
+  const tool = await dependencies.toolResults.open(requestId, unlock.startId);
   if (
     tool.completion.completionDigest !== unlock.toolCompletionDigest ||
     tool.completion.runId !== plan.runId ||
@@ -241,7 +251,7 @@ export async function executeCopilotFailureDiagnosisModel(
   });
   if (
     !dependencies.gateway.supportsSuccessfulCompletionSink(
-      dependencies.successfulCompletion,
+      dependencies.successfulCompletion as ModelInvocationSuccessfulCompletionSink,
     )
   ) {
     throw new InvalidCopilotFailureDiagnosisModelExecutionError(

packages/ql3-ai/src/model-gateway/gateway.ts

@@ -39,6 +39,12 @@ import {
   type ModelPriceCatalogResolver,
 } from '../pricing/pricing';
 
+export {
+  InvalidModelInvocationSuccessfulCompletionRouterError,
+  MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS,
+  ModelInvocationSuccessfulCompletionRouter,
+} from './successfulCompletionRouter';
+
 export const MAX_MODEL_GATEWAY_CONCURRENCY = 64;
 
 export class ModelProviderUnavailableError extends Error {
@@ -126,6 +132,9 @@ export interface BoundedModelGatewayOptions {
 }
 
 export interface ModelInvocationSuccessfulCompletionSink {
+  supportsSuccessfulCompletionSink?(
+    sink: ModelInvocationSuccessfulCompletionSink,
+  ): boolean;
   record(
     audit: Readonly<ModelInvocationAuditRecord>,
     result: Readonly<GenerateResult>,
@@ -372,7 +381,11 @@ export class BoundedModelGateway {
   supportsSuccessfulCompletionSink(
     sink: ModelInvocationSuccessfulCompletionSink,
   ): boolean {
-    return this.#successfulCompletion === sink;
+    const configured = this.#successfulCompletion;
+    return (
+      configured === sink ||
+      configured?.supportsSuccessfulCompletionSink?.(sink) === true
+    );
   }
 
   async #prepare(

packages/ql3-ai/src/model-gateway/successfulCompletionRouter.ts

@@ -0,0 +1,80 @@
+import type { ModelInvocationSuccessfulCompletionSink } from './gateway';
+import type {
+  GenerateResult,
+  ModelInvocationAuditRecord,
+  ModelInvocationAuditResult,
+} from './model';
+
+export const MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS = 8;
+
+export class InvalidModelInvocationSuccessfulCompletionRouterError extends TypeError {
+  readonly code = 'MODEL_INVOCATION_SUCCESSFUL_COMPLETION_ROUTER_INVALID';
+
+  constructor() {
+    super('Model invocation successful completion router is invalid');
+    this.name = 'InvalidModelInvocationSuccessfulCompletionRouterError';
+  }
+}
+
+/** Bounded dispatch for mutually exclusive durable output domains. */
+export class ModelInvocationSuccessfulCompletionRouter
+  implements ModelInvocationSuccessfulCompletionSink
+{
+  readonly #sinks: readonly ModelInvocationSuccessfulCompletionSink[];
+
+  constructor(sinks: readonly ModelInvocationSuccessfulCompletionSink[]) {
+    if (
+      !Array.isArray(sinks) ||
+      sinks.length < 2 ||
+      sinks.length > MAX_MODEL_INVOCATION_SUCCESSFUL_COMPLETION_SINKS ||
+      new Set(sinks).size !== sinks.length ||
+      sinks.some(
+        (sink) =>
+          !sink ||
+          typeof sink !== 'object' ||
+          typeof sink.record !== 'function',
+      )
+    ) {
+      throw new InvalidModelInvocationSuccessfulCompletionRouterError();
+    }
+    this.#sinks = Object.freeze([...sinks]);
+    Object.freeze(this);
+  }
+
+  supportsSuccessfulCompletionSink(
+    sink: ModelInvocationSuccessfulCompletionSink,
+  ): boolean {
+    return this.#sinks.some(
+      (candidate) =>
+        candidate === sink ||
+        candidate.supportsSuccessfulCompletionSink?.(sink) === true,
+    );
+  }
+
+  async record(
+    audit: Readonly<ModelInvocationAuditRecord>,
+    result: Readonly<GenerateResult>,
+  ): Promise<
+    Readonly<
+      | { handled: false }
+      | { handled: true; disposition: ModelInvocationAuditResult }
+    >
+  > {
+    for (const sink of this.#sinks) {
+      const routed = await sink.record(audit, result);
+      if (
+        !routed ||
+        typeof routed !== 'object' ||
+        Array.isArray(routed) ||
+        (routed.handled !== true && routed.handled !== false)
+      ) {
+        throw new InvalidModelInvocationSuccessfulCompletionRouterError();
+      }
+      if (routed.handled) return routed;
+      if (Object.keys(routed).length !== 1) {
+        throw new InvalidModelInvocationSuccessfulCompletionRouterError();
+      }
+    }
+    return Object.freeze({ handled: false as const });
+  }
+}

packages/ql3-ai/src/prompt/postgres-plugin-package-prompt-application/bootstrap.ts

@@ -15,6 +15,10 @@ import {
 import type { PluginPackagePromptOutputReadService } from '../../prompt-output/pluginPackagePromptOutputRead';
 import type { PluginPackagePromptExecutionOutputReadService } from '../../prompt-output/pluginPackagePromptExecutionOutputRead';
 import { bootstrapModelGatewayProfile } from '../../profile/profileComposition';
+import {
+  ModelInvocationSuccessfulCompletionRouter,
+  type ModelInvocationSuccessfulCompletionSink,
+} from '../../model-gateway/gateway';
 import {
   PostgresPluginPackagePromptApplicationUnavailableError,
   unavailable,
@@ -49,6 +53,8 @@ function assertEnabledOptions(
     (options.confirmActive !== undefined &&
       typeof options.confirmActive !== 'function') ||
     (options.now !== undefined && typeof options.now !== 'function') ||
+    (options.createAdditionalSuccessfulCompletion !== undefined &&
+      typeof options.createAdditionalSuccessfulCompletion !== 'function') ||
     (options.promptOutputKeys !== undefined &&
       (!options.promptOutputKeys ||
         typeof options.promptOutputKeys !== 'object' ||
@@ -118,17 +124,36 @@ export async function bootstrapPostgresPluginPackagePromptApplication(
         });
       },
       loadProviders: options.loadProviders,
-      ...(options.promptOutputKeys === undefined
+      ...(options.promptOutputKeys === undefined &&
+      options.createAdditionalSuccessfulCompletion === undefined
         ? {}
         : {
             createSuccessfulCompletion: (coordinator) => {
-              durableOutput =
-                new PluginPackagePromptOutputCompletionCoordinator({
+              const sinks: ModelInvocationSuccessfulCompletionSink[] = [];
+              if (options.promptOutputKeys !== undefined) {
+                durableOutput =
+                  new PluginPackagePromptOutputCompletionCoordinator({
                   coordinator,
                   keys: options.promptOutputKeys!,
                   ...(options.now === undefined ? {} : { now: options.now }),
                 });
-              return durableOutput;
+                sinks.push(durableOutput);
+              }
+              if (options.createAdditionalSuccessfulCompletion !== undefined) {
+                const additionalSuccessfulCompletion =
+                  options.createAdditionalSuccessfulCompletion(coordinator);
+                if (
+                  !additionalSuccessfulCompletion ||
+                  typeof additionalSuccessfulCompletion.record !== 'function'
+                ) {
+                  throw new TypeError(
+                    'PostgreSQL Package Prompt additional completion is invalid',
+                  );
+                }
+                sinks.push(additionalSuccessfulCompletion);
+              }
+              if (sinks.length === 1) return sinks[0]!;
+              return new ModelInvocationSuccessfulCompletionRouter(sinks);
             },
           }),
       audit: options.audit,

packages/ql3-ai/src/prompt/postgres-plugin-package-prompt-application/contracts.ts

@@ -21,6 +21,8 @@ import type {
   ModelGatewayProfileAudit,
   ModelGatewayProviderAuthority,
 } from '../../profile/profileComposition';
+import type { DurableModelInvocationCoordinator } from '../../model-invocation/durableModelInvocationCoordinator';
+import type { ModelInvocationSuccessfulCompletionSink } from '../../model-gateway/gateway';
 import type { PluginPackagePromptCatalogCapability } from '../pluginPackagePromptCatalog';
 import type { PluginPackagePromptExecutionInspectionRepository } from '../pluginPackagePromptExecutionInspection';
 
@@ -80,6 +82,9 @@ export type BootstrapPostgresPluginPackagePromptApplicationOptions =
       maxConcurrent?: number;
       recoveryLimit?: number;
       now?: () => number;
+      createAdditionalSuccessfulCompletion?: (
+        coordinator: DurableModelInvocationCoordinator,
+      ) => ModelInvocationSuccessfulCompletionSink;
       promptOutputKeys?: PluginPackagePromptOutputArtifactKeyProvider;
       promptOutputRead?: Readonly<{
         authorizer: PluginPackagePromptOutputArtifactReadAuthorizer;