mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): compose cluster copilot diagnosis
This commit is contained in:
whyour
@@ -35,6 +35,11 @@
|
||||
"require": "./dist/application-runtime/aiProductionApplication.js",
|
||||
"default": "./dist/application-runtime/aiProductionApplication.js"
|
||||
},
|
||||
"./copilot-production": {
|
||||
"types": "./dist/application-runtime/copilot/failureDiagnosisComposition.d.ts",
|
||||
"require": "./dist/application-runtime/copilot/failureDiagnosisComposition.js",
|
||||
"default": "./dist/application-runtime/copilot/failureDiagnosisComposition.js"
|
||||
},
|
||||
"./failure-diagnosis-output-keyring": {
|
||||
"types": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.d.ts",
|
||||
"require": "./dist/copilot/failure-diagnosis/outputProjectedKeyring.js",
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile';
|
||||
import type { DurableModelInvocationCoordinator } from '@qinglong/ai/durable-model-invocation';
|
||||
import { CopilotFailureDiagnosisModelCompletionCoordinator } from '@qinglong/ai/failure-diagnosis-model-execution';
|
||||
import type { CopilotFailureDiagnosisApplicationService } from '@qinglong/ai/failure-diagnosis-application';
|
||||
import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential';
|
||||
import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage';
|
||||
import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority';
|
||||
@@ -22,12 +25,19 @@ import {
|
||||
startProductionClusterControlApplication,
|
||||
type ProductionClusterControlApplicationOptions,
|
||||
} from './productionApplication';
|
||||
import {
|
||||
createProductionClusterCopilotFailureDiagnosis,
|
||||
prepareProductionClusterCopilotFailureDiagnosisProjection,
|
||||
type ClusterCopilotFailureDiagnosisProjection,
|
||||
type CreateProductionClusterCopilotFailureDiagnosisOptions,
|
||||
} from './copilot/failureDiagnosisComposition';
|
||||
|
||||
export interface EnabledProductionClusterAiConfig {
|
||||
readonly enabled: true;
|
||||
readonly providerAuthorityFile: string;
|
||||
readonly secretRootDirectory: string;
|
||||
readonly promptOutputKeyringRootDirectory?: string;
|
||||
readonly copilot?: Readonly<ClusterCopilotFailureDiagnosisProjection>;
|
||||
readonly maxConcurrent: number;
|
||||
readonly recoveryLimit: number;
|
||||
readonly databaseMaxConnections: number;
|
||||
@@ -41,8 +51,18 @@ export interface ProductionClusterAiControlApplicationOptions {
|
||||
) => void | Promise<void>;
|
||||
readonly startControl?: typeof startProductionClusterControlApplication;
|
||||
readonly bootstrapPrompt?: typeof bootstrapPostgresPluginPackagePromptApplication;
|
||||
readonly createCopilot?: (
|
||||
options: CreateProductionClusterCopilotFailureDiagnosisOptions,
|
||||
) => Promise<Readonly<CopilotFailureDiagnosisApplicationService>>;
|
||||
readonly openAiDatabase?: ReturnType<typeof createPostgresDatabaseOpener>;
|
||||
}
|
||||
|
||||
export type ProductionClusterAiControlApplicationResult = Extract<
|
||||
ClusterControlApplicationResult,
|
||||
{ readonly status: 'active' }
|
||||
> &
|
||||
Readonly<{ copilot?: Readonly<CopilotFailureDiagnosisApplicationService> }>;
|
||||
|
||||
export class ProductionClusterAiConfigError extends TypeError {
|
||||
readonly code = 'QL3_CLUSTER_AI_CONFIG_INVALID';
|
||||
|
||||
@@ -121,6 +141,11 @@ export function loadProductionClusterAiConfig(
|
||||
'QL3_CLUSTER_AI_PROMPT_OUTPUT_ENABLED',
|
||||
false,
|
||||
);
|
||||
const copilotEnabled = booleanValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_COPILOT_ENABLED',
|
||||
false,
|
||||
);
|
||||
return Object.freeze({
|
||||
enabled: true,
|
||||
providerAuthorityFile: requiredPath(
|
||||
@@ -139,6 +164,28 @@ export function loadProductionClusterAiConfig(
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
...(copilotEnabled
|
||||
? {
|
||||
copilot: Object.freeze({
|
||||
configFile: requiredPath(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_COPILOT_CONFIG_FILE',
|
||||
),
|
||||
invocationKeyringRootDirectory: requiredPath(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_COPILOT_INVOCATION_KEYRING_ROOT',
|
||||
),
|
||||
resultKeyringRootDirectory: requiredPath(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_COPILOT_RESULT_KEYRING_ROOT',
|
||||
),
|
||||
outputKeyringRootDirectory: requiredPath(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_COPILOT_OUTPUT_KEYRING_ROOT',
|
||||
),
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
maxConcurrent: boundedInteger(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_MAX_CONCURRENT',
|
||||
@@ -187,7 +234,7 @@ function aiDatabaseOpener(
|
||||
*/
|
||||
export async function startProductionClusterAiControlApplication(
|
||||
options: ProductionClusterAiControlApplicationOptions,
|
||||
): Promise<ClusterControlApplicationResult> {
|
||||
): Promise<ProductionClusterAiControlApplicationResult> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
@@ -200,9 +247,26 @@ export async function startProductionClusterAiControlApplication(
|
||||
options.startControl ?? startProductionClusterControlApplication;
|
||||
const bootstrapPrompt =
|
||||
options.bootstrapPrompt ?? bootstrapPostgresPluginPackagePromptApplication;
|
||||
if (typeof startControl !== 'function' || typeof bootstrapPrompt !== 'function') {
|
||||
const createCopilot =
|
||||
options.createCopilot ?? createProductionClusterCopilotFailureDiagnosis;
|
||||
if (
|
||||
typeof startControl !== 'function' ||
|
||||
typeof bootstrapPrompt !== 'function' ||
|
||||
typeof createCopilot !== 'function' ||
|
||||
(options.openAiDatabase !== undefined &&
|
||||
typeof options.openAiDatabase !== 'function')
|
||||
) {
|
||||
throw new TypeError('Production Cluster AI application factories are invalid');
|
||||
}
|
||||
const copilotArtifactStore = options.control.workerIngress?.artifactStore;
|
||||
if (
|
||||
options.ai.copilot !== undefined &&
|
||||
typeof copilotArtifactStore?.readLogRange !== 'function'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster Copilot requires the bounded Worker log Artifact read capability',
|
||||
);
|
||||
}
|
||||
const secretMaterial =
|
||||
await createProjectedModelProviderSecretMaterialProvider({
|
||||
rootDirectory: options.ai.secretRootDirectory,
|
||||
@@ -213,6 +277,12 @@ export async function startProductionClusterAiControlApplication(
|
||||
: await createPluginPackagePromptOutputProjectedKeyring({
|
||||
rootDirectory: options.ai.promptOutputKeyringRootDirectory,
|
||||
});
|
||||
const preparedCopilot =
|
||||
options.ai.copilot === undefined
|
||||
? undefined
|
||||
: await prepareProductionClusterCopilotFailureDiagnosisProjection(
|
||||
options.ai.copilot,
|
||||
);
|
||||
let aiDatabase:
|
||||
| Awaited<ReturnType<ReturnType<typeof createPostgresDatabaseOpener>>>
|
||||
| undefined;
|
||||
@@ -230,6 +300,12 @@ export async function startProductionClusterAiControlApplication(
|
||||
| BootstrapPostgresPluginPackagePromptApplicationResult
|
||||
| undefined;
|
||||
let controlApplication: ClusterControlApplicationResult | undefined;
|
||||
let copilotApplication:
|
||||
| Readonly<CopilotFailureDiagnosisApplicationService>
|
||||
| undefined;
|
||||
let copilotSuccessfulCompletion:
|
||||
| CopilotFailureDiagnosisModelCompletionCoordinator
|
||||
| undefined;
|
||||
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
let promptOutputPolicy: ProjectPolicyEngine | undefined;
|
||||
const promptOutputReadAuthorizer = Object.freeze({
|
||||
@@ -271,11 +347,9 @@ export async function startProductionClusterAiControlApplication(
|
||||
return stopPromise;
|
||||
};
|
||||
try {
|
||||
const openDatabase = aiDatabaseOpener(
|
||||
options.control.config,
|
||||
options.ai,
|
||||
onAiUnavailable,
|
||||
);
|
||||
const openDatabase =
|
||||
options.openAiDatabase ??
|
||||
aiDatabaseOpener(options.control.config, options.ai, onAiUnavailable);
|
||||
promptApplication = await bootstrapPrompt({
|
||||
enabled: true,
|
||||
async openDatabase() {
|
||||
@@ -311,10 +385,41 @@ export async function startProductionClusterAiControlApplication(
|
||||
promptOutputKeys,
|
||||
promptOutputRead: { authorizer: promptOutputReadAuthorizer },
|
||||
}),
|
||||
...(preparedCopilot === undefined
|
||||
? {}
|
||||
: {
|
||||
createAdditionalSuccessfulCompletion(
|
||||
coordinator: DurableModelInvocationCoordinator,
|
||||
) {
|
||||
if (copilotSuccessfulCompletion) {
|
||||
throw new Error(
|
||||
'Cluster Copilot completion was created more than once',
|
||||
);
|
||||
}
|
||||
copilotSuccessfulCompletion =
|
||||
new CopilotFailureDiagnosisModelCompletionCoordinator({
|
||||
coordinator,
|
||||
keys: preparedCopilot.outputKeys,
|
||||
});
|
||||
return copilotSuccessfulCompletion;
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (promptApplication.status !== 'active') {
|
||||
throw new Error('Cluster AI Prompt application did not activate');
|
||||
}
|
||||
if (preparedCopilot !== undefined) {
|
||||
if (!copilotSuccessfulCompletion || !aiDatabase || !copilotArtifactStore) {
|
||||
throw new Error('Cluster Copilot shared authorities did not activate');
|
||||
}
|
||||
copilotApplication = await createCopilot({
|
||||
pool: aiDatabase.pool,
|
||||
gateway: promptApplication.capability,
|
||||
prepared: preparedCopilot,
|
||||
successfulCompletion: copilotSuccessfulCompletion,
|
||||
artifactStore: copilotArtifactStore,
|
||||
});
|
||||
}
|
||||
controlApplication = await startControl({
|
||||
...options.control,
|
||||
promptCatalog: {
|
||||
@@ -356,6 +461,9 @@ export async function startProductionClusterAiControlApplication(
|
||||
? 'unavailable'
|
||||
: activeControl.availabilityStatus();
|
||||
},
|
||||
...(copilotApplication === undefined
|
||||
? {}
|
||||
: { copilot: copilotApplication }),
|
||||
stop,
|
||||
});
|
||||
} catch (error) {
|
||||
|
||||
+466
@@ -0,0 +1,466 @@
|
||||
import { Buffer } from 'node:buffer';
|
||||
import { basename, dirname } from 'node:path';
|
||||
|
||||
import {
|
||||
CopilotFailureDiagnosisApplicationService,
|
||||
type CopilotFailureDiagnosisApplicationDependencies,
|
||||
} from '@qinglong/ai/failure-diagnosis-application';
|
||||
import type { PrepareCopilotFailureDiagnosisModelIntent } from '@qinglong/ai/failure-diagnosis-execution-admission';
|
||||
import {
|
||||
CopilotFailureDiagnosisModelCompletionCoordinator,
|
||||
executeCopilotFailureDiagnosisModel,
|
||||
type CopilotFailureDiagnosisToolResultReader,
|
||||
} from '@qinglong/ai/failure-diagnosis-model-execution';
|
||||
import {
|
||||
executeCopilotFailureDiagnosisTool,
|
||||
restoreCopilotFailureDiagnosisTrustedToolAuthority,
|
||||
} from '@qinglong/ai/failure-diagnosis-tool-execution';
|
||||
import { PostgresCopilotFailureDiagnosisAdmissionRepository } from '@qinglong/ai/postgres-failure-diagnosis-admission-storage';
|
||||
import { PostgresCopilotFailureDiagnosisModelRepository } from '@qinglong/ai/postgres-failure-diagnosis-model-execution-storage';
|
||||
import { PostgresCopilotFailureDiagnosisToolUnlockRepository } from '@qinglong/ai/postgres-failure-diagnosis-tool-execution-storage';
|
||||
import type { ActiveModelGatewayCapability } from '@qinglong/ai/profile';
|
||||
import {
|
||||
PostgresProjectPolicyRepository,
|
||||
PostgresProjectToolDefinitionSnapshotRepository,
|
||||
PostgresRunAttemptLogRetentionClaimRepository,
|
||||
PostgresRunRepository,
|
||||
PostgresStepRunRepository,
|
||||
PostgresToolExecutionCompletionRepository,
|
||||
PostgresToolExecutionFailureCompletionRepository,
|
||||
PostgresToolExecutionStartBarrierRepository,
|
||||
PostgresToolInvocationArtifactRepository,
|
||||
PostgresToolResultKeyCatalogReader,
|
||||
PostgresToolResultRekeyReader,
|
||||
type QingLongPostgresPool,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
import {
|
||||
BuiltInRunLogExcerptToolAdapter,
|
||||
} from '@qinglong/runtime-core/builtin-run-log-excerpt-tool';
|
||||
import type { RunAttemptLogReadPort } from '@qinglong/runtime-core/builtin-run-log-excerpt-projection';
|
||||
import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
|
||||
import { RunAttemptLogReadService } from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
import { openTrustedToolSuccessCompletion } from '@qinglong/runtime-core/trusted-tool-completion';
|
||||
import { TrustedToolExecutionAdapterRegistry } from '@qinglong/runtime-core/trusted-tool-execution';
|
||||
|
||||
import type { ClusterRemoteWorkerArtifactStore } from '../../remote-execution/remoteWorkerCompletionService';
|
||||
import { PrivateProjectedFileReader } from '../../security/privateProjectedFile';
|
||||
import { createClusterToolInvocationProjectedKeyring } from '../../trusted-tool/key-management/toolInvocationProjectedKeyring';
|
||||
import { createClusterToolResultProjectedKeyring } from '../../trusted-tool/key-management/toolResultProjectedKeyring';
|
||||
import { createClusterCopilotFailureDiagnosisOutputProjectedKeyring } from '../../copilot/failure-diagnosis/outputProjectedKeyring';
|
||||
|
||||
export const CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA =
|
||||
'qinglong/cluster-copilot-failure-diagnosis-config@v1' as const;
|
||||
export const MAX_CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_BYTES = 16 * 1024;
|
||||
|
||||
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const MODEL_BOUNDARIES = ['on_device', 'external'] as const;
|
||||
const RESPONSE_LANGUAGES = ['en', 'zh-CN'] as const;
|
||||
const EGRESS_SCHEMA = 'qinglong/copilot-model-egress-policy@v1' as const;
|
||||
|
||||
export interface ClusterCopilotFailureDiagnosisConfig {
|
||||
readonly schema: typeof CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA;
|
||||
readonly provider: string;
|
||||
readonly model: string;
|
||||
readonly modelBoundary: 'on_device' | 'external';
|
||||
readonly responseLanguage: 'en' | 'zh-CN';
|
||||
readonly maxOutputTokens: number;
|
||||
readonly executionTimeoutMs: number;
|
||||
readonly egressPolicy: PrepareCopilotFailureDiagnosisModelIntent['egressPolicy'];
|
||||
}
|
||||
|
||||
export interface ClusterCopilotFailureDiagnosisProjection {
|
||||
readonly configFile: string;
|
||||
readonly invocationKeyringRootDirectory: string;
|
||||
readonly resultKeyringRootDirectory: string;
|
||||
readonly outputKeyringRootDirectory: string;
|
||||
}
|
||||
|
||||
export interface CreateProductionClusterCopilotFailureDiagnosisOptions {
|
||||
readonly pool: QingLongPostgresPool;
|
||||
readonly gateway: ActiveModelGatewayCapability;
|
||||
readonly prepared: PreparedClusterCopilotFailureDiagnosisProjection;
|
||||
readonly successfulCompletion: CopilotFailureDiagnosisModelCompletionCoordinator;
|
||||
readonly artifactStore: ClusterRemoteWorkerArtifactStore;
|
||||
}
|
||||
|
||||
export interface PreparedClusterCopilotFailureDiagnosisProjection {
|
||||
readonly config: Readonly<ClusterCopilotFailureDiagnosisConfig>;
|
||||
readonly invocationKeys: Awaited<
|
||||
ReturnType<typeof createClusterToolInvocationProjectedKeyring>
|
||||
>;
|
||||
readonly resultKeys: Awaited<
|
||||
ReturnType<typeof createClusterToolResultProjectedKeyring>
|
||||
>;
|
||||
readonly outputKeys: Awaited<
|
||||
ReturnType<typeof createClusterCopilotFailureDiagnosisOutputProjectedKeyring>
|
||||
>;
|
||||
}
|
||||
|
||||
export class ClusterCopilotFailureDiagnosisCompositionError extends Error {
|
||||
readonly code = 'QL3_CLUSTER_COPILOT_FAILURE_DIAGNOSIS_COMPOSITION_INVALID';
|
||||
|
||||
constructor(message: string, options?: ErrorOptions) {
|
||||
super(`Cluster Copilot failure diagnosis composition is invalid: ${message}`, options);
|
||||
this.name = 'ClusterCopilotFailureDiagnosisCompositionError';
|
||||
}
|
||||
}
|
||||
|
||||
function invalid(message: string, cause?: unknown): never {
|
||||
throw new ClusterCopilotFailureDiagnosisCompositionError(message, {
|
||||
cause: cause instanceof Error ? cause : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function record(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
return invalid(`${label} must be an object`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function exactKeys(
|
||||
value: Readonly<Record<string, unknown>>,
|
||||
expected: readonly string[],
|
||||
label: string,
|
||||
): void {
|
||||
const actual = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
actual.length !== canonical.length ||
|
||||
actual.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
return invalid(`${label} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function identity(value: unknown, label: string): string {
|
||||
if (typeof value !== 'string' || !IDENTITY.test(value)) {
|
||||
return invalid(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function integer(
|
||||
value: unknown,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
label: string,
|
||||
): number {
|
||||
if (
|
||||
typeof value !== 'number' ||
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < minimum ||
|
||||
value > maximum
|
||||
) {
|
||||
return invalid(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizeClusterCopilotFailureDiagnosisConfig(
|
||||
value: unknown,
|
||||
): Readonly<ClusterCopilotFailureDiagnosisConfig> {
|
||||
const candidate = record(value, 'configuration');
|
||||
exactKeys(
|
||||
candidate,
|
||||
[
|
||||
'egressPolicy',
|
||||
'executionTimeoutMs',
|
||||
'maxOutputTokens',
|
||||
'model',
|
||||
'modelBoundary',
|
||||
'provider',
|
||||
'responseLanguage',
|
||||
'schema',
|
||||
],
|
||||
'configuration',
|
||||
);
|
||||
if (candidate.schema !== CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA) {
|
||||
return invalid('schema is invalid');
|
||||
}
|
||||
if (!MODEL_BOUNDARIES.includes(candidate.modelBoundary as never)) {
|
||||
return invalid('model boundary is invalid');
|
||||
}
|
||||
if (!RESPONSE_LANGUAGES.includes(candidate.responseLanguage as never)) {
|
||||
return invalid('response language is invalid');
|
||||
}
|
||||
const egress = record(candidate.egressPolicy, 'egress policy');
|
||||
exactKeys(
|
||||
egress,
|
||||
[
|
||||
'maxInputBytes',
|
||||
'maxOutputTokens',
|
||||
'potentiallySensitiveDataBoundaries',
|
||||
'revision',
|
||||
'schema',
|
||||
],
|
||||
'egress policy',
|
||||
);
|
||||
if (egress.schema !== EGRESS_SCHEMA) return invalid('egress schema is invalid');
|
||||
const selected = egress.potentiallySensitiveDataBoundaries;
|
||||
if (
|
||||
!Array.isArray(selected) ||
|
||||
selected.length < 1 ||
|
||||
selected.length > MODEL_BOUNDARIES.length ||
|
||||
selected.some((entry) => !MODEL_BOUNDARIES.includes(entry as never)) ||
|
||||
new Set(selected).size !== selected.length ||
|
||||
MODEL_BOUNDARIES.filter((entry) => selected.includes(entry)).some(
|
||||
(entry, index) => entry !== selected[index],
|
||||
) ||
|
||||
!selected.includes(candidate.modelBoundary)
|
||||
) {
|
||||
return invalid('egress model boundaries are invalid');
|
||||
}
|
||||
const egressMaxOutputTokens = integer(
|
||||
egress.maxOutputTokens,
|
||||
1,
|
||||
4_096,
|
||||
'egress max output tokens',
|
||||
);
|
||||
const maxOutputTokens = integer(
|
||||
candidate.maxOutputTokens,
|
||||
1,
|
||||
egressMaxOutputTokens,
|
||||
'max output tokens',
|
||||
);
|
||||
return Object.freeze({
|
||||
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA,
|
||||
provider: identity(candidate.provider, 'provider'),
|
||||
model: identity(candidate.model, 'model'),
|
||||
modelBoundary: candidate.modelBoundary as 'on_device' | 'external',
|
||||
responseLanguage: candidate.responseLanguage as 'en' | 'zh-CN',
|
||||
maxOutputTokens,
|
||||
executionTimeoutMs: integer(
|
||||
candidate.executionTimeoutMs,
|
||||
1,
|
||||
5 * 60_000,
|
||||
'execution timeout',
|
||||
),
|
||||
egressPolicy: Object.freeze({
|
||||
schema: EGRESS_SCHEMA,
|
||||
revision: identity(egress.revision, 'egress revision'),
|
||||
potentiallySensitiveDataBoundaries: Object.freeze([...selected]) as (
|
||||
| 'on_device'
|
||||
| 'external'
|
||||
)[],
|
||||
maxInputBytes: integer(
|
||||
egress.maxInputBytes,
|
||||
1,
|
||||
64 * 1024,
|
||||
'egress max input bytes',
|
||||
),
|
||||
maxOutputTokens: egressMaxOutputTokens,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function canonicalClusterCopilotFailureDiagnosisConfig(
|
||||
value: unknown,
|
||||
): Buffer {
|
||||
return Buffer.from(
|
||||
`${JSON.stringify(normalizeClusterCopilotFailureDiagnosisConfig(value))}\n`,
|
||||
'utf8',
|
||||
);
|
||||
}
|
||||
|
||||
export async function loadClusterCopilotFailureDiagnosisConfig(
|
||||
configFile: string,
|
||||
): Promise<Readonly<ClusterCopilotFailureDiagnosisConfig>> {
|
||||
let bytes: Buffer | undefined;
|
||||
let canonical: Buffer | undefined;
|
||||
try {
|
||||
const reader = new PrivateProjectedFileReader({
|
||||
rootDirectory: dirname(configFile),
|
||||
minimumBytes: 1,
|
||||
maximumBytes: MAX_CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_BYTES,
|
||||
access: 'read_only_keyring',
|
||||
});
|
||||
bytes = await reader.read(basename(configFile));
|
||||
const parsed = JSON.parse(bytes.toString('utf8')) as unknown;
|
||||
const config = normalizeClusterCopilotFailureDiagnosisConfig(parsed);
|
||||
canonical = canonicalClusterCopilotFailureDiagnosisConfig(config);
|
||||
if (!canonical.equals(bytes)) return invalid('file is not canonical');
|
||||
return config;
|
||||
} catch (cause) {
|
||||
return cause instanceof ClusterCopilotFailureDiagnosisCompositionError
|
||||
? invalid(cause.message, cause)
|
||||
: invalid('projected configuration is unavailable', cause);
|
||||
} finally {
|
||||
bytes?.fill(0);
|
||||
canonical?.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
function modelIntent(
|
||||
config: Readonly<ClusterCopilotFailureDiagnosisConfig>,
|
||||
): Readonly<PrepareCopilotFailureDiagnosisModelIntent> {
|
||||
return Object.freeze({
|
||||
provider: config.provider,
|
||||
model: config.model,
|
||||
modelBoundary: config.modelBoundary,
|
||||
responseLanguage: config.responseLanguage,
|
||||
maxOutputTokens: config.maxOutputTokens,
|
||||
egressPolicy: config.egressPolicy,
|
||||
});
|
||||
}
|
||||
|
||||
export async function prepareProductionClusterCopilotFailureDiagnosisProjection(
|
||||
projection: ClusterCopilotFailureDiagnosisProjection,
|
||||
): Promise<Readonly<PreparedClusterCopilotFailureDiagnosisProjection>> {
|
||||
if (!projection || typeof projection !== 'object' || Array.isArray(projection)) {
|
||||
return invalid('projection is invalid');
|
||||
}
|
||||
const [config, invocationKeys, resultKeys, outputKeys] = await Promise.all([
|
||||
loadClusterCopilotFailureDiagnosisConfig(projection.configFile),
|
||||
createClusterToolInvocationProjectedKeyring({
|
||||
rootDirectory: projection.invocationKeyringRootDirectory,
|
||||
}),
|
||||
createClusterToolResultProjectedKeyring({
|
||||
rootDirectory: projection.resultKeyringRootDirectory,
|
||||
}),
|
||||
createClusterCopilotFailureDiagnosisOutputProjectedKeyring({
|
||||
rootDirectory: projection.outputKeyringRootDirectory,
|
||||
}),
|
||||
]);
|
||||
return Object.freeze({ config, invocationKeys, resultKeys, outputKeys });
|
||||
}
|
||||
|
||||
export async function createProductionClusterCopilotFailureDiagnosis(
|
||||
options: CreateProductionClusterCopilotFailureDiagnosisOptions,
|
||||
): Promise<Readonly<CopilotFailureDiagnosisApplicationService>> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
typeof options.pool?.query !== 'function' ||
|
||||
typeof options.pool?.connect !== 'function' ||
|
||||
typeof options.gateway?.generate !== 'function' ||
|
||||
typeof options.gateway?.supportsSuccessfulCompletionSink !== 'function' ||
|
||||
typeof options.successfulCompletion?.begin !== 'function' ||
|
||||
typeof options.successfulCompletion?.record !== 'function' ||
|
||||
!options.prepared ||
|
||||
typeof options.artifactStore?.readLogRange !== 'function'
|
||||
) {
|
||||
return invalid('dependencies are unavailable');
|
||||
}
|
||||
if (
|
||||
!options.gateway.supportsSuccessfulCompletionSink(
|
||||
options.successfulCompletion,
|
||||
)
|
||||
) {
|
||||
return invalid('shared Model completion authority is unavailable');
|
||||
}
|
||||
const { config, invocationKeys, resultKeys } = options.prepared;
|
||||
const admissions = new PostgresCopilotFailureDiagnosisAdmissionRepository(
|
||||
options.pool,
|
||||
);
|
||||
const snapshots = new PostgresProjectToolDefinitionSnapshotRepository(
|
||||
options.pool,
|
||||
);
|
||||
const runs = new PostgresRunRepository(options.pool);
|
||||
const artifacts = new PostgresToolInvocationArtifactRepository(options.pool);
|
||||
const stepRuns = new PostgresStepRunRepository(options.pool);
|
||||
const barriers = new PostgresToolExecutionStartBarrierRepository(options.pool);
|
||||
const completions = new PostgresToolExecutionCompletionRepository(options.pool);
|
||||
const failureCompletions =
|
||||
new PostgresToolExecutionFailureCompletionRepository(options.pool);
|
||||
const resultKeyCatalog = new PostgresToolResultKeyCatalogReader(options.pool);
|
||||
const resultRekeys = new PostgresToolResultRekeyReader(options.pool);
|
||||
const unlocks = new PostgresCopilotFailureDiagnosisToolUnlockRepository(
|
||||
options.pool,
|
||||
);
|
||||
const models = new PostgresCopilotFailureDiagnosisModelRepository(options.pool);
|
||||
const logReader = new RunAttemptLogReadService(
|
||||
runs,
|
||||
Object.freeze({
|
||||
read: options.artifactStore.readLogRange.bind(options.artifactStore),
|
||||
}),
|
||||
{
|
||||
executorType: 'remote_worker',
|
||||
artifactIdPattern: /^wlog-[a-f0-9]{30}$/,
|
||||
maximumReadBytes: 256 * 1024,
|
||||
activeMissingIsPending: true,
|
||||
},
|
||||
new PostgresRunAttemptLogRetentionClaimRepository(options.pool),
|
||||
);
|
||||
const logs: RunAttemptLogReadPort = Object.freeze({
|
||||
read: logReader.read.bind(logReader),
|
||||
});
|
||||
const successfulCompletion = options.successfulCompletion;
|
||||
const toolResults: CopilotFailureDiagnosisToolResultReader = Object.freeze({
|
||||
async open(requestId: string, startId: string) {
|
||||
const plan = await admissions.findPlanByRequestId(requestId);
|
||||
if (!plan) return invalid('diagnosis plan is unavailable');
|
||||
const snapshot = await snapshots.findCurrent(plan.projectId);
|
||||
if (!snapshot) return invalid('Tool snapshot is unavailable');
|
||||
const authority = restoreCopilotFailureDiagnosisTrustedToolAuthority(
|
||||
plan,
|
||||
snapshot.snapshot,
|
||||
);
|
||||
const definitions = authority.bindings.definitionRegistry();
|
||||
const adapters = new TrustedToolExecutionAdapterRegistry(
|
||||
authority.bindings,
|
||||
[
|
||||
new BuiltInRunLogExcerptToolAdapter(
|
||||
authority.binding,
|
||||
'cluster-control',
|
||||
definitions,
|
||||
logs,
|
||||
),
|
||||
],
|
||||
);
|
||||
return openTrustedToolSuccessCompletion(startId, {
|
||||
completions,
|
||||
barriers,
|
||||
resultKeyCatalog,
|
||||
resultRekeys,
|
||||
resultKeys,
|
||||
adapters,
|
||||
});
|
||||
},
|
||||
});
|
||||
const policy = new ProjectPolicyEngine(
|
||||
new PostgresProjectPolicyRepository(options.pool),
|
||||
);
|
||||
const tool = Object.freeze({
|
||||
admissions,
|
||||
snapshots,
|
||||
artifacts,
|
||||
invocationKeys,
|
||||
resultKeys,
|
||||
stepRuns,
|
||||
runs,
|
||||
barriers,
|
||||
completions,
|
||||
failureCompletions,
|
||||
resultKeyCatalog,
|
||||
resultRekeys,
|
||||
logs,
|
||||
unlocks,
|
||||
});
|
||||
const model = Object.freeze({
|
||||
admissions,
|
||||
unlocks,
|
||||
toolResults,
|
||||
modelInvocations: models,
|
||||
outputs: models,
|
||||
gateway: options.gateway,
|
||||
successfulCompletion,
|
||||
finalizations: models,
|
||||
});
|
||||
const dependencies: CopilotFailureDiagnosisApplicationDependencies = {
|
||||
admissions,
|
||||
snapshots,
|
||||
runs,
|
||||
artifacts,
|
||||
invocationKeys,
|
||||
authorizer: policy,
|
||||
tool,
|
||||
model,
|
||||
executeTool: executeCopilotFailureDiagnosisTool,
|
||||
executeModel: executeCopilotFailureDiagnosisModel,
|
||||
modelIntent: modelIntent(config),
|
||||
executionTimeoutMs: config.executionTimeoutMs,
|
||||
};
|
||||
return new CopilotFailureDiagnosisApplicationService(dependencies);
|
||||
}
|
||||
@@ -13,6 +13,18 @@ const {
|
||||
canonicalPluginPackagePromptOutputKeyringManifest,
|
||||
PLUGIN_PACKAGE_PROMPT_OUTPUT_KEYRING_MANIFEST_SCHEMA,
|
||||
} = require('@qinglong/ai/plugin-package-prompt-output-keyring-manifest');
|
||||
const {
|
||||
canonicalClusterToolInvocationKeyringManifest,
|
||||
CLUSTER_TOOL_INVOCATION_KEYRING_MANIFEST_SCHEMA,
|
||||
} = require('@qinglong/cluster-control/trusted-tool-invocation-keyring');
|
||||
const {
|
||||
canonicalClusterToolResultKeyringManifest,
|
||||
CLUSTER_TOOL_RESULT_KEYRING_MANIFEST_SCHEMA,
|
||||
} = require('@qinglong/cluster-control/trusted-tool-result-keyring');
|
||||
const {
|
||||
canonicalClusterCopilotFailureDiagnosisOutputKeyringManifest,
|
||||
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_KEYRING_MANIFEST_SCHEMA,
|
||||
} = require('@qinglong/cluster-control/failure-diagnosis-output-keyring');
|
||||
|
||||
function enabledEnvironment(overrides = {}) {
|
||||
return {
|
||||
@@ -74,6 +86,161 @@ test('AI config is fail-closed and bounded behind the explicit process flag', ()
|
||||
),
|
||||
/QL3_CLUSTER_AI_PROMPT_OUTPUT_KEYRING_ROOT is invalid/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
loadProductionClusterAiConfig(
|
||||
enabledEnvironment({ QL3_CLUSTER_AI_COPILOT_ENABLED: 'true' }),
|
||||
),
|
||||
/QL3_CLUSTER_AI_COPILOT_CONFIG_FILE is invalid/,
|
||||
);
|
||||
assert.deepEqual(
|
||||
loadProductionClusterAiConfig(
|
||||
enabledEnvironment({
|
||||
QL3_CLUSTER_AI_COPILOT_ENABLED: 'true',
|
||||
QL3_CLUSTER_AI_COPILOT_CONFIG_FILE: '/run/ql3/copilot/config.json',
|
||||
QL3_CLUSTER_AI_COPILOT_INVOCATION_KEYRING_ROOT: '/run/ql3/invocation',
|
||||
QL3_CLUSTER_AI_COPILOT_RESULT_KEYRING_ROOT: '/run/ql3/result',
|
||||
QL3_CLUSTER_AI_COPILOT_OUTPUT_KEYRING_ROOT: '/run/ql3/output',
|
||||
}),
|
||||
).copilot,
|
||||
{
|
||||
configFile: '/run/ql3/copilot/config.json',
|
||||
invocationKeyringRootDirectory: '/run/ql3/invocation',
|
||||
resultKeyringRootDirectory: '/run/ql3/result',
|
||||
outputKeyringRootDirectory: '/run/ql3/output',
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
async function projectedFile(root, name, bytes) {
|
||||
await writeFile(join(root, name), bytes, { mode: 0o440 });
|
||||
await chmod(join(root, name), 0o440);
|
||||
}
|
||||
|
||||
test('Copilot composition is explicit, shares the Prompt gateway and exposes no route', async () => {
|
||||
const secretRoot = await mkdtemp(join(tmpdir(), 'ql3-cluster-ai-secret-'));
|
||||
const configRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-config-'));
|
||||
const invocationRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-invocation-'));
|
||||
const resultRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-result-'));
|
||||
const outputRoot = await mkdtemp(join(tmpdir(), 'ql3-copilot-output-'));
|
||||
const key = Buffer.alloc(32, 0x55).toString('base64url');
|
||||
const config = Buffer.from(`${JSON.stringify({
|
||||
schema: 'qinglong/cluster-copilot-failure-diagnosis-config@v1',
|
||||
provider: 'provider-primary',
|
||||
model: 'model-diagnosis',
|
||||
modelBoundary: 'external',
|
||||
responseLanguage: 'zh-CN',
|
||||
maxOutputTokens: 512,
|
||||
executionTimeoutMs: 60_000,
|
||||
egressPolicy: {
|
||||
schema: 'qinglong/copilot-model-egress-policy@v1',
|
||||
revision: 'cluster-copilot-v1',
|
||||
potentiallySensitiveDataBoundaries: ['external'],
|
||||
maxInputBytes: 64 * 1024,
|
||||
maxOutputTokens: 1024,
|
||||
},
|
||||
})}\n`);
|
||||
const invocation = canonicalClusterToolInvocationKeyringManifest({
|
||||
schema: CLUSTER_TOOL_INVOCATION_KEYRING_MANIFEST_SCHEMA,
|
||||
activeKeyId: 'invocation-key-1',
|
||||
keys: { 'invocation-key-1': key },
|
||||
});
|
||||
const result = canonicalClusterToolResultKeyringManifest({
|
||||
schema: CLUSTER_TOOL_RESULT_KEYRING_MANIFEST_SCHEMA,
|
||||
keys: { 'result-key-1': key },
|
||||
});
|
||||
const output = canonicalClusterCopilotFailureDiagnosisOutputKeyringManifest({
|
||||
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_OUTPUT_KEYRING_MANIFEST_SCHEMA,
|
||||
activeKeyId: 'output-key-1',
|
||||
keys: { 'output-key-1': key },
|
||||
});
|
||||
const gateway = {
|
||||
generate() {},
|
||||
supportsSuccessfulCompletionSink(sink) {
|
||||
return sink === registeredSink;
|
||||
},
|
||||
};
|
||||
const fakePool = { query() {}, connect() {} };
|
||||
const artifactStore = { put() {}, inspect() {}, readLogRange() {} };
|
||||
const copilot = Object.freeze({ execute() {} });
|
||||
let registeredSink;
|
||||
let created;
|
||||
let controlOptions;
|
||||
try {
|
||||
await Promise.all([
|
||||
projectedFile(configRoot, 'config.json', config),
|
||||
projectedFile(invocationRoot, 'keyring.json', invocation),
|
||||
projectedFile(resultRoot, 'keyring.json', result),
|
||||
projectedFile(outputRoot, 'keyring.json', output),
|
||||
]);
|
||||
const application = await startProductionClusterAiControlApplication({
|
||||
control: {
|
||||
config: controlConfig(),
|
||||
workerIngress: {
|
||||
config: { enabled: true },
|
||||
artifactStore,
|
||||
},
|
||||
},
|
||||
ai: {
|
||||
enabled: true,
|
||||
providerAuthorityFile: '/unused/providers.json',
|
||||
secretRootDirectory: secretRoot,
|
||||
copilot: {
|
||||
configFile: join(configRoot, 'config.json'),
|
||||
invocationKeyringRootDirectory: invocationRoot,
|
||||
resultKeyringRootDirectory: resultRoot,
|
||||
outputKeyringRootDirectory: outputRoot,
|
||||
},
|
||||
maxConcurrent: 1,
|
||||
recoveryLimit: 1,
|
||||
databaseMaxConnections: 1,
|
||||
},
|
||||
audit() {},
|
||||
async openAiDatabase() {
|
||||
return { pool: fakePool, async close() {} };
|
||||
},
|
||||
async bootstrapPrompt(options) {
|
||||
await options.openDatabase();
|
||||
registeredSink = options.createAdditionalSuccessfulCompletion({
|
||||
async recordWithAtomicSuccess() {},
|
||||
});
|
||||
return {
|
||||
status: 'active', profile: 'cluster', readiness: {}, capability: gateway,
|
||||
prompts: {}, promptCatalog: {}, promptExecutions: {},
|
||||
promptExecutionInspections: {}, async stop() { return 'stopped'; },
|
||||
};
|
||||
},
|
||||
async createCopilot(options) {
|
||||
created = options;
|
||||
return copilot;
|
||||
},
|
||||
async startControl(options) {
|
||||
controlOptions = options;
|
||||
return {
|
||||
status: 'active', address: { host: '127.0.0.1', port: 5800 },
|
||||
evidence: {}, recovery: { safe: true, remaining: 0, failed: 0 },
|
||||
unavailable: new Promise(() => {}), availabilityStatus() { return 'ready'; },
|
||||
async stop() { return 'stopped'; },
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.equal(application.copilot, copilot);
|
||||
assert.equal(created.pool, fakePool);
|
||||
assert.equal(created.gateway, gateway);
|
||||
assert.equal(created.successfulCompletion, registeredSink);
|
||||
assert.equal(created.artifactStore, artifactStore);
|
||||
assert.equal('copilot' in controlOptions, false);
|
||||
assert.equal(await application.stop(), 'stopped');
|
||||
} finally {
|
||||
config.fill(0); invocation.fill(0); result.fill(0); output.fill(0);
|
||||
await Promise.all([
|
||||
rm(secretRoot, { recursive: true, force: true }),
|
||||
rm(configRoot, { recursive: true, force: true }),
|
||||
rm(invocationRoot, { recursive: true, force: true }),
|
||||
rm(resultRoot, { recursive: true, force: true }),
|
||||
rm(outputRoot, { recursive: true, force: true }),
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
test('explicit AI composition injects one reviewed Prompt capability and drains it after HTTP control', async () => {
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { chmod, mkdtemp, rm, writeFile } = require('node:fs/promises');
|
||||
const { tmpdir } = require('node:os');
|
||||
const { join } = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA,
|
||||
ClusterCopilotFailureDiagnosisCompositionError,
|
||||
canonicalClusterCopilotFailureDiagnosisConfig,
|
||||
loadClusterCopilotFailureDiagnosisConfig,
|
||||
normalizeClusterCopilotFailureDiagnosisConfig,
|
||||
} = require('@qinglong/cluster-control/copilot-production');
|
||||
|
||||
function config(overrides = {}) {
|
||||
return {
|
||||
schema: CLUSTER_COPILOT_FAILURE_DIAGNOSIS_CONFIG_SCHEMA,
|
||||
provider: 'provider-primary',
|
||||
model: 'model-diagnosis',
|
||||
modelBoundary: 'external',
|
||||
responseLanguage: 'zh-CN',
|
||||
maxOutputTokens: 512,
|
||||
executionTimeoutMs: 60_000,
|
||||
egressPolicy: {
|
||||
schema: 'qinglong/copilot-model-egress-policy@v1',
|
||||
revision: 'cluster-copilot-v1',
|
||||
potentiallySensitiveDataBoundaries: ['external'],
|
||||
maxInputBytes: 64 * 1024,
|
||||
maxOutputTokens: 1024,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('normalizes one bounded deployment-owned Copilot model intent', () => {
|
||||
const normalized = normalizeClusterCopilotFailureDiagnosisConfig(config());
|
||||
assert.equal(normalized.provider, 'provider-primary');
|
||||
assert.equal(normalized.executionTimeoutMs, 60_000);
|
||||
assert.equal(Object.isFrozen(normalized), true);
|
||||
assert.equal(Object.isFrozen(normalized.egressPolicy), true);
|
||||
assert.throws(
|
||||
() => normalizeClusterCopilotFailureDiagnosisConfig(config({ extra: true })),
|
||||
ClusterCopilotFailureDiagnosisCompositionError,
|
||||
);
|
||||
assert.throws(
|
||||
() => normalizeClusterCopilotFailureDiagnosisConfig(config({
|
||||
modelBoundary: 'on_device',
|
||||
})),
|
||||
/egress model boundaries/,
|
||||
);
|
||||
assert.throws(
|
||||
() => normalizeClusterCopilotFailureDiagnosisConfig(config({
|
||||
executionTimeoutMs: 300_001,
|
||||
})),
|
||||
/execution timeout/,
|
||||
);
|
||||
});
|
||||
|
||||
test('loads only canonical read-only projected Copilot configuration', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'ql3-copilot-config-test-'));
|
||||
const file = join(root, 'config.json');
|
||||
const canonical = canonicalClusterCopilotFailureDiagnosisConfig(config());
|
||||
try {
|
||||
await writeFile(file, canonical, { mode: 0o440 });
|
||||
await chmod(file, 0o440);
|
||||
assert.deepEqual(
|
||||
await loadClusterCopilotFailureDiagnosisConfig(file),
|
||||
normalizeClusterCopilotFailureDiagnosisConfig(config()),
|
||||
);
|
||||
|
||||
await chmod(file, 0o640);
|
||||
await writeFile(file, Buffer.from(`${JSON.stringify(config(), null, 2)}\n`), {
|
||||
mode: 0o440,
|
||||
});
|
||||
await chmod(file, 0o440);
|
||||
await assert.rejects(
|
||||
loadClusterCopilotFailureDiagnosisConfig(file),
|
||||
/not canonical/,
|
||||
);
|
||||
|
||||
await chmod(file, 0o640);
|
||||
await writeFile(file, canonical, { mode: 0o640 });
|
||||
await chmod(file, 0o640);
|
||||
await assert.rejects(
|
||||
loadClusterCopilotFailureDiagnosisConfig(file),
|
||||
ClusterCopilotFailureDiagnosisCompositionError,
|
||||
);
|
||||
} finally {
|
||||
canonical.fill(0);
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user