mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 09:58:46 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,365 @@
|
||||
import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile';
|
||||
import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential';
|
||||
import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage';
|
||||
import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority';
|
||||
import { createProjectedModelProviderSecretMaterialProvider } from '@qinglong/ai/projected-model-provider-secret-material';
|
||||
import { createPluginPackagePromptOutputProjectedKeyring } from '@qinglong/ai/plugin-package-prompt-output-projected-keyring';
|
||||
import {
|
||||
bootstrapPostgresPluginPackagePromptApplication,
|
||||
type BootstrapPostgresPluginPackagePromptApplicationResult,
|
||||
} from '@qinglong/ai/postgres-plugin-package-prompt-application';
|
||||
import { createPostgresDatabaseOpener } from '@qinglong/cluster-postgres/runtime';
|
||||
import { PostgresProjectPolicyRepository } from '@qinglong/cluster-postgres/project-policy';
|
||||
import type { ClusterControlStopResult } from '@qinglong/runtime-core';
|
||||
import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
|
||||
|
||||
import type { ClusterControlApplicationResult } from './application';
|
||||
import type {
|
||||
ClusterControlEnvironment,
|
||||
EnabledClusterControlConfig,
|
||||
} from '../production-process/config';
|
||||
import {
|
||||
startProductionClusterControlApplication,
|
||||
type ProductionClusterControlApplicationOptions,
|
||||
} from './productionApplication';
|
||||
|
||||
export interface EnabledProductionClusterAiConfig {
|
||||
readonly enabled: true;
|
||||
readonly providerAuthorityFile: string;
|
||||
readonly secretRootDirectory: string;
|
||||
readonly promptOutputKeyringRootDirectory?: string;
|
||||
readonly maxConcurrent: number;
|
||||
readonly recoveryLimit: number;
|
||||
readonly databaseMaxConnections: number;
|
||||
}
|
||||
|
||||
export interface ProductionClusterAiControlApplicationOptions {
|
||||
readonly control: ProductionClusterControlApplicationOptions;
|
||||
readonly ai: EnabledProductionClusterAiConfig;
|
||||
readonly audit: (
|
||||
record: Readonly<ModelGatewayProfileAudit>,
|
||||
) => void | Promise<void>;
|
||||
readonly startControl?: typeof startProductionClusterControlApplication;
|
||||
readonly bootstrapPrompt?: typeof bootstrapPostgresPluginPackagePromptApplication;
|
||||
}
|
||||
|
||||
export class ProductionClusterAiConfigError extends TypeError {
|
||||
readonly code = 'QL3_CLUSTER_AI_CONFIG_INVALID';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Cluster AI configuration is invalid: ${message}`);
|
||||
this.name = 'ProductionClusterAiConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
function booleanValue(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
defaultValue: boolean,
|
||||
): boolean {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return defaultValue;
|
||||
if (value === 'true') return true;
|
||||
if (value === 'false') return false;
|
||||
throw new ProductionClusterAiConfigError(`${name} must be true or false`);
|
||||
}
|
||||
|
||||
function boundedInteger(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
fallback: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return fallback;
|
||||
if (!/^\d+$/.test(value)) {
|
||||
throw new ProductionClusterAiConfigError(`${name} must be an integer`);
|
||||
}
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
|
||||
throw new ProductionClusterAiConfigError(
|
||||
`${name} must be between ${minimum} and ${maximum}`,
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function requiredPath(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
): string {
|
||||
const value = environment[name];
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 2 ||
|
||||
value.length > 4096 ||
|
||||
!value.startsWith('/') ||
|
||||
/[\0\r\n]/.test(value)
|
||||
) {
|
||||
throw new ProductionClusterAiConfigError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Parsed only by the explicit AI process entrypoint; the default CLI ignores it. */
|
||||
export function loadProductionClusterAiConfig(
|
||||
environment: ClusterControlEnvironment,
|
||||
): EnabledProductionClusterAiConfig {
|
||||
if (
|
||||
!environment ||
|
||||
typeof environment !== 'object' ||
|
||||
Array.isArray(environment) ||
|
||||
!booleanValue(environment, 'QL3_CLUSTER_AI_ENABLED', false)
|
||||
) {
|
||||
throw new ProductionClusterAiConfigError(
|
||||
'QL3_CLUSTER_AI_ENABLED must be true',
|
||||
);
|
||||
}
|
||||
const promptOutputEnabled = booleanValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_PROMPT_OUTPUT_ENABLED',
|
||||
false,
|
||||
);
|
||||
return Object.freeze({
|
||||
enabled: true,
|
||||
providerAuthorityFile: requiredPath(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_PROVIDER_AUTHORITY_FILE',
|
||||
),
|
||||
secretRootDirectory: requiredPath(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_SECRET_ROOT',
|
||||
),
|
||||
...(promptOutputEnabled
|
||||
? {
|
||||
promptOutputKeyringRootDirectory: requiredPath(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_PROMPT_OUTPUT_KEYRING_ROOT',
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
maxConcurrent: boundedInteger(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_MAX_CONCURRENT',
|
||||
4,
|
||||
1,
|
||||
64,
|
||||
),
|
||||
recoveryLimit: boundedInteger(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_RECOVERY_LIMIT',
|
||||
32,
|
||||
1,
|
||||
128,
|
||||
),
|
||||
databaseMaxConnections: boundedInteger(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_DATABASE_MAX_CONNECTIONS',
|
||||
4,
|
||||
1,
|
||||
16,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function aiDatabaseOpener(
|
||||
control: EnabledClusterControlConfig,
|
||||
ai: EnabledProductionClusterAiConfig,
|
||||
onUnavailable: (error: Error) => void,
|
||||
) {
|
||||
return createPostgresDatabaseOpener({
|
||||
role: 'runtime',
|
||||
connection: control.database.connection,
|
||||
pool: {
|
||||
...control.database.pool,
|
||||
applicationName: 'qinglong-cluster-ai',
|
||||
maxConnections: ai.databaseMaxConnections,
|
||||
},
|
||||
onPoolError: onUnavailable,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit AI-enabled composition root. It keeps the normal control image and
|
||||
* process AI-free while sharing the reviewed authentication/Policy pipeline
|
||||
* and route registry when the separate AI image entrypoint is selected.
|
||||
*/
|
||||
export async function startProductionClusterAiControlApplication(
|
||||
options: ProductionClusterAiControlApplicationOptions,
|
||||
): Promise<ClusterControlApplicationResult> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
typeof options.audit !== 'function'
|
||||
) {
|
||||
throw new TypeError('Production Cluster AI application options are invalid');
|
||||
}
|
||||
const startControl =
|
||||
options.startControl ?? startProductionClusterControlApplication;
|
||||
const bootstrapPrompt =
|
||||
options.bootstrapPrompt ?? bootstrapPostgresPluginPackagePromptApplication;
|
||||
if (typeof startControl !== 'function' || typeof bootstrapPrompt !== 'function') {
|
||||
throw new TypeError('Production Cluster AI application factories are invalid');
|
||||
}
|
||||
const secretMaterial =
|
||||
await createProjectedModelProviderSecretMaterialProvider({
|
||||
rootDirectory: options.ai.secretRootDirectory,
|
||||
});
|
||||
const promptOutputKeys =
|
||||
options.ai.promptOutputKeyringRootDirectory === undefined
|
||||
? undefined
|
||||
: await createPluginPackagePromptOutputProjectedKeyring({
|
||||
rootDirectory: options.ai.promptOutputKeyringRootDirectory,
|
||||
});
|
||||
let aiDatabase:
|
||||
| Awaited<ReturnType<ReturnType<typeof createPostgresDatabaseOpener>>>
|
||||
| undefined;
|
||||
let resolveAiUnavailable: ((error: Error) => void) | undefined;
|
||||
let aiUnavailableError: Error | undefined;
|
||||
const aiUnavailable = new Promise<Error>((resolve) => {
|
||||
resolveAiUnavailable = resolve;
|
||||
});
|
||||
const onAiUnavailable = (error: Error): void => {
|
||||
aiUnavailableError ??= error;
|
||||
resolveAiUnavailable?.(aiUnavailableError);
|
||||
resolveAiUnavailable = undefined;
|
||||
};
|
||||
let promptApplication:
|
||||
| BootstrapPostgresPluginPackagePromptApplicationResult
|
||||
| undefined;
|
||||
let controlApplication: ClusterControlApplicationResult | undefined;
|
||||
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
let promptOutputPolicy: ProjectPolicyEngine | undefined;
|
||||
const promptOutputReadAuthorizer = Object.freeze({
|
||||
async authorize(request: Readonly<{
|
||||
principal: Parameters<ProjectPolicyEngine['authorize']>[0];
|
||||
projectId: string;
|
||||
}>) {
|
||||
if (!aiDatabase) {
|
||||
throw new Error('Cluster AI database is unavailable during output read');
|
||||
}
|
||||
promptOutputPolicy ??= new ProjectPolicyEngine(
|
||||
new PostgresProjectPolicyRepository(aiDatabase.pool),
|
||||
);
|
||||
const decision = await promptOutputPolicy.authorize(
|
||||
request.principal,
|
||||
request.projectId,
|
||||
'artifact.read',
|
||||
);
|
||||
return decision.effect === 'allow'
|
||||
? Object.freeze({ effect: 'allow' as const })
|
||||
: Object.freeze({
|
||||
effect: decision.effect,
|
||||
reasonCode: 'artifact_read_denied',
|
||||
});
|
||||
},
|
||||
});
|
||||
const stop = async (): Promise<ClusterControlStopResult> => {
|
||||
stopPromise ??= (async () => {
|
||||
const controlResult =
|
||||
controlApplication?.status === 'active'
|
||||
? await controlApplication.stop()
|
||||
: 'stopped';
|
||||
const promptResult = await promptApplication?.stop();
|
||||
return controlResult === 'stopped' &&
|
||||
(promptResult === undefined || promptResult === 'stopped')
|
||||
? 'stopped'
|
||||
: 'timed_out';
|
||||
})();
|
||||
return stopPromise;
|
||||
};
|
||||
try {
|
||||
const openDatabase = aiDatabaseOpener(
|
||||
options.control.config,
|
||||
options.ai,
|
||||
onAiUnavailable,
|
||||
);
|
||||
promptApplication = await bootstrapPrompt({
|
||||
enabled: true,
|
||||
async openDatabase() {
|
||||
if (aiDatabase) {
|
||||
throw new Error('Cluster AI database was opened more than once');
|
||||
}
|
||||
aiDatabase = await openDatabase();
|
||||
return aiDatabase;
|
||||
},
|
||||
async loadProviders() {
|
||||
if (!aiDatabase) {
|
||||
throw new Error('Cluster AI database is unavailable during provider load');
|
||||
}
|
||||
const credentialStorage = new PostgresModelProviderCredentialReader(
|
||||
aiDatabase.pool,
|
||||
);
|
||||
const credentials = new BoundModelProviderCredentialProvider({
|
||||
bindings: credentialStorage,
|
||||
secrets: secretMaterial,
|
||||
audit: credentialStorage,
|
||||
});
|
||||
return loadProjectedModelGatewayProviderAuthority({
|
||||
configFile: options.ai.providerAuthorityFile,
|
||||
credentials,
|
||||
});
|
||||
},
|
||||
audit: options.audit,
|
||||
maxConcurrent: options.ai.maxConcurrent,
|
||||
recoveryLimit: options.ai.recoveryLimit,
|
||||
...(promptOutputKeys === undefined
|
||||
? {}
|
||||
: {
|
||||
promptOutputKeys,
|
||||
promptOutputRead: { authorizer: promptOutputReadAuthorizer },
|
||||
}),
|
||||
});
|
||||
if (promptApplication.status !== 'active') {
|
||||
throw new Error('Cluster AI Prompt application did not activate');
|
||||
}
|
||||
controlApplication = await startControl({
|
||||
...options.control,
|
||||
promptCatalog: {
|
||||
capability: promptApplication.promptCatalog,
|
||||
},
|
||||
promptExecution: {
|
||||
capability: promptApplication.promptExecutions,
|
||||
},
|
||||
promptExecutionInspection: {
|
||||
capability: promptApplication.promptExecutionInspections,
|
||||
},
|
||||
...(promptApplication.promptOutputs === undefined
|
||||
? {}
|
||||
: {
|
||||
promptOutputRead: {
|
||||
capability: promptApplication.promptOutputs,
|
||||
},
|
||||
}),
|
||||
...(promptApplication.promptExecutionOutputs === undefined
|
||||
? {}
|
||||
: {
|
||||
promptExecutionOutputRead: {
|
||||
capability: promptApplication.promptExecutionOutputs,
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (controlApplication.status !== 'active') {
|
||||
throw new Error('AI-enabled cluster-control did not activate');
|
||||
}
|
||||
const activeControl = controlApplication;
|
||||
return Object.freeze({
|
||||
status: 'active' as const,
|
||||
address: activeControl.address,
|
||||
evidence: activeControl.evidence,
|
||||
recovery: activeControl.recovery,
|
||||
unavailable: Promise.race([activeControl.unavailable, aiUnavailable]),
|
||||
availabilityStatus() {
|
||||
return aiUnavailableError
|
||||
? 'unavailable'
|
||||
: activeControl.availabilityStatus();
|
||||
},
|
||||
stop,
|
||||
});
|
||||
} catch (error) {
|
||||
await stop().catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
import type {
|
||||
ClusterControlActivationAudit,
|
||||
ClusterControlActivationStack,
|
||||
ClusterControlReadinessEvidence,
|
||||
ClusterControlRuntimeActivationResult,
|
||||
ClusterControlStartupRecoverySummary,
|
||||
ClusterControlStopResult,
|
||||
DeploymentProfile,
|
||||
OpenPostgresDatabase,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
bootstrapClusterControlRuntime,
|
||||
type ClusterControlAssemblyInput,
|
||||
type ClusterControlRecoveryRuntimeOptions,
|
||||
type ClusterRunCancellationConvergenceRuntimeOptions,
|
||||
type ClusterSchedulerRuntimeOptions,
|
||||
type ClusterWorkerRuntimeDependencies,
|
||||
} from './clusterControlRuntime';
|
||||
import { assertClusterControlApiCredentialPepper } from '../authentication/apiCredentialAuthenticator';
|
||||
import {
|
||||
startClusterControlHttpSurface,
|
||||
type ClusterControlAdmissionPipeline,
|
||||
type ClusterControlHttpAddress,
|
||||
type ClusterControlHttpSurfaceOptions,
|
||||
} from '../transport/httpSurface';
|
||||
import type { ClusterControlAvailabilitySource } from '../database/availability';
|
||||
|
||||
export interface ClusterControlApplicationStack {
|
||||
reconcile(): Promise<ClusterControlStartupRecoverySummary>;
|
||||
startLifecycles(): Promise<boolean>;
|
||||
admission: ClusterControlAdmissionPipeline;
|
||||
stop(): Promise<ClusterControlStopResult>;
|
||||
}
|
||||
|
||||
export interface ClusterControlApplicationOptions {
|
||||
readonly enabled?: boolean;
|
||||
readonly profile: DeploymentProfile;
|
||||
readonly apiCredentialPepper?: string;
|
||||
readonly recovery?: ClusterControlRecoveryRuntimeOptions;
|
||||
readonly scheduler?: ClusterSchedulerRuntimeOptions;
|
||||
readonly cancellationConvergence?: ClusterRunCancellationConvergenceRuntimeOptions;
|
||||
readonly workerRuntime?: ClusterWorkerRuntimeDependencies;
|
||||
readonly openDatabase: OpenPostgresDatabase;
|
||||
readonly availability: ClusterControlAvailabilitySource;
|
||||
readonly http: ClusterControlHttpSurfaceOptions;
|
||||
readonly create: (
|
||||
input: ClusterControlAssemblyInput,
|
||||
) => ClusterControlApplicationStack;
|
||||
readonly audit: (
|
||||
record: ClusterControlActivationAudit,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export type ClusterControlApplicationResult =
|
||||
| { readonly status: 'disabled'; stop(): Promise<'stopped'> }
|
||||
| {
|
||||
readonly status: 'active';
|
||||
readonly address: ClusterControlHttpAddress;
|
||||
readonly evidence: ClusterControlReadinessEvidence;
|
||||
readonly recovery: ClusterControlStartupRecoverySummary;
|
||||
readonly unavailable: Promise<Error>;
|
||||
availabilityStatus(): 'ready' | 'unavailable' | 'stopped';
|
||||
stop(): Promise<ClusterControlStopResult>;
|
||||
};
|
||||
|
||||
export class ClusterControlDatabaseUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_CONTROL_DATABASE_UNAVAILABLE';
|
||||
|
||||
constructor() {
|
||||
super('Cluster-control database became unavailable');
|
||||
this.name = 'ClusterControlDatabaseUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function inactiveBootstrap(
|
||||
options: ClusterControlApplicationOptions,
|
||||
): Promise<ClusterControlRuntimeActivationResult> {
|
||||
return bootstrapClusterControlRuntime({
|
||||
...(options.enabled === undefined ? {} : { enabled: options.enabled }),
|
||||
profile: options.profile,
|
||||
...(options.apiCredentialPepper === undefined
|
||||
? {}
|
||||
: { apiCredentialPepper: options.apiCredentialPepper }),
|
||||
...(options.recovery === undefined ? {} : { recovery: options.recovery }),
|
||||
...(options.scheduler === undefined
|
||||
? {}
|
||||
: { scheduler: options.scheduler }),
|
||||
...(options.cancellationConvergence === undefined
|
||||
? {}
|
||||
: { cancellationConvergence: options.cancellationConvergence }),
|
||||
...(options.workerRuntime === undefined
|
||||
? {}
|
||||
: { workerRuntime: options.workerRuntime }),
|
||||
openDatabase: options.openDatabase,
|
||||
create() {
|
||||
throw new Error('Inactive cluster-control unexpectedly created a stack');
|
||||
},
|
||||
audit: options.audit,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the cluster probe surface before database readiness, then installs the
|
||||
* /api/v3 admission handler only after recovery and lifecycles are safe. Stop
|
||||
* withdraws and drains admission before stack, Pool and listener shutdown.
|
||||
*/
|
||||
export async function startClusterControlApplication(
|
||||
options: ClusterControlApplicationOptions,
|
||||
): Promise<ClusterControlApplicationResult> {
|
||||
const enabled = options.enabled ?? false;
|
||||
if (!enabled || options.profile !== 'cluster-control') {
|
||||
const inactive = await inactiveBootstrap(options);
|
||||
if (inactive.status !== 'disabled') {
|
||||
throw new Error('Inactive cluster-control unexpectedly became active');
|
||||
}
|
||||
return inactive;
|
||||
}
|
||||
|
||||
assertClusterControlApiCredentialPepper(options.apiCredentialPepper ?? '');
|
||||
const apiCredentialPepper = options.apiCredentialPepper!;
|
||||
if (
|
||||
!options.availability ||
|
||||
typeof options.availability.subscribe !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster-control availability source is invalid');
|
||||
}
|
||||
|
||||
const http = await startClusterControlHttpSurface(options.http);
|
||||
let activation: ClusterControlRuntimeActivationResult | undefined;
|
||||
let unavailableError: Error | undefined;
|
||||
let unavailableStopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
let availabilityStatus: 'ready' | 'unavailable' | 'stopped' = 'ready';
|
||||
let unsubscribeAvailability: (() => void) | undefined;
|
||||
let resolveUnavailable: ((error: Error) => void) | undefined;
|
||||
const unavailable = new Promise<Error>((resolve) => {
|
||||
resolveUnavailable = resolve;
|
||||
});
|
||||
const withdrawForUnavailable = (error: Error): Promise<void> => {
|
||||
unavailableError ??= error;
|
||||
availabilityStatus = 'unavailable';
|
||||
resolveUnavailable?.(unavailableError);
|
||||
resolveUnavailable = undefined;
|
||||
if (!activation || activation.status === 'disabled')
|
||||
return Promise.resolve();
|
||||
unavailableStopPromise ??= activation.stop();
|
||||
return unavailableStopPromise.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
};
|
||||
try {
|
||||
unsubscribeAvailability = options.availability.subscribe(
|
||||
withdrawForUnavailable,
|
||||
);
|
||||
activation = await bootstrapClusterControlRuntime({
|
||||
enabled: true,
|
||||
profile: options.profile,
|
||||
apiCredentialPepper,
|
||||
...(options.recovery === undefined ? {} : { recovery: options.recovery }),
|
||||
...(options.scheduler === undefined
|
||||
? {}
|
||||
: { scheduler: options.scheduler }),
|
||||
...(options.cancellationConvergence === undefined
|
||||
? {}
|
||||
: { cancellationConvergence: options.cancellationConvergence }),
|
||||
...(options.workerRuntime === undefined
|
||||
? {}
|
||||
: { workerRuntime: options.workerRuntime }),
|
||||
openDatabase: options.openDatabase,
|
||||
create(input): ClusterControlActivationStack {
|
||||
const application = options.create(input);
|
||||
if (
|
||||
!application ||
|
||||
typeof application !== 'object' ||
|
||||
!application.admission ||
|
||||
typeof application.admission.prepare !== 'function'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster-control application stack has no admission pipeline',
|
||||
);
|
||||
}
|
||||
return {
|
||||
reconcile: () => application.reconcile(),
|
||||
startLifecycles: () => application.startLifecycles(),
|
||||
installAdmission: () =>
|
||||
http.installAdmission(input.evidence, application.admission),
|
||||
stop: () => application.stop(),
|
||||
};
|
||||
},
|
||||
audit: options.audit,
|
||||
});
|
||||
if (activation.status === 'disabled') {
|
||||
unsubscribeAvailability();
|
||||
await http.close();
|
||||
return activation;
|
||||
}
|
||||
if (unavailableError) {
|
||||
await withdrawForUnavailable(unavailableError);
|
||||
throw new ClusterControlDatabaseUnavailableError();
|
||||
}
|
||||
const activeActivation = activation;
|
||||
|
||||
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
return {
|
||||
status: 'active',
|
||||
address: http.address,
|
||||
evidence: activeActivation.evidence,
|
||||
recovery: activeActivation.recovery,
|
||||
unavailable,
|
||||
availabilityStatus: () => availabilityStatus,
|
||||
stop() {
|
||||
if (stopPromise) return stopPromise;
|
||||
availabilityStatus = 'stopped';
|
||||
unsubscribeAvailability?.();
|
||||
unsubscribeAvailability = undefined;
|
||||
stopPromise = (async () => {
|
||||
let result: ClusterControlStopResult | undefined;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
result = await activeActivation.stop();
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
await http.close();
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
return result!;
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
unsubscribeAvailability?.();
|
||||
try {
|
||||
await http.close();
|
||||
} catch {
|
||||
// Preserve the readiness/assembly/activation failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,842 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
activateClusterControlRuntime,
|
||||
ClusterRunLostRetryCoordinator,
|
||||
ClusterRunCancellationConvergenceCoordinator,
|
||||
ClusterControlRecoveryEvidenceRegistry,
|
||||
ClusterControlRecoveryConvergenceVerifier,
|
||||
ClusterControlRecoverySupervisor,
|
||||
ClusterControlStartupRecoveryCoordinator,
|
||||
EvidenceBasedClusterControlRecoveryProcessor,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_CLAIMS_PER_PASS,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_CLAIM_LEASE_MS,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_EVIDENCE_TIMEOUT_MS,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_RETRY_DELAY_MS,
|
||||
MAX_CLUSTER_CONTROL_STARTUP_RECOVERY_PASSES,
|
||||
MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGE_SIZE,
|
||||
MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGES_PER_CYCLE,
|
||||
type ClusterControlActivationAudit,
|
||||
type ClusterControlActivationStack,
|
||||
type ClusterControlReadinessEvidence,
|
||||
type ClusterControlRecoveryExecutorEvidenceProvider,
|
||||
type ClusterControlRuntimeActivationResult,
|
||||
type ClusterControlStopResult,
|
||||
type DeploymentProfile,
|
||||
type OpenPostgresDatabase,
|
||||
type PostgresDatabaseResource,
|
||||
type PostgresPool,
|
||||
type ProjectPolicyRepository,
|
||||
type RunRepository,
|
||||
type ClusterRunCancellationConvergenceCycleResult,
|
||||
} from '@qinglong/runtime-core';
|
||||
import type { ClusterRunCancellationRepository } from '@qinglong/runtime-core/cluster-run-cancellation';
|
||||
import type { ProjectRunListReader } from '@qinglong/runtime-core/project-run-list';
|
||||
import type { ClusterScheduleStore } from '@qinglong/runtime-core/cluster-scheduler';
|
||||
import type { TaskDefinitionSource } from '@qinglong/runtime-core/task-definition';
|
||||
import type { TriggerSource } from '@qinglong/runtime-core/trigger';
|
||||
import type { ClusterTaskExecutionRevisionSource } from '@qinglong/runtime-core/cluster-execution-revision';
|
||||
import type {
|
||||
ProjectToolDefinitionSnapshotRepository,
|
||||
ProjectToolDefinitionSnapshotSourceRepository,
|
||||
} from '@qinglong/runtime-core/project-tool-definition-snapshot';
|
||||
import type { StepRunRepository } from '@qinglong/runtime-core/step-run';
|
||||
import type { ToolExecutionCompletionRepository } from '@qinglong/runtime-core/tool-execution-completion';
|
||||
import type { ToolExecutionFailureCompletionRepository } from '@qinglong/runtime-core/tool-execution-failure-completion';
|
||||
import type { ToolExecutionStartBarrierRepository } from '@qinglong/runtime-core/tool-execution-start-barrier';
|
||||
import type { ToolInvocationArtifactRepository } from '@qinglong/runtime-core/tool-invocation-artifact';
|
||||
import type { ToolResultKeyCatalogReader } from '@qinglong/runtime-core/tool-result-key-catalog';
|
||||
import type { ToolExecutionResultRekeyReader } from '@qinglong/runtime-core/tool-result-rekey';
|
||||
import {
|
||||
assertPostgresSchemaReady,
|
||||
PostgresClusterControlRecoverySource,
|
||||
PostgresClusterControlRecoveryClaimRepository,
|
||||
PostgresClusterControlRecoveryResolutionRepository,
|
||||
PostgresClusterRuntimeRecoverySource,
|
||||
PostgresClusterRunLostRetryRepository,
|
||||
PostgresClusterRunCancellationRepository,
|
||||
PostgresClusterRunCancellationConvergenceRepository,
|
||||
PostgresProjectPolicyRepository,
|
||||
PostgresApiCredentialRepository,
|
||||
PostgresSecurityAuditRepository,
|
||||
PostgresRunRepository,
|
||||
PostgresWorkerExecutionAttestationRepository,
|
||||
PostgresTaskDefinitionSource,
|
||||
PostgresTaskExecutionRevisionSource,
|
||||
PostgresTriggerSource,
|
||||
PostgresClusterScheduleRepository,
|
||||
PostgresRemoteWorkerAttestationEvidenceProvider,
|
||||
PostgresProjectToolDefinitionSnapshotRepository,
|
||||
PostgresStepRunRepository,
|
||||
PostgresToolExecutionCompletionRepository,
|
||||
PostgresToolExecutionFailureCompletionRepository,
|
||||
PostgresToolExecutionStartBarrierRepository,
|
||||
PostgresToolInvocationArtifactRepository,
|
||||
PostgresToolResultKeyCatalogReader,
|
||||
PostgresToolResultRekeyReader,
|
||||
type PostgresSchemaReadinessReport,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
import { PostgresPluginPackageWorkflowFrontierRepository } from '@qinglong/cluster-postgres/plugin-package-workflow-frontier';
|
||||
import { PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository } from '@qinglong/cluster-postgres/plugin-package-workflow-task-attempt-admission';
|
||||
import { PostgresTaskStartRepository } from '@qinglong/cluster-postgres/task-start';
|
||||
import { PostgresPluginPackageAutomationPublicationRepository } from '@qinglong/cluster-postgres/plugin-package-automation-publication';
|
||||
import { PostgresPluginPackageMaterializedRevisionRepository } from '@qinglong/cluster-postgres/plugin-package-materialized-revision';
|
||||
import {
|
||||
PostgresAuthorizedPluginPackageWorkflowAdmissionRepository,
|
||||
PostgresAuthorizedPluginPackageWorkflowRunEventListRepository,
|
||||
PostgresAuthorizedPluginPackageWorkflowRunInspectionRepository,
|
||||
PostgresAuthorizedPluginPackageWorkflowRunListRepository,
|
||||
PostgresAuthorizedPluginPackageWorkflowStepRunListRepository,
|
||||
} from '@qinglong/cluster-postgres/plugin-package-workflow-administration';
|
||||
import {
|
||||
assertClusterControlApiCredentialPepper,
|
||||
createClusterControlApiCredentialAuthenticator,
|
||||
} from '../authentication/apiCredentialAuthenticator';
|
||||
import type {
|
||||
ClusterControlRequestAuthenticator,
|
||||
ClusterControlSecurityAuditSink,
|
||||
} from '../transport/admissionPipeline';
|
||||
import {
|
||||
MAX_CLUSTER_SCHEDULE_CLAIMS_PER_CYCLE,
|
||||
ClusterSchedulerCoordinator,
|
||||
ClusterSchedulerLifecycle,
|
||||
type ClusterSchedulerCycleSummary,
|
||||
} from '../scheduling/scheduler';
|
||||
import { ClusterWorkflowSchedulerCoordinator } from '../scheduling/workflowScheduler';
|
||||
import { ClusterRuntimeSchedulerCoordinator } from '../scheduling/runtimeScheduler';
|
||||
import { ClusterRunCancellationConvergenceLifecycle } from '../run/runCancellationLifecycle';
|
||||
import type { TaskStartRepository } from '@qinglong/runtime-core/task-start';
|
||||
import {
|
||||
createClusterWorkerRuntimePort,
|
||||
type ClusterWorkerRuntimeDependencies,
|
||||
type ClusterWorkerRuntimePort,
|
||||
} from '../remote-execution/workerRuntimePort';
|
||||
import {
|
||||
createClusterPluginPackageWorkflowAdministrationCapability,
|
||||
type ClusterPluginPackageWorkflowAdministrationCapability,
|
||||
} from '../plugin-package/workflow/pluginPackageWorkflowAdministration';
|
||||
|
||||
export interface ClusterTrustedToolStorage {
|
||||
readonly invocationArtifacts: ToolInvocationArtifactRepository;
|
||||
readonly stepRuns: StepRunRepository;
|
||||
readonly startBarriers: ToolExecutionStartBarrierRepository;
|
||||
readonly completions: ToolExecutionCompletionRepository;
|
||||
readonly failureCompletions: ToolExecutionFailureCompletionRepository;
|
||||
readonly resultKeyCatalog: ToolResultKeyCatalogReader;
|
||||
readonly resultRekeys: ToolExecutionResultRekeyReader;
|
||||
readonly toolDefinitionSnapshots: ProjectToolDefinitionSnapshotRepository &
|
||||
ProjectToolDefinitionSnapshotSourceRepository;
|
||||
}
|
||||
|
||||
export interface ClusterControlAssemblyInput {
|
||||
readonly evidence: ClusterControlReadinessEvidence;
|
||||
readonly policies: ProjectPolicyRepository;
|
||||
readonly runs: RunRepository & ProjectRunListReader;
|
||||
readonly runCancellation: ClusterRunCancellationRepository;
|
||||
readonly taskStart: TaskStartRepository;
|
||||
readonly taskDefinitions: TaskDefinitionSource;
|
||||
readonly taskExecutionRevisions: ClusterTaskExecutionRevisionSource;
|
||||
readonly triggers: TriggerSource;
|
||||
readonly schedules: ClusterScheduleStore;
|
||||
readonly trustedToolStorage: ClusterTrustedToolStorage;
|
||||
readonly authenticator: ClusterControlRequestAuthenticator;
|
||||
readonly securityAudit: ClusterControlSecurityAuditSink;
|
||||
readonly workflowAdministration: ClusterPluginPackageWorkflowAdministrationCapability;
|
||||
readonly workerRuntime?: ClusterWorkerRuntimePort;
|
||||
}
|
||||
|
||||
export interface ClusterControlRecoveryRuntimeOptions {
|
||||
readonly ownerId: string;
|
||||
readonly providers?: readonly ClusterControlRecoveryExecutorEvidenceProvider[];
|
||||
readonly claimLimit?: number;
|
||||
readonly claimLeaseMs?: number;
|
||||
readonly retryDelayMs?: number;
|
||||
readonly providerTimeoutMs?: number;
|
||||
readonly maxStartupPasses?: number;
|
||||
}
|
||||
|
||||
export interface ClusterSchedulerRuntimeOptions {
|
||||
readonly ownerId?: string;
|
||||
readonly claimLeaseMs?: number;
|
||||
readonly maxClaimsPerCycle?: number;
|
||||
readonly misfireGraceMs?: number;
|
||||
readonly intervalMs?: number;
|
||||
readonly stopTimeoutMs?: number;
|
||||
readonly onDiagnostic?: (
|
||||
error: unknown,
|
||||
summary?: ClusterSchedulerCycleSummary,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ClusterRunCancellationConvergenceRuntimeOptions {
|
||||
readonly pageSize?: number;
|
||||
readonly maxPages?: number;
|
||||
readonly intervalMs?: number;
|
||||
readonly stopTimeoutMs?: number;
|
||||
readonly onDiagnostic?: (
|
||||
error: unknown,
|
||||
summary?: Readonly<ClusterRunCancellationConvergenceCycleResult>,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ClusterControlBootstrapOptions {
|
||||
readonly enabled?: boolean;
|
||||
readonly profile: DeploymentProfile;
|
||||
readonly apiCredentialPepper?: string;
|
||||
readonly recovery?: ClusterControlRecoveryRuntimeOptions;
|
||||
readonly scheduler?: ClusterSchedulerRuntimeOptions;
|
||||
readonly cancellationConvergence?: ClusterRunCancellationConvergenceRuntimeOptions;
|
||||
readonly workerRuntime?: ClusterWorkerRuntimeDependencies;
|
||||
readonly openDatabase: OpenPostgresDatabase;
|
||||
readonly create: (
|
||||
input: ClusterControlAssemblyInput,
|
||||
) => ClusterControlActivationStack;
|
||||
readonly audit: (
|
||||
record: ClusterControlActivationAudit,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
interface PreparedRecoveryRuntime {
|
||||
readonly providers: readonly ClusterControlRecoveryExecutorEvidenceProvider[];
|
||||
readonly providerTimeoutMs: number;
|
||||
readonly ownerId: string;
|
||||
readonly claimLimit: number;
|
||||
readonly claimLeaseMs: number;
|
||||
readonly retryDelayMs: number;
|
||||
readonly maxStartupPasses: number;
|
||||
}
|
||||
|
||||
interface PreparedSchedulerRuntime {
|
||||
readonly ownerId: string;
|
||||
readonly claimLeaseMs: number;
|
||||
readonly maxClaimsPerCycle: number;
|
||||
readonly misfireGraceMs: number;
|
||||
readonly intervalMs: number;
|
||||
readonly stopTimeoutMs: number;
|
||||
readonly onDiagnostic?: ClusterSchedulerRuntimeOptions['onDiagnostic'];
|
||||
}
|
||||
|
||||
interface PreparedCancellationConvergenceRuntime {
|
||||
readonly pageSize: number;
|
||||
readonly maxPages: number;
|
||||
readonly intervalMs: number;
|
||||
readonly stopTimeoutMs: number;
|
||||
readonly onDiagnostic?: ClusterRunCancellationConvergenceRuntimeOptions['onDiagnostic'];
|
||||
}
|
||||
|
||||
function boundedInteger(
|
||||
name: string,
|
||||
value: number | undefined,
|
||||
fallback: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
const normalized = value ?? fallback;
|
||||
if (
|
||||
!Number.isSafeInteger(normalized) ||
|
||||
normalized < minimum ||
|
||||
normalized > maximum
|
||||
) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function prepareRecoveryRuntime(
|
||||
options: ClusterControlRecoveryRuntimeOptions | undefined,
|
||||
): PreparedRecoveryRuntime {
|
||||
if (!options) {
|
||||
throw new TypeError(
|
||||
'Enabled cluster-control requires bounded recovery configuration',
|
||||
);
|
||||
}
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(options.ownerId)) {
|
||||
throw new TypeError('Cluster-control recovery ownerId is invalid');
|
||||
}
|
||||
const claimLeaseMs = boundedInteger(
|
||||
'Cluster-control recovery claim lease',
|
||||
options.claimLeaseMs,
|
||||
30_000,
|
||||
1_000,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_CLAIM_LEASE_MS,
|
||||
);
|
||||
const providerTimeoutMs = boundedInteger(
|
||||
'Cluster-control recovery evidence timeout',
|
||||
options.providerTimeoutMs,
|
||||
5_000,
|
||||
1,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_EVIDENCE_TIMEOUT_MS,
|
||||
);
|
||||
if (providerTimeoutMs + 250 > claimLeaseMs) {
|
||||
throw new RangeError(
|
||||
'Cluster-control recovery evidence timeout must leave at least 250ms for fenced settlement',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
providers: Object.freeze([...(options.providers ?? [])]),
|
||||
providerTimeoutMs,
|
||||
ownerId: options.ownerId,
|
||||
claimLimit: boundedInteger(
|
||||
'Cluster-control recovery claim limit',
|
||||
options.claimLimit,
|
||||
16,
|
||||
1,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_CLAIMS_PER_PASS,
|
||||
),
|
||||
claimLeaseMs,
|
||||
retryDelayMs: boundedInteger(
|
||||
'Cluster-control recovery retry delay',
|
||||
options.retryDelayMs,
|
||||
5_000,
|
||||
0,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_RETRY_DELAY_MS,
|
||||
),
|
||||
maxStartupPasses: boundedInteger(
|
||||
'Cluster-control startup recovery passes',
|
||||
options.maxStartupPasses,
|
||||
8,
|
||||
1,
|
||||
MAX_CLUSTER_CONTROL_STARTUP_RECOVERY_PASSES,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function prepareSchedulerRuntime(
|
||||
options: ClusterSchedulerRuntimeOptions | undefined,
|
||||
fallbackOwnerId: string,
|
||||
): PreparedSchedulerRuntime {
|
||||
const allowedKeys = new Set([
|
||||
'claimLeaseMs',
|
||||
'intervalMs',
|
||||
'maxClaimsPerCycle',
|
||||
'misfireGraceMs',
|
||||
'onDiagnostic',
|
||||
'ownerId',
|
||||
'stopTimeoutMs',
|
||||
]);
|
||||
if (
|
||||
options !== undefined &&
|
||||
(!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => !allowedKeys.has(key)))
|
||||
) {
|
||||
throw new TypeError('Cluster scheduler configuration is invalid');
|
||||
}
|
||||
const ownerId = options?.ownerId ?? fallbackOwnerId;
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(ownerId)) {
|
||||
throw new TypeError('Cluster scheduler ownerId is invalid');
|
||||
}
|
||||
if (
|
||||
options?.onDiagnostic !== undefined &&
|
||||
typeof options.onDiagnostic !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster scheduler diagnostic sink is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
ownerId,
|
||||
claimLeaseMs: boundedInteger(
|
||||
'Cluster scheduler claim lease',
|
||||
options?.claimLeaseMs,
|
||||
30_000,
|
||||
1_000,
|
||||
60_000,
|
||||
),
|
||||
maxClaimsPerCycle: boundedInteger(
|
||||
'Cluster scheduler claim budget',
|
||||
options?.maxClaimsPerCycle,
|
||||
16,
|
||||
1,
|
||||
MAX_CLUSTER_SCHEDULE_CLAIMS_PER_CYCLE,
|
||||
),
|
||||
misfireGraceMs: boundedInteger(
|
||||
'Cluster scheduler misfire grace',
|
||||
options?.misfireGraceMs,
|
||||
30_000,
|
||||
0,
|
||||
5 * 60_000,
|
||||
),
|
||||
intervalMs: boundedInteger(
|
||||
'Cluster scheduler interval',
|
||||
options?.intervalMs,
|
||||
1_000,
|
||||
250,
|
||||
60 * 60_000,
|
||||
),
|
||||
stopTimeoutMs: boundedInteger(
|
||||
'Cluster scheduler stop timeout',
|
||||
options?.stopTimeoutMs,
|
||||
10_000,
|
||||
100,
|
||||
30_000,
|
||||
),
|
||||
...(options?.onDiagnostic === undefined
|
||||
? {}
|
||||
: { onDiagnostic: options.onDiagnostic }),
|
||||
});
|
||||
}
|
||||
|
||||
function prepareCancellationConvergenceRuntime(
|
||||
options: ClusterRunCancellationConvergenceRuntimeOptions | undefined,
|
||||
): PreparedCancellationConvergenceRuntime {
|
||||
const allowedKeys = new Set([
|
||||
'intervalMs',
|
||||
'maxPages',
|
||||
'onDiagnostic',
|
||||
'pageSize',
|
||||
'stopTimeoutMs',
|
||||
]);
|
||||
if (
|
||||
options !== undefined &&
|
||||
(!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => !allowedKeys.has(key)))
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster Run cancellation convergence configuration is invalid',
|
||||
);
|
||||
}
|
||||
if (
|
||||
options?.onDiagnostic !== undefined &&
|
||||
typeof options.onDiagnostic !== 'function'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster Run cancellation convergence diagnostic sink is invalid',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
pageSize: boundedInteger(
|
||||
'Cluster Run cancellation convergence page size',
|
||||
options?.pageSize,
|
||||
32,
|
||||
1,
|
||||
MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGE_SIZE,
|
||||
),
|
||||
maxPages: boundedInteger(
|
||||
'Cluster Run cancellation convergence page limit',
|
||||
options?.maxPages,
|
||||
4,
|
||||
1,
|
||||
MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGES_PER_CYCLE,
|
||||
),
|
||||
intervalMs: boundedInteger(
|
||||
'Cluster Run cancellation convergence interval',
|
||||
options?.intervalMs,
|
||||
1_000,
|
||||
250,
|
||||
60 * 60_000,
|
||||
),
|
||||
stopTimeoutMs: boundedInteger(
|
||||
'Cluster Run cancellation convergence stop timeout',
|
||||
options?.stopTimeoutMs,
|
||||
10_000,
|
||||
100,
|
||||
30_000,
|
||||
),
|
||||
...(options?.onDiagnostic === undefined
|
||||
? {}
|
||||
: { onDiagnostic: options.onDiagnostic }),
|
||||
});
|
||||
}
|
||||
|
||||
function readinessEvidence(
|
||||
report: PostgresSchemaReadinessReport,
|
||||
): ClusterControlReadinessEvidence {
|
||||
return Object.freeze({
|
||||
contractName: report.contractName,
|
||||
contractVersion: report.contractVersion,
|
||||
serverMajor: report.serverMajor,
|
||||
migrationIds: Object.freeze([...report.migrationIds]),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns the cluster database around the readiness-first activation gate.
|
||||
* Repository and service construction happens through create() only after the
|
||||
* runtime role, migration history and catalog contract are proven ready.
|
||||
*/
|
||||
export async function bootstrapClusterControlRuntime(
|
||||
options: ClusterControlBootstrapOptions,
|
||||
): Promise<ClusterControlRuntimeActivationResult> {
|
||||
let recoveryRuntime: PreparedRecoveryRuntime | undefined;
|
||||
let schedulerRuntime: PreparedSchedulerRuntime | undefined;
|
||||
let cancellationConvergenceRuntime:
|
||||
| PreparedCancellationConvergenceRuntime
|
||||
| undefined;
|
||||
let recoveryRegistry: ClusterControlRecoveryEvidenceRegistry | undefined;
|
||||
if ((options.enabled ?? false) && options.profile === 'cluster-control') {
|
||||
assertClusterControlApiCredentialPepper(options.apiCredentialPepper ?? '');
|
||||
recoveryRuntime = prepareRecoveryRuntime(options.recovery);
|
||||
schedulerRuntime = prepareSchedulerRuntime(
|
||||
options.scheduler,
|
||||
recoveryRuntime.ownerId,
|
||||
);
|
||||
cancellationConvergenceRuntime = prepareCancellationConvergenceRuntime(
|
||||
options.cancellationConvergence,
|
||||
);
|
||||
}
|
||||
let database: PostgresDatabaseResource | undefined;
|
||||
let closePromise: Promise<void> | undefined;
|
||||
const closeDatabase = (): Promise<void> => {
|
||||
if (!database) return Promise.resolve();
|
||||
closePromise ??= Promise.resolve().then(() => database!.close());
|
||||
return closePromise;
|
||||
};
|
||||
|
||||
try {
|
||||
const activation = await activateClusterControlRuntime({
|
||||
...(options.enabled === undefined ? {} : { enabled: options.enabled }),
|
||||
profile: options.profile,
|
||||
readiness: {
|
||||
async assertReady() {
|
||||
if (database) {
|
||||
throw new Error(
|
||||
'Cluster-control database was opened more than once',
|
||||
);
|
||||
}
|
||||
database = await options.openDatabase();
|
||||
return readinessEvidence(
|
||||
await assertPostgresSchemaReady(database.pool),
|
||||
);
|
||||
},
|
||||
},
|
||||
create(evidence) {
|
||||
if (!database) {
|
||||
throw new Error(
|
||||
'Cluster-control database is unavailable after readiness',
|
||||
);
|
||||
}
|
||||
const recovery = new PostgresClusterControlRecoverySource(
|
||||
database.pool,
|
||||
);
|
||||
const recoveryClaims =
|
||||
new PostgresClusterControlRecoveryClaimRepository(database.pool);
|
||||
const recoveryTransitions =
|
||||
new PostgresClusterControlRecoveryResolutionRepository(database.pool);
|
||||
if (!recoveryRuntime) {
|
||||
throw new Error(
|
||||
'Cluster-control recovery runtime is unavailable after readiness',
|
||||
);
|
||||
}
|
||||
if (!schedulerRuntime) {
|
||||
throw new Error(
|
||||
'Cluster scheduler runtime is unavailable after readiness',
|
||||
);
|
||||
}
|
||||
if (!cancellationConvergenceRuntime) {
|
||||
throw new Error(
|
||||
'Cluster Run cancellation convergence runtime is unavailable after readiness',
|
||||
);
|
||||
}
|
||||
const remoteAttestations =
|
||||
new PostgresWorkerExecutionAttestationRepository(database.pool);
|
||||
recoveryRegistry = new ClusterControlRecoveryEvidenceRegistry(
|
||||
[
|
||||
...recoveryRuntime.providers,
|
||||
new PostgresRemoteWorkerAttestationEvidenceProvider(
|
||||
database.pool,
|
||||
remoteAttestations,
|
||||
),
|
||||
],
|
||||
{ timeoutMs: recoveryRuntime.providerTimeoutMs },
|
||||
);
|
||||
const recoveryProcessor =
|
||||
new EvidenceBasedClusterControlRecoveryProcessor(
|
||||
recoveryTransitions,
|
||||
recoveryRegistry,
|
||||
{ retryDelayMs: recoveryRuntime.retryDelayMs },
|
||||
);
|
||||
const recoverySupervisor = new ClusterControlRecoverySupervisor(
|
||||
recoveryClaims,
|
||||
recoveryProcessor,
|
||||
{
|
||||
ownerId: recoveryRuntime.ownerId,
|
||||
limit: recoveryRuntime.claimLimit,
|
||||
leaseMs: recoveryRuntime.claimLeaseMs,
|
||||
retryDelayMs: recoveryRuntime.retryDelayMs,
|
||||
},
|
||||
);
|
||||
const recoveryCoordinator =
|
||||
new ClusterControlStartupRecoveryCoordinator(recoverySupervisor, {
|
||||
maxPasses: recoveryRuntime.maxStartupPasses,
|
||||
});
|
||||
const runtimeRecoverySupervisor = new ClusterControlRecoverySupervisor(
|
||||
new PostgresClusterControlRecoveryClaimRepository(
|
||||
database.pool,
|
||||
randomUUID,
|
||||
(queryable) => new PostgresClusterRuntimeRecoverySource(queryable),
|
||||
),
|
||||
recoveryProcessor,
|
||||
{
|
||||
ownerId: recoveryRuntime.ownerId,
|
||||
limit: recoveryRuntime.claimLimit,
|
||||
leaseMs: recoveryRuntime.claimLeaseMs,
|
||||
retryDelayMs: recoveryRuntime.retryDelayMs,
|
||||
},
|
||||
);
|
||||
const schedules = new PostgresClusterScheduleRepository(database.pool);
|
||||
const runs = new PostgresRunRepository(database.pool);
|
||||
const trustedToolStorage: ClusterTrustedToolStorage = Object.freeze({
|
||||
invocationArtifacts: new PostgresToolInvocationArtifactRepository(
|
||||
database.pool,
|
||||
),
|
||||
stepRuns: new PostgresStepRunRepository(database.pool),
|
||||
startBarriers: new PostgresToolExecutionStartBarrierRepository(
|
||||
database.pool,
|
||||
),
|
||||
completions: new PostgresToolExecutionCompletionRepository(
|
||||
database.pool,
|
||||
),
|
||||
failureCompletions:
|
||||
new PostgresToolExecutionFailureCompletionRepository(database.pool),
|
||||
resultKeyCatalog: new PostgresToolResultKeyCatalogReader(
|
||||
database.pool,
|
||||
),
|
||||
resultRekeys: new PostgresToolResultRekeyReader(database.pool),
|
||||
toolDefinitionSnapshots:
|
||||
new PostgresProjectToolDefinitionSnapshotRepository(database.pool),
|
||||
});
|
||||
const workflowScheduler = new ClusterWorkflowSchedulerCoordinator(
|
||||
new ClusterSchedulerCoordinator(schedules, {
|
||||
ownerId: schedulerRuntime.ownerId,
|
||||
claimLeaseMs: schedulerRuntime.claimLeaseMs,
|
||||
maxClaimsPerCycle: schedulerRuntime.maxClaimsPerCycle,
|
||||
misfireGraceMs: schedulerRuntime.misfireGraceMs,
|
||||
}),
|
||||
new PostgresPluginPackageWorkflowFrontierRepository(database.pool),
|
||||
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
|
||||
database.pool,
|
||||
),
|
||||
{
|
||||
frontierPageSize: 32,
|
||||
frontierMaxPages: 4,
|
||||
taskAttemptPageSize: 32,
|
||||
taskAttemptMaxPages: 4,
|
||||
},
|
||||
);
|
||||
const runtimeScheduler = new ClusterRuntimeSchedulerCoordinator(
|
||||
runtimeRecoverySupervisor,
|
||||
new ClusterRunLostRetryCoordinator(
|
||||
new PostgresClusterRunLostRetryRepository(database.pool),
|
||||
{ pageSize: 16 },
|
||||
),
|
||||
workflowScheduler,
|
||||
);
|
||||
const schedulerLifecycle = new ClusterSchedulerLifecycle(
|
||||
runtimeScheduler,
|
||||
{
|
||||
intervalMs: schedulerRuntime.intervalMs,
|
||||
stopTimeoutMs: schedulerRuntime.stopTimeoutMs,
|
||||
...(schedulerRuntime.onDiagnostic === undefined
|
||||
? {}
|
||||
: { onDiagnostic: schedulerRuntime.onDiagnostic }),
|
||||
},
|
||||
);
|
||||
const cancellationConvergenceLifecycle =
|
||||
new ClusterRunCancellationConvergenceLifecycle(
|
||||
new ClusterRunCancellationConvergenceCoordinator(
|
||||
new PostgresClusterRunCancellationConvergenceRepository(
|
||||
database.pool,
|
||||
),
|
||||
{
|
||||
pageSize: cancellationConvergenceRuntime.pageSize,
|
||||
maxPages: cancellationConvergenceRuntime.maxPages,
|
||||
},
|
||||
),
|
||||
{
|
||||
intervalMs: cancellationConvergenceRuntime.intervalMs,
|
||||
stopTimeoutMs: cancellationConvergenceRuntime.stopTimeoutMs,
|
||||
...(cancellationConvergenceRuntime.onDiagnostic === undefined
|
||||
? {}
|
||||
: {
|
||||
onDiagnostic: cancellationConvergenceRuntime.onDiagnostic,
|
||||
}),
|
||||
},
|
||||
);
|
||||
const runCancellation = new PostgresClusterRunCancellationRepository(
|
||||
database.pool,
|
||||
);
|
||||
const taskStart = new PostgresTaskStartRepository(database.pool);
|
||||
const application = options.create({
|
||||
evidence,
|
||||
authenticator: createClusterControlApiCredentialAuthenticator(
|
||||
new PostgresApiCredentialRepository(database.pool),
|
||||
options.apiCredentialPepper ?? '',
|
||||
),
|
||||
policies: new PostgresProjectPolicyRepository(database.pool),
|
||||
runs,
|
||||
runCancellation,
|
||||
taskStart,
|
||||
taskDefinitions: new PostgresTaskDefinitionSource(database.pool),
|
||||
taskExecutionRevisions: new PostgresTaskExecutionRevisionSource(
|
||||
database.pool,
|
||||
),
|
||||
triggers: new PostgresTriggerSource(database.pool),
|
||||
schedules,
|
||||
trustedToolStorage,
|
||||
securityAudit: new PostgresSecurityAuditRepository(database.pool),
|
||||
workflowAdministration:
|
||||
createClusterPluginPackageWorkflowAdministrationCapability(
|
||||
new PostgresPluginPackageAutomationPublicationRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresPluginPackageMaterializedRevisionRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresAuthorizedPluginPackageWorkflowAdmissionRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresAuthorizedPluginPackageWorkflowRunInspectionRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresAuthorizedPluginPackageWorkflowRunListRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresAuthorizedPluginPackageWorkflowStepRunListRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresAuthorizedPluginPackageWorkflowRunEventListRepository(
|
||||
database.pool,
|
||||
),
|
||||
runCancellation,
|
||||
),
|
||||
...(options.workerRuntime === undefined
|
||||
? {}
|
||||
: {
|
||||
workerRuntime: createClusterWorkerRuntimePort(
|
||||
database.pool,
|
||||
options.workerRuntime,
|
||||
),
|
||||
}),
|
||||
});
|
||||
const convergence = new ClusterControlRecoveryConvergenceVerifier(
|
||||
recovery,
|
||||
);
|
||||
return {
|
||||
async reconcile() {
|
||||
const outstanding = await convergence.verify();
|
||||
if (!outstanding.safe) {
|
||||
const system = await recoveryCoordinator.reconcile();
|
||||
if (
|
||||
!system.safe ||
|
||||
system.remaining !== 0 ||
|
||||
system.failed !== 0
|
||||
) {
|
||||
return system;
|
||||
}
|
||||
}
|
||||
const summary = await application.reconcile();
|
||||
if (
|
||||
!summary.safe ||
|
||||
summary.remaining !== 0 ||
|
||||
summary.failed !== 0
|
||||
) {
|
||||
return summary;
|
||||
}
|
||||
return convergence.verify();
|
||||
},
|
||||
async startLifecycles() {
|
||||
if (!(await application.startLifecycles())) return false;
|
||||
schedulerLifecycle.start();
|
||||
cancellationConvergenceLifecycle.start();
|
||||
return true;
|
||||
},
|
||||
installAdmission: () => application.installAdmission(),
|
||||
async stop() {
|
||||
recoveryRegistry?.dispose();
|
||||
let schedulerStatus: 'stopped' | 'timed_out' = 'stopped';
|
||||
let cancellationStatus: 'stopped' | 'timed_out' = 'stopped';
|
||||
let applicationStatus: ClusterControlStopResult = 'stopped';
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
cancellationStatus = (
|
||||
await cancellationConvergenceLifecycle.stopAndDrain()
|
||||
).status;
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
schedulerStatus = (await schedulerLifecycle.stopAndDrain())
|
||||
.status;
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
try {
|
||||
applicationStatus = await application.stop();
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
return cancellationStatus === 'timed_out' ||
|
||||
schedulerStatus === 'timed_out' ||
|
||||
applicationStatus === 'timed_out'
|
||||
? 'timed_out'
|
||||
: 'stopped';
|
||||
},
|
||||
};
|
||||
},
|
||||
audit: options.audit,
|
||||
});
|
||||
if (activation.status === 'disabled') return activation;
|
||||
|
||||
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
return {
|
||||
...activation,
|
||||
stop() {
|
||||
if (stopPromise) return stopPromise;
|
||||
stopPromise = (async () => {
|
||||
let result: ClusterControlStopResult | undefined;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
result = await activation.stop();
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
await closeDatabase();
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
return result!;
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
recoveryRegistry?.dispose();
|
||||
try {
|
||||
await closeDatabase();
|
||||
} catch {
|
||||
// Preserve the readiness/assembly/activation failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export type {
|
||||
ClusterControlActivationAudit,
|
||||
ClusterControlActivationStack,
|
||||
ClusterControlReadinessEvidence,
|
||||
ClusterControlRuntimeActivationResult,
|
||||
ClusterControlStopResult,
|
||||
DeploymentProfile,
|
||||
OpenPostgresDatabase,
|
||||
PostgresDatabaseResource,
|
||||
PostgresPool,
|
||||
ProjectPolicyRepository,
|
||||
RunRepository,
|
||||
ClusterRunCancellationRepository,
|
||||
ClusterControlRequestAuthenticator,
|
||||
ClusterControlSecurityAuditSink,
|
||||
ClusterControlRecoveryExecutorEvidenceProvider,
|
||||
ClusterScheduleStore,
|
||||
};
|
||||
|
||||
export { ClusterRunCancellationConvergenceLifecycle } from '../run/runCancellationLifecycle';
|
||||
|
||||
export * from '../scheduling/scheduler';
|
||||
export * from '../scheduling/workflowScheduler';
|
||||
export * from '../scheduling/runtimeScheduler';
|
||||
export * from '../remote-execution/remoteWorkerDispatcher';
|
||||
export * from '../remote-execution/workerRuntimePort';
|
||||
@@ -0,0 +1,410 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type {
|
||||
ClusterControlStartupRecoverySummary,
|
||||
ClusterControlStopResult,
|
||||
} from '@qinglong/runtime-core';
|
||||
import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import {
|
||||
startClusterControlApplication,
|
||||
type ClusterControlApplicationOptions,
|
||||
type ClusterControlApplicationResult,
|
||||
type ClusterControlApplicationStack,
|
||||
} from './application';
|
||||
import {
|
||||
createClusterControlDatabaseBinding,
|
||||
type EnabledClusterControlConfig,
|
||||
} from '../production-process/config';
|
||||
import {
|
||||
createClusterControlAdmissionPipeline,
|
||||
createClusterControlProjectPolicyAuthorizer,
|
||||
} from '../transport/admissionPipeline';
|
||||
import { createClusterControlRouteRegistry } from '../transport/routeRegistry';
|
||||
import { CLUSTER_CONTROL_HTTP_DEFAULTS } from '../transport/httpSurface';
|
||||
import { createClusterControlRunReadRoute } from '../run/runReadRoute';
|
||||
import { createClusterControlRunListRoute } from '../run/runListRoute';
|
||||
import { createClusterControlRunEventListRoute } from '../run/runEventListRoute';
|
||||
import { createClusterControlRunStepListRoute } from '../run/runStepListRoute';
|
||||
import { createClusterControlTaskListRoute } from '../task/taskListRoute';
|
||||
import { createClusterControlTaskReadRoute } from '../task/taskReadRoute';
|
||||
import { createClusterControlTaskStartRoute } from '../task/taskStartRoute';
|
||||
import {
|
||||
createClusterControlPluginPackagePromptExecutionRoute,
|
||||
type ClusterPluginPackagePromptExecutionCapability,
|
||||
} from '../plugin-package/prompt/pluginPackagePromptExecutionRoute';
|
||||
import {
|
||||
createClusterControlPluginPackagePromptCatalogRoute,
|
||||
type ClusterPluginPackagePromptCatalogCapability,
|
||||
} from '../plugin-package/prompt/pluginPackagePromptCatalogRoute';
|
||||
import {
|
||||
createClusterControlPluginPackagePromptOutputReadRoute,
|
||||
type ClusterPluginPackagePromptOutputReadCapability,
|
||||
} from '../plugin-package/prompt/pluginPackagePromptOutputReadRoute';
|
||||
import {
|
||||
createClusterControlPluginPackagePromptExecutionInspectionRoute,
|
||||
type ClusterPluginPackagePromptExecutionInspectionCapability,
|
||||
} from '../plugin-package/prompt/pluginPackagePromptExecutionInspectionRoute';
|
||||
import {
|
||||
createClusterControlPluginPackagePromptExecutionOutputReadRoute,
|
||||
type ClusterPluginPackagePromptExecutionOutputReadCapability,
|
||||
} from '../plugin-package/prompt/pluginPackagePromptExecutionOutputReadRoute';
|
||||
import {
|
||||
createClusterControlRunCancellationRoute,
|
||||
type ClusterRunCancellationEventIdFactory,
|
||||
} from '../run/runCancellationRoute';
|
||||
import type { ClusterControlAssemblyInput } from './clusterControlRuntime';
|
||||
import type { ClusterRemoteWorkerArtifactStore } from '../remote-execution/remoteWorkerCompletionService';
|
||||
import type { EnabledClusterWorkerIngressConfig } from '../worker-ingress/workerIngressConfig';
|
||||
import {
|
||||
startProductionClusterWorkerIngress,
|
||||
type ProductionClusterWorkerIngressOptions as ProductionClusterWorkerIngressStarterOptions,
|
||||
} from '../worker-ingress/productionWorkerIngress';
|
||||
import type { ClusterWorkerIngressApplicationResult } from '../worker-ingress/workerIngressApplication';
|
||||
import { createClusterControlPluginPackageWorkflowRoutes } from '../plugin-package/workflow/pluginPackageWorkflowRoute';
|
||||
|
||||
export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
|
||||
'task.get',
|
||||
'task.list',
|
||||
'task.start',
|
||||
'run.get',
|
||||
'run.list',
|
||||
'run.events.list',
|
||||
'run.steps.list',
|
||||
'run.cancel',
|
||||
'workflow.read',
|
||||
'workflow.run.read',
|
||||
'workflow.run.list',
|
||||
'workflow.step.list',
|
||||
'workflow.event.list',
|
||||
'workflow.start',
|
||||
'workflow.cancel',
|
||||
] as const);
|
||||
export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS =
|
||||
Object.freeze([
|
||||
'prompt.read',
|
||||
'prompt.execute',
|
||||
'prompt.execution.read',
|
||||
'prompt.execution.output.read',
|
||||
'prompt.output.read',
|
||||
] as const);
|
||||
|
||||
export interface ProductionClusterControlAssemblyOptions {
|
||||
readonly createEventId?: ClusterRunCancellationEventIdFactory;
|
||||
readonly promptCatalog?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptCatalogCapability;
|
||||
}>;
|
||||
readonly promptExecution?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionCapability;
|
||||
readonly maxExecutionMs?: number;
|
||||
readonly now?: () => number;
|
||||
}>;
|
||||
readonly promptExecutionInspection?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionInspectionCapability;
|
||||
readonly now?: () => number;
|
||||
}>;
|
||||
readonly promptOutputRead?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptOutputReadCapability;
|
||||
}>;
|
||||
readonly promptExecutionOutputRead?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionOutputReadCapability;
|
||||
}>;
|
||||
readonly workerIngress?: Readonly<{
|
||||
readonly config: EnabledClusterWorkerIngressConfig;
|
||||
readonly onDiagnostic?: (error: unknown) => void | Promise<void>;
|
||||
}>;
|
||||
readonly startWorkerIngress?: (
|
||||
options: ProductionClusterWorkerIngressStarterOptions,
|
||||
) => Promise<ClusterWorkerIngressApplicationResult>;
|
||||
}
|
||||
|
||||
export interface ProductionClusterWorkerIngressOptions {
|
||||
readonly config: EnabledClusterWorkerIngressConfig;
|
||||
readonly artifactStore: ClusterRemoteWorkerArtifactStore;
|
||||
readonly secretProvider?: RemoteWorkerSecretValueProvider;
|
||||
readonly onDiagnostic?: (error: unknown) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ProductionClusterControlApplicationOptions
|
||||
extends Omit<
|
||||
ClusterControlApplicationOptions,
|
||||
| 'create'
|
||||
| 'enabled'
|
||||
| 'profile'
|
||||
| 'apiCredentialPepper'
|
||||
| 'openDatabase'
|
||||
| 'availability'
|
||||
| 'http'
|
||||
| 'workerRuntime'
|
||||
> {
|
||||
readonly config: EnabledClusterControlConfig;
|
||||
readonly createEventId?: ClusterRunCancellationEventIdFactory;
|
||||
readonly promptCatalog?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptCatalogCapability;
|
||||
}>;
|
||||
readonly promptExecution?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionCapability;
|
||||
}>;
|
||||
readonly promptExecutionInspection?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionInspectionCapability;
|
||||
}>;
|
||||
readonly promptOutputRead?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptOutputReadCapability;
|
||||
}>;
|
||||
readonly promptExecutionOutputRead?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionOutputReadCapability;
|
||||
}>;
|
||||
readonly workerIngress?: ProductionClusterWorkerIngressOptions;
|
||||
}
|
||||
|
||||
const SAFE_RECOVERY: Readonly<ClusterControlStartupRecoverySummary> =
|
||||
Object.freeze({ safe: true, remaining: 0, failed: 0 });
|
||||
|
||||
function eventIdFactory(
|
||||
candidate: ClusterRunCancellationEventIdFactory | undefined,
|
||||
): ClusterRunCancellationEventIdFactory {
|
||||
if (candidate !== undefined && typeof candidate !== 'function') {
|
||||
throw new TypeError(
|
||||
'Production cluster-control event ID factory is invalid',
|
||||
);
|
||||
}
|
||||
return candidate ?? randomUUID;
|
||||
}
|
||||
|
||||
/**
|
||||
* The reviewed production business surface. Bootstrap still owns PostgreSQL,
|
||||
* startup recovery, the scheduler and cancellation convergence; this stack
|
||||
* owns only the exact route allowlist and its admission pipeline.
|
||||
*/
|
||||
export function createProductionClusterControlApplicationStack(
|
||||
input: ClusterControlAssemblyInput,
|
||||
options: ProductionClusterControlAssemblyOptions = {},
|
||||
): ClusterControlApplicationStack {
|
||||
const createEventId = eventIdFactory(options.createEventId);
|
||||
const workerIngressStarter =
|
||||
options.startWorkerIngress ?? startProductionClusterWorkerIngress;
|
||||
if (typeof workerIngressStarter !== 'function') {
|
||||
throw new TypeError('Production Worker ingress starter is invalid');
|
||||
}
|
||||
const routeDefinitions = [
|
||||
createClusterControlTaskReadRoute(input.taskDefinitions),
|
||||
createClusterControlTaskListRoute(input.taskDefinitions),
|
||||
createClusterControlTaskStartRoute(input.taskStart, createEventId),
|
||||
createClusterControlRunReadRoute(input.runs),
|
||||
createClusterControlRunListRoute(input.runs),
|
||||
createClusterControlRunEventListRoute(input.runs),
|
||||
createClusterControlRunStepListRoute(
|
||||
input.runs,
|
||||
input.trustedToolStorage.stepRuns,
|
||||
),
|
||||
createClusterControlRunCancellationRoute(
|
||||
input.runCancellation,
|
||||
createEventId,
|
||||
),
|
||||
...createClusterControlPluginPackageWorkflowRoutes(
|
||||
input.workflowAdministration,
|
||||
Date.now,
|
||||
createEventId,
|
||||
),
|
||||
...(options.promptCatalog === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlPluginPackagePromptCatalogRoute(
|
||||
options.promptCatalog.capability,
|
||||
),
|
||||
]),
|
||||
...(options.promptExecution === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlPluginPackagePromptExecutionRoute(
|
||||
options.promptExecution.capability,
|
||||
{
|
||||
...(options.promptExecution.maxExecutionMs === undefined
|
||||
? {}
|
||||
: { maxExecutionMs: options.promptExecution.maxExecutionMs }),
|
||||
...(options.promptExecution.now === undefined
|
||||
? {}
|
||||
: { now: options.promptExecution.now }),
|
||||
createEventId,
|
||||
},
|
||||
),
|
||||
]),
|
||||
...(options.promptExecutionInspection === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlPluginPackagePromptExecutionInspectionRoute(
|
||||
options.promptExecutionInspection.capability,
|
||||
{
|
||||
...(options.promptExecutionInspection.now === undefined
|
||||
? {}
|
||||
: { now: options.promptExecutionInspection.now }),
|
||||
createEventId,
|
||||
},
|
||||
),
|
||||
]),
|
||||
...(options.promptOutputRead === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlPluginPackagePromptOutputReadRoute(
|
||||
options.promptOutputRead.capability,
|
||||
),
|
||||
]),
|
||||
...(options.promptExecutionOutputRead === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlPluginPackagePromptExecutionOutputReadRoute(
|
||||
options.promptExecutionOutputRead.capability,
|
||||
),
|
||||
]),
|
||||
];
|
||||
const routes = createClusterControlRouteRegistry(routeDefinitions);
|
||||
const expectedRouteCount =
|
||||
PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS.length +
|
||||
(options.promptCatalog === undefined ? 0 : 1) +
|
||||
(options.promptExecution === undefined ? 0 : 1) +
|
||||
(options.promptExecutionInspection === undefined ? 0 : 1) +
|
||||
(options.promptOutputRead === undefined ? 0 : 1) +
|
||||
(options.promptExecutionOutputRead === undefined ? 0 : 1);
|
||||
if (routes.size !== expectedRouteCount) {
|
||||
throw new Error('Production cluster-control route allowlist is incomplete');
|
||||
}
|
||||
const admission = createClusterControlAdmissionPipeline({
|
||||
routes,
|
||||
authenticator: input.authenticator,
|
||||
policy: createClusterControlProjectPolicyAuthorizer(input.policies),
|
||||
audit: input.securityAudit,
|
||||
});
|
||||
let workerIngress:
|
||||
| Extract<ClusterWorkerIngressApplicationResult, { status: 'active' }>
|
||||
| undefined;
|
||||
let workerIngressStart:
|
||||
| Promise<
|
||||
Extract<ClusterWorkerIngressApplicationResult, { status: 'active' }>
|
||||
>
|
||||
| undefined;
|
||||
let workerIngressUnavailable: unknown;
|
||||
let workerIngressStop: Promise<'stopped'> | undefined;
|
||||
const stopWorkerIngress = (): Promise<'stopped'> => {
|
||||
if (!workerIngress) return Promise.resolve('stopped' as const);
|
||||
workerIngressStop ??= workerIngress.stop();
|
||||
return workerIngressStop;
|
||||
};
|
||||
const startWorkerIngress = async (): Promise<void> => {
|
||||
const ingress = options.workerIngress;
|
||||
if (!ingress) return;
|
||||
if (!input.workerRuntime) {
|
||||
throw new Error(
|
||||
'Production Worker ingress requires an injected runtime service port',
|
||||
);
|
||||
}
|
||||
workerIngressStart ??= (async () => {
|
||||
const result = await workerIngressStarter({
|
||||
config: ingress.config,
|
||||
runtime: input.workerRuntime!,
|
||||
onPoolError(error) {
|
||||
workerIngressUnavailable ??= error;
|
||||
void Promise.resolve(ingress.onDiagnostic?.(error)).catch(
|
||||
() => undefined,
|
||||
);
|
||||
void stopWorkerIngress().catch(() => undefined);
|
||||
},
|
||||
});
|
||||
if (result.status !== 'active') {
|
||||
throw new Error('Production Worker ingress did not activate');
|
||||
}
|
||||
workerIngress = result;
|
||||
if (workerIngressUnavailable !== undefined) {
|
||||
await stopWorkerIngress();
|
||||
throw new Error(
|
||||
'Production Worker ingress database became unavailable during activation',
|
||||
);
|
||||
}
|
||||
return result;
|
||||
})();
|
||||
await workerIngressStart;
|
||||
};
|
||||
|
||||
return Object.freeze({
|
||||
async reconcile(): Promise<ClusterControlStartupRecoverySummary> {
|
||||
return SAFE_RECOVERY;
|
||||
},
|
||||
async startLifecycles(): Promise<boolean> {
|
||||
await startWorkerIngress();
|
||||
return true;
|
||||
},
|
||||
admission,
|
||||
async stop(): Promise<ClusterControlStopResult> {
|
||||
await stopWorkerIngress();
|
||||
return 'stopped';
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** Starts cluster-control with the reviewed production route allowlist. */
|
||||
export function startProductionClusterControlApplication(
|
||||
options: ProductionClusterControlApplicationOptions,
|
||||
): Promise<ClusterControlApplicationResult> {
|
||||
const createEventId = eventIdFactory(options.createEventId);
|
||||
const {
|
||||
createEventId: _ignoredCreateEventId,
|
||||
config,
|
||||
workerIngress,
|
||||
promptCatalog,
|
||||
promptExecution,
|
||||
promptExecutionInspection,
|
||||
promptOutputRead,
|
||||
promptExecutionOutputRead,
|
||||
...applicationOptions
|
||||
} = options;
|
||||
const database = createClusterControlDatabaseBinding(config);
|
||||
return startClusterControlApplication({
|
||||
...applicationOptions,
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
apiCredentialPepper: config.security.apiCredentialPepper,
|
||||
http: config.http,
|
||||
...(workerIngress === undefined
|
||||
? {}
|
||||
: {
|
||||
workerRuntime: {
|
||||
artifactStore: workerIngress.artifactStore,
|
||||
...(workerIngress.secretProvider === undefined
|
||||
? {}
|
||||
: { secretProvider: workerIngress.secretProvider }),
|
||||
},
|
||||
}),
|
||||
...database,
|
||||
create: (input) =>
|
||||
createProductionClusterControlApplicationStack(input, {
|
||||
createEventId,
|
||||
...(promptCatalog === undefined ? {} : { promptCatalog }),
|
||||
...(promptExecution === undefined
|
||||
? {}
|
||||
: {
|
||||
promptExecution: {
|
||||
capability: promptExecution.capability,
|
||||
maxExecutionMs: Math.max(
|
||||
1,
|
||||
(config.http.requestTimeoutMs ??
|
||||
CLUSTER_CONTROL_HTTP_DEFAULTS.requestTimeoutMs) - 100,
|
||||
),
|
||||
},
|
||||
}),
|
||||
...(promptExecutionInspection === undefined
|
||||
? {}
|
||||
: { promptExecutionInspection }),
|
||||
...(promptOutputRead === undefined ? {} : { promptOutputRead }),
|
||||
...(promptExecutionOutputRead === undefined
|
||||
? {}
|
||||
: { promptExecutionOutputRead }),
|
||||
...(workerIngress === undefined
|
||||
? {}
|
||||
: {
|
||||
workerIngress: {
|
||||
config: workerIngress.config,
|
||||
...(workerIngress.onDiagnostic === undefined
|
||||
? {}
|
||||
: { onDiagnostic: workerIngress.onDiagnostic }),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user