mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 18:08:20 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile';
|
||||
|
||||
import {
|
||||
loadProductionClusterAiConfig,
|
||||
startProductionClusterAiControlApplication,
|
||||
} from './application-runtime/aiProductionApplication';
|
||||
import {
|
||||
runProductionClusterControlProcess,
|
||||
type ClusterControlProcessEvent,
|
||||
type ClusterControlProcessSignal,
|
||||
type ClusterControlProcessSignalSource,
|
||||
} from './production-process/processApplication';
|
||||
|
||||
const USAGE = 'Usage: ql3-cluster-control-ai';
|
||||
|
||||
const nodeSignals: ClusterControlProcessSignalSource = Object.freeze({
|
||||
subscribe(listener: (signal: ClusterControlProcessSignal) => void) {
|
||||
const handlers = Object.freeze({
|
||||
SIGINT: () => listener('SIGINT' as const),
|
||||
SIGTERM: () => listener('SIGTERM' as const),
|
||||
});
|
||||
process.once('SIGINT', handlers.SIGINT);
|
||||
process.once('SIGTERM', handlers.SIGTERM);
|
||||
return () => {
|
||||
process.off('SIGINT', handlers.SIGINT);
|
||||
process.off('SIGTERM', handlers.SIGTERM);
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
function write(record: object): void {
|
||||
process.stdout.write(`${JSON.stringify(record)}\n`);
|
||||
}
|
||||
|
||||
function emit(record: ClusterControlProcessEvent): void {
|
||||
write(record);
|
||||
}
|
||||
|
||||
function audit(record: Readonly<ModelGatewayProfileAudit>): void {
|
||||
write(
|
||||
Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-cluster-ai',
|
||||
level: record.state === 'failed' ? 'error' : 'info',
|
||||
event: 'activation',
|
||||
profile: record.profile,
|
||||
state: record.state,
|
||||
...(record.maxConcurrent === undefined
|
||||
? {}
|
||||
: { maxConcurrent: record.maxConcurrent }),
|
||||
...(record.recoveryLimit === undefined
|
||||
? {}
|
||||
: { recoveryLimit: record.recoveryLimit }),
|
||||
...(record.recovered === undefined
|
||||
? {}
|
||||
: { recovered: record.recovered }),
|
||||
...(record.alreadyCompleted === undefined
|
||||
? {}
|
||||
: { alreadyCompleted: record.alreadyCompleted }),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function main(argv: readonly string[]): Promise<void> {
|
||||
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
|
||||
process.stdout.write(`${USAGE}\n`);
|
||||
return;
|
||||
}
|
||||
if (argv.length !== 0) {
|
||||
process.stderr.write(
|
||||
`${JSON.stringify({
|
||||
code: 'QL3_CLUSTER_AI_CLI_USAGE_INVALID',
|
||||
message: USAGE,
|
||||
})}\n`,
|
||||
);
|
||||
process.exitCode = 64;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const ai = loadProductionClusterAiConfig(process.env);
|
||||
const stopResult = await runProductionClusterControlProcess({
|
||||
environment: process.env,
|
||||
signals: nodeSignals,
|
||||
emit,
|
||||
start: (control) =>
|
||||
startProductionClusterAiControlApplication({ control, ai, audit }),
|
||||
});
|
||||
if (stopResult !== 'stopped') process.exitCode = 1;
|
||||
} catch (error) {
|
||||
const candidate = error as { readonly name?: unknown; readonly code?: unknown };
|
||||
process.stderr.write(
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-cluster-ai',
|
||||
level: 'error',
|
||||
event: 'process_failed',
|
||||
name:
|
||||
typeof candidate?.name === 'string' ? candidate.name : 'Error',
|
||||
...(typeof candidate?.code === 'string'
|
||||
? { code: candidate.code }
|
||||
: {}),
|
||||
})}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
void main(process.argv.slice(2));
|
||||
@@ -0,0 +1,365 @@
|
||||
import type { ModelGatewayProfileAudit } from '@qinglong/ai/profile';
|
||||
import { BoundModelProviderCredentialProvider } from '@qinglong/ai/provider-credential';
|
||||
import { PostgresModelProviderCredentialReader } from '@qinglong/ai/postgres-model-provider-credential-storage';
|
||||
import { loadProjectedModelGatewayProviderAuthority } from '@qinglong/ai/projected-model-gateway-authority';
|
||||
import { createProjectedModelProviderSecretMaterialProvider } from '@qinglong/ai/projected-model-provider-secret-material';
|
||||
import { createPluginPackagePromptOutputProjectedKeyring } from '@qinglong/ai/plugin-package-prompt-output-projected-keyring';
|
||||
import {
|
||||
bootstrapPostgresPluginPackagePromptApplication,
|
||||
type BootstrapPostgresPluginPackagePromptApplicationResult,
|
||||
} from '@qinglong/ai/postgres-plugin-package-prompt-application';
|
||||
import { createPostgresDatabaseOpener } from '@qinglong/cluster-postgres/runtime';
|
||||
import { PostgresProjectPolicyRepository } from '@qinglong/cluster-postgres/project-policy';
|
||||
import type { ClusterControlStopResult } from '@qinglong/runtime-core';
|
||||
import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
|
||||
|
||||
import type { ClusterControlApplicationResult } from './application';
|
||||
import type {
|
||||
ClusterControlEnvironment,
|
||||
EnabledClusterControlConfig,
|
||||
} from '../production-process/config';
|
||||
import {
|
||||
startProductionClusterControlApplication,
|
||||
type ProductionClusterControlApplicationOptions,
|
||||
} from './productionApplication';
|
||||
|
||||
export interface EnabledProductionClusterAiConfig {
|
||||
readonly enabled: true;
|
||||
readonly providerAuthorityFile: string;
|
||||
readonly secretRootDirectory: string;
|
||||
readonly promptOutputKeyringRootDirectory?: string;
|
||||
readonly maxConcurrent: number;
|
||||
readonly recoveryLimit: number;
|
||||
readonly databaseMaxConnections: number;
|
||||
}
|
||||
|
||||
export interface ProductionClusterAiControlApplicationOptions {
|
||||
readonly control: ProductionClusterControlApplicationOptions;
|
||||
readonly ai: EnabledProductionClusterAiConfig;
|
||||
readonly audit: (
|
||||
record: Readonly<ModelGatewayProfileAudit>,
|
||||
) => void | Promise<void>;
|
||||
readonly startControl?: typeof startProductionClusterControlApplication;
|
||||
readonly bootstrapPrompt?: typeof bootstrapPostgresPluginPackagePromptApplication;
|
||||
}
|
||||
|
||||
export class ProductionClusterAiConfigError extends TypeError {
|
||||
readonly code = 'QL3_CLUSTER_AI_CONFIG_INVALID';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Cluster AI configuration is invalid: ${message}`);
|
||||
this.name = 'ProductionClusterAiConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
function booleanValue(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
defaultValue: boolean,
|
||||
): boolean {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return defaultValue;
|
||||
if (value === 'true') return true;
|
||||
if (value === 'false') return false;
|
||||
throw new ProductionClusterAiConfigError(`${name} must be true or false`);
|
||||
}
|
||||
|
||||
function boundedInteger(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
fallback: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return fallback;
|
||||
if (!/^\d+$/.test(value)) {
|
||||
throw new ProductionClusterAiConfigError(`${name} must be an integer`);
|
||||
}
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
|
||||
throw new ProductionClusterAiConfigError(
|
||||
`${name} must be between ${minimum} and ${maximum}`,
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function requiredPath(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
): string {
|
||||
const value = environment[name];
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 2 ||
|
||||
value.length > 4096 ||
|
||||
!value.startsWith('/') ||
|
||||
/[\0\r\n]/.test(value)
|
||||
) {
|
||||
throw new ProductionClusterAiConfigError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Parsed only by the explicit AI process entrypoint; the default CLI ignores it. */
|
||||
export function loadProductionClusterAiConfig(
|
||||
environment: ClusterControlEnvironment,
|
||||
): EnabledProductionClusterAiConfig {
|
||||
if (
|
||||
!environment ||
|
||||
typeof environment !== 'object' ||
|
||||
Array.isArray(environment) ||
|
||||
!booleanValue(environment, 'QL3_CLUSTER_AI_ENABLED', false)
|
||||
) {
|
||||
throw new ProductionClusterAiConfigError(
|
||||
'QL3_CLUSTER_AI_ENABLED must be true',
|
||||
);
|
||||
}
|
||||
const promptOutputEnabled = booleanValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_PROMPT_OUTPUT_ENABLED',
|
||||
false,
|
||||
);
|
||||
return Object.freeze({
|
||||
enabled: true,
|
||||
providerAuthorityFile: requiredPath(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_PROVIDER_AUTHORITY_FILE',
|
||||
),
|
||||
secretRootDirectory: requiredPath(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_SECRET_ROOT',
|
||||
),
|
||||
...(promptOutputEnabled
|
||||
? {
|
||||
promptOutputKeyringRootDirectory: requiredPath(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_PROMPT_OUTPUT_KEYRING_ROOT',
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
maxConcurrent: boundedInteger(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_MAX_CONCURRENT',
|
||||
4,
|
||||
1,
|
||||
64,
|
||||
),
|
||||
recoveryLimit: boundedInteger(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_RECOVERY_LIMIT',
|
||||
32,
|
||||
1,
|
||||
128,
|
||||
),
|
||||
databaseMaxConnections: boundedInteger(
|
||||
environment,
|
||||
'QL3_CLUSTER_AI_DATABASE_MAX_CONNECTIONS',
|
||||
4,
|
||||
1,
|
||||
16,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function aiDatabaseOpener(
|
||||
control: EnabledClusterControlConfig,
|
||||
ai: EnabledProductionClusterAiConfig,
|
||||
onUnavailable: (error: Error) => void,
|
||||
) {
|
||||
return createPostgresDatabaseOpener({
|
||||
role: 'runtime',
|
||||
connection: control.database.connection,
|
||||
pool: {
|
||||
...control.database.pool,
|
||||
applicationName: 'qinglong-cluster-ai',
|
||||
maxConnections: ai.databaseMaxConnections,
|
||||
},
|
||||
onPoolError: onUnavailable,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Explicit AI-enabled composition root. It keeps the normal control image and
|
||||
* process AI-free while sharing the reviewed authentication/Policy pipeline
|
||||
* and route registry when the separate AI image entrypoint is selected.
|
||||
*/
|
||||
export async function startProductionClusterAiControlApplication(
|
||||
options: ProductionClusterAiControlApplicationOptions,
|
||||
): Promise<ClusterControlApplicationResult> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
typeof options.audit !== 'function'
|
||||
) {
|
||||
throw new TypeError('Production Cluster AI application options are invalid');
|
||||
}
|
||||
const startControl =
|
||||
options.startControl ?? startProductionClusterControlApplication;
|
||||
const bootstrapPrompt =
|
||||
options.bootstrapPrompt ?? bootstrapPostgresPluginPackagePromptApplication;
|
||||
if (typeof startControl !== 'function' || typeof bootstrapPrompt !== 'function') {
|
||||
throw new TypeError('Production Cluster AI application factories are invalid');
|
||||
}
|
||||
const secretMaterial =
|
||||
await createProjectedModelProviderSecretMaterialProvider({
|
||||
rootDirectory: options.ai.secretRootDirectory,
|
||||
});
|
||||
const promptOutputKeys =
|
||||
options.ai.promptOutputKeyringRootDirectory === undefined
|
||||
? undefined
|
||||
: await createPluginPackagePromptOutputProjectedKeyring({
|
||||
rootDirectory: options.ai.promptOutputKeyringRootDirectory,
|
||||
});
|
||||
let aiDatabase:
|
||||
| Awaited<ReturnType<ReturnType<typeof createPostgresDatabaseOpener>>>
|
||||
| undefined;
|
||||
let resolveAiUnavailable: ((error: Error) => void) | undefined;
|
||||
let aiUnavailableError: Error | undefined;
|
||||
const aiUnavailable = new Promise<Error>((resolve) => {
|
||||
resolveAiUnavailable = resolve;
|
||||
});
|
||||
const onAiUnavailable = (error: Error): void => {
|
||||
aiUnavailableError ??= error;
|
||||
resolveAiUnavailable?.(aiUnavailableError);
|
||||
resolveAiUnavailable = undefined;
|
||||
};
|
||||
let promptApplication:
|
||||
| BootstrapPostgresPluginPackagePromptApplicationResult
|
||||
| undefined;
|
||||
let controlApplication: ClusterControlApplicationResult | undefined;
|
||||
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
let promptOutputPolicy: ProjectPolicyEngine | undefined;
|
||||
const promptOutputReadAuthorizer = Object.freeze({
|
||||
async authorize(request: Readonly<{
|
||||
principal: Parameters<ProjectPolicyEngine['authorize']>[0];
|
||||
projectId: string;
|
||||
}>) {
|
||||
if (!aiDatabase) {
|
||||
throw new Error('Cluster AI database is unavailable during output read');
|
||||
}
|
||||
promptOutputPolicy ??= new ProjectPolicyEngine(
|
||||
new PostgresProjectPolicyRepository(aiDatabase.pool),
|
||||
);
|
||||
const decision = await promptOutputPolicy.authorize(
|
||||
request.principal,
|
||||
request.projectId,
|
||||
'artifact.read',
|
||||
);
|
||||
return decision.effect === 'allow'
|
||||
? Object.freeze({ effect: 'allow' as const })
|
||||
: Object.freeze({
|
||||
effect: decision.effect,
|
||||
reasonCode: 'artifact_read_denied',
|
||||
});
|
||||
},
|
||||
});
|
||||
const stop = async (): Promise<ClusterControlStopResult> => {
|
||||
stopPromise ??= (async () => {
|
||||
const controlResult =
|
||||
controlApplication?.status === 'active'
|
||||
? await controlApplication.stop()
|
||||
: 'stopped';
|
||||
const promptResult = await promptApplication?.stop();
|
||||
return controlResult === 'stopped' &&
|
||||
(promptResult === undefined || promptResult === 'stopped')
|
||||
? 'stopped'
|
||||
: 'timed_out';
|
||||
})();
|
||||
return stopPromise;
|
||||
};
|
||||
try {
|
||||
const openDatabase = aiDatabaseOpener(
|
||||
options.control.config,
|
||||
options.ai,
|
||||
onAiUnavailable,
|
||||
);
|
||||
promptApplication = await bootstrapPrompt({
|
||||
enabled: true,
|
||||
async openDatabase() {
|
||||
if (aiDatabase) {
|
||||
throw new Error('Cluster AI database was opened more than once');
|
||||
}
|
||||
aiDatabase = await openDatabase();
|
||||
return aiDatabase;
|
||||
},
|
||||
async loadProviders() {
|
||||
if (!aiDatabase) {
|
||||
throw new Error('Cluster AI database is unavailable during provider load');
|
||||
}
|
||||
const credentialStorage = new PostgresModelProviderCredentialReader(
|
||||
aiDatabase.pool,
|
||||
);
|
||||
const credentials = new BoundModelProviderCredentialProvider({
|
||||
bindings: credentialStorage,
|
||||
secrets: secretMaterial,
|
||||
audit: credentialStorage,
|
||||
});
|
||||
return loadProjectedModelGatewayProviderAuthority({
|
||||
configFile: options.ai.providerAuthorityFile,
|
||||
credentials,
|
||||
});
|
||||
},
|
||||
audit: options.audit,
|
||||
maxConcurrent: options.ai.maxConcurrent,
|
||||
recoveryLimit: options.ai.recoveryLimit,
|
||||
...(promptOutputKeys === undefined
|
||||
? {}
|
||||
: {
|
||||
promptOutputKeys,
|
||||
promptOutputRead: { authorizer: promptOutputReadAuthorizer },
|
||||
}),
|
||||
});
|
||||
if (promptApplication.status !== 'active') {
|
||||
throw new Error('Cluster AI Prompt application did not activate');
|
||||
}
|
||||
controlApplication = await startControl({
|
||||
...options.control,
|
||||
promptCatalog: {
|
||||
capability: promptApplication.promptCatalog,
|
||||
},
|
||||
promptExecution: {
|
||||
capability: promptApplication.promptExecutions,
|
||||
},
|
||||
promptExecutionInspection: {
|
||||
capability: promptApplication.promptExecutionInspections,
|
||||
},
|
||||
...(promptApplication.promptOutputs === undefined
|
||||
? {}
|
||||
: {
|
||||
promptOutputRead: {
|
||||
capability: promptApplication.promptOutputs,
|
||||
},
|
||||
}),
|
||||
...(promptApplication.promptExecutionOutputs === undefined
|
||||
? {}
|
||||
: {
|
||||
promptExecutionOutputRead: {
|
||||
capability: promptApplication.promptExecutionOutputs,
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (controlApplication.status !== 'active') {
|
||||
throw new Error('AI-enabled cluster-control did not activate');
|
||||
}
|
||||
const activeControl = controlApplication;
|
||||
return Object.freeze({
|
||||
status: 'active' as const,
|
||||
address: activeControl.address,
|
||||
evidence: activeControl.evidence,
|
||||
recovery: activeControl.recovery,
|
||||
unavailable: Promise.race([activeControl.unavailable, aiUnavailable]),
|
||||
availabilityStatus() {
|
||||
return aiUnavailableError
|
||||
? 'unavailable'
|
||||
: activeControl.availabilityStatus();
|
||||
},
|
||||
stop,
|
||||
});
|
||||
} catch (error) {
|
||||
await stop().catch(() => undefined);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
import type {
|
||||
ClusterControlActivationAudit,
|
||||
ClusterControlActivationStack,
|
||||
ClusterControlReadinessEvidence,
|
||||
ClusterControlRuntimeActivationResult,
|
||||
ClusterControlStartupRecoverySummary,
|
||||
ClusterControlStopResult,
|
||||
DeploymentProfile,
|
||||
OpenPostgresDatabase,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
bootstrapClusterControlRuntime,
|
||||
type ClusterControlAssemblyInput,
|
||||
type ClusterControlRecoveryRuntimeOptions,
|
||||
type ClusterRunCancellationConvergenceRuntimeOptions,
|
||||
type ClusterSchedulerRuntimeOptions,
|
||||
type ClusterWorkerRuntimeDependencies,
|
||||
} from './clusterControlRuntime';
|
||||
import { assertClusterControlApiCredentialPepper } from '../authentication/apiCredentialAuthenticator';
|
||||
import {
|
||||
startClusterControlHttpSurface,
|
||||
type ClusterControlAdmissionPipeline,
|
||||
type ClusterControlHttpAddress,
|
||||
type ClusterControlHttpSurfaceOptions,
|
||||
} from '../transport/httpSurface';
|
||||
import type { ClusterControlAvailabilitySource } from '../database/availability';
|
||||
|
||||
export interface ClusterControlApplicationStack {
|
||||
reconcile(): Promise<ClusterControlStartupRecoverySummary>;
|
||||
startLifecycles(): Promise<boolean>;
|
||||
admission: ClusterControlAdmissionPipeline;
|
||||
stop(): Promise<ClusterControlStopResult>;
|
||||
}
|
||||
|
||||
export interface ClusterControlApplicationOptions {
|
||||
readonly enabled?: boolean;
|
||||
readonly profile: DeploymentProfile;
|
||||
readonly apiCredentialPepper?: string;
|
||||
readonly recovery?: ClusterControlRecoveryRuntimeOptions;
|
||||
readonly scheduler?: ClusterSchedulerRuntimeOptions;
|
||||
readonly cancellationConvergence?: ClusterRunCancellationConvergenceRuntimeOptions;
|
||||
readonly workerRuntime?: ClusterWorkerRuntimeDependencies;
|
||||
readonly openDatabase: OpenPostgresDatabase;
|
||||
readonly availability: ClusterControlAvailabilitySource;
|
||||
readonly http: ClusterControlHttpSurfaceOptions;
|
||||
readonly create: (
|
||||
input: ClusterControlAssemblyInput,
|
||||
) => ClusterControlApplicationStack;
|
||||
readonly audit: (
|
||||
record: ClusterControlActivationAudit,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export type ClusterControlApplicationResult =
|
||||
| { readonly status: 'disabled'; stop(): Promise<'stopped'> }
|
||||
| {
|
||||
readonly status: 'active';
|
||||
readonly address: ClusterControlHttpAddress;
|
||||
readonly evidence: ClusterControlReadinessEvidence;
|
||||
readonly recovery: ClusterControlStartupRecoverySummary;
|
||||
readonly unavailable: Promise<Error>;
|
||||
availabilityStatus(): 'ready' | 'unavailable' | 'stopped';
|
||||
stop(): Promise<ClusterControlStopResult>;
|
||||
};
|
||||
|
||||
export class ClusterControlDatabaseUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_CONTROL_DATABASE_UNAVAILABLE';
|
||||
|
||||
constructor() {
|
||||
super('Cluster-control database became unavailable');
|
||||
this.name = 'ClusterControlDatabaseUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function inactiveBootstrap(
|
||||
options: ClusterControlApplicationOptions,
|
||||
): Promise<ClusterControlRuntimeActivationResult> {
|
||||
return bootstrapClusterControlRuntime({
|
||||
...(options.enabled === undefined ? {} : { enabled: options.enabled }),
|
||||
profile: options.profile,
|
||||
...(options.apiCredentialPepper === undefined
|
||||
? {}
|
||||
: { apiCredentialPepper: options.apiCredentialPepper }),
|
||||
...(options.recovery === undefined ? {} : { recovery: options.recovery }),
|
||||
...(options.scheduler === undefined
|
||||
? {}
|
||||
: { scheduler: options.scheduler }),
|
||||
...(options.cancellationConvergence === undefined
|
||||
? {}
|
||||
: { cancellationConvergence: options.cancellationConvergence }),
|
||||
...(options.workerRuntime === undefined
|
||||
? {}
|
||||
: { workerRuntime: options.workerRuntime }),
|
||||
openDatabase: options.openDatabase,
|
||||
create() {
|
||||
throw new Error('Inactive cluster-control unexpectedly created a stack');
|
||||
},
|
||||
audit: options.audit,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the cluster probe surface before database readiness, then installs the
|
||||
* /api/v3 admission handler only after recovery and lifecycles are safe. Stop
|
||||
* withdraws and drains admission before stack, Pool and listener shutdown.
|
||||
*/
|
||||
export async function startClusterControlApplication(
|
||||
options: ClusterControlApplicationOptions,
|
||||
): Promise<ClusterControlApplicationResult> {
|
||||
const enabled = options.enabled ?? false;
|
||||
if (!enabled || options.profile !== 'cluster-control') {
|
||||
const inactive = await inactiveBootstrap(options);
|
||||
if (inactive.status !== 'disabled') {
|
||||
throw new Error('Inactive cluster-control unexpectedly became active');
|
||||
}
|
||||
return inactive;
|
||||
}
|
||||
|
||||
assertClusterControlApiCredentialPepper(options.apiCredentialPepper ?? '');
|
||||
const apiCredentialPepper = options.apiCredentialPepper!;
|
||||
if (
|
||||
!options.availability ||
|
||||
typeof options.availability.subscribe !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster-control availability source is invalid');
|
||||
}
|
||||
|
||||
const http = await startClusterControlHttpSurface(options.http);
|
||||
let activation: ClusterControlRuntimeActivationResult | undefined;
|
||||
let unavailableError: Error | undefined;
|
||||
let unavailableStopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
let availabilityStatus: 'ready' | 'unavailable' | 'stopped' = 'ready';
|
||||
let unsubscribeAvailability: (() => void) | undefined;
|
||||
let resolveUnavailable: ((error: Error) => void) | undefined;
|
||||
const unavailable = new Promise<Error>((resolve) => {
|
||||
resolveUnavailable = resolve;
|
||||
});
|
||||
const withdrawForUnavailable = (error: Error): Promise<void> => {
|
||||
unavailableError ??= error;
|
||||
availabilityStatus = 'unavailable';
|
||||
resolveUnavailable?.(unavailableError);
|
||||
resolveUnavailable = undefined;
|
||||
if (!activation || activation.status === 'disabled')
|
||||
return Promise.resolve();
|
||||
unavailableStopPromise ??= activation.stop();
|
||||
return unavailableStopPromise.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
};
|
||||
try {
|
||||
unsubscribeAvailability = options.availability.subscribe(
|
||||
withdrawForUnavailable,
|
||||
);
|
||||
activation = await bootstrapClusterControlRuntime({
|
||||
enabled: true,
|
||||
profile: options.profile,
|
||||
apiCredentialPepper,
|
||||
...(options.recovery === undefined ? {} : { recovery: options.recovery }),
|
||||
...(options.scheduler === undefined
|
||||
? {}
|
||||
: { scheduler: options.scheduler }),
|
||||
...(options.cancellationConvergence === undefined
|
||||
? {}
|
||||
: { cancellationConvergence: options.cancellationConvergence }),
|
||||
...(options.workerRuntime === undefined
|
||||
? {}
|
||||
: { workerRuntime: options.workerRuntime }),
|
||||
openDatabase: options.openDatabase,
|
||||
create(input): ClusterControlActivationStack {
|
||||
const application = options.create(input);
|
||||
if (
|
||||
!application ||
|
||||
typeof application !== 'object' ||
|
||||
!application.admission ||
|
||||
typeof application.admission.prepare !== 'function'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster-control application stack has no admission pipeline',
|
||||
);
|
||||
}
|
||||
return {
|
||||
reconcile: () => application.reconcile(),
|
||||
startLifecycles: () => application.startLifecycles(),
|
||||
installAdmission: () =>
|
||||
http.installAdmission(input.evidence, application.admission),
|
||||
stop: () => application.stop(),
|
||||
};
|
||||
},
|
||||
audit: options.audit,
|
||||
});
|
||||
if (activation.status === 'disabled') {
|
||||
unsubscribeAvailability();
|
||||
await http.close();
|
||||
return activation;
|
||||
}
|
||||
if (unavailableError) {
|
||||
await withdrawForUnavailable(unavailableError);
|
||||
throw new ClusterControlDatabaseUnavailableError();
|
||||
}
|
||||
const activeActivation = activation;
|
||||
|
||||
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
return {
|
||||
status: 'active',
|
||||
address: http.address,
|
||||
evidence: activeActivation.evidence,
|
||||
recovery: activeActivation.recovery,
|
||||
unavailable,
|
||||
availabilityStatus: () => availabilityStatus,
|
||||
stop() {
|
||||
if (stopPromise) return stopPromise;
|
||||
availabilityStatus = 'stopped';
|
||||
unsubscribeAvailability?.();
|
||||
unsubscribeAvailability = undefined;
|
||||
stopPromise = (async () => {
|
||||
let result: ClusterControlStopResult | undefined;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
result = await activeActivation.stop();
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
await http.close();
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
return result!;
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
unsubscribeAvailability?.();
|
||||
try {
|
||||
await http.close();
|
||||
} catch {
|
||||
// Preserve the readiness/assembly/activation failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,842 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
activateClusterControlRuntime,
|
||||
ClusterRunLostRetryCoordinator,
|
||||
ClusterRunCancellationConvergenceCoordinator,
|
||||
ClusterControlRecoveryEvidenceRegistry,
|
||||
ClusterControlRecoveryConvergenceVerifier,
|
||||
ClusterControlRecoverySupervisor,
|
||||
ClusterControlStartupRecoveryCoordinator,
|
||||
EvidenceBasedClusterControlRecoveryProcessor,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_CLAIMS_PER_PASS,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_CLAIM_LEASE_MS,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_EVIDENCE_TIMEOUT_MS,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_RETRY_DELAY_MS,
|
||||
MAX_CLUSTER_CONTROL_STARTUP_RECOVERY_PASSES,
|
||||
MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGE_SIZE,
|
||||
MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGES_PER_CYCLE,
|
||||
type ClusterControlActivationAudit,
|
||||
type ClusterControlActivationStack,
|
||||
type ClusterControlReadinessEvidence,
|
||||
type ClusterControlRecoveryExecutorEvidenceProvider,
|
||||
type ClusterControlRuntimeActivationResult,
|
||||
type ClusterControlStopResult,
|
||||
type DeploymentProfile,
|
||||
type OpenPostgresDatabase,
|
||||
type PostgresDatabaseResource,
|
||||
type PostgresPool,
|
||||
type ProjectPolicyRepository,
|
||||
type RunRepository,
|
||||
type ClusterRunCancellationConvergenceCycleResult,
|
||||
} from '@qinglong/runtime-core';
|
||||
import type { ClusterRunCancellationRepository } from '@qinglong/runtime-core/cluster-run-cancellation';
|
||||
import type { ProjectRunListReader } from '@qinglong/runtime-core/project-run-list';
|
||||
import type { ClusterScheduleStore } from '@qinglong/runtime-core/cluster-scheduler';
|
||||
import type { TaskDefinitionSource } from '@qinglong/runtime-core/task-definition';
|
||||
import type { TriggerSource } from '@qinglong/runtime-core/trigger';
|
||||
import type { ClusterTaskExecutionRevisionSource } from '@qinglong/runtime-core/cluster-execution-revision';
|
||||
import type {
|
||||
ProjectToolDefinitionSnapshotRepository,
|
||||
ProjectToolDefinitionSnapshotSourceRepository,
|
||||
} from '@qinglong/runtime-core/project-tool-definition-snapshot';
|
||||
import type { StepRunRepository } from '@qinglong/runtime-core/step-run';
|
||||
import type { ToolExecutionCompletionRepository } from '@qinglong/runtime-core/tool-execution-completion';
|
||||
import type { ToolExecutionFailureCompletionRepository } from '@qinglong/runtime-core/tool-execution-failure-completion';
|
||||
import type { ToolExecutionStartBarrierRepository } from '@qinglong/runtime-core/tool-execution-start-barrier';
|
||||
import type { ToolInvocationArtifactRepository } from '@qinglong/runtime-core/tool-invocation-artifact';
|
||||
import type { ToolResultKeyCatalogReader } from '@qinglong/runtime-core/tool-result-key-catalog';
|
||||
import type { ToolExecutionResultRekeyReader } from '@qinglong/runtime-core/tool-result-rekey';
|
||||
import {
|
||||
assertPostgresSchemaReady,
|
||||
PostgresClusterControlRecoverySource,
|
||||
PostgresClusterControlRecoveryClaimRepository,
|
||||
PostgresClusterControlRecoveryResolutionRepository,
|
||||
PostgresClusterRuntimeRecoverySource,
|
||||
PostgresClusterRunLostRetryRepository,
|
||||
PostgresClusterRunCancellationRepository,
|
||||
PostgresClusterRunCancellationConvergenceRepository,
|
||||
PostgresProjectPolicyRepository,
|
||||
PostgresApiCredentialRepository,
|
||||
PostgresSecurityAuditRepository,
|
||||
PostgresRunRepository,
|
||||
PostgresWorkerExecutionAttestationRepository,
|
||||
PostgresTaskDefinitionSource,
|
||||
PostgresTaskExecutionRevisionSource,
|
||||
PostgresTriggerSource,
|
||||
PostgresClusterScheduleRepository,
|
||||
PostgresRemoteWorkerAttestationEvidenceProvider,
|
||||
PostgresProjectToolDefinitionSnapshotRepository,
|
||||
PostgresStepRunRepository,
|
||||
PostgresToolExecutionCompletionRepository,
|
||||
PostgresToolExecutionFailureCompletionRepository,
|
||||
PostgresToolExecutionStartBarrierRepository,
|
||||
PostgresToolInvocationArtifactRepository,
|
||||
PostgresToolResultKeyCatalogReader,
|
||||
PostgresToolResultRekeyReader,
|
||||
type PostgresSchemaReadinessReport,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
import { PostgresPluginPackageWorkflowFrontierRepository } from '@qinglong/cluster-postgres/plugin-package-workflow-frontier';
|
||||
import { PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository } from '@qinglong/cluster-postgres/plugin-package-workflow-task-attempt-admission';
|
||||
import { PostgresTaskStartRepository } from '@qinglong/cluster-postgres/task-start';
|
||||
import { PostgresPluginPackageAutomationPublicationRepository } from '@qinglong/cluster-postgres/plugin-package-automation-publication';
|
||||
import { PostgresPluginPackageMaterializedRevisionRepository } from '@qinglong/cluster-postgres/plugin-package-materialized-revision';
|
||||
import {
|
||||
PostgresAuthorizedPluginPackageWorkflowAdmissionRepository,
|
||||
PostgresAuthorizedPluginPackageWorkflowRunEventListRepository,
|
||||
PostgresAuthorizedPluginPackageWorkflowRunInspectionRepository,
|
||||
PostgresAuthorizedPluginPackageWorkflowRunListRepository,
|
||||
PostgresAuthorizedPluginPackageWorkflowStepRunListRepository,
|
||||
} from '@qinglong/cluster-postgres/plugin-package-workflow-administration';
|
||||
import {
|
||||
assertClusterControlApiCredentialPepper,
|
||||
createClusterControlApiCredentialAuthenticator,
|
||||
} from '../authentication/apiCredentialAuthenticator';
|
||||
import type {
|
||||
ClusterControlRequestAuthenticator,
|
||||
ClusterControlSecurityAuditSink,
|
||||
} from '../transport/admissionPipeline';
|
||||
import {
|
||||
MAX_CLUSTER_SCHEDULE_CLAIMS_PER_CYCLE,
|
||||
ClusterSchedulerCoordinator,
|
||||
ClusterSchedulerLifecycle,
|
||||
type ClusterSchedulerCycleSummary,
|
||||
} from '../scheduling/scheduler';
|
||||
import { ClusterWorkflowSchedulerCoordinator } from '../scheduling/workflowScheduler';
|
||||
import { ClusterRuntimeSchedulerCoordinator } from '../scheduling/runtimeScheduler';
|
||||
import { ClusterRunCancellationConvergenceLifecycle } from '../run/runCancellationLifecycle';
|
||||
import type { TaskStartRepository } from '@qinglong/runtime-core/task-start';
|
||||
import {
|
||||
createClusterWorkerRuntimePort,
|
||||
type ClusterWorkerRuntimeDependencies,
|
||||
type ClusterWorkerRuntimePort,
|
||||
} from '../remote-execution/workerRuntimePort';
|
||||
import {
|
||||
createClusterPluginPackageWorkflowAdministrationCapability,
|
||||
type ClusterPluginPackageWorkflowAdministrationCapability,
|
||||
} from '../plugin-package/workflow/pluginPackageWorkflowAdministration';
|
||||
|
||||
export interface ClusterTrustedToolStorage {
|
||||
readonly invocationArtifacts: ToolInvocationArtifactRepository;
|
||||
readonly stepRuns: StepRunRepository;
|
||||
readonly startBarriers: ToolExecutionStartBarrierRepository;
|
||||
readonly completions: ToolExecutionCompletionRepository;
|
||||
readonly failureCompletions: ToolExecutionFailureCompletionRepository;
|
||||
readonly resultKeyCatalog: ToolResultKeyCatalogReader;
|
||||
readonly resultRekeys: ToolExecutionResultRekeyReader;
|
||||
readonly toolDefinitionSnapshots: ProjectToolDefinitionSnapshotRepository &
|
||||
ProjectToolDefinitionSnapshotSourceRepository;
|
||||
}
|
||||
|
||||
export interface ClusterControlAssemblyInput {
|
||||
readonly evidence: ClusterControlReadinessEvidence;
|
||||
readonly policies: ProjectPolicyRepository;
|
||||
readonly runs: RunRepository & ProjectRunListReader;
|
||||
readonly runCancellation: ClusterRunCancellationRepository;
|
||||
readonly taskStart: TaskStartRepository;
|
||||
readonly taskDefinitions: TaskDefinitionSource;
|
||||
readonly taskExecutionRevisions: ClusterTaskExecutionRevisionSource;
|
||||
readonly triggers: TriggerSource;
|
||||
readonly schedules: ClusterScheduleStore;
|
||||
readonly trustedToolStorage: ClusterTrustedToolStorage;
|
||||
readonly authenticator: ClusterControlRequestAuthenticator;
|
||||
readonly securityAudit: ClusterControlSecurityAuditSink;
|
||||
readonly workflowAdministration: ClusterPluginPackageWorkflowAdministrationCapability;
|
||||
readonly workerRuntime?: ClusterWorkerRuntimePort;
|
||||
}
|
||||
|
||||
export interface ClusterControlRecoveryRuntimeOptions {
|
||||
readonly ownerId: string;
|
||||
readonly providers?: readonly ClusterControlRecoveryExecutorEvidenceProvider[];
|
||||
readonly claimLimit?: number;
|
||||
readonly claimLeaseMs?: number;
|
||||
readonly retryDelayMs?: number;
|
||||
readonly providerTimeoutMs?: number;
|
||||
readonly maxStartupPasses?: number;
|
||||
}
|
||||
|
||||
export interface ClusterSchedulerRuntimeOptions {
|
||||
readonly ownerId?: string;
|
||||
readonly claimLeaseMs?: number;
|
||||
readonly maxClaimsPerCycle?: number;
|
||||
readonly misfireGraceMs?: number;
|
||||
readonly intervalMs?: number;
|
||||
readonly stopTimeoutMs?: number;
|
||||
readonly onDiagnostic?: (
|
||||
error: unknown,
|
||||
summary?: ClusterSchedulerCycleSummary,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ClusterRunCancellationConvergenceRuntimeOptions {
|
||||
readonly pageSize?: number;
|
||||
readonly maxPages?: number;
|
||||
readonly intervalMs?: number;
|
||||
readonly stopTimeoutMs?: number;
|
||||
readonly onDiagnostic?: (
|
||||
error: unknown,
|
||||
summary?: Readonly<ClusterRunCancellationConvergenceCycleResult>,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ClusterControlBootstrapOptions {
|
||||
readonly enabled?: boolean;
|
||||
readonly profile: DeploymentProfile;
|
||||
readonly apiCredentialPepper?: string;
|
||||
readonly recovery?: ClusterControlRecoveryRuntimeOptions;
|
||||
readonly scheduler?: ClusterSchedulerRuntimeOptions;
|
||||
readonly cancellationConvergence?: ClusterRunCancellationConvergenceRuntimeOptions;
|
||||
readonly workerRuntime?: ClusterWorkerRuntimeDependencies;
|
||||
readonly openDatabase: OpenPostgresDatabase;
|
||||
readonly create: (
|
||||
input: ClusterControlAssemblyInput,
|
||||
) => ClusterControlActivationStack;
|
||||
readonly audit: (
|
||||
record: ClusterControlActivationAudit,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
interface PreparedRecoveryRuntime {
|
||||
readonly providers: readonly ClusterControlRecoveryExecutorEvidenceProvider[];
|
||||
readonly providerTimeoutMs: number;
|
||||
readonly ownerId: string;
|
||||
readonly claimLimit: number;
|
||||
readonly claimLeaseMs: number;
|
||||
readonly retryDelayMs: number;
|
||||
readonly maxStartupPasses: number;
|
||||
}
|
||||
|
||||
interface PreparedSchedulerRuntime {
|
||||
readonly ownerId: string;
|
||||
readonly claimLeaseMs: number;
|
||||
readonly maxClaimsPerCycle: number;
|
||||
readonly misfireGraceMs: number;
|
||||
readonly intervalMs: number;
|
||||
readonly stopTimeoutMs: number;
|
||||
readonly onDiagnostic?: ClusterSchedulerRuntimeOptions['onDiagnostic'];
|
||||
}
|
||||
|
||||
interface PreparedCancellationConvergenceRuntime {
|
||||
readonly pageSize: number;
|
||||
readonly maxPages: number;
|
||||
readonly intervalMs: number;
|
||||
readonly stopTimeoutMs: number;
|
||||
readonly onDiagnostic?: ClusterRunCancellationConvergenceRuntimeOptions['onDiagnostic'];
|
||||
}
|
||||
|
||||
function boundedInteger(
|
||||
name: string,
|
||||
value: number | undefined,
|
||||
fallback: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
const normalized = value ?? fallback;
|
||||
if (
|
||||
!Number.isSafeInteger(normalized) ||
|
||||
normalized < minimum ||
|
||||
normalized > maximum
|
||||
) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function prepareRecoveryRuntime(
|
||||
options: ClusterControlRecoveryRuntimeOptions | undefined,
|
||||
): PreparedRecoveryRuntime {
|
||||
if (!options) {
|
||||
throw new TypeError(
|
||||
'Enabled cluster-control requires bounded recovery configuration',
|
||||
);
|
||||
}
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(options.ownerId)) {
|
||||
throw new TypeError('Cluster-control recovery ownerId is invalid');
|
||||
}
|
||||
const claimLeaseMs = boundedInteger(
|
||||
'Cluster-control recovery claim lease',
|
||||
options.claimLeaseMs,
|
||||
30_000,
|
||||
1_000,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_CLAIM_LEASE_MS,
|
||||
);
|
||||
const providerTimeoutMs = boundedInteger(
|
||||
'Cluster-control recovery evidence timeout',
|
||||
options.providerTimeoutMs,
|
||||
5_000,
|
||||
1,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_EVIDENCE_TIMEOUT_MS,
|
||||
);
|
||||
if (providerTimeoutMs + 250 > claimLeaseMs) {
|
||||
throw new RangeError(
|
||||
'Cluster-control recovery evidence timeout must leave at least 250ms for fenced settlement',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
providers: Object.freeze([...(options.providers ?? [])]),
|
||||
providerTimeoutMs,
|
||||
ownerId: options.ownerId,
|
||||
claimLimit: boundedInteger(
|
||||
'Cluster-control recovery claim limit',
|
||||
options.claimLimit,
|
||||
16,
|
||||
1,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_CLAIMS_PER_PASS,
|
||||
),
|
||||
claimLeaseMs,
|
||||
retryDelayMs: boundedInteger(
|
||||
'Cluster-control recovery retry delay',
|
||||
options.retryDelayMs,
|
||||
5_000,
|
||||
0,
|
||||
MAX_CLUSTER_CONTROL_RECOVERY_RETRY_DELAY_MS,
|
||||
),
|
||||
maxStartupPasses: boundedInteger(
|
||||
'Cluster-control startup recovery passes',
|
||||
options.maxStartupPasses,
|
||||
8,
|
||||
1,
|
||||
MAX_CLUSTER_CONTROL_STARTUP_RECOVERY_PASSES,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function prepareSchedulerRuntime(
|
||||
options: ClusterSchedulerRuntimeOptions | undefined,
|
||||
fallbackOwnerId: string,
|
||||
): PreparedSchedulerRuntime {
|
||||
const allowedKeys = new Set([
|
||||
'claimLeaseMs',
|
||||
'intervalMs',
|
||||
'maxClaimsPerCycle',
|
||||
'misfireGraceMs',
|
||||
'onDiagnostic',
|
||||
'ownerId',
|
||||
'stopTimeoutMs',
|
||||
]);
|
||||
if (
|
||||
options !== undefined &&
|
||||
(!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => !allowedKeys.has(key)))
|
||||
) {
|
||||
throw new TypeError('Cluster scheduler configuration is invalid');
|
||||
}
|
||||
const ownerId = options?.ownerId ?? fallbackOwnerId;
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(ownerId)) {
|
||||
throw new TypeError('Cluster scheduler ownerId is invalid');
|
||||
}
|
||||
if (
|
||||
options?.onDiagnostic !== undefined &&
|
||||
typeof options.onDiagnostic !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster scheduler diagnostic sink is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
ownerId,
|
||||
claimLeaseMs: boundedInteger(
|
||||
'Cluster scheduler claim lease',
|
||||
options?.claimLeaseMs,
|
||||
30_000,
|
||||
1_000,
|
||||
60_000,
|
||||
),
|
||||
maxClaimsPerCycle: boundedInteger(
|
||||
'Cluster scheduler claim budget',
|
||||
options?.maxClaimsPerCycle,
|
||||
16,
|
||||
1,
|
||||
MAX_CLUSTER_SCHEDULE_CLAIMS_PER_CYCLE,
|
||||
),
|
||||
misfireGraceMs: boundedInteger(
|
||||
'Cluster scheduler misfire grace',
|
||||
options?.misfireGraceMs,
|
||||
30_000,
|
||||
0,
|
||||
5 * 60_000,
|
||||
),
|
||||
intervalMs: boundedInteger(
|
||||
'Cluster scheduler interval',
|
||||
options?.intervalMs,
|
||||
1_000,
|
||||
250,
|
||||
60 * 60_000,
|
||||
),
|
||||
stopTimeoutMs: boundedInteger(
|
||||
'Cluster scheduler stop timeout',
|
||||
options?.stopTimeoutMs,
|
||||
10_000,
|
||||
100,
|
||||
30_000,
|
||||
),
|
||||
...(options?.onDiagnostic === undefined
|
||||
? {}
|
||||
: { onDiagnostic: options.onDiagnostic }),
|
||||
});
|
||||
}
|
||||
|
||||
function prepareCancellationConvergenceRuntime(
|
||||
options: ClusterRunCancellationConvergenceRuntimeOptions | undefined,
|
||||
): PreparedCancellationConvergenceRuntime {
|
||||
const allowedKeys = new Set([
|
||||
'intervalMs',
|
||||
'maxPages',
|
||||
'onDiagnostic',
|
||||
'pageSize',
|
||||
'stopTimeoutMs',
|
||||
]);
|
||||
if (
|
||||
options !== undefined &&
|
||||
(!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => !allowedKeys.has(key)))
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster Run cancellation convergence configuration is invalid',
|
||||
);
|
||||
}
|
||||
if (
|
||||
options?.onDiagnostic !== undefined &&
|
||||
typeof options.onDiagnostic !== 'function'
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster Run cancellation convergence diagnostic sink is invalid',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
pageSize: boundedInteger(
|
||||
'Cluster Run cancellation convergence page size',
|
||||
options?.pageSize,
|
||||
32,
|
||||
1,
|
||||
MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGE_SIZE,
|
||||
),
|
||||
maxPages: boundedInteger(
|
||||
'Cluster Run cancellation convergence page limit',
|
||||
options?.maxPages,
|
||||
4,
|
||||
1,
|
||||
MAX_CLUSTER_RUN_CANCELLATION_CONVERGENCE_PAGES_PER_CYCLE,
|
||||
),
|
||||
intervalMs: boundedInteger(
|
||||
'Cluster Run cancellation convergence interval',
|
||||
options?.intervalMs,
|
||||
1_000,
|
||||
250,
|
||||
60 * 60_000,
|
||||
),
|
||||
stopTimeoutMs: boundedInteger(
|
||||
'Cluster Run cancellation convergence stop timeout',
|
||||
options?.stopTimeoutMs,
|
||||
10_000,
|
||||
100,
|
||||
30_000,
|
||||
),
|
||||
...(options?.onDiagnostic === undefined
|
||||
? {}
|
||||
: { onDiagnostic: options.onDiagnostic }),
|
||||
});
|
||||
}
|
||||
|
||||
function readinessEvidence(
|
||||
report: PostgresSchemaReadinessReport,
|
||||
): ClusterControlReadinessEvidence {
|
||||
return Object.freeze({
|
||||
contractName: report.contractName,
|
||||
contractVersion: report.contractVersion,
|
||||
serverMajor: report.serverMajor,
|
||||
migrationIds: Object.freeze([...report.migrationIds]),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns the cluster database around the readiness-first activation gate.
|
||||
* Repository and service construction happens through create() only after the
|
||||
* runtime role, migration history and catalog contract are proven ready.
|
||||
*/
|
||||
export async function bootstrapClusterControlRuntime(
|
||||
options: ClusterControlBootstrapOptions,
|
||||
): Promise<ClusterControlRuntimeActivationResult> {
|
||||
let recoveryRuntime: PreparedRecoveryRuntime | undefined;
|
||||
let schedulerRuntime: PreparedSchedulerRuntime | undefined;
|
||||
let cancellationConvergenceRuntime:
|
||||
| PreparedCancellationConvergenceRuntime
|
||||
| undefined;
|
||||
let recoveryRegistry: ClusterControlRecoveryEvidenceRegistry | undefined;
|
||||
if ((options.enabled ?? false) && options.profile === 'cluster-control') {
|
||||
assertClusterControlApiCredentialPepper(options.apiCredentialPepper ?? '');
|
||||
recoveryRuntime = prepareRecoveryRuntime(options.recovery);
|
||||
schedulerRuntime = prepareSchedulerRuntime(
|
||||
options.scheduler,
|
||||
recoveryRuntime.ownerId,
|
||||
);
|
||||
cancellationConvergenceRuntime = prepareCancellationConvergenceRuntime(
|
||||
options.cancellationConvergence,
|
||||
);
|
||||
}
|
||||
let database: PostgresDatabaseResource | undefined;
|
||||
let closePromise: Promise<void> | undefined;
|
||||
const closeDatabase = (): Promise<void> => {
|
||||
if (!database) return Promise.resolve();
|
||||
closePromise ??= Promise.resolve().then(() => database!.close());
|
||||
return closePromise;
|
||||
};
|
||||
|
||||
try {
|
||||
const activation = await activateClusterControlRuntime({
|
||||
...(options.enabled === undefined ? {} : { enabled: options.enabled }),
|
||||
profile: options.profile,
|
||||
readiness: {
|
||||
async assertReady() {
|
||||
if (database) {
|
||||
throw new Error(
|
||||
'Cluster-control database was opened more than once',
|
||||
);
|
||||
}
|
||||
database = await options.openDatabase();
|
||||
return readinessEvidence(
|
||||
await assertPostgresSchemaReady(database.pool),
|
||||
);
|
||||
},
|
||||
},
|
||||
create(evidence) {
|
||||
if (!database) {
|
||||
throw new Error(
|
||||
'Cluster-control database is unavailable after readiness',
|
||||
);
|
||||
}
|
||||
const recovery = new PostgresClusterControlRecoverySource(
|
||||
database.pool,
|
||||
);
|
||||
const recoveryClaims =
|
||||
new PostgresClusterControlRecoveryClaimRepository(database.pool);
|
||||
const recoveryTransitions =
|
||||
new PostgresClusterControlRecoveryResolutionRepository(database.pool);
|
||||
if (!recoveryRuntime) {
|
||||
throw new Error(
|
||||
'Cluster-control recovery runtime is unavailable after readiness',
|
||||
);
|
||||
}
|
||||
if (!schedulerRuntime) {
|
||||
throw new Error(
|
||||
'Cluster scheduler runtime is unavailable after readiness',
|
||||
);
|
||||
}
|
||||
if (!cancellationConvergenceRuntime) {
|
||||
throw new Error(
|
||||
'Cluster Run cancellation convergence runtime is unavailable after readiness',
|
||||
);
|
||||
}
|
||||
const remoteAttestations =
|
||||
new PostgresWorkerExecutionAttestationRepository(database.pool);
|
||||
recoveryRegistry = new ClusterControlRecoveryEvidenceRegistry(
|
||||
[
|
||||
...recoveryRuntime.providers,
|
||||
new PostgresRemoteWorkerAttestationEvidenceProvider(
|
||||
database.pool,
|
||||
remoteAttestations,
|
||||
),
|
||||
],
|
||||
{ timeoutMs: recoveryRuntime.providerTimeoutMs },
|
||||
);
|
||||
const recoveryProcessor =
|
||||
new EvidenceBasedClusterControlRecoveryProcessor(
|
||||
recoveryTransitions,
|
||||
recoveryRegistry,
|
||||
{ retryDelayMs: recoveryRuntime.retryDelayMs },
|
||||
);
|
||||
const recoverySupervisor = new ClusterControlRecoverySupervisor(
|
||||
recoveryClaims,
|
||||
recoveryProcessor,
|
||||
{
|
||||
ownerId: recoveryRuntime.ownerId,
|
||||
limit: recoveryRuntime.claimLimit,
|
||||
leaseMs: recoveryRuntime.claimLeaseMs,
|
||||
retryDelayMs: recoveryRuntime.retryDelayMs,
|
||||
},
|
||||
);
|
||||
const recoveryCoordinator =
|
||||
new ClusterControlStartupRecoveryCoordinator(recoverySupervisor, {
|
||||
maxPasses: recoveryRuntime.maxStartupPasses,
|
||||
});
|
||||
const runtimeRecoverySupervisor = new ClusterControlRecoverySupervisor(
|
||||
new PostgresClusterControlRecoveryClaimRepository(
|
||||
database.pool,
|
||||
randomUUID,
|
||||
(queryable) => new PostgresClusterRuntimeRecoverySource(queryable),
|
||||
),
|
||||
recoveryProcessor,
|
||||
{
|
||||
ownerId: recoveryRuntime.ownerId,
|
||||
limit: recoveryRuntime.claimLimit,
|
||||
leaseMs: recoveryRuntime.claimLeaseMs,
|
||||
retryDelayMs: recoveryRuntime.retryDelayMs,
|
||||
},
|
||||
);
|
||||
const schedules = new PostgresClusterScheduleRepository(database.pool);
|
||||
const runs = new PostgresRunRepository(database.pool);
|
||||
const trustedToolStorage: ClusterTrustedToolStorage = Object.freeze({
|
||||
invocationArtifacts: new PostgresToolInvocationArtifactRepository(
|
||||
database.pool,
|
||||
),
|
||||
stepRuns: new PostgresStepRunRepository(database.pool),
|
||||
startBarriers: new PostgresToolExecutionStartBarrierRepository(
|
||||
database.pool,
|
||||
),
|
||||
completions: new PostgresToolExecutionCompletionRepository(
|
||||
database.pool,
|
||||
),
|
||||
failureCompletions:
|
||||
new PostgresToolExecutionFailureCompletionRepository(database.pool),
|
||||
resultKeyCatalog: new PostgresToolResultKeyCatalogReader(
|
||||
database.pool,
|
||||
),
|
||||
resultRekeys: new PostgresToolResultRekeyReader(database.pool),
|
||||
toolDefinitionSnapshots:
|
||||
new PostgresProjectToolDefinitionSnapshotRepository(database.pool),
|
||||
});
|
||||
const workflowScheduler = new ClusterWorkflowSchedulerCoordinator(
|
||||
new ClusterSchedulerCoordinator(schedules, {
|
||||
ownerId: schedulerRuntime.ownerId,
|
||||
claimLeaseMs: schedulerRuntime.claimLeaseMs,
|
||||
maxClaimsPerCycle: schedulerRuntime.maxClaimsPerCycle,
|
||||
misfireGraceMs: schedulerRuntime.misfireGraceMs,
|
||||
}),
|
||||
new PostgresPluginPackageWorkflowFrontierRepository(database.pool),
|
||||
new PostgresPluginPackageWorkflowTaskAttemptAdmissionRepository(
|
||||
database.pool,
|
||||
),
|
||||
{
|
||||
frontierPageSize: 32,
|
||||
frontierMaxPages: 4,
|
||||
taskAttemptPageSize: 32,
|
||||
taskAttemptMaxPages: 4,
|
||||
},
|
||||
);
|
||||
const runtimeScheduler = new ClusterRuntimeSchedulerCoordinator(
|
||||
runtimeRecoverySupervisor,
|
||||
new ClusterRunLostRetryCoordinator(
|
||||
new PostgresClusterRunLostRetryRepository(database.pool),
|
||||
{ pageSize: 16 },
|
||||
),
|
||||
workflowScheduler,
|
||||
);
|
||||
const schedulerLifecycle = new ClusterSchedulerLifecycle(
|
||||
runtimeScheduler,
|
||||
{
|
||||
intervalMs: schedulerRuntime.intervalMs,
|
||||
stopTimeoutMs: schedulerRuntime.stopTimeoutMs,
|
||||
...(schedulerRuntime.onDiagnostic === undefined
|
||||
? {}
|
||||
: { onDiagnostic: schedulerRuntime.onDiagnostic }),
|
||||
},
|
||||
);
|
||||
const cancellationConvergenceLifecycle =
|
||||
new ClusterRunCancellationConvergenceLifecycle(
|
||||
new ClusterRunCancellationConvergenceCoordinator(
|
||||
new PostgresClusterRunCancellationConvergenceRepository(
|
||||
database.pool,
|
||||
),
|
||||
{
|
||||
pageSize: cancellationConvergenceRuntime.pageSize,
|
||||
maxPages: cancellationConvergenceRuntime.maxPages,
|
||||
},
|
||||
),
|
||||
{
|
||||
intervalMs: cancellationConvergenceRuntime.intervalMs,
|
||||
stopTimeoutMs: cancellationConvergenceRuntime.stopTimeoutMs,
|
||||
...(cancellationConvergenceRuntime.onDiagnostic === undefined
|
||||
? {}
|
||||
: {
|
||||
onDiagnostic: cancellationConvergenceRuntime.onDiagnostic,
|
||||
}),
|
||||
},
|
||||
);
|
||||
const runCancellation = new PostgresClusterRunCancellationRepository(
|
||||
database.pool,
|
||||
);
|
||||
const taskStart = new PostgresTaskStartRepository(database.pool);
|
||||
const application = options.create({
|
||||
evidence,
|
||||
authenticator: createClusterControlApiCredentialAuthenticator(
|
||||
new PostgresApiCredentialRepository(database.pool),
|
||||
options.apiCredentialPepper ?? '',
|
||||
),
|
||||
policies: new PostgresProjectPolicyRepository(database.pool),
|
||||
runs,
|
||||
runCancellation,
|
||||
taskStart,
|
||||
taskDefinitions: new PostgresTaskDefinitionSource(database.pool),
|
||||
taskExecutionRevisions: new PostgresTaskExecutionRevisionSource(
|
||||
database.pool,
|
||||
),
|
||||
triggers: new PostgresTriggerSource(database.pool),
|
||||
schedules,
|
||||
trustedToolStorage,
|
||||
securityAudit: new PostgresSecurityAuditRepository(database.pool),
|
||||
workflowAdministration:
|
||||
createClusterPluginPackageWorkflowAdministrationCapability(
|
||||
new PostgresPluginPackageAutomationPublicationRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresPluginPackageMaterializedRevisionRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresAuthorizedPluginPackageWorkflowAdmissionRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresAuthorizedPluginPackageWorkflowRunInspectionRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresAuthorizedPluginPackageWorkflowRunListRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresAuthorizedPluginPackageWorkflowStepRunListRepository(
|
||||
database.pool,
|
||||
),
|
||||
new PostgresAuthorizedPluginPackageWorkflowRunEventListRepository(
|
||||
database.pool,
|
||||
),
|
||||
runCancellation,
|
||||
),
|
||||
...(options.workerRuntime === undefined
|
||||
? {}
|
||||
: {
|
||||
workerRuntime: createClusterWorkerRuntimePort(
|
||||
database.pool,
|
||||
options.workerRuntime,
|
||||
),
|
||||
}),
|
||||
});
|
||||
const convergence = new ClusterControlRecoveryConvergenceVerifier(
|
||||
recovery,
|
||||
);
|
||||
return {
|
||||
async reconcile() {
|
||||
const outstanding = await convergence.verify();
|
||||
if (!outstanding.safe) {
|
||||
const system = await recoveryCoordinator.reconcile();
|
||||
if (
|
||||
!system.safe ||
|
||||
system.remaining !== 0 ||
|
||||
system.failed !== 0
|
||||
) {
|
||||
return system;
|
||||
}
|
||||
}
|
||||
const summary = await application.reconcile();
|
||||
if (
|
||||
!summary.safe ||
|
||||
summary.remaining !== 0 ||
|
||||
summary.failed !== 0
|
||||
) {
|
||||
return summary;
|
||||
}
|
||||
return convergence.verify();
|
||||
},
|
||||
async startLifecycles() {
|
||||
if (!(await application.startLifecycles())) return false;
|
||||
schedulerLifecycle.start();
|
||||
cancellationConvergenceLifecycle.start();
|
||||
return true;
|
||||
},
|
||||
installAdmission: () => application.installAdmission(),
|
||||
async stop() {
|
||||
recoveryRegistry?.dispose();
|
||||
let schedulerStatus: 'stopped' | 'timed_out' = 'stopped';
|
||||
let cancellationStatus: 'stopped' | 'timed_out' = 'stopped';
|
||||
let applicationStatus: ClusterControlStopResult = 'stopped';
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
cancellationStatus = (
|
||||
await cancellationConvergenceLifecycle.stopAndDrain()
|
||||
).status;
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
schedulerStatus = (await schedulerLifecycle.stopAndDrain())
|
||||
.status;
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
try {
|
||||
applicationStatus = await application.stop();
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
return cancellationStatus === 'timed_out' ||
|
||||
schedulerStatus === 'timed_out' ||
|
||||
applicationStatus === 'timed_out'
|
||||
? 'timed_out'
|
||||
: 'stopped';
|
||||
},
|
||||
};
|
||||
},
|
||||
audit: options.audit,
|
||||
});
|
||||
if (activation.status === 'disabled') return activation;
|
||||
|
||||
let stopPromise: Promise<ClusterControlStopResult> | undefined;
|
||||
return {
|
||||
...activation,
|
||||
stop() {
|
||||
if (stopPromise) return stopPromise;
|
||||
stopPromise = (async () => {
|
||||
let result: ClusterControlStopResult | undefined;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
result = await activation.stop();
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
await closeDatabase();
|
||||
} catch (error) {
|
||||
primaryError ??= error;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
return result!;
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
recoveryRegistry?.dispose();
|
||||
try {
|
||||
await closeDatabase();
|
||||
} catch {
|
||||
// Preserve the readiness/assembly/activation failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export type {
|
||||
ClusterControlActivationAudit,
|
||||
ClusterControlActivationStack,
|
||||
ClusterControlReadinessEvidence,
|
||||
ClusterControlRuntimeActivationResult,
|
||||
ClusterControlStopResult,
|
||||
DeploymentProfile,
|
||||
OpenPostgresDatabase,
|
||||
PostgresDatabaseResource,
|
||||
PostgresPool,
|
||||
ProjectPolicyRepository,
|
||||
RunRepository,
|
||||
ClusterRunCancellationRepository,
|
||||
ClusterControlRequestAuthenticator,
|
||||
ClusterControlSecurityAuditSink,
|
||||
ClusterControlRecoveryExecutorEvidenceProvider,
|
||||
ClusterScheduleStore,
|
||||
};
|
||||
|
||||
export { ClusterRunCancellationConvergenceLifecycle } from '../run/runCancellationLifecycle';
|
||||
|
||||
export * from '../scheduling/scheduler';
|
||||
export * from '../scheduling/workflowScheduler';
|
||||
export * from '../scheduling/runtimeScheduler';
|
||||
export * from '../remote-execution/remoteWorkerDispatcher';
|
||||
export * from '../remote-execution/workerRuntimePort';
|
||||
@@ -0,0 +1,410 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type {
|
||||
ClusterControlStartupRecoverySummary,
|
||||
ClusterControlStopResult,
|
||||
} from '@qinglong/runtime-core';
|
||||
import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import {
|
||||
startClusterControlApplication,
|
||||
type ClusterControlApplicationOptions,
|
||||
type ClusterControlApplicationResult,
|
||||
type ClusterControlApplicationStack,
|
||||
} from './application';
|
||||
import {
|
||||
createClusterControlDatabaseBinding,
|
||||
type EnabledClusterControlConfig,
|
||||
} from '../production-process/config';
|
||||
import {
|
||||
createClusterControlAdmissionPipeline,
|
||||
createClusterControlProjectPolicyAuthorizer,
|
||||
} from '../transport/admissionPipeline';
|
||||
import { createClusterControlRouteRegistry } from '../transport/routeRegistry';
|
||||
import { CLUSTER_CONTROL_HTTP_DEFAULTS } from '../transport/httpSurface';
|
||||
import { createClusterControlRunReadRoute } from '../run/runReadRoute';
|
||||
import { createClusterControlRunListRoute } from '../run/runListRoute';
|
||||
import { createClusterControlRunEventListRoute } from '../run/runEventListRoute';
|
||||
import { createClusterControlRunStepListRoute } from '../run/runStepListRoute';
|
||||
import { createClusterControlTaskListRoute } from '../task/taskListRoute';
|
||||
import { createClusterControlTaskReadRoute } from '../task/taskReadRoute';
|
||||
import { createClusterControlTaskStartRoute } from '../task/taskStartRoute';
|
||||
import {
|
||||
createClusterControlPluginPackagePromptExecutionRoute,
|
||||
type ClusterPluginPackagePromptExecutionCapability,
|
||||
} from '../plugin-package/prompt/pluginPackagePromptExecutionRoute';
|
||||
import {
|
||||
createClusterControlPluginPackagePromptCatalogRoute,
|
||||
type ClusterPluginPackagePromptCatalogCapability,
|
||||
} from '../plugin-package/prompt/pluginPackagePromptCatalogRoute';
|
||||
import {
|
||||
createClusterControlPluginPackagePromptOutputReadRoute,
|
||||
type ClusterPluginPackagePromptOutputReadCapability,
|
||||
} from '../plugin-package/prompt/pluginPackagePromptOutputReadRoute';
|
||||
import {
|
||||
createClusterControlPluginPackagePromptExecutionInspectionRoute,
|
||||
type ClusterPluginPackagePromptExecutionInspectionCapability,
|
||||
} from '../plugin-package/prompt/pluginPackagePromptExecutionInspectionRoute';
|
||||
import {
|
||||
createClusterControlPluginPackagePromptExecutionOutputReadRoute,
|
||||
type ClusterPluginPackagePromptExecutionOutputReadCapability,
|
||||
} from '../plugin-package/prompt/pluginPackagePromptExecutionOutputReadRoute';
|
||||
import {
|
||||
createClusterControlRunCancellationRoute,
|
||||
type ClusterRunCancellationEventIdFactory,
|
||||
} from '../run/runCancellationRoute';
|
||||
import type { ClusterControlAssemblyInput } from './clusterControlRuntime';
|
||||
import type { ClusterRemoteWorkerArtifactStore } from '../remote-execution/remoteWorkerCompletionService';
|
||||
import type { EnabledClusterWorkerIngressConfig } from '../worker-ingress/workerIngressConfig';
|
||||
import {
|
||||
startProductionClusterWorkerIngress,
|
||||
type ProductionClusterWorkerIngressOptions as ProductionClusterWorkerIngressStarterOptions,
|
||||
} from '../worker-ingress/productionWorkerIngress';
|
||||
import type { ClusterWorkerIngressApplicationResult } from '../worker-ingress/workerIngressApplication';
|
||||
import { createClusterControlPluginPackageWorkflowRoutes } from '../plugin-package/workflow/pluginPackageWorkflowRoute';
|
||||
|
||||
export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
|
||||
'task.get',
|
||||
'task.list',
|
||||
'task.start',
|
||||
'run.get',
|
||||
'run.list',
|
||||
'run.events.list',
|
||||
'run.steps.list',
|
||||
'run.cancel',
|
||||
'workflow.read',
|
||||
'workflow.run.read',
|
||||
'workflow.run.list',
|
||||
'workflow.step.list',
|
||||
'workflow.event.list',
|
||||
'workflow.start',
|
||||
'workflow.cancel',
|
||||
] as const);
|
||||
export const PRODUCTION_CLUSTER_CONTROL_OPTIONAL_ROUTE_OPERATIONS =
|
||||
Object.freeze([
|
||||
'prompt.read',
|
||||
'prompt.execute',
|
||||
'prompt.execution.read',
|
||||
'prompt.execution.output.read',
|
||||
'prompt.output.read',
|
||||
] as const);
|
||||
|
||||
export interface ProductionClusterControlAssemblyOptions {
|
||||
readonly createEventId?: ClusterRunCancellationEventIdFactory;
|
||||
readonly promptCatalog?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptCatalogCapability;
|
||||
}>;
|
||||
readonly promptExecution?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionCapability;
|
||||
readonly maxExecutionMs?: number;
|
||||
readonly now?: () => number;
|
||||
}>;
|
||||
readonly promptExecutionInspection?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionInspectionCapability;
|
||||
readonly now?: () => number;
|
||||
}>;
|
||||
readonly promptOutputRead?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptOutputReadCapability;
|
||||
}>;
|
||||
readonly promptExecutionOutputRead?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionOutputReadCapability;
|
||||
}>;
|
||||
readonly workerIngress?: Readonly<{
|
||||
readonly config: EnabledClusterWorkerIngressConfig;
|
||||
readonly onDiagnostic?: (error: unknown) => void | Promise<void>;
|
||||
}>;
|
||||
readonly startWorkerIngress?: (
|
||||
options: ProductionClusterWorkerIngressStarterOptions,
|
||||
) => Promise<ClusterWorkerIngressApplicationResult>;
|
||||
}
|
||||
|
||||
export interface ProductionClusterWorkerIngressOptions {
|
||||
readonly config: EnabledClusterWorkerIngressConfig;
|
||||
readonly artifactStore: ClusterRemoteWorkerArtifactStore;
|
||||
readonly secretProvider?: RemoteWorkerSecretValueProvider;
|
||||
readonly onDiagnostic?: (error: unknown) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ProductionClusterControlApplicationOptions
|
||||
extends Omit<
|
||||
ClusterControlApplicationOptions,
|
||||
| 'create'
|
||||
| 'enabled'
|
||||
| 'profile'
|
||||
| 'apiCredentialPepper'
|
||||
| 'openDatabase'
|
||||
| 'availability'
|
||||
| 'http'
|
||||
| 'workerRuntime'
|
||||
> {
|
||||
readonly config: EnabledClusterControlConfig;
|
||||
readonly createEventId?: ClusterRunCancellationEventIdFactory;
|
||||
readonly promptCatalog?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptCatalogCapability;
|
||||
}>;
|
||||
readonly promptExecution?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionCapability;
|
||||
}>;
|
||||
readonly promptExecutionInspection?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionInspectionCapability;
|
||||
}>;
|
||||
readonly promptOutputRead?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptOutputReadCapability;
|
||||
}>;
|
||||
readonly promptExecutionOutputRead?: Readonly<{
|
||||
readonly capability: ClusterPluginPackagePromptExecutionOutputReadCapability;
|
||||
}>;
|
||||
readonly workerIngress?: ProductionClusterWorkerIngressOptions;
|
||||
}
|
||||
|
||||
const SAFE_RECOVERY: Readonly<ClusterControlStartupRecoverySummary> =
|
||||
Object.freeze({ safe: true, remaining: 0, failed: 0 });
|
||||
|
||||
function eventIdFactory(
|
||||
candidate: ClusterRunCancellationEventIdFactory | undefined,
|
||||
): ClusterRunCancellationEventIdFactory {
|
||||
if (candidate !== undefined && typeof candidate !== 'function') {
|
||||
throw new TypeError(
|
||||
'Production cluster-control event ID factory is invalid',
|
||||
);
|
||||
}
|
||||
return candidate ?? randomUUID;
|
||||
}
|
||||
|
||||
/**
|
||||
* The reviewed production business surface. Bootstrap still owns PostgreSQL,
|
||||
* startup recovery, the scheduler and cancellation convergence; this stack
|
||||
* owns only the exact route allowlist and its admission pipeline.
|
||||
*/
|
||||
export function createProductionClusterControlApplicationStack(
|
||||
input: ClusterControlAssemblyInput,
|
||||
options: ProductionClusterControlAssemblyOptions = {},
|
||||
): ClusterControlApplicationStack {
|
||||
const createEventId = eventIdFactory(options.createEventId);
|
||||
const workerIngressStarter =
|
||||
options.startWorkerIngress ?? startProductionClusterWorkerIngress;
|
||||
if (typeof workerIngressStarter !== 'function') {
|
||||
throw new TypeError('Production Worker ingress starter is invalid');
|
||||
}
|
||||
const routeDefinitions = [
|
||||
createClusterControlTaskReadRoute(input.taskDefinitions),
|
||||
createClusterControlTaskListRoute(input.taskDefinitions),
|
||||
createClusterControlTaskStartRoute(input.taskStart, createEventId),
|
||||
createClusterControlRunReadRoute(input.runs),
|
||||
createClusterControlRunListRoute(input.runs),
|
||||
createClusterControlRunEventListRoute(input.runs),
|
||||
createClusterControlRunStepListRoute(
|
||||
input.runs,
|
||||
input.trustedToolStorage.stepRuns,
|
||||
),
|
||||
createClusterControlRunCancellationRoute(
|
||||
input.runCancellation,
|
||||
createEventId,
|
||||
),
|
||||
...createClusterControlPluginPackageWorkflowRoutes(
|
||||
input.workflowAdministration,
|
||||
Date.now,
|
||||
createEventId,
|
||||
),
|
||||
...(options.promptCatalog === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlPluginPackagePromptCatalogRoute(
|
||||
options.promptCatalog.capability,
|
||||
),
|
||||
]),
|
||||
...(options.promptExecution === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlPluginPackagePromptExecutionRoute(
|
||||
options.promptExecution.capability,
|
||||
{
|
||||
...(options.promptExecution.maxExecutionMs === undefined
|
||||
? {}
|
||||
: { maxExecutionMs: options.promptExecution.maxExecutionMs }),
|
||||
...(options.promptExecution.now === undefined
|
||||
? {}
|
||||
: { now: options.promptExecution.now }),
|
||||
createEventId,
|
||||
},
|
||||
),
|
||||
]),
|
||||
...(options.promptExecutionInspection === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlPluginPackagePromptExecutionInspectionRoute(
|
||||
options.promptExecutionInspection.capability,
|
||||
{
|
||||
...(options.promptExecutionInspection.now === undefined
|
||||
? {}
|
||||
: { now: options.promptExecutionInspection.now }),
|
||||
createEventId,
|
||||
},
|
||||
),
|
||||
]),
|
||||
...(options.promptOutputRead === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlPluginPackagePromptOutputReadRoute(
|
||||
options.promptOutputRead.capability,
|
||||
),
|
||||
]),
|
||||
...(options.promptExecutionOutputRead === undefined
|
||||
? []
|
||||
: [
|
||||
createClusterControlPluginPackagePromptExecutionOutputReadRoute(
|
||||
options.promptExecutionOutputRead.capability,
|
||||
),
|
||||
]),
|
||||
];
|
||||
const routes = createClusterControlRouteRegistry(routeDefinitions);
|
||||
const expectedRouteCount =
|
||||
PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS.length +
|
||||
(options.promptCatalog === undefined ? 0 : 1) +
|
||||
(options.promptExecution === undefined ? 0 : 1) +
|
||||
(options.promptExecutionInspection === undefined ? 0 : 1) +
|
||||
(options.promptOutputRead === undefined ? 0 : 1) +
|
||||
(options.promptExecutionOutputRead === undefined ? 0 : 1);
|
||||
if (routes.size !== expectedRouteCount) {
|
||||
throw new Error('Production cluster-control route allowlist is incomplete');
|
||||
}
|
||||
const admission = createClusterControlAdmissionPipeline({
|
||||
routes,
|
||||
authenticator: input.authenticator,
|
||||
policy: createClusterControlProjectPolicyAuthorizer(input.policies),
|
||||
audit: input.securityAudit,
|
||||
});
|
||||
let workerIngress:
|
||||
| Extract<ClusterWorkerIngressApplicationResult, { status: 'active' }>
|
||||
| undefined;
|
||||
let workerIngressStart:
|
||||
| Promise<
|
||||
Extract<ClusterWorkerIngressApplicationResult, { status: 'active' }>
|
||||
>
|
||||
| undefined;
|
||||
let workerIngressUnavailable: unknown;
|
||||
let workerIngressStop: Promise<'stopped'> | undefined;
|
||||
const stopWorkerIngress = (): Promise<'stopped'> => {
|
||||
if (!workerIngress) return Promise.resolve('stopped' as const);
|
||||
workerIngressStop ??= workerIngress.stop();
|
||||
return workerIngressStop;
|
||||
};
|
||||
const startWorkerIngress = async (): Promise<void> => {
|
||||
const ingress = options.workerIngress;
|
||||
if (!ingress) return;
|
||||
if (!input.workerRuntime) {
|
||||
throw new Error(
|
||||
'Production Worker ingress requires an injected runtime service port',
|
||||
);
|
||||
}
|
||||
workerIngressStart ??= (async () => {
|
||||
const result = await workerIngressStarter({
|
||||
config: ingress.config,
|
||||
runtime: input.workerRuntime!,
|
||||
onPoolError(error) {
|
||||
workerIngressUnavailable ??= error;
|
||||
void Promise.resolve(ingress.onDiagnostic?.(error)).catch(
|
||||
() => undefined,
|
||||
);
|
||||
void stopWorkerIngress().catch(() => undefined);
|
||||
},
|
||||
});
|
||||
if (result.status !== 'active') {
|
||||
throw new Error('Production Worker ingress did not activate');
|
||||
}
|
||||
workerIngress = result;
|
||||
if (workerIngressUnavailable !== undefined) {
|
||||
await stopWorkerIngress();
|
||||
throw new Error(
|
||||
'Production Worker ingress database became unavailable during activation',
|
||||
);
|
||||
}
|
||||
return result;
|
||||
})();
|
||||
await workerIngressStart;
|
||||
};
|
||||
|
||||
return Object.freeze({
|
||||
async reconcile(): Promise<ClusterControlStartupRecoverySummary> {
|
||||
return SAFE_RECOVERY;
|
||||
},
|
||||
async startLifecycles(): Promise<boolean> {
|
||||
await startWorkerIngress();
|
||||
return true;
|
||||
},
|
||||
admission,
|
||||
async stop(): Promise<ClusterControlStopResult> {
|
||||
await stopWorkerIngress();
|
||||
return 'stopped';
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** Starts cluster-control with the reviewed production route allowlist. */
|
||||
export function startProductionClusterControlApplication(
|
||||
options: ProductionClusterControlApplicationOptions,
|
||||
): Promise<ClusterControlApplicationResult> {
|
||||
const createEventId = eventIdFactory(options.createEventId);
|
||||
const {
|
||||
createEventId: _ignoredCreateEventId,
|
||||
config,
|
||||
workerIngress,
|
||||
promptCatalog,
|
||||
promptExecution,
|
||||
promptExecutionInspection,
|
||||
promptOutputRead,
|
||||
promptExecutionOutputRead,
|
||||
...applicationOptions
|
||||
} = options;
|
||||
const database = createClusterControlDatabaseBinding(config);
|
||||
return startClusterControlApplication({
|
||||
...applicationOptions,
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
apiCredentialPepper: config.security.apiCredentialPepper,
|
||||
http: config.http,
|
||||
...(workerIngress === undefined
|
||||
? {}
|
||||
: {
|
||||
workerRuntime: {
|
||||
artifactStore: workerIngress.artifactStore,
|
||||
...(workerIngress.secretProvider === undefined
|
||||
? {}
|
||||
: { secretProvider: workerIngress.secretProvider }),
|
||||
},
|
||||
}),
|
||||
...database,
|
||||
create: (input) =>
|
||||
createProductionClusterControlApplicationStack(input, {
|
||||
createEventId,
|
||||
...(promptCatalog === undefined ? {} : { promptCatalog }),
|
||||
...(promptExecution === undefined
|
||||
? {}
|
||||
: {
|
||||
promptExecution: {
|
||||
capability: promptExecution.capability,
|
||||
maxExecutionMs: Math.max(
|
||||
1,
|
||||
(config.http.requestTimeoutMs ??
|
||||
CLUSTER_CONTROL_HTTP_DEFAULTS.requestTimeoutMs) - 100,
|
||||
),
|
||||
},
|
||||
}),
|
||||
...(promptExecutionInspection === undefined
|
||||
? {}
|
||||
: { promptExecutionInspection }),
|
||||
...(promptOutputRead === undefined ? {} : { promptOutputRead }),
|
||||
...(promptExecutionOutputRead === undefined
|
||||
? {}
|
||||
: { promptExecutionOutputRead }),
|
||||
...(workerIngress === undefined
|
||||
? {}
|
||||
: {
|
||||
workerIngress: {
|
||||
config: workerIngress.config,
|
||||
...(workerIngress.onDiagnostic === undefined
|
||||
? {}
|
||||
: { onDiagnostic: workerIngress.onDiagnostic }),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,778 @@
|
||||
// Artifact owns immutable S3 evidence, checksum validation, and conditional promotion.
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { Readable } from 'node:stream';
|
||||
import {
|
||||
ChecksumAlgorithm,
|
||||
ChecksumMode,
|
||||
CopyObjectCommand,
|
||||
DeleteObjectCommand,
|
||||
HeadObjectCommand,
|
||||
MetadataDirective,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
ServerSideEncryption,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import {
|
||||
MAX_REMOTE_WORKER_ARTIFACT_BYTES,
|
||||
REMOTE_WORKER_ARTIFACT_CONTENT_TYPE,
|
||||
normalizeRemoteWorkerArtifactReceipt,
|
||||
type RemoteWorkerArtifactReceipt,
|
||||
} from '@qinglong/runtime-core/remote-worker-completion';
|
||||
import type {
|
||||
ClusterRemoteWorkerArtifactLookup,
|
||||
ClusterRemoteWorkerArtifactStorageCommand,
|
||||
ClusterRemoteWorkerArtifactStore,
|
||||
} from '../remote-execution/remoteWorkerCompletionService';
|
||||
|
||||
const DEFAULT_PREFIX = 'qinglong/v3/worker-artifacts';
|
||||
const METADATA_SCHEMA = 'qinglong-remote-worker-artifact-v1';
|
||||
const TEMPORARY_METADATA_SCHEMA =
|
||||
'qinglong-remote-worker-artifact-temporary-v1';
|
||||
const SHA256_PATTERN = /^[a-f0-9]{64}$/;
|
||||
const BUCKET_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/;
|
||||
const PREFIX_PATTERN = /^[A-Za-z0-9][A-Za-z0-9/_=-]{0,254}$/;
|
||||
const TEMPORARY_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
|
||||
type S3SendClient = Pick<S3Client, 'send'>;
|
||||
|
||||
export type S3ClusterRemoteWorkerArtifactEncryption =
|
||||
| Readonly<{ readonly mode: 's3' }>
|
||||
| Readonly<{ readonly mode: 'kms'; readonly keyId: string }>;
|
||||
|
||||
export interface S3ClusterRemoteWorkerArtifactStoreDiagnostic {
|
||||
readonly operation: 'temporary_object_cleanup';
|
||||
}
|
||||
|
||||
export interface S3ClusterRemoteWorkerArtifactStoreOptions {
|
||||
readonly client: S3SendClient;
|
||||
readonly bucket: string;
|
||||
readonly prefix?: string;
|
||||
readonly expectedBucketOwner?: string;
|
||||
readonly encryption: S3ClusterRemoteWorkerArtifactEncryption;
|
||||
readonly createTemporaryId?: () => string;
|
||||
readonly onDiagnostic?: (
|
||||
error: unknown,
|
||||
context: Readonly<S3ClusterRemoteWorkerArtifactStoreDiagnostic>,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface S3ClusterRemoteWorkerArtifactClientOptions {
|
||||
readonly region: string;
|
||||
readonly endpoint?: string;
|
||||
readonly forcePathStyle?: boolean;
|
||||
}
|
||||
|
||||
export function createS3ClusterRemoteWorkerArtifactClient(
|
||||
options: S3ClusterRemoteWorkerArtifactClientOptions,
|
||||
): S3Client {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
!/^[a-z0-9][a-z0-9-]{0,62}$/.test(options.region) ||
|
||||
(options.endpoint !== undefined &&
|
||||
typeof options.endpoint !== 'string') ||
|
||||
(options.forcePathStyle !== undefined &&
|
||||
typeof options.forcePathStyle !== 'boolean')
|
||||
) {
|
||||
throw configurationError('client options are invalid');
|
||||
}
|
||||
return new S3Client({
|
||||
region: options.region,
|
||||
...(options.endpoint === undefined
|
||||
? {}
|
||||
: { endpoint: options.endpoint }),
|
||||
forcePathStyle: options.forcePathStyle ?? false,
|
||||
});
|
||||
}
|
||||
|
||||
export class S3ClusterRemoteWorkerArtifactStoreError extends Error {
|
||||
constructor(
|
||||
readonly reason: 'unavailable' | 'integrity_mismatch',
|
||||
options?: ErrorOptions,
|
||||
) {
|
||||
super(`S3 Remote Worker Artifact store failed: ${reason}`, options);
|
||||
this.name = 'S3ClusterRemoteWorkerArtifactStoreError';
|
||||
}
|
||||
}
|
||||
|
||||
interface PreparedOptions {
|
||||
readonly client: S3SendClient;
|
||||
readonly bucket: string;
|
||||
readonly prefix: string;
|
||||
readonly expectedBucketOwner?: string;
|
||||
readonly encryption: Readonly<{
|
||||
readonly ServerSideEncryption: 'AES256' | 'aws:kms';
|
||||
readonly SSEKMSKeyId?: string;
|
||||
}>;
|
||||
readonly createTemporaryId: () => string;
|
||||
readonly onDiagnostic?: S3ClusterRemoteWorkerArtifactStoreOptions['onDiagnostic'];
|
||||
}
|
||||
|
||||
type ArtifactAuthority = Readonly<{
|
||||
projectId: string;
|
||||
runId: string;
|
||||
attemptId: string;
|
||||
logArtifactId: string;
|
||||
}>;
|
||||
|
||||
type NormalizedStorageCommand = ArtifactAuthority &
|
||||
Readonly<{
|
||||
byteLength: number;
|
||||
truncated?: boolean;
|
||||
}>;
|
||||
|
||||
const DIAGNOSTIC_CONTEXT = Object.freeze({
|
||||
operation: 'temporary_object_cleanup' as const,
|
||||
});
|
||||
|
||||
function configurationError(message: string): TypeError {
|
||||
return new TypeError(`S3 Remote Worker Artifact store is invalid: ${message}`);
|
||||
}
|
||||
|
||||
function prepareOptions(
|
||||
options: S3ClusterRemoteWorkerArtifactStoreOptions,
|
||||
): PreparedOptions {
|
||||
const allowedKeys = new Set([
|
||||
'bucket',
|
||||
'client',
|
||||
'createTemporaryId',
|
||||
'encryption',
|
||||
'expectedBucketOwner',
|
||||
'onDiagnostic',
|
||||
'prefix',
|
||||
]);
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => !allowedKeys.has(key)) ||
|
||||
typeof options.client?.send !== 'function'
|
||||
) {
|
||||
throw configurationError('options are invalid');
|
||||
}
|
||||
if (
|
||||
!BUCKET_PATTERN.test(options.bucket) ||
|
||||
options.bucket.includes('..') ||
|
||||
/^\d{1,3}(?:\.\d{1,3}){3}$/.test(options.bucket)
|
||||
) {
|
||||
throw configurationError('bucket is invalid');
|
||||
}
|
||||
const prefix = options.prefix ?? DEFAULT_PREFIX;
|
||||
if (
|
||||
!PREFIX_PATTERN.test(prefix) ||
|
||||
prefix.startsWith('/') ||
|
||||
prefix.endsWith('/') ||
|
||||
prefix.includes('//') ||
|
||||
prefix.split('/').some((segment) => segment === '.' || segment === '..')
|
||||
) {
|
||||
throw configurationError('prefix is invalid');
|
||||
}
|
||||
if (
|
||||
options.expectedBucketOwner !== undefined &&
|
||||
!/^\d{12}$/.test(options.expectedBucketOwner)
|
||||
) {
|
||||
throw configurationError('expected bucket owner is invalid');
|
||||
}
|
||||
const encryption = options.encryption;
|
||||
if (!encryption || typeof encryption !== 'object' || Array.isArray(encryption)) {
|
||||
throw configurationError('encryption is required');
|
||||
}
|
||||
let preparedEncryption: PreparedOptions['encryption'];
|
||||
if (
|
||||
encryption.mode === 's3' &&
|
||||
Object.keys(encryption).length === 1
|
||||
) {
|
||||
preparedEncryption = Object.freeze({
|
||||
ServerSideEncryption: ServerSideEncryption.AES256,
|
||||
});
|
||||
} else if (
|
||||
encryption.mode === 'kms' &&
|
||||
Object.keys(encryption).length === 2 &&
|
||||
typeof encryption.keyId === 'string' &&
|
||||
encryption.keyId.length >= 1 &&
|
||||
encryption.keyId.length <= 2048 &&
|
||||
!/[\u0000-\u001f\u007f]/.test(encryption.keyId)
|
||||
) {
|
||||
preparedEncryption = Object.freeze({
|
||||
ServerSideEncryption: ServerSideEncryption.aws_kms,
|
||||
SSEKMSKeyId: encryption.keyId,
|
||||
});
|
||||
} else {
|
||||
throw configurationError('encryption is invalid');
|
||||
}
|
||||
if (
|
||||
options.createTemporaryId !== undefined &&
|
||||
typeof options.createTemporaryId !== 'function'
|
||||
) {
|
||||
throw configurationError('temporary ID factory is invalid');
|
||||
}
|
||||
if (
|
||||
options.onDiagnostic !== undefined &&
|
||||
typeof options.onDiagnostic !== 'function'
|
||||
) {
|
||||
throw configurationError('diagnostic sink is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
client: options.client,
|
||||
bucket: options.bucket,
|
||||
prefix,
|
||||
...(options.expectedBucketOwner === undefined
|
||||
? {}
|
||||
: { expectedBucketOwner: options.expectedBucketOwner }),
|
||||
encryption: preparedEncryption,
|
||||
createTemporaryId: options.createTemporaryId ?? randomUUID,
|
||||
...(options.onDiagnostic === undefined
|
||||
? {}
|
||||
: { onDiagnostic: options.onDiagnostic }),
|
||||
});
|
||||
}
|
||||
|
||||
function receiptCandidate(
|
||||
authority: ArtifactAuthority,
|
||||
byteLength: number,
|
||||
sha256: string,
|
||||
truncated?: boolean,
|
||||
status: RemoteWorkerArtifactReceipt['status'] = 'stored',
|
||||
): Readonly<RemoteWorkerArtifactReceipt> {
|
||||
return normalizeRemoteWorkerArtifactReceipt({
|
||||
status,
|
||||
...authority,
|
||||
byteLength,
|
||||
sha256,
|
||||
...(truncated === undefined ? {} : { truncated }),
|
||||
});
|
||||
}
|
||||
|
||||
function exactObjectShape(
|
||||
value: unknown,
|
||||
required: readonly string[],
|
||||
optional: readonly string[],
|
||||
name: string,
|
||||
): asserts value is Readonly<Record<string, unknown>> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw configurationError(`${name} is invalid`);
|
||||
}
|
||||
const keys = Object.keys(value);
|
||||
const allowed = new Set([...required, ...optional]);
|
||||
if (
|
||||
required.some((key) => !Object.hasOwn(value, key)) ||
|
||||
keys.some((key) => !allowed.has(key))
|
||||
) {
|
||||
throw configurationError(`${name} shape is invalid`);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeStorageCommand(
|
||||
command: Readonly<ClusterRemoteWorkerArtifactStorageCommand>,
|
||||
): NormalizedStorageCommand {
|
||||
exactObjectShape(
|
||||
command,
|
||||
['attemptId', 'byteLength', 'logArtifactId', 'projectId', 'runId'],
|
||||
['truncated'],
|
||||
'storage command',
|
||||
);
|
||||
const normalized = receiptCandidate(
|
||||
command as unknown as ClusterRemoteWorkerArtifactStorageCommand,
|
||||
command.byteLength as number,
|
||||
'0'.repeat(64),
|
||||
command.truncated as boolean | undefined,
|
||||
);
|
||||
return Object.freeze({
|
||||
projectId: normalized.projectId,
|
||||
runId: normalized.runId,
|
||||
attemptId: normalized.attemptId,
|
||||
logArtifactId: normalized.logArtifactId,
|
||||
byteLength: normalized.byteLength,
|
||||
...(normalized.truncated === undefined
|
||||
? {}
|
||||
: { truncated: normalized.truncated }),
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeLookup(
|
||||
lookup: Readonly<ClusterRemoteWorkerArtifactLookup>,
|
||||
): ArtifactAuthority {
|
||||
exactObjectShape(
|
||||
lookup,
|
||||
['attemptId', 'logArtifactId', 'projectId', 'runId'],
|
||||
[],
|
||||
'lookup',
|
||||
);
|
||||
const normalized = receiptCandidate(
|
||||
lookup as unknown as ClusterRemoteWorkerArtifactLookup,
|
||||
0,
|
||||
'0'.repeat(64),
|
||||
);
|
||||
return Object.freeze({
|
||||
projectId: normalized.projectId,
|
||||
runId: normalized.runId,
|
||||
attemptId: normalized.attemptId,
|
||||
logArtifactId: normalized.logArtifactId,
|
||||
});
|
||||
}
|
||||
|
||||
function lookupFromCommand(
|
||||
command: NormalizedStorageCommand,
|
||||
): ArtifactAuthority {
|
||||
return Object.freeze({
|
||||
projectId: command.projectId,
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
logArtifactId: command.logArtifactId,
|
||||
});
|
||||
}
|
||||
|
||||
function identityDigest(authority: ArtifactAuthority): string {
|
||||
return createHash('sha256')
|
||||
.update('qinglong/remote-worker-artifact-identity@v1\0', 'utf8')
|
||||
.update(authority.projectId, 'utf8')
|
||||
.update('\0', 'utf8')
|
||||
.update(authority.runId, 'utf8')
|
||||
.update('\0', 'utf8')
|
||||
.update(authority.attemptId, 'utf8')
|
||||
.update('\0', 'utf8')
|
||||
.update(authority.logArtifactId, 'utf8')
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
function fieldDigest(domain: string, value: string): string {
|
||||
return createHash('sha256')
|
||||
.update(`qinglong/remote-worker-artifact-${domain}@v1\0`, 'utf8')
|
||||
.update(value, 'utf8')
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
function finalObjectKey(prefix: string, authority: ArtifactAuthority): string {
|
||||
const digest = identityDigest(authority);
|
||||
return `${prefix}/objects/${digest.slice(0, 2)}/${digest}`;
|
||||
}
|
||||
|
||||
function temporaryObjectKey(prefix: string, createId: () => string): string {
|
||||
const id = createId();
|
||||
if (typeof id !== 'string' || !TEMPORARY_ID_PATTERN.test(id)) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('unavailable');
|
||||
}
|
||||
return `${prefix}/temporary/${id}`;
|
||||
}
|
||||
|
||||
function temporaryOwnershipDigest(): string {
|
||||
const authority = randomBytes(32);
|
||||
try {
|
||||
return createHash('sha256')
|
||||
.update('qinglong/remote-worker-artifact-temporary-owner@v1\0', 'utf8')
|
||||
.update(authority)
|
||||
.digest('hex');
|
||||
} finally {
|
||||
authority.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
function finalMetadata(
|
||||
command: NormalizedStorageCommand,
|
||||
sha256: string,
|
||||
): Readonly<Record<string, string>> {
|
||||
return Object.freeze({
|
||||
'ql3-schema': METADATA_SCHEMA,
|
||||
'ql3-project-sha256': fieldDigest('project', command.projectId),
|
||||
'ql3-run-sha256': fieldDigest('run', command.runId),
|
||||
'ql3-attempt-sha256': fieldDigest('attempt', command.attemptId),
|
||||
'ql3-log-artifact-sha256': fieldDigest(
|
||||
'log-artifact',
|
||||
command.logArtifactId,
|
||||
),
|
||||
'ql3-byte-length': String(command.byteLength),
|
||||
'ql3-content-sha256': sha256,
|
||||
'ql3-truncated': command.truncated === undefined
|
||||
? 'omitted'
|
||||
: String(command.truncated),
|
||||
});
|
||||
}
|
||||
|
||||
function canonicalChecksum(value: unknown): string {
|
||||
if (typeof value !== 'string' || value.length !== 44) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
const decoded = Buffer.from(value, 'base64');
|
||||
try {
|
||||
if (decoded.byteLength !== 32 || decoded.toString('base64') !== value) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
return decoded.toString('hex');
|
||||
} finally {
|
||||
decoded.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
function metadataValue(
|
||||
metadata: Readonly<Record<string, string | undefined>> | undefined,
|
||||
name: string,
|
||||
): string {
|
||||
const value = metadata?.[name];
|
||||
if (typeof value !== 'string') {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseStoredReceipt(
|
||||
authority: ArtifactAuthority,
|
||||
output: Readonly<{
|
||||
ContentLength?: number | undefined;
|
||||
ContentType?: string | undefined;
|
||||
ChecksumSHA256?: string | undefined;
|
||||
Metadata?: Readonly<Record<string, string | undefined>> | undefined;
|
||||
}>,
|
||||
): Readonly<RemoteWorkerArtifactReceipt> {
|
||||
const metadata = output.Metadata;
|
||||
const lengthText = metadataValue(metadata, 'ql3-byte-length');
|
||||
const byteLength = Number(lengthText);
|
||||
const sha256 = metadataValue(metadata, 'ql3-content-sha256');
|
||||
const truncatedText = metadataValue(metadata, 'ql3-truncated');
|
||||
if (
|
||||
metadataValue(metadata, 'ql3-schema') !== METADATA_SCHEMA ||
|
||||
metadataValue(metadata, 'ql3-project-sha256') !==
|
||||
fieldDigest('project', authority.projectId) ||
|
||||
metadataValue(metadata, 'ql3-run-sha256') !==
|
||||
fieldDigest('run', authority.runId) ||
|
||||
metadataValue(metadata, 'ql3-attempt-sha256') !==
|
||||
fieldDigest('attempt', authority.attemptId) ||
|
||||
metadataValue(metadata, 'ql3-log-artifact-sha256') !==
|
||||
fieldDigest('log-artifact', authority.logArtifactId) ||
|
||||
!Number.isSafeInteger(byteLength) ||
|
||||
byteLength < 0 ||
|
||||
byteLength > MAX_REMOTE_WORKER_ARTIFACT_BYTES ||
|
||||
lengthText !== String(byteLength) ||
|
||||
output.ContentLength !== byteLength ||
|
||||
output.ContentType !== REMOTE_WORKER_ARTIFACT_CONTENT_TYPE ||
|
||||
!SHA256_PATTERN.test(sha256) ||
|
||||
canonicalChecksum(output.ChecksumSHA256) !== sha256 ||
|
||||
!['omitted', 'true', 'false'].includes(truncatedText)
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
return receiptCandidate(
|
||||
authority,
|
||||
byteLength,
|
||||
sha256,
|
||||
truncatedText === 'omitted' ? undefined : truncatedText === 'true',
|
||||
'already_stored',
|
||||
);
|
||||
}
|
||||
|
||||
function isNotFound(error: unknown): boolean {
|
||||
if (!error || typeof error !== 'object') return false;
|
||||
const value = error as {
|
||||
name?: unknown;
|
||||
Code?: unknown;
|
||||
$metadata?: { httpStatusCode?: unknown };
|
||||
};
|
||||
return value.name === 'NotFound' ||
|
||||
value.name === 'NoSuchKey' ||
|
||||
value.Code === 'NoSuchKey' ||
|
||||
value.$metadata?.httpStatusCode === 404;
|
||||
}
|
||||
|
||||
function requestOptions(signal?: AbortSignal): { abortSignal: AbortSignal } | undefined {
|
||||
return signal === undefined ? undefined : { abortSignal: signal };
|
||||
}
|
||||
|
||||
function copySource(bucket: string, key: string): string {
|
||||
return [bucket, ...key.split('/')]
|
||||
.map((segment) => encodeURIComponent(segment))
|
||||
.join('/');
|
||||
}
|
||||
|
||||
class ArtifactContentDigest {
|
||||
private readonly hash = createHash('sha256');
|
||||
private consumedBytes = 0;
|
||||
private complete = false;
|
||||
private digestValue?: string;
|
||||
|
||||
constructor(
|
||||
private readonly content: AsyncIterable<Uint8Array>,
|
||||
private readonly expectedBytes: number,
|
||||
private readonly signal?: AbortSignal,
|
||||
) {
|
||||
if (!content || typeof content[Symbol.asyncIterator] !== 'function') {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('unavailable');
|
||||
}
|
||||
}
|
||||
|
||||
async *stream(): AsyncGenerator<Buffer> {
|
||||
if (this.complete || this.consumedBytes !== 0) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('unavailable');
|
||||
}
|
||||
if (this.signal?.aborted) throw this.signal.reason;
|
||||
for await (const chunk of this.content) {
|
||||
if (this.signal?.aborted) throw this.signal.reason;
|
||||
if (!(chunk instanceof Uint8Array) || chunk.byteLength === 0) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
this.consumedBytes += chunk.byteLength;
|
||||
if (this.consumedBytes > this.expectedBytes) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
this.hash.update(chunk);
|
||||
yield Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
|
||||
}
|
||||
if (this.consumedBytes !== this.expectedBytes) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('integrity_mismatch');
|
||||
}
|
||||
if (this.signal?.aborted) throw this.signal.reason;
|
||||
this.digestValue = this.hash.digest('hex');
|
||||
this.complete = true;
|
||||
}
|
||||
|
||||
async consume(): Promise<string> {
|
||||
for await (const _chunk of this.stream()) {
|
||||
// Hash a replay without allocating or contacting object storage.
|
||||
}
|
||||
return this.digest();
|
||||
}
|
||||
|
||||
digest(): string {
|
||||
if (!this.complete || !this.digestValue) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('unavailable');
|
||||
}
|
||||
return this.digestValue;
|
||||
}
|
||||
|
||||
isComplete(): boolean {
|
||||
return this.complete;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared immutable S3 adapter. A unique temporary upload is checksummed first,
|
||||
* then promoted by one destination-conditional server-side copy. Permanent
|
||||
* objects are never overwritten or deleted by this adapter.
|
||||
*/
|
||||
export class S3ClusterRemoteWorkerArtifactStore
|
||||
implements ClusterRemoteWorkerArtifactStore {
|
||||
private readonly options: PreparedOptions;
|
||||
|
||||
constructor(options: S3ClusterRemoteWorkerArtifactStoreOptions) {
|
||||
this.options = prepareOptions(options);
|
||||
}
|
||||
|
||||
async inspect(
|
||||
lookup: Readonly<ClusterRemoteWorkerArtifactLookup>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt> | undefined> {
|
||||
const authority = normalizeLookup(lookup);
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
try {
|
||||
const output = await this.options.client.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: finalObjectKey(this.options.prefix, authority),
|
||||
ChecksumMode: ChecksumMode.ENABLED,
|
||||
...(this.options.expectedBucketOwner === undefined
|
||||
? {}
|
||||
: { ExpectedBucketOwner: this.options.expectedBucketOwner }),
|
||||
}),
|
||||
requestOptions(signal),
|
||||
);
|
||||
return parseStoredReceipt(authority, output);
|
||||
} catch (error) {
|
||||
if (isNotFound(error)) return undefined;
|
||||
if (error instanceof S3ClusterRemoteWorkerArtifactStoreError) throw error;
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('unavailable', {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async put(
|
||||
value: Readonly<ClusterRemoteWorkerArtifactStorageCommand>,
|
||||
content: AsyncIterable<Uint8Array>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt>> {
|
||||
const command = normalizeStorageCommand(value);
|
||||
const digest = new ArtifactContentDigest(
|
||||
content,
|
||||
command.byteLength,
|
||||
signal,
|
||||
);
|
||||
const lookup = lookupFromCommand(command);
|
||||
const existing = await this.inspect(lookup, signal);
|
||||
if (existing) {
|
||||
const incomingSha256 = await digest.consume();
|
||||
if (
|
||||
existing.byteLength !== command.byteLength ||
|
||||
existing.truncated !== command.truncated ||
|
||||
existing.sha256 !== incomingSha256
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError(
|
||||
'integrity_mismatch',
|
||||
);
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
|
||||
const temporaryKey = temporaryObjectKey(
|
||||
this.options.prefix,
|
||||
this.options.createTemporaryId,
|
||||
);
|
||||
const temporaryOwner = temporaryOwnershipDigest();
|
||||
let temporaryOwned = false;
|
||||
let result: Readonly<RemoteWorkerArtifactReceipt> | undefined;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
const body = Readable.from(digest.stream(), { objectMode: false });
|
||||
try {
|
||||
await this.options.client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: temporaryKey,
|
||||
Body: body,
|
||||
ContentLength: command.byteLength,
|
||||
ContentType: REMOTE_WORKER_ARTIFACT_CONTENT_TYPE,
|
||||
ChecksumAlgorithm: ChecksumAlgorithm.SHA256,
|
||||
IfNoneMatch: '*',
|
||||
Metadata: {
|
||||
'ql3-schema': TEMPORARY_METADATA_SCHEMA,
|
||||
'ql3-owner-sha256': temporaryOwner,
|
||||
},
|
||||
...this.options.encryption,
|
||||
...(this.options.expectedBucketOwner === undefined
|
||||
? {}
|
||||
: { ExpectedBucketOwner: this.options.expectedBucketOwner }),
|
||||
}),
|
||||
requestOptions(signal),
|
||||
);
|
||||
temporaryOwned = true;
|
||||
} catch (error) {
|
||||
if (!digest.isComplete()) throw error;
|
||||
} finally {
|
||||
body.destroy();
|
||||
}
|
||||
const sha256 = digest.digest();
|
||||
await this.assertTemporaryObject(
|
||||
temporaryKey,
|
||||
temporaryOwner,
|
||||
command.byteLength,
|
||||
sha256,
|
||||
signal,
|
||||
);
|
||||
temporaryOwned = true;
|
||||
|
||||
let copied = false;
|
||||
try {
|
||||
await this.options.client.send(
|
||||
new CopyObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: finalObjectKey(this.options.prefix, command),
|
||||
CopySource: copySource(this.options.bucket, temporaryKey),
|
||||
ContentType: REMOTE_WORKER_ARTIFACT_CONTENT_TYPE,
|
||||
MetadataDirective: MetadataDirective.REPLACE,
|
||||
Metadata: finalMetadata(command, sha256),
|
||||
ChecksumAlgorithm: ChecksumAlgorithm.SHA256,
|
||||
IfNoneMatch: '*',
|
||||
...this.options.encryption,
|
||||
...(this.options.expectedBucketOwner === undefined
|
||||
? {}
|
||||
: {
|
||||
ExpectedBucketOwner: this.options.expectedBucketOwner,
|
||||
CopySourceExpectedBucketOwner:
|
||||
this.options.expectedBucketOwner,
|
||||
}),
|
||||
}),
|
||||
requestOptions(signal),
|
||||
);
|
||||
copied = true;
|
||||
} catch {
|
||||
// A 409/412 race or a lost successful response is resolved only by
|
||||
// inspecting the immutable destination below.
|
||||
}
|
||||
const stored = await this.inspect(lookup, signal);
|
||||
if (
|
||||
!stored ||
|
||||
stored.byteLength !== command.byteLength ||
|
||||
stored.truncated !== command.truncated ||
|
||||
stored.sha256 !== sha256
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError(
|
||||
'integrity_mismatch',
|
||||
);
|
||||
}
|
||||
result = Object.freeze({
|
||||
...stored,
|
||||
status: copied ? 'stored' as const : 'already_stored' as const,
|
||||
});
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
}
|
||||
|
||||
if (temporaryOwned) {
|
||||
try {
|
||||
await this.options.client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: temporaryKey,
|
||||
...(this.options.expectedBucketOwner === undefined
|
||||
? {}
|
||||
: { ExpectedBucketOwner: this.options.expectedBucketOwner }),
|
||||
}),
|
||||
requestOptions(signal),
|
||||
);
|
||||
} catch (error) {
|
||||
if (primaryError === undefined) {
|
||||
try {
|
||||
await this.options.onDiagnostic?.(error, DIAGNOSTIC_CONTEXT);
|
||||
} catch {
|
||||
// Diagnostics cannot reverse a durable immutable promotion.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (primaryError !== undefined) {
|
||||
if (primaryError instanceof S3ClusterRemoteWorkerArtifactStoreError) {
|
||||
throw primaryError;
|
||||
}
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('unavailable', {
|
||||
cause: primaryError,
|
||||
});
|
||||
}
|
||||
return result!;
|
||||
}
|
||||
|
||||
private async assertTemporaryObject(
|
||||
key: string,
|
||||
ownerSha256: string,
|
||||
byteLength: number,
|
||||
sha256: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
let output;
|
||||
try {
|
||||
output = await this.options.client.send(
|
||||
new HeadObjectCommand({
|
||||
Bucket: this.options.bucket,
|
||||
Key: key,
|
||||
ChecksumMode: ChecksumMode.ENABLED,
|
||||
...(this.options.expectedBucketOwner === undefined
|
||||
? {}
|
||||
: { ExpectedBucketOwner: this.options.expectedBucketOwner }),
|
||||
}),
|
||||
requestOptions(signal),
|
||||
);
|
||||
} catch (error) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError('unavailable', {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
if (
|
||||
output.ContentLength !== byteLength ||
|
||||
output.ContentType !== REMOTE_WORKER_ARTIFACT_CONTENT_TYPE ||
|
||||
output.Metadata?.['ql3-schema'] !== TEMPORARY_METADATA_SCHEMA ||
|
||||
output.Metadata?.['ql3-owner-sha256'] !== ownerSha256 ||
|
||||
canonicalChecksum(output.ChecksumSHA256) !== sha256
|
||||
) {
|
||||
throw new S3ClusterRemoteWorkerArtifactStoreError(
|
||||
'integrity_mismatch',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
// Artifact owns lazy production binding without widening the Worker runtime port.
|
||||
import {
|
||||
createS3ClusterRemoteWorkerArtifactClient,
|
||||
S3ClusterRemoteWorkerArtifactStore,
|
||||
} from './s3ArtifactStore';
|
||||
import type { ClusterRemoteWorkerArtifactStore } from '../remote-execution/remoteWorkerCompletionService';
|
||||
import type { ClusterWorkerArtifactS3Config } from '../worker-ingress/workerIngressConfig';
|
||||
|
||||
export interface ClusterWorkerArtifactBinding {
|
||||
readonly store: ClusterRemoteWorkerArtifactStore;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export function createClusterWorkerArtifactBinding(
|
||||
config: ClusterWorkerArtifactS3Config,
|
||||
): Readonly<ClusterWorkerArtifactBinding> {
|
||||
if (!config || typeof config !== 'object' || Array.isArray(config)) {
|
||||
throw new TypeError('Cluster Worker Artifact binding config is invalid');
|
||||
}
|
||||
const client = createS3ClusterRemoteWorkerArtifactClient({
|
||||
region: config.region,
|
||||
...(config.endpoint === undefined
|
||||
? {}
|
||||
: { endpoint: config.endpoint }),
|
||||
forcePathStyle: config.forcePathStyle,
|
||||
});
|
||||
const store = new S3ClusterRemoteWorkerArtifactStore({
|
||||
client,
|
||||
bucket: config.bucket,
|
||||
...(config.prefix === undefined ? {} : { prefix: config.prefix }),
|
||||
...(config.expectedBucketOwner === undefined
|
||||
? {}
|
||||
: { expectedBucketOwner: config.expectedBucketOwner }),
|
||||
encryption: config.encryption,
|
||||
});
|
||||
let closed = false;
|
||||
return Object.freeze({
|
||||
store,
|
||||
async close() {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
client.destroy();
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
// Authentication owns credential verification and bounded Principal issuance.
|
||||
import { createHmac, timingSafeEqual } from 'node:crypto';
|
||||
import {
|
||||
ApiCredentialUnavailableError,
|
||||
LEGACY_API_CREDENTIAL_PEPPER_KEY_ID,
|
||||
assertApiCredentialPepperKeyId,
|
||||
normalizeApiCredentialRecord,
|
||||
type ApiCredentialRepository,
|
||||
} from '@qinglong/runtime-core/api-credential';
|
||||
import {
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPrincipal,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
import type { ClusterControlAdmissionMetadata } from '../transport/httpSurface';
|
||||
import type { ClusterControlRequestAuthenticator } from '../transport/admissionPipeline';
|
||||
|
||||
export const CLUSTER_CONTROL_API_CREDENTIAL_LIMITS = Object.freeze({
|
||||
principalTtlMs: 60_000,
|
||||
maxPrincipalTtlMs: 300_000,
|
||||
secretBytes: 32,
|
||||
});
|
||||
|
||||
export class ClusterControlApiCredentialConfigurationError extends TypeError {
|
||||
constructor(message: string) {
|
||||
super(
|
||||
`Cluster-control API credential configuration is invalid: ${message}`,
|
||||
);
|
||||
this.name = 'ClusterControlApiCredentialConfigurationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterControlApiCredentialUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_CONTROL_API_CREDENTIAL_UNAVAILABLE';
|
||||
|
||||
constructor() {
|
||||
super('Cluster-control API credential authentication is unavailable');
|
||||
this.name = 'ClusterControlApiCredentialUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export interface ClusterControlApiCredentialAuthenticatorOptions {
|
||||
readonly principalTtlMs?: number;
|
||||
readonly pepperKeyId?: string;
|
||||
readonly now?: () => number;
|
||||
}
|
||||
|
||||
const AUTHORIZATION_PATTERN =
|
||||
/^Bearer ql3c_([A-Za-z0-9][A-Za-z0-9._:-]{0,63})_([A-Za-z0-9_-]{43})$/;
|
||||
const PEPPER_PATTERN = /^[A-Za-z0-9_-]{43}$/;
|
||||
const DIGEST_DOMAIN = Buffer.from('qinglong-api-credential-v1\0', 'utf8');
|
||||
|
||||
function decodeSecret(name: string, value: string): Buffer {
|
||||
if (typeof value !== 'string' || !PEPPER_PATTERN.test(value)) {
|
||||
throw new ClusterControlApiCredentialConfigurationError(
|
||||
`${name} must be canonical base64url for 32 bytes`,
|
||||
);
|
||||
}
|
||||
const decoded = Buffer.from(value, 'base64url');
|
||||
if (
|
||||
decoded.byteLength !== CLUSTER_CONTROL_API_CREDENTIAL_LIMITS.secretBytes ||
|
||||
decoded.toString('base64url') !== value
|
||||
) {
|
||||
throw new ClusterControlApiCredentialConfigurationError(
|
||||
`${name} must be canonical base64url for 32 bytes`,
|
||||
);
|
||||
}
|
||||
return decoded;
|
||||
}
|
||||
|
||||
export function assertClusterControlApiCredentialPepper(value: string): void {
|
||||
const decoded = decodeSecret('pepper', value);
|
||||
decoded.fill(0);
|
||||
}
|
||||
|
||||
function principalTtl(value: number | undefined): number {
|
||||
const resolved =
|
||||
value ?? CLUSTER_CONTROL_API_CREDENTIAL_LIMITS.principalTtlMs;
|
||||
if (
|
||||
!Number.isSafeInteger(resolved) ||
|
||||
resolved < 1_000 ||
|
||||
resolved > CLUSTER_CONTROL_API_CREDENTIAL_LIMITS.maxPrincipalTtlMs
|
||||
) {
|
||||
throw new ClusterControlApiCredentialConfigurationError(
|
||||
'principalTtlMs is invalid',
|
||||
);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function digest(pepper: Buffer, credentialId: string, secret: Buffer): Buffer {
|
||||
return createHmac('sha256', pepper)
|
||||
.update(DIGEST_DOMAIN)
|
||||
.update(credentialId, 'utf8')
|
||||
.update('\0', 'utf8')
|
||||
.update(secret)
|
||||
.digest();
|
||||
}
|
||||
|
||||
export function apiCredentialSecretDigest(
|
||||
pepperBase64Url: string,
|
||||
credentialId: string,
|
||||
secretBase64Url: string,
|
||||
): string {
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/.test(credentialId)) {
|
||||
throw new ClusterControlApiCredentialConfigurationError(
|
||||
'credentialId is invalid',
|
||||
);
|
||||
}
|
||||
const pepper = decodeSecret('pepper', pepperBase64Url);
|
||||
const secret = decodeSecret('secret', secretBase64Url);
|
||||
let result: Buffer | undefined;
|
||||
try {
|
||||
result = digest(pepper, credentialId, secret);
|
||||
return result.toString('hex');
|
||||
} finally {
|
||||
result?.fill(0);
|
||||
pepper.fill(0);
|
||||
secret.fill(0);
|
||||
}
|
||||
}
|
||||
|
||||
function parseAuthorization(
|
||||
metadata: ClusterControlAdmissionMetadata,
|
||||
): { readonly credentialId: string; readonly secret: Buffer } | null {
|
||||
const value = metadata.headers.authorization;
|
||||
if (typeof value !== 'string') return null;
|
||||
const match = AUTHORIZATION_PATTERN.exec(value);
|
||||
if (!match) return null;
|
||||
let secret: Buffer;
|
||||
try {
|
||||
secret = decodeSecret('bearer secret', match[2]!);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return Object.freeze({ credentialId: match[1]!, secret });
|
||||
}
|
||||
|
||||
export function createClusterControlApiCredentialAuthenticator(
|
||||
repository: ApiCredentialRepository,
|
||||
pepperBase64Url: string,
|
||||
options: ClusterControlApiCredentialAuthenticatorOptions = {},
|
||||
): ClusterControlRequestAuthenticator {
|
||||
if (!repository || typeof repository.resolve !== 'function') {
|
||||
throw new ClusterControlApiCredentialConfigurationError(
|
||||
'repository is invalid',
|
||||
);
|
||||
}
|
||||
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
||||
throw new ClusterControlApiCredentialConfigurationError(
|
||||
'options are invalid',
|
||||
);
|
||||
}
|
||||
const keys = Object.keys(options);
|
||||
if (
|
||||
keys.some(
|
||||
(key) =>
|
||||
key !== 'principalTtlMs' && key !== 'pepperKeyId' && key !== 'now',
|
||||
)
|
||||
) {
|
||||
throw new ClusterControlApiCredentialConfigurationError(
|
||||
'options shape is invalid',
|
||||
);
|
||||
}
|
||||
if (options.now !== undefined && typeof options.now !== 'function') {
|
||||
throw new ClusterControlApiCredentialConfigurationError('now is invalid');
|
||||
}
|
||||
const pepperKeyId =
|
||||
options.pepperKeyId ?? LEGACY_API_CREDENTIAL_PEPPER_KEY_ID;
|
||||
try {
|
||||
assertApiCredentialPepperKeyId(pepperKeyId);
|
||||
} catch {
|
||||
throw new ClusterControlApiCredentialConfigurationError(
|
||||
'pepperKeyId is invalid',
|
||||
);
|
||||
}
|
||||
const pepper = decodeSecret('pepper', pepperBase64Url);
|
||||
const ttlMs = principalTtl(options.principalTtlMs);
|
||||
const now = options.now ?? Date.now;
|
||||
|
||||
return Object.freeze({
|
||||
async authenticate(
|
||||
metadata: ClusterControlAdmissionMetadata,
|
||||
): Promise<Readonly<SecurityPrincipal> | null> {
|
||||
const parsed = parseAuthorization(metadata);
|
||||
if (!parsed) return null;
|
||||
const presentedDigest = digest(
|
||||
pepper,
|
||||
parsed.credentialId,
|
||||
parsed.secret,
|
||||
);
|
||||
parsed.secret.fill(0);
|
||||
let candidate;
|
||||
try {
|
||||
candidate = await repository.resolve(parsed.credentialId);
|
||||
} catch (error) {
|
||||
presentedDigest.fill(0);
|
||||
if (error instanceof ApiCredentialUnavailableError) {
|
||||
throw new ClusterControlApiCredentialUnavailableError();
|
||||
}
|
||||
throw new ClusterControlApiCredentialUnavailableError();
|
||||
}
|
||||
if (metadata.signal.aborted) {
|
||||
presentedDigest.fill(0);
|
||||
throw new ClusterControlApiCredentialUnavailableError();
|
||||
}
|
||||
let record;
|
||||
try {
|
||||
record = candidate ? normalizeApiCredentialRecord(candidate) : null;
|
||||
} catch {
|
||||
presentedDigest.fill(0);
|
||||
throw new ClusterControlApiCredentialUnavailableError();
|
||||
}
|
||||
if (record && record.pepperKeyId !== pepperKeyId) {
|
||||
presentedDigest.fill(0);
|
||||
throw new ClusterControlApiCredentialUnavailableError();
|
||||
}
|
||||
const storedDigest = record
|
||||
? Buffer.from(record.secretDigest, 'hex')
|
||||
: Buffer.alloc(32);
|
||||
const matches = timingSafeEqual(presentedDigest, storedDigest);
|
||||
presentedDigest.fill(0);
|
||||
storedDigest.fill(0);
|
||||
if (!record || !matches) return null;
|
||||
const nowMs = now();
|
||||
if (
|
||||
!Number.isSafeInteger(nowMs) ||
|
||||
nowMs < 0 ||
|
||||
record.state !== 'active' ||
|
||||
record.subjectStatus !== 'active' ||
|
||||
record.notBeforeAtMs > nowMs ||
|
||||
record.expiresAtMs <= nowMs
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const expiresAtMs = Math.min(record.expiresAtMs, nowMs + ttlMs);
|
||||
try {
|
||||
return normalizeSecurityPrincipal(
|
||||
{
|
||||
subject: record.subject,
|
||||
authenticationId: `api_credential:${record.credentialId}:${record.version}`,
|
||||
authenticatedAtMs: nowMs,
|
||||
expiresAtMs,
|
||||
assurance:
|
||||
record.subject.type === 'user' ? 'single_factor' : 'service',
|
||||
},
|
||||
nowMs,
|
||||
);
|
||||
} catch {
|
||||
throw new ClusterControlApiCredentialUnavailableError();
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
// Authentication owns its bounded pre-body overload shield.
|
||||
import { createHmac, randomBytes } from 'node:crypto';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
|
||||
export interface ClusterControlAuthenticationShieldOptions {
|
||||
readonly windowMs: number;
|
||||
readonly maxRequestsPerPeer: number;
|
||||
readonly maxRequestsGlobal: number;
|
||||
readonly maxTrackedPeers: number;
|
||||
readonly now?: () => number;
|
||||
}
|
||||
|
||||
export type ClusterControlAuthenticationShieldRejectionReason =
|
||||
| 'capacity'
|
||||
| 'clock'
|
||||
| 'global'
|
||||
| 'peer';
|
||||
|
||||
export type ClusterControlAuthenticationShieldResult =
|
||||
| {
|
||||
readonly allowed: true;
|
||||
/**
|
||||
* Returns this provisional attempt budget after the pre-body admission
|
||||
* preflight has succeeded. Idempotent and scoped to the exact windows
|
||||
* consumed by this result.
|
||||
*/
|
||||
refund(): void;
|
||||
}
|
||||
| {
|
||||
readonly allowed: false;
|
||||
readonly reason: ClusterControlAuthenticationShieldRejectionReason;
|
||||
readonly retryAfterMs: number;
|
||||
};
|
||||
|
||||
export interface ClusterControlAuthenticationShield {
|
||||
consume(
|
||||
peerAddress: string | undefined,
|
||||
): ClusterControlAuthenticationShieldResult;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
interface PeerWindow {
|
||||
readonly startedAt: number;
|
||||
readonly count: number;
|
||||
}
|
||||
|
||||
const FINGERPRINT_KEY_BYTES = 32;
|
||||
const MAX_PEER_ADDRESS_BYTES = 128;
|
||||
const MAX_PRUNE_PER_ATTEMPT = 64;
|
||||
const UNKNOWN_PEER = '<unknown-transport-peer>';
|
||||
|
||||
function normalizedPeerAddress(peerAddress: string | undefined): string {
|
||||
if (
|
||||
typeof peerAddress !== 'string' ||
|
||||
peerAddress.length === 0 ||
|
||||
Buffer.byteLength(peerAddress) > MAX_PEER_ADDRESS_BYTES ||
|
||||
/[\0\r\n]/.test(peerAddress)
|
||||
) {
|
||||
return UNKNOWN_PEER;
|
||||
}
|
||||
return peerAddress;
|
||||
}
|
||||
|
||||
function remainingWindow(
|
||||
now: number,
|
||||
startedAt: number,
|
||||
windowMs: number,
|
||||
): number {
|
||||
return Math.max(1, Math.ceil(windowMs - (now - startedAt)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a process-local overload shield for authentication attempts. It is
|
||||
* deliberately not an authorization or distributed quota authority: every
|
||||
* cluster-control replica owns a bounded, disposable window.
|
||||
*/
|
||||
export function createClusterControlAuthenticationShield(
|
||||
options: ClusterControlAuthenticationShieldOptions,
|
||||
): ClusterControlAuthenticationShield {
|
||||
const now = options.now ?? (() => performance.now());
|
||||
const fingerprintKey = randomBytes(FINGERPRINT_KEY_BYTES);
|
||||
const peers = new Map<string, PeerWindow>();
|
||||
let globalWindow: PeerWindow | undefined;
|
||||
let lastNow = 0;
|
||||
let closed = false;
|
||||
|
||||
const fingerprint = (peerAddress: string | undefined): string =>
|
||||
createHmac('sha256', fingerprintKey)
|
||||
.update('qinglong.cluster-control.authentication-peer\0')
|
||||
.update(normalizedPeerAddress(peerAddress))
|
||||
.digest('base64url');
|
||||
|
||||
const pruneExpired = (currentTime: number): void => {
|
||||
let scanned = 0;
|
||||
for (const [key, window] of peers) {
|
||||
if (scanned >= MAX_PRUNE_PER_ATTEMPT) return;
|
||||
scanned += 1;
|
||||
if (currentTime - window.startedAt >= options.windowMs) {
|
||||
peers.delete(key);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const accepted = (
|
||||
peer: string,
|
||||
peerStartedAt: number,
|
||||
globalStartedAt: number,
|
||||
): ClusterControlAuthenticationShieldResult => {
|
||||
let completed = false;
|
||||
return Object.freeze({
|
||||
allowed: true as const,
|
||||
refund() {
|
||||
if (completed || closed) return;
|
||||
completed = true;
|
||||
if (
|
||||
globalWindow?.startedAt === globalStartedAt &&
|
||||
globalWindow.count > 0
|
||||
) {
|
||||
globalWindow = {
|
||||
startedAt: globalWindow.startedAt,
|
||||
count: globalWindow.count - 1,
|
||||
};
|
||||
}
|
||||
const currentPeerWindow = peers.get(peer);
|
||||
if (
|
||||
currentPeerWindow?.startedAt === peerStartedAt &&
|
||||
currentPeerWindow.count > 0
|
||||
) {
|
||||
if (currentPeerWindow.count === 1) peers.delete(peer);
|
||||
else {
|
||||
peers.set(peer, {
|
||||
startedAt: currentPeerWindow.startedAt,
|
||||
count: currentPeerWindow.count - 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
consume(peerAddress) {
|
||||
if (closed) {
|
||||
return Object.freeze({
|
||||
allowed: false,
|
||||
reason: 'clock',
|
||||
retryAfterMs: options.windowMs,
|
||||
});
|
||||
}
|
||||
|
||||
let currentTime: number;
|
||||
try {
|
||||
currentTime = now();
|
||||
} catch {
|
||||
return Object.freeze({
|
||||
allowed: false,
|
||||
reason: 'clock',
|
||||
retryAfterMs: options.windowMs,
|
||||
});
|
||||
}
|
||||
if (
|
||||
!Number.isFinite(currentTime) ||
|
||||
currentTime < 0 ||
|
||||
currentTime < lastNow
|
||||
) {
|
||||
return Object.freeze({
|
||||
allowed: false,
|
||||
reason: 'clock',
|
||||
retryAfterMs: options.windowMs,
|
||||
});
|
||||
}
|
||||
lastNow = currentTime;
|
||||
|
||||
if (
|
||||
!globalWindow ||
|
||||
currentTime - globalWindow.startedAt >= options.windowMs
|
||||
) {
|
||||
globalWindow = { startedAt: currentTime, count: 0 };
|
||||
}
|
||||
if (globalWindow.count >= options.maxRequestsGlobal) {
|
||||
return Object.freeze({
|
||||
allowed: false,
|
||||
reason: 'global',
|
||||
retryAfterMs: remainingWindow(
|
||||
currentTime,
|
||||
globalWindow.startedAt,
|
||||
options.windowMs,
|
||||
),
|
||||
});
|
||||
}
|
||||
globalWindow = {
|
||||
startedAt: globalWindow.startedAt,
|
||||
count: globalWindow.count + 1,
|
||||
};
|
||||
|
||||
const peer = fingerprint(peerAddress);
|
||||
let peerWindow = peers.get(peer);
|
||||
if (
|
||||
peerWindow &&
|
||||
currentTime - peerWindow.startedAt >= options.windowMs
|
||||
) {
|
||||
peers.delete(peer);
|
||||
peerWindow = undefined;
|
||||
}
|
||||
if (peerWindow) {
|
||||
if (peerWindow.count >= options.maxRequestsPerPeer) {
|
||||
return Object.freeze({
|
||||
allowed: false,
|
||||
reason: 'peer',
|
||||
retryAfterMs: remainingWindow(
|
||||
currentTime,
|
||||
peerWindow.startedAt,
|
||||
options.windowMs,
|
||||
),
|
||||
});
|
||||
}
|
||||
peers.delete(peer);
|
||||
peers.set(peer, {
|
||||
startedAt: peerWindow.startedAt,
|
||||
count: peerWindow.count + 1,
|
||||
});
|
||||
return accepted(peer, peerWindow.startedAt, globalWindow.startedAt);
|
||||
}
|
||||
|
||||
if (peers.size >= options.maxTrackedPeers) pruneExpired(currentTime);
|
||||
if (peers.size >= options.maxTrackedPeers) {
|
||||
return Object.freeze({
|
||||
allowed: false,
|
||||
reason: 'capacity',
|
||||
retryAfterMs: options.windowMs,
|
||||
});
|
||||
}
|
||||
peers.set(peer, { startedAt: currentTime, count: 1 });
|
||||
return accepted(peer, currentTime, globalWindow.startedAt);
|
||||
},
|
||||
close() {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
peers.clear();
|
||||
globalWindow = undefined;
|
||||
fingerprintKey.fill(0);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import {
|
||||
runProductionClusterControlProcess,
|
||||
type ClusterControlProcessEvent,
|
||||
type ClusterControlProcessSignal,
|
||||
type ClusterControlProcessSignalSource,
|
||||
} from './production-process/processApplication';
|
||||
|
||||
const USAGE = 'Usage: ql3-cluster-control';
|
||||
|
||||
const nodeSignals: ClusterControlProcessSignalSource = Object.freeze({
|
||||
subscribe(
|
||||
listener: (signal: ClusterControlProcessSignal) => void,
|
||||
) {
|
||||
const handlers: Readonly<
|
||||
Record<ClusterControlProcessSignal, () => void>
|
||||
> = Object.freeze({
|
||||
SIGINT: () => listener('SIGINT'),
|
||||
SIGTERM: () => listener('SIGTERM'),
|
||||
});
|
||||
process.once('SIGINT', handlers.SIGINT);
|
||||
process.once('SIGTERM', handlers.SIGTERM);
|
||||
return () => {
|
||||
process.off('SIGINT', handlers.SIGINT);
|
||||
process.off('SIGTERM', handlers.SIGTERM);
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
function emit(record: ClusterControlProcessEvent): void {
|
||||
process.stdout.write(`${JSON.stringify(record)}\n`);
|
||||
}
|
||||
|
||||
function failureFact(error: unknown): Readonly<Record<string, unknown>> {
|
||||
const candidate = error as {
|
||||
readonly name?: unknown;
|
||||
readonly code?: unknown;
|
||||
};
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-cluster-control',
|
||||
level: 'error',
|
||||
event: 'process_failed',
|
||||
name:
|
||||
typeof candidate?.name === 'string' && candidate.name.length <= 128
|
||||
? candidate.name
|
||||
: 'Error',
|
||||
...(typeof candidate?.code === 'string' && candidate.code.length <= 128
|
||||
? { code: candidate.code }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function main(argv: readonly string[]): Promise<void> {
|
||||
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
|
||||
process.stdout.write(`${USAGE}\n`);
|
||||
return;
|
||||
}
|
||||
if (argv.length !== 0) {
|
||||
process.stderr.write(
|
||||
`${JSON.stringify({
|
||||
code: 'QL3_CLUSTER_CONTROL_CLI_USAGE_INVALID',
|
||||
message: USAGE,
|
||||
})}\n`,
|
||||
);
|
||||
process.exitCode = 64;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const stopResult = await runProductionClusterControlProcess({
|
||||
environment: process.env,
|
||||
signals: nodeSignals,
|
||||
emit,
|
||||
});
|
||||
if (stopResult !== 'stopped') process.exitCode = 1;
|
||||
} catch (error) {
|
||||
process.stderr.write(`${JSON.stringify(failureFact(error))}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
void main(process.argv.slice(2));
|
||||
@@ -0,0 +1,93 @@
|
||||
// Database owns the one-way Pool failure to admission-withdrawal fence.
|
||||
export type ClusterControlAvailabilityStatus =
|
||||
| 'available'
|
||||
| 'unavailable'
|
||||
| 'disposed';
|
||||
|
||||
export type ClusterControlUnavailableListener = (
|
||||
error: Error,
|
||||
) => void | Promise<void>;
|
||||
|
||||
export interface ClusterControlAvailabilitySource {
|
||||
subscribe(listener: ClusterControlUnavailableListener): () => void;
|
||||
}
|
||||
|
||||
export type ClusterControlAvailabilitySignalResult =
|
||||
| 'signaled'
|
||||
| 'already_unavailable'
|
||||
| 'disposed';
|
||||
|
||||
/**
|
||||
* A bounded one-way bridge from pg.Pool availability errors to the application
|
||||
* admission owner. It deliberately has one listener and no timer, retry loop,
|
||||
* error history or path back to available.
|
||||
*/
|
||||
export class ClusterControlAvailabilityFence
|
||||
implements ClusterControlAvailabilitySource
|
||||
{
|
||||
private currentStatus: ClusterControlAvailabilityStatus = 'available';
|
||||
private listener: ClusterControlUnavailableListener | undefined;
|
||||
private reason: Error | undefined;
|
||||
private notification: Promise<void> | undefined;
|
||||
|
||||
get status(): ClusterControlAvailabilityStatus {
|
||||
return this.currentStatus;
|
||||
}
|
||||
|
||||
subscribe(listener: ClusterControlUnavailableListener): () => void {
|
||||
if (typeof listener !== 'function') {
|
||||
throw new TypeError('Cluster-control availability listener is invalid');
|
||||
}
|
||||
if (this.currentStatus === 'disposed') {
|
||||
throw new Error('Cluster-control availability fence is disposed');
|
||||
}
|
||||
if (this.listener) {
|
||||
throw new Error('Cluster-control availability listener is already bound');
|
||||
}
|
||||
this.listener = listener;
|
||||
if (this.currentStatus === 'unavailable') {
|
||||
// An early signal has already returned to its producer. The subscriber
|
||||
// owns the delayed notification, so contain its rejection here.
|
||||
void this.notify().catch(() => undefined);
|
||||
}
|
||||
let subscribed = true;
|
||||
return () => {
|
||||
if (!subscribed) return;
|
||||
subscribed = false;
|
||||
if (this.listener === listener) this.listener = undefined;
|
||||
};
|
||||
}
|
||||
|
||||
signal(error: Error): Promise<ClusterControlAvailabilitySignalResult> {
|
||||
if (!(error instanceof Error)) {
|
||||
return Promise.reject(
|
||||
new TypeError('Cluster-control availability error is invalid'),
|
||||
);
|
||||
}
|
||||
if (this.currentStatus === 'disposed') return Promise.resolve('disposed');
|
||||
if (this.currentStatus === 'unavailable') {
|
||||
return (this.notification ?? Promise.resolve()).then(
|
||||
() => 'already_unavailable' as const,
|
||||
);
|
||||
}
|
||||
this.currentStatus = 'unavailable';
|
||||
this.reason = error;
|
||||
return this.notify().then(() => 'signaled' as const);
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
if (this.currentStatus === 'disposed') return;
|
||||
this.currentStatus = 'disposed';
|
||||
this.listener = undefined;
|
||||
this.reason = undefined;
|
||||
}
|
||||
|
||||
private notify(): Promise<void> {
|
||||
if (this.notification) return this.notification;
|
||||
if (!this.listener || !this.reason) return Promise.resolve();
|
||||
const listener = this.listener;
|
||||
const reason = this.reason;
|
||||
this.notification = Promise.resolve().then(() => listener(reason));
|
||||
return this.notification;
|
||||
}
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_PLUGIN_PACKAGE_PROMPT_CATALOG_RESPONSE_SCHEMA =
|
||||
'qinglong/plugin-package-prompt-catalog@v1' as const;
|
||||
|
||||
export interface ClusterPluginPackagePromptCatalogCapability {
|
||||
inspect(
|
||||
projectId: string,
|
||||
packageName: string,
|
||||
): Promise<
|
||||
Readonly<{
|
||||
schema: typeof CLUSTER_PLUGIN_PACKAGE_PROMPT_CATALOG_RESPONSE_SCHEMA;
|
||||
projectId: string;
|
||||
packageName: string;
|
||||
found: boolean;
|
||||
publicationState: 'active' | 'withdrawn' | 'absent' | null;
|
||||
prompts: readonly Readonly<{
|
||||
id: string;
|
||||
name: string;
|
||||
description: string | null;
|
||||
parameters: readonly Readonly<{
|
||||
name: string;
|
||||
description: string | null;
|
||||
required: boolean;
|
||||
}>[];
|
||||
}>[];
|
||||
}>
|
||||
>;
|
||||
}
|
||||
|
||||
const PACKAGE_NAME = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
export function createClusterControlPluginPackagePromptCatalogRoute(
|
||||
capability: ClusterPluginPackagePromptCatalogCapability,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (!capability || typeof capability.inspect !== 'function') {
|
||||
throw new TypeError('Cluster-control Prompt catalog capability is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/packages/{packageName}/prompts',
|
||||
operationId: 'prompt.read',
|
||||
permission: 'model.invoke',
|
||||
projectParameter: 'projectId' as const,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (
|
||||
authorized.request.body !== null ||
|
||||
authorized.projectId === null ||
|
||||
typeof parameters.packageName !== 'string' ||
|
||||
!PACKAGE_NAME.test(parameters.packageName)
|
||||
) {
|
||||
return response(400, { code: 'invalid_prompt_catalog_request' });
|
||||
}
|
||||
try {
|
||||
const result = await capability.inspect(
|
||||
authorized.projectId,
|
||||
parameters.packageName,
|
||||
);
|
||||
if (
|
||||
result.schema !==
|
||||
CLUSTER_PLUGIN_PACKAGE_PROMPT_CATALOG_RESPONSE_SCHEMA ||
|
||||
result.projectId !== authorized.projectId ||
|
||||
result.packageName !== parameters.packageName
|
||||
) {
|
||||
return response(503, { code: 'prompt_catalog_unavailable' });
|
||||
}
|
||||
return response(200, result);
|
||||
} catch {
|
||||
return response(503, { code: 'prompt_catalog_unavailable' });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import {
|
||||
normalizeSecurityAuditRecord,
|
||||
type SecurityAuditRecord,
|
||||
} from '@qinglong/runtime-core/security-audit';
|
||||
import type {
|
||||
SecurityPolicyFence,
|
||||
SecuritySubject,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_CONTROL_PLUGIN_PACKAGE_PROMPT_EXECUTION_INSPECTION_ROUTE =
|
||||
Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/packages/{packageName}/prompts/{promptId}/executions/{executionRequestId}',
|
||||
operationId: 'prompt.execution.read',
|
||||
permission: 'run.read',
|
||||
projectParameter: 'projectId',
|
||||
});
|
||||
|
||||
export interface ClusterPluginPackagePromptExecutionInspectionRouteOptions {
|
||||
readonly now?: () => number;
|
||||
readonly createEventId?: () => string;
|
||||
}
|
||||
|
||||
export interface ClusterPluginPackagePromptExecutionInspectionCapability {
|
||||
inspectAuthorized(input: Readonly<{
|
||||
projectId: string;
|
||||
packageName: string;
|
||||
promptId: string;
|
||||
executionRequestId: string;
|
||||
actor: Readonly<SecuritySubject>;
|
||||
fence: Readonly<SecurityPolicyFence>;
|
||||
audit: Readonly<SecurityAuditRecord>;
|
||||
}>): Promise<Readonly<{ found: boolean }>>;
|
||||
}
|
||||
|
||||
const PACKAGE_NAME = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
|
||||
const PROMPT_ID = /^[a-z][a-z0-9-]{0,62}$/;
|
||||
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/;
|
||||
const UUID_V4 =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | null {
|
||||
return error &&
|
||||
typeof error === 'object' &&
|
||||
'code' in error &&
|
||||
typeof error.code === 'string'
|
||||
? error.code
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Exact, content-free recovery read keyed by the caller-known requestId. */
|
||||
export function createClusterControlPluginPackagePromptExecutionInspectionRoute(
|
||||
capability: ClusterPluginPackagePromptExecutionInspectionCapability,
|
||||
options: ClusterPluginPackagePromptExecutionInspectionRouteOptions = {},
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (
|
||||
!capability ||
|
||||
typeof capability.inspectAuthorized !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function') ||
|
||||
(options.createEventId !== undefined &&
|
||||
typeof options.createEventId !== 'function')
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster-control Prompt execution inspection capability is invalid',
|
||||
);
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
const createEventId = options.createEventId ?? randomUUID;
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_PLUGIN_PACKAGE_PROMPT_EXECUTION_INSPECTION_ROUTE,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
const observedAtMs = now();
|
||||
const auditEventId = createEventId();
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
if (
|
||||
authorized.projectId === null ||
|
||||
typeof parameters.packageName !== 'string' ||
|
||||
!PACKAGE_NAME.test(parameters.packageName) ||
|
||||
typeof parameters.promptId !== 'string' ||
|
||||
!PROMPT_ID.test(parameters.promptId) ||
|
||||
typeof parameters.executionRequestId !== 'string' ||
|
||||
!IDENTITY.test(parameters.executionRequestId) ||
|
||||
!authorized.policyFence ||
|
||||
authorized.policyFence.bindingVersion === null ||
|
||||
!Number.isSafeInteger(observedAtMs) ||
|
||||
observedAtMs < 0 ||
|
||||
typeof auditEventId !== 'string' ||
|
||||
!UUID_V4.test(auditEventId)
|
||||
) {
|
||||
return response(503, { code: 'prompt_execution_inspection_unavailable' });
|
||||
}
|
||||
try {
|
||||
const result = await capability.inspectAuthorized({
|
||||
projectId: authorized.projectId,
|
||||
packageName: parameters.packageName,
|
||||
promptId: parameters.promptId,
|
||||
executionRequestId: parameters.executionRequestId,
|
||||
actor: authorized.principal.subject,
|
||||
fence: {
|
||||
projectVersion: authorized.policyFence.projectVersion,
|
||||
bindingVersion: authorized.policyFence.bindingVersion,
|
||||
},
|
||||
audit: normalizeSecurityAuditRecord({
|
||||
eventId: auditEventId,
|
||||
requestId: authorized.request.requestId,
|
||||
operationId: 'prompt.execution.read',
|
||||
projectId: authorized.projectId,
|
||||
subject: authorized.principal.subject,
|
||||
authenticationId: authorized.principal.authenticationId,
|
||||
outcome: 'allowed',
|
||||
reasons: ['project_policy_allowed'],
|
||||
fence: authorized.policyFence,
|
||||
occurredAtMs: observedAtMs,
|
||||
}),
|
||||
});
|
||||
return result.found
|
||||
? response(200, { ...result })
|
||||
: response(404, { code: 'prompt_execution_not_found' });
|
||||
} catch (error) {
|
||||
return errorCode(error) ===
|
||||
'PLUGIN_PACKAGE_PROMPT_EXECUTION_INSPECTION_AUTHORIZATION_FENCE_CONFLICT'
|
||||
? response(409, { code: 'authorization_fence_conflict' })
|
||||
: response(503, {
|
||||
code: 'prompt_execution_inspection_unavailable',
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
+277
@@ -0,0 +1,277 @@
|
||||
// Plugin Package Prompt owns request-keyed durable output recovery.
|
||||
import type { SecurityPrincipal } from '@qinglong/runtime-core/security';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_OUTPUT_READ_RESPONSE_SCHEMA =
|
||||
'qinglong/cluster-plugin-package-prompt-execution-output-read-response@v1' as const;
|
||||
export const CLUSTER_CONTROL_PLUGIN_PACKAGE_PROMPT_EXECUTION_OUTPUT_READ_ROUTE =
|
||||
Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/packages/{packageName}/prompts/{promptId}/executions/{executionRequestId}/output',
|
||||
operationId: 'prompt.execution.output.read',
|
||||
permission: 'artifact.read',
|
||||
projectParameter: 'projectId',
|
||||
});
|
||||
|
||||
export interface ClusterPluginPackagePromptExecutionOutputReadCapability {
|
||||
read(command: Readonly<{
|
||||
principal: Readonly<SecurityPrincipal>;
|
||||
projectId: string;
|
||||
packageName: string;
|
||||
promptId: string;
|
||||
executionRequestId: string;
|
||||
}>): Promise<unknown>;
|
||||
}
|
||||
|
||||
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/;
|
||||
const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
|
||||
const PACKAGE_NAME = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
|
||||
const PROMPT_ID = /^[a-z][a-z0-9-]{0,62}$/;
|
||||
const MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/;
|
||||
const DIGEST = /^[0-9a-f]{64}$/;
|
||||
const KEY_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
||||
const FINISH_REASONS = new Set([
|
||||
'stop',
|
||||
'length',
|
||||
'content_filter',
|
||||
'tool_call',
|
||||
'unknown',
|
||||
]);
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function exactRecord(
|
||||
value: unknown,
|
||||
required: readonly string[],
|
||||
optional: readonly string[] = [],
|
||||
): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
|
||||
const record = value as Record<string, unknown>;
|
||||
const keys = Object.keys(record).sort();
|
||||
const expected = [
|
||||
...required,
|
||||
...optional.filter((key) => key in record),
|
||||
].sort();
|
||||
return keys.length === expected.length &&
|
||||
keys.every((key, index) => key === expected[index])
|
||||
? record
|
||||
: null;
|
||||
}
|
||||
|
||||
function nonNegativeInteger(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && (value as number) >= 0;
|
||||
}
|
||||
|
||||
function exactTarget(
|
||||
value: Record<string, unknown>,
|
||||
expected: Readonly<{
|
||||
projectId: string;
|
||||
packageName: string;
|
||||
promptId: string;
|
||||
executionRequestId: string;
|
||||
}>,
|
||||
): boolean {
|
||||
return (
|
||||
value.projectId === expected.projectId &&
|
||||
value.packageName === expected.packageName &&
|
||||
value.promptId === expected.promptId &&
|
||||
value.executionRequestId === expected.executionRequestId
|
||||
);
|
||||
}
|
||||
|
||||
function availableView(
|
||||
value: unknown,
|
||||
expected: Readonly<{
|
||||
projectId: string;
|
||||
packageName: string;
|
||||
promptId: string;
|
||||
executionRequestId: string;
|
||||
}>,
|
||||
): Readonly<Record<string, unknown>> | null {
|
||||
const envelope = exactRecord(value, [
|
||||
'executionRequestId',
|
||||
'packageName',
|
||||
'projectId',
|
||||
'promptId',
|
||||
'reference',
|
||||
'result',
|
||||
'schema',
|
||||
'status',
|
||||
]);
|
||||
if (
|
||||
!envelope ||
|
||||
envelope.schema !==
|
||||
'qinglong/plugin-package-prompt-execution-output-read-result@v1' ||
|
||||
envelope.status !== 'available' ||
|
||||
!exactTarget(envelope, expected)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const reference = exactRecord(envelope.reference, [
|
||||
'algorithm',
|
||||
'artifactDigest',
|
||||
'artifactId',
|
||||
'contentDigest',
|
||||
'invocationId',
|
||||
'keyId',
|
||||
'outputBytes',
|
||||
'projectId',
|
||||
'retentionEligibleAtMs',
|
||||
'retentionPolicyDigest',
|
||||
'runId',
|
||||
'schema',
|
||||
'stepRunId',
|
||||
]);
|
||||
const result = exactRecord(envelope.result, [
|
||||
'finishReason',
|
||||
'model',
|
||||
'provider',
|
||||
'text',
|
||||
'usage',
|
||||
]);
|
||||
const usage = result
|
||||
? exactRecord(
|
||||
result.usage,
|
||||
['inputTokens', 'outputTokens', 'totalTokens'],
|
||||
['costMicros'],
|
||||
)
|
||||
: null;
|
||||
if (
|
||||
!reference ||
|
||||
!result ||
|
||||
!usage ||
|
||||
reference.schema !==
|
||||
'qinglong/plugin-package-prompt-output-artifact-reference@v1' ||
|
||||
reference.algorithm !== 'aes-256-gcm' ||
|
||||
reference.projectId !== expected.projectId ||
|
||||
typeof reference.runId !== 'string' ||
|
||||
!RUN_ID.test(reference.runId) ||
|
||||
typeof reference.artifactId !== 'string' ||
|
||||
!IDENTITY.test(reference.artifactId) ||
|
||||
typeof reference.artifactDigest !== 'string' ||
|
||||
!DIGEST.test(reference.artifactDigest) ||
|
||||
typeof reference.stepRunId !== 'string' ||
|
||||
!IDENTITY.test(reference.stepRunId) ||
|
||||
typeof reference.invocationId !== 'string' ||
|
||||
!IDENTITY.test(reference.invocationId) ||
|
||||
typeof reference.contentDigest !== 'string' ||
|
||||
!DIGEST.test(reference.contentDigest) ||
|
||||
!nonNegativeInteger(reference.outputBytes) ||
|
||||
reference.outputBytes > 1024 * 1024 ||
|
||||
typeof reference.retentionPolicyDigest !== 'string' ||
|
||||
!DIGEST.test(reference.retentionPolicyDigest) ||
|
||||
!nonNegativeInteger(reference.retentionEligibleAtMs) ||
|
||||
typeof reference.keyId !== 'string' ||
|
||||
!KEY_ID.test(reference.keyId) ||
|
||||
typeof result.provider !== 'string' ||
|
||||
!MODEL_ID.test(result.provider) ||
|
||||
typeof result.model !== 'string' ||
|
||||
!MODEL_ID.test(result.model) ||
|
||||
typeof result.text !== 'string' ||
|
||||
Buffer.byteLength(result.text, 'utf8') > 1024 * 1024 ||
|
||||
!FINISH_REASONS.has(result.finishReason as string) ||
|
||||
!nonNegativeInteger(usage.inputTokens) ||
|
||||
!nonNegativeInteger(usage.outputTokens) ||
|
||||
!nonNegativeInteger(usage.totalTokens) ||
|
||||
usage.totalTokens !== usage.inputTokens + usage.outputTokens ||
|
||||
(usage.costMicros !== undefined && !nonNegativeInteger(usage.costMicros))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return Object.freeze({
|
||||
schema:
|
||||
CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_OUTPUT_READ_RESPONSE_SCHEMA,
|
||||
status: 'available',
|
||||
...expected,
|
||||
reference: Object.freeze({ ...reference }),
|
||||
result: Object.freeze({
|
||||
provider: result.provider,
|
||||
model: result.model,
|
||||
text: result.text,
|
||||
finishReason: result.finishReason,
|
||||
usage: Object.freeze({ ...usage }),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterControlPluginPackagePromptExecutionOutputReadRoute(
|
||||
capability: ClusterPluginPackagePromptExecutionOutputReadCapability,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (!capability || typeof capability.read !== 'function') {
|
||||
throw new TypeError(
|
||||
'Cluster-control Prompt execution output read capability is invalid',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_PLUGIN_PACKAGE_PROMPT_EXECUTION_OUTPUT_READ_ROUTE,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (
|
||||
authorized.request.body !== null ||
|
||||
authorized.projectId === null ||
|
||||
typeof parameters.packageName !== 'string' ||
|
||||
!PACKAGE_NAME.test(parameters.packageName) ||
|
||||
typeof parameters.promptId !== 'string' ||
|
||||
!PROMPT_ID.test(parameters.promptId) ||
|
||||
typeof parameters.executionRequestId !== 'string' ||
|
||||
!IDENTITY.test(parameters.executionRequestId)
|
||||
) {
|
||||
return response(400, {
|
||||
code: 'invalid_prompt_execution_output_read_request',
|
||||
});
|
||||
}
|
||||
const expected = Object.freeze({
|
||||
projectId: authorized.projectId,
|
||||
packageName: parameters.packageName,
|
||||
promptId: parameters.promptId,
|
||||
executionRequestId: parameters.executionRequestId,
|
||||
});
|
||||
try {
|
||||
const result = await capability.read({
|
||||
principal: authorized.principal,
|
||||
...expected,
|
||||
});
|
||||
const notFound = exactRecord(result, [
|
||||
'executionRequestId',
|
||||
'packageName',
|
||||
'projectId',
|
||||
'promptId',
|
||||
'schema',
|
||||
'status',
|
||||
]);
|
||||
if (
|
||||
notFound &&
|
||||
notFound.schema ===
|
||||
'qinglong/plugin-package-prompt-execution-output-read-result@v1' &&
|
||||
notFound.status === 'not_found' &&
|
||||
exactTarget(notFound, expected)
|
||||
) {
|
||||
return response(404, { code: 'prompt_execution_output_not_found' });
|
||||
}
|
||||
const view = availableView(result, expected);
|
||||
return view
|
||||
? response(200, view)
|
||||
: response(503, {
|
||||
code: 'prompt_execution_output_read_unavailable',
|
||||
});
|
||||
} catch {
|
||||
return response(503, {
|
||||
code: 'prompt_execution_output_read_unavailable',
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
+400
@@ -0,0 +1,400 @@
|
||||
// Plugin Package Prompt owns bounded, Policy-fenced model execution admission.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { SecurityPrincipal } from '@qinglong/runtime-core/security';
|
||||
import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_REQUEST_SCHEMA =
|
||||
'qinglong/cluster-plugin-package-prompt-execution-request@v2' as const;
|
||||
export const CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_RESPONSE_SCHEMA =
|
||||
'qinglong/cluster-plugin-package-prompt-execution-response@v2' as const;
|
||||
export const CLUSTER_CONTROL_PLUGIN_PACKAGE_PROMPT_EXECUTION_ROUTE =
|
||||
Object.freeze({
|
||||
method: 'POST' as const,
|
||||
path: '/api/v3/projects/{projectId}/packages/{packageName}/prompts/{promptId}/executions',
|
||||
operationId: 'prompt.execute',
|
||||
permission: 'model.invoke',
|
||||
projectParameter: 'projectId',
|
||||
});
|
||||
|
||||
export const CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_LIMITS = Object.freeze({
|
||||
maxParameters: 64,
|
||||
maxParameterValueBytes: 64 * 1024,
|
||||
maxOutputTokens: 32_768,
|
||||
maxExecutionMs: 120_000,
|
||||
minOutputRetentionMs: 60 * 60_000,
|
||||
maxOutputRetentionMs: 365 * 24 * 60 * 60_000,
|
||||
});
|
||||
|
||||
export type ClusterPluginPackagePromptOutputIntent =
|
||||
| Readonly<{ mode: 'live_only' }>
|
||||
| Readonly<{
|
||||
mode: 'durable_artifact';
|
||||
retentionPolicy: Readonly<{
|
||||
revision: string;
|
||||
retentionMs: number;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export interface ClusterPluginPackagePromptExecutionCommand {
|
||||
readonly projectId: string;
|
||||
readonly packageName: string;
|
||||
readonly promptId: string;
|
||||
readonly requestId: string;
|
||||
readonly traceId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<{
|
||||
readonly projectVersion: number;
|
||||
readonly bindingVersion: number;
|
||||
}>;
|
||||
readonly parameters: Readonly<Record<string, string>>;
|
||||
readonly provider: string;
|
||||
readonly model: string;
|
||||
readonly maxOutputTokens: number;
|
||||
readonly temperature?: number;
|
||||
readonly deadlineAtMs: number;
|
||||
readonly plannedAtMs: number;
|
||||
readonly output?: Readonly<ClusterPluginPackagePromptOutputIntent>;
|
||||
readonly signal: AbortSignal;
|
||||
}
|
||||
|
||||
export interface ClusterPluginPackagePromptExecutionCapability {
|
||||
execute(command: Readonly<ClusterPluginPackagePromptExecutionCommand>): Promise<
|
||||
Readonly<{
|
||||
readonly status: 'executed' | 'resumed' | 'existing';
|
||||
readonly admission: Readonly<{
|
||||
readonly requestId: string;
|
||||
readonly invocationId: string;
|
||||
readonly runId: string;
|
||||
readonly stepRunId: string;
|
||||
}>;
|
||||
readonly finalization: Readonly<{ readonly runStatus: string }>;
|
||||
readonly result: unknown | null;
|
||||
readonly outputArtifact?: unknown;
|
||||
}>
|
||||
>;
|
||||
}
|
||||
|
||||
export interface ClusterPluginPackagePromptExecutionRouteOptions {
|
||||
readonly maxExecutionMs?: number;
|
||||
readonly now?: () => number;
|
||||
readonly createEventId?: () => string;
|
||||
}
|
||||
|
||||
class InvalidPromptExecutionRequestError extends TypeError {}
|
||||
|
||||
const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const PACKAGE_NAME = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
|
||||
const UUID_V4 =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
|
||||
function invalid(): never {
|
||||
throw new InvalidPromptExecutionRequestError();
|
||||
}
|
||||
|
||||
function dataRecord(value: unknown): Record<string, unknown> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
(Object.getPrototypeOf(value) !== Object.prototype &&
|
||||
Object.getPrototypeOf(value) !== null)
|
||||
) {
|
||||
return invalid();
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function exactKeys(
|
||||
value: Record<string, unknown>,
|
||||
required: readonly string[],
|
||||
optional: readonly string[] = [],
|
||||
): void {
|
||||
const allowed = new Set([...required, ...optional]);
|
||||
const keys = Object.keys(value);
|
||||
if (
|
||||
required.some((key) => !keys.includes(key)) ||
|
||||
keys.some((key) => !allowed.has(key))
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
}
|
||||
|
||||
function identifier(value: unknown): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER.test(value)) return invalid();
|
||||
return value;
|
||||
}
|
||||
|
||||
function positiveInteger(value: unknown, maximum: number): number {
|
||||
if (
|
||||
typeof value !== 'number' ||
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < 1 ||
|
||||
value > maximum
|
||||
) {
|
||||
return invalid();
|
||||
}
|
||||
return value as number;
|
||||
}
|
||||
|
||||
function parameters(value: unknown): Readonly<Record<string, string>> {
|
||||
const record = dataRecord(value);
|
||||
const names = Object.keys(record).sort();
|
||||
if (
|
||||
names.length > CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_LIMITS.maxParameters
|
||||
) {
|
||||
return invalid();
|
||||
}
|
||||
const normalized = Object.create(null) as Record<string, string>;
|
||||
for (const name of names) {
|
||||
const parameter = record[name];
|
||||
if (
|
||||
!/^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(name) ||
|
||||
typeof parameter !== 'string' ||
|
||||
Buffer.byteLength(parameter, 'utf8') >
|
||||
CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_LIMITS.maxParameterValueBytes
|
||||
) {
|
||||
return invalid();
|
||||
}
|
||||
normalized[name] = parameter;
|
||||
}
|
||||
return Object.freeze(normalized);
|
||||
}
|
||||
|
||||
function outputIntent(
|
||||
value: unknown,
|
||||
): Readonly<ClusterPluginPackagePromptOutputIntent> {
|
||||
const output = dataRecord(value);
|
||||
if (output.mode === 'live_only') {
|
||||
exactKeys(output, ['mode']);
|
||||
return Object.freeze({ mode: 'live_only' as const });
|
||||
}
|
||||
if (output.mode !== 'durable_artifact') return invalid();
|
||||
exactKeys(output, ['mode', 'retentionPolicy']);
|
||||
const retention = dataRecord(output.retentionPolicy);
|
||||
exactKeys(retention, ['retentionMs', 'revision']);
|
||||
if (
|
||||
typeof retention.revision !== 'string' ||
|
||||
!/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/.test(retention.revision) ||
|
||||
!Number.isSafeInteger(retention.retentionMs) ||
|
||||
(retention.retentionMs as number) <
|
||||
CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_LIMITS.minOutputRetentionMs ||
|
||||
(retention.retentionMs as number) >
|
||||
CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_LIMITS.maxOutputRetentionMs
|
||||
) {
|
||||
return invalid();
|
||||
}
|
||||
return Object.freeze({
|
||||
mode: 'durable_artifact' as const,
|
||||
retentionPolicy: Object.freeze({
|
||||
revision: retention.revision,
|
||||
retentionMs: retention.retentionMs as number,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | null {
|
||||
if (
|
||||
!error ||
|
||||
typeof error !== 'object' ||
|
||||
!('code' in error) ||
|
||||
typeof error.code !== 'string'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return error.code;
|
||||
}
|
||||
|
||||
function executionError(error: unknown): ClusterControlAdmissionResponse {
|
||||
const code = errorCode(error);
|
||||
if (
|
||||
code === 'PLUGIN_PACKAGE_PROMPT_ADMISSION_NOT_ALLOWED' ||
|
||||
code === 'PLUGIN_PACKAGE_PROMPT_ADMISSION_CONFLICT' ||
|
||||
code === 'PLUGIN_PACKAGE_PROMPT_EXECUTION_IN_PROGRESS' ||
|
||||
code === 'PLUGIN_PACKAGE_PROMPT_RESOLUTION_REQUIRED' ||
|
||||
code === 'MODEL_INVOCATION_CONFLICT' ||
|
||||
code === 'MODEL_INVOCATION_REPLAY_BLOCKED'
|
||||
) {
|
||||
return response(409, { code: 'prompt_execution_conflict' });
|
||||
}
|
||||
if (code === 'MODEL_GATEWAY_BUSY' || code === 'MODEL_PROJECT_QUOTA_EXCEEDED') {
|
||||
return response(429, { code: 'prompt_execution_capacity_exceeded' });
|
||||
}
|
||||
if (code === 'MODEL_POLICY_DENIED' || code === 'MODEL_BUDGET_EXCEEDED') {
|
||||
return response(422, { code: 'prompt_execution_policy_rejected' });
|
||||
}
|
||||
if (code === 'MODEL_INVOCATION_DEADLINE_EXCEEDED') {
|
||||
return response(504, { code: 'prompt_execution_deadline_exceeded' });
|
||||
}
|
||||
if (code === 'MODEL_INVOCATION_ABORTED') {
|
||||
return response(408, { code: 'prompt_execution_aborted' });
|
||||
}
|
||||
if (code === 'PLUGIN_PACKAGE_PROMPT_EXECUTION_PLAN_INVALID') {
|
||||
return response(400, { code: 'invalid_prompt_execution_request' });
|
||||
}
|
||||
return response(503, { code: 'prompt_execution_unavailable' });
|
||||
}
|
||||
|
||||
function parseBody(value: unknown, maximumExecutionMs: number) {
|
||||
const body = dataRecord(value);
|
||||
exactKeys(
|
||||
body,
|
||||
[
|
||||
'schema',
|
||||
'requestId',
|
||||
'traceId',
|
||||
'parameters',
|
||||
'provider',
|
||||
'model',
|
||||
'maxOutputTokens',
|
||||
'timeoutMs',
|
||||
],
|
||||
['output', 'temperature'],
|
||||
);
|
||||
if (body.schema !== CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_REQUEST_SCHEMA) {
|
||||
return invalid();
|
||||
}
|
||||
const temperature = body.temperature;
|
||||
const output =
|
||||
body.output === undefined ? undefined : outputIntent(body.output);
|
||||
if (
|
||||
temperature !== undefined &&
|
||||
(typeof temperature !== 'number' ||
|
||||
!Number.isFinite(temperature) ||
|
||||
temperature < 0 ||
|
||||
temperature > 2)
|
||||
) {
|
||||
return invalid();
|
||||
}
|
||||
return Object.freeze({
|
||||
requestId: identifier(body.requestId),
|
||||
traceId: identifier(body.traceId),
|
||||
parameters: parameters(body.parameters),
|
||||
provider: identifier(body.provider),
|
||||
model: identifier(body.model),
|
||||
maxOutputTokens: positiveInteger(
|
||||
body.maxOutputTokens,
|
||||
CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_LIMITS.maxOutputTokens,
|
||||
),
|
||||
timeoutMs: positiveInteger(body.timeoutMs, maximumExecutionMs),
|
||||
...(output === undefined ? {} : { output }),
|
||||
...(temperature === undefined ? {} : { temperature }),
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterControlPluginPackagePromptExecutionRoute(
|
||||
capability: ClusterPluginPackagePromptExecutionCapability,
|
||||
options: ClusterPluginPackagePromptExecutionRouteOptions = {},
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (!capability || typeof capability.execute !== 'function') {
|
||||
throw new TypeError('Cluster-control Prompt execution capability is invalid');
|
||||
}
|
||||
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
||||
throw new TypeError('Cluster-control Prompt execution route options are invalid');
|
||||
}
|
||||
const maximumExecutionMs =
|
||||
options.maxExecutionMs ??
|
||||
CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_LIMITS.maxExecutionMs;
|
||||
if (
|
||||
!Number.isSafeInteger(maximumExecutionMs) ||
|
||||
maximumExecutionMs < 1 ||
|
||||
maximumExecutionMs >
|
||||
CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_LIMITS.maxExecutionMs ||
|
||||
(options.now !== undefined && typeof options.now !== 'function') ||
|
||||
(options.createEventId !== undefined &&
|
||||
typeof options.createEventId !== 'function')
|
||||
) {
|
||||
throw new TypeError('Cluster-control Prompt execution route options are invalid');
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
const createEventId = options.createEventId ?? randomUUID;
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_PLUGIN_PACKAGE_PROMPT_EXECUTION_ROUTE,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
routeParameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
let body;
|
||||
try {
|
||||
body = parseBody(authorized.request.body, maximumExecutionMs);
|
||||
} catch {
|
||||
return response(400, { code: 'invalid_prompt_execution_request' });
|
||||
}
|
||||
const projectId = authorized.projectId;
|
||||
const packageName = routeParameters.packageName;
|
||||
const promptId = routeParameters.promptId;
|
||||
const fence = authorized.policyFence;
|
||||
const plannedAtMs = now();
|
||||
const auditEventId = createEventId();
|
||||
if (
|
||||
projectId === null ||
|
||||
typeof packageName !== 'string' ||
|
||||
!PACKAGE_NAME.test(packageName) ||
|
||||
typeof promptId !== 'string' ||
|
||||
!IDENTIFIER.test(promptId) ||
|
||||
!fence ||
|
||||
fence.bindingVersion === null ||
|
||||
!Number.isSafeInteger(plannedAtMs) ||
|
||||
plannedAtMs < 0 ||
|
||||
typeof auditEventId !== 'string' ||
|
||||
!UUID_V4.test(auditEventId)
|
||||
) {
|
||||
return response(503, { code: 'prompt_execution_unavailable' });
|
||||
}
|
||||
try {
|
||||
const result = await capability.execute({
|
||||
projectId,
|
||||
packageName,
|
||||
promptId,
|
||||
requestId: body.requestId,
|
||||
traceId: body.traceId,
|
||||
auditEventId,
|
||||
principal: authorized.principal,
|
||||
policyFence: Object.freeze({
|
||||
projectVersion: fence.projectVersion,
|
||||
bindingVersion: fence.bindingVersion,
|
||||
}),
|
||||
parameters: body.parameters,
|
||||
provider: body.provider,
|
||||
model: body.model,
|
||||
maxOutputTokens: body.maxOutputTokens,
|
||||
...(body.temperature === undefined
|
||||
? {}
|
||||
: { temperature: body.temperature }),
|
||||
...(body.output === undefined ? {} : { output: body.output }),
|
||||
plannedAtMs,
|
||||
deadlineAtMs: plannedAtMs + body.timeoutMs,
|
||||
signal: authorized.request.signal,
|
||||
});
|
||||
return response(200, {
|
||||
schema: CLUSTER_PLUGIN_PACKAGE_PROMPT_EXECUTION_RESPONSE_SCHEMA,
|
||||
status: result.status,
|
||||
replayed: result.status === 'existing',
|
||||
requestId: result.admission.requestId,
|
||||
invocationId: result.admission.invocationId,
|
||||
runId: result.admission.runId,
|
||||
stepRunId: result.admission.stepRunId,
|
||||
runStatus: result.finalization.runStatus,
|
||||
result: result.result,
|
||||
...(result.outputArtifact === undefined
|
||||
? {}
|
||||
: { outputArtifact: result.outputArtifact }),
|
||||
});
|
||||
} catch (error) {
|
||||
return executionError(error);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
// Plugin Package Prompt owns its capability-free durable output projection.
|
||||
import type { SecurityPrincipal } from '@qinglong/runtime-core/security';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_PLUGIN_PACKAGE_PROMPT_OUTPUT_READ_RESPONSE_SCHEMA =
|
||||
'qinglong/cluster-plugin-package-prompt-output-read-response@v1' as const;
|
||||
export const CLUSTER_CONTROL_PLUGIN_PACKAGE_PROMPT_OUTPUT_READ_ROUTE =
|
||||
Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/prompt-output-artifacts/{artifactId}',
|
||||
operationId: 'prompt.output.read',
|
||||
permission: 'artifact.read',
|
||||
projectParameter: 'projectId',
|
||||
allowedQuery: Object.freeze(['artifact_digest']),
|
||||
});
|
||||
|
||||
export interface ClusterPluginPackagePromptOutputReadCapability {
|
||||
read(command: Readonly<{
|
||||
principal: Readonly<SecurityPrincipal>;
|
||||
projectId: string;
|
||||
runId: string;
|
||||
artifactId: string;
|
||||
artifactDigest: string;
|
||||
}>): Promise<unknown>;
|
||||
}
|
||||
|
||||
const IDENTITY = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/;
|
||||
const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,35}$/;
|
||||
const MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/;
|
||||
const DIGEST = /^[0-9a-f]{64}$/;
|
||||
const KEY_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
||||
const FINISH_REASONS = new Set([
|
||||
'stop',
|
||||
'length',
|
||||
'content_filter',
|
||||
'tool_call',
|
||||
'unknown',
|
||||
]);
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function exactRecord(
|
||||
value: unknown,
|
||||
required: readonly string[],
|
||||
optional: readonly string[] = [],
|
||||
): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
|
||||
const record = value as Record<string, unknown>;
|
||||
const keys = Object.keys(record).sort();
|
||||
const expected = [...required, ...optional.filter((key) => key in record)].sort();
|
||||
return keys.length === expected.length &&
|
||||
keys.every((key, index) => key === expected[index])
|
||||
? record
|
||||
: null;
|
||||
}
|
||||
|
||||
function nonNegativeInteger(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && (value as number) >= 0;
|
||||
}
|
||||
|
||||
function availableView(
|
||||
value: unknown,
|
||||
expected: Readonly<{
|
||||
projectId: string;
|
||||
runId: string;
|
||||
artifactId: string;
|
||||
artifactDigest: string;
|
||||
}>,
|
||||
): Readonly<Record<string, unknown>> | null {
|
||||
const resultEnvelope = exactRecord(value, [
|
||||
'schema',
|
||||
'status',
|
||||
'reference',
|
||||
'result',
|
||||
]);
|
||||
if (
|
||||
!resultEnvelope ||
|
||||
resultEnvelope.schema !==
|
||||
'qinglong/plugin-package-prompt-output-read-result@v1' ||
|
||||
resultEnvelope.status !== 'available'
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const reference = exactRecord(resultEnvelope.reference, [
|
||||
'algorithm',
|
||||
'artifactDigest',
|
||||
'artifactId',
|
||||
'contentDigest',
|
||||
'invocationId',
|
||||
'keyId',
|
||||
'outputBytes',
|
||||
'projectId',
|
||||
'retentionEligibleAtMs',
|
||||
'retentionPolicyDigest',
|
||||
'runId',
|
||||
'schema',
|
||||
'stepRunId',
|
||||
]);
|
||||
const result = exactRecord(resultEnvelope.result, [
|
||||
'finishReason',
|
||||
'model',
|
||||
'provider',
|
||||
'text',
|
||||
'usage',
|
||||
]);
|
||||
const usage = result ? exactRecord(result.usage, [
|
||||
'inputTokens',
|
||||
'outputTokens',
|
||||
'totalTokens',
|
||||
], ['costMicros']) : null;
|
||||
if (
|
||||
!reference ||
|
||||
!result ||
|
||||
!usage ||
|
||||
reference.schema !==
|
||||
'qinglong/plugin-package-prompt-output-artifact-reference@v1' ||
|
||||
reference.algorithm !== 'aes-256-gcm' ||
|
||||
reference.projectId !== expected.projectId ||
|
||||
reference.runId !== expected.runId ||
|
||||
reference.artifactId !== expected.artifactId ||
|
||||
reference.artifactDigest !== expected.artifactDigest ||
|
||||
typeof reference.stepRunId !== 'string' ||
|
||||
!IDENTITY.test(reference.stepRunId) ||
|
||||
typeof reference.invocationId !== 'string' ||
|
||||
!IDENTITY.test(reference.invocationId) ||
|
||||
typeof reference.contentDigest !== 'string' ||
|
||||
!DIGEST.test(reference.contentDigest) ||
|
||||
!nonNegativeInteger(reference.outputBytes) ||
|
||||
reference.outputBytes > 1024 * 1024 ||
|
||||
typeof reference.retentionPolicyDigest !== 'string' ||
|
||||
!DIGEST.test(reference.retentionPolicyDigest) ||
|
||||
!nonNegativeInteger(reference.retentionEligibleAtMs) ||
|
||||
typeof reference.keyId !== 'string' ||
|
||||
!KEY_ID.test(reference.keyId) ||
|
||||
typeof result.provider !== 'string' ||
|
||||
!MODEL_ID.test(result.provider) ||
|
||||
typeof result.model !== 'string' ||
|
||||
!MODEL_ID.test(result.model) ||
|
||||
typeof result.text !== 'string' ||
|
||||
Buffer.byteLength(result.text, 'utf8') > 1024 * 1024 ||
|
||||
!FINISH_REASONS.has(result.finishReason as string) ||
|
||||
!nonNegativeInteger(usage.inputTokens) ||
|
||||
!nonNegativeInteger(usage.outputTokens) ||
|
||||
!nonNegativeInteger(usage.totalTokens) ||
|
||||
usage.totalTokens !== usage.inputTokens + usage.outputTokens ||
|
||||
(usage.costMicros !== undefined && !nonNegativeInteger(usage.costMicros))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return Object.freeze({
|
||||
schema: CLUSTER_PLUGIN_PACKAGE_PROMPT_OUTPUT_READ_RESPONSE_SCHEMA,
|
||||
status: 'available',
|
||||
reference: Object.freeze({ ...reference }),
|
||||
result: Object.freeze({
|
||||
provider: result.provider,
|
||||
model: result.model,
|
||||
text: result.text,
|
||||
finishReason: result.finishReason,
|
||||
usage: Object.freeze({ ...usage }),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export function createClusterControlPluginPackagePromptOutputReadRoute(
|
||||
capability: ClusterPluginPackagePromptOutputReadCapability,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (!capability || typeof capability.read !== 'function') {
|
||||
throw new TypeError('Cluster-control Prompt output read capability is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_PLUGIN_PACKAGE_PROMPT_OUTPUT_READ_ROUTE,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
const artifactDigestValues =
|
||||
authorized.request.query.artifact_digest;
|
||||
if (
|
||||
authorized.request.body !== null ||
|
||||
authorized.projectId === null ||
|
||||
typeof parameters.runId !== 'string' ||
|
||||
!RUN_ID.test(parameters.runId) ||
|
||||
typeof parameters.artifactId !== 'string' ||
|
||||
!IDENTITY.test(parameters.artifactId) ||
|
||||
!Array.isArray(artifactDigestValues) ||
|
||||
artifactDigestValues.length !== 1 ||
|
||||
typeof artifactDigestValues[0] !== 'string' ||
|
||||
!DIGEST.test(artifactDigestValues[0])
|
||||
) {
|
||||
return response(400, { code: 'invalid_prompt_output_read_request' });
|
||||
}
|
||||
const expected = Object.freeze({
|
||||
projectId: authorized.projectId,
|
||||
runId: parameters.runId,
|
||||
artifactId: parameters.artifactId,
|
||||
artifactDigest: artifactDigestValues[0],
|
||||
});
|
||||
try {
|
||||
const result = await capability.read({
|
||||
principal: authorized.principal,
|
||||
...expected,
|
||||
});
|
||||
const notFound = exactRecord(result, ['schema', 'status']);
|
||||
if (
|
||||
notFound &&
|
||||
notFound.schema ===
|
||||
'qinglong/plugin-package-prompt-output-read-result@v1' &&
|
||||
notFound.status === 'not_found'
|
||||
) {
|
||||
return response(404, { code: 'prompt_output_not_found' });
|
||||
}
|
||||
const view = availableView(result, expected);
|
||||
return view
|
||||
? response(200, view)
|
||||
: response(503, { code: 'prompt_output_read_unavailable' });
|
||||
} catch {
|
||||
return response(503, { code: 'prompt_output_read_unavailable' });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
// Plugin Package Prompt owns its stable execution and output-read route surface.
|
||||
export * from './pluginPackagePromptExecutionRoute';
|
||||
export * from './pluginPackagePromptCatalogRoute';
|
||||
export * from './pluginPackagePromptExecutionInspectionRoute';
|
||||
export * from './pluginPackagePromptExecutionOutputReadRoute';
|
||||
export * from './pluginPackagePromptOutputReadRoute';
|
||||
+535
@@ -0,0 +1,535 @@
|
||||
// Plugin Package Workflow owns inspection and durable authorized admission.
|
||||
import type {
|
||||
SecurityPolicyFence,
|
||||
SecurityPrincipal,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
import { normalizeSecurityAuditRecord } from '@qinglong/runtime-core/security-audit';
|
||||
import type {
|
||||
PluginPackageAutomationPublication,
|
||||
PluginPackageAutomationPublicationRepository,
|
||||
} from '@qinglong/runtime-core/plugin-package-automation-publication';
|
||||
import type {
|
||||
PluginPackageMaterializedRevision,
|
||||
PluginPackageMaterializedRevisionRepository,
|
||||
PluginPackageWorkflowResource,
|
||||
} from '@qinglong/runtime-core/plugin-package-resource-materialization';
|
||||
import type {
|
||||
PluginPackageWorkflowAdministrationRepository,
|
||||
PluginPackageWorkflowRunEventListRepository,
|
||||
PluginPackageWorkflowRunEventListResult,
|
||||
PluginPackageWorkflowRunInspectionRepository,
|
||||
PluginPackageWorkflowRunInspectionResult,
|
||||
PluginPackageWorkflowRunListRepository,
|
||||
PluginPackageWorkflowRunListResult,
|
||||
PluginPackageWorkflowStepRunListRepository,
|
||||
PluginPackageWorkflowStepRunListResult,
|
||||
} from '@qinglong/runtime-core/plugin-package-workflow-administration';
|
||||
import type {
|
||||
ClusterRunCancellationRepository,
|
||||
ClusterRunCancellationResult,
|
||||
} from '@qinglong/runtime-core/cluster-run-cancellation';
|
||||
import {
|
||||
createPluginPackageWorkflowExecutionPlan,
|
||||
type PluginPackageWorkflowAdmissionReceipt,
|
||||
type PluginPackageWorkflowExecutionPlan,
|
||||
} from '@qinglong/runtime-core/plugin-package-workflow-execution-plan';
|
||||
import {
|
||||
TaskSpecSemanticRegistry,
|
||||
createBuiltInTaskSpecSemanticRegistry,
|
||||
} from '@qinglong/runtime-core/task-spec-semantic';
|
||||
|
||||
export interface ClusterPluginPackageWorkflowSummary {
|
||||
readonly id: string;
|
||||
readonly name: string;
|
||||
readonly enabled: boolean;
|
||||
readonly steps: readonly Readonly<{
|
||||
id: string;
|
||||
task: string;
|
||||
needs: readonly string[];
|
||||
}>[];
|
||||
}
|
||||
|
||||
export interface StartClusterPluginPackageWorkflowCommand {
|
||||
readonly projectId: string;
|
||||
readonly packageName: string;
|
||||
readonly workflowId: string;
|
||||
readonly planId: string;
|
||||
readonly runId: string;
|
||||
readonly stepRunIds: Readonly<Record<string, string>>;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
readonly plannedAtMs: number;
|
||||
}
|
||||
|
||||
export interface CancelClusterPluginPackageWorkflowCommand {
|
||||
readonly projectId: string;
|
||||
readonly packageName: string;
|
||||
readonly workflowId: string;
|
||||
readonly runId: string;
|
||||
readonly mutationId: string;
|
||||
readonly eventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
}
|
||||
|
||||
export interface InspectClusterPluginPackageWorkflowRunCommand {
|
||||
readonly projectId: string;
|
||||
readonly packageName: string;
|
||||
readonly workflowId: string;
|
||||
readonly runId: string;
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
readonly observedAtMs: number;
|
||||
}
|
||||
|
||||
export interface ListClusterPluginPackageWorkflowRunsCommand {
|
||||
readonly projectId: string;
|
||||
readonly packageName: string;
|
||||
readonly workflowId: string;
|
||||
readonly limit: number;
|
||||
readonly after: Readonly<{ admittedAtMs: number; runId: string }> | null;
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
readonly observedAtMs: number;
|
||||
}
|
||||
|
||||
export interface ListClusterPluginPackageWorkflowStepRunsCommand {
|
||||
readonly projectId: string;
|
||||
readonly packageName: string;
|
||||
readonly workflowId: string;
|
||||
readonly runId: string;
|
||||
readonly limit: number;
|
||||
readonly after: Readonly<{ stepKey: string; id: string }> | null;
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
readonly observedAtMs: number;
|
||||
}
|
||||
|
||||
export interface ListClusterPluginPackageWorkflowRunEventsCommand {
|
||||
readonly projectId: string;
|
||||
readonly packageName: string;
|
||||
readonly workflowId: string;
|
||||
readonly runId: string;
|
||||
readonly limit: number;
|
||||
readonly afterSequence: number;
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
readonly observedAtMs: number;
|
||||
}
|
||||
|
||||
export interface ClusterPluginPackageWorkflowAdministrationCapability {
|
||||
inspect(
|
||||
projectId: string,
|
||||
packageName: string,
|
||||
): Promise<
|
||||
Readonly<{
|
||||
found: boolean;
|
||||
publicationState: PluginPackageAutomationPublication['state'] | null;
|
||||
workflows: readonly Readonly<ClusterPluginPackageWorkflowSummary>[];
|
||||
}>
|
||||
>;
|
||||
start(command: Readonly<StartClusterPluginPackageWorkflowCommand>): Promise<
|
||||
Readonly<{
|
||||
status: 'created' | 'existing';
|
||||
plan: Readonly<PluginPackageWorkflowExecutionPlan>;
|
||||
receipt: Readonly<PluginPackageWorkflowAdmissionReceipt>;
|
||||
}>
|
||||
>;
|
||||
cancel(
|
||||
command: Readonly<CancelClusterPluginPackageWorkflowCommand>,
|
||||
): Promise<Readonly<ClusterRunCancellationResult>>;
|
||||
inspectRun(
|
||||
command: Readonly<InspectClusterPluginPackageWorkflowRunCommand>,
|
||||
): Promise<Readonly<PluginPackageWorkflowRunInspectionResult>>;
|
||||
listRuns(
|
||||
command: Readonly<ListClusterPluginPackageWorkflowRunsCommand>,
|
||||
): Promise<Readonly<PluginPackageWorkflowRunListResult>>;
|
||||
listStepRuns(
|
||||
command: Readonly<ListClusterPluginPackageWorkflowStepRunsCommand>,
|
||||
): Promise<Readonly<PluginPackageWorkflowStepRunListResult>>;
|
||||
listRunEvents(
|
||||
command: Readonly<ListClusterPluginPackageWorkflowRunEventsCommand>,
|
||||
): Promise<Readonly<PluginPackageWorkflowRunEventListResult>>;
|
||||
}
|
||||
|
||||
export class ClusterPluginPackageWorkflowNotFoundError extends Error {
|
||||
readonly code = 'CLUSTER_PLUGIN_PACKAGE_WORKFLOW_NOT_FOUND';
|
||||
|
||||
constructor() {
|
||||
super('Active Plugin Package Workflow is not available');
|
||||
this.name = 'ClusterPluginPackageWorkflowNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterPluginPackageWorkflowConflictError extends Error {
|
||||
readonly code = 'CLUSTER_PLUGIN_PACKAGE_WORKFLOW_CONFLICT';
|
||||
|
||||
constructor() {
|
||||
super('Plugin Package Workflow request conflicts with durable identity');
|
||||
this.name = 'ClusterPluginPackageWorkflowConflictError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterPluginPackageWorkflowUnavailableError extends Error {
|
||||
readonly code = 'CLUSTER_PLUGIN_PACKAGE_WORKFLOW_UNAVAILABLE';
|
||||
|
||||
constructor() {
|
||||
super('Plugin Package Workflow administration is unavailable');
|
||||
this.name = 'ClusterPluginPackageWorkflowUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function summary(
|
||||
workflow: Readonly<PluginPackageWorkflowResource>,
|
||||
): Readonly<ClusterPluginPackageWorkflowSummary> {
|
||||
return Object.freeze({
|
||||
id: workflow.id,
|
||||
name: workflow.name,
|
||||
enabled: workflow.enabled,
|
||||
steps: Object.freeze(
|
||||
workflow.steps.map((step) =>
|
||||
Object.freeze({
|
||||
id: step.id,
|
||||
task: step.task,
|
||||
needs: Object.freeze([...step.needs]),
|
||||
}),
|
||||
),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function sameReplay(
|
||||
plan: Readonly<PluginPackageWorkflowExecutionPlan>,
|
||||
command: Readonly<StartClusterPluginPackageWorkflowCommand>,
|
||||
): boolean {
|
||||
const requested = Object.entries(command.stepRunIds).sort(([a], [b]) =>
|
||||
a.localeCompare(b),
|
||||
);
|
||||
const stored = plan.steps
|
||||
.map((step) => [step.stepKey, step.stepRunId] as const)
|
||||
.sort(([a], [b]) => a.localeCompare(b));
|
||||
return (
|
||||
plan.planId === command.planId &&
|
||||
plan.runId === command.runId &&
|
||||
plan.target.projectId === command.projectId &&
|
||||
plan.target.packageName === command.packageName &&
|
||||
plan.target.workflowId === command.workflowId &&
|
||||
stored.length === requested.length &&
|
||||
stored.every(
|
||||
([key, id], index) =>
|
||||
key === requested[index]?.[0] && id === requested[index]?.[1],
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function createClusterPluginPackageWorkflowAdministrationCapability(
|
||||
publications: Pick<
|
||||
PluginPackageAutomationPublicationRepository,
|
||||
'findCurrent'
|
||||
>,
|
||||
revisions: Pick<PluginPackageMaterializedRevisionRepository, 'find'>,
|
||||
admissions: PluginPackageWorkflowAdministrationRepository,
|
||||
runInspections: PluginPackageWorkflowRunInspectionRepository,
|
||||
runLists: PluginPackageWorkflowRunListRepository,
|
||||
stepRunLists: PluginPackageWorkflowStepRunListRepository,
|
||||
runEventLists: PluginPackageWorkflowRunEventListRepository,
|
||||
cancellations: ClusterRunCancellationRepository,
|
||||
taskSpecSemanticRegistry: TaskSpecSemanticRegistry = createBuiltInTaskSpecSemanticRegistry(),
|
||||
): ClusterPluginPackageWorkflowAdministrationCapability {
|
||||
if (
|
||||
!publications ||
|
||||
typeof publications.findCurrent !== 'function' ||
|
||||
!revisions ||
|
||||
typeof revisions.find !== 'function' ||
|
||||
!admissions ||
|
||||
typeof admissions.findPlanByPlanId !== 'function' ||
|
||||
typeof admissions.admitAuthorized !== 'function' ||
|
||||
!runInspections ||
|
||||
typeof runInspections.inspectRunAuthorized !== 'function' ||
|
||||
!runLists ||
|
||||
typeof runLists.listRunsAuthorized !== 'function' ||
|
||||
!stepRunLists ||
|
||||
typeof stepRunLists.listStepRunsAuthorized !== 'function' ||
|
||||
!runEventLists ||
|
||||
typeof runEventLists.listRunEventsAuthorized !== 'function' ||
|
||||
!cancellations ||
|
||||
typeof cancellations.requestUserCancellation !== 'function' ||
|
||||
!(taskSpecSemanticRegistry instanceof TaskSpecSemanticRegistry)
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster Plugin Package Workflow administration dependencies are invalid',
|
||||
);
|
||||
}
|
||||
|
||||
async function currentTarget(
|
||||
projectId: string,
|
||||
packageName: string,
|
||||
): Promise<Readonly<{
|
||||
publication: Readonly<PluginPackageAutomationPublication>;
|
||||
revision: Readonly<PluginPackageMaterializedRevision>;
|
||||
}> | null> {
|
||||
try {
|
||||
const publication = await publications.findCurrent(
|
||||
projectId,
|
||||
packageName,
|
||||
);
|
||||
if (!publication) return null;
|
||||
const revision = await revisions.find(
|
||||
publication.target.generationDigest,
|
||||
);
|
||||
if (
|
||||
!revision ||
|
||||
revision.revisionDigest !==
|
||||
publication.target.materializedRevisionDigest
|
||||
) {
|
||||
throw new ClusterPluginPackageWorkflowUnavailableError();
|
||||
}
|
||||
return Object.freeze({ publication, revision });
|
||||
} catch (error) {
|
||||
if (error instanceof ClusterPluginPackageWorkflowUnavailableError) {
|
||||
throw error;
|
||||
}
|
||||
throw new ClusterPluginPackageWorkflowUnavailableError();
|
||||
}
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
async inspect(projectId: string, packageName: string) {
|
||||
const target = await currentTarget(projectId, packageName);
|
||||
return target
|
||||
? Object.freeze({
|
||||
found: true,
|
||||
publicationState: target.publication.state,
|
||||
workflows: Object.freeze(
|
||||
target.publication.definitions.workflows.map(summary),
|
||||
),
|
||||
})
|
||||
: Object.freeze({
|
||||
found: false,
|
||||
publicationState: null,
|
||||
workflows: Object.freeze([]),
|
||||
});
|
||||
},
|
||||
|
||||
async start(command: Readonly<StartClusterPluginPackageWorkflowCommand>) {
|
||||
let plan = await admissions.findPlanByPlanId(command.planId);
|
||||
if (plan) {
|
||||
if (!sameReplay(plan, command)) {
|
||||
throw new ClusterPluginPackageWorkflowConflictError();
|
||||
}
|
||||
} else {
|
||||
const target = await currentTarget(
|
||||
command.projectId,
|
||||
command.packageName,
|
||||
);
|
||||
const workflow = target?.publication.definitions.workflows.find(
|
||||
({ id }) => id === command.workflowId,
|
||||
);
|
||||
if (
|
||||
!target ||
|
||||
target.publication.state !== 'active' ||
|
||||
!workflow?.enabled
|
||||
) {
|
||||
throw new ClusterPluginPackageWorkflowNotFoundError();
|
||||
}
|
||||
try {
|
||||
plan = createPluginPackageWorkflowExecutionPlan({
|
||||
planId: command.planId,
|
||||
runId: command.runId,
|
||||
workflowId: command.workflowId,
|
||||
stepRunIds: command.stepRunIds,
|
||||
publication: target.publication,
|
||||
revision: target.revision,
|
||||
taskSpecSemanticRegistry,
|
||||
plannedAtMs: command.plannedAtMs,
|
||||
});
|
||||
} catch {
|
||||
throw new ClusterPluginPackageWorkflowConflictError();
|
||||
}
|
||||
}
|
||||
if (command.policyFence.bindingVersion === null) {
|
||||
throw new ClusterPluginPackageWorkflowUnavailableError();
|
||||
}
|
||||
const admitted = await admissions.admitAuthorized({
|
||||
plan,
|
||||
actor: command.principal.subject,
|
||||
fence: {
|
||||
projectVersion: command.policyFence.projectVersion,
|
||||
bindingVersion: command.policyFence.bindingVersion,
|
||||
},
|
||||
audit: normalizeSecurityAuditRecord({
|
||||
eventId: command.planId,
|
||||
requestId: command.planId,
|
||||
operationId: 'workflow.start',
|
||||
projectId: command.projectId,
|
||||
subject: command.principal.subject,
|
||||
authenticationId: command.principal.authenticationId,
|
||||
outcome: 'allowed',
|
||||
reasons: ['project_policy_allowed'],
|
||||
fence: command.policyFence,
|
||||
occurredAtMs: plan.plannedAtMs,
|
||||
}),
|
||||
});
|
||||
return Object.freeze({
|
||||
status: admitted.status,
|
||||
plan,
|
||||
receipt: admitted.receipt,
|
||||
});
|
||||
},
|
||||
|
||||
async cancel(command: Readonly<CancelClusterPluginPackageWorkflowCommand>) {
|
||||
if (command.policyFence.bindingVersion === null) {
|
||||
throw new ClusterPluginPackageWorkflowUnavailableError();
|
||||
}
|
||||
return cancellations.requestUserCancellation({
|
||||
projectId: command.projectId,
|
||||
runId: command.runId,
|
||||
mutationId: command.mutationId,
|
||||
eventId: command.eventId,
|
||||
subject: command.principal.subject,
|
||||
policyFence: command.policyFence,
|
||||
workflowTarget: {
|
||||
packageName: command.packageName,
|
||||
workflowId: command.workflowId,
|
||||
},
|
||||
});
|
||||
},
|
||||
|
||||
async inspectRun(
|
||||
command: Readonly<InspectClusterPluginPackageWorkflowRunCommand>,
|
||||
) {
|
||||
if (command.policyFence.bindingVersion === null) {
|
||||
throw new ClusterPluginPackageWorkflowUnavailableError();
|
||||
}
|
||||
return runInspections.inspectRunAuthorized({
|
||||
projectId: command.projectId,
|
||||
packageName: command.packageName,
|
||||
workflowId: command.workflowId,
|
||||
runId: command.runId,
|
||||
actor: command.principal.subject,
|
||||
fence: {
|
||||
projectVersion: command.policyFence.projectVersion,
|
||||
bindingVersion: command.policyFence.bindingVersion,
|
||||
},
|
||||
audit: normalizeSecurityAuditRecord({
|
||||
eventId: command.auditEventId,
|
||||
requestId: command.requestId,
|
||||
operationId: 'workflow.run.read',
|
||||
projectId: command.projectId,
|
||||
subject: command.principal.subject,
|
||||
authenticationId: command.principal.authenticationId,
|
||||
outcome: 'allowed',
|
||||
reasons: ['project_policy_allowed'],
|
||||
fence: command.policyFence,
|
||||
occurredAtMs: command.observedAtMs,
|
||||
}),
|
||||
});
|
||||
},
|
||||
|
||||
async listRuns(
|
||||
command: Readonly<ListClusterPluginPackageWorkflowRunsCommand>,
|
||||
) {
|
||||
if (command.policyFence.bindingVersion === null) {
|
||||
throw new ClusterPluginPackageWorkflowUnavailableError();
|
||||
}
|
||||
return runLists.listRunsAuthorized({
|
||||
projectId: command.projectId,
|
||||
packageName: command.packageName,
|
||||
workflowId: command.workflowId,
|
||||
limit: command.limit,
|
||||
after: command.after,
|
||||
actor: command.principal.subject,
|
||||
fence: {
|
||||
projectVersion: command.policyFence.projectVersion,
|
||||
bindingVersion: command.policyFence.bindingVersion,
|
||||
},
|
||||
audit: normalizeSecurityAuditRecord({
|
||||
eventId: command.auditEventId,
|
||||
requestId: command.requestId,
|
||||
operationId: 'workflow.run.list',
|
||||
projectId: command.projectId,
|
||||
subject: command.principal.subject,
|
||||
authenticationId: command.principal.authenticationId,
|
||||
outcome: 'allowed',
|
||||
reasons: ['project_policy_allowed'],
|
||||
fence: command.policyFence,
|
||||
occurredAtMs: command.observedAtMs,
|
||||
}),
|
||||
});
|
||||
},
|
||||
|
||||
async listStepRuns(
|
||||
command: Readonly<ListClusterPluginPackageWorkflowStepRunsCommand>,
|
||||
) {
|
||||
if (command.policyFence.bindingVersion === null) {
|
||||
throw new ClusterPluginPackageWorkflowUnavailableError();
|
||||
}
|
||||
return stepRunLists.listStepRunsAuthorized({
|
||||
projectId: command.projectId,
|
||||
packageName: command.packageName,
|
||||
workflowId: command.workflowId,
|
||||
runId: command.runId,
|
||||
limit: command.limit,
|
||||
after: command.after,
|
||||
actor: command.principal.subject,
|
||||
fence: {
|
||||
projectVersion: command.policyFence.projectVersion,
|
||||
bindingVersion: command.policyFence.bindingVersion,
|
||||
},
|
||||
audit: normalizeSecurityAuditRecord({
|
||||
eventId: command.auditEventId,
|
||||
requestId: command.requestId,
|
||||
operationId: 'workflow.step.list',
|
||||
projectId: command.projectId,
|
||||
subject: command.principal.subject,
|
||||
authenticationId: command.principal.authenticationId,
|
||||
outcome: 'allowed',
|
||||
reasons: ['project_policy_allowed'],
|
||||
fence: command.policyFence,
|
||||
occurredAtMs: command.observedAtMs,
|
||||
}),
|
||||
});
|
||||
},
|
||||
|
||||
async listRunEvents(
|
||||
command: Readonly<ListClusterPluginPackageWorkflowRunEventsCommand>,
|
||||
) {
|
||||
if (command.policyFence.bindingVersion === null) {
|
||||
throw new ClusterPluginPackageWorkflowUnavailableError();
|
||||
}
|
||||
return runEventLists.listRunEventsAuthorized({
|
||||
projectId: command.projectId,
|
||||
packageName: command.packageName,
|
||||
workflowId: command.workflowId,
|
||||
runId: command.runId,
|
||||
limit: command.limit,
|
||||
afterSequence: command.afterSequence,
|
||||
actor: command.principal.subject,
|
||||
fence: {
|
||||
projectVersion: command.policyFence.projectVersion,
|
||||
bindingVersion: command.policyFence.bindingVersion,
|
||||
},
|
||||
audit: normalizeSecurityAuditRecord({
|
||||
eventId: command.auditEventId,
|
||||
requestId: command.requestId,
|
||||
operationId: 'workflow.event.list',
|
||||
projectId: command.projectId,
|
||||
subject: command.principal.subject,
|
||||
authenticationId: command.principal.authenticationId,
|
||||
outcome: 'allowed',
|
||||
reasons: ['project_policy_allowed'],
|
||||
fence: command.policyFence,
|
||||
occurredAtMs: command.observedAtMs,
|
||||
}),
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
+707
@@ -0,0 +1,707 @@
|
||||
// Plugin Package Workflow owns its bounded inspect/start/cancel transport adapter.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
CLUSTER_RUN_CANCELLATION_SCHEMA,
|
||||
createClusterRunCancellationResponseBody,
|
||||
parseClusterRunCancellationRequestBody,
|
||||
} from '@qinglong/runtime-core/cluster-run-cancellation';
|
||||
import {
|
||||
DEFAULT_PLUGIN_PACKAGE_WORKFLOW_RUN_EVENT_LIST_PAGE_SIZE,
|
||||
DEFAULT_PLUGIN_PACKAGE_WORKFLOW_RUN_LIST_PAGE_SIZE,
|
||||
DEFAULT_PLUGIN_PACKAGE_WORKFLOW_STEP_RUN_LIST_PAGE_SIZE,
|
||||
MAX_PLUGIN_PACKAGE_WORKFLOW_RUN_EVENT_LIST_PAGE_SIZE,
|
||||
MAX_PLUGIN_PACKAGE_WORKFLOW_RUN_LIST_PAGE_SIZE,
|
||||
MAX_PLUGIN_PACKAGE_WORKFLOW_STEP_RUN_LIST_PAGE_SIZE,
|
||||
} from '@qinglong/runtime-core/plugin-package-workflow-administration';
|
||||
import type { ClusterControlAdmissionResponse } from '../../transport/httpSurface';
|
||||
import type { ClusterPluginPackageWorkflowAdministrationCapability } from './pluginPackageWorkflowAdministration';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_PLUGIN_PACKAGE_WORKFLOW_LIST_RESPONSE_SCHEMA =
|
||||
'qinglong/cluster-plugin-package-workflow-list@v1' as const;
|
||||
export const CLUSTER_PLUGIN_PACKAGE_WORKFLOW_START_REQUEST_SCHEMA =
|
||||
'qinglong/cluster-plugin-package-workflow-start-request@v1' as const;
|
||||
export const CLUSTER_PLUGIN_PACKAGE_WORKFLOW_START_RESPONSE_SCHEMA =
|
||||
'qinglong/cluster-plugin-package-workflow-start-response@v1' as const;
|
||||
|
||||
const UUID_V4 =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const PACKAGE_NAME = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
|
||||
const RESOURCE_ID = /^[a-z][a-z0-9-]{0,62}$/;
|
||||
|
||||
function parseRunListQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): Readonly<{
|
||||
limit: number;
|
||||
after: Readonly<{ admittedAtMs: number; runId: string }> | null;
|
||||
}> {
|
||||
const limitValues = query.limit;
|
||||
const admittedAtValues = query.after_admitted_at_ms;
|
||||
const runIdValues = query.after_run_id;
|
||||
if (
|
||||
(limitValues !== undefined && limitValues.length !== 1) ||
|
||||
(admittedAtValues !== undefined && admittedAtValues.length !== 1) ||
|
||||
(runIdValues !== undefined && runIdValues.length !== 1) ||
|
||||
(admittedAtValues === undefined) !== (runIdValues === undefined)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const limit =
|
||||
limitValues === undefined
|
||||
? DEFAULT_PLUGIN_PACKAGE_WORKFLOW_RUN_LIST_PAGE_SIZE
|
||||
: Number(limitValues[0]);
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_PLUGIN_PACKAGE_WORKFLOW_RUN_LIST_PAGE_SIZE ||
|
||||
(limitValues !== undefined && String(limit) !== limitValues[0])
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
if (admittedAtValues === undefined || runIdValues === undefined) {
|
||||
return Object.freeze({ limit, after: null });
|
||||
}
|
||||
const admittedAtMs = Number(admittedAtValues[0]);
|
||||
const runId = runIdValues[0]!;
|
||||
if (
|
||||
!Number.isSafeInteger(admittedAtMs) ||
|
||||
admittedAtMs < 0 ||
|
||||
String(admittedAtMs) !== admittedAtValues[0] ||
|
||||
!UUID_V4.test(runId)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return Object.freeze({
|
||||
limit,
|
||||
after: Object.freeze({ admittedAtMs, runId }),
|
||||
});
|
||||
}
|
||||
|
||||
function parseStepRunListQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): Readonly<{
|
||||
limit: number;
|
||||
after: Readonly<{ stepKey: string; id: string }> | null;
|
||||
}> {
|
||||
const limitValues = query.limit;
|
||||
const stepKeyValues = query.after_step_key;
|
||||
const stepRunIdValues = query.after_step_run_id;
|
||||
if (
|
||||
(limitValues !== undefined && limitValues.length !== 1) ||
|
||||
(stepKeyValues !== undefined && stepKeyValues.length !== 1) ||
|
||||
(stepRunIdValues !== undefined && stepRunIdValues.length !== 1) ||
|
||||
(stepKeyValues === undefined) !== (stepRunIdValues === undefined)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const limit =
|
||||
limitValues === undefined
|
||||
? DEFAULT_PLUGIN_PACKAGE_WORKFLOW_STEP_RUN_LIST_PAGE_SIZE
|
||||
: Number(limitValues[0]);
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_PLUGIN_PACKAGE_WORKFLOW_STEP_RUN_LIST_PAGE_SIZE ||
|
||||
(limitValues !== undefined && String(limit) !== limitValues[0])
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
if (stepKeyValues === undefined || stepRunIdValues === undefined) {
|
||||
return Object.freeze({ limit, after: null });
|
||||
}
|
||||
const stepKey = stepKeyValues[0]!;
|
||||
const id = stepRunIdValues[0]!;
|
||||
if (!RESOURCE_ID.test(stepKey) || !UUID_V4.test(id)) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return Object.freeze({
|
||||
limit,
|
||||
after: Object.freeze({ stepKey, id }),
|
||||
});
|
||||
}
|
||||
|
||||
function parseRunEventListQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): Readonly<{ limit: number; afterSequence: number }> {
|
||||
const limitValues = query.limit;
|
||||
const afterValues = query.after_sequence;
|
||||
if (
|
||||
(limitValues !== undefined && limitValues.length !== 1) ||
|
||||
(afterValues !== undefined && afterValues.length !== 1)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const limit =
|
||||
limitValues === undefined
|
||||
? DEFAULT_PLUGIN_PACKAGE_WORKFLOW_RUN_EVENT_LIST_PAGE_SIZE
|
||||
: Number(limitValues[0]);
|
||||
const afterSequence = afterValues === undefined ? 0 : Number(afterValues[0]);
|
||||
if (
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit < 1 ||
|
||||
limit > MAX_PLUGIN_PACKAGE_WORKFLOW_RUN_EVENT_LIST_PAGE_SIZE ||
|
||||
(limitValues !== undefined && String(limit) !== limitValues[0]) ||
|
||||
!Number.isSafeInteger(afterSequence) ||
|
||||
afterSequence < 0 ||
|
||||
(afterValues !== undefined && String(afterSequence) !== afterValues[0])
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return Object.freeze({ limit, afterSequence });
|
||||
}
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function parseBody(value: unknown): Readonly<{
|
||||
planId: string;
|
||||
runId: string;
|
||||
stepRunIds: Readonly<Record<string, string>>;
|
||||
}> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const body = value as Record<string, unknown>;
|
||||
if (
|
||||
Object.keys(body).sort().join(',') !== 'planId,runId,schema,stepRunIds' ||
|
||||
body.schema !== CLUSTER_PLUGIN_PACKAGE_WORKFLOW_START_REQUEST_SCHEMA ||
|
||||
typeof body.planId !== 'string' ||
|
||||
!UUID_V4.test(body.planId) ||
|
||||
typeof body.runId !== 'string' ||
|
||||
!UUID_V4.test(body.runId) ||
|
||||
!body.stepRunIds ||
|
||||
typeof body.stepRunIds !== 'object' ||
|
||||
Array.isArray(body.stepRunIds) ||
|
||||
Object.getPrototypeOf(body.stepRunIds) !== Object.prototype
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const entries = Object.entries(body.stepRunIds as Record<string, unknown>);
|
||||
if (
|
||||
entries.length < 1 ||
|
||||
entries.length > 128 ||
|
||||
entries.some(
|
||||
([key, id]) =>
|
||||
!RESOURCE_ID.test(key) || typeof id !== 'string' || !UUID_V4.test(id),
|
||||
) ||
|
||||
new Set(entries.map(([, id]) => id)).size !== entries.length
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return Object.freeze({
|
||||
planId: body.planId,
|
||||
runId: body.runId,
|
||||
stepRunIds: Object.freeze(
|
||||
Object.fromEntries(entries) as Record<string, string>,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
function errorResponse(error: unknown): ClusterControlAdmissionResponse {
|
||||
const code =
|
||||
error && typeof error === 'object' && 'code' in error
|
||||
? (error as { code?: unknown }).code
|
||||
: null;
|
||||
if (code === 'CLUSTER_PLUGIN_PACKAGE_WORKFLOW_NOT_FOUND') {
|
||||
return response(404, { code: 'workflow_not_found' });
|
||||
}
|
||||
if (
|
||||
code === 'CLUSTER_PLUGIN_PACKAGE_WORKFLOW_CONFLICT' ||
|
||||
code === 'PLUGIN_PACKAGE_WORKFLOW_ADMINISTRATION_MUTATION_CONFLICT' ||
|
||||
code === 'PLUGIN_PACKAGE_WORKFLOW_ADMISSION_CONFLICT'
|
||||
) {
|
||||
return response(409, { code: 'workflow_start_conflict' });
|
||||
}
|
||||
if (
|
||||
code ===
|
||||
'PLUGIN_PACKAGE_WORKFLOW_ADMINISTRATION_AUTHORIZATION_FENCE_CONFLICT'
|
||||
) {
|
||||
return response(409, { code: 'authorization_fence_changed' });
|
||||
}
|
||||
if (code === 'CLUSTER_RUN_CANCELLATION_NOT_FOUND') {
|
||||
return response(404, { code: 'workflow_run_not_found' });
|
||||
}
|
||||
if (code === 'CLUSTER_RUN_CANCELLATION_FENCE_REJECTED') {
|
||||
const candidateReason =
|
||||
error && typeof error === 'object' && 'reason' in error
|
||||
? (error as { reason?: unknown }).reason
|
||||
: null;
|
||||
const reason =
|
||||
candidateReason === 'authorization_changed' ||
|
||||
candidateReason === 'project_mismatch' ||
|
||||
candidateReason === 'state_mismatch'
|
||||
? candidateReason
|
||||
: 'state_mismatch';
|
||||
return response(409, {
|
||||
code: 'workflow_cancellation_fence_rejected',
|
||||
reason,
|
||||
});
|
||||
}
|
||||
return response(503, { code: 'workflow_administration_unavailable' });
|
||||
}
|
||||
|
||||
function runInspectionErrorResponse(
|
||||
error: unknown,
|
||||
): ClusterControlAdmissionResponse {
|
||||
const code =
|
||||
error && typeof error === 'object' && 'code' in error
|
||||
? (error as { code?: unknown }).code
|
||||
: null;
|
||||
if (
|
||||
code ===
|
||||
'PLUGIN_PACKAGE_WORKFLOW_ADMINISTRATION_AUTHORIZATION_FENCE_CONFLICT'
|
||||
) {
|
||||
return response(409, { code: 'authorization_fence_changed' });
|
||||
}
|
||||
return response(503, { code: 'workflow_run_query_unavailable' });
|
||||
}
|
||||
|
||||
function runListErrorResponse(error: unknown): ClusterControlAdmissionResponse {
|
||||
const code =
|
||||
error && typeof error === 'object' && 'code' in error
|
||||
? (error as { code?: unknown }).code
|
||||
: null;
|
||||
if (
|
||||
code ===
|
||||
'PLUGIN_PACKAGE_WORKFLOW_ADMINISTRATION_AUTHORIZATION_FENCE_CONFLICT'
|
||||
) {
|
||||
return response(409, { code: 'authorization_fence_changed' });
|
||||
}
|
||||
return response(503, { code: 'workflow_run_list_unavailable' });
|
||||
}
|
||||
|
||||
function stepRunListErrorResponse(
|
||||
error: unknown,
|
||||
): ClusterControlAdmissionResponse {
|
||||
const code =
|
||||
error && typeof error === 'object' && 'code' in error
|
||||
? (error as { code?: unknown }).code
|
||||
: null;
|
||||
if (
|
||||
code ===
|
||||
'PLUGIN_PACKAGE_WORKFLOW_ADMINISTRATION_AUTHORIZATION_FENCE_CONFLICT'
|
||||
) {
|
||||
return response(409, { code: 'authorization_fence_changed' });
|
||||
}
|
||||
return response(503, { code: 'workflow_step_run_query_unavailable' });
|
||||
}
|
||||
|
||||
function runEventListErrorResponse(
|
||||
error: unknown,
|
||||
): ClusterControlAdmissionResponse {
|
||||
const code =
|
||||
error && typeof error === 'object' && 'code' in error
|
||||
? (error as { code?: unknown }).code
|
||||
: null;
|
||||
if (
|
||||
code ===
|
||||
'PLUGIN_PACKAGE_WORKFLOW_ADMINISTRATION_AUTHORIZATION_FENCE_CONFLICT'
|
||||
) {
|
||||
return response(409, { code: 'authorization_fence_changed' });
|
||||
}
|
||||
return response(503, { code: 'workflow_run_event_query_unavailable' });
|
||||
}
|
||||
|
||||
export function createClusterControlPluginPackageWorkflowRoutes(
|
||||
capability: ClusterPluginPackageWorkflowAdministrationCapability,
|
||||
now: () => number = Date.now,
|
||||
createEventId: () => string = randomUUID,
|
||||
): readonly Readonly<ClusterControlRouteDefinition>[] {
|
||||
if (
|
||||
!capability ||
|
||||
typeof capability.inspect !== 'function' ||
|
||||
typeof capability.inspectRun !== 'function' ||
|
||||
typeof capability.listRuns !== 'function' ||
|
||||
typeof capability.listStepRuns !== 'function' ||
|
||||
typeof capability.listRunEvents !== 'function' ||
|
||||
typeof capability.start !== 'function' ||
|
||||
typeof capability.cancel !== 'function' ||
|
||||
typeof now !== 'function' ||
|
||||
typeof createEventId !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster-control Workflow capability is invalid');
|
||||
}
|
||||
const common = {
|
||||
projectParameter: 'projectId' as const,
|
||||
};
|
||||
return Object.freeze([
|
||||
Object.freeze({
|
||||
...common,
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/packages/{packageName}/workflows',
|
||||
operationId: 'workflow.read',
|
||||
permission: 'run.read',
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (
|
||||
authorized.projectId === null ||
|
||||
typeof parameters.packageName !== 'string' ||
|
||||
!PACKAGE_NAME.test(parameters.packageName)
|
||||
) {
|
||||
return response(503, { code: 'workflow_administration_unavailable' });
|
||||
}
|
||||
try {
|
||||
const result = await capability.inspect(
|
||||
authorized.projectId,
|
||||
parameters.packageName,
|
||||
);
|
||||
return response(200, {
|
||||
schema: CLUSTER_PLUGIN_PACKAGE_WORKFLOW_LIST_RESPONSE_SCHEMA,
|
||||
...result,
|
||||
});
|
||||
} catch (error) {
|
||||
return errorResponse(error);
|
||||
}
|
||||
},
|
||||
}),
|
||||
Object.freeze({
|
||||
...common,
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/packages/{packageName}/workflows/{workflowId}/runs',
|
||||
operationId: 'workflow.run.list',
|
||||
permission: 'run.read',
|
||||
allowedQuery: Object.freeze([
|
||||
'after_admitted_at_ms',
|
||||
'after_run_id',
|
||||
'limit',
|
||||
]),
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
let page;
|
||||
try {
|
||||
page = parseRunListQuery(authorized.request.query);
|
||||
} catch {
|
||||
return response(400, { code: 'invalid_workflow_run_query' });
|
||||
}
|
||||
const observedAtMs = now();
|
||||
if (
|
||||
authorized.projectId === null ||
|
||||
typeof parameters.packageName !== 'string' ||
|
||||
!PACKAGE_NAME.test(parameters.packageName) ||
|
||||
typeof parameters.workflowId !== 'string' ||
|
||||
!RESOURCE_ID.test(parameters.workflowId) ||
|
||||
!authorized.policyFence ||
|
||||
authorized.policyFence.bindingVersion === null ||
|
||||
!Number.isSafeInteger(observedAtMs) ||
|
||||
observedAtMs < 0
|
||||
) {
|
||||
return response(503, { code: 'workflow_run_list_unavailable' });
|
||||
}
|
||||
try {
|
||||
const result = await capability.listRuns({
|
||||
projectId: authorized.projectId,
|
||||
packageName: parameters.packageName,
|
||||
workflowId: parameters.workflowId,
|
||||
limit: page.limit,
|
||||
after: page.after,
|
||||
requestId: authorized.request.requestId,
|
||||
auditEventId: createEventId(),
|
||||
principal: authorized.principal,
|
||||
policyFence: authorized.policyFence,
|
||||
observedAtMs,
|
||||
});
|
||||
return response(200, { ...result });
|
||||
} catch (error) {
|
||||
return runListErrorResponse(error);
|
||||
}
|
||||
},
|
||||
}),
|
||||
Object.freeze({
|
||||
...common,
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/packages/{packageName}/workflows/{workflowId}/runs/{runId}',
|
||||
operationId: 'workflow.run.read',
|
||||
permission: 'run.read',
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
const observedAtMs = now();
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
if (
|
||||
authorized.projectId === null ||
|
||||
typeof parameters.packageName !== 'string' ||
|
||||
!PACKAGE_NAME.test(parameters.packageName) ||
|
||||
typeof parameters.workflowId !== 'string' ||
|
||||
!RESOURCE_ID.test(parameters.workflowId) ||
|
||||
typeof parameters.runId !== 'string' ||
|
||||
!UUID_V4.test(parameters.runId) ||
|
||||
!authorized.policyFence ||
|
||||
authorized.policyFence.bindingVersion === null ||
|
||||
!Number.isSafeInteger(observedAtMs) ||
|
||||
observedAtMs < 0
|
||||
) {
|
||||
return response(503, { code: 'workflow_run_query_unavailable' });
|
||||
}
|
||||
try {
|
||||
const result = await capability.inspectRun({
|
||||
projectId: authorized.projectId,
|
||||
packageName: parameters.packageName,
|
||||
workflowId: parameters.workflowId,
|
||||
runId: parameters.runId,
|
||||
requestId: authorized.request.requestId,
|
||||
auditEventId: createEventId(),
|
||||
principal: authorized.principal,
|
||||
policyFence: authorized.policyFence,
|
||||
observedAtMs,
|
||||
});
|
||||
return result.found
|
||||
? response(200, { ...result })
|
||||
: response(404, { code: 'workflow_run_not_found' });
|
||||
} catch (error) {
|
||||
return runInspectionErrorResponse(error);
|
||||
}
|
||||
},
|
||||
}),
|
||||
Object.freeze({
|
||||
...common,
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/packages/{packageName}/workflows/{workflowId}/runs/{runId}/steps',
|
||||
operationId: 'workflow.step.list',
|
||||
permission: 'run.read',
|
||||
allowedQuery: Object.freeze([
|
||||
'after_step_key',
|
||||
'after_step_run_id',
|
||||
'limit',
|
||||
]),
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
let page;
|
||||
try {
|
||||
page = parseStepRunListQuery(authorized.request.query);
|
||||
} catch {
|
||||
return response(400, { code: 'invalid_step_run_query' });
|
||||
}
|
||||
const observedAtMs = now();
|
||||
if (
|
||||
authorized.projectId === null ||
|
||||
typeof parameters.packageName !== 'string' ||
|
||||
!PACKAGE_NAME.test(parameters.packageName) ||
|
||||
typeof parameters.workflowId !== 'string' ||
|
||||
!RESOURCE_ID.test(parameters.workflowId) ||
|
||||
typeof parameters.runId !== 'string' ||
|
||||
!UUID_V4.test(parameters.runId) ||
|
||||
!authorized.policyFence ||
|
||||
authorized.policyFence.bindingVersion === null ||
|
||||
!Number.isSafeInteger(observedAtMs) ||
|
||||
observedAtMs < 0
|
||||
) {
|
||||
return response(503, { code: 'workflow_step_run_query_unavailable' });
|
||||
}
|
||||
try {
|
||||
const result = await capability.listStepRuns({
|
||||
projectId: authorized.projectId,
|
||||
packageName: parameters.packageName,
|
||||
workflowId: parameters.workflowId,
|
||||
runId: parameters.runId,
|
||||
limit: page.limit,
|
||||
after: page.after,
|
||||
requestId: authorized.request.requestId,
|
||||
auditEventId: createEventId(),
|
||||
principal: authorized.principal,
|
||||
policyFence: authorized.policyFence,
|
||||
observedAtMs,
|
||||
});
|
||||
return result.found
|
||||
? response(200, { ...result })
|
||||
: response(404, { code: 'workflow_run_not_found' });
|
||||
} catch (error) {
|
||||
return stepRunListErrorResponse(error);
|
||||
}
|
||||
},
|
||||
}),
|
||||
Object.freeze({
|
||||
...common,
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/packages/{packageName}/workflows/{workflowId}/runs/{runId}/events',
|
||||
operationId: 'workflow.event.list',
|
||||
permission: 'run.read',
|
||||
allowedQuery: Object.freeze(['after_sequence', 'limit']),
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
let page;
|
||||
try {
|
||||
page = parseRunEventListQuery(authorized.request.query);
|
||||
} catch {
|
||||
return response(400, { code: 'invalid_run_event_query' });
|
||||
}
|
||||
const observedAtMs = now();
|
||||
if (
|
||||
authorized.projectId === null ||
|
||||
typeof parameters.packageName !== 'string' ||
|
||||
!PACKAGE_NAME.test(parameters.packageName) ||
|
||||
typeof parameters.workflowId !== 'string' ||
|
||||
!RESOURCE_ID.test(parameters.workflowId) ||
|
||||
typeof parameters.runId !== 'string' ||
|
||||
!UUID_V4.test(parameters.runId) ||
|
||||
!authorized.policyFence ||
|
||||
authorized.policyFence.bindingVersion === null ||
|
||||
!Number.isSafeInteger(observedAtMs) ||
|
||||
observedAtMs < 0
|
||||
) {
|
||||
return response(503, {
|
||||
code: 'workflow_run_event_query_unavailable',
|
||||
});
|
||||
}
|
||||
try {
|
||||
const result = await capability.listRunEvents({
|
||||
projectId: authorized.projectId,
|
||||
packageName: parameters.packageName,
|
||||
workflowId: parameters.workflowId,
|
||||
runId: parameters.runId,
|
||||
limit: page.limit,
|
||||
afterSequence: page.afterSequence,
|
||||
requestId: authorized.request.requestId,
|
||||
auditEventId: createEventId(),
|
||||
principal: authorized.principal,
|
||||
policyFence: authorized.policyFence,
|
||||
observedAtMs,
|
||||
});
|
||||
return result.found
|
||||
? response(200, { ...result })
|
||||
: response(404, { code: 'workflow_run_not_found' });
|
||||
} catch (error) {
|
||||
return runEventListErrorResponse(error);
|
||||
}
|
||||
},
|
||||
}),
|
||||
Object.freeze({
|
||||
...common,
|
||||
method: 'POST' as const,
|
||||
path: '/api/v3/projects/{projectId}/packages/{packageName}/workflows/{workflowId}/runs',
|
||||
operationId: 'workflow.start',
|
||||
permission: 'run.start',
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
let body;
|
||||
try {
|
||||
body = parseBody(authorized.request.body);
|
||||
} catch {
|
||||
return response(400, { code: 'invalid_workflow_start_request' });
|
||||
}
|
||||
const plannedAtMs = now();
|
||||
if (
|
||||
authorized.projectId === null ||
|
||||
typeof parameters.packageName !== 'string' ||
|
||||
!PACKAGE_NAME.test(parameters.packageName) ||
|
||||
typeof parameters.workflowId !== 'string' ||
|
||||
!RESOURCE_ID.test(parameters.workflowId) ||
|
||||
!authorized.policyFence ||
|
||||
authorized.policyFence.bindingVersion === null ||
|
||||
!Number.isSafeInteger(plannedAtMs) ||
|
||||
plannedAtMs < 0
|
||||
) {
|
||||
return response(503, { code: 'workflow_administration_unavailable' });
|
||||
}
|
||||
try {
|
||||
const result = await capability.start({
|
||||
projectId: authorized.projectId,
|
||||
packageName: parameters.packageName,
|
||||
workflowId: parameters.workflowId,
|
||||
planId: body.planId,
|
||||
runId: body.runId,
|
||||
stepRunIds: body.stepRunIds,
|
||||
principal: authorized.principal,
|
||||
policyFence: authorized.policyFence,
|
||||
plannedAtMs,
|
||||
});
|
||||
return response(result.status === 'created' ? 201 : 200, {
|
||||
schema: CLUSTER_PLUGIN_PACKAGE_WORKFLOW_START_RESPONSE_SCHEMA,
|
||||
status: result.status,
|
||||
replayed: result.status === 'existing',
|
||||
planId: result.plan.planId,
|
||||
runId: result.plan.runId,
|
||||
receiptDigest: result.receipt.receiptDigest,
|
||||
});
|
||||
} catch (error) {
|
||||
return errorResponse(error);
|
||||
}
|
||||
},
|
||||
}),
|
||||
Object.freeze({
|
||||
...common,
|
||||
method: 'POST' as const,
|
||||
path: '/api/v3/projects/{projectId}/packages/{packageName}/workflows/{workflowId}/runs/{runId}/cancellation',
|
||||
operationId: 'workflow.cancel',
|
||||
permission: 'run.stop',
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
let body;
|
||||
try {
|
||||
body = parseClusterRunCancellationRequestBody(
|
||||
authorized.request.body,
|
||||
);
|
||||
} catch {
|
||||
return response(400, {
|
||||
code: 'invalid_workflow_cancellation_request',
|
||||
schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
|
||||
});
|
||||
}
|
||||
if (
|
||||
authorized.projectId === null ||
|
||||
typeof parameters.packageName !== 'string' ||
|
||||
!PACKAGE_NAME.test(parameters.packageName) ||
|
||||
typeof parameters.workflowId !== 'string' ||
|
||||
!RESOURCE_ID.test(parameters.workflowId) ||
|
||||
typeof parameters.runId !== 'string' ||
|
||||
!UUID_V4.test(parameters.runId) ||
|
||||
!authorized.policyFence ||
|
||||
authorized.policyFence.bindingVersion === null
|
||||
) {
|
||||
return response(503, {
|
||||
code: 'workflow_administration_unavailable',
|
||||
});
|
||||
}
|
||||
try {
|
||||
const result = await capability.cancel({
|
||||
projectId: authorized.projectId,
|
||||
packageName: parameters.packageName,
|
||||
workflowId: parameters.workflowId,
|
||||
runId: parameters.runId,
|
||||
mutationId: body.mutationId,
|
||||
eventId: createEventId(),
|
||||
principal: authorized.principal,
|
||||
policyFence: authorized.policyFence,
|
||||
});
|
||||
return response(
|
||||
result.status === 'accepted' ? 202 : 200,
|
||||
createClusterRunCancellationResponseBody(result),
|
||||
);
|
||||
} catch (error) {
|
||||
return errorResponse(error);
|
||||
}
|
||||
},
|
||||
}),
|
||||
]);
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
import type {
|
||||
DeploymentProfile,
|
||||
OpenPostgresDatabase,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
createPostgresDatabaseOpener,
|
||||
isPostgresTlsDnsServername,
|
||||
loadPostgresConnectionEnvironment,
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
type PostgresConnectionOptions,
|
||||
type PostgresPoolOptions,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
import { ClusterControlAvailabilityFence } from '../database/availability';
|
||||
import type { ClusterControlHttpSurfaceOptions } from '../transport/httpSurface';
|
||||
|
||||
export type ClusterControlEnvironment = Readonly<
|
||||
Record<string, string | undefined>
|
||||
>;
|
||||
|
||||
export interface DisabledClusterControlConfig {
|
||||
readonly enabled: false;
|
||||
readonly profile: DeploymentProfile;
|
||||
}
|
||||
|
||||
export interface EnabledClusterControlConfig {
|
||||
readonly enabled: true;
|
||||
readonly profile: 'cluster-control';
|
||||
readonly http: ClusterControlHttpSurfaceOptions;
|
||||
readonly database: Readonly<{
|
||||
connection: PostgresConnectionOptions;
|
||||
pool: PostgresPoolOptions;
|
||||
}>;
|
||||
readonly security: Readonly<{
|
||||
apiCredentialPepper: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export type ClusterControlConfig =
|
||||
| DisabledClusterControlConfig
|
||||
| EnabledClusterControlConfig;
|
||||
|
||||
export interface ClusterControlDatabaseBinding {
|
||||
readonly availability: ClusterControlAvailabilityFence;
|
||||
readonly openDatabase: OpenPostgresDatabase;
|
||||
}
|
||||
|
||||
export class ClusterControlConfigError extends TypeError {
|
||||
constructor(message: string) {
|
||||
super(`Cluster-control configuration is invalid: ${message}`);
|
||||
this.name = 'ClusterControlConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
const PROFILES = new Set<DeploymentProfile>([
|
||||
'edge',
|
||||
'standalone',
|
||||
'cluster-control',
|
||||
'worker',
|
||||
]);
|
||||
|
||||
function booleanValue(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
defaultValue: boolean,
|
||||
): boolean {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return defaultValue;
|
||||
if (value === 'true') return true;
|
||||
if (value === 'false') return false;
|
||||
throw new ClusterControlConfigError(`${name} must be true or false`);
|
||||
}
|
||||
|
||||
function integerValue(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
defaultValue: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return defaultValue;
|
||||
if (!/^\d+$/.test(value)) {
|
||||
throw new ClusterControlConfigError(`${name} must be an integer`);
|
||||
}
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
|
||||
throw new ClusterControlConfigError(
|
||||
`${name} must be between ${minimum} and ${maximum}`,
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function boundedValue(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
maximumLength: number,
|
||||
required = false,
|
||||
): string | undefined {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') {
|
||||
if (required) throw new ClusterControlConfigError(`${name} is required`);
|
||||
return undefined;
|
||||
}
|
||||
if (value.length > maximumLength || /[\0\r\n]/.test(value)) {
|
||||
throw new ClusterControlConfigError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function deploymentProfile(
|
||||
environment: ClusterControlEnvironment,
|
||||
): DeploymentProfile {
|
||||
const value = environment.QL_DEPLOYMENT_PROFILE ?? 'standalone';
|
||||
if (!PROFILES.has(value as DeploymentProfile)) {
|
||||
throw new ClusterControlConfigError('QL_DEPLOYMENT_PROFILE is invalid');
|
||||
}
|
||||
return value as DeploymentProfile;
|
||||
}
|
||||
|
||||
function runtimeConnection(
|
||||
environment: ClusterControlEnvironment,
|
||||
): PostgresConnectionOptions {
|
||||
let connection: PostgresConnectionOptions;
|
||||
try {
|
||||
connection = loadPostgresConnectionEnvironment(environment, {
|
||||
connectionString: 'QL3_POSTGRES_RUNTIME_URL',
|
||||
host: 'QL3_POSTGRES_RUNTIME_HOST',
|
||||
port: 'QL3_POSTGRES_RUNTIME_PORT',
|
||||
database: 'QL3_POSTGRES_RUNTIME_DATABASE',
|
||||
user: 'QL3_POSTGRES_RUNTIME_USER',
|
||||
password: 'QL3_POSTGRES_RUNTIME_PASSWORD',
|
||||
});
|
||||
} catch (error) {
|
||||
throw new ClusterControlConfigError(
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: 'PostgreSQL runtime connection is invalid',
|
||||
);
|
||||
}
|
||||
|
||||
const mode = environment.QL3_POSTGRES_TLS_MODE ?? 'verify-full';
|
||||
if (mode !== 'verify-full' && mode !== 'disable') {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_POSTGRES_TLS_MODE must be verify-full or disable',
|
||||
);
|
||||
}
|
||||
if (
|
||||
mode === 'disable' &&
|
||||
!booleanValue(environment, 'QL3_POSTGRES_ALLOW_INSECURE', false)
|
||||
) {
|
||||
throw new ClusterControlConfigError(
|
||||
'disabling PostgreSQL TLS requires QL3_POSTGRES_ALLOW_INSECURE=true',
|
||||
);
|
||||
}
|
||||
const servername = boundedValue(
|
||||
environment,
|
||||
'QL3_POSTGRES_TLS_SERVERNAME',
|
||||
253,
|
||||
);
|
||||
if (mode === 'verify-full' && !isPostgresTlsDnsServername(servername)) {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_POSTGRES_TLS_SERVERNAME must be an explicit DNS name for verify-full',
|
||||
);
|
||||
}
|
||||
const certificateAuthorityFile = boundedValue(
|
||||
environment,
|
||||
'QL3_POSTGRES_TLS_CA_FILE',
|
||||
4096,
|
||||
);
|
||||
if (mode === 'disable' && certificateAuthorityFile !== undefined) {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_POSTGRES_TLS_CA_FILE cannot be used when TLS is disabled',
|
||||
);
|
||||
}
|
||||
let certificateAuthority: string | undefined;
|
||||
if (certificateAuthorityFile !== undefined) {
|
||||
try {
|
||||
certificateAuthority = loadPostgresCertificateAuthorityFile(
|
||||
certificateAuthorityFile,
|
||||
);
|
||||
} catch {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_POSTGRES_TLS_CA_FILE must contain a bounded trusted CA bundle',
|
||||
);
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
...connection,
|
||||
tls:
|
||||
mode === 'disable'
|
||||
? Object.freeze({ mode: 'disable' as const })
|
||||
: Object.freeze({
|
||||
mode: 'verify-full' as const,
|
||||
...(certificateAuthority === undefined
|
||||
? {}
|
||||
: { ca: certificateAuthority }),
|
||||
servername: servername!,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function apiCredentialPepper(environment: ClusterControlEnvironment): string {
|
||||
const value = boundedValue(
|
||||
environment,
|
||||
'QL3_API_CREDENTIAL_PEPPER',
|
||||
64,
|
||||
true,
|
||||
)!;
|
||||
if (!/^[A-Za-z0-9_-]{43}$/.test(value)) {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_API_CREDENTIAL_PEPPER must be canonical base64url for 32 bytes',
|
||||
);
|
||||
}
|
||||
const decoded = Buffer.from(value, 'base64url');
|
||||
if (decoded.byteLength !== 32 || decoded.toString('base64url') !== value) {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_API_CREDENTIAL_PEPPER must be canonical base64url for 32 bytes',
|
||||
);
|
||||
}
|
||||
decoded.fill(0);
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the profile gate before reading PostgreSQL configuration. A disabled
|
||||
* cluster-control therefore does not touch its runtime credential source.
|
||||
*/
|
||||
export function loadClusterControlConfig(
|
||||
environment: ClusterControlEnvironment,
|
||||
): ClusterControlConfig {
|
||||
if (
|
||||
!environment ||
|
||||
typeof environment !== 'object' ||
|
||||
Array.isArray(environment)
|
||||
) {
|
||||
throw new ClusterControlConfigError('environment must be an object');
|
||||
}
|
||||
const profile = deploymentProfile(environment);
|
||||
const enabled = booleanValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_CONTROL_ENABLED',
|
||||
false,
|
||||
);
|
||||
if (!enabled) return Object.freeze({ enabled: false, profile });
|
||||
if (profile !== 'cluster-control') {
|
||||
throw new ClusterControlConfigError(
|
||||
'enabled runtime requires QL_DEPLOYMENT_PROFILE=cluster-control',
|
||||
);
|
||||
}
|
||||
|
||||
const applicationName =
|
||||
boundedValue(environment, 'QL3_POSTGRES_APPLICATION_NAME', 63) ??
|
||||
'qinglong-cluster-runtime';
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,62}$/.test(applicationName)) {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_POSTGRES_APPLICATION_NAME is invalid',
|
||||
);
|
||||
}
|
||||
const host =
|
||||
boundedValue(environment, 'QL3_CLUSTER_HTTP_HOST', 253) ?? '0.0.0.0';
|
||||
const config: EnabledClusterControlConfig = {
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
http: Object.freeze({
|
||||
host,
|
||||
port: integerValue(environment, 'QL3_CLUSTER_HTTP_PORT', 5800, 1, 65_535),
|
||||
maxBodyBytes: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_HTTP_MAX_BODY_BYTES',
|
||||
1024 * 1024,
|
||||
1024,
|
||||
4 * 1024 * 1024,
|
||||
),
|
||||
maxInFlightRequests: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_HTTP_MAX_IN_FLIGHT',
|
||||
64,
|
||||
1,
|
||||
1024,
|
||||
),
|
||||
authenticationRateWindowMs: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_AUTH_RATE_WINDOW_MS',
|
||||
60_000,
|
||||
1_000,
|
||||
60 * 60_000,
|
||||
),
|
||||
authenticationRatePerPeer: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_AUTH_RATE_PER_PEER',
|
||||
300,
|
||||
1,
|
||||
1_000_000,
|
||||
),
|
||||
authenticationRateGlobal: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_AUTH_RATE_GLOBAL',
|
||||
1_200,
|
||||
1,
|
||||
1_000_000,
|
||||
),
|
||||
authenticationRateMaxPeers: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_AUTH_RATE_MAX_PEERS',
|
||||
4_096,
|
||||
1,
|
||||
65_536,
|
||||
),
|
||||
requestTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_HTTP_REQUEST_TIMEOUT_MS',
|
||||
15_000,
|
||||
100,
|
||||
120_000,
|
||||
),
|
||||
drainTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_HTTP_DRAIN_TIMEOUT_MS',
|
||||
10_000,
|
||||
100,
|
||||
120_000,
|
||||
),
|
||||
}),
|
||||
database: Object.freeze({
|
||||
connection: runtimeConnection(environment),
|
||||
pool: Object.freeze({
|
||||
applicationName,
|
||||
maxConnections: integerValue(
|
||||
environment,
|
||||
'QL3_POSTGRES_MAX_CONNECTIONS',
|
||||
8,
|
||||
1,
|
||||
64,
|
||||
),
|
||||
connectionTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_POSTGRES_CONNECTION_TIMEOUT_MS',
|
||||
5_000,
|
||||
100,
|
||||
60_000,
|
||||
),
|
||||
}),
|
||||
}),
|
||||
security: Object.freeze({
|
||||
apiCredentialPepper: apiCredentialPepper(environment),
|
||||
}),
|
||||
};
|
||||
return Object.freeze(config);
|
||||
}
|
||||
|
||||
export function createClusterControlDatabaseBinding(
|
||||
config: EnabledClusterControlConfig,
|
||||
): ClusterControlDatabaseBinding {
|
||||
if (!config?.enabled || config.profile !== 'cluster-control') {
|
||||
throw new ClusterControlConfigError(
|
||||
'database binding requires an enabled cluster-control config',
|
||||
);
|
||||
}
|
||||
const availability = new ClusterControlAvailabilityFence();
|
||||
const openDatabase = createPostgresDatabaseOpener({
|
||||
role: 'runtime',
|
||||
connection: config.database.connection,
|
||||
pool: config.database.pool,
|
||||
onPoolError(error) {
|
||||
// pg emits idle-client errors outside a request Promise. They are an
|
||||
// availability signal, never a callback exception or transaction retry.
|
||||
void availability.signal(error).catch(() => undefined);
|
||||
},
|
||||
});
|
||||
return Object.freeze({ availability, openDatabase });
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
import type {
|
||||
ClusterControlActivationAudit,
|
||||
ClusterControlStopResult,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
loadClusterControlConfig,
|
||||
type ClusterControlEnvironment,
|
||||
type EnabledClusterControlConfig,
|
||||
} from './config';
|
||||
import {
|
||||
startProductionClusterControlApplication,
|
||||
type ProductionClusterControlApplicationOptions,
|
||||
} from '../application-runtime/productionApplication';
|
||||
import {
|
||||
ClusterControlDatabaseUnavailableError,
|
||||
type ClusterControlApplicationResult,
|
||||
} from '../application-runtime/application';
|
||||
import {
|
||||
loadClusterWorkerIngressConfig,
|
||||
type EnabledClusterWorkerIngressConfig,
|
||||
} from '../worker-ingress/workerIngressConfig';
|
||||
import type { ClusterWorkerArtifactBinding } from '../artifact/workerArtifactBinding';
|
||||
import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
|
||||
export type ClusterControlProcessSignal = 'SIGINT' | 'SIGTERM';
|
||||
|
||||
export interface ClusterControlProcessEvent {
|
||||
readonly schemaVersion: 1;
|
||||
readonly component: 'qinglong3-cluster-control';
|
||||
readonly level: 'info' | 'error';
|
||||
readonly event: string;
|
||||
readonly replicaId: string;
|
||||
readonly signal?: ClusterControlProcessSignal;
|
||||
readonly stopResult?: ClusterControlStopResult;
|
||||
readonly address?: Readonly<{ host: string; port: number }>;
|
||||
readonly activation?: ClusterControlActivationAudit;
|
||||
readonly diagnostic?: Readonly<{
|
||||
scope:
|
||||
| 'scheduler'
|
||||
| 'cancellation-convergence'
|
||||
| 'database'
|
||||
| 'worker-ingress';
|
||||
name: string;
|
||||
code?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface ClusterControlProcessSignalSource {
|
||||
subscribe(
|
||||
listener: (signal: ClusterControlProcessSignal) => void,
|
||||
): () => void;
|
||||
}
|
||||
|
||||
export type ProductionClusterControlStarter = (
|
||||
options: ProductionClusterControlApplicationOptions,
|
||||
) => Promise<ClusterControlApplicationResult>;
|
||||
|
||||
export type ClusterWorkerArtifactBindingFactory = (
|
||||
config: EnabledClusterWorkerIngressConfig['artifact'],
|
||||
) => Promise<Readonly<ClusterWorkerArtifactBinding>>;
|
||||
|
||||
export type ClusterWorkerSecretProviderFactory = (
|
||||
config: NonNullable<EnabledClusterWorkerIngressConfig['secret']>,
|
||||
) => Promise<Readonly<RemoteWorkerSecretValueProvider>>;
|
||||
|
||||
export interface ProductionClusterControlProcessOptions {
|
||||
readonly environment: ClusterControlEnvironment;
|
||||
readonly signals: ClusterControlProcessSignalSource;
|
||||
readonly emit: (event: ClusterControlProcessEvent) => void | Promise<void>;
|
||||
readonly start?: ProductionClusterControlStarter;
|
||||
readonly createWorkerArtifactBinding?: ClusterWorkerArtifactBindingFactory;
|
||||
readonly createWorkerSecretProvider?: ClusterWorkerSecretProviderFactory;
|
||||
readonly workerSecretProvider?: RemoteWorkerSecretValueProvider;
|
||||
}
|
||||
|
||||
export class ClusterControlProcessError extends Error {
|
||||
readonly code:
|
||||
| 'QL3_CLUSTER_CONTROL_PROCESS_CONFIG_INVALID'
|
||||
| 'QL3_CLUSTER_CONTROL_PROCESS_DISABLED';
|
||||
|
||||
constructor(
|
||||
code: ClusterControlProcessError['code'],
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ClusterControlProcessError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
const REPLICA_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
|
||||
function processConfiguration(environment: ClusterControlEnvironment): {
|
||||
readonly config: EnabledClusterControlConfig;
|
||||
readonly workerIngress?: EnabledClusterWorkerIngressConfig;
|
||||
readonly replicaId: string;
|
||||
} {
|
||||
const config = loadClusterControlConfig(environment);
|
||||
if (!config.enabled) {
|
||||
throw new ClusterControlProcessError(
|
||||
'QL3_CLUSTER_CONTROL_PROCESS_DISABLED',
|
||||
'The cluster-control process requires an enabled cluster-control profile',
|
||||
);
|
||||
}
|
||||
const replicaId = environment.QL3_CLUSTER_REPLICA_ID;
|
||||
if (
|
||||
typeof replicaId !== 'string' ||
|
||||
!REPLICA_ID_PATTERN.test(replicaId)
|
||||
) {
|
||||
throw new ClusterControlProcessError(
|
||||
'QL3_CLUSTER_CONTROL_PROCESS_CONFIG_INVALID',
|
||||
'QL3_CLUSTER_REPLICA_ID must be a stable safe identifier',
|
||||
);
|
||||
}
|
||||
const workerIngress = loadClusterWorkerIngressConfig(environment);
|
||||
return Object.freeze({
|
||||
config,
|
||||
replicaId,
|
||||
...(workerIngress.enabled ? { workerIngress } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function createWorkerArtifactBinding(
|
||||
config: EnabledClusterWorkerIngressConfig['artifact'],
|
||||
): Promise<Readonly<ClusterWorkerArtifactBinding>> {
|
||||
const binding = await import('../artifact/workerArtifactBinding.js');
|
||||
return binding.createClusterWorkerArtifactBinding(config);
|
||||
}
|
||||
|
||||
async function createWorkerSecretProvider(
|
||||
config: NonNullable<EnabledClusterWorkerIngressConfig['secret']>,
|
||||
): Promise<Readonly<RemoteWorkerSecretValueProvider>> {
|
||||
if (config.provider !== 'mounted-files') {
|
||||
throw new TypeError('Cluster Worker Secret provider is unsupported');
|
||||
}
|
||||
const provider = await import('../remote-execution/mountedSecretProvider.js');
|
||||
return provider.createClusterMountedSecretProvider({
|
||||
rootDirectory: config.rootDirectory,
|
||||
});
|
||||
}
|
||||
|
||||
function diagnosticFact(
|
||||
scope: ClusterControlProcessEvent['diagnostic'] extends infer T
|
||||
? T extends { readonly scope: infer TScope }
|
||||
? TScope
|
||||
: never
|
||||
: never,
|
||||
error: unknown,
|
||||
): NonNullable<ClusterControlProcessEvent['diagnostic']> {
|
||||
const candidate = error as {
|
||||
readonly name?: unknown;
|
||||
readonly code?: unknown;
|
||||
};
|
||||
return Object.freeze({
|
||||
scope,
|
||||
name:
|
||||
typeof candidate?.name === 'string' && candidate.name.length <= 128
|
||||
? candidate.name
|
||||
: 'Error',
|
||||
...(typeof candidate?.code === 'string' && candidate.code.length <= 128
|
||||
? { code: candidate.code }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
function event(
|
||||
replicaId: string,
|
||||
values: Omit<
|
||||
ClusterControlProcessEvent,
|
||||
'schemaVersion' | 'component' | 'replicaId'
|
||||
>,
|
||||
): ClusterControlProcessEvent {
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-cluster-control',
|
||||
replicaId,
|
||||
...values,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns exactly one production cluster-control process. It installs signal
|
||||
* handling before startup, derives every lease owner from the stable replica
|
||||
* identity, and withdraws admission through the application stop contract.
|
||||
*/
|
||||
export async function runProductionClusterControlProcess(
|
||||
options: ProductionClusterControlProcessOptions,
|
||||
): Promise<ClusterControlStopResult> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
typeof options.emit !== 'function' ||
|
||||
typeof options.signals?.subscribe !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster-control process options are invalid');
|
||||
}
|
||||
const { config, replicaId, workerIngress } = processConfiguration(
|
||||
options.environment,
|
||||
);
|
||||
const start = options.start ?? startProductionClusterControlApplication;
|
||||
if (typeof start !== 'function') {
|
||||
throw new TypeError('Cluster-control process starter is invalid');
|
||||
}
|
||||
|
||||
let resolveSignal:
|
||||
| ((signal: ClusterControlProcessSignal) => void)
|
||||
| undefined;
|
||||
const requestedSignal = new Promise<ClusterControlProcessSignal>((resolve) => {
|
||||
resolveSignal = resolve;
|
||||
});
|
||||
let acceptedSignal = false;
|
||||
const unsubscribe = options.signals.subscribe((signal) => {
|
||||
if (acceptedSignal) return;
|
||||
acceptedSignal = true;
|
||||
resolveSignal?.(signal);
|
||||
});
|
||||
|
||||
let artifactBinding: Readonly<ClusterWorkerArtifactBinding> | undefined;
|
||||
let workerSecretProvider = options.workerSecretProvider;
|
||||
let application: ClusterControlApplicationResult | undefined;
|
||||
let applicationStopStarted = false;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
if (workerIngress) {
|
||||
const createBinding =
|
||||
options.createWorkerArtifactBinding ?? createWorkerArtifactBinding;
|
||||
if (typeof createBinding !== 'function') {
|
||||
throw new TypeError(
|
||||
'Cluster Worker Artifact binding factory is invalid',
|
||||
);
|
||||
}
|
||||
artifactBinding = await createBinding(workerIngress.artifact);
|
||||
if (
|
||||
workerIngress.secret !== undefined &&
|
||||
workerSecretProvider === undefined
|
||||
) {
|
||||
const createProvider =
|
||||
options.createWorkerSecretProvider ?? createWorkerSecretProvider;
|
||||
if (typeof createProvider !== 'function') {
|
||||
throw new TypeError(
|
||||
'Cluster Worker Secret provider factory is invalid',
|
||||
);
|
||||
}
|
||||
workerSecretProvider = await createProvider(workerIngress.secret);
|
||||
}
|
||||
if (
|
||||
workerSecretProvider !== undefined &&
|
||||
typeof workerSecretProvider.resolve !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster Worker Secret provider is invalid');
|
||||
}
|
||||
}
|
||||
application = await start({
|
||||
config,
|
||||
recovery: { ownerId: replicaId },
|
||||
scheduler: {
|
||||
ownerId: replicaId,
|
||||
onDiagnostic(error) {
|
||||
void Promise.resolve(
|
||||
options.emit(
|
||||
event(replicaId, {
|
||||
level: 'error',
|
||||
event: 'runtime_diagnostic',
|
||||
diagnostic: diagnosticFact('scheduler', error),
|
||||
}),
|
||||
),
|
||||
).catch(() => undefined);
|
||||
},
|
||||
},
|
||||
cancellationConvergence: {
|
||||
onDiagnostic(error) {
|
||||
void Promise.resolve(
|
||||
options.emit(
|
||||
event(replicaId, {
|
||||
level: 'error',
|
||||
event: 'runtime_diagnostic',
|
||||
diagnostic: diagnosticFact(
|
||||
'cancellation-convergence',
|
||||
error,
|
||||
),
|
||||
}),
|
||||
),
|
||||
).catch(() => undefined);
|
||||
},
|
||||
},
|
||||
...(workerIngress === undefined
|
||||
? {}
|
||||
: {
|
||||
workerIngress: {
|
||||
config: workerIngress,
|
||||
artifactStore: artifactBinding!.store,
|
||||
...(workerSecretProvider === undefined
|
||||
? {}
|
||||
: { secretProvider: workerSecretProvider }),
|
||||
onDiagnostic(error: unknown) {
|
||||
void Promise.resolve(
|
||||
options.emit(
|
||||
event(replicaId, {
|
||||
level: 'error',
|
||||
event: 'runtime_diagnostic',
|
||||
diagnostic: diagnosticFact(
|
||||
'worker-ingress',
|
||||
error,
|
||||
),
|
||||
}),
|
||||
),
|
||||
).catch(() => undefined);
|
||||
},
|
||||
},
|
||||
}),
|
||||
audit(record) {
|
||||
return options.emit(
|
||||
event(replicaId, {
|
||||
level: record.state === 'failed' ? 'error' : 'info',
|
||||
event: 'activation',
|
||||
activation: Object.freeze({ ...record }),
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
if (application.status !== 'active') {
|
||||
throw new ClusterControlProcessError(
|
||||
'QL3_CLUSTER_CONTROL_PROCESS_DISABLED',
|
||||
'The cluster-control process did not activate',
|
||||
);
|
||||
}
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: 'info',
|
||||
event: 'listening',
|
||||
address: application.address,
|
||||
}),
|
||||
);
|
||||
if (workerIngress) {
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: 'info',
|
||||
event: 'worker_ingress_listening',
|
||||
address: Object.freeze({
|
||||
host: workerIngress.http.host ?? '0.0.0.0',
|
||||
port: workerIngress.http.port ?? 5801,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
const termination = await Promise.race([
|
||||
requestedSignal.then((signal) =>
|
||||
Object.freeze({ kind: 'signal' as const, signal }),
|
||||
),
|
||||
application.unavailable.then((error) =>
|
||||
Object.freeze({ kind: 'database-unavailable' as const, error }),
|
||||
),
|
||||
]);
|
||||
if (termination.kind === 'database-unavailable') {
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: 'error',
|
||||
event: 'database_unavailable',
|
||||
diagnostic: diagnosticFact('database', termination.error),
|
||||
}),
|
||||
);
|
||||
applicationStopStarted = true;
|
||||
const stopResult = await application.stop();
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: stopResult === 'stopped' ? 'info' : 'error',
|
||||
event: 'stopped',
|
||||
stopResult,
|
||||
}),
|
||||
);
|
||||
throw new ClusterControlDatabaseUnavailableError();
|
||||
}
|
||||
const signal = termination.signal;
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: 'info',
|
||||
event: 'shutdown_requested',
|
||||
signal,
|
||||
}),
|
||||
);
|
||||
applicationStopStarted = true;
|
||||
const stopResult = await application.stop();
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: stopResult === 'stopped' ? 'info' : 'error',
|
||||
event: 'stopped',
|
||||
stopResult,
|
||||
}),
|
||||
);
|
||||
return stopResult;
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
throw error;
|
||||
} finally {
|
||||
unsubscribe();
|
||||
resolveSignal = undefined;
|
||||
let cleanupError: unknown;
|
||||
if (
|
||||
application?.status === 'active' &&
|
||||
!applicationStopStarted
|
||||
) {
|
||||
try {
|
||||
applicationStopStarted = true;
|
||||
await application.stop();
|
||||
} catch (error) {
|
||||
cleanupError = error;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await artifactBinding?.close();
|
||||
} catch (error) {
|
||||
cleanupError ??= error;
|
||||
}
|
||||
if (cleanupError && primaryError === undefined) throw cleanupError;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
// Remote Execution owns mounted Secret resolution for authenticated delivery.
|
||||
import { createHash } from 'node:crypto';
|
||||
import { constants } from 'node:fs';
|
||||
import {
|
||||
lstat,
|
||||
open,
|
||||
realpath,
|
||||
} from 'node:fs/promises';
|
||||
import {
|
||||
isAbsolute,
|
||||
join,
|
||||
normalize,
|
||||
parse,
|
||||
relative,
|
||||
} from 'node:path';
|
||||
import {
|
||||
MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES,
|
||||
MAX_REMOTE_SECRET_VALUE_BYTES,
|
||||
normalizeRemoteWorkerSecretDeliveryAuthority,
|
||||
type RemoteWorkerSecretDeliveryAuthority,
|
||||
type RemoteWorkerSecretResolution,
|
||||
type RemoteWorkerSecretValueProvider,
|
||||
} from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import { parseSecretRef } from '@qinglong/runtime-core/secret-reference';
|
||||
|
||||
const MAX_SECRET_ROOT_BYTES = 4096;
|
||||
const SECRET_FILE_NAME = /^[0-9a-f]{64}$/;
|
||||
|
||||
export interface ClusterMountedSecretProviderOptions {
|
||||
/**
|
||||
* Read-only directory whose file names are SHA-256(canonical SecretRef).
|
||||
* Kubernetes projected-volume symlinks are accepted only when their resolved
|
||||
* regular file remains below this directory.
|
||||
*/
|
||||
readonly rootDirectory: string;
|
||||
}
|
||||
|
||||
export class ClusterMountedSecretProviderError extends Error {
|
||||
readonly code = 'QL3_CLUSTER_MOUNTED_SECRET_UNAVAILABLE';
|
||||
|
||||
constructor(
|
||||
readonly reason:
|
||||
| 'invalid_configuration'
|
||||
| 'root_unavailable'
|
||||
| 'material_unavailable',
|
||||
options?: ErrorOptions,
|
||||
) {
|
||||
super(`Cluster mounted Secret provider failed: ${reason}`, options);
|
||||
this.name = 'ClusterMountedSecretProviderError';
|
||||
}
|
||||
}
|
||||
|
||||
function rootDirectory(value: string): string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
!isAbsolute(value) ||
|
||||
parse(value).root === value ||
|
||||
normalize(value) !== value ||
|
||||
value.includes('\0') ||
|
||||
Buffer.byteLength(value, 'utf8') > MAX_SECRET_ROOT_BYTES
|
||||
) {
|
||||
throw new ClusterMountedSecretProviderError('invalid_configuration');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Kubernetes Secret keys cannot contain a SecretRef directly. This stable,
|
||||
* non-reversible name also prevents Project/name input from becoming a path.
|
||||
*/
|
||||
export function clusterMountedSecretFileName(secretRef: string): string {
|
||||
let canonical: string;
|
||||
try {
|
||||
const parsed = parseSecretRef(secretRef);
|
||||
canonical = secretRef;
|
||||
if (
|
||||
parsed.projectId.length < 1 ||
|
||||
parsed.name.length < 1
|
||||
) throw new Error('invalid SecretRef');
|
||||
} catch (error) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'invalid_configuration',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
const name = createHash('sha256').update(canonical, 'utf8').digest('hex');
|
||||
if (!SECRET_FILE_NAME.test(name)) {
|
||||
throw new ClusterMountedSecretProviderError('invalid_configuration');
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
function remainsBelow(root: string, candidate: string): boolean {
|
||||
const suffix = relative(root, candidate);
|
||||
return (
|
||||
suffix.length > 0 &&
|
||||
!isAbsolute(suffix) &&
|
||||
suffix !== '..' &&
|
||||
!suffix.startsWith(`..${process.platform === 'win32' ? '\\' : '/'}`)
|
||||
);
|
||||
}
|
||||
|
||||
async function resolvedRoot(path: string): Promise<string> {
|
||||
try {
|
||||
const configured = await lstat(path);
|
||||
if (!configured.isDirectory() || configured.isSymbolicLink()) {
|
||||
throw new Error('root is not a direct directory');
|
||||
}
|
||||
return await realpath(path);
|
||||
} catch (error) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'root_unavailable',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function readMaterial(
|
||||
root: string,
|
||||
secretRef: string,
|
||||
): Promise<Buffer> {
|
||||
const candidate = join(root, clusterMountedSecretFileName(secretRef));
|
||||
let handle;
|
||||
try {
|
||||
const target = await realpath(candidate);
|
||||
if (!remainsBelow(root, target)) {
|
||||
throw new Error('material escaped its root');
|
||||
}
|
||||
handle = await open(
|
||||
target,
|
||||
constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0),
|
||||
);
|
||||
const stat = await handle.stat();
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.nlink !== 1 ||
|
||||
stat.size < 0 ||
|
||||
stat.size > MAX_REMOTE_SECRET_VALUE_BYTES ||
|
||||
(stat.mode & 0o111) !== 0 ||
|
||||
(stat.mode & 0o027) !== 0
|
||||
) {
|
||||
throw new Error('material metadata is unsafe');
|
||||
}
|
||||
const bytes = await handle.readFile();
|
||||
if (
|
||||
bytes.byteLength !== stat.size ||
|
||||
bytes.byteLength > MAX_REMOTE_SECRET_VALUE_BYTES ||
|
||||
(await realpath(candidate)) !== target
|
||||
) {
|
||||
bytes.fill(0);
|
||||
throw new Error('material changed while reading');
|
||||
}
|
||||
return bytes;
|
||||
} catch (error) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'material_unavailable',
|
||||
{ cause: error },
|
||||
);
|
||||
} finally {
|
||||
await handle?.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
function secretValue(bytes: Buffer): string {
|
||||
try {
|
||||
const value = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
||||
if (value.includes('\0')) {
|
||||
throw new Error('Secret contains NUL');
|
||||
}
|
||||
return value;
|
||||
} catch (error) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'material_unavailable',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A zero-client, zero-watcher Cluster provider for Kubernetes Secret, CSI or
|
||||
* operator-managed projected files. Every authorized delivery resolves the
|
||||
* active files again, so atomic projection replacement rotates material
|
||||
* without a timer, cache, control restart or Kubernetes API permission.
|
||||
*/
|
||||
export class ClusterMountedSecretProvider
|
||||
implements RemoteWorkerSecretValueProvider
|
||||
{
|
||||
private readonly rootDirectory: string;
|
||||
|
||||
constructor(options: ClusterMountedSecretProviderOptions) {
|
||||
if (!options || typeof options !== 'object' || Array.isArray(options)) {
|
||||
throw new ClusterMountedSecretProviderError('invalid_configuration');
|
||||
}
|
||||
this.rootDirectory = rootDirectory(options.rootDirectory);
|
||||
}
|
||||
|
||||
async verify(): Promise<void> {
|
||||
await resolvedRoot(this.rootDirectory);
|
||||
}
|
||||
|
||||
async resolve(
|
||||
authority: Readonly<RemoteWorkerSecretDeliveryAuthority>,
|
||||
): Promise<Readonly<RemoteWorkerSecretResolution>> {
|
||||
let normalized: Readonly<RemoteWorkerSecretDeliveryAuthority>;
|
||||
try {
|
||||
normalized = normalizeRemoteWorkerSecretDeliveryAuthority(authority);
|
||||
} catch (error) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'material_unavailable',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
const root = await resolvedRoot(this.rootDirectory);
|
||||
const buffers: Buffer[] = [];
|
||||
try {
|
||||
const values = [];
|
||||
let totalBytes = 0;
|
||||
for (const secretRef of normalized.secretRefs) {
|
||||
const bytes = await readMaterial(root, secretRef);
|
||||
buffers.push(bytes);
|
||||
totalBytes += bytes.byteLength;
|
||||
if (totalBytes > MAX_REMOTE_SECRET_DELIVERY_TOTAL_VALUE_BYTES) {
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'material_unavailable',
|
||||
);
|
||||
}
|
||||
values.push(
|
||||
Object.freeze({
|
||||
secretRef,
|
||||
value: secretValue(bytes),
|
||||
}),
|
||||
);
|
||||
}
|
||||
let disposed = false;
|
||||
return Object.freeze({
|
||||
values: Object.freeze(values),
|
||||
dispose() {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
for (const bytes of buffers) bytes.fill(0);
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
for (const bytes of buffers) bytes.fill(0);
|
||||
if (error instanceof ClusterMountedSecretProviderError) throw error;
|
||||
throw new ClusterMountedSecretProviderError(
|
||||
'material_unavailable',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function createClusterMountedSecretProvider(
|
||||
options: ClusterMountedSecretProviderOptions,
|
||||
): Promise<Readonly<ClusterMountedSecretProvider>> {
|
||||
const provider = new ClusterMountedSecretProvider(options);
|
||||
await provider.verify();
|
||||
return provider;
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// Remote execution owns Worker-bound activation acknowledgements and start failure fencing.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type {
|
||||
AcknowledgeRemoteRunRunningCommand,
|
||||
AcknowledgeRemoteRunStartingCommand,
|
||||
FailRemoteRunStartCommand,
|
||||
RemoteRunActivationRepository,
|
||||
RemoteRunActivationResult,
|
||||
} from '@qinglong/runtime-core/remote-activation';
|
||||
|
||||
export interface ClusterRemoteRunActivationPrincipal {
|
||||
readonly workerId: string;
|
||||
}
|
||||
|
||||
type ServerOwnedStartingFields = 'workerId' | 'eventId';
|
||||
type ServerOwnedRunningFields = 'workerId' | 'attemptEventId' | 'runEventId';
|
||||
|
||||
export type AcknowledgeClusterRemoteRunStartingCommand = Omit<
|
||||
AcknowledgeRemoteRunStartingCommand,
|
||||
ServerOwnedStartingFields
|
||||
>;
|
||||
|
||||
export type AcknowledgeClusterRemoteRunRunningCommand = Omit<
|
||||
AcknowledgeRemoteRunRunningCommand,
|
||||
ServerOwnedRunningFields
|
||||
>;
|
||||
|
||||
export type FailClusterRemoteRunStartCommand = Omit<
|
||||
FailRemoteRunStartCommand,
|
||||
ServerOwnedRunningFields
|
||||
>;
|
||||
|
||||
export interface ClusterRemoteRunActivationServiceOptions {
|
||||
readonly createEventId?: () => string;
|
||||
}
|
||||
|
||||
export class ClusterRemoteRunActivationService {
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RemoteRunActivationRepository,
|
||||
options: ClusterRemoteRunActivationServiceOptions = {},
|
||||
) {
|
||||
if (
|
||||
!repository ||
|
||||
typeof repository.acknowledgeStarting !== 'function' ||
|
||||
typeof repository.acknowledgeRunning !== 'function' ||
|
||||
typeof repository.failStart !== 'function'
|
||||
) {
|
||||
throw new TypeError('Remote Run activation repository is invalid');
|
||||
}
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => key !== 'createEventId')
|
||||
) {
|
||||
throw new TypeError('Remote Run activation service options are invalid');
|
||||
}
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
if (typeof this.createEventId !== 'function') {
|
||||
throw new TypeError('Remote Run activation event ID factory is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
acknowledgeStarting(
|
||||
principal: ClusterRemoteRunActivationPrincipal,
|
||||
command: AcknowledgeClusterRemoteRunStartingCommand,
|
||||
): Promise<Readonly<RemoteRunActivationResult>> {
|
||||
this.assertPrincipal(principal);
|
||||
return this.repository.acknowledgeStarting({
|
||||
...command,
|
||||
workerId: principal.workerId,
|
||||
eventId: this.createEventId(),
|
||||
});
|
||||
}
|
||||
|
||||
acknowledgeRunning(
|
||||
principal: ClusterRemoteRunActivationPrincipal,
|
||||
command: AcknowledgeClusterRemoteRunRunningCommand,
|
||||
): Promise<Readonly<RemoteRunActivationResult>> {
|
||||
this.assertPrincipal(principal);
|
||||
return this.repository.acknowledgeRunning({
|
||||
...command,
|
||||
workerId: principal.workerId,
|
||||
attemptEventId: this.createEventId(),
|
||||
runEventId: this.createEventId(),
|
||||
});
|
||||
}
|
||||
|
||||
failStart(
|
||||
principal: ClusterRemoteRunActivationPrincipal,
|
||||
command: FailClusterRemoteRunStartCommand,
|
||||
): Promise<Readonly<RemoteRunActivationResult>> {
|
||||
this.assertPrincipal(principal);
|
||||
return this.repository.failStart({
|
||||
...command,
|
||||
workerId: principal.workerId,
|
||||
attemptEventId: this.createEventId(),
|
||||
runEventId: this.createEventId(),
|
||||
});
|
||||
}
|
||||
|
||||
private assertPrincipal(principal: ClusterRemoteRunActivationPrincipal): void {
|
||||
if (
|
||||
!principal ||
|
||||
typeof principal !== 'object' ||
|
||||
Array.isArray(principal) ||
|
||||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(principal.workerId)
|
||||
) {
|
||||
throw new TypeError('Remote Run activation principal is invalid');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
// Remote execution owns immutable Artifact admission and fenced Worker completion.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
InvalidRemoteWorkerCompletionError,
|
||||
MAX_REMOTE_WORKER_ARTIFACT_HEADER_BYTES,
|
||||
RemoteWorkerCompletionFenceRejectedError,
|
||||
RemoteWorkerCompletionUnavailableError,
|
||||
normalizeRemoteWorkerArtifactReceipt,
|
||||
normalizeRemoteWorkerCompletionCommand,
|
||||
normalizeRemoteWorkerCompletionResult,
|
||||
parseRemoteWorkerArtifactUploadHeader,
|
||||
type RemoteWorkerArtifactReceipt,
|
||||
type RemoteWorkerArtifactUploadAuthorityRepository,
|
||||
type RemoteWorkerArtifactUploadCommand,
|
||||
type RemoteWorkerCompletionCommand,
|
||||
type RemoteWorkerCompletionRepository,
|
||||
type RemoteWorkerCompletionResult,
|
||||
} from '@qinglong/runtime-core/remote-worker-completion';
|
||||
|
||||
export interface ClusterRemoteWorkerArtifactStorageCommand {
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly logArtifactId: string;
|
||||
readonly byteLength: number;
|
||||
readonly truncated?: boolean;
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerArtifactLookup {
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly logArtifactId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Production implementations must be shared by every cluster-control replica
|
||||
* and provide immutable, digest-authenticated put-if-absent semantics.
|
||||
*/
|
||||
export interface ClusterRemoteWorkerArtifactStore {
|
||||
put(
|
||||
command: Readonly<ClusterRemoteWorkerArtifactStorageCommand>,
|
||||
content: AsyncIterable<Uint8Array>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt>>;
|
||||
inspect(
|
||||
lookup: Readonly<ClusterRemoteWorkerArtifactLookup>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt> | undefined>;
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerArtifactUploadInput {
|
||||
readonly workerId: string;
|
||||
readonly workerSessionId: string;
|
||||
readonly contentLength: number;
|
||||
readonly chunks: AsyncIterable<Uint8Array>;
|
||||
readonly signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerCompletionServiceOptions {
|
||||
readonly createEventId?: () => string;
|
||||
}
|
||||
|
||||
class BoundedArtifactStreamReader {
|
||||
private readonly iterator: AsyncIterator<Uint8Array>;
|
||||
private pending: Uint8Array | undefined;
|
||||
private pendingOffset = 0;
|
||||
|
||||
constructor(
|
||||
source: AsyncIterable<Uint8Array>,
|
||||
private readonly signal?: AbortSignal,
|
||||
) {
|
||||
if (!source || typeof source[Symbol.asyncIterator] !== 'function') {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload stream is invalid',
|
||||
);
|
||||
}
|
||||
this.iterator = source[Symbol.asyncIterator]();
|
||||
}
|
||||
|
||||
async readExactly(byteLength: number): Promise<Buffer> {
|
||||
const result = Buffer.allocUnsafe(byteLength);
|
||||
let written = 0;
|
||||
try {
|
||||
while (written < byteLength) {
|
||||
const chunk = await this.nextChunk();
|
||||
if (!chunk) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload stream ended before its header',
|
||||
);
|
||||
}
|
||||
const available = chunk.byteLength - this.pendingOffset;
|
||||
const copied = Math.min(available, byteLength - written);
|
||||
Buffer.from(
|
||||
chunk.buffer,
|
||||
chunk.byteOffset + this.pendingOffset,
|
||||
copied,
|
||||
).copy(result, written);
|
||||
written += copied;
|
||||
this.pendingOffset += copied;
|
||||
if (this.pendingOffset === chunk.byteLength) {
|
||||
this.pending = undefined;
|
||||
this.pendingOffset = 0;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
result.fill(0);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
content(byteLength: number): Readonly<{
|
||||
chunks: AsyncIterable<Uint8Array>;
|
||||
isComplete(): boolean;
|
||||
}> {
|
||||
let complete = false;
|
||||
let started = false;
|
||||
const self = this;
|
||||
const chunks = Object.freeze({
|
||||
async *[Symbol.asyncIterator](): AsyncGenerator<Uint8Array> {
|
||||
if (started) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact content can only be consumed once',
|
||||
);
|
||||
}
|
||||
started = true;
|
||||
let total = 0;
|
||||
while (true) {
|
||||
const chunk = await self.nextChunk();
|
||||
if (!chunk) break;
|
||||
const bytes = chunk.subarray(self.pendingOffset);
|
||||
self.pending = undefined;
|
||||
self.pendingOffset = 0;
|
||||
total += bytes.byteLength;
|
||||
if (total > byteLength) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact content exceeds its declared length',
|
||||
);
|
||||
}
|
||||
yield bytes;
|
||||
}
|
||||
if (total !== byteLength) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact content does not match its declared length',
|
||||
);
|
||||
}
|
||||
complete = true;
|
||||
},
|
||||
});
|
||||
return Object.freeze({ chunks, isComplete: () => complete });
|
||||
}
|
||||
|
||||
private async nextChunk(): Promise<Uint8Array | undefined> {
|
||||
if (this.signal?.aborted) throw this.signal.reason;
|
||||
if (this.pending) return this.pending;
|
||||
const next = await this.iterator.next();
|
||||
if (next.done) return undefined;
|
||||
if (!(next.value instanceof Uint8Array) || next.value.byteLength === 0) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload chunk is invalid',
|
||||
);
|
||||
}
|
||||
this.pending = next.value;
|
||||
this.pendingOffset = 0;
|
||||
return this.pending;
|
||||
}
|
||||
}
|
||||
|
||||
function storageCommand(
|
||||
command: RemoteWorkerArtifactUploadCommand,
|
||||
): Readonly<ClusterRemoteWorkerArtifactStorageCommand> {
|
||||
return Object.freeze({
|
||||
projectId: command.projectId,
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
logArtifactId: command.logArtifactId,
|
||||
byteLength: command.byteLength,
|
||||
...(command.truncated === undefined
|
||||
? {}
|
||||
: { truncated: command.truncated }),
|
||||
});
|
||||
}
|
||||
|
||||
function assertReceiptMatches(
|
||||
command: ClusterRemoteWorkerArtifactStorageCommand,
|
||||
value: RemoteWorkerArtifactReceipt,
|
||||
): Readonly<RemoteWorkerArtifactReceipt> {
|
||||
const receipt = normalizeRemoteWorkerArtifactReceipt(value);
|
||||
if (
|
||||
receipt.projectId !== command.projectId ||
|
||||
receipt.runId !== command.runId ||
|
||||
receipt.attemptId !== command.attemptId ||
|
||||
receipt.logArtifactId !== command.logArtifactId ||
|
||||
receipt.byteLength !== command.byteLength ||
|
||||
receipt.truncated !== command.truncated
|
||||
) {
|
||||
throw new RemoteWorkerCompletionUnavailableError();
|
||||
}
|
||||
return receipt;
|
||||
}
|
||||
|
||||
function eventId(factory: () => string): string {
|
||||
const value = factory();
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > 36 ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) {
|
||||
throw new RemoteWorkerCompletionUnavailableError();
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export class ClusterRemoteWorkerArtifactService {
|
||||
constructor(
|
||||
private readonly authority: RemoteWorkerArtifactUploadAuthorityRepository,
|
||||
private readonly store: ClusterRemoteWorkerArtifactStore,
|
||||
) {
|
||||
if (
|
||||
typeof authority?.authorizeArtifactUpload !== 'function' ||
|
||||
typeof store?.put !== 'function' ||
|
||||
typeof store?.inspect !== 'function'
|
||||
) {
|
||||
throw new TypeError('Remote Worker Artifact service is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async upload(
|
||||
input: ClusterRemoteWorkerArtifactUploadInput,
|
||||
): Promise<Readonly<RemoteWorkerArtifactReceipt>> {
|
||||
if (
|
||||
!input ||
|
||||
!Number.isSafeInteger(input.contentLength) ||
|
||||
input.contentLength < 6
|
||||
) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload envelope is invalid',
|
||||
);
|
||||
}
|
||||
const reader = new BoundedArtifactStreamReader(input.chunks, input.signal);
|
||||
const prefix = await reader.readExactly(4);
|
||||
const headerLength = prefix.readUInt32BE(0);
|
||||
prefix.fill(0);
|
||||
if (
|
||||
headerLength < 2 ||
|
||||
headerLength > MAX_REMOTE_WORKER_ARTIFACT_HEADER_BYTES
|
||||
) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload header length is invalid',
|
||||
);
|
||||
}
|
||||
const header = await reader.readExactly(headerLength);
|
||||
let command: Readonly<RemoteWorkerArtifactUploadCommand>;
|
||||
try {
|
||||
command = parseRemoteWorkerArtifactUploadHeader(header, {
|
||||
workerId: input.workerId,
|
||||
workerSessionId: input.workerSessionId,
|
||||
});
|
||||
} finally {
|
||||
header.fill(0);
|
||||
}
|
||||
if (input.contentLength !== 4 + headerLength + command.byteLength) {
|
||||
throw new InvalidRemoteWorkerCompletionError(
|
||||
'Artifact upload envelope length does not match its header',
|
||||
);
|
||||
}
|
||||
try {
|
||||
await this.authority.authorizeArtifactUpload(command);
|
||||
const target = storageCommand(command);
|
||||
const content = reader.content(command.byteLength);
|
||||
const receipt = await this.store.put(
|
||||
target,
|
||||
content.chunks,
|
||||
input.signal,
|
||||
);
|
||||
if (!content.isComplete()) {
|
||||
throw new Error('Artifact store did not consume the complete body');
|
||||
}
|
||||
return assertReceiptMatches(target, receipt);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof InvalidRemoteWorkerCompletionError ||
|
||||
error instanceof RemoteWorkerCompletionFenceRejectedError ||
|
||||
error instanceof RemoteWorkerCompletionUnavailableError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new RemoteWorkerCompletionUnavailableError({ cause: error });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterRemoteWorkerCompletionService {
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RemoteWorkerCompletionRepository,
|
||||
private readonly store: Pick<ClusterRemoteWorkerArtifactStore, 'inspect'>,
|
||||
options: ClusterRemoteWorkerCompletionServiceOptions = {},
|
||||
) {
|
||||
if (
|
||||
typeof repository?.complete !== 'function' ||
|
||||
typeof store?.inspect !== 'function' ||
|
||||
(options.createEventId !== undefined &&
|
||||
typeof options.createEventId !== 'function')
|
||||
) {
|
||||
throw new TypeError('Remote Worker completion service is invalid');
|
||||
}
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
}
|
||||
|
||||
async complete(
|
||||
value: RemoteWorkerCompletionCommand,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readonly<RemoteWorkerCompletionResult>> {
|
||||
const command = normalizeRemoteWorkerCompletionCommand(value);
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
const lookup = Object.freeze({
|
||||
projectId: command.projectId,
|
||||
runId: command.runId,
|
||||
attemptId: command.attemptId,
|
||||
logArtifactId: command.artifact.logArtifactId,
|
||||
});
|
||||
let stored: Readonly<RemoteWorkerArtifactReceipt> | undefined;
|
||||
try {
|
||||
stored = await this.store.inspect(lookup, signal);
|
||||
} catch (error) {
|
||||
throw new RemoteWorkerCompletionUnavailableError({ cause: error });
|
||||
}
|
||||
if (!stored) {
|
||||
throw new RemoteWorkerCompletionFenceRejectedError(
|
||||
command.attemptId,
|
||||
'state_mismatch',
|
||||
);
|
||||
}
|
||||
const receipt = assertReceiptMatches(
|
||||
{
|
||||
...lookup,
|
||||
byteLength: command.artifact.byteLength,
|
||||
...(command.artifact.truncated === undefined
|
||||
? {}
|
||||
: { truncated: command.artifact.truncated }),
|
||||
},
|
||||
stored,
|
||||
);
|
||||
if (receipt.sha256 !== command.artifact.sha256) {
|
||||
throw new RemoteWorkerCompletionFenceRejectedError(
|
||||
command.attemptId,
|
||||
'replay_mismatch',
|
||||
);
|
||||
}
|
||||
try {
|
||||
const result = normalizeRemoteWorkerCompletionResult(
|
||||
await this.repository.complete(Object.freeze({
|
||||
...command,
|
||||
attemptEventId: eventId(this.createEventId),
|
||||
runEventId: eventId(this.createEventId),
|
||||
})),
|
||||
);
|
||||
if (
|
||||
result.runId !== command.runId ||
|
||||
result.attemptId !== command.attemptId ||
|
||||
result.callbackSequence !== command.callbackSequence
|
||||
) {
|
||||
throw new Error('Remote Worker completion authority drifted');
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof RemoteWorkerCompletionFenceRejectedError ||
|
||||
error instanceof RemoteWorkerCompletionUnavailableError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new RemoteWorkerCompletionUnavailableError({ cause: error });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
// Remote execution owns bounded offer selection, placement, and lease claiming.
|
||||
import { randomBytes, randomUUID } from 'node:crypto';
|
||||
import type {
|
||||
ClusterDispatchCandidate,
|
||||
ClusterDispatchCandidateCursor,
|
||||
ClusterDispatchSource,
|
||||
ClusterRemoteExecutionOffer,
|
||||
} from '@qinglong/runtime-core/remote-dispatch';
|
||||
import {
|
||||
assertRemoteDispatchPageSize,
|
||||
createClusterRemoteExecutionOffer,
|
||||
evaluateRemoteWorkerPlacement,
|
||||
leaseTokenMatchesDigest,
|
||||
normalizeClusterDispatchCandidate,
|
||||
} from '@qinglong/runtime-core/remote-dispatch';
|
||||
import type {
|
||||
ClusterTaskExecutionRevision,
|
||||
ClusterTaskExecutionRevisionSource,
|
||||
} from '@qinglong/runtime-core/cluster-execution-revision';
|
||||
import type {
|
||||
ClaimRunDispatchLeaseResult,
|
||||
RunDispatchLeaseRepository,
|
||||
WorkerSessionRecord,
|
||||
WorkerSessionRepository,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
assertRunDispatchId,
|
||||
assertRunDispatchLeaseDuration,
|
||||
assertRunDispatchLeaseToken,
|
||||
assertWorkerId,
|
||||
assertWorkerSessionId,
|
||||
} from '@qinglong/runtime-core';
|
||||
import { parseTaskDefinitionRevisionRef } from '@qinglong/runtime-core/task-definition-execution-compiler';
|
||||
|
||||
const DEFAULT_PAGE_SIZE = 8;
|
||||
const DEFAULT_MAX_PAGES = 2;
|
||||
const DEFAULT_MAX_CLAIMS = 8;
|
||||
const DEFAULT_LEASE_MS = 30_000;
|
||||
const MAX_PAGES = 16;
|
||||
const MAX_CLAIMS = 64;
|
||||
|
||||
export interface ClusterRemoteWorkerOfferPrincipal {
|
||||
readonly workerId: string;
|
||||
}
|
||||
|
||||
export interface ClaimClusterRemoteWorkerOfferCommand {
|
||||
readonly workerSessionId: string;
|
||||
readonly workerGeneration: number;
|
||||
/** Worker-generated stable idempotency key for this poll attempt. */
|
||||
readonly offerId: string;
|
||||
/** Worker-generated high-entropy capability; PostgreSQL stores only its digest. */
|
||||
readonly leaseToken: string;
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerOfferStats {
|
||||
readonly pages: number;
|
||||
readonly candidates: number;
|
||||
readonly plansUnavailable: number;
|
||||
readonly placementMismatches: number;
|
||||
readonly claimAttempts: number;
|
||||
readonly claimRaces: number;
|
||||
}
|
||||
|
||||
type MutableClusterRemoteWorkerOfferStats = {
|
||||
-readonly [Key in keyof ClusterRemoteWorkerOfferStats]: ClusterRemoteWorkerOfferStats[Key];
|
||||
};
|
||||
|
||||
export type ClaimClusterRemoteWorkerOfferResult =
|
||||
| Readonly<{
|
||||
status: 'offered';
|
||||
offer: ClusterRemoteExecutionOffer;
|
||||
stats: ClusterRemoteWorkerOfferStats;
|
||||
truncated: boolean;
|
||||
}>
|
||||
| Readonly<{
|
||||
status: 'idle';
|
||||
reason:
|
||||
| 'worker_unavailable'
|
||||
| 'no_candidates'
|
||||
| 'no_match'
|
||||
| 'plans_unavailable'
|
||||
| 'claim_raced'
|
||||
| 'claim_budget_exhausted'
|
||||
| 'scan_budget_exhausted';
|
||||
stats: ClusterRemoteWorkerOfferStats;
|
||||
truncated: boolean;
|
||||
}>;
|
||||
|
||||
export class ClusterRemoteWorkerOfferFenceRejectedError extends Error {
|
||||
readonly code = 'REMOTE_WORKER_OFFER_FENCED';
|
||||
|
||||
constructor() {
|
||||
super('Remote Worker offer authority was fenced');
|
||||
this.name = 'ClusterRemoteWorkerOfferFenceRejectedError';
|
||||
}
|
||||
}
|
||||
|
||||
export interface ClusterRemoteWorkerOfferClaimServiceOptions {
|
||||
readonly pageSize?: number;
|
||||
readonly maxPages?: number;
|
||||
readonly maxClaimAttempts?: number;
|
||||
readonly leaseDurationMs?: number;
|
||||
readonly createEventId?: () => string;
|
||||
}
|
||||
|
||||
function bounded(name: string, value: number, minimum: number, maximum: number): number {
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
||||
throw new RangeError(`${name} must be between ${minimum} and ${maximum}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function emptyStats(): MutableClusterRemoteWorkerOfferStats {
|
||||
return {
|
||||
pages: 0,
|
||||
candidates: 0,
|
||||
plansUnavailable: 0,
|
||||
placementMismatches: 0,
|
||||
claimAttempts: 0,
|
||||
claimRaces: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function cursor(candidate: ClusterDispatchCandidate): ClusterDispatchCandidateCursor {
|
||||
return Object.freeze({
|
||||
priority: candidate.priority,
|
||||
queuedAtMs: candidate.queuedAtMs,
|
||||
attemptCreatedAtMs: candidate.attemptCreatedAtMs,
|
||||
attemptId: candidate.attemptId,
|
||||
});
|
||||
}
|
||||
|
||||
export class ClusterRemoteWorkerOfferClaimService {
|
||||
private readonly pageSize: number;
|
||||
private readonly maxPages: number;
|
||||
private readonly maxClaimAttempts: number;
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly source: ClusterDispatchSource,
|
||||
private readonly workers: Pick<WorkerSessionRepository, 'findById'>,
|
||||
private readonly revisions: ClusterTaskExecutionRevisionSource,
|
||||
private readonly leases: Pick<RunDispatchLeaseRepository, 'claim'>,
|
||||
options: ClusterRemoteWorkerOfferClaimServiceOptions = {},
|
||||
) {
|
||||
if (
|
||||
!source || typeof source.listClusterDispatchCandidates !== 'function' ||
|
||||
typeof source.findClusterDispatchRecovery !== 'function' ||
|
||||
!workers || typeof workers.findById !== 'function' ||
|
||||
!revisions || typeof revisions.resolveClusterTaskExecutionRevision !== 'function' ||
|
||||
!leases || typeof leases.claim !== 'function'
|
||||
) throw new TypeError('Remote Worker offer service dependencies are invalid');
|
||||
const allowed = new Set([
|
||||
'createEventId', 'leaseDurationMs', 'maxClaimAttempts', 'maxPages', 'pageSize',
|
||||
]);
|
||||
if (!options || typeof options !== 'object' || Array.isArray(options) || Object.keys(options).some((key) => !allowed.has(key))) {
|
||||
throw new TypeError('Remote Worker offer service options are invalid');
|
||||
}
|
||||
this.pageSize = options.pageSize ?? DEFAULT_PAGE_SIZE;
|
||||
assertRemoteDispatchPageSize(this.pageSize);
|
||||
this.maxPages = bounded('Remote Worker offer maxPages', options.maxPages ?? DEFAULT_MAX_PAGES, 1, MAX_PAGES);
|
||||
this.maxClaimAttempts = bounded('Remote Worker offer maxClaimAttempts', options.maxClaimAttempts ?? DEFAULT_MAX_CLAIMS, 1, MAX_CLAIMS);
|
||||
this.leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_MS;
|
||||
assertRunDispatchLeaseDuration(this.leaseDurationMs);
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
if (typeof this.createEventId !== 'function') {
|
||||
throw new TypeError('Remote Worker offer event ID factory is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async claimNext(
|
||||
principal: ClusterRemoteWorkerOfferPrincipal,
|
||||
command: ClaimClusterRemoteWorkerOfferCommand,
|
||||
): Promise<ClaimClusterRemoteWorkerOfferResult> {
|
||||
this.assertCommand(principal, command);
|
||||
const stats = emptyStats();
|
||||
const recovered = await this.source.findClusterDispatchRecovery(command.offerId);
|
||||
if (recovered) {
|
||||
if (
|
||||
recovered.lease.status !== 'leased' ||
|
||||
recovered.lease.expiresAtMs <= recovered.observedAtMs ||
|
||||
!recovered.workerCurrent ||
|
||||
recovered.lease.workerId !== principal.workerId ||
|
||||
recovered.lease.workerSessionId !== command.workerSessionId ||
|
||||
recovered.lease.workerGeneration !== command.workerGeneration ||
|
||||
!leaseTokenMatchesDigest(command.leaseToken, recovered.lease.leaseTokenDigest)
|
||||
) throw new ClusterRemoteWorkerOfferFenceRejectedError();
|
||||
const revision = await this.resolveRevision(recovered.candidate);
|
||||
if (!revision) throw new ClusterRemoteWorkerOfferFenceRejectedError();
|
||||
return Object.freeze({
|
||||
status: 'offered' as const,
|
||||
offer: this.offer(
|
||||
'lease_recovery', command, recovered.candidate,
|
||||
recovered.lease, revision, 0,
|
||||
),
|
||||
stats: Object.freeze(stats),
|
||||
truncated: false,
|
||||
});
|
||||
}
|
||||
|
||||
let after: ClusterDispatchCandidateCursor | undefined;
|
||||
let worker: WorkerSessionRecord | null | undefined;
|
||||
let sawCandidate = false;
|
||||
let sawMatch = false;
|
||||
let sawRace = false;
|
||||
let lastTruncated = false;
|
||||
for (let pageIndex = 0; pageIndex < this.maxPages; pageIndex += 1) {
|
||||
const page = await this.source.listClusterDispatchCandidates({
|
||||
limit: this.pageSize,
|
||||
...(after === undefined ? {} : { after }),
|
||||
});
|
||||
stats.pages += 1;
|
||||
lastTruncated = page.truncated;
|
||||
if (page.candidates.length > this.pageSize) {
|
||||
throw new RangeError('Remote Worker candidate source exceeded page size');
|
||||
}
|
||||
worker ??= await this.workers.findById(principal.workerId);
|
||||
if (
|
||||
!worker || worker.sessionId !== command.workerSessionId ||
|
||||
worker.generation !== command.workerGeneration ||
|
||||
worker.status !== 'online' || worker.availableSlots < 1 ||
|
||||
worker.leaseExpiresAtMs <= page.observedAtMs
|
||||
) return this.idle('worker_unavailable', stats, false);
|
||||
|
||||
for (const rawCandidate of page.candidates) {
|
||||
const candidate = normalizeClusterDispatchCandidate(rawCandidate);
|
||||
sawCandidate = true;
|
||||
stats.candidates += 1;
|
||||
const revision = await this.resolveRevision(candidate);
|
||||
if (!revision) {
|
||||
stats.plansUnavailable += 1;
|
||||
continue;
|
||||
}
|
||||
const placement = evaluateRemoteWorkerPlacement(
|
||||
worker,
|
||||
revision.placement ?? {},
|
||||
page.observedAtMs,
|
||||
);
|
||||
if (!placement.matches) {
|
||||
stats.placementMismatches += 1;
|
||||
continue;
|
||||
}
|
||||
sawMatch = true;
|
||||
if (stats.claimAttempts >= this.maxClaimAttempts) {
|
||||
return this.idle('claim_budget_exhausted', stats, true);
|
||||
}
|
||||
const eventId = this.createEventId();
|
||||
assertRunDispatchId('eventId', eventId);
|
||||
stats.claimAttempts += 1;
|
||||
const claim = await this.leases.claim({
|
||||
runId: candidate.runId,
|
||||
attemptId: candidate.attemptId,
|
||||
workerId: principal.workerId,
|
||||
workerSessionId: command.workerSessionId,
|
||||
workerGeneration: command.workerGeneration,
|
||||
leaseToken: command.leaseToken,
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
eventId,
|
||||
offerId: command.offerId,
|
||||
});
|
||||
if (claim.status === 'claimed' || claim.status === 'idempotent') {
|
||||
return Object.freeze({
|
||||
status: 'offered' as const,
|
||||
offer: this.offer(
|
||||
'new_claim', command, candidate, claim.lease, revision,
|
||||
placement.score,
|
||||
),
|
||||
stats: Object.freeze(stats),
|
||||
truncated: page.truncated,
|
||||
});
|
||||
}
|
||||
if (claim.status === 'worker_unavailable' || claim.status === 'capacity_exhausted') {
|
||||
return this.idle('worker_unavailable', stats, false);
|
||||
}
|
||||
stats.claimRaces += 1;
|
||||
sawRace = true;
|
||||
}
|
||||
if (!page.truncated || page.candidates.length === 0) break;
|
||||
const last = page.candidates.at(-1);
|
||||
if (!last) break;
|
||||
const next = page.next ?? cursor(last);
|
||||
if (after && next.attemptId === after.attemptId) {
|
||||
throw new Error('Remote Worker candidate cursor did not advance');
|
||||
}
|
||||
after = next;
|
||||
}
|
||||
if (lastTruncated) return this.idle('scan_budget_exhausted', stats, true);
|
||||
if (!sawCandidate) return this.idle('no_candidates', stats, false);
|
||||
if (stats.plansUnavailable === stats.candidates) return this.idle('plans_unavailable', stats, false);
|
||||
return this.idle(sawRace ? 'claim_raced' : sawMatch ? 'claim_raced' : 'no_match', stats, false);
|
||||
}
|
||||
|
||||
private async resolveRevision(
|
||||
candidate: ClusterDispatchCandidate,
|
||||
): Promise<ClusterTaskExecutionRevision | null> {
|
||||
let sourceRevision: number;
|
||||
try {
|
||||
sourceRevision = parseTaskDefinitionRevisionRef(candidate.taskRevision).revision;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const revision = await this.revisions.resolveClusterTaskExecutionRevision({
|
||||
projectId: candidate.projectId,
|
||||
taskId: candidate.taskId,
|
||||
sourceRevision,
|
||||
});
|
||||
if (
|
||||
!revision || revision.projectId !== candidate.projectId ||
|
||||
revision.taskId !== candidate.taskId ||
|
||||
revision.taskRevision !== candidate.taskRevision
|
||||
) return null;
|
||||
return revision;
|
||||
}
|
||||
|
||||
private offer(
|
||||
deliveryKind: ClusterRemoteExecutionOffer['deliveryKind'],
|
||||
command: ClaimClusterRemoteWorkerOfferCommand,
|
||||
candidate: ClusterDispatchCandidate,
|
||||
lease: Extract<ClaimRunDispatchLeaseResult, { lease: unknown }>['lease'],
|
||||
revision: ClusterTaskExecutionRevision,
|
||||
placementScore: number,
|
||||
): ClusterRemoteExecutionOffer {
|
||||
return createClusterRemoteExecutionOffer({
|
||||
offerId: command.offerId,
|
||||
deliveryKind,
|
||||
executionDigest: revision.contentDigest,
|
||||
candidate,
|
||||
worker: {
|
||||
workerId: lease.workerId,
|
||||
sessionId: lease.workerSessionId,
|
||||
generation: lease.workerGeneration,
|
||||
},
|
||||
lease,
|
||||
leaseToken: command.leaseToken,
|
||||
executionRevision: revision,
|
||||
placementScore,
|
||||
});
|
||||
}
|
||||
|
||||
private idle(
|
||||
reason: Extract<ClaimClusterRemoteWorkerOfferResult, { status: 'idle' }>['reason'],
|
||||
stats: ClusterRemoteWorkerOfferStats,
|
||||
truncated: boolean,
|
||||
): ClaimClusterRemoteWorkerOfferResult {
|
||||
return Object.freeze({
|
||||
status: 'idle' as const,
|
||||
reason,
|
||||
stats: Object.freeze({ ...stats }),
|
||||
truncated,
|
||||
});
|
||||
}
|
||||
|
||||
private assertCommand(
|
||||
principal: ClusterRemoteWorkerOfferPrincipal,
|
||||
command: ClaimClusterRemoteWorkerOfferCommand,
|
||||
): void {
|
||||
if (!principal || typeof principal !== 'object' || Array.isArray(principal)) {
|
||||
throw new TypeError('Remote Worker offer principal is invalid');
|
||||
}
|
||||
assertWorkerId(principal.workerId);
|
||||
if (!command || typeof command !== 'object' || Array.isArray(command)) {
|
||||
throw new TypeError('Remote Worker offer command is invalid');
|
||||
}
|
||||
const keys = Object.keys(command).sort().join(',');
|
||||
if (keys !== 'leaseToken,offerId,workerGeneration,workerSessionId') {
|
||||
throw new TypeError('Remote Worker offer command shape is invalid');
|
||||
}
|
||||
assertWorkerSessionId(command.workerSessionId);
|
||||
if (!Number.isSafeInteger(command.workerGeneration) || command.workerGeneration < 1) {
|
||||
throw new RangeError('Remote Worker offer generation is invalid');
|
||||
}
|
||||
assertRunDispatchId('offerId', command.offerId);
|
||||
assertRunDispatchLeaseToken(command.leaseToken);
|
||||
}
|
||||
}
|
||||
|
||||
export function createRemoteWorkerLeaseToken(): string {
|
||||
return randomBytes(32).toString('base64url');
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
// Remote execution owns fenced Worker lease renewal, release, and timeout authority.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
RemoteWorkerLeaseControlUnavailableError,
|
||||
assertRemoteWorkerLeaseControlDuration,
|
||||
normalizeRemoteWorkerLeaseControlCommand,
|
||||
normalizeRemoteWorkerLeaseControlResult,
|
||||
type RemoteWorkerLeaseControlCommand,
|
||||
type RemoteWorkerLeaseControlRepository,
|
||||
type RemoteWorkerLeaseControlResult,
|
||||
} from '@qinglong/runtime-core/remote-worker-lease-control';
|
||||
|
||||
export interface ClusterRemoteWorkerLeaseControlServiceOptions {
|
||||
readonly leaseDurationMs?: number;
|
||||
readonly createEventId?: () => string;
|
||||
}
|
||||
|
||||
function eventId(factory: () => string): string {
|
||||
const value = factory();
|
||||
if (
|
||||
typeof value !== 'string' || value.length < 1 || value.length > 36 ||
|
||||
/[\u0000-\u001f\u007f]/.test(value)
|
||||
) throw new RemoteWorkerLeaseControlUnavailableError();
|
||||
return value;
|
||||
}
|
||||
|
||||
export class ClusterRemoteWorkerLeaseControlService {
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly createEventId: () => string;
|
||||
|
||||
constructor(
|
||||
private readonly repository: RemoteWorkerLeaseControlRepository,
|
||||
options: ClusterRemoteWorkerLeaseControlServiceOptions = {},
|
||||
) {
|
||||
if (
|
||||
typeof repository?.control !== 'function' ||
|
||||
(options.createEventId !== undefined &&
|
||||
typeof options.createEventId !== 'function')
|
||||
) throw new TypeError('Remote Worker lease control service is invalid');
|
||||
const leaseDurationMs = options.leaseDurationMs ?? 30_000;
|
||||
assertRemoteWorkerLeaseControlDuration(leaseDurationMs);
|
||||
this.leaseDurationMs = leaseDurationMs;
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
}
|
||||
|
||||
async control(
|
||||
value: RemoteWorkerLeaseControlCommand,
|
||||
): Promise<Readonly<RemoteWorkerLeaseControlResult>> {
|
||||
const command = normalizeRemoteWorkerLeaseControlCommand(value);
|
||||
return normalizeRemoteWorkerLeaseControlResult(
|
||||
await this.repository.control(Object.freeze({
|
||||
...command,
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
timeoutEventId: eventId(this.createEventId),
|
||||
})),
|
||||
);
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
// Remote execution owns offer-bound Secret delivery without retaining plaintext authority.
|
||||
import {
|
||||
InvalidRemoteWorkerSecretDeliveryError,
|
||||
RemoteWorkerSecretDeliveryFenceRejectedError,
|
||||
RemoteWorkerSecretDeliveryUnavailableError,
|
||||
createRemoteWorkerSecretDeliveryResponseBody,
|
||||
normalizeRemoteWorkerSecretDeliveryAuthority,
|
||||
normalizeRemoteWorkerSecretDeliveryCommand,
|
||||
type RemoteWorkerSecretDeliveryAuthorityRepository,
|
||||
type RemoteWorkerSecretDeliveryCommand,
|
||||
type RemoteWorkerSecretDeliveryResult,
|
||||
type RemoteWorkerSecretValueProvider,
|
||||
} from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
|
||||
export interface ClusterRemoteWorkerSecretDeliveryPrincipal {
|
||||
readonly workerId: string;
|
||||
}
|
||||
|
||||
export type ClusterRemoteWorkerSecretDeliveryCommand = Omit<
|
||||
RemoteWorkerSecretDeliveryCommand,
|
||||
'workerId'
|
||||
>;
|
||||
|
||||
export class ClusterRemoteWorkerSecretDeliveryService {
|
||||
constructor(
|
||||
private readonly authority: RemoteWorkerSecretDeliveryAuthorityRepository,
|
||||
private readonly secrets: RemoteWorkerSecretValueProvider,
|
||||
) {
|
||||
if (
|
||||
!authority ||
|
||||
typeof authority.authorize !== 'function' ||
|
||||
!secrets ||
|
||||
typeof secrets.resolve !== 'function'
|
||||
) throw new TypeError('Remote Worker Secret delivery service is invalid');
|
||||
}
|
||||
|
||||
async deliver(
|
||||
principal: ClusterRemoteWorkerSecretDeliveryPrincipal,
|
||||
input: ClusterRemoteWorkerSecretDeliveryCommand,
|
||||
): Promise<Readonly<RemoteWorkerSecretDeliveryResult>> {
|
||||
if (
|
||||
!principal ||
|
||||
typeof principal !== 'object' ||
|
||||
Array.isArray(principal) ||
|
||||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(principal.workerId)
|
||||
) throw new TypeError('Remote Worker Secret delivery principal is invalid');
|
||||
const command = normalizeRemoteWorkerSecretDeliveryCommand({
|
||||
...input,
|
||||
workerId: principal.workerId,
|
||||
});
|
||||
let authorized;
|
||||
try {
|
||||
authorized = normalizeRemoteWorkerSecretDeliveryAuthority(
|
||||
await this.authority.authorize(command),
|
||||
);
|
||||
if (
|
||||
authorized.workerId !== command.workerId ||
|
||||
authorized.workerSessionId !== command.workerSessionId ||
|
||||
authorized.workerGeneration !== command.workerGeneration ||
|
||||
authorized.runId !== command.runId ||
|
||||
authorized.attemptId !== command.attemptId ||
|
||||
authorized.projectId !== command.projectId ||
|
||||
authorized.taskId !== command.taskId ||
|
||||
authorized.taskRevision !== command.taskRevision ||
|
||||
authorized.executionDigest !== command.executionDigest ||
|
||||
authorized.offerId !== command.offerId ||
|
||||
authorized.leaseGeneration !== command.leaseGeneration ||
|
||||
authorized.leaseVersion !== command.expectedLeaseVersion ||
|
||||
JSON.stringify(authorized.secretRefs) !== JSON.stringify(command.secretRefs)
|
||||
) throw new InvalidRemoteWorkerSecretDeliveryError(
|
||||
'repository authority does not match command',
|
||||
);
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof RemoteWorkerSecretDeliveryFenceRejectedError ||
|
||||
error instanceof RemoteWorkerSecretDeliveryUnavailableError
|
||||
) throw error;
|
||||
throw new RemoteWorkerSecretDeliveryUnavailableError();
|
||||
}
|
||||
let resolution;
|
||||
try {
|
||||
resolution = await this.secrets.resolve(authorized);
|
||||
} catch {
|
||||
throw new RemoteWorkerSecretDeliveryUnavailableError();
|
||||
}
|
||||
if (!resolution) throw new RemoteWorkerSecretDeliveryUnavailableError();
|
||||
try {
|
||||
if (
|
||||
typeof resolution !== 'object' ||
|
||||
Array.isArray(resolution) ||
|
||||
Object.keys(resolution).some((key) => key !== 'values' && key !== 'dispose') ||
|
||||
(resolution.dispose !== undefined &&
|
||||
typeof resolution.dispose !== 'function')
|
||||
) throw new InvalidRemoteWorkerSecretDeliveryError(
|
||||
'provider response shape is invalid',
|
||||
);
|
||||
const body = createRemoteWorkerSecretDeliveryResponseBody({
|
||||
runId: authorized.runId,
|
||||
attemptId: authorized.attemptId,
|
||||
offerId: authorized.offerId,
|
||||
executionDigest: authorized.executionDigest,
|
||||
values: resolution.values,
|
||||
}, authorized.secretRefs);
|
||||
return Object.freeze({
|
||||
runId: body.runId,
|
||||
attemptId: body.attemptId,
|
||||
offerId: body.offerId,
|
||||
executionDigest: body.executionDigest,
|
||||
values: body.values,
|
||||
...(resolution.dispose === undefined
|
||||
? {}
|
||||
: { dispose: resolution.dispose }),
|
||||
});
|
||||
} catch (error) {
|
||||
try { await resolution.dispose?.(); } catch { /* preserve root */ }
|
||||
if (error instanceof InvalidRemoteWorkerSecretDeliveryError) {
|
||||
throw new RemoteWorkerSecretDeliveryUnavailableError();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Remote execution owns the least-privilege assembly of Worker-facing runtime capabilities.
|
||||
import type { PostgresPool } from '@qinglong/runtime-core';
|
||||
import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import {
|
||||
PostgresClusterDispatchSource,
|
||||
PostgresRemoteRunActivationRepository,
|
||||
PostgresRemoteWorkerCompletionRepository,
|
||||
PostgresRemoteWorkerLeaseControlRepository,
|
||||
PostgresRemoteWorkerSecretDeliveryAuthorityRepository,
|
||||
PostgresRunDispatchLeaseRepository,
|
||||
PostgresTaskExecutionRevisionSource,
|
||||
PostgresWorkerSessionRepository,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
import {
|
||||
ClusterRemoteWorkerOfferClaimService,
|
||||
} from './remoteWorkerDispatcher';
|
||||
import {
|
||||
ClusterRemoteRunActivationService,
|
||||
} from './remoteRunActivationService';
|
||||
import {
|
||||
ClusterRemoteWorkerSecretDeliveryService,
|
||||
} from './remoteWorkerSecretDeliveryService';
|
||||
import {
|
||||
ClusterRemoteWorkerArtifactService,
|
||||
ClusterRemoteWorkerCompletionService,
|
||||
type ClusterRemoteWorkerArtifactStore,
|
||||
} from './remoteWorkerCompletionService';
|
||||
import {
|
||||
ClusterRemoteWorkerLeaseControlService,
|
||||
} from './remoteWorkerLeaseControlService';
|
||||
import type { WorkerIngressPipelineOptions } from '../worker-ingress/workerIngressPipeline';
|
||||
|
||||
export interface ClusterWorkerRuntimeDependencies {
|
||||
readonly artifactStore: ClusterRemoteWorkerArtifactStore;
|
||||
readonly secretProvider?: RemoteWorkerSecretValueProvider;
|
||||
}
|
||||
|
||||
/**
|
||||
* The in-process capability boundary from the runtime authority to the
|
||||
* Worker-facing transport. It exposes reviewed operations, never the runtime
|
||||
* Pool or mutation repositories.
|
||||
*/
|
||||
export interface ClusterWorkerRuntimePort {
|
||||
readonly offers: NonNullable<WorkerIngressPipelineOptions['offers']>;
|
||||
readonly activation: NonNullable<WorkerIngressPipelineOptions['activation']>;
|
||||
readonly secrets?: NonNullable<WorkerIngressPipelineOptions['secrets']>;
|
||||
readonly artifacts: NonNullable<WorkerIngressPipelineOptions['artifacts']>;
|
||||
readonly completion: NonNullable<WorkerIngressPipelineOptions['completion']>;
|
||||
readonly leaseControl: NonNullable<
|
||||
WorkerIngressPipelineOptions['leaseControl']
|
||||
>;
|
||||
}
|
||||
|
||||
export function createClusterWorkerRuntimePort(
|
||||
pool: PostgresPool,
|
||||
dependencies: ClusterWorkerRuntimeDependencies,
|
||||
): Readonly<ClusterWorkerRuntimePort> {
|
||||
if (!pool || typeof pool.query !== 'function') {
|
||||
throw new TypeError('Cluster Worker runtime Pool is invalid');
|
||||
}
|
||||
if (
|
||||
!dependencies ||
|
||||
typeof dependencies !== 'object' ||
|
||||
Array.isArray(dependencies)
|
||||
) {
|
||||
throw new TypeError('Cluster Worker runtime dependencies are invalid');
|
||||
}
|
||||
|
||||
const workerSessions = new PostgresWorkerSessionRepository(pool);
|
||||
const completionRepository =
|
||||
new PostgresRemoteWorkerCompletionRepository(pool);
|
||||
const secretProvider = dependencies.secretProvider;
|
||||
return Object.freeze({
|
||||
offers: new ClusterRemoteWorkerOfferClaimService(
|
||||
new PostgresClusterDispatchSource(pool),
|
||||
workerSessions,
|
||||
new PostgresTaskExecutionRevisionSource(pool),
|
||||
new PostgresRunDispatchLeaseRepository(pool),
|
||||
),
|
||||
activation: new ClusterRemoteRunActivationService(
|
||||
new PostgresRemoteRunActivationRepository(pool),
|
||||
),
|
||||
...(secretProvider === undefined
|
||||
? {}
|
||||
: {
|
||||
secrets: new ClusterRemoteWorkerSecretDeliveryService(
|
||||
new PostgresRemoteWorkerSecretDeliveryAuthorityRepository(pool),
|
||||
secretProvider,
|
||||
),
|
||||
}),
|
||||
artifacts: new ClusterRemoteWorkerArtifactService(
|
||||
completionRepository,
|
||||
dependencies.artifactStore,
|
||||
),
|
||||
completion: new ClusterRemoteWorkerCompletionService(
|
||||
completionRepository,
|
||||
dependencies.artifactStore,
|
||||
),
|
||||
leaseControl: new ClusterRemoteWorkerLeaseControlService(
|
||||
new PostgresRemoteWorkerLeaseControlRepository(pool),
|
||||
),
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
// Run owns bounded convergence of durable cancellation intent to terminal state.
|
||||
import type {
|
||||
ClusterRunCancellationConvergenceCoordinator,
|
||||
ClusterRunCancellationConvergenceCycleResult,
|
||||
} from '@qinglong/runtime-core/cluster-run-cancellation-convergence';
|
||||
|
||||
export interface ClusterRunCancellationConvergenceLifecycleOptions {
|
||||
readonly intervalMs: number;
|
||||
readonly stopTimeoutMs: number;
|
||||
readonly onDiagnostic?: (
|
||||
error: unknown,
|
||||
summary?: Readonly<ClusterRunCancellationConvergenceCycleResult>,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ClusterRunCancellationConvergenceLifecycleStopSummary {
|
||||
readonly status: 'stopped' | 'timed_out';
|
||||
}
|
||||
|
||||
/** One constant-cost cadence for all pending non-executing Run cancellations. */
|
||||
export class ClusterRunCancellationConvergenceLifecycle {
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private inFlight:
|
||||
| Promise<Readonly<ClusterRunCancellationConvergenceCycleResult>>
|
||||
| undefined;
|
||||
private stopPromise:
|
||||
| Promise<ClusterRunCancellationConvergenceLifecycleStopSummary>
|
||||
| undefined;
|
||||
private running = false;
|
||||
private stopping = false;
|
||||
|
||||
constructor(
|
||||
private readonly coordinator: Pick<
|
||||
ClusterRunCancellationConvergenceCoordinator,
|
||||
'reconcile'
|
||||
>,
|
||||
private readonly options: ClusterRunCancellationConvergenceLifecycleOptions,
|
||||
) {
|
||||
if (
|
||||
typeof coordinator?.reconcile !== 'function' ||
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
!Number.isSafeInteger(options.intervalMs) ||
|
||||
options.intervalMs < 250 ||
|
||||
options.intervalMs > 60 * 60_000 ||
|
||||
!Number.isSafeInteger(options.stopTimeoutMs) ||
|
||||
options.stopTimeoutMs < 100 ||
|
||||
options.stopTimeoutMs > 30_000 ||
|
||||
(options.onDiagnostic !== undefined &&
|
||||
typeof options.onDiagnostic !== 'function')
|
||||
) {
|
||||
throw new TypeError('Cluster Run cancellation lifecycle options are invalid');
|
||||
}
|
||||
}
|
||||
|
||||
start(): 'started' {
|
||||
if (!this.running && !this.stopping) {
|
||||
this.running = true;
|
||||
this.schedule();
|
||||
}
|
||||
return 'started';
|
||||
}
|
||||
|
||||
runOnce(): Promise<Readonly<ClusterRunCancellationConvergenceCycleResult>> {
|
||||
if (this.stopping) {
|
||||
return Promise.reject(
|
||||
new Error('Cluster Run cancellation lifecycle is stopping'),
|
||||
);
|
||||
}
|
||||
if (this.inFlight) return this.inFlight;
|
||||
const work = this.coordinator.reconcile().finally(() => {
|
||||
if (this.inFlight === work) this.inFlight = undefined;
|
||||
});
|
||||
this.inFlight = work;
|
||||
return work;
|
||||
}
|
||||
|
||||
stopAndDrain(): Promise<ClusterRunCancellationConvergenceLifecycleStopSummary> {
|
||||
if (this.stopPromise) return this.stopPromise;
|
||||
this.stopping = true;
|
||||
this.running = false;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
this.stopPromise = (async () => {
|
||||
const work = this.inFlight;
|
||||
if (!work) return Object.freeze({ status: 'stopped' as const });
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
work.then(
|
||||
() => Object.freeze({ status: 'stopped' as const }),
|
||||
() => Object.freeze({ status: 'stopped' as const }),
|
||||
),
|
||||
new Promise<ClusterRunCancellationConvergenceLifecycleStopSummary>(
|
||||
(resolve) => {
|
||||
timeout = setTimeout(
|
||||
() => resolve(Object.freeze({ status: 'timed_out' as const })),
|
||||
this.options.stopTimeoutMs,
|
||||
);
|
||||
timeout.unref?.();
|
||||
},
|
||||
),
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
}
|
||||
})();
|
||||
return this.stopPromise;
|
||||
}
|
||||
|
||||
private schedule(): void {
|
||||
if (!this.running || this.timer) return;
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = undefined;
|
||||
if (!this.running) return;
|
||||
void this.runOnce()
|
||||
.then((summary) => this.diagnostic(undefined, summary))
|
||||
.catch((error) => this.diagnostic(error))
|
||||
.finally(() => this.schedule());
|
||||
}, this.options.intervalMs);
|
||||
this.timer.unref?.();
|
||||
}
|
||||
|
||||
private async diagnostic(
|
||||
error: unknown,
|
||||
summary?: Readonly<ClusterRunCancellationConvergenceCycleResult>,
|
||||
): Promise<void> {
|
||||
if (this.stopping) return;
|
||||
try {
|
||||
await this.options.onDiagnostic?.(error, summary);
|
||||
} catch {
|
||||
// Diagnostics cannot own or stop convergence.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
// Run owns its Policy-fenced durable cancellation mutation route.
|
||||
import {
|
||||
CLUSTER_RUN_CANCELLATION_SCHEMA,
|
||||
ClusterRunCancellationFenceRejectedError,
|
||||
ClusterRunCancellationNotFoundError,
|
||||
ClusterRunCancellationUnavailableError,
|
||||
InvalidClusterRunCancellationError,
|
||||
createClusterRunCancellationResponseBody,
|
||||
parseClusterRunCancellationRequestBody,
|
||||
type ClusterRunCancellationRepository,
|
||||
} from '@qinglong/runtime-core/cluster-run-cancellation';
|
||||
import type { ClusterControlAdmissionResponse } from '../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_CONTROL_RUN_CANCELLATION_ROUTE = Object.freeze({
|
||||
method: 'POST' as const,
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/cancellation',
|
||||
operationId: 'run.cancel',
|
||||
permission: 'run.stop',
|
||||
projectParameter: 'projectId',
|
||||
});
|
||||
|
||||
export type ClusterRunCancellationEventIdFactory = () => string;
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
/**
|
||||
* Publishes a durable cancellation command. Authentication, authorization and
|
||||
* the first audit complete in admission; the repository revalidates the exact
|
||||
* policy fence in the same transaction that writes the Run intent and Event.
|
||||
*/
|
||||
export function createClusterControlRunCancellationRoute(
|
||||
repository: ClusterRunCancellationRepository,
|
||||
createEventId: ClusterRunCancellationEventIdFactory,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (
|
||||
!repository ||
|
||||
typeof repository.requestUserCancellation !== 'function' ||
|
||||
typeof createEventId !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster-control Run cancellation route is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_RUN_CANCELLATION_ROUTE,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
let body;
|
||||
try {
|
||||
body = parseClusterRunCancellationRequestBody(
|
||||
authorized.request.body,
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidClusterRunCancellationError) {
|
||||
return response(400, {
|
||||
code: 'invalid_run_cancellation_request',
|
||||
schema: CLUSTER_RUN_CANCELLATION_SCHEMA,
|
||||
});
|
||||
}
|
||||
return response(503, { code: 'run_cancellation_unavailable' });
|
||||
}
|
||||
const projectId = authorized.projectId;
|
||||
const runId = parameters.runId;
|
||||
if (
|
||||
projectId === null ||
|
||||
typeof runId !== 'string' ||
|
||||
runId.length < 1 ||
|
||||
!authorized.policyFence ||
|
||||
authorized.policyFence.bindingVersion === null
|
||||
) {
|
||||
return response(503, { code: 'run_cancellation_unavailable' });
|
||||
}
|
||||
try {
|
||||
const result = await repository.requestUserCancellation({
|
||||
projectId,
|
||||
runId,
|
||||
mutationId: body.mutationId,
|
||||
eventId: createEventId(),
|
||||
subject: authorized.principal.subject,
|
||||
policyFence: authorized.policyFence,
|
||||
});
|
||||
return response(
|
||||
result.status === 'accepted' ? 202 : 200,
|
||||
createClusterRunCancellationResponseBody(result),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof ClusterRunCancellationNotFoundError) {
|
||||
return response(404, { code: 'run_not_found' });
|
||||
}
|
||||
if (error instanceof ClusterRunCancellationFenceRejectedError) {
|
||||
return response(409, {
|
||||
code: 'run_cancellation_fence_rejected',
|
||||
reason: error.reason,
|
||||
});
|
||||
}
|
||||
if (
|
||||
error instanceof InvalidClusterRunCancellationError ||
|
||||
error instanceof ClusterRunCancellationUnavailableError
|
||||
) {
|
||||
return response(503, { code: 'run_cancellation_unavailable' });
|
||||
}
|
||||
return response(503, { code: 'run_cancellation_unavailable' });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import {
|
||||
BoundedRunEventListProjectionUnavailableError,
|
||||
InvalidBoundedRunEventListProjectionError,
|
||||
executeBoundedRunEventListProjection,
|
||||
} from '@qinglong/runtime-core/bounded-run-event-list-projection';
|
||||
import type { RunRepositoryReader } from '@qinglong/runtime-core/run-repository';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_CONTROL_RUN_EVENT_LIST_ROUTE = Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/events',
|
||||
operationId: 'run.events.list',
|
||||
permission: 'run.read',
|
||||
projectParameter: 'projectId',
|
||||
allowedQuery: Object.freeze(['after_sequence', 'limit']),
|
||||
});
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function parseQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): Readonly<{ afterSequence?: number; limit?: number }> {
|
||||
const afterValues = query.after_sequence;
|
||||
const limitValues = query.limit;
|
||||
if (
|
||||
(afterValues !== undefined && afterValues.length !== 1) ||
|
||||
(limitValues !== undefined && limitValues.length !== 1)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const rawAfter = afterValues?.[0];
|
||||
const afterSequence = rawAfter === undefined ? undefined : Number(rawAfter);
|
||||
const rawLimit = limitValues?.[0];
|
||||
const limit = rawLimit === undefined ? undefined : Number(rawLimit);
|
||||
if (
|
||||
(rawAfter !== undefined &&
|
||||
(!Number.isSafeInteger(afterSequence) ||
|
||||
Number(afterSequence) < 0 ||
|
||||
Number(afterSequence) > 2_147_483_647 ||
|
||||
String(afterSequence) !== rawAfter)) ||
|
||||
(rawLimit !== undefined &&
|
||||
(!Number.isSafeInteger(limit) ||
|
||||
Number(limit) < 1 ||
|
||||
Number(limit) > 64 ||
|
||||
String(limit) !== rawLimit))
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return Object.freeze({
|
||||
...(afterSequence === undefined ? {} : { afterSequence }),
|
||||
...(limit === undefined ? {} : { limit }),
|
||||
});
|
||||
}
|
||||
|
||||
function validateRunEventListQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): void {
|
||||
parseQuery(query);
|
||||
}
|
||||
|
||||
export function createClusterControlRunEventListRoute(
|
||||
runs: Pick<RunRepositoryReader, 'findRunById' | 'listEvents'>,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (
|
||||
!runs ||
|
||||
typeof runs.findRunById !== 'function' ||
|
||||
typeof runs.listEvents !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster-control Run event list repository is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_RUN_EVENT_LIST_ROUTE,
|
||||
validateQuery: validateRunEventListQuery,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
if (authorized.projectId === null) {
|
||||
return response(503, { code: 'run_event_list_unavailable' });
|
||||
}
|
||||
let input;
|
||||
try {
|
||||
input = parseQuery(authorized.request.query);
|
||||
} catch {
|
||||
return response(400, { code: 'invalid_run_event_list_query' });
|
||||
}
|
||||
try {
|
||||
const result = await executeBoundedRunEventListProjection(
|
||||
runs,
|
||||
authorized.projectId,
|
||||
parameters.runId!,
|
||||
input,
|
||||
);
|
||||
if (!result.found) {
|
||||
return response(404, { code: 'run_not_found' });
|
||||
}
|
||||
const { found: _found, ...timeline } = result;
|
||||
return response(200, { ...timeline });
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof InvalidBoundedRunEventListProjectionError ||
|
||||
error instanceof BoundedRunEventListProjectionUnavailableError
|
||||
) {
|
||||
return response(503, { code: 'run_event_list_unavailable' });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
import {
|
||||
BoundedRunListProjectionUnavailableError,
|
||||
InvalidBoundedRunListProjectionError,
|
||||
executeBoundedRunListProjection,
|
||||
} from '@qinglong/runtime-core/bounded-run-list-projection';
|
||||
import type { ProjectRunListReader } from '@qinglong/runtime-core/project-run-list';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
} from '../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_CONTROL_RUN_LIST_ROUTE = Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/runs',
|
||||
operationId: 'run.list',
|
||||
permission: 'run.read',
|
||||
projectParameter: 'projectId',
|
||||
allowedQuery: Object.freeze([
|
||||
'after_created_at_ms',
|
||||
'after_run_id',
|
||||
'limit',
|
||||
]),
|
||||
});
|
||||
|
||||
const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function parseQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): Readonly<{
|
||||
limit?: number;
|
||||
after?: Readonly<{ createdAtMs: number; runId: string }>;
|
||||
}> {
|
||||
const limitValues = query.limit;
|
||||
const createdAtValues = query.after_created_at_ms;
|
||||
const runIdValues = query.after_run_id;
|
||||
if (
|
||||
(limitValues !== undefined && limitValues.length !== 1) ||
|
||||
(createdAtValues !== undefined && createdAtValues.length !== 1) ||
|
||||
(runIdValues !== undefined && runIdValues.length !== 1) ||
|
||||
(createdAtValues === undefined) !== (runIdValues === undefined)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const rawLimit = limitValues?.[0];
|
||||
const limit = rawLimit === undefined ? undefined : Number(rawLimit);
|
||||
if (
|
||||
rawLimit !== undefined &&
|
||||
(!Number.isSafeInteger(limit) ||
|
||||
Number(limit) < 1 ||
|
||||
Number(limit) > 64 ||
|
||||
String(limit) !== rawLimit)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const rawCreatedAtMs = createdAtValues?.[0];
|
||||
const runId = runIdValues?.[0];
|
||||
if (rawCreatedAtMs === undefined || runId === undefined) {
|
||||
return Object.freeze({ ...(limit === undefined ? {} : { limit }) });
|
||||
}
|
||||
const createdAtMs = Number(rawCreatedAtMs);
|
||||
if (
|
||||
!Number.isSafeInteger(createdAtMs) ||
|
||||
createdAtMs < 0 ||
|
||||
String(createdAtMs) !== rawCreatedAtMs ||
|
||||
!RUN_ID_PATTERN.test(runId)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return Object.freeze({
|
||||
...(limit === undefined ? {} : { limit }),
|
||||
after: Object.freeze({ createdAtMs, runId }),
|
||||
});
|
||||
}
|
||||
|
||||
function validateRunListQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): void {
|
||||
parseQuery(query);
|
||||
}
|
||||
|
||||
export function createClusterControlRunListRoute(
|
||||
runs: ProjectRunListReader,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (!runs || typeof runs.listRunsByProject !== 'function') {
|
||||
throw new TypeError('Cluster-control Run list repository is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_RUN_LIST_ROUTE,
|
||||
validateQuery: validateRunListQuery,
|
||||
async handle(authorized: ClusterControlAuthorizedOperationRequest) {
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
if (authorized.projectId === null) {
|
||||
return response(503, { code: 'run_list_unavailable' });
|
||||
}
|
||||
let input;
|
||||
try {
|
||||
input = parseQuery(authorized.request.query);
|
||||
} catch {
|
||||
return response(400, { code: 'invalid_run_list_query' });
|
||||
}
|
||||
try {
|
||||
const result = await executeBoundedRunListProjection(
|
||||
runs,
|
||||
authorized.projectId,
|
||||
input,
|
||||
);
|
||||
return response(200, { ...result });
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof InvalidBoundedRunListProjectionError ||
|
||||
error instanceof BoundedRunListProjectionUnavailableError
|
||||
) {
|
||||
return response(503, { code: 'run_list_unavailable' });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
// Run owns its bounded read projection and masks cross-Project storage facts.
|
||||
import {
|
||||
EXECUTION_ORIGINS,
|
||||
RUN_STATUSES,
|
||||
type ExecutionOrigin,
|
||||
type ExecutionOwner,
|
||||
type RunRecord,
|
||||
type RunRepositoryReader,
|
||||
type RunStatus,
|
||||
} from '@qinglong/runtime-core';
|
||||
import type { ClusterControlAdmissionResponse } from '../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../transport/routeRegistry';
|
||||
|
||||
export interface ClusterControlRunReadRepository
|
||||
extends Pick<RunRepositoryReader, 'findRunById'> {}
|
||||
|
||||
export interface ClusterControlRunView {
|
||||
readonly id: string;
|
||||
readonly projectId: string;
|
||||
readonly taskId: string;
|
||||
readonly taskRevision: string;
|
||||
readonly status: RunStatus;
|
||||
readonly version: number;
|
||||
readonly eventSequence: number;
|
||||
readonly priority: number;
|
||||
readonly executionOrigin: ExecutionOrigin;
|
||||
readonly executionOwner: ExecutionOwner;
|
||||
readonly createdAtMs: number;
|
||||
readonly queuedAtMs: number | null;
|
||||
readonly startedAtMs: number | null;
|
||||
readonly finishedAtMs: number | null;
|
||||
}
|
||||
|
||||
export const CLUSTER_CONTROL_RUN_READ_ROUTE = Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}',
|
||||
operationId: 'run.get',
|
||||
permission: 'run.read',
|
||||
projectParameter: 'projectId',
|
||||
});
|
||||
|
||||
const CONTROL_CHARACTER_PATTERN = /[\u0000-\u001f\u007f]/;
|
||||
|
||||
function boundedText(value: unknown, maximum: number): value is string {
|
||||
return (
|
||||
typeof value === 'string' &&
|
||||
value.length > 0 &&
|
||||
value.length <= maximum &&
|
||||
!CONTROL_CHARACTER_PATTERN.test(value)
|
||||
);
|
||||
}
|
||||
|
||||
function nonNegativeInteger(value: unknown): value is number {
|
||||
return Number.isSafeInteger(value) && Number(value) >= 0;
|
||||
}
|
||||
|
||||
function optionalTimestamp(value: unknown): value is number | undefined {
|
||||
return value === undefined || nonNegativeInteger(value);
|
||||
}
|
||||
|
||||
function projectRunView(
|
||||
run: RunRecord,
|
||||
runId: string,
|
||||
): Readonly<ClusterControlRunView> | null {
|
||||
if (
|
||||
!run ||
|
||||
typeof run !== 'object' ||
|
||||
Array.isArray(run) ||
|
||||
run.id !== runId ||
|
||||
!boundedText(run.id, 128) ||
|
||||
!boundedText(run.projectId, 128) ||
|
||||
!boundedText(run.taskId, 255) ||
|
||||
!boundedText(run.taskRevision, 255) ||
|
||||
!RUN_STATUSES.includes(run.status) ||
|
||||
!EXECUTION_ORIGINS.includes(run.executionOrigin) ||
|
||||
(run.executionOwner !== 'legacy' && run.executionOwner !== 'runtime') ||
|
||||
!Number.isSafeInteger(run.version) ||
|
||||
run.version < 0 ||
|
||||
!nonNegativeInteger(run.eventSequence) ||
|
||||
!Number.isSafeInteger(run.priority) ||
|
||||
!nonNegativeInteger(run.createdAtMs) ||
|
||||
!optionalTimestamp(run.queuedAtMs) ||
|
||||
!optionalTimestamp(run.startedAtMs) ||
|
||||
!optionalTimestamp(run.finishedAtMs)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return Object.freeze({
|
||||
id: run.id,
|
||||
projectId: run.projectId,
|
||||
taskId: run.taskId,
|
||||
taskRevision: run.taskRevision,
|
||||
status: run.status,
|
||||
version: run.version,
|
||||
eventSequence: run.eventSequence,
|
||||
priority: run.priority,
|
||||
executionOrigin: run.executionOrigin,
|
||||
executionOwner: run.executionOwner,
|
||||
createdAtMs: run.createdAtMs,
|
||||
queuedAtMs: run.queuedAtMs ?? null,
|
||||
startedAtMs: run.startedAtMs ?? null,
|
||||
finishedAtMs: run.finishedAtMs ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
/**
|
||||
* Defines the first reviewed cluster-control business route. The response is a
|
||||
* deliberately low-sensitive projection: refs, trigger identity, request IDs,
|
||||
* executor handles, error summaries and output locations never cross the wire.
|
||||
*/
|
||||
export function createClusterControlRunReadRoute(
|
||||
repository: ClusterControlRunReadRepository,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (!repository || typeof repository.findRunById !== 'function') {
|
||||
throw new TypeError('Cluster-control Run read repository is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_RUN_READ_ROUTE,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
const runId = parameters.runId;
|
||||
if (!boundedText(runId, 128)) {
|
||||
return response(503, { code: 'run_query_unavailable' });
|
||||
}
|
||||
let run: RunRecord | null;
|
||||
try {
|
||||
run = await repository.findRunById(runId);
|
||||
} catch {
|
||||
return response(503, { code: 'run_query_unavailable' });
|
||||
}
|
||||
if (!run) {
|
||||
return response(404, { code: 'run_not_found' });
|
||||
}
|
||||
const view = projectRunView(run, runId);
|
||||
if (!view) {
|
||||
return response(503, { code: 'run_query_unavailable' });
|
||||
}
|
||||
if (view.projectId !== authorized.projectId) {
|
||||
return response(404, { code: 'run_not_found' });
|
||||
}
|
||||
return response(200, { run: view });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export * from './runCancellationRoute';
|
||||
export * from './runListRoute';
|
||||
export * from './runEventListRoute';
|
||||
export * from './runStepListRoute';
|
||||
@@ -0,0 +1,130 @@
|
||||
import {
|
||||
BoundedRunStepListProjectionUnavailableError,
|
||||
InvalidBoundedRunStepListProjectionError,
|
||||
executeBoundedRunStepListProjection,
|
||||
} from '@qinglong/runtime-core/bounded-run-step-list-projection';
|
||||
import type { RunRepositoryReader } from '@qinglong/runtime-core/run-repository';
|
||||
import type { StepRunRepository } from '@qinglong/runtime-core/step-run';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_CONTROL_RUN_STEP_LIST_ROUTE = Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/runs/{runId}/steps',
|
||||
operationId: 'run.steps.list',
|
||||
permission: 'run.read',
|
||||
projectParameter: 'projectId',
|
||||
allowedQuery: Object.freeze(['after_step_key', 'after_step_run_id', 'limit']),
|
||||
});
|
||||
|
||||
const IDENTITY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function parseQuery(query: Readonly<Record<string, readonly string[]>>) {
|
||||
const stepKeyValues = query.after_step_key;
|
||||
const stepRunIdValues = query.after_step_run_id;
|
||||
const limitValues = query.limit;
|
||||
if (
|
||||
(stepKeyValues !== undefined && stepKeyValues.length !== 1) ||
|
||||
(stepRunIdValues !== undefined && stepRunIdValues.length !== 1) ||
|
||||
(limitValues !== undefined && limitValues.length !== 1) ||
|
||||
(stepKeyValues === undefined) !== (stepRunIdValues === undefined)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const stepKey = stepKeyValues?.[0];
|
||||
const stepRunId = stepRunIdValues?.[0];
|
||||
const rawLimit = limitValues?.[0];
|
||||
const limit = rawLimit === undefined ? undefined : Number(rawLimit);
|
||||
if (
|
||||
(stepKey !== undefined && !IDENTITY_PATTERN.test(stepKey)) ||
|
||||
(stepRunId !== undefined && !IDENTITY_PATTERN.test(stepRunId)) ||
|
||||
(rawLimit !== undefined &&
|
||||
(!Number.isSafeInteger(limit) ||
|
||||
Number(limit) < 1 ||
|
||||
Number(limit) > 64 ||
|
||||
String(limit) !== rawLimit))
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return Object.freeze({
|
||||
...(limit === undefined ? {} : { limit }),
|
||||
...(stepKey === undefined || stepRunId === undefined
|
||||
? {}
|
||||
: { after: Object.freeze({ stepKey, stepRunId }) }),
|
||||
});
|
||||
}
|
||||
|
||||
function validateRunStepListQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): void {
|
||||
parseQuery(query);
|
||||
}
|
||||
|
||||
export function createClusterControlRunStepListRoute(
|
||||
runs: Pick<RunRepositoryReader, 'findRunById'>,
|
||||
stepRuns: Pick<StepRunRepository, 'listByRun'>,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (
|
||||
!runs ||
|
||||
typeof runs.findRunById !== 'function' ||
|
||||
!stepRuns ||
|
||||
typeof stepRuns.listByRun !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster-control Run Step list repository is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_RUN_STEP_LIST_ROUTE,
|
||||
validateQuery: validateRunStepListQuery,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
if (authorized.projectId === null) {
|
||||
return response(503, { code: 'run_step_list_unavailable' });
|
||||
}
|
||||
let input;
|
||||
try {
|
||||
input = parseQuery(authorized.request.query);
|
||||
} catch {
|
||||
return response(400, { code: 'invalid_run_step_list_query' });
|
||||
}
|
||||
try {
|
||||
const result = await executeBoundedRunStepListProjection(
|
||||
runs,
|
||||
stepRuns,
|
||||
authorized.projectId,
|
||||
parameters.runId!,
|
||||
input,
|
||||
);
|
||||
if (!result.found) {
|
||||
return response(404, { code: 'run_not_found' });
|
||||
}
|
||||
const { found: _found, ...page } = result;
|
||||
return response(200, { ...page });
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof InvalidBoundedRunStepListProjectionError ||
|
||||
error instanceof BoundedRunStepListProjectionUnavailableError
|
||||
) {
|
||||
return response(503, { code: 'run_step_list_unavailable' });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
// Scheduling owns the bounded Cron expression adapter used by the Cluster cadence.
|
||||
import type {
|
||||
LocalCronNextOccurrence,
|
||||
LocalCronSchedule,
|
||||
} from '@qinglong/runtime-core/local-scheduler';
|
||||
|
||||
interface CronerJob {
|
||||
nextRun(after: Date): Date | null;
|
||||
stop(): void;
|
||||
}
|
||||
|
||||
interface CronerConstructor {
|
||||
new (
|
||||
expression: string,
|
||||
options: Readonly<{
|
||||
timezone: string;
|
||||
paused: true;
|
||||
unref: true;
|
||||
}>,
|
||||
): CronerJob;
|
||||
}
|
||||
|
||||
export const cronerClusterNextOccurrence: LocalCronNextOccurrence = (
|
||||
schedule: LocalCronSchedule,
|
||||
afterMs: number,
|
||||
): number => {
|
||||
let job: CronerJob | undefined;
|
||||
try {
|
||||
const { Cron } = require('croner') as Readonly<{
|
||||
Cron: CronerConstructor;
|
||||
}>;
|
||||
job = new Cron(schedule.expression, {
|
||||
timezone: schedule.timezone,
|
||||
paused: true,
|
||||
unref: true,
|
||||
});
|
||||
const next = job.nextRun(new Date(afterMs));
|
||||
if (!(next instanceof Date)) {
|
||||
throw new Error('cron has no next occurrence');
|
||||
}
|
||||
return next.getTime();
|
||||
} finally {
|
||||
job?.stop();
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,69 @@
|
||||
// Scheduling owns recovery and lost-retry ordering inside the shared cadence.
|
||||
import type {
|
||||
ClusterControlStartupRecoverySummary,
|
||||
ClusterRunLostRetryPageResult,
|
||||
} from '@qinglong/runtime-core';
|
||||
|
||||
import type {
|
||||
ClusterSchedulerCoordinator,
|
||||
ClusterSchedulerCycleSummary,
|
||||
} from './scheduler';
|
||||
|
||||
export interface ClusterRuntimeSchedulerMaintenanceSummary {
|
||||
readonly recovery: Readonly<ClusterControlStartupRecoverySummary>;
|
||||
readonly lostRetry: Readonly<ClusterRunLostRetryPageResult>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reuses the scheduler's single non-overlapping cadence for runtime recovery
|
||||
* and lost retry. It owns no timer, connection, cursor or per-Run state.
|
||||
*/
|
||||
export class ClusterRuntimeSchedulerCoordinator {
|
||||
private inFlight: Promise<ClusterSchedulerCycleSummary> | undefined;
|
||||
private latestMaintenance:
|
||||
| Readonly<ClusterRuntimeSchedulerMaintenanceSummary>
|
||||
| undefined;
|
||||
|
||||
constructor(
|
||||
private readonly recovery: Readonly<{
|
||||
reconcile(): Promise<ClusterControlStartupRecoverySummary>;
|
||||
}>,
|
||||
private readonly lostRetry: Readonly<{
|
||||
reconcile(): Promise<Readonly<ClusterRunLostRetryPageResult>>;
|
||||
}>,
|
||||
private readonly scheduler: Pick<
|
||||
ClusterSchedulerCoordinator,
|
||||
'scheduleOnce'
|
||||
>,
|
||||
) {
|
||||
if (
|
||||
typeof recovery?.reconcile !== 'function' ||
|
||||
typeof lostRetry?.reconcile !== 'function' ||
|
||||
typeof scheduler?.scheduleOnce !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster runtime scheduler coordinator is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
scheduleOnce(): Promise<ClusterSchedulerCycleSummary> {
|
||||
if (this.inFlight) return this.inFlight;
|
||||
const work = this.runCycle().finally(() => {
|
||||
if (this.inFlight === work) this.inFlight = undefined;
|
||||
});
|
||||
this.inFlight = work;
|
||||
return work;
|
||||
}
|
||||
|
||||
latestMaintenanceSummary():
|
||||
| Readonly<ClusterRuntimeSchedulerMaintenanceSummary>
|
||||
| undefined {
|
||||
return this.latestMaintenance;
|
||||
}
|
||||
|
||||
private async runCycle(): Promise<ClusterSchedulerCycleSummary> {
|
||||
const recovery = await this.recovery.reconcile();
|
||||
const lostRetry = await this.lostRetry.reconcile();
|
||||
this.latestMaintenance = Object.freeze({ recovery, lostRetry });
|
||||
return this.scheduler.scheduleOnce();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
// Scheduling owns bounded trigger claiming and the single non-overlapping lifecycle timer.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
MAX_CLUSTER_SCHEDULE_CLAIM_LEASE_MS,
|
||||
MIN_CLUSTER_SCHEDULE_CLAIM_LEASE_MS,
|
||||
resolveClusterScheduleDecision,
|
||||
type ClusterScheduleStore,
|
||||
} from '@qinglong/runtime-core/cluster-scheduler';
|
||||
import type { LocalCronNextOccurrence } from '@qinglong/runtime-core/local-scheduler';
|
||||
import { cronerClusterNextOccurrence } from './cronerSchedule';
|
||||
|
||||
export const MAX_CLUSTER_SCHEDULE_CLAIMS_PER_CYCLE = 256;
|
||||
|
||||
export interface ClusterSchedulerCoordinatorOptions {
|
||||
readonly ownerId: string;
|
||||
readonly claimLeaseMs?: number;
|
||||
readonly maxClaimsPerCycle?: number;
|
||||
readonly misfireGraceMs?: number;
|
||||
readonly createId?: () => string;
|
||||
readonly nextOccurrence?: LocalCronNextOccurrence;
|
||||
readonly onAdmitted?: (
|
||||
runId: string,
|
||||
attemptId: string,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ClusterSchedulerCycleSummary {
|
||||
readonly firstClaimAcquiredAtMs: number | null;
|
||||
readonly lastClaimAcquiredAtMs: number | null;
|
||||
readonly claimed: number;
|
||||
readonly initialized: number;
|
||||
readonly skipped: number;
|
||||
readonly admitted: number;
|
||||
readonly raced: number;
|
||||
readonly saturated: boolean;
|
||||
}
|
||||
|
||||
const OWNER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const COORDINATOR_OPTION_KEYS = new Set([
|
||||
'claimLeaseMs',
|
||||
'createId',
|
||||
'maxClaimsPerCycle',
|
||||
'misfireGraceMs',
|
||||
'nextOccurrence',
|
||||
'onAdmitted',
|
||||
'ownerId',
|
||||
]);
|
||||
|
||||
export class ClusterSchedulerCoordinator {
|
||||
private readonly ownerId: string;
|
||||
private readonly claimLeaseMs: number;
|
||||
private readonly maxClaimsPerCycle: number;
|
||||
private readonly misfireGraceMs: number;
|
||||
private readonly createId: () => string;
|
||||
private readonly nextOccurrence: LocalCronNextOccurrence;
|
||||
private readonly onAdmitted?: ClusterSchedulerCoordinatorOptions['onAdmitted'];
|
||||
|
||||
constructor(
|
||||
private readonly schedules: ClusterScheduleStore,
|
||||
options: ClusterSchedulerCoordinatorOptions,
|
||||
) {
|
||||
this.ownerId = options?.ownerId ?? '';
|
||||
this.claimLeaseMs = options?.claimLeaseMs ?? 30_000;
|
||||
this.maxClaimsPerCycle = options?.maxClaimsPerCycle ?? 16;
|
||||
this.misfireGraceMs = options?.misfireGraceMs ?? 30_000;
|
||||
this.createId = options?.createId ?? randomUUID;
|
||||
this.nextOccurrence =
|
||||
options?.nextOccurrence ?? cronerClusterNextOccurrence;
|
||||
this.onAdmitted = options?.onAdmitted;
|
||||
if (
|
||||
!schedules ||
|
||||
typeof schedules.claimNextClusterSchedule !== 'function' ||
|
||||
typeof schedules.commitClusterScheduleDecision !== 'function' ||
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => !COORDINATOR_OPTION_KEYS.has(key)) ||
|
||||
!OWNER_PATTERN.test(this.ownerId) ||
|
||||
!Number.isSafeInteger(this.claimLeaseMs) ||
|
||||
this.claimLeaseMs < MIN_CLUSTER_SCHEDULE_CLAIM_LEASE_MS ||
|
||||
this.claimLeaseMs > MAX_CLUSTER_SCHEDULE_CLAIM_LEASE_MS ||
|
||||
!Number.isSafeInteger(this.maxClaimsPerCycle) ||
|
||||
this.maxClaimsPerCycle < 1 ||
|
||||
this.maxClaimsPerCycle > MAX_CLUSTER_SCHEDULE_CLAIMS_PER_CYCLE ||
|
||||
!Number.isSafeInteger(this.misfireGraceMs) ||
|
||||
this.misfireGraceMs < 0 ||
|
||||
this.misfireGraceMs > 5 * 60_000 ||
|
||||
typeof this.createId !== 'function' ||
|
||||
typeof this.nextOccurrence !== 'function' ||
|
||||
(this.onAdmitted !== undefined && typeof this.onAdmitted !== 'function')
|
||||
) {
|
||||
throw new TypeError('Cluster scheduler coordinator options are invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async scheduleOnce(): Promise<ClusterSchedulerCycleSummary> {
|
||||
const stats: {
|
||||
firstClaimAcquiredAtMs: number | null;
|
||||
lastClaimAcquiredAtMs: number | null;
|
||||
claimed: number;
|
||||
initialized: number;
|
||||
skipped: number;
|
||||
admitted: number;
|
||||
raced: number;
|
||||
saturated: boolean;
|
||||
} = {
|
||||
firstClaimAcquiredAtMs: null,
|
||||
lastClaimAcquiredAtMs: null,
|
||||
claimed: 0,
|
||||
initialized: 0,
|
||||
skipped: 0,
|
||||
admitted: 0,
|
||||
raced: 0,
|
||||
saturated: false,
|
||||
};
|
||||
while (stats.claimed < this.maxClaimsPerCycle) {
|
||||
const claimToken = this.createId();
|
||||
const claimed = await this.schedules.claimNextClusterSchedule({
|
||||
ownerId: this.ownerId,
|
||||
claimToken,
|
||||
leaseMs: this.claimLeaseMs,
|
||||
});
|
||||
if (!claimed) break;
|
||||
if (
|
||||
claimed.claimOwner !== this.ownerId ||
|
||||
claimed.claimToken !== claimToken ||
|
||||
claimed.claimExpiresAtMs !==
|
||||
claimed.claimAcquiredAtMs + this.claimLeaseMs
|
||||
) {
|
||||
throw new TypeError('Cluster scheduler store returned a foreign claim');
|
||||
}
|
||||
stats.claimed += 1;
|
||||
stats.firstClaimAcquiredAtMs ??= claimed.claimAcquiredAtMs;
|
||||
stats.lastClaimAcquiredAtMs = claimed.claimAcquiredAtMs;
|
||||
const decision = resolveClusterScheduleDecision(
|
||||
claimed,
|
||||
this.misfireGraceMs,
|
||||
this.nextOccurrence,
|
||||
);
|
||||
const admitted = decision.disposition === 'admit';
|
||||
const result = await this.schedules.commitClusterScheduleDecision({
|
||||
claim: claimed,
|
||||
decision,
|
||||
...(admitted
|
||||
? {
|
||||
runId: this.createId(),
|
||||
attemptId: this.createId(),
|
||||
createdEventId: this.createId(),
|
||||
queuedEventId: this.createId(),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
if (result.status === 'raced') {
|
||||
stats.raced += 1;
|
||||
continue;
|
||||
}
|
||||
if (result.disposition === 'initialize') stats.initialized += 1;
|
||||
if (result.disposition === 'skip') stats.skipped += 1;
|
||||
if (result.status === 'admitted') {
|
||||
stats.admitted += 1;
|
||||
await this.onAdmitted?.(result.runId, result.attemptId);
|
||||
}
|
||||
}
|
||||
stats.saturated = stats.claimed === this.maxClaimsPerCycle;
|
||||
return Object.freeze(stats);
|
||||
}
|
||||
}
|
||||
|
||||
export interface ClusterSchedulerLifecycleOptions {
|
||||
readonly intervalMs: number;
|
||||
readonly stopTimeoutMs: number;
|
||||
readonly onDiagnostic?: (
|
||||
error: unknown,
|
||||
summary?: ClusterSchedulerCycleSummary,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ClusterSchedulerLifecycleStopSummary {
|
||||
readonly status: 'stopped' | 'timed_out';
|
||||
}
|
||||
|
||||
export class ClusterSchedulerLifecycle {
|
||||
private timer: NodeJS.Timeout | undefined;
|
||||
private inFlight: Promise<ClusterSchedulerCycleSummary> | undefined;
|
||||
private stopPromise:
|
||||
| Promise<ClusterSchedulerLifecycleStopSummary>
|
||||
| undefined;
|
||||
private running = false;
|
||||
private stopping = false;
|
||||
|
||||
constructor(
|
||||
private readonly scheduler: Pick<
|
||||
ClusterSchedulerCoordinator,
|
||||
'scheduleOnce'
|
||||
>,
|
||||
private readonly options: ClusterSchedulerLifecycleOptions,
|
||||
) {
|
||||
if (
|
||||
!scheduler ||
|
||||
typeof scheduler.scheduleOnce !== 'function' ||
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
!Number.isSafeInteger(options.intervalMs) ||
|
||||
options.intervalMs < 250 ||
|
||||
options.intervalMs > 60 * 60_000 ||
|
||||
!Number.isSafeInteger(options.stopTimeoutMs) ||
|
||||
options.stopTimeoutMs < 100 ||
|
||||
options.stopTimeoutMs > 30_000 ||
|
||||
(options.onDiagnostic !== undefined &&
|
||||
typeof options.onDiagnostic !== 'function')
|
||||
) {
|
||||
throw new TypeError('Cluster scheduler lifecycle options are invalid');
|
||||
}
|
||||
}
|
||||
|
||||
start(): 'started' {
|
||||
if (!this.running && !this.stopping) {
|
||||
this.running = true;
|
||||
this.schedule();
|
||||
}
|
||||
return 'started';
|
||||
}
|
||||
|
||||
runOnce(): Promise<ClusterSchedulerCycleSummary> {
|
||||
if (this.stopping) {
|
||||
return Promise.reject(
|
||||
new Error('Cluster scheduler lifecycle is stopping'),
|
||||
);
|
||||
}
|
||||
if (this.inFlight) return this.inFlight;
|
||||
const work = this.scheduler.scheduleOnce().finally(() => {
|
||||
if (this.inFlight === work) this.inFlight = undefined;
|
||||
});
|
||||
this.inFlight = work;
|
||||
return work;
|
||||
}
|
||||
|
||||
stopAndDrain(): Promise<ClusterSchedulerLifecycleStopSummary> {
|
||||
if (this.stopPromise) return this.stopPromise;
|
||||
this.stopping = true;
|
||||
this.running = false;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
this.timer = undefined;
|
||||
this.stopPromise = (async () => {
|
||||
const work = this.inFlight;
|
||||
if (!work) return Object.freeze({ status: 'stopped' as const });
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
work.then(
|
||||
() => Object.freeze({ status: 'stopped' as const }),
|
||||
() => Object.freeze({ status: 'stopped' as const }),
|
||||
),
|
||||
new Promise<ClusterSchedulerLifecycleStopSummary>((resolve) => {
|
||||
timeout = setTimeout(
|
||||
() => resolve(Object.freeze({ status: 'timed_out' as const })),
|
||||
this.options.stopTimeoutMs,
|
||||
);
|
||||
timeout.unref?.();
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
}
|
||||
})();
|
||||
return this.stopPromise;
|
||||
}
|
||||
|
||||
private schedule(): void {
|
||||
if (!this.running || this.timer) return;
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = undefined;
|
||||
if (!this.running) return;
|
||||
void this.runOnce()
|
||||
.then((summary) => this.diagnostic(undefined, summary))
|
||||
.catch((error) => this.diagnostic(error))
|
||||
.finally(() => this.schedule());
|
||||
}, this.options.intervalMs);
|
||||
this.timer.unref?.();
|
||||
}
|
||||
|
||||
private async diagnostic(
|
||||
error: unknown,
|
||||
summary?: ClusterSchedulerCycleSummary,
|
||||
): Promise<void> {
|
||||
if (this.stopping) return;
|
||||
try {
|
||||
await this.options.onDiagnostic?.(error, summary);
|
||||
} catch {
|
||||
// Diagnostics cannot own or stop scheduling.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
// Scheduling owns Workflow frontier and Task Attempt admission on the shared cadence.
|
||||
import type {
|
||||
PluginPackageWorkflowFrontierCursor,
|
||||
PluginPackageWorkflowFrontierRepository,
|
||||
} from '@qinglong/runtime-core/plugin-package-workflow-frontier';
|
||||
import type {
|
||||
PluginPackageWorkflowTaskAttemptAdmissionCursor,
|
||||
PluginPackageWorkflowTaskAttemptAdmissionRepository,
|
||||
} from '@qinglong/runtime-core/plugin-package-workflow-task-attempt-admission';
|
||||
|
||||
import type {
|
||||
ClusterSchedulerCoordinator,
|
||||
ClusterSchedulerCycleSummary,
|
||||
} from './scheduler';
|
||||
|
||||
export interface ClusterWorkflowSchedulerOptions {
|
||||
readonly frontierPageSize: number;
|
||||
readonly frontierMaxPages: number;
|
||||
readonly taskAttemptPageSize: number;
|
||||
readonly taskAttemptMaxPages: number;
|
||||
}
|
||||
|
||||
export interface ClusterWorkflowSchedulerCycleSummary {
|
||||
readonly frontierPages: number;
|
||||
readonly frontierScanned: number;
|
||||
readonly frontierAdvanced: number;
|
||||
readonly frontierTruncated: boolean;
|
||||
readonly taskAttemptPages: number;
|
||||
readonly taskAttemptsScanned: number;
|
||||
readonly taskAttemptsCreated: number;
|
||||
readonly taskAttemptsExisting: number;
|
||||
readonly taskAttemptsTruncated: boolean;
|
||||
}
|
||||
|
||||
function bounded(
|
||||
label: string,
|
||||
value: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
if (!Number.isSafeInteger(value) || value < 1 || value > maximum) {
|
||||
throw new RangeError(`${label} must be between 1 and ${maximum}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function nextFrontierCursor(
|
||||
current: PluginPackageWorkflowFrontierCursor | undefined,
|
||||
next: PluginPackageWorkflowFrontierCursor | undefined,
|
||||
): PluginPackageWorkflowFrontierCursor {
|
||||
if (
|
||||
!next ||
|
||||
(current !== undefined &&
|
||||
(next.admittedAtMs < current.admittedAtMs ||
|
||||
(next.admittedAtMs === current.admittedAtMs &&
|
||||
next.planDigest <= current.planDigest)))
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster Workflow frontier continuation did not advance',
|
||||
);
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function nextTaskAttemptCursor(
|
||||
current: PluginPackageWorkflowTaskAttemptAdmissionCursor | undefined,
|
||||
next: PluginPackageWorkflowTaskAttemptAdmissionCursor | undefined,
|
||||
): PluginPackageWorkflowTaskAttemptAdmissionCursor {
|
||||
if (
|
||||
!next ||
|
||||
(current !== undefined &&
|
||||
(next.readyAtMs < current.readyAtMs ||
|
||||
(next.readyAtMs === current.readyAtMs &&
|
||||
next.stepRunId <= current.stepRunId)))
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster Workflow Task Attempt continuation did not advance',
|
||||
);
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extends the existing Cluster Scheduler cadence with Workflow frontier and
|
||||
* Task Attempt admission. It owns no timer, connection, watcher, or
|
||||
* per-Workflow state.
|
||||
*/
|
||||
export class ClusterWorkflowSchedulerCoordinator {
|
||||
private readonly frontierPageSize: number;
|
||||
private readonly frontierMaxPages: number;
|
||||
private readonly taskAttemptPageSize: number;
|
||||
private readonly taskAttemptMaxPages: number;
|
||||
private inFlight: Promise<ClusterSchedulerCycleSummary> | undefined;
|
||||
private latestWorkflow:
|
||||
| Readonly<ClusterWorkflowSchedulerCycleSummary>
|
||||
| undefined;
|
||||
|
||||
constructor(
|
||||
private readonly scheduler: Pick<
|
||||
ClusterSchedulerCoordinator,
|
||||
'scheduleOnce'
|
||||
>,
|
||||
private readonly frontier: PluginPackageWorkflowFrontierRepository,
|
||||
private readonly taskAttempts: PluginPackageWorkflowTaskAttemptAdmissionRepository,
|
||||
options: ClusterWorkflowSchedulerOptions,
|
||||
) {
|
||||
if (
|
||||
typeof scheduler?.scheduleOnce !== 'function' ||
|
||||
typeof frontier?.listCandidates !== 'function' ||
|
||||
typeof frontier?.advance !== 'function' ||
|
||||
typeof taskAttempts?.listCandidates !== 'function' ||
|
||||
typeof taskAttempts?.admit !== 'function' ||
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options)
|
||||
) {
|
||||
throw new TypeError('Cluster Workflow scheduler is invalid');
|
||||
}
|
||||
this.frontierPageSize = bounded(
|
||||
'Cluster Workflow frontier page size',
|
||||
options.frontierPageSize,
|
||||
64,
|
||||
);
|
||||
this.frontierMaxPages = bounded(
|
||||
'Cluster Workflow frontier page limit',
|
||||
options.frontierMaxPages,
|
||||
16,
|
||||
);
|
||||
this.taskAttemptPageSize = bounded(
|
||||
'Cluster Workflow Task Attempt page size',
|
||||
options.taskAttemptPageSize,
|
||||
64,
|
||||
);
|
||||
this.taskAttemptMaxPages = bounded(
|
||||
'Cluster Workflow Task Attempt page limit',
|
||||
options.taskAttemptMaxPages,
|
||||
16,
|
||||
);
|
||||
}
|
||||
|
||||
scheduleOnce(): Promise<ClusterSchedulerCycleSummary> {
|
||||
if (this.inFlight) return this.inFlight;
|
||||
const work = this.runCycle().finally(() => {
|
||||
if (this.inFlight === work) this.inFlight = undefined;
|
||||
});
|
||||
this.inFlight = work;
|
||||
return work;
|
||||
}
|
||||
|
||||
latestWorkflowSummary():
|
||||
| Readonly<ClusterWorkflowSchedulerCycleSummary>
|
||||
| undefined {
|
||||
return this.latestWorkflow;
|
||||
}
|
||||
|
||||
private async runCycle(): Promise<ClusterSchedulerCycleSummary> {
|
||||
const scheduler = await this.scheduler.scheduleOnce();
|
||||
const frontier = await this.advanceFrontier();
|
||||
const taskAttempts = await this.admitTaskAttempts();
|
||||
this.latestWorkflow = Object.freeze({
|
||||
...frontier,
|
||||
...taskAttempts,
|
||||
});
|
||||
return scheduler;
|
||||
}
|
||||
|
||||
private async advanceFrontier(): Promise<Readonly<{
|
||||
frontierPages: number;
|
||||
frontierScanned: number;
|
||||
frontierAdvanced: number;
|
||||
frontierTruncated: boolean;
|
||||
}>> {
|
||||
let frontierPages = 0;
|
||||
let frontierScanned = 0;
|
||||
let frontierAdvanced = 0;
|
||||
let frontierTruncated = false;
|
||||
let after: PluginPackageWorkflowFrontierCursor | undefined;
|
||||
for (let index = 0; index < this.frontierMaxPages; index += 1) {
|
||||
const page = await this.frontier.listCandidates({
|
||||
limit: this.frontierPageSize,
|
||||
...(after === undefined ? {} : { after }),
|
||||
});
|
||||
if (page.candidates.length > this.frontierPageSize) {
|
||||
throw new RangeError(
|
||||
'Cluster Workflow frontier exceeded its page size',
|
||||
);
|
||||
}
|
||||
frontierPages += 1;
|
||||
frontierScanned += page.candidates.length;
|
||||
for (const candidate of page.candidates) {
|
||||
await this.frontier.advance(candidate.runId);
|
||||
frontierAdvanced += 1;
|
||||
}
|
||||
frontierTruncated = page.truncated;
|
||||
if (!page.truncated) break;
|
||||
after = nextFrontierCursor(after, page.next);
|
||||
}
|
||||
return Object.freeze({
|
||||
frontierPages,
|
||||
frontierScanned,
|
||||
frontierAdvanced,
|
||||
frontierTruncated,
|
||||
});
|
||||
}
|
||||
|
||||
private async admitTaskAttempts(): Promise<Readonly<{
|
||||
taskAttemptPages: number;
|
||||
taskAttemptsScanned: number;
|
||||
taskAttemptsCreated: number;
|
||||
taskAttemptsExisting: number;
|
||||
taskAttemptsTruncated: boolean;
|
||||
}>> {
|
||||
let taskAttemptPages = 0;
|
||||
let taskAttemptsScanned = 0;
|
||||
let taskAttemptsCreated = 0;
|
||||
let taskAttemptsExisting = 0;
|
||||
let taskAttemptsTruncated = false;
|
||||
let after:
|
||||
| PluginPackageWorkflowTaskAttemptAdmissionCursor
|
||||
| undefined;
|
||||
for (let index = 0; index < this.taskAttemptMaxPages; index += 1) {
|
||||
const page = await this.taskAttempts.listCandidates({
|
||||
limit: this.taskAttemptPageSize,
|
||||
...(after === undefined ? {} : { after }),
|
||||
});
|
||||
if (page.candidates.length > this.taskAttemptPageSize) {
|
||||
throw new RangeError(
|
||||
'Cluster Workflow Task Attempt source exceeded its page size',
|
||||
);
|
||||
}
|
||||
taskAttemptPages += 1;
|
||||
taskAttemptsScanned += page.candidates.length;
|
||||
for (const candidate of page.candidates) {
|
||||
const admitted = await this.taskAttempts.admit(
|
||||
candidate.runId,
|
||||
candidate.stepRunId,
|
||||
);
|
||||
if (admitted.status === 'created') taskAttemptsCreated += 1;
|
||||
else taskAttemptsExisting += 1;
|
||||
}
|
||||
taskAttemptsTruncated = page.truncated;
|
||||
if (!page.truncated) break;
|
||||
after = nextTaskAttemptCursor(after, page.next);
|
||||
}
|
||||
return Object.freeze({
|
||||
taskAttemptPages,
|
||||
taskAttemptsScanned,
|
||||
taskAttemptsCreated,
|
||||
taskAttemptsExisting,
|
||||
taskAttemptsTruncated,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import {
|
||||
BoundedTaskListProjectionUnavailableError,
|
||||
InvalidBoundedTaskListProjectionError,
|
||||
executeBoundedTaskListProjection,
|
||||
} from '@qinglong/runtime-core/bounded-task-list-projection';
|
||||
import type { TaskDefinitionSource } from '@qinglong/runtime-core/task-definition';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
} from '../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_CONTROL_TASK_LIST_ROUTE = Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/tasks',
|
||||
operationId: 'task.list',
|
||||
permission: 'task.read',
|
||||
projectParameter: 'projectId',
|
||||
allowedQuery: Object.freeze(['after_task_id', 'limit']),
|
||||
});
|
||||
|
||||
const TASK_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
function parseQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): Readonly<{
|
||||
limit?: number;
|
||||
after?: Readonly<{ taskId: string }>;
|
||||
}> {
|
||||
const limitValues = query.limit;
|
||||
const taskIdValues = query.after_task_id;
|
||||
if (
|
||||
(limitValues !== undefined && limitValues.length !== 1) ||
|
||||
(taskIdValues !== undefined && taskIdValues.length !== 1)
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
const rawLimit = limitValues?.[0];
|
||||
const limit = rawLimit === undefined ? undefined : Number(rawLimit);
|
||||
const taskId = taskIdValues?.[0];
|
||||
if (
|
||||
(rawLimit !== undefined &&
|
||||
(!Number.isSafeInteger(limit) ||
|
||||
Number(limit) < 1 ||
|
||||
Number(limit) > 64 ||
|
||||
String(limit) !== rawLimit)) ||
|
||||
(taskId !== undefined && !TASK_ID_PATTERN.test(taskId))
|
||||
) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return Object.freeze({
|
||||
...(limit === undefined ? {} : { limit }),
|
||||
...(taskId === undefined
|
||||
? {}
|
||||
: { after: Object.freeze({ taskId }) }),
|
||||
});
|
||||
}
|
||||
|
||||
function validateTaskListQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
): void {
|
||||
parseQuery(query);
|
||||
}
|
||||
|
||||
export function createClusterControlTaskListRoute(
|
||||
tasks: Pick<TaskDefinitionSource, 'listTaskDefinitions'>,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (!tasks || typeof tasks.listTaskDefinitions !== 'function') {
|
||||
throw new TypeError('Cluster-control Task list repository is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_TASK_LIST_ROUTE,
|
||||
validateQuery: validateTaskListQuery,
|
||||
async handle(authorized: ClusterControlAuthorizedOperationRequest) {
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
if (authorized.projectId === null) {
|
||||
return response(503, { code: 'task_list_unavailable' });
|
||||
}
|
||||
let input;
|
||||
try {
|
||||
input = parseQuery(authorized.request.query);
|
||||
} catch {
|
||||
return response(400, { code: 'invalid_task_list_query' });
|
||||
}
|
||||
try {
|
||||
const result = await executeBoundedTaskListProjection(
|
||||
tasks,
|
||||
authorized.projectId,
|
||||
input,
|
||||
);
|
||||
return response(200, { ...result });
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof InvalidBoundedTaskListProjectionError ||
|
||||
error instanceof BoundedTaskListProjectionUnavailableError
|
||||
) {
|
||||
return response(503, { code: 'task_list_unavailable' });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export * from './taskReadRoute';
|
||||
export * from './taskStartRoute';
|
||||
@@ -0,0 +1,70 @@
|
||||
import {
|
||||
BoundedTaskReadProjectionUnavailableError,
|
||||
InvalidBoundedTaskReadProjectionError,
|
||||
executeBoundedTaskReadProjection,
|
||||
} from '@qinglong/runtime-core/bounded-task-read-projection';
|
||||
import type { TaskDefinitionSource } from '@qinglong/runtime-core/task-definition';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_CONTROL_TASK_READ_ROUTE = Object.freeze({
|
||||
method: 'GET' as const,
|
||||
path: '/api/v3/projects/{projectId}/tasks/{taskId}',
|
||||
operationId: 'task.get',
|
||||
permission: 'task.read',
|
||||
projectParameter: 'projectId',
|
||||
});
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
export function createClusterControlTaskReadRoute(
|
||||
tasks: Pick<TaskDefinitionSource, 'findCurrentTaskDefinition'>,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (!tasks || typeof tasks.findCurrentTaskDefinition !== 'function') {
|
||||
throw new TypeError('Cluster-control Task read repository is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_TASK_READ_ROUTE,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
if (authorized.request.body !== null) {
|
||||
return response(400, { code: 'invalid_request_body' });
|
||||
}
|
||||
if (authorized.projectId === null) {
|
||||
return response(503, { code: 'task_query_unavailable' });
|
||||
}
|
||||
try {
|
||||
const projection = await executeBoundedTaskReadProjection(
|
||||
tasks,
|
||||
authorized.projectId,
|
||||
parameters.taskId!,
|
||||
);
|
||||
if (projection.found !== true) {
|
||||
return response(404, { code: 'task_not_found' });
|
||||
}
|
||||
const { found: _found, ...task } = projection;
|
||||
return response(200, { task: Object.freeze(task) });
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof InvalidBoundedTaskReadProjectionError ||
|
||||
error instanceof BoundedTaskReadProjectionUnavailableError
|
||||
) {
|
||||
return response(503, { code: 'task_query_unavailable' });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
import {
|
||||
TASK_START_SCHEMA,
|
||||
InvalidTaskStartError,
|
||||
TaskStartFenceRejectedError,
|
||||
TaskStartNotFoundError,
|
||||
TaskStartUnavailableError,
|
||||
createTaskStartResponseBody,
|
||||
parseTaskStartRequestBody,
|
||||
type TaskStartRepository,
|
||||
} from '@qinglong/runtime-core/task-start';
|
||||
|
||||
import type { ClusterControlAdmissionResponse } from '../transport/httpSurface';
|
||||
import type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRouteDefinition,
|
||||
ClusterControlRouteParameters,
|
||||
} from '../transport/routeRegistry';
|
||||
|
||||
export const CLUSTER_CONTROL_TASK_START_ROUTE = Object.freeze({
|
||||
method: 'POST' as const,
|
||||
path: '/api/v3/projects/{projectId}/tasks/{taskId}/runs',
|
||||
operationId: 'task.start',
|
||||
permission: 'run.start',
|
||||
projectParameter: 'projectId',
|
||||
});
|
||||
|
||||
export type ClusterTaskStartIdFactory = () => string;
|
||||
|
||||
function response(
|
||||
statusCode: number,
|
||||
body: Readonly<Record<string, unknown>>,
|
||||
): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body: Object.freeze(body) });
|
||||
}
|
||||
|
||||
export function createClusterControlTaskStartRoute(
|
||||
repository: TaskStartRepository,
|
||||
createId: ClusterTaskStartIdFactory,
|
||||
): Readonly<ClusterControlRouteDefinition> {
|
||||
if (
|
||||
!repository ||
|
||||
typeof repository.startTask !== 'function' ||
|
||||
typeof createId !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster-control Task start route is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
...CLUSTER_CONTROL_TASK_START_ROUTE,
|
||||
async handle(
|
||||
authorized: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
) {
|
||||
let body;
|
||||
try {
|
||||
body = parseTaskStartRequestBody(authorized.request.body);
|
||||
} catch (error) {
|
||||
if (error instanceof InvalidTaskStartError) {
|
||||
return response(400, {
|
||||
code: 'invalid_task_start_request',
|
||||
schema: TASK_START_SCHEMA,
|
||||
});
|
||||
}
|
||||
return response(503, { code: 'task_start_unavailable' });
|
||||
}
|
||||
const projectId = authorized.projectId;
|
||||
const taskId = parameters.taskId;
|
||||
if (
|
||||
projectId === null ||
|
||||
typeof taskId !== 'string' ||
|
||||
taskId.length < 1 ||
|
||||
!authorized.policyFence ||
|
||||
authorized.policyFence.bindingVersion === null
|
||||
) {
|
||||
return response(503, { code: 'task_start_unavailable' });
|
||||
}
|
||||
try {
|
||||
const result = await repository.startTask({
|
||||
projectId,
|
||||
taskId,
|
||||
mutationId: body.mutationId,
|
||||
expectedRevision: body.expectedRevision,
|
||||
expectedContentDigest: body.expectedContentDigest,
|
||||
runId: createId(),
|
||||
attemptId: createId(),
|
||||
createdEventId: createId(),
|
||||
queuedEventId: createId(),
|
||||
subject: authorized.principal.subject,
|
||||
policyFence: authorized.policyFence,
|
||||
});
|
||||
return response(
|
||||
result.status === 'accepted' ? 202 : 200,
|
||||
createTaskStartResponseBody(result),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof TaskStartNotFoundError) {
|
||||
return response(404, { code: 'task_not_found' });
|
||||
}
|
||||
if (error instanceof TaskStartFenceRejectedError) {
|
||||
return response(409, {
|
||||
code: 'task_start_fence_rejected',
|
||||
reason: error.reason,
|
||||
});
|
||||
}
|
||||
if (
|
||||
error instanceof InvalidTaskStartError ||
|
||||
error instanceof TaskStartUnavailableError
|
||||
) {
|
||||
return response(503, { code: 'task_start_unavailable' });
|
||||
}
|
||||
return response(503, { code: 'task_start_unavailable' });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
// Transport owns authenticated, Policy-fenced and synchronously audited admission.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
normalizeSecurityPolicyDecision,
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPolicyDecision,
|
||||
type SecurityPrincipal,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
import {
|
||||
normalizeSecurityAuditRecord,
|
||||
type SecurityAuditOutcome,
|
||||
type SecurityAuditRecord,
|
||||
type SecurityAuditSink,
|
||||
} from '@qinglong/runtime-core/security-audit';
|
||||
import {
|
||||
ProjectPolicyEngine,
|
||||
normalizeProjectPermission,
|
||||
type ProjectPolicyRepository,
|
||||
} from '@qinglong/runtime-core/project-policy';
|
||||
import type {
|
||||
ClusterControlAdmissionMetadata,
|
||||
ClusterControlAdmissionPipeline,
|
||||
} from './httpSurface';
|
||||
import {
|
||||
ClusterControlRouteResolutionError,
|
||||
isClusterControlRouteRegistry,
|
||||
type ClusterControlAuthorizedOperationRequest,
|
||||
type ClusterControlRoute,
|
||||
type ClusterControlRouteRegistry,
|
||||
} from './routeRegistry';
|
||||
|
||||
export type {
|
||||
ClusterControlAuthorizedOperationRequest,
|
||||
ClusterControlRoute,
|
||||
ClusterControlRouteRegistry,
|
||||
} from './routeRegistry';
|
||||
|
||||
export interface ClusterControlRequestAuthenticator {
|
||||
authenticate(
|
||||
request: ClusterControlAdmissionMetadata,
|
||||
): SecurityPrincipal | null | Promise<SecurityPrincipal | null>;
|
||||
}
|
||||
|
||||
export interface ClusterControlPolicyRequest {
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly operationId: string;
|
||||
readonly permission: string;
|
||||
readonly projectId: string | null;
|
||||
readonly signal: AbortSignal;
|
||||
}
|
||||
|
||||
export interface ClusterControlPolicyAuthorizer {
|
||||
authorize(
|
||||
request: ClusterControlPolicyRequest,
|
||||
): SecurityPolicyDecision | Promise<SecurityPolicyDecision>;
|
||||
}
|
||||
|
||||
export type ClusterControlSecurityAuditOutcome = SecurityAuditOutcome;
|
||||
export type ClusterControlSecurityAuditRecord = SecurityAuditRecord;
|
||||
export type ClusterControlSecurityAuditSink = SecurityAuditSink;
|
||||
|
||||
export interface ClusterControlAdmissionPipelineOptions {
|
||||
readonly routes: ClusterControlRouteRegistry;
|
||||
readonly authenticator: ClusterControlRequestAuthenticator;
|
||||
readonly policy: ClusterControlPolicyAuthorizer;
|
||||
readonly audit: ClusterControlSecurityAuditSink;
|
||||
readonly now?: () => number;
|
||||
}
|
||||
|
||||
export class ClusterControlAdmissionSecurityError extends Error {
|
||||
constructor(
|
||||
readonly statusCode: number,
|
||||
readonly code: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ClusterControlAdmissionSecurityError';
|
||||
}
|
||||
}
|
||||
|
||||
/** Adapts the shared Project Policy engine to cluster admission. */
|
||||
export function createClusterControlProjectPolicyAuthorizer(
|
||||
repository: ProjectPolicyRepository,
|
||||
): ClusterControlPolicyAuthorizer {
|
||||
const engine = new ProjectPolicyEngine(repository);
|
||||
return Object.freeze({
|
||||
authorize(request: ClusterControlPolicyRequest) {
|
||||
if (request.projectId === null) {
|
||||
return Object.freeze({
|
||||
effect: 'deny' as const,
|
||||
reasons: Object.freeze(['project_scope_required']),
|
||||
fence: null,
|
||||
});
|
||||
}
|
||||
return engine.authorize(
|
||||
request.principal,
|
||||
request.projectId,
|
||||
normalizeProjectPermission(request.permission),
|
||||
);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function securityError(
|
||||
statusCode: number,
|
||||
code: string,
|
||||
message: string,
|
||||
): ClusterControlAdmissionSecurityError {
|
||||
return new ClusterControlAdmissionSecurityError(statusCode, code, message);
|
||||
}
|
||||
|
||||
async function recordSecurityAudit(
|
||||
audit: ClusterControlSecurityAuditSink,
|
||||
record: Omit<SecurityAuditRecord, 'eventId' | 'occurredAtMs'>,
|
||||
now: () => number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await audit.record(
|
||||
normalizeSecurityAuditRecord({
|
||||
...record,
|
||||
eventId: randomUUID(),
|
||||
occurredAtMs: now(),
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
throw securityError(
|
||||
503,
|
||||
'security_audit_unavailable',
|
||||
'Cluster-control security audit is unavailable',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a fail-closed, two-phase admission pipeline. Route matching,
|
||||
* authentication, policy evaluation and durable security audit all complete
|
||||
* before the returned operation is allowed to receive a request body.
|
||||
*/
|
||||
export function createClusterControlAdmissionPipeline(
|
||||
options: ClusterControlAdmissionPipelineOptions,
|
||||
): ClusterControlAdmissionPipeline {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
!isClusterControlRouteRegistry(options.routes) ||
|
||||
typeof options.authenticator?.authenticate !== 'function' ||
|
||||
typeof options.policy?.authorize !== 'function' ||
|
||||
typeof options.audit?.record !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function')
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Cluster-control admission pipeline options are invalid',
|
||||
);
|
||||
}
|
||||
const now = options.now ?? Date.now;
|
||||
|
||||
return Object.freeze({
|
||||
async prepare(request: ClusterControlAdmissionMetadata) {
|
||||
let route: ClusterControlRoute;
|
||||
try {
|
||||
const resolved = await options.routes.resolve(request);
|
||||
if (!resolved) {
|
||||
throw securityError(
|
||||
404,
|
||||
'route_not_found',
|
||||
'Cluster-control route was not found',
|
||||
);
|
||||
}
|
||||
route = resolved;
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof ClusterControlAdmissionSecurityError ||
|
||||
error instanceof ClusterControlRouteResolutionError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw securityError(
|
||||
503,
|
||||
'route_resolution_unavailable',
|
||||
'Cluster-control route resolution is unavailable',
|
||||
);
|
||||
}
|
||||
|
||||
let candidate: SecurityPrincipal | null;
|
||||
try {
|
||||
candidate = await options.authenticator.authenticate(request);
|
||||
} catch {
|
||||
await recordSecurityAudit(
|
||||
options.audit,
|
||||
{
|
||||
requestId: request.requestId,
|
||||
operationId: route.operationId,
|
||||
projectId: route.projectId,
|
||||
subject: null,
|
||||
authenticationId: null,
|
||||
outcome: 'authentication_unavailable',
|
||||
reasons: Object.freeze(['authentication_unavailable']),
|
||||
fence: null,
|
||||
},
|
||||
now,
|
||||
);
|
||||
throw securityError(
|
||||
503,
|
||||
'authentication_unavailable',
|
||||
'Cluster-control authentication is unavailable',
|
||||
);
|
||||
}
|
||||
if (!candidate) {
|
||||
await recordSecurityAudit(
|
||||
options.audit,
|
||||
{
|
||||
requestId: request.requestId,
|
||||
operationId: route.operationId,
|
||||
projectId: route.projectId,
|
||||
subject: null,
|
||||
authenticationId: null,
|
||||
outcome: 'authentication_rejected',
|
||||
reasons: Object.freeze(['authentication_rejected']),
|
||||
fence: null,
|
||||
},
|
||||
now,
|
||||
);
|
||||
throw securityError(
|
||||
401,
|
||||
'authentication_required',
|
||||
'Cluster-control authentication is required',
|
||||
);
|
||||
}
|
||||
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(candidate, now());
|
||||
} catch {
|
||||
await recordSecurityAudit(
|
||||
options.audit,
|
||||
{
|
||||
requestId: request.requestId,
|
||||
operationId: route.operationId,
|
||||
projectId: route.projectId,
|
||||
subject: null,
|
||||
authenticationId: null,
|
||||
outcome: 'authentication_unavailable',
|
||||
reasons: Object.freeze(['invalid_principal']),
|
||||
fence: null,
|
||||
},
|
||||
now,
|
||||
);
|
||||
throw securityError(
|
||||
503,
|
||||
'authentication_unavailable',
|
||||
'Cluster-control authentication is unavailable',
|
||||
);
|
||||
}
|
||||
|
||||
let decision: Readonly<SecurityPolicyDecision>;
|
||||
try {
|
||||
decision = normalizeSecurityPolicyDecision(
|
||||
await options.policy.authorize(
|
||||
Object.freeze({
|
||||
principal,
|
||||
operationId: route.operationId,
|
||||
permission: route.permission,
|
||||
projectId: route.projectId,
|
||||
signal: request.signal,
|
||||
}),
|
||||
),
|
||||
);
|
||||
} catch {
|
||||
await recordSecurityAudit(
|
||||
options.audit,
|
||||
{
|
||||
requestId: request.requestId,
|
||||
operationId: route.operationId,
|
||||
projectId: route.projectId,
|
||||
subject: principal.subject,
|
||||
authenticationId: principal.authenticationId,
|
||||
outcome: 'authorization_unavailable',
|
||||
reasons: Object.freeze(['authorization_unavailable']),
|
||||
fence: null,
|
||||
},
|
||||
now,
|
||||
);
|
||||
throw securityError(
|
||||
503,
|
||||
'authorization_unavailable',
|
||||
'Cluster-control authorization is unavailable',
|
||||
);
|
||||
}
|
||||
|
||||
const outcome =
|
||||
decision.effect === 'allow'
|
||||
? 'allowed'
|
||||
: decision.effect === 'require_approval'
|
||||
? 'approval_required'
|
||||
: 'denied';
|
||||
await recordSecurityAudit(
|
||||
options.audit,
|
||||
{
|
||||
requestId: request.requestId,
|
||||
operationId: route.operationId,
|
||||
projectId: route.projectId,
|
||||
subject: principal.subject,
|
||||
authenticationId: principal.authenticationId,
|
||||
outcome,
|
||||
reasons: decision.reasons,
|
||||
fence: decision.fence,
|
||||
},
|
||||
now,
|
||||
);
|
||||
if (decision.effect === 'deny') {
|
||||
throw securityError(
|
||||
403,
|
||||
'forbidden',
|
||||
'Cluster-control operation is forbidden',
|
||||
);
|
||||
}
|
||||
if (decision.effect === 'require_approval') {
|
||||
throw securityError(
|
||||
403,
|
||||
'approval_required',
|
||||
'Cluster-control operation requires approval',
|
||||
);
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
handle(body: unknown | null) {
|
||||
return route.handle(
|
||||
Object.freeze({
|
||||
request: Object.freeze({ ...request, body }),
|
||||
principal,
|
||||
operationId: route.operationId,
|
||||
permission: route.permission,
|
||||
projectId: route.projectId,
|
||||
policyFence: decision.fence,
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,452 @@
|
||||
// Transport owns bounded route compilation, resolution and query validation.
|
||||
import type {
|
||||
SecurityPolicyFence,
|
||||
SecurityPrincipal,
|
||||
} from '@qinglong/runtime-core/security';
|
||||
import type {
|
||||
ClusterControlAdmissionMetadata,
|
||||
ClusterControlAdmissionRequest,
|
||||
ClusterControlAdmissionResponse,
|
||||
ClusterControlHttpMethod,
|
||||
} from './httpSurface';
|
||||
|
||||
export const CLUSTER_CONTROL_ROUTE_REGISTRY_LIMITS = Object.freeze({
|
||||
maxRoutes: 256,
|
||||
maxPathBytes: 1024,
|
||||
maxPathSegments: 16,
|
||||
maxPathParameters: 8,
|
||||
maxQueryParameters: 16,
|
||||
maxQueryValuesPerParameter: 16,
|
||||
maxQueryValueBytes: 1024,
|
||||
});
|
||||
|
||||
export type ClusterControlRouteParameters = Readonly<Record<string, string>>;
|
||||
|
||||
export interface ClusterControlAuthorizedOperationRequest {
|
||||
readonly request: ClusterControlAdmissionRequest;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly operationId: string;
|
||||
readonly permission: string;
|
||||
readonly projectId: string | null;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence> | null;
|
||||
}
|
||||
|
||||
export interface ClusterControlRouteDefinition {
|
||||
readonly method: ClusterControlHttpMethod;
|
||||
readonly path: string;
|
||||
readonly operationId: string;
|
||||
readonly permission: string;
|
||||
readonly projectParameter: string | null;
|
||||
readonly allowedQuery?: readonly string[];
|
||||
readonly validateQuery?: (
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
) => void;
|
||||
handle(
|
||||
request: ClusterControlAuthorizedOperationRequest,
|
||||
parameters: ClusterControlRouteParameters,
|
||||
): ClusterControlAdmissionResponse | Promise<ClusterControlAdmissionResponse>;
|
||||
}
|
||||
|
||||
export interface ClusterControlRoute {
|
||||
readonly operationId: string;
|
||||
readonly permission: string;
|
||||
readonly projectId: string | null;
|
||||
handle(
|
||||
request: ClusterControlAuthorizedOperationRequest,
|
||||
): ClusterControlAdmissionResponse | Promise<ClusterControlAdmissionResponse>;
|
||||
}
|
||||
|
||||
export interface ClusterControlRouteResolver {
|
||||
resolve(request: ClusterControlAdmissionMetadata): ClusterControlRoute | null;
|
||||
}
|
||||
|
||||
export interface ClusterControlRouteRegistry
|
||||
extends ClusterControlRouteResolver {
|
||||
readonly contractVersion: 1;
|
||||
readonly size: number;
|
||||
}
|
||||
|
||||
export class ClusterControlRouteRegistryConfigurationError extends TypeError {
|
||||
constructor(message: string) {
|
||||
super(`Cluster-control route registry is invalid: ${message}`);
|
||||
this.name = 'ClusterControlRouteRegistryConfigurationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusterControlRouteResolutionError extends Error {
|
||||
constructor(
|
||||
readonly statusCode: 400,
|
||||
readonly code: 'invalid_route_path' | 'invalid_route_query',
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ClusterControlRouteResolutionError';
|
||||
}
|
||||
}
|
||||
|
||||
type CompiledSegment =
|
||||
| { readonly kind: 'literal'; readonly value: string }
|
||||
| { readonly kind: 'parameter'; readonly name: string };
|
||||
|
||||
interface CompiledRoute {
|
||||
readonly method: ClusterControlHttpMethod;
|
||||
readonly operationId: string;
|
||||
readonly permission: string;
|
||||
readonly projectParameter: string | null;
|
||||
readonly segments: readonly CompiledSegment[];
|
||||
readonly allowedQuery: ReadonlySet<string>;
|
||||
readonly validateQuery?: ClusterControlRouteDefinition['validateQuery'];
|
||||
readonly handle: ClusterControlRouteDefinition['handle'];
|
||||
}
|
||||
|
||||
const HTTP_METHODS = new Set<ClusterControlHttpMethod>([
|
||||
'DELETE',
|
||||
'GET',
|
||||
'PATCH',
|
||||
'POST',
|
||||
'PUT',
|
||||
]);
|
||||
const OPERATION_PATTERN = /^[a-z][a-z0-9_.:-]{0,127}$/;
|
||||
const PERMISSION_PATTERN = /^[a-z][a-z0-9_.:*:-]{0,127}$/;
|
||||
const LITERAL_SEGMENT_PATTERN = /^[a-z][a-z0-9-]{0,63}$/;
|
||||
const PARAMETER_NAME_PATTERN = /^[a-z][A-Za-z0-9]{0,63}$/;
|
||||
const PARAMETER_SEGMENT_PATTERN = /^\{([a-z][A-Za-z0-9]{0,63})\}$/;
|
||||
const PARAMETER_VALUE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const QUERY_NAME_PATTERN = /^[a-z][a-z0-9_]{0,63}$/;
|
||||
const CONTROL_CHARACTER_PATTERN = /[\u0000-\u001f\u007f]/;
|
||||
const DEFINITION_KEYS = new Set([
|
||||
'allowedQuery',
|
||||
'handle',
|
||||
'method',
|
||||
'operationId',
|
||||
'path',
|
||||
'permission',
|
||||
'projectParameter',
|
||||
'validateQuery',
|
||||
]);
|
||||
const reviewedRegistries = new WeakSet<object>();
|
||||
|
||||
function configurationError(
|
||||
message: string,
|
||||
): ClusterControlRouteRegistryConfigurationError {
|
||||
return new ClusterControlRouteRegistryConfigurationError(message);
|
||||
}
|
||||
|
||||
function exactDefinitionShape(value: ClusterControlRouteDefinition): void {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw configurationError('each route must be an object');
|
||||
}
|
||||
const keys = Object.keys(value);
|
||||
if (
|
||||
keys.some((key) => !DEFINITION_KEYS.has(key)) ||
|
||||
!keys.includes('method') ||
|
||||
!keys.includes('path') ||
|
||||
!keys.includes('operationId') ||
|
||||
!keys.includes('permission') ||
|
||||
!keys.includes('projectParameter') ||
|
||||
!keys.includes('handle')
|
||||
) {
|
||||
throw configurationError('route shape is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function compilePath(path: string): readonly CompiledSegment[] {
|
||||
if (
|
||||
typeof path !== 'string' ||
|
||||
Buffer.byteLength(path) >
|
||||
CLUSTER_CONTROL_ROUTE_REGISTRY_LIMITS.maxPathBytes ||
|
||||
!path.startsWith('/api/v3/') ||
|
||||
path.endsWith('/') ||
|
||||
path.includes('//') ||
|
||||
path.includes('%') ||
|
||||
path.includes('\\') ||
|
||||
CONTROL_CHARACTER_PATTERN.test(path)
|
||||
) {
|
||||
throw configurationError('route path must be a canonical /api/v3 path');
|
||||
}
|
||||
const rawSegments = path.slice(1).split('/');
|
||||
if (
|
||||
rawSegments.length > CLUSTER_CONTROL_ROUTE_REGISTRY_LIMITS.maxPathSegments
|
||||
) {
|
||||
throw configurationError('route path has too many segments');
|
||||
}
|
||||
const parameterNames = new Set<string>();
|
||||
const compiled = rawSegments.map((segment): CompiledSegment => {
|
||||
const parameter = PARAMETER_SEGMENT_PATTERN.exec(segment)?.[1];
|
||||
if (parameter) {
|
||||
if (parameterNames.has(parameter)) {
|
||||
throw configurationError('route path repeats a parameter');
|
||||
}
|
||||
parameterNames.add(parameter);
|
||||
return Object.freeze({ kind: 'parameter', name: parameter });
|
||||
}
|
||||
if (!LITERAL_SEGMENT_PATTERN.test(segment)) {
|
||||
throw configurationError('route path contains an invalid segment');
|
||||
}
|
||||
return Object.freeze({ kind: 'literal', value: segment });
|
||||
});
|
||||
if (
|
||||
parameterNames.size >
|
||||
CLUSTER_CONTROL_ROUTE_REGISTRY_LIMITS.maxPathParameters
|
||||
) {
|
||||
throw configurationError('route path has too many parameters');
|
||||
}
|
||||
return Object.freeze(compiled);
|
||||
}
|
||||
|
||||
function compileAllowedQuery(
|
||||
value: readonly string[] | undefined,
|
||||
): ReadonlySet<string> {
|
||||
if (value === undefined) return new Set<string>();
|
||||
if (
|
||||
!Array.isArray(value) ||
|
||||
value.length > CLUSTER_CONTROL_ROUTE_REGISTRY_LIMITS.maxQueryParameters
|
||||
) {
|
||||
throw configurationError('allowedQuery is invalid');
|
||||
}
|
||||
const names = new Set<string>();
|
||||
for (const name of value) {
|
||||
if (!QUERY_NAME_PATTERN.test(name) || names.has(name)) {
|
||||
throw configurationError('allowedQuery contains an invalid name');
|
||||
}
|
||||
names.add(name);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
function compileRoute(
|
||||
definition: ClusterControlRouteDefinition,
|
||||
): CompiledRoute {
|
||||
exactDefinitionShape(definition);
|
||||
if (!HTTP_METHODS.has(definition.method)) {
|
||||
throw configurationError('route method is invalid');
|
||||
}
|
||||
if (!OPERATION_PATTERN.test(definition.operationId)) {
|
||||
throw configurationError('route operationId is invalid');
|
||||
}
|
||||
if (!PERMISSION_PATTERN.test(definition.permission)) {
|
||||
throw configurationError('route permission is invalid');
|
||||
}
|
||||
if (typeof definition.handle !== 'function') {
|
||||
throw configurationError('route handler is invalid');
|
||||
}
|
||||
if (
|
||||
definition.validateQuery !== undefined &&
|
||||
typeof definition.validateQuery !== 'function'
|
||||
) {
|
||||
throw configurationError('route query validator is invalid');
|
||||
}
|
||||
const segments = compilePath(definition.path);
|
||||
const parameterNames = new Set(
|
||||
segments.flatMap((segment) =>
|
||||
segment.kind === 'parameter' ? [segment.name] : [],
|
||||
),
|
||||
);
|
||||
if (
|
||||
definition.projectParameter !== null &&
|
||||
(!PARAMETER_NAME_PATTERN.test(definition.projectParameter) ||
|
||||
!parameterNames.has(definition.projectParameter))
|
||||
) {
|
||||
throw configurationError(
|
||||
'projectParameter must name one declared path parameter',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
method: definition.method,
|
||||
operationId: definition.operationId,
|
||||
permission: definition.permission,
|
||||
projectParameter: definition.projectParameter,
|
||||
segments,
|
||||
allowedQuery: compileAllowedQuery(definition.allowedQuery),
|
||||
...(definition.validateQuery === undefined
|
||||
? {}
|
||||
: { validateQuery: definition.validateQuery }),
|
||||
handle: definition.handle,
|
||||
});
|
||||
}
|
||||
|
||||
function routesOverlap(left: CompiledRoute, right: CompiledRoute): boolean {
|
||||
if (
|
||||
left.method !== right.method ||
|
||||
left.segments.length !== right.segments.length
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return left.segments.every((segment, index) => {
|
||||
const other = right.segments[index]!;
|
||||
return (
|
||||
segment.kind === 'parameter' ||
|
||||
other.kind === 'parameter' ||
|
||||
segment.value === other.value
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function validateRequestPath(path: string): readonly string[] {
|
||||
if (
|
||||
typeof path !== 'string' ||
|
||||
Buffer.byteLength(path) >
|
||||
CLUSTER_CONTROL_ROUTE_REGISTRY_LIMITS.maxPathBytes ||
|
||||
!(path === '/api/v3' || path.startsWith('/api/v3/')) ||
|
||||
path.endsWith('/') ||
|
||||
path.includes('//') ||
|
||||
path.includes('%') ||
|
||||
path.includes('\\') ||
|
||||
CONTROL_CHARACTER_PATTERN.test(path)
|
||||
) {
|
||||
throw new ClusterControlRouteResolutionError(
|
||||
400,
|
||||
'invalid_route_path',
|
||||
'Cluster-control route path is invalid',
|
||||
);
|
||||
}
|
||||
const segments = path.slice(1).split('/');
|
||||
if (
|
||||
segments.length > CLUSTER_CONTROL_ROUTE_REGISTRY_LIMITS.maxPathSegments ||
|
||||
segments.some((segment) => !PARAMETER_VALUE_PATTERN.test(segment))
|
||||
) {
|
||||
throw new ClusterControlRouteResolutionError(
|
||||
400,
|
||||
'invalid_route_path',
|
||||
'Cluster-control route path is invalid',
|
||||
);
|
||||
}
|
||||
return segments;
|
||||
}
|
||||
|
||||
function validateQuery(
|
||||
query: Readonly<Record<string, readonly string[]>>,
|
||||
allowed: ReadonlySet<string>,
|
||||
): void {
|
||||
if (!query || typeof query !== 'object' || Array.isArray(query)) {
|
||||
throw new ClusterControlRouteResolutionError(
|
||||
400,
|
||||
'invalid_route_query',
|
||||
'Cluster-control route query is invalid',
|
||||
);
|
||||
}
|
||||
const names = Object.keys(query);
|
||||
for (const name of names) {
|
||||
const values = query[name];
|
||||
if (
|
||||
!allowed.has(name) ||
|
||||
!Array.isArray(values) ||
|
||||
values.length === 0 ||
|
||||
values.length >
|
||||
CLUSTER_CONTROL_ROUTE_REGISTRY_LIMITS.maxQueryValuesPerParameter ||
|
||||
values.some(
|
||||
(value) =>
|
||||
typeof value !== 'string' ||
|
||||
Buffer.byteLength(value) >
|
||||
CLUSTER_CONTROL_ROUTE_REGISTRY_LIMITS.maxQueryValueBytes ||
|
||||
CONTROL_CHARACTER_PATTERN.test(value),
|
||||
)
|
||||
) {
|
||||
throw new ClusterControlRouteResolutionError(
|
||||
400,
|
||||
'invalid_route_query',
|
||||
'Cluster-control route query is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function matchRoute(
|
||||
route: CompiledRoute,
|
||||
method: ClusterControlHttpMethod,
|
||||
segments: readonly string[],
|
||||
): ClusterControlRouteParameters | null {
|
||||
if (route.method !== method || route.segments.length !== segments.length) {
|
||||
return null;
|
||||
}
|
||||
const parameters = Object.create(null) as Record<string, string>;
|
||||
for (let index = 0; index < segments.length; index += 1) {
|
||||
const definition = route.segments[index]!;
|
||||
const value = segments[index]!;
|
||||
if (definition.kind === 'literal') {
|
||||
if (definition.value !== value) return null;
|
||||
} else {
|
||||
parameters[definition.name] = value;
|
||||
}
|
||||
}
|
||||
return Object.freeze(parameters);
|
||||
}
|
||||
|
||||
/** Returns true only for an object created by the reviewed registry factory. */
|
||||
export function isClusterControlRouteRegistry(
|
||||
value: unknown,
|
||||
): value is ClusterControlRouteRegistry {
|
||||
return (
|
||||
!!value &&
|
||||
typeof value === 'object' &&
|
||||
reviewedRegistries.has(value as object)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compiles a bounded, immutable and non-overlapping route table. Route-owned
|
||||
* operation, permission and Project scope are resolved before authentication.
|
||||
*/
|
||||
export function createClusterControlRouteRegistry(
|
||||
definitions: readonly ClusterControlRouteDefinition[],
|
||||
): ClusterControlRouteRegistry {
|
||||
if (
|
||||
!Array.isArray(definitions) ||
|
||||
definitions.length > CLUSTER_CONTROL_ROUTE_REGISTRY_LIMITS.maxRoutes
|
||||
) {
|
||||
throw configurationError('definitions must be a bounded array');
|
||||
}
|
||||
const routes = definitions.map(compileRoute);
|
||||
const operationIds = new Set<string>();
|
||||
for (let index = 0; index < routes.length; index += 1) {
|
||||
const route = routes[index]!;
|
||||
if (operationIds.has(route.operationId)) {
|
||||
throw configurationError('operationId must be unique');
|
||||
}
|
||||
operationIds.add(route.operationId);
|
||||
for (let otherIndex = 0; otherIndex < index; otherIndex += 1) {
|
||||
if (routesOverlap(route, routes[otherIndex]!)) {
|
||||
throw configurationError('route definitions overlap');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const registry: ClusterControlRouteRegistry = {
|
||||
contractVersion: 1,
|
||||
size: routes.length,
|
||||
resolve(request) {
|
||||
const segments = validateRequestPath(request.path);
|
||||
for (const route of routes) {
|
||||
const parameters = matchRoute(route, request.method, segments);
|
||||
if (!parameters) continue;
|
||||
validateQuery(request.query, route.allowedQuery);
|
||||
if (route.validateQuery) {
|
||||
try {
|
||||
route.validateQuery(request.query);
|
||||
} catch {
|
||||
throw new ClusterControlRouteResolutionError(
|
||||
400,
|
||||
'invalid_route_query',
|
||||
'Cluster-control route query is invalid',
|
||||
);
|
||||
}
|
||||
}
|
||||
const projectId =
|
||||
route.projectParameter === null
|
||||
? null
|
||||
: parameters[route.projectParameter]!;
|
||||
return Object.freeze({
|
||||
operationId: route.operationId,
|
||||
permission: route.permission,
|
||||
projectId,
|
||||
handle(request: ClusterControlAuthorizedOperationRequest) {
|
||||
return route.handle(request, parameters);
|
||||
},
|
||||
});
|
||||
}
|
||||
return null;
|
||||
},
|
||||
};
|
||||
reviewedRegistries.add(registry);
|
||||
return Object.freeze(registry);
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
// Cluster Control Worker Ingress boundary; keep production PostgreSQL composition explicit.
|
||||
import {
|
||||
assertPostgresWorkerIngressSchemaReady,
|
||||
PostgresSecurityAuditRepository,
|
||||
PostgresWorkerCredentialRepository,
|
||||
PostgresWorkerExecutionAttestationRepository,
|
||||
PostgresWorkerSessionRepository,
|
||||
} from '@qinglong/cluster-postgres/worker-ingress';
|
||||
import {
|
||||
startClusterWorkerIngressApplication,
|
||||
type ClusterWorkerIngressApplicationResult,
|
||||
} from './workerIngressApplication';
|
||||
import {
|
||||
createClusterWorkerIngressDatabaseOpener,
|
||||
createClusterWorkerIngressHttpOptions,
|
||||
type EnabledClusterWorkerIngressConfig,
|
||||
} from './workerIngressConfig';
|
||||
import {
|
||||
createWorkerCredentialAuthenticator,
|
||||
} from './workerCredentialAuthenticator';
|
||||
import {
|
||||
createWorkerIngressAdmissionPipeline,
|
||||
} from './workerIngressPipeline';
|
||||
import type { ClusterWorkerRuntimePort } from '../remote-execution/workerRuntimePort';
|
||||
|
||||
export interface ProductionClusterWorkerIngressOptions {
|
||||
readonly config: EnabledClusterWorkerIngressConfig;
|
||||
readonly runtime: ClusterWorkerRuntimePort;
|
||||
readonly onPoolError?: (error: Error) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the reviewed Worker-facing listener. The worker-ingress Pool is used
|
||||
* only for authentication, Session, attestation and audit authority. Every
|
||||
* Run/Attempt/Lease mutation crosses the injected runtime capability port.
|
||||
*/
|
||||
export async function startProductionClusterWorkerIngress(
|
||||
options: ProductionClusterWorkerIngressOptions,
|
||||
): Promise<ClusterWorkerIngressApplicationResult> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
!options.config?.enabled ||
|
||||
!options.runtime
|
||||
) {
|
||||
throw new TypeError('Production Worker ingress options are invalid');
|
||||
}
|
||||
if (
|
||||
options.onPoolError !== undefined &&
|
||||
typeof options.onPoolError !== 'function'
|
||||
) {
|
||||
throw new TypeError('Production Worker ingress Pool error sink is invalid');
|
||||
}
|
||||
const http = await createClusterWorkerIngressHttpOptions(options.config);
|
||||
const openDatabase = createClusterWorkerIngressDatabaseOpener(
|
||||
options.config,
|
||||
(error) => options.onPoolError?.(error),
|
||||
);
|
||||
return startClusterWorkerIngressApplication({
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
workerCredentialPepper:
|
||||
options.config.security.workerCredentialPepper,
|
||||
openDatabase,
|
||||
http,
|
||||
async create({ database, workerCredentialPepper }) {
|
||||
const report = await assertPostgresWorkerIngressSchemaReady(
|
||||
database.pool,
|
||||
);
|
||||
return Object.freeze({
|
||||
evidence: Object.freeze({
|
||||
contractName: report.contractName,
|
||||
contractVersion: report.contractVersion,
|
||||
serverMajor: report.serverMajor,
|
||||
migrationIds: Object.freeze([...report.migrationIds]),
|
||||
}),
|
||||
pipeline: createWorkerIngressAdmissionPipeline({
|
||||
authenticator: createWorkerCredentialAuthenticator(
|
||||
new PostgresWorkerCredentialRepository(database.pool),
|
||||
workerCredentialPepper,
|
||||
),
|
||||
workers: new PostgresWorkerSessionRepository(database.pool),
|
||||
attestations: new PostgresWorkerExecutionAttestationRepository(
|
||||
database.pool,
|
||||
),
|
||||
audit: new PostgresSecurityAuditRepository(database.pool),
|
||||
offers: options.runtime.offers,
|
||||
activation: options.runtime.activation,
|
||||
...(options.runtime.secrets === undefined
|
||||
? {}
|
||||
: { secrets: options.runtime.secrets }),
|
||||
artifacts: options.runtime.artifacts,
|
||||
completion: options.runtime.completion,
|
||||
leaseControl: options.runtime.leaseControl,
|
||||
}),
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Cluster Control Worker Ingress boundary; keep Worker credential authentication explicit.
|
||||
import { timingSafeEqual } from 'node:crypto';
|
||||
import {
|
||||
WorkerCredentialUnavailableError,
|
||||
normalizeWorkerCredentialRecord,
|
||||
type WorkerCredentialRepository,
|
||||
} from '@qinglong/runtime-core/worker-credential';
|
||||
import {
|
||||
assertWorkerCredentialPepper,
|
||||
workerCredentialSecretDigest,
|
||||
} from '@qinglong/runtime-core/worker-credential-token';
|
||||
import type { ClusterControlAdmissionMetadata } from '../transport/httpSurface';
|
||||
|
||||
export interface AuthenticatedWorkerPrincipal {
|
||||
readonly workerId: string;
|
||||
readonly credentialId: string;
|
||||
readonly credentialVersion: number;
|
||||
readonly authenticationId: string;
|
||||
readonly authenticatedAtMs: number;
|
||||
readonly expiresAtMs: number;
|
||||
}
|
||||
|
||||
export interface WorkerCredentialAuthenticator {
|
||||
authenticate(
|
||||
metadata: ClusterControlAdmissionMetadata,
|
||||
): Promise<Readonly<AuthenticatedWorkerPrincipal> | null>;
|
||||
}
|
||||
|
||||
const AUTHORIZATION =
|
||||
/^Worker ql3w_([A-Za-z0-9][A-Za-z0-9._:-]{0,63})_([A-Za-z0-9_-]{43})$/;
|
||||
|
||||
export function createWorkerCredentialAuthenticator(
|
||||
repository: WorkerCredentialRepository,
|
||||
pepper: string,
|
||||
options: Readonly<{ now?: () => number; principalTtlMs?: number }> = {},
|
||||
): WorkerCredentialAuthenticator {
|
||||
if (!repository || typeof repository.resolve !== 'function') {
|
||||
throw new TypeError('Worker credential authenticator repository is invalid');
|
||||
}
|
||||
assertWorkerCredentialPepper(pepper);
|
||||
const now = options.now ?? Date.now;
|
||||
const principalTtlMs = options.principalTtlMs ?? 60_000;
|
||||
if (
|
||||
!Number.isSafeInteger(principalTtlMs) ||
|
||||
principalTtlMs < 1_000 ||
|
||||
principalTtlMs > 300_000
|
||||
) {
|
||||
throw new RangeError('Worker credential principal TTL is invalid');
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
async authenticate(metadata: ClusterControlAdmissionMetadata) {
|
||||
const header = metadata.headers.authorization;
|
||||
if (typeof header !== 'string') return null;
|
||||
const match = AUTHORIZATION.exec(header);
|
||||
if (!match) return null;
|
||||
let presented: Buffer | undefined;
|
||||
try {
|
||||
presented = Buffer.from(
|
||||
workerCredentialSecretDigest(pepper, match[1]!, match[2]!),
|
||||
'hex',
|
||||
);
|
||||
const candidate = await repository.resolve(match[1]!);
|
||||
if (metadata.signal.aborted) throw new WorkerCredentialUnavailableError();
|
||||
const record = candidate ? normalizeWorkerCredentialRecord(candidate) : null;
|
||||
const stored = record
|
||||
? Buffer.from(record.secretDigest, 'hex')
|
||||
: Buffer.alloc(32);
|
||||
const matches = timingSafeEqual(presented, stored);
|
||||
stored.fill(0);
|
||||
if (!record || !matches) return null;
|
||||
const nowMs = now();
|
||||
if (
|
||||
!Number.isSafeInteger(nowMs) ||
|
||||
nowMs < 0 ||
|
||||
record.state !== 'active' ||
|
||||
record.notBeforeAtMs > nowMs ||
|
||||
record.expiresAtMs <= nowMs
|
||||
) return null;
|
||||
return Object.freeze({
|
||||
workerId: record.workerId,
|
||||
credentialId: record.credentialId,
|
||||
credentialVersion: record.version,
|
||||
authenticationId: `worker_credential:${record.credentialId}:${record.version}`,
|
||||
authenticatedAtMs: nowMs,
|
||||
expiresAtMs: Math.min(record.expiresAtMs, nowMs + principalTtlMs),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof WorkerCredentialUnavailableError) throw error;
|
||||
throw new WorkerCredentialUnavailableError();
|
||||
} finally {
|
||||
presented?.fill(0);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
// Cluster Control Worker Ingress boundary; keep listener lifecycle authority explicit.
|
||||
import type {
|
||||
ClusterControlReadinessEvidence,
|
||||
ClusterControlAdmissionDisposer,
|
||||
DeploymentProfile,
|
||||
OpenPostgresDatabase,
|
||||
PostgresDatabaseResource,
|
||||
} from '@qinglong/runtime-core';
|
||||
import { assertWorkerCredentialPepper } from '@qinglong/runtime-core/worker-credential-token';
|
||||
import { MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES } from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import {
|
||||
startClusterControlHttpSurface,
|
||||
type ClusterControlAdmissionPipeline,
|
||||
type ClusterControlHttpAddress,
|
||||
type ClusterControlHttpSurfaceOptions,
|
||||
type ClusterControlMutualTlsOptions,
|
||||
} from '../transport/httpSurface';
|
||||
|
||||
export interface ClusterWorkerIngressAssemblyInput {
|
||||
readonly database: PostgresDatabaseResource;
|
||||
readonly workerCredentialPepper: string;
|
||||
}
|
||||
|
||||
export interface ClusterWorkerIngressAssembly {
|
||||
readonly evidence: ClusterControlReadinessEvidence;
|
||||
readonly pipeline: ClusterControlAdmissionPipeline;
|
||||
}
|
||||
|
||||
export interface ClusterWorkerIngressApplicationOptions {
|
||||
readonly enabled?: boolean;
|
||||
readonly profile: DeploymentProfile;
|
||||
readonly workerCredentialPepper?: string;
|
||||
readonly openDatabase: OpenPostgresDatabase;
|
||||
readonly http: ClusterControlHttpSurfaceOptions;
|
||||
readonly create: (
|
||||
input: ClusterWorkerIngressAssemblyInput,
|
||||
) => ClusterWorkerIngressAssembly | Promise<ClusterWorkerIngressAssembly>;
|
||||
}
|
||||
|
||||
export type ClusterWorkerIngressApplicationResult =
|
||||
| { readonly status: 'disabled'; stop(): Promise<'stopped'> }
|
||||
| {
|
||||
readonly status: 'active';
|
||||
readonly protocol: 'https';
|
||||
readonly transport: 'mutual-tls';
|
||||
readonly address: ClusterControlHttpAddress;
|
||||
readonly evidence: ClusterControlReadinessEvidence;
|
||||
reloadTransport(options: ClusterControlMutualTlsOptions): number;
|
||||
stop(): Promise<'stopped'>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Separate Worker-facing composition root. It owns a dedicated listener and a
|
||||
* worker-ingress database resource. Storage readiness and repositories are
|
||||
* supplied by the outer composition root, so this transport layer cannot
|
||||
* acquire Project Policy, dispatch, recovery-claim or DDL authority itself.
|
||||
*/
|
||||
export async function startClusterWorkerIngressApplication(
|
||||
options: ClusterWorkerIngressApplicationOptions,
|
||||
): Promise<ClusterWorkerIngressApplicationResult> {
|
||||
if (!(options.enabled ?? false)) {
|
||||
return Object.freeze({
|
||||
status: 'disabled',
|
||||
async stop() {
|
||||
return 'stopped' as const;
|
||||
},
|
||||
});
|
||||
}
|
||||
if (options.profile !== 'cluster-control') {
|
||||
throw new TypeError('Worker ingress requires cluster-control profile');
|
||||
}
|
||||
if (typeof options.create !== 'function') {
|
||||
throw new TypeError('Worker ingress assembly factory is required');
|
||||
}
|
||||
if (!options.http?.mutualTls) {
|
||||
throw new TypeError('Worker ingress requires mutual TLS');
|
||||
}
|
||||
assertWorkerCredentialPepper(options.workerCredentialPepper ?? '');
|
||||
const bodyLimit = options.http.maxBodyBytes ?? 64 * 1024;
|
||||
if (
|
||||
!Number.isSafeInteger(bodyLimit) ||
|
||||
bodyLimit < 1024 ||
|
||||
bodyLimit > 64 * 1024
|
||||
) {
|
||||
throw new RangeError(
|
||||
'Worker ingress body limit must be between 1 KiB and 64 KiB',
|
||||
);
|
||||
}
|
||||
|
||||
let database: PostgresDatabaseResource | undefined;
|
||||
const http = await startClusterControlHttpSurface({
|
||||
...options.http,
|
||||
maxBodyBytes: bodyLimit,
|
||||
maxResponseBytes: Math.min(
|
||||
options.http.maxResponseBytes ?? MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES,
|
||||
MAX_REMOTE_SECRET_DELIVERY_RESPONSE_BYTES,
|
||||
),
|
||||
maxInFlightRequests: Math.min(options.http.maxInFlightRequests ?? 64, 256),
|
||||
});
|
||||
let disposeAdmission: ClusterControlAdmissionDisposer | undefined;
|
||||
try {
|
||||
database = await options.openDatabase();
|
||||
const assembly = await options.create({
|
||||
database,
|
||||
workerCredentialPepper: options.workerCredentialPepper!,
|
||||
});
|
||||
disposeAdmission = http.installAdmission(
|
||||
assembly.evidence,
|
||||
assembly.pipeline,
|
||||
);
|
||||
let stopPromise: Promise<'stopped'> | undefined;
|
||||
return Object.freeze({
|
||||
status: 'active' as const,
|
||||
protocol: 'https' as const,
|
||||
transport: 'mutual-tls' as const,
|
||||
address: http.address,
|
||||
evidence: assembly.evidence,
|
||||
reloadTransport(mutualTls: ClusterControlMutualTlsOptions) {
|
||||
return http.reloadMutualTls(mutualTls);
|
||||
},
|
||||
stop() {
|
||||
stopPromise ??= (async () => {
|
||||
let primary: unknown;
|
||||
try {
|
||||
await disposeAdmission?.();
|
||||
} catch (error) {
|
||||
primary = error;
|
||||
}
|
||||
try {
|
||||
await database?.close();
|
||||
} catch (error) {
|
||||
primary ??= error;
|
||||
}
|
||||
try {
|
||||
await http.close();
|
||||
} catch (error) {
|
||||
primary ??= error;
|
||||
}
|
||||
if (primary) throw primary;
|
||||
return 'stopped' as const;
|
||||
})();
|
||||
return stopPromise;
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await disposeAdmission?.();
|
||||
} catch {
|
||||
/* preserve root */
|
||||
}
|
||||
try {
|
||||
await database?.close();
|
||||
} catch {
|
||||
/* preserve root */
|
||||
}
|
||||
try {
|
||||
await http.close();
|
||||
} catch {
|
||||
/* preserve root */
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export * from './workerCredentialAuthenticator';
|
||||
export * from './workerIngressPipeline';
|
||||
export * from '../remote-execution/remoteRunActivationService';
|
||||
export * from '../remote-execution/remoteWorkerSecretDeliveryService';
|
||||
export * from '../remote-execution/remoteWorkerCompletionService';
|
||||
export * from '../remote-execution/remoteWorkerLeaseControlService';
|
||||
@@ -0,0 +1,864 @@
|
||||
// Cluster Control Worker Ingress boundary; keep fail-closed deployment configuration explicit.
|
||||
import {
|
||||
createPrivateKey,
|
||||
createPublicKey,
|
||||
timingSafeEqual,
|
||||
X509Certificate,
|
||||
type KeyObject,
|
||||
} from 'node:crypto';
|
||||
import { constants } from 'node:fs';
|
||||
import { open } from 'node:fs/promises';
|
||||
import { isAbsolute } from 'node:path';
|
||||
import type {
|
||||
DeploymentProfile,
|
||||
OpenPostgresDatabase,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
createPostgresDatabaseOpener,
|
||||
isPostgresTlsDnsServername,
|
||||
loadPostgresConnectionEnvironment,
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
type PostgresConnectionOptions,
|
||||
type PostgresPoolOptions,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
import type {
|
||||
ClusterControlHttpSurfaceOptions,
|
||||
ClusterControlMutualTlsOptions,
|
||||
} from '../transport/httpSurface';
|
||||
|
||||
export type ClusterWorkerIngressEnvironment = Readonly<
|
||||
Record<string, string | undefined>
|
||||
>;
|
||||
|
||||
export interface DisabledClusterWorkerIngressConfig {
|
||||
readonly enabled: false;
|
||||
readonly profile: DeploymentProfile;
|
||||
}
|
||||
|
||||
export interface ClusterWorkerArtifactS3Config {
|
||||
readonly bucket: string;
|
||||
readonly region: string;
|
||||
readonly prefix?: string;
|
||||
readonly expectedBucketOwner?: string;
|
||||
readonly endpoint?: string;
|
||||
readonly forcePathStyle: boolean;
|
||||
readonly encryption:
|
||||
| Readonly<{ readonly mode: 's3' }>
|
||||
| Readonly<{ readonly mode: 'kms'; readonly keyId: string }>;
|
||||
}
|
||||
|
||||
export interface ClusterWorkerMountedSecretConfig {
|
||||
readonly provider: 'mounted-files';
|
||||
readonly rootDirectory: string;
|
||||
}
|
||||
|
||||
export interface EnabledClusterWorkerIngressConfig {
|
||||
readonly enabled: true;
|
||||
readonly profile: 'cluster-control';
|
||||
readonly http: Omit<ClusterControlHttpSurfaceOptions, 'mutualTls'>;
|
||||
readonly transport: Readonly<{
|
||||
readonly privateKeyFile: string;
|
||||
readonly certificateFile: string;
|
||||
readonly clientCertificateAuthorityFile: string;
|
||||
readonly clientCertificateRevocationListFile?: string;
|
||||
}>;
|
||||
readonly database: Readonly<{
|
||||
readonly connection: PostgresConnectionOptions;
|
||||
readonly pool: PostgresPoolOptions;
|
||||
}>;
|
||||
readonly security: Readonly<{
|
||||
readonly workerCredentialPepper: string;
|
||||
}>;
|
||||
readonly artifact: Readonly<ClusterWorkerArtifactS3Config>;
|
||||
readonly secret?: Readonly<ClusterWorkerMountedSecretConfig>;
|
||||
}
|
||||
|
||||
export type ClusterWorkerIngressConfig =
|
||||
| DisabledClusterWorkerIngressConfig
|
||||
| EnabledClusterWorkerIngressConfig;
|
||||
|
||||
export class ClusterWorkerIngressConfigError extends TypeError {
|
||||
constructor(message: string) {
|
||||
super(`Worker ingress configuration is invalid: ${message}`);
|
||||
this.name = 'ClusterWorkerIngressConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
const PROFILES = new Set<DeploymentProfile>([
|
||||
'edge',
|
||||
'standalone',
|
||||
'cluster-control',
|
||||
'worker',
|
||||
]);
|
||||
const MAX_TLS_FILE_BYTES = 1024 * 1024;
|
||||
|
||||
function booleanValue(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
name: string,
|
||||
defaultValue: boolean,
|
||||
): boolean {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return defaultValue;
|
||||
if (value === 'true') return true;
|
||||
if (value === 'false') return false;
|
||||
throw new ClusterWorkerIngressConfigError(`${name} must be true or false`);
|
||||
}
|
||||
|
||||
function integerValue(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
name: string,
|
||||
defaultValue: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return defaultValue;
|
||||
if (!/^\d+$/.test(value)) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} must be an integer`);
|
||||
}
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
`${name} must be between ${minimum} and ${maximum}`,
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function boundedValue(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
name: string,
|
||||
maximumLength: number,
|
||||
required = false,
|
||||
): string | undefined {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') {
|
||||
if (required) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} is required`);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
if (value.length > maximumLength || /[\0\r\n]/.test(value)) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function deploymentProfile(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): DeploymentProfile {
|
||||
const value = environment.QL_DEPLOYMENT_PROFILE ?? 'standalone';
|
||||
if (!PROFILES.has(value as DeploymentProfile)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL_DEPLOYMENT_PROFILE is invalid',
|
||||
);
|
||||
}
|
||||
return value as DeploymentProfile;
|
||||
}
|
||||
|
||||
function absoluteFile(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
name: string,
|
||||
): string {
|
||||
const value = boundedValue(environment, name, 4096, true)!;
|
||||
if (!isAbsolute(value)) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} must be absolute`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function optionalAbsoluteFile(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
name: string,
|
||||
): string | undefined {
|
||||
const value = boundedValue(environment, name, 4096);
|
||||
if (value === undefined) return undefined;
|
||||
if (!isAbsolute(value)) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} must be absolute`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function workerCredentialPepper(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): string {
|
||||
const value = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_CREDENTIAL_PEPPER',
|
||||
64,
|
||||
true,
|
||||
)!;
|
||||
if (!/^[A-Za-z0-9_-]{43}$/.test(value)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_CREDENTIAL_PEPPER must be canonical base64url for 32 bytes',
|
||||
);
|
||||
}
|
||||
const decoded = Buffer.from(value, 'base64url');
|
||||
const canonical =
|
||||
decoded.byteLength === 32 && decoded.toString('base64url') === value;
|
||||
decoded.fill(0);
|
||||
if (!canonical) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_CREDENTIAL_PEPPER must be canonical base64url for 32 bytes',
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function databaseConnection(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): PostgresConnectionOptions {
|
||||
let connection: PostgresConnectionOptions;
|
||||
try {
|
||||
connection = loadPostgresConnectionEnvironment(environment, {
|
||||
connectionString: 'QL3_POSTGRES_WORKER_INGRESS_URL',
|
||||
host: 'QL3_POSTGRES_WORKER_INGRESS_HOST',
|
||||
port: 'QL3_POSTGRES_WORKER_INGRESS_PORT',
|
||||
database: 'QL3_POSTGRES_WORKER_INGRESS_DATABASE',
|
||||
user: 'QL3_POSTGRES_WORKER_INGRESS_USER',
|
||||
password: 'QL3_POSTGRES_WORKER_INGRESS_PASSWORD',
|
||||
});
|
||||
} catch (error) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: 'PostgreSQL Worker ingress connection is invalid',
|
||||
);
|
||||
}
|
||||
const mode =
|
||||
environment.QL3_WORKER_INGRESS_POSTGRES_TLS_MODE ?? 'verify-full';
|
||||
if (mode !== 'verify-full' && mode !== 'disable') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_MODE must be verify-full or disable',
|
||||
);
|
||||
}
|
||||
if (
|
||||
mode === 'disable' &&
|
||||
!booleanValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_ALLOW_INSECURE',
|
||||
false,
|
||||
)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'disabling PostgreSQL TLS requires QL3_WORKER_INGRESS_POSTGRES_ALLOW_INSECURE=true',
|
||||
);
|
||||
}
|
||||
const servername = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_SERVERNAME',
|
||||
253,
|
||||
);
|
||||
if (mode === 'verify-full' && !isPostgresTlsDnsServername(servername)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_SERVERNAME must be an explicit DNS name for verify-full',
|
||||
);
|
||||
}
|
||||
const certificateAuthorityFile = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_CA_FILE',
|
||||
4096,
|
||||
);
|
||||
if (mode === 'disable' && certificateAuthorityFile !== undefined) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_CA_FILE cannot be used when TLS is disabled',
|
||||
);
|
||||
}
|
||||
let certificateAuthority: string | undefined;
|
||||
if (certificateAuthorityFile !== undefined) {
|
||||
try {
|
||||
certificateAuthority = loadPostgresCertificateAuthorityFile(
|
||||
certificateAuthorityFile,
|
||||
);
|
||||
} catch {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_INGRESS_POSTGRES_TLS_CA_FILE must contain a bounded trusted CA bundle',
|
||||
);
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
...connection,
|
||||
tls:
|
||||
mode === 'disable'
|
||||
? Object.freeze({ mode: 'disable' as const })
|
||||
: Object.freeze({
|
||||
mode: 'verify-full' as const,
|
||||
...(certificateAuthority === undefined
|
||||
? {}
|
||||
: { ca: certificateAuthority }),
|
||||
servername: servername!,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function workerArtifactS3(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): Readonly<ClusterWorkerArtifactS3Config> {
|
||||
const bucket = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_BUCKET',
|
||||
63,
|
||||
true,
|
||||
)!;
|
||||
if (
|
||||
!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket) ||
|
||||
bucket.includes('..') ||
|
||||
/^\d{1,3}(?:\.\d{1,3}){3}$/.test(bucket)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_BUCKET is invalid',
|
||||
);
|
||||
}
|
||||
const region = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_REGION',
|
||||
63,
|
||||
true,
|
||||
)!;
|
||||
if (!/^[a-z0-9][a-z0-9-]{0,62}$/.test(region)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_REGION is invalid',
|
||||
);
|
||||
}
|
||||
const prefix = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_PREFIX',
|
||||
255,
|
||||
);
|
||||
if (
|
||||
prefix !== undefined &&
|
||||
(
|
||||
!/^[A-Za-z0-9][A-Za-z0-9/_=-]{0,254}$/.test(prefix) ||
|
||||
prefix.startsWith('/') ||
|
||||
prefix.endsWith('/') ||
|
||||
prefix.includes('//') ||
|
||||
prefix.split('/').some((segment) => segment === '.' || segment === '..')
|
||||
)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_PREFIX is invalid',
|
||||
);
|
||||
}
|
||||
const expectedBucketOwner = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_EXPECTED_BUCKET_OWNER',
|
||||
12,
|
||||
);
|
||||
if (
|
||||
expectedBucketOwner !== undefined &&
|
||||
!/^\d{12}$/.test(expectedBucketOwner)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_EXPECTED_BUCKET_OWNER must be 12 digits',
|
||||
);
|
||||
}
|
||||
const endpointValue = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_ENDPOINT',
|
||||
2048,
|
||||
);
|
||||
let endpoint: string | undefined;
|
||||
if (endpointValue !== undefined) {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(endpointValue);
|
||||
} catch {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_ENDPOINT is invalid',
|
||||
);
|
||||
}
|
||||
const allowInsecure = booleanValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_ALLOW_INSECURE',
|
||||
false,
|
||||
);
|
||||
if (
|
||||
(parsed.protocol !== 'https:' &&
|
||||
!(parsed.protocol === 'http:' && allowInsecure)) ||
|
||||
parsed.username !== '' ||
|
||||
parsed.password !== '' ||
|
||||
parsed.search !== '' ||
|
||||
parsed.hash !== '' ||
|
||||
parsed.pathname !== '/'
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_ENDPOINT must be an origin URL; HTTP requires explicit insecure opt-in',
|
||||
);
|
||||
}
|
||||
endpoint = parsed.origin;
|
||||
}
|
||||
const encryptionMode =
|
||||
boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_ENCRYPTION',
|
||||
3,
|
||||
) ?? 's3';
|
||||
if (encryptionMode !== 's3' && encryptionMode !== 'kms') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_ENCRYPTION must be s3 or kms',
|
||||
);
|
||||
}
|
||||
const keyId = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_KMS_KEY_ID',
|
||||
2048,
|
||||
);
|
||||
if (
|
||||
(encryptionMode === 'kms' && keyId === undefined) ||
|
||||
(encryptionMode === 's3' && keyId !== undefined)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_ARTIFACT_S3_KMS_KEY_ID must be present exactly for kms encryption',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
bucket,
|
||||
region,
|
||||
...(prefix === undefined ? {} : { prefix }),
|
||||
...(expectedBucketOwner === undefined
|
||||
? {}
|
||||
: { expectedBucketOwner }),
|
||||
...(endpoint === undefined ? {} : { endpoint }),
|
||||
forcePathStyle: booleanValue(
|
||||
environment,
|
||||
'QL3_WORKER_ARTIFACT_S3_FORCE_PATH_STYLE',
|
||||
false,
|
||||
),
|
||||
encryption:
|
||||
encryptionMode === 's3'
|
||||
? Object.freeze({ mode: 's3' as const })
|
||||
: Object.freeze({ mode: 'kms' as const, keyId: keyId! }),
|
||||
});
|
||||
}
|
||||
|
||||
function workerSecret(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): Readonly<ClusterWorkerMountedSecretConfig> | undefined {
|
||||
const provider = boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_SECRET_PROVIDER',
|
||||
32,
|
||||
);
|
||||
if (provider === undefined || provider === 'disabled') return undefined;
|
||||
if (provider !== 'mounted-files') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_SECRET_PROVIDER must be disabled or mounted-files',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
provider,
|
||||
rootDirectory: absoluteFile(
|
||||
environment,
|
||||
'QL3_WORKER_SECRET_ROOT_DIRECTORY',
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the Profile gate before reading database, Worker secret or TLS file
|
||||
* configuration. Disabled edge/standalone installs therefore remain free of
|
||||
* Worker ingress credential and filesystem requirements.
|
||||
*/
|
||||
export function loadClusterWorkerIngressConfig(
|
||||
environment: ClusterWorkerIngressEnvironment,
|
||||
): ClusterWorkerIngressConfig {
|
||||
if (
|
||||
!environment ||
|
||||
typeof environment !== 'object' ||
|
||||
Array.isArray(environment)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError('environment must be an object');
|
||||
}
|
||||
const profile = deploymentProfile(environment);
|
||||
const enabled = booleanValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_ENABLED',
|
||||
false,
|
||||
);
|
||||
if (!enabled) return Object.freeze({ enabled: false, profile });
|
||||
if (profile !== 'cluster-control') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'enabled ingress requires QL_DEPLOYMENT_PROFILE=cluster-control',
|
||||
);
|
||||
}
|
||||
|
||||
const applicationName =
|
||||
boundedValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_APPLICATION_NAME',
|
||||
63,
|
||||
) ?? 'qinglong-worker-ingress';
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,62}$/.test(applicationName)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'QL3_WORKER_INGRESS_POSTGRES_APPLICATION_NAME is invalid',
|
||||
);
|
||||
}
|
||||
const host =
|
||||
boundedValue(environment, 'QL3_WORKER_INGRESS_HOST', 253) ?? '0.0.0.0';
|
||||
const secret = workerSecret(environment);
|
||||
|
||||
return Object.freeze({
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
http: Object.freeze({
|
||||
host,
|
||||
port: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_PORT',
|
||||
5801,
|
||||
1,
|
||||
65_535,
|
||||
),
|
||||
maxBodyBytes: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_MAX_BODY_BYTES',
|
||||
64 * 1024,
|
||||
1024,
|
||||
64 * 1024,
|
||||
),
|
||||
maxResponseBytes: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_MAX_RESPONSE_BYTES',
|
||||
64 * 1024,
|
||||
1024,
|
||||
64 * 1024,
|
||||
),
|
||||
maxInFlightRequests: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_MAX_IN_FLIGHT',
|
||||
64,
|
||||
1,
|
||||
256,
|
||||
),
|
||||
authenticationRateWindowMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_AUTH_RATE_WINDOW_MS',
|
||||
60_000,
|
||||
1_000,
|
||||
60 * 60_000,
|
||||
),
|
||||
authenticationRatePerPeer: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_AUTH_RATE_PER_PEER',
|
||||
120,
|
||||
1,
|
||||
1_000_000,
|
||||
),
|
||||
authenticationRateGlobal: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_AUTH_RATE_GLOBAL',
|
||||
1_200,
|
||||
1,
|
||||
1_000_000,
|
||||
),
|
||||
authenticationRateMaxPeers: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_AUTH_RATE_MAX_PEERS',
|
||||
4_096,
|
||||
1,
|
||||
65_536,
|
||||
),
|
||||
requestTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_REQUEST_TIMEOUT_MS',
|
||||
15_000,
|
||||
100,
|
||||
120_000,
|
||||
),
|
||||
drainTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_DRAIN_TIMEOUT_MS',
|
||||
10_000,
|
||||
100,
|
||||
120_000,
|
||||
),
|
||||
}),
|
||||
transport: Object.freeze({
|
||||
privateKeyFile: absoluteFile(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_TLS_PRIVATE_KEY_FILE',
|
||||
),
|
||||
certificateFile: absoluteFile(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_TLS_CERTIFICATE_FILE',
|
||||
),
|
||||
clientCertificateAuthorityFile: absoluteFile(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_TLS_CLIENT_CA_FILE',
|
||||
),
|
||||
...(() => {
|
||||
const clientCertificateRevocationListFile = optionalAbsoluteFile(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_TLS_CLIENT_CRL_FILE',
|
||||
);
|
||||
return clientCertificateRevocationListFile === undefined
|
||||
? {}
|
||||
: { clientCertificateRevocationListFile };
|
||||
})(),
|
||||
}),
|
||||
database: Object.freeze({
|
||||
connection: databaseConnection(environment),
|
||||
pool: Object.freeze({
|
||||
applicationName,
|
||||
maxConnections: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_MAX_CONNECTIONS',
|
||||
4,
|
||||
1,
|
||||
16,
|
||||
),
|
||||
connectionTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_WORKER_INGRESS_POSTGRES_CONNECTION_TIMEOUT_MS',
|
||||
5_000,
|
||||
100,
|
||||
60_000,
|
||||
),
|
||||
}),
|
||||
}),
|
||||
security: Object.freeze({
|
||||
workerCredentialPepper: workerCredentialPepper(environment),
|
||||
}),
|
||||
artifact: workerArtifactS3(environment),
|
||||
...(secret === undefined ? {} : { secret }),
|
||||
});
|
||||
}
|
||||
|
||||
async function readTlsFile(
|
||||
path: string,
|
||||
privateMaterial: boolean,
|
||||
): Promise<Buffer> {
|
||||
let handle;
|
||||
try {
|
||||
handle = await open(path, constants.O_RDONLY);
|
||||
const stat = await handle.stat();
|
||||
if (
|
||||
!stat.isFile() ||
|
||||
stat.size < 1 ||
|
||||
stat.size > MAX_TLS_FILE_BYTES ||
|
||||
(privateMaterial && (stat.mode & 0o022) !== 0)
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError('TLS file metadata is unsafe');
|
||||
}
|
||||
const bytes = await handle.readFile();
|
||||
if (bytes.byteLength < 1 || bytes.byteLength > MAX_TLS_FILE_BYTES) {
|
||||
bytes.fill(0);
|
||||
throw new ClusterWorkerIngressConfigError('TLS file size is unsafe');
|
||||
}
|
||||
return bytes;
|
||||
} catch (error) {
|
||||
if (error instanceof ClusterWorkerIngressConfigError) throw error;
|
||||
throw new ClusterWorkerIngressConfigError('TLS material is unavailable');
|
||||
} finally {
|
||||
await handle?.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
function activeCertificate(
|
||||
name: string,
|
||||
bytes: Buffer,
|
||||
now: number,
|
||||
): X509Certificate {
|
||||
let certificate: X509Certificate;
|
||||
try {
|
||||
certificate = new X509Certificate(bytes);
|
||||
} catch {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
`${name} is not an X.509 certificate`,
|
||||
);
|
||||
}
|
||||
const validFrom = Date.parse(certificate.validFrom);
|
||||
const validTo = Date.parse(certificate.validTo);
|
||||
if (
|
||||
!Number.isFinite(validFrom) ||
|
||||
!Number.isFinite(validTo) ||
|
||||
now < validFrom ||
|
||||
now >= validTo
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(`${name} is not currently valid`);
|
||||
}
|
||||
return certificate;
|
||||
}
|
||||
|
||||
function activeCertificateAuthorities(
|
||||
bytes: Buffer,
|
||||
now: number,
|
||||
): readonly Buffer[] {
|
||||
const pem = bytes.toString('utf8');
|
||||
const matches = pem.match(
|
||||
/-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/g,
|
||||
);
|
||||
if (!matches || matches.length < 1 || matches.length > 16) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS client certificate authority bundle must contain 1 to 16 PEM certificates',
|
||||
);
|
||||
}
|
||||
const remainder = matches.reduce(
|
||||
(value, certificate) => value.replace(certificate, ''),
|
||||
pem,
|
||||
);
|
||||
if (remainder.trim() !== '') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS client certificate authority bundle contains unsupported data',
|
||||
);
|
||||
}
|
||||
const authorities: Buffer[] = [];
|
||||
try {
|
||||
for (const match of matches) {
|
||||
const authorityBytes = Buffer.from(`${match}\n`, 'utf8');
|
||||
const authority = activeCertificate(
|
||||
'TLS client certificate authority',
|
||||
authorityBytes,
|
||||
now,
|
||||
);
|
||||
if (!authority.ca) {
|
||||
authorityBytes.fill(0);
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS client certificate authority is not a CA',
|
||||
);
|
||||
}
|
||||
authorities.push(authorityBytes);
|
||||
}
|
||||
return Object.freeze(authorities);
|
||||
} catch (error) {
|
||||
for (const authority of authorities) authority.fill(0);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function certificateRevocationList(bytes: Buffer): Buffer {
|
||||
const value = bytes.toString('utf8').trim();
|
||||
if (
|
||||
!value.startsWith('-----BEGIN X509 CRL-----') ||
|
||||
!value.endsWith('-----END X509 CRL-----')
|
||||
) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS client certificate revocation list is not a PEM CRL',
|
||||
);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
function matchingPrivateKey(
|
||||
privateKey: KeyObject,
|
||||
certificate: X509Certificate,
|
||||
): boolean {
|
||||
const key = createPublicKey(privateKey).export({
|
||||
type: 'spki',
|
||||
format: 'der',
|
||||
});
|
||||
const certificateKey = certificate.publicKey.export({
|
||||
type: 'spki',
|
||||
format: 'der',
|
||||
});
|
||||
return (
|
||||
key.byteLength === certificateKey.byteLength &&
|
||||
timingSafeEqual(key, certificateKey)
|
||||
);
|
||||
}
|
||||
|
||||
export async function loadClusterWorkerIngressMutualTls(
|
||||
config: EnabledClusterWorkerIngressConfig,
|
||||
now: number = Date.now(),
|
||||
): Promise<ClusterControlMutualTlsOptions> {
|
||||
if (!config?.enabled || config.profile !== 'cluster-control') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS material requires an enabled Worker ingress config',
|
||||
);
|
||||
}
|
||||
if (!Number.isSafeInteger(now) || now < 0) {
|
||||
throw new ClusterWorkerIngressConfigError('observation time is invalid');
|
||||
}
|
||||
const keyBytes = await readTlsFile(config.transport.privateKeyFile, true);
|
||||
let certificateBytes: Buffer | undefined;
|
||||
let clientAuthorityBundleBytes: Buffer | undefined;
|
||||
let certificateRevocationListBytes: Buffer | undefined;
|
||||
let clientCertificateAuthorities: readonly Buffer[] = Object.freeze([]);
|
||||
try {
|
||||
let privateKey: KeyObject;
|
||||
try {
|
||||
privateKey = createPrivateKey(keyBytes);
|
||||
} catch {
|
||||
throw new ClusterWorkerIngressConfigError('TLS private key is invalid');
|
||||
}
|
||||
certificateBytes = await readTlsFile(
|
||||
config.transport.certificateFile,
|
||||
false,
|
||||
);
|
||||
clientAuthorityBundleBytes = await readTlsFile(
|
||||
config.transport.clientCertificateAuthorityFile,
|
||||
false,
|
||||
);
|
||||
const certificate = activeCertificate(
|
||||
'TLS server certificate',
|
||||
certificateBytes,
|
||||
now,
|
||||
);
|
||||
clientCertificateAuthorities = activeCertificateAuthorities(
|
||||
clientAuthorityBundleBytes,
|
||||
now,
|
||||
);
|
||||
if (!matchingPrivateKey(privateKey, certificate)) {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'TLS private key does not match the server certificate',
|
||||
);
|
||||
}
|
||||
if (config.transport.clientCertificateRevocationListFile !== undefined) {
|
||||
certificateRevocationListBytes = certificateRevocationList(
|
||||
await readTlsFile(
|
||||
config.transport.clientCertificateRevocationListFile,
|
||||
false,
|
||||
),
|
||||
);
|
||||
}
|
||||
const mutualTls: ClusterControlMutualTlsOptions = Object.freeze({
|
||||
privateKey: keyBytes,
|
||||
certificateChain: certificateBytes,
|
||||
clientCertificateAuthorities,
|
||||
...(certificateRevocationListBytes === undefined
|
||||
? {}
|
||||
: {
|
||||
certificateRevocationLists: Object.freeze([
|
||||
certificateRevocationListBytes,
|
||||
]),
|
||||
}),
|
||||
});
|
||||
clientAuthorityBundleBytes.fill(0);
|
||||
return mutualTls;
|
||||
} catch (error) {
|
||||
keyBytes.fill(0);
|
||||
certificateBytes?.fill(0);
|
||||
clientAuthorityBundleBytes?.fill(0);
|
||||
certificateRevocationListBytes?.fill(0);
|
||||
for (const authority of clientCertificateAuthorities) authority.fill(0);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function createClusterWorkerIngressHttpOptions(
|
||||
config: EnabledClusterWorkerIngressConfig,
|
||||
now: number = Date.now(),
|
||||
): Promise<ClusterControlHttpSurfaceOptions> {
|
||||
const mutualTls = await loadClusterWorkerIngressMutualTls(config, now);
|
||||
return Object.freeze({ ...config.http, mutualTls });
|
||||
}
|
||||
|
||||
export function createClusterWorkerIngressDatabaseOpener(
|
||||
config: EnabledClusterWorkerIngressConfig,
|
||||
onPoolError: (error: Error) => void,
|
||||
): OpenPostgresDatabase {
|
||||
if (!config?.enabled || config.profile !== 'cluster-control') {
|
||||
throw new ClusterWorkerIngressConfigError(
|
||||
'database opener requires an enabled Worker ingress config',
|
||||
);
|
||||
}
|
||||
if (typeof onPoolError !== 'function') {
|
||||
throw new ClusterWorkerIngressConfigError('onPoolError must be a function');
|
||||
}
|
||||
return createPostgresDatabaseOpener({
|
||||
role: 'worker-ingress',
|
||||
connection: config.database.connection,
|
||||
pool: config.database.pool,
|
||||
onPoolError,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,584 @@
|
||||
// Cluster Control Worker Ingress boundary; keep authenticated admission routing explicit.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
WorkerSessionConflictError,
|
||||
WorkerSessionFenceRejectedError,
|
||||
} from '@qinglong/runtime-core';
|
||||
import type {
|
||||
AuthenticatedWorkerSessionRepository,
|
||||
} from '@qinglong/runtime-core/worker-credential-delivery';
|
||||
import {
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
WorkerCredentialDeliveryUnavailableError,
|
||||
} from '@qinglong/runtime-core/worker-credential-delivery';
|
||||
import {
|
||||
InvalidWorkerSessionTransportError,
|
||||
createWorkerSessionHeartbeatResponseBody,
|
||||
createWorkerSessionRegisterResponseBody,
|
||||
createWorkerSessionTransitionResponseBody,
|
||||
parseWorkerSessionHeartbeatRequestBody,
|
||||
parseWorkerSessionRegisterRequestBody,
|
||||
parseWorkerSessionTransitionRequestBody,
|
||||
} from '@qinglong/runtime-core/worker-session-transport';
|
||||
import {
|
||||
WorkerExecutionAttestationFenceRejectedError,
|
||||
WorkerExecutionAttestationUnavailableError,
|
||||
type WorkerExecutionAttestationRepository,
|
||||
} from '@qinglong/runtime-core/worker-attestation';
|
||||
import {
|
||||
WorkerCredentialUnavailableError,
|
||||
} from '@qinglong/runtime-core/worker-credential';
|
||||
import {
|
||||
normalizeSecurityAuditRecord,
|
||||
type SecurityAuditSink,
|
||||
} from '@qinglong/runtime-core/security-audit';
|
||||
import type {
|
||||
ClusterControlAdmissionMetadata,
|
||||
ClusterControlAdmissionPipeline,
|
||||
ClusterControlAdmissionResponse,
|
||||
ClusterControlStreamingAdmissionBody,
|
||||
} from '../transport/httpSurface';
|
||||
import type {
|
||||
AuthenticatedWorkerPrincipal,
|
||||
WorkerCredentialAuthenticator,
|
||||
} from './workerCredentialAuthenticator';
|
||||
import {
|
||||
ClusterRemoteWorkerOfferFenceRejectedError,
|
||||
type ClusterRemoteWorkerOfferClaimService,
|
||||
} from '../remote-execution/remoteWorkerDispatcher';
|
||||
import {
|
||||
RemoteRunActivationFenceRejectedError,
|
||||
RemoteRunActivationUnavailableError,
|
||||
} from '@qinglong/runtime-core/remote-activation';
|
||||
import {
|
||||
createRemoteRunActivationResponseBody,
|
||||
InvalidRemoteRunActivationDeliveryError,
|
||||
} from '@qinglong/runtime-core/remote-activation-delivery';
|
||||
import type { ClusterRemoteRunActivationService } from '../remote-execution/remoteRunActivationService';
|
||||
import {
|
||||
createRemoteExecutionOfferPullBody,
|
||||
InvalidRemoteExecutionOfferDeliveryError,
|
||||
} from '@qinglong/runtime-core/remote-offer-delivery';
|
||||
import {
|
||||
createRemoteWorkerSecretDeliveryResponseBody,
|
||||
InvalidRemoteWorkerSecretDeliveryError,
|
||||
REMOTE_SECRET_DELIVERY_SCHEMA,
|
||||
RemoteWorkerSecretDeliveryFenceRejectedError,
|
||||
RemoteWorkerSecretDeliveryUnavailableError,
|
||||
} from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import type { ClusterRemoteWorkerSecretDeliveryService } from '../remote-execution/remoteWorkerSecretDeliveryService';
|
||||
import {
|
||||
InvalidRemoteWorkerCompletionError,
|
||||
MAX_REMOTE_WORKER_ARTIFACT_BYTES,
|
||||
MAX_REMOTE_WORKER_ARTIFACT_HEADER_BYTES,
|
||||
REMOTE_WORKER_ARTIFACT_CONTENT_TYPE,
|
||||
RemoteWorkerCompletionFenceRejectedError,
|
||||
RemoteWorkerCompletionUnavailableError,
|
||||
createRemoteWorkerArtifactUploadResponseBody,
|
||||
createRemoteWorkerCompletionResponseBody,
|
||||
parseRemoteWorkerCompletionRequestBody,
|
||||
} from '@qinglong/runtime-core/remote-worker-completion';
|
||||
import type {
|
||||
ClusterRemoteWorkerArtifactService,
|
||||
ClusterRemoteWorkerCompletionService,
|
||||
} from '../remote-execution/remoteWorkerCompletionService';
|
||||
import {
|
||||
InvalidRemoteWorkerLeaseControlError,
|
||||
RemoteWorkerLeaseControlFenceRejectedError,
|
||||
RemoteWorkerLeaseControlUnavailableError,
|
||||
createRemoteWorkerLeaseControlResponseBody,
|
||||
parseRemoteWorkerLeaseControlRequestBody,
|
||||
} from '@qinglong/runtime-core/remote-worker-lease-control';
|
||||
import type { ClusterRemoteWorkerLeaseControlService } from '../remote-execution/remoteWorkerLeaseControlService';
|
||||
|
||||
export interface WorkerIngressPipelineOptions {
|
||||
readonly authenticator: WorkerCredentialAuthenticator;
|
||||
readonly workers: AuthenticatedWorkerSessionRepository;
|
||||
readonly attestations: WorkerExecutionAttestationRepository;
|
||||
readonly audit: SecurityAuditSink;
|
||||
readonly offers?: Pick<ClusterRemoteWorkerOfferClaimService, 'claimNext'>;
|
||||
readonly activation?: Pick<
|
||||
ClusterRemoteRunActivationService,
|
||||
'acknowledgeStarting' | 'acknowledgeRunning' | 'failStart'
|
||||
>;
|
||||
readonly secrets?: Pick<ClusterRemoteWorkerSecretDeliveryService, 'deliver'>;
|
||||
readonly artifacts?: Pick<ClusterRemoteWorkerArtifactService, 'upload'>;
|
||||
readonly completion?: Pick<ClusterRemoteWorkerCompletionService, 'complete'>;
|
||||
readonly leaseControl?: Pick<ClusterRemoteWorkerLeaseControlService, 'control'>;
|
||||
readonly now?: () => number;
|
||||
}
|
||||
|
||||
type Operation =
|
||||
| 'register'
|
||||
| 'heartbeat'
|
||||
| 'transition'
|
||||
| 'attestations'
|
||||
| 'offers'
|
||||
| 'starting'
|
||||
| 'running'
|
||||
| 'start-failure'
|
||||
| 'secrets'
|
||||
| 'artifacts'
|
||||
| 'completion'
|
||||
| 'lease-control';
|
||||
|
||||
interface ResolvedRoute {
|
||||
readonly workerId: string;
|
||||
readonly sessionId: string;
|
||||
readonly operation: Operation;
|
||||
}
|
||||
|
||||
const ROUTE = /^\/api\/v3\/worker-ingress\/workers\/([A-Za-z0-9][A-Za-z0-9._:-]{0,127})\/sessions\/([0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})\/(register|heartbeat|transition|attestations|offers|starting|running|start-failure|secrets|artifacts|completion|lease-control)$/;
|
||||
|
||||
function failure(statusCode: number, code: string): Error {
|
||||
return Object.assign(new Error(code), { statusCode, code });
|
||||
}
|
||||
|
||||
function route(metadata: ClusterControlAdmissionMetadata): ResolvedRoute {
|
||||
if (metadata.method !== 'POST' || Object.keys(metadata.query).length !== 0) {
|
||||
throw failure(404, 'worker_route_not_found');
|
||||
}
|
||||
const match = ROUTE.exec(metadata.path);
|
||||
if (!match) throw failure(404, 'worker_route_not_found');
|
||||
return Object.freeze({
|
||||
workerId: match[1]!,
|
||||
sessionId: match[2]!,
|
||||
operation: match[3]! as Operation,
|
||||
});
|
||||
}
|
||||
|
||||
function objectBody(body: unknown | null, keys: readonly string[]): Record<string, unknown> {
|
||||
if (!body || typeof body !== 'object' || Array.isArray(body)) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
const actual = Object.keys(body).sort();
|
||||
const expected = [...keys].sort();
|
||||
if (
|
||||
actual.length !== expected.length ||
|
||||
actual.some((key, index) => key !== expected[index])
|
||||
) throw failure(400, 'invalid_worker_request');
|
||||
return body as Record<string, unknown>;
|
||||
}
|
||||
|
||||
async function audit(
|
||||
sink: SecurityAuditSink,
|
||||
metadata: ClusterControlAdmissionMetadata,
|
||||
operation: Operation,
|
||||
principal: Readonly<AuthenticatedWorkerPrincipal> | null,
|
||||
outcome: 'authentication_rejected' | 'authentication_unavailable' | 'allowed',
|
||||
now: () => number,
|
||||
): Promise<void> {
|
||||
await sink.record(normalizeSecurityAuditRecord({
|
||||
eventId: randomUUID(),
|
||||
requestId: metadata.requestId,
|
||||
operationId: `worker.${operation}`,
|
||||
projectId: null,
|
||||
subject: principal ? { type: 'worker', id: principal.workerId } : null,
|
||||
authenticationId: principal?.authenticationId ?? null,
|
||||
outcome,
|
||||
reasons: [outcome === 'allowed' ? 'worker_credential' : outcome],
|
||||
fence: null,
|
||||
occurredAtMs: now(),
|
||||
}));
|
||||
}
|
||||
|
||||
function response(statusCode: number, body: unknown): ClusterControlAdmissionResponse {
|
||||
return Object.freeze({ statusCode, body });
|
||||
}
|
||||
|
||||
function mapIngressFailure(error: unknown): never {
|
||||
if (error && typeof error === 'object' && 'statusCode' in error) throw error;
|
||||
if (
|
||||
error instanceof WorkerSessionConflictError ||
|
||||
error instanceof WorkerSessionFenceRejectedError ||
|
||||
error instanceof WorkerCredentialDeliveryConflictError
|
||||
) throw failure(409, 'worker_session_fenced');
|
||||
if (error instanceof WorkerExecutionAttestationFenceRejectedError) {
|
||||
throw failure(409, 'worker_attestation_fenced');
|
||||
}
|
||||
if (error instanceof ClusterRemoteWorkerOfferFenceRejectedError) {
|
||||
throw failure(409, 'worker_offer_fenced');
|
||||
}
|
||||
if (error instanceof RemoteRunActivationFenceRejectedError) {
|
||||
throw failure(409, 'worker_activation_fenced');
|
||||
}
|
||||
if (error instanceof RemoteWorkerSecretDeliveryFenceRejectedError) {
|
||||
throw failure(409, 'worker_secret_delivery_fenced');
|
||||
}
|
||||
if (error instanceof RemoteWorkerCompletionFenceRejectedError) {
|
||||
throw failure(409, 'worker_completion_fenced');
|
||||
}
|
||||
if (error instanceof RemoteWorkerLeaseControlFenceRejectedError) {
|
||||
throw failure(409, 'worker_lease_control_fenced');
|
||||
}
|
||||
if (error instanceof InvalidRemoteWorkerCompletionError) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
if (error instanceof InvalidRemoteWorkerLeaseControlError) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
if (error instanceof InvalidWorkerSessionTransportError) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
if (
|
||||
error instanceof InvalidRemoteExecutionOfferDeliveryError ||
|
||||
error instanceof InvalidRemoteRunActivationDeliveryError ||
|
||||
error instanceof InvalidRemoteWorkerSecretDeliveryError
|
||||
) throw failure(503, 'worker_ingress_unavailable');
|
||||
if (
|
||||
error instanceof WorkerExecutionAttestationUnavailableError ||
|
||||
error instanceof WorkerCredentialUnavailableError ||
|
||||
error instanceof WorkerCredentialDeliveryUnavailableError ||
|
||||
error instanceof RemoteRunActivationUnavailableError ||
|
||||
error instanceof RemoteWorkerSecretDeliveryUnavailableError ||
|
||||
error instanceof RemoteWorkerCompletionUnavailableError ||
|
||||
error instanceof RemoteWorkerLeaseControlUnavailableError
|
||||
) throw failure(503, 'worker_ingress_unavailable');
|
||||
if (error instanceof TypeError || error instanceof RangeError) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
throw failure(503, 'worker_ingress_unavailable');
|
||||
}
|
||||
|
||||
export function createWorkerIngressAdmissionPipeline(
|
||||
options: WorkerIngressPipelineOptions,
|
||||
): ClusterControlAdmissionPipeline {
|
||||
if (
|
||||
!options ||
|
||||
typeof options.authenticator?.authenticate !== 'function' ||
|
||||
typeof options.workers?.register !== 'function' ||
|
||||
typeof options.workers?.heartbeatAuthenticated !== 'function' ||
|
||||
typeof options.workers?.transitionAuthenticated !== 'function' ||
|
||||
typeof options.attestations?.submit !== 'function' ||
|
||||
typeof options.audit?.record !== 'function' ||
|
||||
(options.offers !== undefined &&
|
||||
typeof options.offers.claimNext !== 'function') ||
|
||||
(options.activation !== undefined &&
|
||||
(typeof options.activation.acknowledgeStarting !== 'function' ||
|
||||
typeof options.activation.acknowledgeRunning !== 'function' ||
|
||||
typeof options.activation.failStart !== 'function')) ||
|
||||
(options.secrets !== undefined &&
|
||||
typeof options.secrets.deliver !== 'function') ||
|
||||
(options.artifacts !== undefined &&
|
||||
typeof options.artifacts.upload !== 'function') ||
|
||||
(options.completion !== undefined &&
|
||||
typeof options.completion.complete !== 'function') ||
|
||||
(options.leaseControl !== undefined &&
|
||||
typeof options.leaseControl.control !== 'function')
|
||||
) throw new TypeError('Worker ingress pipeline options are invalid');
|
||||
const now = options.now ?? Date.now;
|
||||
|
||||
return Object.freeze({
|
||||
async prepare(metadata: ClusterControlAdmissionMetadata) {
|
||||
const resolved = route(metadata);
|
||||
let principal: Readonly<AuthenticatedWorkerPrincipal> | null;
|
||||
try {
|
||||
principal = await options.authenticator.authenticate(metadata);
|
||||
} catch {
|
||||
try { await audit(options.audit, metadata, resolved.operation, null, 'authentication_unavailable', now); } catch { /* fail below */ }
|
||||
throw failure(503, 'worker_authentication_unavailable');
|
||||
}
|
||||
if (!principal || principal.workerId !== resolved.workerId) {
|
||||
try { await audit(options.audit, metadata, resolved.operation, null, 'authentication_rejected', now); } catch { throw failure(503, 'worker_audit_unavailable'); }
|
||||
throw failure(401, 'worker_authentication_required');
|
||||
}
|
||||
try {
|
||||
await audit(options.audit, metadata, resolved.operation, principal, 'allowed', now);
|
||||
} catch {
|
||||
throw failure(503, 'worker_audit_unavailable');
|
||||
}
|
||||
|
||||
if (resolved.operation === 'artifacts') {
|
||||
return Object.freeze({
|
||||
bodyMode: 'stream' as const,
|
||||
contentType: REMOTE_WORKER_ARTIFACT_CONTENT_TYPE,
|
||||
maximumBodyBytes:
|
||||
4 + MAX_REMOTE_WORKER_ARTIFACT_HEADER_BYTES +
|
||||
MAX_REMOTE_WORKER_ARTIFACT_BYTES,
|
||||
async handleStream(body: ClusterControlStreamingAdmissionBody) {
|
||||
try {
|
||||
if (!options.artifacts) {
|
||||
throw failure(503, 'worker_artifact_unavailable');
|
||||
}
|
||||
const receipt = await options.artifacts.upload({
|
||||
workerId: resolved.workerId,
|
||||
workerSessionId: resolved.sessionId,
|
||||
contentLength: body.contentLength,
|
||||
chunks: body.chunks,
|
||||
signal: metadata.signal,
|
||||
});
|
||||
return response(
|
||||
200,
|
||||
createRemoteWorkerArtifactUploadResponseBody(receipt),
|
||||
);
|
||||
} catch (error) {
|
||||
return mapIngressFailure(error);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
async handle(body: unknown | null) {
|
||||
try {
|
||||
if (resolved.operation === 'register') {
|
||||
const command = parseWorkerSessionRegisterRequestBody(body, {
|
||||
workerId: resolved.workerId,
|
||||
sessionId: resolved.sessionId,
|
||||
});
|
||||
const result = await options.workers.register(command);
|
||||
return response(
|
||||
200,
|
||||
createWorkerSessionRegisterResponseBody(result),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'heartbeat') {
|
||||
const command = parseWorkerSessionHeartbeatRequestBody(body, {
|
||||
workerId: resolved.workerId,
|
||||
sessionId: resolved.sessionId,
|
||||
});
|
||||
const worker = await options.workers.heartbeatAuthenticated(
|
||||
command,
|
||||
{
|
||||
workerId: principal.workerId,
|
||||
credentialId: principal.credentialId,
|
||||
credentialVersion: principal.credentialVersion,
|
||||
},
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
createWorkerSessionHeartbeatResponseBody(worker),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'transition') {
|
||||
const command = parseWorkerSessionTransitionRequestBody(body, {
|
||||
workerId: resolved.workerId,
|
||||
sessionId: resolved.sessionId,
|
||||
});
|
||||
const worker = await options.workers.transitionAuthenticated(
|
||||
command,
|
||||
{
|
||||
workerId: principal.workerId,
|
||||
credentialId: principal.credentialId,
|
||||
credentialVersion: principal.credentialVersion,
|
||||
},
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
createWorkerSessionTransitionResponseBody(worker),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'offers') {
|
||||
if (!options.offers) {
|
||||
throw failure(503, 'worker_offer_unavailable');
|
||||
}
|
||||
const value = objectBody(body, [
|
||||
'workerGeneration', 'offerId', 'leaseToken',
|
||||
]);
|
||||
const result = await options.offers.claimNext(
|
||||
{ workerId: resolved.workerId },
|
||||
{
|
||||
workerSessionId: resolved.sessionId,
|
||||
workerGeneration: value.workerGeneration as number,
|
||||
offerId: value.offerId as string,
|
||||
leaseToken: value.leaseToken as string,
|
||||
},
|
||||
);
|
||||
return response(200, createRemoteExecutionOfferPullBody(result));
|
||||
}
|
||||
if (
|
||||
resolved.operation === 'starting' ||
|
||||
resolved.operation === 'start-failure'
|
||||
) {
|
||||
if (!options.activation) {
|
||||
throw failure(503, 'worker_activation_unavailable');
|
||||
}
|
||||
const value = objectBody(body, [
|
||||
'runId', 'attemptId', 'workerGeneration', 'offerId',
|
||||
'leaseGeneration', 'leaseToken', 'expectedLeaseVersion',
|
||||
]);
|
||||
const command = {
|
||||
runId: value.runId as string,
|
||||
attemptId: value.attemptId as string,
|
||||
workerSessionId: resolved.sessionId,
|
||||
workerGeneration: value.workerGeneration as number,
|
||||
offerId: value.offerId as string,
|
||||
leaseGeneration: value.leaseGeneration as number,
|
||||
leaseToken: value.leaseToken as string,
|
||||
expectedLeaseVersion: value.expectedLeaseVersion as number,
|
||||
};
|
||||
const activation = resolved.operation === 'starting'
|
||||
? await options.activation.acknowledgeStarting(
|
||||
{ workerId: resolved.workerId }, command,
|
||||
)
|
||||
: await options.activation.failStart(
|
||||
{ workerId: resolved.workerId }, command,
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
createRemoteRunActivationResponseBody(activation),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'running') {
|
||||
if (!options.activation) {
|
||||
throw failure(503, 'worker_activation_unavailable');
|
||||
}
|
||||
const value = objectBody(body, [
|
||||
'runId', 'attemptId', 'workerGeneration', 'offerId',
|
||||
'leaseGeneration', 'leaseToken', 'expectedLeaseVersion',
|
||||
'executorHandle', 'logArtifactId', 'callbackSequence',
|
||||
'callbackTokenDigest',
|
||||
]);
|
||||
if (
|
||||
value.logArtifactId !== null &&
|
||||
typeof value.logArtifactId !== 'string'
|
||||
) throw failure(400, 'invalid_worker_request');
|
||||
const activation = await options.activation.acknowledgeRunning(
|
||||
{ workerId: resolved.workerId },
|
||||
{
|
||||
runId: value.runId as string,
|
||||
attemptId: value.attemptId as string,
|
||||
workerSessionId: resolved.sessionId,
|
||||
workerGeneration: value.workerGeneration as number,
|
||||
offerId: value.offerId as string,
|
||||
leaseGeneration: value.leaseGeneration as number,
|
||||
leaseToken: value.leaseToken as string,
|
||||
expectedLeaseVersion: value.expectedLeaseVersion as number,
|
||||
executorHandle: value.executorHandle as string,
|
||||
callbackSequence: value.callbackSequence as number,
|
||||
callbackTokenDigest: value.callbackTokenDigest as string,
|
||||
...(value.logArtifactId === null
|
||||
? {}
|
||||
: { logArtifactId: value.logArtifactId }),
|
||||
},
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
createRemoteRunActivationResponseBody(activation),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'secrets') {
|
||||
if (!options.secrets) {
|
||||
throw failure(503, 'worker_secret_delivery_unavailable');
|
||||
}
|
||||
const value = objectBody(body, [
|
||||
'schema', 'runId', 'attemptId', 'projectId', 'taskId',
|
||||
'taskRevision', 'executionDigest', 'workerGeneration',
|
||||
'offerId', 'leaseGeneration', 'leaseToken',
|
||||
'expectedLeaseVersion', 'secretRefs',
|
||||
]);
|
||||
if (value.schema !== REMOTE_SECRET_DELIVERY_SCHEMA) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
const delivered = await options.secrets.deliver(
|
||||
{ workerId: resolved.workerId },
|
||||
{
|
||||
workerSessionId: resolved.sessionId,
|
||||
workerGeneration: value.workerGeneration as number,
|
||||
runId: value.runId as string,
|
||||
attemptId: value.attemptId as string,
|
||||
projectId: value.projectId as string,
|
||||
taskId: value.taskId as string,
|
||||
taskRevision: value.taskRevision as string,
|
||||
executionDigest: value.executionDigest as string,
|
||||
offerId: value.offerId as string,
|
||||
leaseGeneration: value.leaseGeneration as number,
|
||||
leaseToken: value.leaseToken as string,
|
||||
expectedLeaseVersion: value.expectedLeaseVersion as number,
|
||||
secretRefs: value.secretRefs as string[],
|
||||
},
|
||||
);
|
||||
try {
|
||||
const responseBody = createRemoteWorkerSecretDeliveryResponseBody(
|
||||
delivered,
|
||||
value.secretRefs as string[],
|
||||
);
|
||||
if (
|
||||
responseBody.runId !== value.runId ||
|
||||
responseBody.attemptId !== value.attemptId ||
|
||||
responseBody.offerId !== value.offerId ||
|
||||
responseBody.executionDigest !== value.executionDigest
|
||||
) throw new InvalidRemoteWorkerSecretDeliveryError(
|
||||
'service response authority does not match request',
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
responseBody,
|
||||
);
|
||||
} finally {
|
||||
try { await delivered.dispose?.(); } catch { /* response remains valid */ }
|
||||
}
|
||||
}
|
||||
if (resolved.operation === 'completion') {
|
||||
if (!options.completion) {
|
||||
throw failure(503, 'worker_completion_unavailable');
|
||||
}
|
||||
const command = parseRemoteWorkerCompletionRequestBody(body, {
|
||||
workerId: resolved.workerId,
|
||||
workerSessionId: resolved.sessionId,
|
||||
});
|
||||
const completed = await options.completion.complete(
|
||||
command,
|
||||
metadata.signal,
|
||||
);
|
||||
return response(
|
||||
200,
|
||||
createRemoteWorkerCompletionResponseBody(completed),
|
||||
);
|
||||
}
|
||||
if (resolved.operation === 'lease-control') {
|
||||
if (!options.leaseControl) {
|
||||
throw failure(503, 'worker_lease_control_unavailable');
|
||||
}
|
||||
const command = parseRemoteWorkerLeaseControlRequestBody(body, {
|
||||
workerId: resolved.workerId,
|
||||
workerSessionId: resolved.sessionId,
|
||||
});
|
||||
return response(
|
||||
200,
|
||||
createRemoteWorkerLeaseControlResponseBody(
|
||||
await options.leaseControl.control(command),
|
||||
),
|
||||
);
|
||||
}
|
||||
const value = objectBody(body, [
|
||||
'attestationId', 'runId', 'attemptId', 'sequence', 'state',
|
||||
'workerGeneration', 'leaseTokenDigest', 'leaseGeneration',
|
||||
'leaseVersion', 'offerId', 'callbackSequence', 'executorHandle',
|
||||
'journalRevision',
|
||||
]);
|
||||
if (value.workerGeneration === undefined) {
|
||||
throw failure(400, 'invalid_worker_request');
|
||||
}
|
||||
const result = await options.attestations.submit({
|
||||
attestationId: value.attestationId as string,
|
||||
runId: value.runId as string,
|
||||
attemptId: value.attemptId as string,
|
||||
sequence: value.sequence as number,
|
||||
state: value.state as 'running' | 'stopped',
|
||||
workerId: resolved.workerId,
|
||||
workerSessionId: resolved.sessionId,
|
||||
workerGeneration: value.workerGeneration as number,
|
||||
leaseTokenDigest: value.leaseTokenDigest as string,
|
||||
leaseGeneration: value.leaseGeneration as number,
|
||||
leaseVersion: value.leaseVersion as number,
|
||||
offerId: value.offerId as string,
|
||||
callbackSequence: value.callbackSequence as number,
|
||||
executorHandle: value.executorHandle as string,
|
||||
journalRevision: value.journalRevision as number,
|
||||
});
|
||||
return response(result.status === 'created' ? 201 : 200, {
|
||||
attestationId: result.attestation.attestationId,
|
||||
sequence: result.attestation.sequence,
|
||||
state: result.attestation.state,
|
||||
receivedAtMs: result.attestation.receivedAtMs,
|
||||
replay: result.status === 'existing',
|
||||
});
|
||||
} catch (error) {
|
||||
return mapIngressFailure(error);
|
||||
}
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user