mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 10:32:40 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,372 @@
|
||||
import type {
|
||||
DeploymentProfile,
|
||||
OpenPostgresDatabase,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
createPostgresDatabaseOpener,
|
||||
isPostgresTlsDnsServername,
|
||||
loadPostgresConnectionEnvironment,
|
||||
loadPostgresCertificateAuthorityFile,
|
||||
type PostgresConnectionOptions,
|
||||
type PostgresPoolOptions,
|
||||
} from '@qinglong/cluster-postgres/runtime';
|
||||
import { ClusterControlAvailabilityFence } from '../database/availability';
|
||||
import type { ClusterControlHttpSurfaceOptions } from '../transport/httpSurface';
|
||||
|
||||
export type ClusterControlEnvironment = Readonly<
|
||||
Record<string, string | undefined>
|
||||
>;
|
||||
|
||||
export interface DisabledClusterControlConfig {
|
||||
readonly enabled: false;
|
||||
readonly profile: DeploymentProfile;
|
||||
}
|
||||
|
||||
export interface EnabledClusterControlConfig {
|
||||
readonly enabled: true;
|
||||
readonly profile: 'cluster-control';
|
||||
readonly http: ClusterControlHttpSurfaceOptions;
|
||||
readonly database: Readonly<{
|
||||
connection: PostgresConnectionOptions;
|
||||
pool: PostgresPoolOptions;
|
||||
}>;
|
||||
readonly security: Readonly<{
|
||||
apiCredentialPepper: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export type ClusterControlConfig =
|
||||
| DisabledClusterControlConfig
|
||||
| EnabledClusterControlConfig;
|
||||
|
||||
export interface ClusterControlDatabaseBinding {
|
||||
readonly availability: ClusterControlAvailabilityFence;
|
||||
readonly openDatabase: OpenPostgresDatabase;
|
||||
}
|
||||
|
||||
export class ClusterControlConfigError extends TypeError {
|
||||
constructor(message: string) {
|
||||
super(`Cluster-control configuration is invalid: ${message}`);
|
||||
this.name = 'ClusterControlConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
const PROFILES = new Set<DeploymentProfile>([
|
||||
'edge',
|
||||
'standalone',
|
||||
'cluster-control',
|
||||
'worker',
|
||||
]);
|
||||
|
||||
function booleanValue(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
defaultValue: boolean,
|
||||
): boolean {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return defaultValue;
|
||||
if (value === 'true') return true;
|
||||
if (value === 'false') return false;
|
||||
throw new ClusterControlConfigError(`${name} must be true or false`);
|
||||
}
|
||||
|
||||
function integerValue(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
defaultValue: number,
|
||||
minimum: number,
|
||||
maximum: number,
|
||||
): number {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') return defaultValue;
|
||||
if (!/^\d+$/.test(value)) {
|
||||
throw new ClusterControlConfigError(`${name} must be an integer`);
|
||||
}
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
|
||||
throw new ClusterControlConfigError(
|
||||
`${name} must be between ${minimum} and ${maximum}`,
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function boundedValue(
|
||||
environment: ClusterControlEnvironment,
|
||||
name: string,
|
||||
maximumLength: number,
|
||||
required = false,
|
||||
): string | undefined {
|
||||
const value = environment[name];
|
||||
if (value === undefined || value === '') {
|
||||
if (required) throw new ClusterControlConfigError(`${name} is required`);
|
||||
return undefined;
|
||||
}
|
||||
if (value.length > maximumLength || /[\0\r\n]/.test(value)) {
|
||||
throw new ClusterControlConfigError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function deploymentProfile(
|
||||
environment: ClusterControlEnvironment,
|
||||
): DeploymentProfile {
|
||||
const value = environment.QL_DEPLOYMENT_PROFILE ?? 'standalone';
|
||||
if (!PROFILES.has(value as DeploymentProfile)) {
|
||||
throw new ClusterControlConfigError('QL_DEPLOYMENT_PROFILE is invalid');
|
||||
}
|
||||
return value as DeploymentProfile;
|
||||
}
|
||||
|
||||
function runtimeConnection(
|
||||
environment: ClusterControlEnvironment,
|
||||
): PostgresConnectionOptions {
|
||||
let connection: PostgresConnectionOptions;
|
||||
try {
|
||||
connection = loadPostgresConnectionEnvironment(environment, {
|
||||
connectionString: 'QL3_POSTGRES_RUNTIME_URL',
|
||||
host: 'QL3_POSTGRES_RUNTIME_HOST',
|
||||
port: 'QL3_POSTGRES_RUNTIME_PORT',
|
||||
database: 'QL3_POSTGRES_RUNTIME_DATABASE',
|
||||
user: 'QL3_POSTGRES_RUNTIME_USER',
|
||||
password: 'QL3_POSTGRES_RUNTIME_PASSWORD',
|
||||
});
|
||||
} catch (error) {
|
||||
throw new ClusterControlConfigError(
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: 'PostgreSQL runtime connection is invalid',
|
||||
);
|
||||
}
|
||||
|
||||
const mode = environment.QL3_POSTGRES_TLS_MODE ?? 'verify-full';
|
||||
if (mode !== 'verify-full' && mode !== 'disable') {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_POSTGRES_TLS_MODE must be verify-full or disable',
|
||||
);
|
||||
}
|
||||
if (
|
||||
mode === 'disable' &&
|
||||
!booleanValue(environment, 'QL3_POSTGRES_ALLOW_INSECURE', false)
|
||||
) {
|
||||
throw new ClusterControlConfigError(
|
||||
'disabling PostgreSQL TLS requires QL3_POSTGRES_ALLOW_INSECURE=true',
|
||||
);
|
||||
}
|
||||
const servername = boundedValue(
|
||||
environment,
|
||||
'QL3_POSTGRES_TLS_SERVERNAME',
|
||||
253,
|
||||
);
|
||||
if (mode === 'verify-full' && !isPostgresTlsDnsServername(servername)) {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_POSTGRES_TLS_SERVERNAME must be an explicit DNS name for verify-full',
|
||||
);
|
||||
}
|
||||
const certificateAuthorityFile = boundedValue(
|
||||
environment,
|
||||
'QL3_POSTGRES_TLS_CA_FILE',
|
||||
4096,
|
||||
);
|
||||
if (mode === 'disable' && certificateAuthorityFile !== undefined) {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_POSTGRES_TLS_CA_FILE cannot be used when TLS is disabled',
|
||||
);
|
||||
}
|
||||
let certificateAuthority: string | undefined;
|
||||
if (certificateAuthorityFile !== undefined) {
|
||||
try {
|
||||
certificateAuthority = loadPostgresCertificateAuthorityFile(
|
||||
certificateAuthorityFile,
|
||||
);
|
||||
} catch {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_POSTGRES_TLS_CA_FILE must contain a bounded trusted CA bundle',
|
||||
);
|
||||
}
|
||||
}
|
||||
return Object.freeze({
|
||||
...connection,
|
||||
tls:
|
||||
mode === 'disable'
|
||||
? Object.freeze({ mode: 'disable' as const })
|
||||
: Object.freeze({
|
||||
mode: 'verify-full' as const,
|
||||
...(certificateAuthority === undefined
|
||||
? {}
|
||||
: { ca: certificateAuthority }),
|
||||
servername: servername!,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function apiCredentialPepper(environment: ClusterControlEnvironment): string {
|
||||
const value = boundedValue(
|
||||
environment,
|
||||
'QL3_API_CREDENTIAL_PEPPER',
|
||||
64,
|
||||
true,
|
||||
)!;
|
||||
if (!/^[A-Za-z0-9_-]{43}$/.test(value)) {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_API_CREDENTIAL_PEPPER must be canonical base64url for 32 bytes',
|
||||
);
|
||||
}
|
||||
const decoded = Buffer.from(value, 'base64url');
|
||||
if (decoded.byteLength !== 32 || decoded.toString('base64url') !== value) {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_API_CREDENTIAL_PEPPER must be canonical base64url for 32 bytes',
|
||||
);
|
||||
}
|
||||
decoded.fill(0);
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the profile gate before reading PostgreSQL configuration. A disabled
|
||||
* cluster-control therefore does not touch its runtime credential source.
|
||||
*/
|
||||
export function loadClusterControlConfig(
|
||||
environment: ClusterControlEnvironment,
|
||||
): ClusterControlConfig {
|
||||
if (
|
||||
!environment ||
|
||||
typeof environment !== 'object' ||
|
||||
Array.isArray(environment)
|
||||
) {
|
||||
throw new ClusterControlConfigError('environment must be an object');
|
||||
}
|
||||
const profile = deploymentProfile(environment);
|
||||
const enabled = booleanValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_CONTROL_ENABLED',
|
||||
false,
|
||||
);
|
||||
if (!enabled) return Object.freeze({ enabled: false, profile });
|
||||
if (profile !== 'cluster-control') {
|
||||
throw new ClusterControlConfigError(
|
||||
'enabled runtime requires QL_DEPLOYMENT_PROFILE=cluster-control',
|
||||
);
|
||||
}
|
||||
|
||||
const applicationName =
|
||||
boundedValue(environment, 'QL3_POSTGRES_APPLICATION_NAME', 63) ??
|
||||
'qinglong-cluster-runtime';
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,62}$/.test(applicationName)) {
|
||||
throw new ClusterControlConfigError(
|
||||
'QL3_POSTGRES_APPLICATION_NAME is invalid',
|
||||
);
|
||||
}
|
||||
const host =
|
||||
boundedValue(environment, 'QL3_CLUSTER_HTTP_HOST', 253) ?? '0.0.0.0';
|
||||
const config: EnabledClusterControlConfig = {
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
http: Object.freeze({
|
||||
host,
|
||||
port: integerValue(environment, 'QL3_CLUSTER_HTTP_PORT', 5800, 1, 65_535),
|
||||
maxBodyBytes: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_HTTP_MAX_BODY_BYTES',
|
||||
1024 * 1024,
|
||||
1024,
|
||||
4 * 1024 * 1024,
|
||||
),
|
||||
maxInFlightRequests: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_HTTP_MAX_IN_FLIGHT',
|
||||
64,
|
||||
1,
|
||||
1024,
|
||||
),
|
||||
authenticationRateWindowMs: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_AUTH_RATE_WINDOW_MS',
|
||||
60_000,
|
||||
1_000,
|
||||
60 * 60_000,
|
||||
),
|
||||
authenticationRatePerPeer: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_AUTH_RATE_PER_PEER',
|
||||
300,
|
||||
1,
|
||||
1_000_000,
|
||||
),
|
||||
authenticationRateGlobal: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_AUTH_RATE_GLOBAL',
|
||||
1_200,
|
||||
1,
|
||||
1_000_000,
|
||||
),
|
||||
authenticationRateMaxPeers: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_AUTH_RATE_MAX_PEERS',
|
||||
4_096,
|
||||
1,
|
||||
65_536,
|
||||
),
|
||||
requestTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_HTTP_REQUEST_TIMEOUT_MS',
|
||||
15_000,
|
||||
100,
|
||||
120_000,
|
||||
),
|
||||
drainTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_CLUSTER_HTTP_DRAIN_TIMEOUT_MS',
|
||||
10_000,
|
||||
100,
|
||||
120_000,
|
||||
),
|
||||
}),
|
||||
database: Object.freeze({
|
||||
connection: runtimeConnection(environment),
|
||||
pool: Object.freeze({
|
||||
applicationName,
|
||||
maxConnections: integerValue(
|
||||
environment,
|
||||
'QL3_POSTGRES_MAX_CONNECTIONS',
|
||||
8,
|
||||
1,
|
||||
64,
|
||||
),
|
||||
connectionTimeoutMs: integerValue(
|
||||
environment,
|
||||
'QL3_POSTGRES_CONNECTION_TIMEOUT_MS',
|
||||
5_000,
|
||||
100,
|
||||
60_000,
|
||||
),
|
||||
}),
|
||||
}),
|
||||
security: Object.freeze({
|
||||
apiCredentialPepper: apiCredentialPepper(environment),
|
||||
}),
|
||||
};
|
||||
return Object.freeze(config);
|
||||
}
|
||||
|
||||
export function createClusterControlDatabaseBinding(
|
||||
config: EnabledClusterControlConfig,
|
||||
): ClusterControlDatabaseBinding {
|
||||
if (!config?.enabled || config.profile !== 'cluster-control') {
|
||||
throw new ClusterControlConfigError(
|
||||
'database binding requires an enabled cluster-control config',
|
||||
);
|
||||
}
|
||||
const availability = new ClusterControlAvailabilityFence();
|
||||
const openDatabase = createPostgresDatabaseOpener({
|
||||
role: 'runtime',
|
||||
connection: config.database.connection,
|
||||
pool: config.database.pool,
|
||||
onPoolError(error) {
|
||||
// pg emits idle-client errors outside a request Promise. They are an
|
||||
// availability signal, never a callback exception or transaction retry.
|
||||
void availability.signal(error).catch(() => undefined);
|
||||
},
|
||||
});
|
||||
return Object.freeze({ availability, openDatabase });
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
import type {
|
||||
ClusterControlActivationAudit,
|
||||
ClusterControlStopResult,
|
||||
} from '@qinglong/runtime-core';
|
||||
import {
|
||||
loadClusterControlConfig,
|
||||
type ClusterControlEnvironment,
|
||||
type EnabledClusterControlConfig,
|
||||
} from './config';
|
||||
import {
|
||||
startProductionClusterControlApplication,
|
||||
type ProductionClusterControlApplicationOptions,
|
||||
} from '../application-runtime/productionApplication';
|
||||
import {
|
||||
ClusterControlDatabaseUnavailableError,
|
||||
type ClusterControlApplicationResult,
|
||||
} from '../application-runtime/application';
|
||||
import {
|
||||
loadClusterWorkerIngressConfig,
|
||||
type EnabledClusterWorkerIngressConfig,
|
||||
} from '../worker-ingress/workerIngressConfig';
|
||||
import type { ClusterWorkerArtifactBinding } from '../artifact/workerArtifactBinding';
|
||||
import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
|
||||
export type ClusterControlProcessSignal = 'SIGINT' | 'SIGTERM';
|
||||
|
||||
export interface ClusterControlProcessEvent {
|
||||
readonly schemaVersion: 1;
|
||||
readonly component: 'qinglong3-cluster-control';
|
||||
readonly level: 'info' | 'error';
|
||||
readonly event: string;
|
||||
readonly replicaId: string;
|
||||
readonly signal?: ClusterControlProcessSignal;
|
||||
readonly stopResult?: ClusterControlStopResult;
|
||||
readonly address?: Readonly<{ host: string; port: number }>;
|
||||
readonly activation?: ClusterControlActivationAudit;
|
||||
readonly diagnostic?: Readonly<{
|
||||
scope:
|
||||
| 'scheduler'
|
||||
| 'cancellation-convergence'
|
||||
| 'database'
|
||||
| 'worker-ingress';
|
||||
name: string;
|
||||
code?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface ClusterControlProcessSignalSource {
|
||||
subscribe(
|
||||
listener: (signal: ClusterControlProcessSignal) => void,
|
||||
): () => void;
|
||||
}
|
||||
|
||||
export type ProductionClusterControlStarter = (
|
||||
options: ProductionClusterControlApplicationOptions,
|
||||
) => Promise<ClusterControlApplicationResult>;
|
||||
|
||||
export type ClusterWorkerArtifactBindingFactory = (
|
||||
config: EnabledClusterWorkerIngressConfig['artifact'],
|
||||
) => Promise<Readonly<ClusterWorkerArtifactBinding>>;
|
||||
|
||||
export type ClusterWorkerSecretProviderFactory = (
|
||||
config: NonNullable<EnabledClusterWorkerIngressConfig['secret']>,
|
||||
) => Promise<Readonly<RemoteWorkerSecretValueProvider>>;
|
||||
|
||||
export interface ProductionClusterControlProcessOptions {
|
||||
readonly environment: ClusterControlEnvironment;
|
||||
readonly signals: ClusterControlProcessSignalSource;
|
||||
readonly emit: (event: ClusterControlProcessEvent) => void | Promise<void>;
|
||||
readonly start?: ProductionClusterControlStarter;
|
||||
readonly createWorkerArtifactBinding?: ClusterWorkerArtifactBindingFactory;
|
||||
readonly createWorkerSecretProvider?: ClusterWorkerSecretProviderFactory;
|
||||
readonly workerSecretProvider?: RemoteWorkerSecretValueProvider;
|
||||
}
|
||||
|
||||
export class ClusterControlProcessError extends Error {
|
||||
readonly code:
|
||||
| 'QL3_CLUSTER_CONTROL_PROCESS_CONFIG_INVALID'
|
||||
| 'QL3_CLUSTER_CONTROL_PROCESS_DISABLED';
|
||||
|
||||
constructor(
|
||||
code: ClusterControlProcessError['code'],
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ClusterControlProcessError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
const REPLICA_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
|
||||
function processConfiguration(environment: ClusterControlEnvironment): {
|
||||
readonly config: EnabledClusterControlConfig;
|
||||
readonly workerIngress?: EnabledClusterWorkerIngressConfig;
|
||||
readonly replicaId: string;
|
||||
} {
|
||||
const config = loadClusterControlConfig(environment);
|
||||
if (!config.enabled) {
|
||||
throw new ClusterControlProcessError(
|
||||
'QL3_CLUSTER_CONTROL_PROCESS_DISABLED',
|
||||
'The cluster-control process requires an enabled cluster-control profile',
|
||||
);
|
||||
}
|
||||
const replicaId = environment.QL3_CLUSTER_REPLICA_ID;
|
||||
if (
|
||||
typeof replicaId !== 'string' ||
|
||||
!REPLICA_ID_PATTERN.test(replicaId)
|
||||
) {
|
||||
throw new ClusterControlProcessError(
|
||||
'QL3_CLUSTER_CONTROL_PROCESS_CONFIG_INVALID',
|
||||
'QL3_CLUSTER_REPLICA_ID must be a stable safe identifier',
|
||||
);
|
||||
}
|
||||
const workerIngress = loadClusterWorkerIngressConfig(environment);
|
||||
return Object.freeze({
|
||||
config,
|
||||
replicaId,
|
||||
...(workerIngress.enabled ? { workerIngress } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function createWorkerArtifactBinding(
|
||||
config: EnabledClusterWorkerIngressConfig['artifact'],
|
||||
): Promise<Readonly<ClusterWorkerArtifactBinding>> {
|
||||
const binding = await import('../artifact/workerArtifactBinding.js');
|
||||
return binding.createClusterWorkerArtifactBinding(config);
|
||||
}
|
||||
|
||||
async function createWorkerSecretProvider(
|
||||
config: NonNullable<EnabledClusterWorkerIngressConfig['secret']>,
|
||||
): Promise<Readonly<RemoteWorkerSecretValueProvider>> {
|
||||
if (config.provider !== 'mounted-files') {
|
||||
throw new TypeError('Cluster Worker Secret provider is unsupported');
|
||||
}
|
||||
const provider = await import('../remote-execution/mountedSecretProvider.js');
|
||||
return provider.createClusterMountedSecretProvider({
|
||||
rootDirectory: config.rootDirectory,
|
||||
});
|
||||
}
|
||||
|
||||
function diagnosticFact(
|
||||
scope: ClusterControlProcessEvent['diagnostic'] extends infer T
|
||||
? T extends { readonly scope: infer TScope }
|
||||
? TScope
|
||||
: never
|
||||
: never,
|
||||
error: unknown,
|
||||
): NonNullable<ClusterControlProcessEvent['diagnostic']> {
|
||||
const candidate = error as {
|
||||
readonly name?: unknown;
|
||||
readonly code?: unknown;
|
||||
};
|
||||
return Object.freeze({
|
||||
scope,
|
||||
name:
|
||||
typeof candidate?.name === 'string' && candidate.name.length <= 128
|
||||
? candidate.name
|
||||
: 'Error',
|
||||
...(typeof candidate?.code === 'string' && candidate.code.length <= 128
|
||||
? { code: candidate.code }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
function event(
|
||||
replicaId: string,
|
||||
values: Omit<
|
||||
ClusterControlProcessEvent,
|
||||
'schemaVersion' | 'component' | 'replicaId'
|
||||
>,
|
||||
): ClusterControlProcessEvent {
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
component: 'qinglong3-cluster-control',
|
||||
replicaId,
|
||||
...values,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns exactly one production cluster-control process. It installs signal
|
||||
* handling before startup, derives every lease owner from the stable replica
|
||||
* identity, and withdraws admission through the application stop contract.
|
||||
*/
|
||||
export async function runProductionClusterControlProcess(
|
||||
options: ProductionClusterControlProcessOptions,
|
||||
): Promise<ClusterControlStopResult> {
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
typeof options.emit !== 'function' ||
|
||||
typeof options.signals?.subscribe !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster-control process options are invalid');
|
||||
}
|
||||
const { config, replicaId, workerIngress } = processConfiguration(
|
||||
options.environment,
|
||||
);
|
||||
const start = options.start ?? startProductionClusterControlApplication;
|
||||
if (typeof start !== 'function') {
|
||||
throw new TypeError('Cluster-control process starter is invalid');
|
||||
}
|
||||
|
||||
let resolveSignal:
|
||||
| ((signal: ClusterControlProcessSignal) => void)
|
||||
| undefined;
|
||||
const requestedSignal = new Promise<ClusterControlProcessSignal>((resolve) => {
|
||||
resolveSignal = resolve;
|
||||
});
|
||||
let acceptedSignal = false;
|
||||
const unsubscribe = options.signals.subscribe((signal) => {
|
||||
if (acceptedSignal) return;
|
||||
acceptedSignal = true;
|
||||
resolveSignal?.(signal);
|
||||
});
|
||||
|
||||
let artifactBinding: Readonly<ClusterWorkerArtifactBinding> | undefined;
|
||||
let workerSecretProvider = options.workerSecretProvider;
|
||||
let application: ClusterControlApplicationResult | undefined;
|
||||
let applicationStopStarted = false;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
if (workerIngress) {
|
||||
const createBinding =
|
||||
options.createWorkerArtifactBinding ?? createWorkerArtifactBinding;
|
||||
if (typeof createBinding !== 'function') {
|
||||
throw new TypeError(
|
||||
'Cluster Worker Artifact binding factory is invalid',
|
||||
);
|
||||
}
|
||||
artifactBinding = await createBinding(workerIngress.artifact);
|
||||
if (
|
||||
workerIngress.secret !== undefined &&
|
||||
workerSecretProvider === undefined
|
||||
) {
|
||||
const createProvider =
|
||||
options.createWorkerSecretProvider ?? createWorkerSecretProvider;
|
||||
if (typeof createProvider !== 'function') {
|
||||
throw new TypeError(
|
||||
'Cluster Worker Secret provider factory is invalid',
|
||||
);
|
||||
}
|
||||
workerSecretProvider = await createProvider(workerIngress.secret);
|
||||
}
|
||||
if (
|
||||
workerSecretProvider !== undefined &&
|
||||
typeof workerSecretProvider.resolve !== 'function'
|
||||
) {
|
||||
throw new TypeError('Cluster Worker Secret provider is invalid');
|
||||
}
|
||||
}
|
||||
application = await start({
|
||||
config,
|
||||
recovery: { ownerId: replicaId },
|
||||
scheduler: {
|
||||
ownerId: replicaId,
|
||||
onDiagnostic(error) {
|
||||
void Promise.resolve(
|
||||
options.emit(
|
||||
event(replicaId, {
|
||||
level: 'error',
|
||||
event: 'runtime_diagnostic',
|
||||
diagnostic: diagnosticFact('scheduler', error),
|
||||
}),
|
||||
),
|
||||
).catch(() => undefined);
|
||||
},
|
||||
},
|
||||
cancellationConvergence: {
|
||||
onDiagnostic(error) {
|
||||
void Promise.resolve(
|
||||
options.emit(
|
||||
event(replicaId, {
|
||||
level: 'error',
|
||||
event: 'runtime_diagnostic',
|
||||
diagnostic: diagnosticFact(
|
||||
'cancellation-convergence',
|
||||
error,
|
||||
),
|
||||
}),
|
||||
),
|
||||
).catch(() => undefined);
|
||||
},
|
||||
},
|
||||
...(workerIngress === undefined
|
||||
? {}
|
||||
: {
|
||||
workerIngress: {
|
||||
config: workerIngress,
|
||||
artifactStore: artifactBinding!.store,
|
||||
...(workerSecretProvider === undefined
|
||||
? {}
|
||||
: { secretProvider: workerSecretProvider }),
|
||||
onDiagnostic(error: unknown) {
|
||||
void Promise.resolve(
|
||||
options.emit(
|
||||
event(replicaId, {
|
||||
level: 'error',
|
||||
event: 'runtime_diagnostic',
|
||||
diagnostic: diagnosticFact(
|
||||
'worker-ingress',
|
||||
error,
|
||||
),
|
||||
}),
|
||||
),
|
||||
).catch(() => undefined);
|
||||
},
|
||||
},
|
||||
}),
|
||||
audit(record) {
|
||||
return options.emit(
|
||||
event(replicaId, {
|
||||
level: record.state === 'failed' ? 'error' : 'info',
|
||||
event: 'activation',
|
||||
activation: Object.freeze({ ...record }),
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
if (application.status !== 'active') {
|
||||
throw new ClusterControlProcessError(
|
||||
'QL3_CLUSTER_CONTROL_PROCESS_DISABLED',
|
||||
'The cluster-control process did not activate',
|
||||
);
|
||||
}
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: 'info',
|
||||
event: 'listening',
|
||||
address: application.address,
|
||||
}),
|
||||
);
|
||||
if (workerIngress) {
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: 'info',
|
||||
event: 'worker_ingress_listening',
|
||||
address: Object.freeze({
|
||||
host: workerIngress.http.host ?? '0.0.0.0',
|
||||
port: workerIngress.http.port ?? 5801,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
const termination = await Promise.race([
|
||||
requestedSignal.then((signal) =>
|
||||
Object.freeze({ kind: 'signal' as const, signal }),
|
||||
),
|
||||
application.unavailable.then((error) =>
|
||||
Object.freeze({ kind: 'database-unavailable' as const, error }),
|
||||
),
|
||||
]);
|
||||
if (termination.kind === 'database-unavailable') {
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: 'error',
|
||||
event: 'database_unavailable',
|
||||
diagnostic: diagnosticFact('database', termination.error),
|
||||
}),
|
||||
);
|
||||
applicationStopStarted = true;
|
||||
const stopResult = await application.stop();
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: stopResult === 'stopped' ? 'info' : 'error',
|
||||
event: 'stopped',
|
||||
stopResult,
|
||||
}),
|
||||
);
|
||||
throw new ClusterControlDatabaseUnavailableError();
|
||||
}
|
||||
const signal = termination.signal;
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: 'info',
|
||||
event: 'shutdown_requested',
|
||||
signal,
|
||||
}),
|
||||
);
|
||||
applicationStopStarted = true;
|
||||
const stopResult = await application.stop();
|
||||
await options.emit(
|
||||
event(replicaId, {
|
||||
level: stopResult === 'stopped' ? 'info' : 'error',
|
||||
event: 'stopped',
|
||||
stopResult,
|
||||
}),
|
||||
);
|
||||
return stopResult;
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
throw error;
|
||||
} finally {
|
||||
unsubscribe();
|
||||
resolveSignal = undefined;
|
||||
let cleanupError: unknown;
|
||||
if (
|
||||
application?.status === 'active' &&
|
||||
!applicationStopStarted
|
||||
) {
|
||||
try {
|
||||
applicationStopStarted = true;
|
||||
await application.stop();
|
||||
} catch (error) {
|
||||
cleanupError = error;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await artifactBinding?.close();
|
||||
} catch (error) {
|
||||
cleanupError ??= error;
|
||||
}
|
||||
if (cleanupError && primaryError === undefined) throw cleanupError;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user