open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-23 11:53:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): complete cluster log retention lifecycle

Browse Source
This commit is contained in:
whyour
2026-08-12 04:41:59 +08:00
parent 40628c843d
commit 5d3b40ce3b
20 changed files with 2026 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-cluster-control/src/production-process/config.ts
+101
View File
@@ -12,6 +12,16 @@ import {
} from '@qinglong/cluster-postgres/runtime';
import { ClusterControlAvailabilityFence } from '../database/availability';
import type { ClusterControlHttpSurfaceOptions } from '../transport/httpSurface';
import {
MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_CLAIMS,
MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
MIN_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
} from '@qinglong/runtime-core/cluster-run-attempt-log-retention';
import {
MAX_RUN_ATTEMPT_LOG_RETENTION_MS,
MIN_RUN_ATTEMPT_LOG_RETENTION_MS,
} from '@qinglong/runtime-core/run-attempt-log-retention';
export type ClusterControlEnvironment = Readonly<
Record<string, string | undefined>
@@ -33,6 +43,20 @@ export interface EnabledClusterControlConfig {
readonly security: Readonly<{
apiCredentialPepper: string;
}>;
readonly logRetention:
| Readonly<{ readonly enabled: false }>
| Readonly<{
readonly enabled: true;
readonly retentionMs: number;
readonly claimLimit: number;
readonly leaseMs: number;
readonly maximumCycleMs: number;
readonly retryBaseMs: number;
readonly retryMaximumMs: number;
readonly maximumFailures: number;
readonly intervalMs: number;
readonly stopTimeoutMs: number;
}>;
}
export type ClusterControlConfig =
@@ -222,6 +246,82 @@ function apiCredentialPepper(environment: ClusterControlEnvironment): string {
return value;
}
function logRetentionConfig(
environment: ClusterControlEnvironment,
): EnabledClusterControlConfig['logRetention'] {
if (!booleanValue(environment, 'QL3_CLUSTER_LOG_RETENTION_ENABLED', true)) {
return Object.freeze({ enabled: false as const });
}
const leaseMs = integerValue(
environment,
'QL3_CLUSTER_LOG_RETENTION_LEASE_MS',
30_000,
MIN_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_LEASE_MS,
);
const retryBaseMs = integerValue(
environment,
'QL3_CLUSTER_LOG_RETENTION_RETRY_BASE_MS',
5_000,
0,
MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
);
return Object.freeze({
enabled: true as const,
retentionMs: integerValue(
environment,
'QL3_CLUSTER_LOG_RETENTION_MS',
30 * 24 * 60 * 60_000,
MIN_RUN_ATTEMPT_LOG_RETENTION_MS,
MAX_RUN_ATTEMPT_LOG_RETENTION_MS,
),
claimLimit: integerValue(
environment,
'QL3_CLUSTER_LOG_RETENTION_CLAIM_LIMIT',
4,
1,
MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_CLAIMS,
),
leaseMs,
maximumCycleMs: integerValue(
environment,
'QL3_CLUSTER_LOG_RETENTION_CYCLE_BUDGET_MS',
10_000,
100,
leaseMs - 500,
),
retryBaseMs,
retryMaximumMs: integerValue(
environment,
'QL3_CLUSTER_LOG_RETENTION_RETRY_MAX_MS',
60 * 60_000,
retryBaseMs,
MAX_CLUSTER_RUN_ATTEMPT_LOG_RETENTION_RETRY_DELAY_MS,
),
maximumFailures: integerValue(
environment,
'QL3_CLUSTER_LOG_RETENTION_MAX_FAILURES',
8,
1,
32,
),
intervalMs: integerValue(
environment,
'QL3_CLUSTER_LOG_RETENTION_INTERVAL_MS',
60_000,
1_000,
24 * 60 * 60_000,
),
stopTimeoutMs: integerValue(
environment,
'QL3_CLUSTER_LOG_RETENTION_STOP_TIMEOUT_MS',
10_000,
100,
30_000,
),
});
}
/**
* Parses the profile gate before reading PostgreSQL configuration. A disabled
* cluster-control therefore does not touch its runtime credential source.
@@ -345,6 +445,7 @@ export function loadClusterControlConfig(
security: Object.freeze({
apiCredentialPepper: apiCredentialPepper(environment),
}),
logRetention: logRetentionConfig(environment),
};
return Object.freeze(config);
}
packages/ql3-cluster-control/src/production-process/processApplication.ts
+47 -23
View File
@@ -38,6 +38,7 @@ export interface ClusterControlProcessEvent {
scope:
| 'scheduler'
| 'cancellation-convergence'
| 'log-retention'
| 'database'
| 'worker-ingress';
name: string;
@@ -78,10 +79,7 @@ export class ClusterControlProcessError extends Error {
| 'QL3_CLUSTER_CONTROL_PROCESS_CONFIG_INVALID'
| 'QL3_CLUSTER_CONTROL_PROCESS_DISABLED';
constructor(
code: ClusterControlProcessError['code'],
message: string,
) {
constructor(code: ClusterControlProcessError['code'], message: string) {
super(message);
this.name = 'ClusterControlProcessError';
this.code = code;
@@ -103,10 +101,7 @@ function processConfiguration(environment: ClusterControlEnvironment): {
);
}
const replicaId = environment.QL3_CLUSTER_REPLICA_ID;
if (
typeof replicaId !== 'string' ||
!REPLICA_ID_PATTERN.test(replicaId)
) {
if (typeof replicaId !== 'string' || !REPLICA_ID_PATTERN.test(replicaId)) {
throw new ClusterControlProcessError(
'QL3_CLUSTER_CONTROL_PROCESS_CONFIG_INVALID',
'QL3_CLUSTER_REPLICA_ID must be a stable safe identifier',
@@ -205,9 +200,11 @@ export async function runProductionClusterControlProcess(
let resolveSignal:
| ((signal: ClusterControlProcessSignal) => void)
| undefined;
const requestedSignal = new Promise<ClusterControlProcessSignal>((resolve) => {
resolveSignal = resolve;
});
const requestedSignal = new Promise<ClusterControlProcessSignal>(
(resolve) => {
resolveSignal = resolve;
},
);
let acceptedSignal = false;
const unsubscribe = options.signals.subscribe((signal) => {
if (acceptedSignal) return;
@@ -230,6 +227,14 @@ export async function runProductionClusterControlProcess(
);
}
artifactBinding = await createBinding(workerIngress.artifact);
if (
config.logRetention.enabled &&
typeof artifactBinding?.store?.retire !== 'function'
) {
throw new TypeError(
'Cluster Worker Artifact binding has no log retirement capability',
);
}
if (
workerIngress.secret !== undefined &&
workerSecretProvider === undefined
@@ -274,15 +279,40 @@ export async function runProductionClusterControlProcess(
event(replicaId, {
level: 'error',
event: 'runtime_diagnostic',
diagnostic: diagnosticFact(
'cancellation-convergence',
error,
),
diagnostic: diagnosticFact('cancellation-convergence', error),
}),
),
).catch(() => undefined);
},
},
...(workerIngress !== undefined && config.logRetention.enabled
? {
logRetention: {
store: artifactBinding!.store,
ownerId: replicaId,
retentionMs: config.logRetention.retentionMs,
claimLimit: config.logRetention.claimLimit,
leaseMs: config.logRetention.leaseMs,
maximumCycleMs: config.logRetention.maximumCycleMs,
retryBaseMs: config.logRetention.retryBaseMs,
retryMaximumMs: config.logRetention.retryMaximumMs,
maximumFailures: config.logRetention.maximumFailures,
intervalMs: config.logRetention.intervalMs,
stopTimeoutMs: config.logRetention.stopTimeoutMs,
onDiagnostic(error: unknown) {
void Promise.resolve(
options.emit(
event(replicaId, {
level: 'error',
event: 'runtime_diagnostic',
diagnostic: diagnosticFact('log-retention', error),
}),
),
).catch(() => undefined);
},
},
}
: {}),
...(workerIngress === undefined
? {}
: {
@@ -298,10 +328,7 @@ export async function runProductionClusterControlProcess(
event(replicaId, {
level: 'error',
event: 'runtime_diagnostic',
diagnostic: diagnosticFact(
'worker-ingress',
error,
),
diagnostic: diagnosticFact('worker-ingress', error),
}),
),
).catch(() => undefined);
@@ -395,10 +422,7 @@ export async function runProductionClusterControlProcess(
unsubscribe();
resolveSignal = undefined;
let cleanupError: unknown;
if (
application?.status === 'active' &&
!applicationStopStarted
) {
if (application?.status === 'active' && !applicationStopStarted) {
try {
applicationStopStarted = true;
await application.stop();