mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 18:08:20 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,362 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
ClusterControlProcessError,
|
||||
runProductionClusterControlProcess,
|
||||
} = require('@qinglong/cluster-control/process');
|
||||
|
||||
const BASE_ENV = Object.freeze({
|
||||
QL3_CLUSTER_CONTROL_ENABLED: 'true',
|
||||
QL_DEPLOYMENT_PROFILE: 'cluster-control',
|
||||
QL3_CLUSTER_REPLICA_ID: 'cluster-control-0',
|
||||
QL3_POSTGRES_RUNTIME_URL:
|
||||
'postgresql://ql3_runtime:do-not-log@postgres-rw.internal:5432/qinglong',
|
||||
QL3_POSTGRES_TLS_SERVERNAME: 'postgres-rw.internal',
|
||||
QL3_API_CREDENTIAL_PEPPER: 'A'.repeat(43),
|
||||
});
|
||||
const NEVER_UNAVAILABLE = new Promise(() => {});
|
||||
|
||||
function signalSource(events, signal = 'SIGTERM') {
|
||||
return {
|
||||
subscribe(listener) {
|
||||
events.push('subscribe');
|
||||
queueMicrotask(() => listener(signal));
|
||||
return () => events.push('unsubscribe');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('runs one production replica and drains it on the first signal', async () => {
|
||||
const events = [];
|
||||
const facts = [];
|
||||
const result = await runProductionClusterControlProcess({
|
||||
environment: BASE_ENV,
|
||||
signals: signalSource(events),
|
||||
emit(record) {
|
||||
facts.push(record);
|
||||
},
|
||||
async start(options) {
|
||||
events.push('start');
|
||||
assert.equal(options.config.enabled, true);
|
||||
assert.equal(options.config.profile, 'cluster-control');
|
||||
assert.equal(options.recovery.ownerId, 'cluster-control-0');
|
||||
assert.equal(options.scheduler.ownerId, 'cluster-control-0');
|
||||
await options.audit({
|
||||
state: 'active',
|
||||
contractName: 'qinglong-cluster-control',
|
||||
contractVersion: 16,
|
||||
serverMajor: 18,
|
||||
migrationCount: 16,
|
||||
});
|
||||
options.scheduler.onDiagnostic(
|
||||
Object.assign(new Error('must-not-be-logged'), {
|
||||
code: 'ECONNRESET',
|
||||
}),
|
||||
);
|
||||
return {
|
||||
status: 'active',
|
||||
address: { host: '0.0.0.0', port: 5800 },
|
||||
evidence: {
|
||||
contractName: 'qinglong-cluster-control',
|
||||
contractVersion: 16,
|
||||
serverMajor: 18,
|
||||
migrationIds: [],
|
||||
},
|
||||
recovery: { safe: true, remaining: 0, failed: 0 },
|
||||
unavailable: NEVER_UNAVAILABLE,
|
||||
availabilityStatus: () => 'ready',
|
||||
async stop() {
|
||||
events.push('stop');
|
||||
return 'stopped';
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result, 'stopped');
|
||||
assert.deepEqual(events, ['subscribe', 'start', 'stop', 'unsubscribe']);
|
||||
assert.equal(facts.some((fact) => fact.event === 'activation'), true);
|
||||
assert.equal(facts.some((fact) => fact.event === 'listening'), true);
|
||||
assert.equal(
|
||||
facts.some(
|
||||
(fact) =>
|
||||
fact.event === 'shutdown_requested' && fact.signal === 'SIGTERM',
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(facts.at(-1).event, 'stopped');
|
||||
assert.equal(facts.at(-1).stopResult, 'stopped');
|
||||
const serialized = JSON.stringify(facts);
|
||||
assert.equal(serialized.includes('do-not-log'), false);
|
||||
assert.equal(serialized.includes(BASE_ENV.QL3_API_CREDENTIAL_PEPPER), false);
|
||||
assert.equal(serialized.includes('must-not-be-logged'), false);
|
||||
assert.equal(serialized.includes('ECONNRESET'), true);
|
||||
});
|
||||
|
||||
test('fails closed before startup for a disabled profile or invalid replica id', async () => {
|
||||
let starts = 0;
|
||||
for (const environment of [
|
||||
{
|
||||
QL3_CLUSTER_CONTROL_ENABLED: 'false',
|
||||
QL_DEPLOYMENT_PROFILE: 'standalone',
|
||||
},
|
||||
{ ...BASE_ENV, QL3_CLUSTER_REPLICA_ID: 'unsafe replica' },
|
||||
]) {
|
||||
await assert.rejects(
|
||||
runProductionClusterControlProcess({
|
||||
environment,
|
||||
signals: { subscribe() { return () => {}; } },
|
||||
emit() {},
|
||||
async start() {
|
||||
starts += 1;
|
||||
throw new Error('must not start');
|
||||
},
|
||||
}),
|
||||
ClusterControlProcessError,
|
||||
);
|
||||
}
|
||||
assert.equal(starts, 0);
|
||||
});
|
||||
|
||||
test('propagates timed-out drain and always releases signal ownership', async () => {
|
||||
const events = [];
|
||||
const result = await runProductionClusterControlProcess({
|
||||
environment: BASE_ENV,
|
||||
signals: signalSource(events, 'SIGINT'),
|
||||
emit() {},
|
||||
async start() {
|
||||
return {
|
||||
status: 'active',
|
||||
address: { host: '127.0.0.1', port: 5800 },
|
||||
evidence: {
|
||||
contractName: 'qinglong-cluster-control',
|
||||
contractVersion: 16,
|
||||
serverMajor: 18,
|
||||
migrationIds: [],
|
||||
},
|
||||
recovery: { safe: true, remaining: 0, failed: 0 },
|
||||
unavailable: NEVER_UNAVAILABLE,
|
||||
availabilityStatus: () => 'ready',
|
||||
async stop() {
|
||||
events.push('stop');
|
||||
return 'timed_out';
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.equal(result, 'timed_out');
|
||||
assert.deepEqual(events, ['subscribe', 'stop', 'unsubscribe']);
|
||||
});
|
||||
|
||||
test('starts the optional Worker listener and closes its lazy Artifact binding', async () => {
|
||||
const events = [];
|
||||
const facts = [];
|
||||
const artifactStore = {
|
||||
async put() {},
|
||||
async inspect() {},
|
||||
};
|
||||
const environment = {
|
||||
...BASE_ENV,
|
||||
QL3_WORKER_INGRESS_ENABLED: 'true',
|
||||
QL3_POSTGRES_WORKER_INGRESS_URL:
|
||||
'postgresql://ql3_worker_ingress:secret@postgres-rw.internal:5432/qinglong',
|
||||
QL3_WORKER_INGRESS_POSTGRES_TLS_SERVERNAME: 'postgres-rw.internal',
|
||||
QL3_WORKER_CREDENTIAL_PEPPER: 'A'.repeat(43),
|
||||
QL3_WORKER_INGRESS_TLS_PRIVATE_KEY_FILE: '/run/worker/tls.key',
|
||||
QL3_WORKER_INGRESS_TLS_CERTIFICATE_FILE: '/run/worker/tls.crt',
|
||||
QL3_WORKER_INGRESS_TLS_CLIENT_CA_FILE: '/run/worker/client-ca.crt',
|
||||
QL3_WORKER_ARTIFACT_S3_BUCKET: 'qinglong-worker-artifacts',
|
||||
QL3_WORKER_ARTIFACT_S3_REGION: 'us-east-1',
|
||||
};
|
||||
const result = await runProductionClusterControlProcess({
|
||||
environment,
|
||||
signals: signalSource(events),
|
||||
emit(record) {
|
||||
facts.push(record);
|
||||
},
|
||||
async createWorkerArtifactBinding(config) {
|
||||
events.push(`artifact:${config.bucket}:${config.region}`);
|
||||
return {
|
||||
store: artifactStore,
|
||||
async close() {
|
||||
events.push('close-artifact');
|
||||
},
|
||||
};
|
||||
},
|
||||
async start(options) {
|
||||
events.push('start');
|
||||
assert.equal(options.workerIngress.config.enabled, true);
|
||||
assert.equal(options.workerIngress.artifactStore, artifactStore);
|
||||
return {
|
||||
status: 'active',
|
||||
address: { host: '0.0.0.0', port: 5800 },
|
||||
evidence: {
|
||||
contractName: 'qinglong-cluster-control',
|
||||
contractVersion: 16,
|
||||
serverMajor: 18,
|
||||
migrationIds: [],
|
||||
},
|
||||
recovery: { safe: true, remaining: 0, failed: 0 },
|
||||
unavailable: NEVER_UNAVAILABLE,
|
||||
availabilityStatus: () => 'ready',
|
||||
async stop() {
|
||||
events.push('stop');
|
||||
return 'stopped';
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.equal(result, 'stopped');
|
||||
assert.deepEqual(events, [
|
||||
'subscribe',
|
||||
'artifact:qinglong-worker-artifacts:us-east-1',
|
||||
'start',
|
||||
'stop',
|
||||
'unsubscribe',
|
||||
'close-artifact',
|
||||
]);
|
||||
assert.equal(
|
||||
facts.some((fact) => fact.event === 'worker_ingress_listening'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('creates the configured mounted Secret provider before Worker activation', async () => {
|
||||
const events = [];
|
||||
const artifactStore = {
|
||||
async put() {},
|
||||
async inspect() {},
|
||||
};
|
||||
const provider = { async resolve() {} };
|
||||
const result = await runProductionClusterControlProcess({
|
||||
environment: {
|
||||
...BASE_ENV,
|
||||
QL3_WORKER_INGRESS_ENABLED: 'true',
|
||||
QL3_POSTGRES_WORKER_INGRESS_URL:
|
||||
'postgresql://ql3_worker_ingress:secret@postgres-rw.internal:5432/qinglong',
|
||||
QL3_WORKER_INGRESS_POSTGRES_TLS_SERVERNAME: 'postgres-rw.internal',
|
||||
QL3_WORKER_CREDENTIAL_PEPPER: 'A'.repeat(43),
|
||||
QL3_WORKER_INGRESS_TLS_PRIVATE_KEY_FILE: '/run/worker/tls.key',
|
||||
QL3_WORKER_INGRESS_TLS_CERTIFICATE_FILE: '/run/worker/tls.crt',
|
||||
QL3_WORKER_INGRESS_TLS_CLIENT_CA_FILE: '/run/worker/client-ca.crt',
|
||||
QL3_WORKER_ARTIFACT_S3_BUCKET: 'qinglong-worker-artifacts',
|
||||
QL3_WORKER_ARTIFACT_S3_REGION: 'us-east-1',
|
||||
QL3_WORKER_SECRET_PROVIDER: 'mounted-files',
|
||||
QL3_WORKER_SECRET_ROOT_DIRECTORY: '/run/worker/values',
|
||||
},
|
||||
signals: signalSource(events),
|
||||
emit() {},
|
||||
async createWorkerArtifactBinding() {
|
||||
events.push('artifact');
|
||||
return {
|
||||
store: artifactStore,
|
||||
async close() {
|
||||
events.push('close-artifact');
|
||||
},
|
||||
};
|
||||
},
|
||||
async createWorkerSecretProvider(config) {
|
||||
events.push(`secret:${config.provider}:${config.rootDirectory}`);
|
||||
return provider;
|
||||
},
|
||||
async start(options) {
|
||||
events.push('start');
|
||||
assert.equal(options.workerIngress.secretProvider, provider);
|
||||
return {
|
||||
status: 'active',
|
||||
address: { host: '0.0.0.0', port: 5800 },
|
||||
evidence: {
|
||||
contractName: 'qinglong-cluster-control',
|
||||
contractVersion: 16,
|
||||
serverMajor: 18,
|
||||
migrationIds: [],
|
||||
},
|
||||
recovery: { safe: true, remaining: 0, failed: 0 },
|
||||
unavailable: NEVER_UNAVAILABLE,
|
||||
availabilityStatus: () => 'ready',
|
||||
async stop() {
|
||||
events.push('stop');
|
||||
return 'stopped';
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.equal(result, 'stopped');
|
||||
assert.deepEqual(events, [
|
||||
'subscribe',
|
||||
'artifact',
|
||||
'secret:mounted-files:/run/worker/values',
|
||||
'start',
|
||||
'stop',
|
||||
'unsubscribe',
|
||||
'close-artifact',
|
||||
]);
|
||||
});
|
||||
|
||||
test('fails the process after a database fence drains the active application', async () => {
|
||||
const events = [];
|
||||
const facts = [];
|
||||
let reportUnavailable;
|
||||
const unavailable = new Promise((resolve) => {
|
||||
reportUnavailable = resolve;
|
||||
});
|
||||
const running = runProductionClusterControlProcess({
|
||||
environment: BASE_ENV,
|
||||
signals: {
|
||||
subscribe() {
|
||||
events.push('subscribe');
|
||||
return () => events.push('unsubscribe');
|
||||
},
|
||||
},
|
||||
emit(record) {
|
||||
facts.push(record);
|
||||
},
|
||||
async start() {
|
||||
queueMicrotask(() =>
|
||||
reportUnavailable(
|
||||
Object.assign(new Error('must-not-escape-database-detail'), {
|
||||
code: '57P01',
|
||||
}),
|
||||
),
|
||||
);
|
||||
return {
|
||||
status: 'active',
|
||||
address: { host: '127.0.0.1', port: 5800 },
|
||||
evidence: {
|
||||
contractName: 'qinglong-cluster-control',
|
||||
contractVersion: 52,
|
||||
serverMajor: 18,
|
||||
migrationIds: [],
|
||||
},
|
||||
recovery: { safe: true, remaining: 0, failed: 0 },
|
||||
unavailable,
|
||||
availabilityStatus: () => 'unavailable',
|
||||
async stop() {
|
||||
events.push('stop');
|
||||
return 'stopped';
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
running,
|
||||
(error) => error?.code === 'CLUSTER_CONTROL_DATABASE_UNAVAILABLE',
|
||||
);
|
||||
assert.deepEqual(events, ['subscribe', 'stop', 'unsubscribe']);
|
||||
assert.equal(
|
||||
facts.some(
|
||||
({ event, diagnostic }) =>
|
||||
event === 'database_unavailable' &&
|
||||
diagnostic?.scope === 'database' &&
|
||||
diagnostic?.code === '57P01',
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(facts.some(({ event }) => event === 'shutdown_requested'), false);
|
||||
assert.equal(facts.at(-1).event, 'stopped');
|
||||
assert.equal(
|
||||
JSON.stringify(facts).includes('must-not-escape-database-detail'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user