mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-23 03:18:09 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,787 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const http = require('node:http');
|
||||
const net = require('node:net');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
startClusterControlApplication,
|
||||
} = require('@qinglong/cluster-control/application');
|
||||
const {
|
||||
ClusterControlAvailabilityFence,
|
||||
} = require('@qinglong/cluster-control/availability');
|
||||
const {
|
||||
apiCredentialSecretDigest,
|
||||
} = require('@qinglong/cluster-control/api-credential');
|
||||
const {
|
||||
createClusterControlAdmissionPipeline,
|
||||
createClusterControlProjectPolicyAuthorizer,
|
||||
} = require('@qinglong/cluster-control/admission');
|
||||
const {
|
||||
createClusterControlRouteRegistry,
|
||||
} = require('@qinglong/cluster-control/routes');
|
||||
const {
|
||||
PostgresSchemaReadinessError,
|
||||
postgresqlMainMigrationManifest,
|
||||
} = require('@qinglong/cluster-postgres/runtime');
|
||||
const {
|
||||
postgresqlControlSchemaContract,
|
||||
} = require('@qinglong/cluster-postgres');
|
||||
|
||||
function freePort() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = net.createServer();
|
||||
server.once('error', reject);
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
const address = server.address();
|
||||
server.close((error) => {
|
||||
if (error) reject(error);
|
||||
else resolve(address.port);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function request(port, path, options = {}) {
|
||||
const body =
|
||||
options.body === undefined
|
||||
? undefined
|
||||
: Buffer.from(JSON.stringify(options.body));
|
||||
return new Promise((resolve, reject) => {
|
||||
const outgoing = http.request(
|
||||
{
|
||||
host: '127.0.0.1',
|
||||
port,
|
||||
path,
|
||||
method: options.method ?? 'GET',
|
||||
headers: {
|
||||
connection: 'close',
|
||||
...(body
|
||||
? {
|
||||
'content-type': 'application/json',
|
||||
'content-length': String(body.byteLength),
|
||||
}
|
||||
: {}),
|
||||
...options.headers,
|
||||
},
|
||||
},
|
||||
(response) => {
|
||||
const chunks = [];
|
||||
response.on('data', (chunk) => chunks.push(chunk));
|
||||
response.on('end', () => {
|
||||
const text = Buffer.concat(chunks).toString('utf8');
|
||||
resolve({
|
||||
statusCode: response.statusCode,
|
||||
body: text.length === 0 ? null : JSON.parse(text),
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
outgoing.once('error', reject);
|
||||
if (body) outgoing.write(body);
|
||||
outgoing.end();
|
||||
});
|
||||
}
|
||||
|
||||
function migrationHistory() {
|
||||
return postgresqlMainMigrationManifest.migrations.map((migration, index) => ({
|
||||
streamId: postgresqlMainMigrationManifest.id,
|
||||
dialect: postgresqlMainMigrationManifest.dialect,
|
||||
migrationId: migration.id,
|
||||
checksum: migration.checksum,
|
||||
appliedAtMs: index + 1,
|
||||
}));
|
||||
}
|
||||
|
||||
function admission(handler) {
|
||||
return {
|
||||
async prepare(metadata) {
|
||||
return {
|
||||
handle(body) {
|
||||
return handler({ ...metadata, body });
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function runtimePrivileges() {
|
||||
const privileges = {
|
||||
schema_migrations: [true, false, false, false],
|
||||
schema_capabilities: [true, false, false, false],
|
||||
projects: [true, true, true, false],
|
||||
task_definitions: [true, false, false, false],
|
||||
task_definition_revisions: [true, false, false, false],
|
||||
task_execution_revisions: [true, false, false, false],
|
||||
triggers: [true, false, false, false],
|
||||
trigger_revisions: [true, false, false, false],
|
||||
trigger_schedules: [true, false, true, false],
|
||||
project_role_bindings: [true, true, false, false],
|
||||
identity_subjects: [true, false, false, false],
|
||||
api_credentials: [true, false, false, false],
|
||||
security_audit_events: [false, true, false, false],
|
||||
identity_subject_mutations: [false, false, false, false],
|
||||
api_credential_mutations: [false, false, false, false],
|
||||
runs: [true, true, true, false],
|
||||
step_runs: [true, true, true, false],
|
||||
step_run_mutations: [true, true, false, false],
|
||||
tool_execution_trace_anchors: [true, true, false, false],
|
||||
tool_execution_audit_receipts: [true, true, false, false],
|
||||
tool_execution_start_barriers: [true, true, false, false],
|
||||
tool_execution_start_artifact_bindings: [true, true, false, false],
|
||||
tool_execution_completions: [true, true, false, false],
|
||||
tool_execution_failure_completions: [true, true, false, false],
|
||||
tool_result_key_catalog_generations: [true, false, false, false],
|
||||
tool_execution_result_key_bindings: [true, true, false, false],
|
||||
tool_execution_result_rekey_overlays: [true, false, false, false],
|
||||
tool_execution_result_rekey_heads: [true, false, false, false],
|
||||
tool_result_key_retirement_receipts: [false, false, false, false],
|
||||
tool_invocation_input_artifacts: [true, true, false, false],
|
||||
tool_invocation_preview_artifacts: [true, true, false, false],
|
||||
run_attempts: [true, true, true, false],
|
||||
run_recovery_controls: [true, true, true, false],
|
||||
worker_sessions: [true, true, true, false],
|
||||
run_dispatch_leases: [true, true, true, false],
|
||||
worker_credentials: [false, false, false, false],
|
||||
worker_credential_mutations: [false, false, false, false],
|
||||
worker_credential_deliveries: [false, false, false, false],
|
||||
worker_credential_stage_discards: [false, false, false, false],
|
||||
worker_credential_management_plans: [false, false, false, false],
|
||||
worker_credential_management_quota_buckets: [false, false, false, false],
|
||||
plugin_package_installs: [false, false, false, false],
|
||||
plugin_package_install_heads: [false, false, false, false],
|
||||
plugin_package_install_mutations: [false, false, false, false],
|
||||
approval_requests: [false, false, false, false],
|
||||
approved_action_dispatches: [false, false, false, false],
|
||||
approved_action_executions: [false, false, false, false],
|
||||
plugin_package_install_proposals: [false, false, false, false],
|
||||
plugin_package_admission_receipts: [false, false, false, false],
|
||||
plugin_package_management_quota_buckets: [false, false, false, false],
|
||||
plugin_package_identity_keyset_ledger: [false, false, false, false],
|
||||
plugin_package_materialized_revisions: [false, false, false, false],
|
||||
plugin_package_task_ownerships: [false, false, false, false],
|
||||
plugin_package_task_reconciliations: [false, false, false, false],
|
||||
plugin_package_task_reconciliation_items: [false, false, false, false],
|
||||
project_tool_definition_snapshots: [false, false, false, false],
|
||||
project_tool_definition_snapshot_sources: [false, false, false, false],
|
||||
plugin_package_publisher_provenance: [false, false, false, false],
|
||||
plugin_package_publisher_revocation_impact_items: [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
],
|
||||
plugin_package_publisher_revocation_impacts: [false, false, false, false],
|
||||
plugin_package_publisher_revocation_proposals: [false, false, false, false],
|
||||
plugin_package_publisher_revocation_receipts: [false, false, false, false],
|
||||
plugin_package_publisher_trust_heads: [false, false, false, false],
|
||||
plugin_package_publisher_trust_snapshots: [false, false, false, false],
|
||||
plugin_package_publisher_trust_transition_proposals: [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
],
|
||||
plugin_package_publisher_trust_transition_receipts: [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
],
|
||||
plugin_package_quarantine_events: [false, false, false, false],
|
||||
plugin_package_withdrawal_receipts: [false, false, false, false],
|
||||
plugin_package_withdrawal_tasks: [false, false, false, false],
|
||||
plugin_package_lifecycle_events: [false, false, false, false],
|
||||
plugin_package_lifecycle_heads: [false, false, false, false],
|
||||
plugin_package_lifecycle_receipts: [false, false, false, false],
|
||||
plugin_package_lifecycle_tasks: [false, false, false, false],
|
||||
plugin_package_lifecycle_plans: [false, false, false, false],
|
||||
plugin_package_automation_publications: [true, false, false, false],
|
||||
plugin_package_automation_publication_heads: [true, false, false, false],
|
||||
plugin_package_workflow_admissions: [true, true, false, false],
|
||||
plugin_package_workflow_admission_steps: [true, true, false, false],
|
||||
plugin_package_workflow_task_attempt_admissions: [
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
],
|
||||
worker_execution_attestations: [true, false, false, false],
|
||||
run_events: [true, true, false, false],
|
||||
run_retry_policies: [true, true, true, false],
|
||||
};
|
||||
return Object.entries(privileges).map(
|
||||
([
|
||||
tableName,
|
||||
[selectAllowed, insertAllowed, updateAllowed, deleteAllowed],
|
||||
]) => ({
|
||||
tableName,
|
||||
selectAllowed,
|
||||
insertAllowed,
|
||||
updateAllowed,
|
||||
deleteAllowed,
|
||||
isOwner: false,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function databaseResource(events, options = {}) {
|
||||
const contract = postgresqlControlSchemaContract;
|
||||
let firstQuery = true;
|
||||
const pool = {
|
||||
async query(text, values) {
|
||||
events.push('query');
|
||||
if (firstQuery && options.readinessGate) {
|
||||
firstQuery = false;
|
||||
options.onReadinessQuery?.();
|
||||
await options.readinessGate;
|
||||
}
|
||||
if (text.includes("current_setting('server_version_num')")) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
serverVersionNum: options.serverVersionNum ?? '160014',
|
||||
currentUser: 'ql3_runtime',
|
||||
inRecovery: false,
|
||||
transactionReadOnly: 'off',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."schema_migrations"')) {
|
||||
return { rows: migrationHistory() };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."schema_capabilities"')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
contractName: contract.contractName,
|
||||
contractVersion: contract.contractVersion,
|
||||
migrationId: contract.migrationId,
|
||||
capabilities: contract.capabilities,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM pg_class tables')) {
|
||||
return {
|
||||
rows: contract.tables.flatMap((table) =>
|
||||
table.columns.map((columnName) => ({
|
||||
tableName: table.name,
|
||||
columnName,
|
||||
})),
|
||||
),
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM pg_indexes')) {
|
||||
return { rows: contract.indexes.map((indexName) => ({ indexName })) };
|
||||
}
|
||||
if (text.includes('FROM pg_constraint')) {
|
||||
return {
|
||||
rows: [
|
||||
...contract.checks.map((constraintName) => ({
|
||||
constraintName,
|
||||
constraintType: 'check',
|
||||
})),
|
||||
...contract.foreignKeys.map((constraintName) => ({
|
||||
constraintName,
|
||||
constraintType: 'foreign_key',
|
||||
})),
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM pg_proc routines')) {
|
||||
return {
|
||||
rows: contract.functions.map((definition) => ({
|
||||
functionName: definition.name,
|
||||
identityArguments: definition.identityArguments,
|
||||
owner: definition.owner,
|
||||
securityDefiner: definition.securityDefiner,
|
||||
volatility: definition.volatility,
|
||||
configuration: definition.configuration,
|
||||
publicExecute: false,
|
||||
})),
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM pg_catalog.pg_roles')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
canLogin: true,
|
||||
superuser: false,
|
||||
createDatabase: false,
|
||||
createRole: false,
|
||||
replication: false,
|
||||
bypassRowLevelSecurity: false,
|
||||
databaseConnect: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('has_schema_privilege')) {
|
||||
return { rows: [{ schemaUsage: true, schemaCreate: false }] };
|
||||
}
|
||||
if (text.includes('has_table_privilege')) {
|
||||
return { rows: runtimePrivileges() };
|
||||
}
|
||||
if (text.includes('has_function_privilege')) {
|
||||
return {
|
||||
rows: contract.functions.map(({ name: functionName }) => ({
|
||||
functionName,
|
||||
executeAllowed: [
|
||||
'plugin_package_automation_start_allowed',
|
||||
'plugin_package_workflow_admission_snapshot',
|
||||
'plugin_package_workflow_task_attempt_snapshot',
|
||||
'plugin_package_run_start_allowed',
|
||||
'plugin_package_tool_start_allowed',
|
||||
].includes(functionName),
|
||||
isOwner: false,
|
||||
})),
|
||||
};
|
||||
}
|
||||
if (text.includes('WITH observation AS')) {
|
||||
return {
|
||||
rows: options.recoveryRows ?? [
|
||||
{
|
||||
observedAtMs: '1',
|
||||
kind: null,
|
||||
id: null,
|
||||
runId: null,
|
||||
status: null,
|
||||
createdAtMs: null,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (options.query) return options.query(text, values);
|
||||
throw new Error(`unexpected query: ${text}`);
|
||||
},
|
||||
async connect() {
|
||||
throw new Error('Repository connections are not used by this fixture');
|
||||
},
|
||||
};
|
||||
return {
|
||||
pool,
|
||||
async close() {
|
||||
events.push('close-database');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function baseOptions(events, port, overrides = {}) {
|
||||
return {
|
||||
enabled: true,
|
||||
profile: 'cluster-control',
|
||||
apiCredentialPepper: 'A'.repeat(43),
|
||||
recovery: { ownerId: 'test-replica', providers: [] },
|
||||
availability: new ClusterControlAvailabilityFence(),
|
||||
http: { host: '127.0.0.1', port, drainTimeoutMs: 1000 },
|
||||
async openDatabase() {
|
||||
events.push('open-database');
|
||||
return databaseResource(events);
|
||||
},
|
||||
create({
|
||||
evidence,
|
||||
authenticator,
|
||||
policies,
|
||||
runs,
|
||||
taskDefinitions,
|
||||
taskExecutionRevisions,
|
||||
triggers,
|
||||
schedules,
|
||||
securityAudit,
|
||||
}) {
|
||||
events.push('create-stack');
|
||||
assert.equal(
|
||||
evidence.contractVersion,
|
||||
postgresqlControlSchemaContract.contractVersion,
|
||||
);
|
||||
assert.equal(typeof authenticator.authenticate, 'function');
|
||||
assert.equal(typeof policies.resolve, 'function');
|
||||
assert.equal(typeof runs.transaction, 'function');
|
||||
assert.equal(
|
||||
typeof taskDefinitions.findCurrentTaskDefinition,
|
||||
'function',
|
||||
);
|
||||
assert.equal(
|
||||
typeof taskExecutionRevisions.resolveClusterTaskExecutionRevision,
|
||||
'function',
|
||||
);
|
||||
assert.equal(typeof triggers.findCurrentTrigger, 'function');
|
||||
assert.equal(typeof schedules.claimNextClusterSchedule, 'function');
|
||||
assert.equal(typeof schedules.commitClusterScheduleDecision, 'function');
|
||||
assert.equal(typeof securityAudit.record, 'function');
|
||||
return {
|
||||
async reconcile() {
|
||||
events.push('reconcile');
|
||||
return { safe: true, remaining: 0, failed: 0 };
|
||||
},
|
||||
async startLifecycles() {
|
||||
events.push('start-lifecycles');
|
||||
return true;
|
||||
},
|
||||
admission: admission(async (incoming) => {
|
||||
events.push(`handle:${incoming.path}`);
|
||||
return { statusCode: 202, body: { accepted: true } };
|
||||
}),
|
||||
async stop() {
|
||||
events.push('stop-stack');
|
||||
return 'stopped';
|
||||
},
|
||||
};
|
||||
},
|
||||
audit(record) {
|
||||
events.push(`audit:${record.state}`);
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('disabled and wrong-profile applications never bind or open PostgreSQL', async () => {
|
||||
const disabledEvents = [];
|
||||
const disabled = await startClusterControlApplication(
|
||||
baseOptions(disabledEvents, 0, { enabled: false }),
|
||||
);
|
||||
assert.equal(disabled.status, 'disabled');
|
||||
assert.deepEqual(disabledEvents, ['audit:disabled']);
|
||||
|
||||
const wrongProfileEvents = [];
|
||||
await assert.rejects(
|
||||
startClusterControlApplication(
|
||||
baseOptions(wrongProfileEvents, 0, { profile: 'standalone' }),
|
||||
),
|
||||
/cannot activate cluster-control/,
|
||||
);
|
||||
assert.deepEqual(wrongProfileEvents, []);
|
||||
});
|
||||
|
||||
test('rejects an invalid credential pepper before binding or opening PostgreSQL', async () => {
|
||||
const events = [];
|
||||
await assert.rejects(
|
||||
startClusterControlApplication(
|
||||
baseOptions(events, 0, { apiCredentialPepper: undefined }),
|
||||
),
|
||||
/API credential configuration is invalid/,
|
||||
);
|
||||
assert.deepEqual(events, []);
|
||||
});
|
||||
|
||||
test('rejects an enabled application without an availability source', async () => {
|
||||
const events = [];
|
||||
await assert.rejects(
|
||||
startClusterControlApplication(
|
||||
baseOptions(events, 0, { availability: undefined }),
|
||||
),
|
||||
/availability source is invalid/,
|
||||
);
|
||||
assert.deepEqual(events, []);
|
||||
});
|
||||
|
||||
test('composes bearer authentication, fenced Policy and durable audit on one Pool', async (t) => {
|
||||
const pepper = 'A'.repeat(43);
|
||||
const secret = Buffer.alloc(32, 2).toString('base64url');
|
||||
const digest = apiCredentialSecretDigest(pepper, 'app_primary', secret);
|
||||
const auditWrites = [];
|
||||
const events = [];
|
||||
const port = await freePort();
|
||||
const result = await startClusterControlApplication(
|
||||
baseOptions(events, port, {
|
||||
apiCredentialPepper: pepper,
|
||||
async openDatabase() {
|
||||
events.push('open-database');
|
||||
return databaseResource(events, {
|
||||
async query(text, values) {
|
||||
if (text.includes('FROM "ql3"."api_credentials"')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
credentialId: 'app_primary',
|
||||
version: '1',
|
||||
state: 'active',
|
||||
subjectType: 'api_app',
|
||||
subjectId: 'app_primary',
|
||||
subjectStatus: 'active',
|
||||
pepperKeyId: 'legacy-v1',
|
||||
secretDigest: digest,
|
||||
createdAtMs: '1',
|
||||
notBeforeAtMs: '1',
|
||||
expiresAtMs: String(Date.now() + 60_000),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."projects" AS project')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
projectId: 'default',
|
||||
projectName: 'Default',
|
||||
projectSlug: 'default',
|
||||
projectStatus: 'active',
|
||||
projectVersion: '2',
|
||||
projectCreatedAtMs: '0',
|
||||
projectUpdatedAtMs: '1',
|
||||
bindingProjectId: 'default',
|
||||
bindingSubjectType: 'api_app',
|
||||
bindingSubjectId: 'app_primary',
|
||||
bindingVersion: '3',
|
||||
bindingState: 'active',
|
||||
bindingRole: 'operator',
|
||||
bindingMutationId: 'grant-app',
|
||||
bindingChangedByType: 'user',
|
||||
bindingChangedById: 'usr_owner',
|
||||
bindingCreatedAtMs: '1',
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.startsWith('INSERT INTO "ql3"."security_audit_events"')) {
|
||||
auditWrites.push(values);
|
||||
return { rows: [] };
|
||||
}
|
||||
throw new Error(`unexpected repository query: ${text}`);
|
||||
},
|
||||
});
|
||||
},
|
||||
create({ authenticator, policies, securityAudit }) {
|
||||
events.push('create-stack');
|
||||
return {
|
||||
async reconcile() {
|
||||
return { safe: true, remaining: 0, failed: 0 };
|
||||
},
|
||||
async startLifecycles() {
|
||||
return true;
|
||||
},
|
||||
admission: createClusterControlAdmissionPipeline({
|
||||
routes: createClusterControlRouteRegistry([
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/api/v3/projects/{projectId}/runs',
|
||||
operationId: 'run.create',
|
||||
permission: 'run.start',
|
||||
projectParameter: 'projectId',
|
||||
handle(input) {
|
||||
return {
|
||||
statusCode: 202,
|
||||
body: {
|
||||
accepted: true,
|
||||
subject: input.principal.subject.id,
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
]),
|
||||
authenticator,
|
||||
policy: createClusterControlProjectPolicyAuthorizer(policies),
|
||||
audit: securityAudit,
|
||||
}),
|
||||
async stop() {
|
||||
return 'stopped';
|
||||
},
|
||||
};
|
||||
},
|
||||
}),
|
||||
);
|
||||
t.after(() => result.stop());
|
||||
|
||||
const rejected = await request(port, '/api/v3/projects/default/runs', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'content-length': String(1024 * 1024),
|
||||
},
|
||||
});
|
||||
assert.equal(rejected.statusCode, 401);
|
||||
assert.equal(auditWrites.length, 1);
|
||||
assert.equal(auditWrites[0][7], 'authentication_rejected');
|
||||
|
||||
const accepted = await request(port, '/api/v3/projects/default/runs', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ql3c_app_primary_${secret}`,
|
||||
},
|
||||
body: { taskId: 'task-1' },
|
||||
});
|
||||
assert.equal(accepted.statusCode, 202);
|
||||
assert.deepEqual(accepted.body, {
|
||||
accepted: true,
|
||||
subject: 'app_primary',
|
||||
});
|
||||
assert.equal(auditWrites.length, 2);
|
||||
assert.equal(auditWrites[1][7], 'allowed');
|
||||
assert.equal(auditWrites[1][9], 2);
|
||||
assert.equal(auditWrites[1][10], 3);
|
||||
assert.equal(JSON.stringify(auditWrites).includes(secret), false);
|
||||
});
|
||||
|
||||
test('serves not-ready while auditing PostgreSQL, then opens admission', async () => {
|
||||
const events = [];
|
||||
const port = await freePort();
|
||||
let releaseReadiness;
|
||||
const readinessGate = new Promise((resolve) => {
|
||||
releaseReadiness = resolve;
|
||||
});
|
||||
let readinessStarted;
|
||||
const readinessStartedPromise = new Promise((resolve) => {
|
||||
readinessStarted = resolve;
|
||||
});
|
||||
const starting = startClusterControlApplication(
|
||||
baseOptions(events, port, {
|
||||
async openDatabase() {
|
||||
events.push('open-database');
|
||||
return databaseResource(events, {
|
||||
readinessGate,
|
||||
onReadinessQuery: readinessStarted,
|
||||
});
|
||||
},
|
||||
}),
|
||||
);
|
||||
await readinessStartedPromise;
|
||||
assert.equal((await request(port, '/livez')).statusCode, 200);
|
||||
assert.deepEqual(await request(port, '/readyz'), {
|
||||
statusCode: 503,
|
||||
body: { status: 'not_ready' },
|
||||
});
|
||||
assert.equal(events.includes('create-stack'), false);
|
||||
|
||||
releaseReadiness();
|
||||
const application = await starting;
|
||||
assert.equal(application.status, 'active');
|
||||
assert.deepEqual(await request(port, '/readyz'), {
|
||||
statusCode: 200,
|
||||
body: { status: 'ready' },
|
||||
});
|
||||
assert.deepEqual(
|
||||
await request(port, '/api/v3/runs', {
|
||||
method: 'POST',
|
||||
body: { taskId: 'task-1' },
|
||||
}),
|
||||
{ statusCode: 202, body: { accepted: true } },
|
||||
);
|
||||
assert.equal(await application.stop(), 'stopped');
|
||||
assert.deepEqual(events.slice(-3), [
|
||||
'stop-stack',
|
||||
'audit:stopped',
|
||||
'close-database',
|
||||
]);
|
||||
await assert.rejects(request(port, '/livez'));
|
||||
});
|
||||
|
||||
test('withdraws and drains application admission before stack and Pool stop', async () => {
|
||||
const events = [];
|
||||
const port = await freePort();
|
||||
let entered;
|
||||
const enteredPromise = new Promise((resolve) => {
|
||||
entered = resolve;
|
||||
});
|
||||
let release;
|
||||
const handlerGate = new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const application = await startClusterControlApplication(
|
||||
baseOptions(events, port, {
|
||||
create(input) {
|
||||
const stack = baseOptions([], port).create(input);
|
||||
return {
|
||||
...stack,
|
||||
admission: admission(async () => {
|
||||
events.push('handle:slow');
|
||||
entered();
|
||||
await handlerGate;
|
||||
events.push('handler-finished');
|
||||
return { statusCode: 200, body: { completed: true } };
|
||||
}),
|
||||
async stop() {
|
||||
events.push('stop-stack');
|
||||
return 'stopped';
|
||||
},
|
||||
};
|
||||
},
|
||||
}),
|
||||
);
|
||||
const admitted = request(port, '/api/v3/slow');
|
||||
await enteredPromise;
|
||||
const stopping = application.stop();
|
||||
assert.deepEqual(await request(port, '/readyz'), {
|
||||
statusCode: 503,
|
||||
body: { status: 'not_ready' },
|
||||
});
|
||||
assert.equal(events.includes('stop-stack'), false);
|
||||
release();
|
||||
assert.deepEqual(await admitted, {
|
||||
statusCode: 503,
|
||||
body: { code: 'admission_draining' },
|
||||
});
|
||||
assert.equal(await stopping, 'stopped');
|
||||
assert.ok(events.indexOf('handler-finished') < events.indexOf('stop-stack'));
|
||||
assert.ok(events.indexOf('stop-stack') < events.indexOf('close-database'));
|
||||
});
|
||||
|
||||
test('a database availability signal withdraws admission but keeps liveness', async () => {
|
||||
const events = [];
|
||||
const port = await freePort();
|
||||
const availability = new ClusterControlAvailabilityFence();
|
||||
const application = await startClusterControlApplication(
|
||||
baseOptions(events, port, { availability }),
|
||||
);
|
||||
assert.equal(application.availabilityStatus(), 'ready');
|
||||
assert.equal((await request(port, '/readyz')).statusCode, 200);
|
||||
|
||||
const unavailable = new Error('PostgreSQL connection lost');
|
||||
assert.equal(await availability.signal(unavailable), 'signaled');
|
||||
assert.equal(await application.unavailable, unavailable);
|
||||
assert.equal(application.availabilityStatus(), 'unavailable');
|
||||
assert.deepEqual(await request(port, '/readyz'), {
|
||||
statusCode: 503,
|
||||
body: { status: 'not_ready' },
|
||||
});
|
||||
assert.equal((await request(port, '/livez')).statusCode, 200);
|
||||
assert.deepEqual(
|
||||
await request(port, '/api/v3/runs', {
|
||||
method: 'POST',
|
||||
body: { taskId: 'task-1' },
|
||||
}),
|
||||
{ statusCode: 503, body: { code: 'not_ready' } },
|
||||
);
|
||||
assert.ok(events.indexOf('stop-stack') < events.indexOf('close-database'));
|
||||
|
||||
assert.equal(await application.stop(), 'stopped');
|
||||
assert.equal(application.availabilityStatus(), 'stopped');
|
||||
await assert.rejects(request(port, '/livez'));
|
||||
});
|
||||
|
||||
test('an availability signal raised before startup cannot open admission', async () => {
|
||||
const events = [];
|
||||
const port = await freePort();
|
||||
const availability = new ClusterControlAvailabilityFence();
|
||||
assert.equal(
|
||||
await availability.signal(new Error('PostgreSQL unavailable at startup')),
|
||||
'signaled',
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
startClusterControlApplication(baseOptions(events, port, { availability })),
|
||||
(error) => error?.code === 'CLUSTER_CONTROL_DATABASE_UNAVAILABLE',
|
||||
);
|
||||
assert.ok(events.indexOf('stop-stack') < events.indexOf('close-database'));
|
||||
await assert.rejects(request(port, '/livez'));
|
||||
});
|
||||
|
||||
test('readiness failure closes both PostgreSQL and the probe listener', async () => {
|
||||
const events = [];
|
||||
const port = await freePort();
|
||||
await assert.rejects(
|
||||
startClusterControlApplication(
|
||||
baseOptions(events, port, {
|
||||
async openDatabase() {
|
||||
events.push('open-database');
|
||||
return databaseResource(events, { serverVersionNum: '150018' });
|
||||
},
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'server_version_unsupported',
|
||||
);
|
||||
assert.equal(events.includes('create-stack'), false);
|
||||
assert.deepEqual(events.slice(-2), ['audit:failed', 'close-database']);
|
||||
await assert.rejects(request(port, '/livez'));
|
||||
});
|
||||
Reference in New Issue
Block a user