mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
812 lines
26 KiB
JavaScript
812 lines
26 KiB
JavaScript
const assert = require('node:assert/strict');
|
|
const { test } = require('node:test');
|
|
const {
|
|
PostgresSchemaReadinessError,
|
|
postgresqlControlSchemaContract,
|
|
postgresqlMainMigrationStream,
|
|
} = require('@qinglong/cluster-postgres');
|
|
const { bootstrapClusterControlRuntime } = require('@qinglong/cluster-control');
|
|
|
|
function migrationHistory() {
|
|
return postgresqlMainMigrationStream.migrations.map((migration, index) => ({
|
|
streamId: 'postgresql-main',
|
|
dialect: 'postgresql',
|
|
migrationId: migration.id,
|
|
checksum: migration.checksum,
|
|
appliedAtMs: index + 1,
|
|
}));
|
|
}
|
|
|
|
function runtimePrivileges() {
|
|
const privileges = {
|
|
schema_migrations: [true, false, false, false],
|
|
schema_capabilities: [true, false, false, false],
|
|
projects: [true, true, true, false],
|
|
task_definitions: [true, false, false, false],
|
|
task_definition_revisions: [true, false, false, false],
|
|
task_execution_revisions: [true, false, false, false],
|
|
triggers: [true, false, false, false],
|
|
trigger_revisions: [true, false, false, false],
|
|
trigger_schedules: [true, false, true, false],
|
|
project_role_bindings: [true, true, false, false],
|
|
identity_subjects: [true, false, false, false],
|
|
api_credentials: [true, false, false, false],
|
|
security_audit_events: [false, true, false, false],
|
|
identity_subject_mutations: [false, false, false, false],
|
|
api_credential_mutations: [false, false, false, false],
|
|
runs: [true, true, true, false],
|
|
step_runs: [true, true, true, false],
|
|
step_run_mutations: [true, true, false, false],
|
|
tool_execution_trace_anchors: [true, true, false, false],
|
|
tool_execution_audit_receipts: [true, true, false, false],
|
|
tool_execution_start_barriers: [true, true, false, false],
|
|
tool_execution_start_artifact_bindings: [true, true, false, false],
|
|
tool_execution_completions: [true, true, false, false],
|
|
tool_execution_failure_completions: [true, true, false, false],
|
|
tool_result_key_catalog_generations: [true, false, false, false],
|
|
tool_execution_result_key_bindings: [true, true, false, false],
|
|
tool_execution_result_rekey_overlays: [true, false, false, false],
|
|
tool_execution_result_rekey_heads: [true, false, false, false],
|
|
tool_result_key_retirement_receipts: [false, false, false, false],
|
|
tool_invocation_input_artifacts: [true, true, false, false],
|
|
tool_invocation_preview_artifacts: [true, true, false, false],
|
|
run_attempts: [true, true, true, false],
|
|
run_attempt_log_retention_controls: [true, true, true, true],
|
|
run_attempt_log_artifact_tombstones: [true, true, false, false],
|
|
run_recovery_controls: [true, true, true, false],
|
|
worker_sessions: [true, true, true, false],
|
|
run_dispatch_leases: [true, true, true, false],
|
|
worker_credentials: [false, false, false, false],
|
|
worker_credential_mutations: [false, false, false, false],
|
|
worker_credential_deliveries: [false, false, false, false],
|
|
worker_credential_stage_discards: [false, false, false, false],
|
|
worker_credential_management_plans: [false, false, false, false],
|
|
worker_credential_management_quota_buckets: [false, false, false, false],
|
|
plugin_package_installs: [false, false, false, false],
|
|
plugin_package_install_heads: [false, false, false, false],
|
|
plugin_package_install_mutations: [false, false, false, false],
|
|
approval_requests: [false, false, false, false],
|
|
approved_action_dispatches: [false, false, false, false],
|
|
approved_action_executions: [false, false, false, false],
|
|
approved_action_manual_recovery_resolutions: [false, false, false, false],
|
|
plugin_package_install_proposals: [false, false, false, false],
|
|
plugin_package_admission_receipts: [false, false, false, false],
|
|
plugin_package_management_quota_buckets: [false, false, false, false],
|
|
plugin_package_identity_keyset_ledger: [false, false, false, false],
|
|
plugin_package_materialized_revisions: [false, false, false, false],
|
|
plugin_package_task_ownerships: [false, false, false, false],
|
|
plugin_package_task_reconciliations: [false, false, false, false],
|
|
plugin_package_task_reconciliation_items: [false, false, false, false],
|
|
project_tool_definition_snapshots: [false, false, false, false],
|
|
project_tool_definition_snapshot_sources: [false, false, false, false],
|
|
plugin_package_publisher_provenance: [false, false, false, false],
|
|
plugin_package_publisher_revocation_impact_items: [
|
|
false,
|
|
false,
|
|
false,
|
|
false,
|
|
],
|
|
plugin_package_publisher_revocation_impacts: [false, false, false, false],
|
|
plugin_package_publisher_revocation_proposals: [false, false, false, false],
|
|
plugin_package_publisher_revocation_receipts: [false, false, false, false],
|
|
plugin_package_publisher_trust_heads: [false, false, false, false],
|
|
plugin_package_publisher_trust_snapshots: [false, false, false, false],
|
|
plugin_package_publisher_trust_transition_proposals: [
|
|
false,
|
|
false,
|
|
false,
|
|
false,
|
|
],
|
|
plugin_package_publisher_trust_transition_receipts: [
|
|
false,
|
|
false,
|
|
false,
|
|
false,
|
|
],
|
|
plugin_package_quarantine_events: [false, false, false, false],
|
|
plugin_package_withdrawal_receipts: [false, false, false, false],
|
|
plugin_package_withdrawal_tasks: [false, false, false, false],
|
|
plugin_package_lifecycle_events: [false, false, false, false],
|
|
plugin_package_lifecycle_heads: [false, false, false, false],
|
|
plugin_package_lifecycle_receipts: [false, false, false, false],
|
|
plugin_package_lifecycle_tasks: [false, false, false, false],
|
|
plugin_package_lifecycle_plans: [false, false, false, false],
|
|
plugin_package_automation_publications: [true, false, false, false],
|
|
plugin_package_automation_disposition_events: [false, false, false, false],
|
|
plugin_package_automation_publication_heads: [true, false, false, false],
|
|
plugin_package_secret_binding_approval_plans: [false, false, false, false],
|
|
plugin_package_secret_binding_transition_approval_plans: [
|
|
false,
|
|
false,
|
|
false,
|
|
false,
|
|
],
|
|
plugin_package_secret_bindings: [false, false, false, false],
|
|
plugin_package_secret_binding_transition_receipts: [
|
|
false,
|
|
false,
|
|
false,
|
|
false,
|
|
],
|
|
plugin_package_workflow_admissions: [true, true, false, false],
|
|
plugin_package_workflow_admission_steps: [true, true, false, false],
|
|
plugin_package_workflow_task_attempt_admissions: [true, true, false, false],
|
|
cluster_legacy_env_migration_plans: [false, false, false, false],
|
|
cluster_legacy_env_migration_application_receipts: [
|
|
false,
|
|
false,
|
|
false,
|
|
false,
|
|
],
|
|
cluster_legacy_env_migration_application_tasks: [
|
|
false,
|
|
false,
|
|
false,
|
|
false,
|
|
],
|
|
cluster_legacy_env_migration_application_triggers: [
|
|
false,
|
|
false,
|
|
false,
|
|
false,
|
|
],
|
|
worker_execution_attestations: [true, false, false, false],
|
|
run_events: [true, true, false, false],
|
|
run_cancellation_dispatches: [true, true, true, false],
|
|
run_retry_policies: [true, true, true, false],
|
|
};
|
|
return Object.entries(privileges).map(
|
|
([
|
|
tableName,
|
|
[selectAllowed, insertAllowed, updateAllowed, deleteAllowed],
|
|
]) => ({
|
|
tableName,
|
|
selectAllowed,
|
|
insertAllowed,
|
|
updateAllowed,
|
|
deleteAllowed,
|
|
isOwner: false,
|
|
}),
|
|
);
|
|
}
|
|
|
|
function databaseResource(events, overrides = {}) {
|
|
const contract = postgresqlControlSchemaContract;
|
|
const pool = {
|
|
async query(text) {
|
|
events.push(
|
|
`query:${
|
|
events.filter((event) => event.startsWith('query:')).length + 1
|
|
}`,
|
|
);
|
|
if (text.includes("current_setting('server_version_num')")) {
|
|
return {
|
|
rows: [
|
|
{
|
|
serverVersionNum: overrides.serverVersionNum ?? '160014',
|
|
currentUser: 'ql3_runtime',
|
|
inRecovery: false,
|
|
transactionReadOnly: 'off',
|
|
},
|
|
],
|
|
};
|
|
}
|
|
if (text.includes('FROM "ql3"."schema_migrations"')) {
|
|
return { rows: migrationHistory() };
|
|
}
|
|
if (text.includes('FROM "ql3"."schema_capabilities"')) {
|
|
return {
|
|
rows: [
|
|
{
|
|
contractName: contract.contractName,
|
|
contractVersion: contract.contractVersion,
|
|
migrationId: contract.migrationId,
|
|
capabilities: contract.capabilities,
|
|
},
|
|
],
|
|
};
|
|
}
|
|
if (text.includes('FROM pg_class tables')) {
|
|
return {
|
|
rows: contract.tables.flatMap((table) =>
|
|
table.columns.map((columnName) => ({
|
|
tableName: table.name,
|
|
columnName,
|
|
})),
|
|
),
|
|
};
|
|
}
|
|
if (text.includes('FROM pg_indexes')) {
|
|
return {
|
|
rows: contract.indexes.map((indexName) => ({ indexName })),
|
|
};
|
|
}
|
|
if (text.includes('FROM pg_constraint')) {
|
|
return {
|
|
rows: [
|
|
...contract.checks.map((constraintName) => ({
|
|
constraintName,
|
|
constraintType: 'check',
|
|
})),
|
|
...contract.foreignKeys.map((constraintName) => ({
|
|
constraintName,
|
|
constraintType: 'foreign_key',
|
|
})),
|
|
],
|
|
};
|
|
}
|
|
if (text.includes('FROM pg_proc routines')) {
|
|
return {
|
|
rows: contract.functions.map((definition) => ({
|
|
functionName: definition.name,
|
|
identityArguments: definition.identityArguments,
|
|
owner: definition.owner,
|
|
securityDefiner: definition.securityDefiner,
|
|
volatility: definition.volatility,
|
|
configuration: definition.configuration,
|
|
publicExecute: false,
|
|
})),
|
|
};
|
|
}
|
|
if (text.includes('FROM pg_trigger triggers')) {
|
|
return {
|
|
rows: contract.triggers.map((definition) => ({
|
|
triggerName: definition.name,
|
|
tableName: definition.tableName,
|
|
functionName: definition.functionName,
|
|
enabled: 'O',
|
|
})),
|
|
};
|
|
}
|
|
if (text.includes('FROM pg_catalog.pg_roles')) {
|
|
return {
|
|
rows: [
|
|
{
|
|
canLogin: true,
|
|
superuser: false,
|
|
createDatabase: false,
|
|
createRole: false,
|
|
replication: false,
|
|
bypassRowLevelSecurity: false,
|
|
databaseConnect: true,
|
|
},
|
|
],
|
|
};
|
|
}
|
|
if (text.includes('has_schema_privilege')) {
|
|
return { rows: [{ schemaUsage: true, schemaCreate: false }] };
|
|
}
|
|
if (text.includes('has_table_privilege')) {
|
|
return { rows: runtimePrivileges() };
|
|
}
|
|
if (text.includes('has_function_privilege')) {
|
|
return {
|
|
rows: contract.functions.map(({ name: functionName }) => ({
|
|
functionName,
|
|
executeAllowed: [
|
|
'plugin_package_automation_start_allowed',
|
|
'plugin_package_workflow_admission_snapshot',
|
|
'plugin_package_workflow_task_attempt_snapshot',
|
|
'plugin_package_run_start_allowed',
|
|
'plugin_package_tool_start_allowed',
|
|
'lock_run_management_policy_fence',
|
|
].includes(functionName),
|
|
isOwner: false,
|
|
})),
|
|
};
|
|
}
|
|
if (
|
|
overrides.runtimeScans &&
|
|
text.includes('JOIN LATERAL') &&
|
|
text.includes('"ql3"."run_retry_policies"')
|
|
) {
|
|
events.push('scan:lost-retry');
|
|
return { rows: [], rowCount: 0 };
|
|
}
|
|
if (
|
|
text.includes('WITH observation AS') &&
|
|
!text.includes('FROM "ql3"."trigger_schedules"')
|
|
) {
|
|
return {
|
|
rows: overrides.recoveryRows ?? [
|
|
{
|
|
observedAtMs: '1',
|
|
kind: null,
|
|
id: null,
|
|
runId: null,
|
|
status: null,
|
|
createdAtMs: null,
|
|
},
|
|
],
|
|
};
|
|
}
|
|
if (
|
|
overrides.runtimeScans &&
|
|
text.includes('FROM "ql3"."trigger_schedules"')
|
|
) {
|
|
events.push('scan:schedules');
|
|
return { rows: [], rowCount: 0 };
|
|
}
|
|
if (
|
|
overrides.runtimeScans &&
|
|
text.includes(
|
|
'FROM "ql3"."plugin_package_workflow_admissions" AS admission',
|
|
)
|
|
) {
|
|
events.push('scan:workflow-frontier');
|
|
return { rows: [], rowCount: 0 };
|
|
}
|
|
throw new Error(`unexpected query: ${text}`);
|
|
},
|
|
async connect() {
|
|
if (overrides.runtimeScans) {
|
|
return {
|
|
async query(text) {
|
|
if (
|
|
text === 'BEGIN ISOLATION LEVEL READ COMMITTED' ||
|
|
text.startsWith('SET LOCAL ') ||
|
|
text === 'COMMIT' ||
|
|
text === 'ROLLBACK'
|
|
) {
|
|
return { rows: [], rowCount: 0 };
|
|
}
|
|
if (
|
|
text.includes('attempt.lease_expires_at_ms') &&
|
|
text.includes('LEFT JOIN candidates AS candidate')
|
|
) {
|
|
events.push('scan:runtime-recovery');
|
|
return {
|
|
rows: [
|
|
{
|
|
observedAtMs: '1',
|
|
attemptId: null,
|
|
runId: null,
|
|
status: null,
|
|
createdAtMs: null,
|
|
},
|
|
],
|
|
rowCount: 1,
|
|
};
|
|
}
|
|
if (
|
|
text.includes(
|
|
'"ql3"."plugin_package_workflow_task_attempt_admissions"',
|
|
)
|
|
) {
|
|
events.push('scan:workflow-task-attempts');
|
|
return { rows: [], rowCount: 0 };
|
|
}
|
|
throw new Error(`unexpected repository query: ${text}`);
|
|
},
|
|
release() {
|
|
events.push('release:workflow-task-attempts');
|
|
},
|
|
};
|
|
}
|
|
throw new Error('Repository connections are not used during bootstrap');
|
|
},
|
|
};
|
|
return {
|
|
pool,
|
|
async close() {
|
|
events.push('close-database');
|
|
if (overrides.closeError) throw overrides.closeError;
|
|
},
|
|
};
|
|
}
|
|
|
|
function activationStack(
|
|
events,
|
|
recovery = { safe: true, remaining: 0, failed: 0 },
|
|
) {
|
|
return {
|
|
async reconcile() {
|
|
events.push('reconcile');
|
|
return recovery;
|
|
},
|
|
async startLifecycles() {
|
|
events.push('start-lifecycles');
|
|
return true;
|
|
},
|
|
installAdmission() {
|
|
events.push('install-admission');
|
|
return () => events.push('dispose-admission');
|
|
},
|
|
async stop() {
|
|
events.push('stop-stack');
|
|
return 'stopped';
|
|
},
|
|
};
|
|
}
|
|
|
|
function bootstrapOptions(events, overrides = {}) {
|
|
return {
|
|
enabled: true,
|
|
profile: 'cluster-control',
|
|
apiCredentialPepper: 'A'.repeat(43),
|
|
recovery: { ownerId: 'test-replica', providers: [] },
|
|
async openDatabase() {
|
|
events.push('open-database');
|
|
return databaseResource(events);
|
|
},
|
|
create(input) {
|
|
const {
|
|
evidence,
|
|
authenticator,
|
|
policies,
|
|
runs,
|
|
runAttemptLogRetention,
|
|
runCancellation,
|
|
taskDefinitions,
|
|
taskExecutionRevisions,
|
|
triggers,
|
|
schedules,
|
|
trustedToolStorage,
|
|
securityAudit,
|
|
} = input;
|
|
events.push('create-stack');
|
|
assert.equal('pool' in input, false);
|
|
assert.equal('recovery' in input, false);
|
|
assert.equal('recoveryClaims' in input, false);
|
|
assert.equal('recoveryTransitions' in input, false);
|
|
assert.equal(
|
|
evidence.contractVersion,
|
|
postgresqlControlSchemaContract.contractVersion,
|
|
);
|
|
assert.equal(typeof authenticator.authenticate, 'function');
|
|
assert.equal(typeof policies.resolve, 'function');
|
|
assert.equal(typeof runs.transaction, 'function');
|
|
assert.equal(typeof runAttemptLogRetention.inspect, 'function');
|
|
assert.equal(typeof runCancellation.requestUserCancellation, 'function');
|
|
assert.equal(
|
|
typeof taskDefinitions.findCurrentTaskDefinition,
|
|
'function',
|
|
);
|
|
assert.equal('appendTaskDefinitionRevision' in taskDefinitions, false);
|
|
assert.equal(
|
|
typeof taskExecutionRevisions.resolveClusterTaskExecutionRevision,
|
|
'function',
|
|
);
|
|
assert.equal(typeof triggers.findCurrentTrigger, 'function');
|
|
assert.equal('appendTriggerRevision' in triggers, false);
|
|
assert.equal(typeof schedules.claimNextClusterSchedule, 'function');
|
|
assert.equal(typeof schedules.commitClusterScheduleDecision, 'function');
|
|
assert.equal(Object.isFrozen(trustedToolStorage), true);
|
|
assert.equal(
|
|
typeof trustedToolStorage.invocationArtifacts.findInput,
|
|
'function',
|
|
);
|
|
assert.equal(typeof trustedToolStorage.stepRuns.findById, 'function');
|
|
assert.equal(
|
|
typeof trustedToolStorage.startBarriers.findByStartId,
|
|
'function',
|
|
);
|
|
assert.equal(
|
|
typeof trustedToolStorage.completions.findByStartId,
|
|
'function',
|
|
);
|
|
assert.equal(
|
|
typeof trustedToolStorage.failureCompletions.findByStartId,
|
|
'function',
|
|
);
|
|
assert.equal(
|
|
typeof trustedToolStorage.resultKeyCatalog.findCurrent,
|
|
'function',
|
|
);
|
|
assert.equal(trustedToolStorage.resultKeyCatalog.append, undefined);
|
|
assert.equal(
|
|
typeof trustedToolStorage.resultRekeys.findHeadByArtifactId,
|
|
'function',
|
|
);
|
|
assert.equal(trustedToolStorage.resultRekeys.append, undefined);
|
|
assert.equal(
|
|
typeof trustedToolStorage.toolDefinitionSnapshots.findCurrent,
|
|
'function',
|
|
);
|
|
assert.equal(typeof securityAudit.record, 'function');
|
|
return activationStack(events);
|
|
},
|
|
audit(record) {
|
|
events.push(`audit:${record.state}`);
|
|
},
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
test('disabled and wrong-profile bootstrap never opens PostgreSQL', async () => {
|
|
const disabledEvents = [];
|
|
const disabled = await bootstrapClusterControlRuntime(
|
|
bootstrapOptions(disabledEvents, { enabled: false }),
|
|
);
|
|
assert.equal(disabled.status, 'disabled');
|
|
assert.deepEqual(disabledEvents, ['audit:disabled']);
|
|
|
|
const wrongProfileEvents = [];
|
|
await assert.rejects(
|
|
bootstrapClusterControlRuntime(
|
|
bootstrapOptions(wrongProfileEvents, { profile: 'edge' }),
|
|
),
|
|
/cannot activate cluster-control/,
|
|
);
|
|
assert.deepEqual(wrongProfileEvents, []);
|
|
});
|
|
|
|
test('rejects a missing credential pepper before opening PostgreSQL', async () => {
|
|
const events = [];
|
|
await assert.rejects(
|
|
bootstrapClusterControlRuntime(
|
|
bootstrapOptions(events, { apiCredentialPepper: undefined }),
|
|
),
|
|
/API credential configuration is invalid/,
|
|
);
|
|
assert.deepEqual(events, []);
|
|
});
|
|
|
|
test('rejects missing or unbounded recovery configuration before opening PostgreSQL', async () => {
|
|
const missingEvents = [];
|
|
await assert.rejects(
|
|
bootstrapClusterControlRuntime(
|
|
bootstrapOptions(missingEvents, { recovery: undefined }),
|
|
),
|
|
/requires bounded recovery configuration/,
|
|
);
|
|
assert.deepEqual(missingEvents, []);
|
|
|
|
const timeoutEvents = [];
|
|
await assert.rejects(
|
|
bootstrapClusterControlRuntime(
|
|
bootstrapOptions(timeoutEvents, {
|
|
recovery: {
|
|
ownerId: 'test-replica',
|
|
claimLeaseMs: 1_000,
|
|
providerTimeoutMs: 900,
|
|
},
|
|
}),
|
|
),
|
|
/250ms for fenced settlement/,
|
|
);
|
|
assert.deepEqual(timeoutEvents, []);
|
|
|
|
const localClockEvents = [];
|
|
await assert.rejects(
|
|
bootstrapClusterControlRuntime(
|
|
bootstrapOptions(localClockEvents, {
|
|
scheduler: { clock: () => 1 },
|
|
}),
|
|
),
|
|
/scheduler configuration is invalid/,
|
|
);
|
|
assert.deepEqual(localClockEvents, []);
|
|
});
|
|
|
|
test('readiness failure closes the database before returning the root error', async () => {
|
|
const events = [];
|
|
await assert.rejects(
|
|
bootstrapClusterControlRuntime(
|
|
bootstrapOptions(events, {
|
|
async openDatabase() {
|
|
events.push('open-database');
|
|
return databaseResource(events, { serverVersionNum: '150018' });
|
|
},
|
|
}),
|
|
),
|
|
(error) =>
|
|
error instanceof PostgresSchemaReadinessError &&
|
|
error.code === 'server_version_unsupported',
|
|
);
|
|
assert.equal(events.includes('create-stack'), false);
|
|
assert.deepEqual(events.slice(-2), ['audit:failed', 'close-database']);
|
|
});
|
|
|
|
test('opens once, assembles after readiness, and closes after stack shutdown', async () => {
|
|
const events = [];
|
|
const result = await bootstrapClusterControlRuntime(bootstrapOptions(events));
|
|
assert.equal(result.status, 'active');
|
|
assert.equal(events.filter((event) => event === 'open-database').length, 1);
|
|
assert.ok(events.indexOf('create-stack') > events.lastIndexOf('query:8'));
|
|
const first = result.stop();
|
|
assert.equal(first, result.stop());
|
|
assert.equal(await first, 'stopped');
|
|
assert.deepEqual(events.slice(-4), [
|
|
'dispose-admission',
|
|
'stop-stack',
|
|
'audit:stopped',
|
|
'close-database',
|
|
]);
|
|
});
|
|
|
|
test('injects reviewed Worker operations without exposing the runtime Pool', async () => {
|
|
const events = [];
|
|
const artifactStore = {
|
|
async put() {
|
|
throw new Error('not invoked during assembly');
|
|
},
|
|
async inspect() {
|
|
throw new Error('not invoked during assembly');
|
|
},
|
|
async readLogRange() {
|
|
throw new Error('not invoked during assembly');
|
|
},
|
|
};
|
|
const result = await bootstrapClusterControlRuntime(
|
|
bootstrapOptions(events, {
|
|
workerRuntime: { artifactStore },
|
|
create(input) {
|
|
events.push('create-stack');
|
|
assert.equal('pool' in input, false);
|
|
assert.equal(Object.isFrozen(input.workerRuntime), true);
|
|
assert.equal(typeof input.workerRuntime.offers.claimNext, 'function');
|
|
assert.equal(
|
|
typeof input.workerRuntime.activation.acknowledgeStarting,
|
|
'function',
|
|
);
|
|
assert.equal(input.workerRuntime.secrets, undefined);
|
|
assert.equal(typeof input.workerRuntime.artifacts.upload, 'function');
|
|
assert.equal(
|
|
typeof input.workerRuntime.completion.complete,
|
|
'function',
|
|
);
|
|
assert.equal(
|
|
typeof input.workerRuntime.leaseControl.control,
|
|
'function',
|
|
);
|
|
assert.equal(
|
|
typeof input.workerRuntime.runAttemptLogRead.read,
|
|
'function',
|
|
);
|
|
return activationStack(events);
|
|
},
|
|
}),
|
|
);
|
|
assert.equal(result.status, 'active');
|
|
assert.equal(await result.stop(), 'stopped');
|
|
});
|
|
|
|
test('production scheduler cadence scans schedules, Workflow frontier, and Task Attempt admission', async () => {
|
|
const events = [];
|
|
let resolveCycle;
|
|
const cycle = new Promise((resolve) => {
|
|
resolveCycle = resolve;
|
|
});
|
|
const result = await bootstrapClusterControlRuntime(
|
|
bootstrapOptions(events, {
|
|
scheduler: {
|
|
intervalMs: 250,
|
|
onDiagnostic(error, summary) {
|
|
resolveCycle({ error, summary });
|
|
},
|
|
},
|
|
cancellationConvergence: { intervalMs: 60 * 60_000 },
|
|
async openDatabase() {
|
|
events.push('open-database');
|
|
return databaseResource(events, { runtimeScans: true });
|
|
},
|
|
}),
|
|
);
|
|
|
|
const observed = await Promise.race([
|
|
cycle,
|
|
new Promise((_, reject) => {
|
|
const timeout = setTimeout(
|
|
() => reject(new Error('production scheduler cycle did not run')),
|
|
2_000,
|
|
);
|
|
timeout.unref?.();
|
|
}),
|
|
]);
|
|
assert.equal(observed.error, undefined);
|
|
assert.deepEqual(observed.summary, {
|
|
firstClaimAcquiredAtMs: null,
|
|
lastClaimAcquiredAtMs: null,
|
|
claimed: 0,
|
|
initialized: 0,
|
|
skipped: 0,
|
|
admitted: 0,
|
|
raced: 0,
|
|
saturated: false,
|
|
});
|
|
assert.deepEqual(
|
|
events.filter((event) => event.startsWith('scan:')),
|
|
[
|
|
'scan:runtime-recovery',
|
|
'scan:lost-retry',
|
|
'scan:schedules',
|
|
'scan:workflow-frontier',
|
|
'scan:workflow-task-attempts',
|
|
],
|
|
);
|
|
assert.equal(await result.stop(), 'stopped');
|
|
assert.ok(events.includes('release:workflow-task-attempts'));
|
|
});
|
|
|
|
test('unsafe recovery stops the stack and closes the database', async () => {
|
|
const events = [];
|
|
await assert.rejects(
|
|
bootstrapClusterControlRuntime(
|
|
bootstrapOptions(events, {
|
|
create({ runs }) {
|
|
events.push('create-stack');
|
|
assert.equal(typeof runs.findRunById, 'function');
|
|
return activationStack(events, {
|
|
safe: false,
|
|
remaining: 1,
|
|
failed: 0,
|
|
});
|
|
},
|
|
}),
|
|
),
|
|
/did not converge safely/,
|
|
);
|
|
assert.equal(events.includes('install-admission'), false);
|
|
assert.deepEqual(events.slice(-3), [
|
|
'stop-stack',
|
|
'audit:failed',
|
|
'close-database',
|
|
]);
|
|
});
|
|
|
|
test('bootstrap-owned recovery blocks a false-safe stack when the claim store is unavailable', async () => {
|
|
const events = [];
|
|
await assert.rejects(
|
|
bootstrapClusterControlRuntime(
|
|
bootstrapOptions(events, {
|
|
async openDatabase() {
|
|
events.push('open-database');
|
|
return databaseResource(events, {
|
|
recoveryRows: [
|
|
{
|
|
observedAtMs: '1',
|
|
kind: 'run',
|
|
id: 'run-1',
|
|
runId: 'run-1',
|
|
status: 'running',
|
|
createdAtMs: '1',
|
|
},
|
|
{
|
|
observedAtMs: '1',
|
|
kind: 'attempt',
|
|
id: 'attempt-1',
|
|
runId: 'run-1',
|
|
status: 'running',
|
|
createdAtMs: '2',
|
|
},
|
|
],
|
|
});
|
|
},
|
|
}),
|
|
),
|
|
(error) =>
|
|
error?.name === 'ClusterControlRecoveryStoreError' &&
|
|
error.retryable === true,
|
|
);
|
|
assert.equal(events.includes('install-admission'), false);
|
|
assert.equal(events.includes('reconcile'), false);
|
|
assert.deepEqual(events.slice(-3), [
|
|
'stop-stack',
|
|
'audit:failed',
|
|
'close-database',
|
|
]);
|
|
});
|
|
|
|
test('database close failure does not skip stack shutdown and remains idempotent', async () => {
|
|
const events = [];
|
|
const closeError = new Error('database close failed');
|
|
const result = await bootstrapClusterControlRuntime(
|
|
bootstrapOptions(events, {
|
|
async openDatabase() {
|
|
events.push('open-database');
|
|
return databaseResource(events, { closeError });
|
|
},
|
|
}),
|
|
);
|
|
const first = result.stop();
|
|
assert.equal(first, result.stop());
|
|
await assert.rejects(first, (error) => error === closeError);
|
|
assert.deepEqual(events.slice(-4), [
|
|
'dispose-admission',
|
|
'stop-stack',
|
|
'audit:stopped',
|
|
'close-database',
|
|
]);
|
|
});
|