open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-20 16:07:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add strong cluster run stop

Browse Source
This commit is contained in:
whyour
2026-08-12 08:21:28 +08:00
parent c0ab62e64a
commit dd370b2842
22 changed files with 1602 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-cluster-postgres/test/clusterRunCancellationRepository.test.cjs
+155 -54
View File
@@ -41,25 +41,38 @@ function fixture(options = {}) {
const normalized = sql.replace(/\s+/g, ' ').trim();
calls.push({ sql: normalized, params });
if (
normalized.startsWith('BEGIN') || normalized === 'COMMIT' ||
normalized === 'ROLLBACK' || normalized.startsWith('SELECT set_config')
) return { rows: [], rowCount: 0 };
normalized.startsWith('BEGIN') ||
normalized === 'COMMIT' ||
normalized === 'ROLLBACK' ||
normalized.startsWith('SELECT set_config')
)
return { rows: [], rowCount: 0 };
if (normalized.includes('lock_run_management_policy_fence')) {
return {
rows: [{ matches: options.policyMatches ?? true }],
rowCount: 1,
};
}
if (normalized.includes('FROM "ql3"."projects"')) {
return {
rows: options.projectRows ?? [{
projectStatus: 'active',
projectVersion: 2,
}],
rows: options.projectRows ?? [
{
projectStatus: 'active',
projectVersion: 2,
},
],
rowCount: 1,
};
}
if (normalized.includes('FROM "ql3"."project_role_bindings"')) {
return {
rows: options.bindingRows ?? [{
bindingVersion: 3,
bindingState: 'active',
bindingRole: 'operator',
}],
rows: options.bindingRows ?? [
{
bindingVersion: 3,
bindingState: 'active',
bindingRole: 'operator',
},
],
rowCount: 1,
};
}
@@ -73,9 +86,7 @@ function fixture(options = {}) {
};
}
if (
normalized.includes(
'FROM "ql3"."plugin_package_workflow_admissions"',
)
normalized.includes('FROM "ql3"."plugin_package_workflow_admissions"')
) {
const rows = options.workflowAdmissionRows ?? [
{
@@ -91,25 +102,40 @@ function fixture(options = {}) {
}
if (normalized.startsWith('UPDATE "ql3"."runs"')) {
return {
rows: options.updatedRows ?? [run({
runVersion: 5,
eventSequence: 7,
cancelRequestedAtMs: options.nowMs ?? 1_000,
cancelReason: 'user',
})],
rows: options.updatedRows ?? [
run({
runVersion: 5,
eventSequence: 7,
cancelRequestedAtMs: options.nowMs ?? 1_000,
cancelReason: 'user',
}),
],
rowCount: options.updatedRows?.length ?? 1,
};
}
if (normalized.startsWith('INSERT INTO "ql3"."run_events"')) {
return { rows: [], rowCount: 1 };
}
if (normalized.startsWith('INSERT INTO "ql3"."security_audit_events"')) {
return {
rows: options.auditInserted === false ? [] : [{ eventId: params[0] }],
rowCount: options.auditInserted === false ? 0 : 1,
};
}
if (normalized.includes('FROM "ql3"."security_audit_events"')) {
return { rows: options.auditReplayRows ?? [], rowCount: 0 };
}
throw new Error(`Unexpected SQL: ${normalized}`);
},
release() { calls.push({ sql: 'RELEASE', params: [] }); },
release() {
calls.push({ sql: 'RELEASE', params: [] });
},
};
return {
repository: new PostgresClusterRunCancellationRepository({
async connect() { return client; },
async connect() {
return client;
},
}),
calls,
};
@@ -127,23 +153,26 @@ test('revalidates policy authority and commits one database-timed intent', async
cancelRequestedAtMs: 1_000,
cancelReason: 'user',
});
const projectIndex = calls.findIndex(({ sql }) =>
sql.includes('FROM "ql3"."projects"'));
const bindingIndex = calls.findIndex(({ sql }) =>
sql.includes('FROM "ql3"."project_role_bindings"'));
const policyIndex = calls.findIndex(({ sql }) =>
sql.includes('lock_run_management_policy_fence'),
);
const runIndex = calls.findIndex(({ sql }) =>
sql.includes('FROM "ql3"."runs"'));
assert.ok(projectIndex < bindingIndex && bindingIndex < runIndex);
const update = calls.find(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'));
sql.includes('FROM "ql3"."runs"'),
);
assert.ok(policyIndex >= 0 && policyIndex < runIndex);
const update = calls.find(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"'));
assert.deepEqual(update.params, ['run-1', 1_000, 5, 7, 4]);
const event = calls.find(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"'));
sql.startsWith('INSERT INTO "ql3"."run_events"'),
);
assert.equal(event.params[0], command().eventId);
assert.equal(event.params[3], 'user-cancel:mutation-1');
assert.equal(event.params[4], 'user');
assert.equal(JSON.parse(event.params[6]).reason, 'user');
assert.equal(calls.some(({ sql }) => sql === 'COMMIT'), true);
assert.equal(
calls.some(({ sql }) => sql === 'COMMIT'),
true,
);
});
test('returns existing intent and terminal state without adding an event', async () => {
@@ -154,8 +183,12 @@ test('returns existing intent and terminal state without adding an event', async
(await existing.repository.requestUserCancellation(command())).status,
'already_requested',
);
assert.equal(existing.calls.some(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"')), false);
assert.equal(
existing.calls.some(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"'),
),
false,
);
const terminal = fixture({
runRows: [run({ runStatus: 'succeeded', runVersion: 5 })],
@@ -164,20 +197,24 @@ test('returns existing intent and terminal state without adding an event', async
(await terminal.repository.requestUserCancellation(command())).status,
'already_terminal',
);
assert.equal(terminal.calls.some(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"')), false);
assert.equal(
terminal.calls.some(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"')),
false,
);
});
test('accepts cancellation for a lost Run that still owns retry authority', async () => {
const { repository } = fixture({
runRows: [run({ runStatus: 'lost' })],
updatedRows: [run({
runStatus: 'lost',
runVersion: 5,
eventSequence: 7,
cancelRequestedAtMs: 1_000,
cancelReason: 'user',
})],
updatedRows: [
run({
runStatus: 'lost',
runVersion: 5,
eventSequence: 7,
cancelRequestedAtMs: 1_000,
cancelReason: 'user',
}),
],
});
assert.equal(
(await repository.requestUserCancellation(command())).status,
@@ -187,11 +224,7 @@ test('accepts cancellation for a lost Run that still owns retry authority', asyn
test('rejects a revoked policy fence before locking the Run', async () => {
const { repository, calls } = fixture({
bindingRows: [{
bindingVersion: 4,
bindingState: 'revoked',
bindingRole: null,
}],
policyMatches: false,
});
await assert.rejects(
repository.requestUserCancellation(command()),
@@ -199,8 +232,78 @@ test('rejects a revoked policy fence before locking the Run', async () => {
error instanceof ClusterRunCancellationFenceRejectedError &&
error.reason === 'authorization_changed',
);
assert.equal(calls.some(({ sql }) => sql.includes('FROM "ql3"."runs"')), false);
assert.equal(calls.some(({ sql }) => sql === 'ROLLBACK'), true);
assert.equal(
calls.some(({ sql }) => sql.includes('FROM "ql3"."runs"')),
false,
);
assert.equal(
calls.some(({ sql }) => sql === 'ROLLBACK'),
true,
);
});
test('atomically records strong management audit and exact audit replay', async () => {
const { repository, calls } = fixture();
const { subject: _subject, ...baseCommand } = command();
const managed = {
...baseCommand,
mutationId: '019f0000-0000-4000-8000-000000000001',
requestId: 'request-stop-1',
auditEventId: '019f0000-0000-4000-8000-000000000002',
principal: {
subject: command().subject,
authenticationId: 'oidc:run-management-1',
authenticatedAtMs: 900,
expiresAtMs: 2_000,
assurance: 'hardware',
},
};
const result = await repository.requestUserCancellationAudited(managed);
assert.equal(result.status, 'accepted');
const audit = calls.find(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
);
assert.equal(audit.params[0], managed.auditEventId);
assert.equal(audit.params[1], managed.requestId);
assert.equal(audit.params[5], managed.principal.authenticationId);
assert.ok(
calls.findIndex(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"')) <
calls.findIndex(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
),
);
assert.ok(
calls.findIndex(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
) < calls.findIndex(({ sql }) => sql === 'COMMIT'),
);
const replay = fixture({
runRows: [run({ cancelRequestedAtMs: 1_000, cancelReason: 'user' })],
auditInserted: false,
auditReplayRows: [
{
requestId: managed.requestId,
operationId: 'run.stop',
projectId: managed.projectId,
subjectType: 'user',
subjectId: 'user-1',
authenticationId: managed.principal.authenticationId,
outcome: 'allowed',
reasons: ['role_grant', 'strong_authentication'],
projectVersion: 2,
bindingVersion: 3,
},
],
});
assert.equal(
(await replay.repository.requestUserCancellationAudited(managed)).status,
'already_requested',
);
assert.equal(
replay.calls.some(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"')),
false,
);
});
test('masks cross-Project and missing Runs', async () => {
@@ -255,9 +358,7 @@ test('binds Workflow cancellation to the immutable admission target', async () =
ClusterRunCancellationNotFoundError,
);
assert.equal(
rejected.calls.some(({ sql }) =>
sql.startsWith('UPDATE "ql3"."runs"'),
),
rejected.calls.some(({ sql }) => sql.startsWith('UPDATE "ql3"."runs"')),
false,
);
}
packages/ql3-cluster-postgres/test/postgresqlMigrationDefinitions.test.cjs
+42 -27
View File
@@ -1,6 +1,8 @@
const assert = require('node:assert/strict');
const { test } = require('node:test');
const { postgresqlControlSchemaContract } = require('../dist/schema/schemaContract');
const {
postgresqlControlSchemaContract,
} = require('../dist/schema/schemaContract');
const {
postgresqlMainMigrationManifest,
} = require('../dist/migration/migrationManifest');
@@ -105,6 +107,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
'pg-0054-approval-management-boundary',
'pg-0055-run-attempt-log-retention',
'pg-0056-run-management-boundary',
'pg-0057-run-management-stop-boundary',
],
);
for (const migration of postgresqlMainMigrationStream.migrations) {
@@ -520,6 +523,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
checksum:
'7aa2b2ade67cdfa6839d4af02209906646a68adfd6c12c4dddeb854021da72b8',
},
{
id: 'pg-0057-run-management-stop-boundary',
checksum:
'ab2d0eee3d85a937e1e87243b1fd1e75181529122b64026303488404162e4ba7',
},
];
assert.deepEqual(
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
@@ -1588,7 +1596,10 @@ test('advances capability v45 with generation-bound Workflow Task attempts', asy
/CREATE FUNCTION "ql3"\."plugin_package_workflow_task_attempt_snapshot"/,
);
assert.match(sql, /SECURITY DEFINER/);
assert.match(sql, /FOR KEY SHARE OF workflow, source, reconciliation, item, execution/);
assert.match(
sql,
/FOR KEY SHARE OF workflow, source, reconciliation, item, execution/,
);
assert.match(
sql,
/GRANT SELECT, INSERT[\s\S]*plugin_package_workflow_task_attempt_admissions[\s\S]*TO ql3_runtime/,
@@ -1611,16 +1622,11 @@ test('advances capability v45 with generation-bound Workflow Task attempts', asy
sql,
/migration_id\s*=\s*'pg-0045-plugin-package-workflow-admissions'/,
);
assert.match(
sql,
/"plugin_package_workflow_task_attempt_admission":1/,
);
assert.match(sql, /"plugin_package_workflow_task_attempt_admission":1/);
});
test('advances capability v46 with split Worker credential management authorities', async () => {
const migration = migrationById(
'pg-0047-worker-credential-management-plans',
);
const migration = migrationById('pg-0047-worker-credential-management-plans');
const statements = [];
await migration.up({
async query(statement) {
@@ -1629,10 +1635,7 @@ test('advances capability v46 with split Worker credential management authoritie
},
});
const sql = statements.join('\n');
assert.match(
sql,
/CREATE TABLE "ql3"\."worker_credential_management_plans"/,
);
assert.match(sql, /CREATE TABLE "ql3"\."worker_credential_management_plans"/);
assert.match(sql, /'ql3_worker_credential_manager'/);
assert.match(sql, /'ql3_worker_credential_executor'/);
assert.match(
@@ -1676,10 +1679,7 @@ test('advances capability v47 without invalidating preapproved Worker credential
},
});
const sql = statements.join('\n');
assert.match(
sql,
/DROP CONSTRAINT ql3_worker_credentials_lifetime_check/,
);
assert.match(sql, /DROP CONSTRAINT ql3_worker_credentials_lifetime_check/);
assert.match(
sql,
/expires_at_ms > GREATEST\(created_at_ms, not_before_at_ms\)/,
@@ -1747,7 +1747,10 @@ test('advances capability v49 with durable Worker credential management boundari
},
});
const sql = statements.join('\n');
assert.match(sql, /CREATE TABLE "ql3"\."worker_credential_management_quota_buckets"/);
assert.match(
sql,
/CREATE TABLE "ql3"\."worker_credential_management_quota_buckets"/,
);
assert.match(sql, /TO ql3_worker_credential_manager/);
assert.doesNotMatch(
sql,
@@ -1820,10 +1823,7 @@ test('advances capability v51 with a restart-safe automation identity keyset led
assert.match(sql, /contract_version = 51/);
assert.match(sql, /"automation_management_identity_keyset_ledger":1/);
assert.match(sql, /contract_version = 50/);
assert.match(
sql,
/migration_id = 'pg-0051-automation-management-boundary'/,
);
assert.match(sql, /migration_id = 'pg-0051-automation-management-boundary'/);
});
test('advances capability v52 with a bounded Workflow Run history index', async () => {
@@ -1905,10 +1905,7 @@ test('advances capability v54 with durable Cluster log retention authority', asy
assert.match(sql, /contract_version = 54/);
assert.match(sql, /"run_attempt_log_retention":1/);
assert.match(sql, /contract_version = 53/);
assert.match(
sql,
/migration_id = 'pg-0054-approval-management-boundary'/,
);
assert.match(sql, /migration_id = 'pg-0054-approval-management-boundary'/);
});
test('advances capability v55 with isolated strong Run management authority', async () => {
@@ -1932,8 +1929,26 @@ test('advances capability v55 with isolated strong Run management authority', as
assert.match(sql, /contract_version = 55/);
assert.match(sql, /"run_management_boundary":1/);
assert.match(sql, /contract_version = 54/);
assert.match(sql, /migration_id = 'pg-0055-run-attempt-log-retention'/);
});
test('advances capability v56 with column-scoped Run stop authority', async () => {
const migration = migrationById('pg-0057-run-management-stop-boundary');
const statements = [];
await migration.up({
async query(statement) {
statements.push(statement);
return { rows: [] };
},
});
const sql = statements.join('\n');
assert.match(
sql,
/migration_id = 'pg-0055-run-attempt-log-retention'/,
/GRANT UPDATE \(cancel_requested_at_ms, cancel_reason, version, event_sequence\) ON "ql3"\."runs" TO ql3_run_manager/,
);
assert.doesNotMatch(sql, /GRANT UPDATE ON "ql3"\."runs" TO ql3_run_manager/);
assert.match(sql, /contract_version = 56/);
assert.match(sql, /"run_management_stop":1/);
assert.match(sql, /contract_version = 55/);
assert.match(sql, /migration_id = 'pg-0056-run-management-boundary'/);
});
packages/ql3-cluster-postgres/test/postgresqlSchemaReadiness.test.cjs
+71 -23
View File
@@ -13,7 +13,9 @@ const {
assertPostgresWorkerCredentialManagerSchemaReady,
assertPostgresWorkerIngressSchemaReady,
} = require('../dist/schema/schemaReadiness');
const { postgresqlControlSchemaContract } = require('../dist/schema/schemaContract');
const {
postgresqlControlSchemaContract,
} = require('../dist/schema/schemaContract');
const { postgresqlMainMigrationStream } = require('../dist/migrations');
function validHistory() {
@@ -86,12 +88,7 @@ function validPrivileges() {
plugin_package_automation_publication_heads: [true, false, false, false],
plugin_package_workflow_admissions: [true, true, false, false],
plugin_package_workflow_admission_steps: [true, true, false, false],
plugin_package_workflow_task_attempt_admissions: [
true,
true,
false,
false,
],
plugin_package_workflow_task_attempt_admissions: [true, true, false, false],
plugin_package_publisher_provenance: [false, false, false, false],
plugin_package_publisher_revocation_receipts: [false, false, false, false],
plugin_package_publisher_revocation_impacts: [false, false, false, false],
@@ -523,11 +520,11 @@ function workerCredentialPrivileges(kind) {
]
: []),
...(manager
? []
: [
'approved_action_dispatches',
'approved_action_executions',
'worker_credentials',
? []
: [
'approved_action_dispatches',
'approved_action_executions',
'worker_credentials',
'worker_credential_mutations',
'worker_credential_deliveries',
'worker_credential_stage_discards',
@@ -714,6 +711,26 @@ function queryable(overrides = {}) {
],
};
}
if (text.includes('has_column_privilege')) {
assert.match(text, /format\('%I\.%I', \$1::text, 'runs'\)/);
const columns = contract.tables.find(
({ name }) => name === 'runs',
).columns;
const allowed = new Set([
'cancel_requested_at_ms',
'cancel_reason',
'version',
'event_sequence',
]);
return {
rows:
overrides.runManagerColumnPrivileges ??
columns.map((columnName) => ({
columnName,
updateAllowed: allowed.has(columnName),
})),
};
}
if (text.includes('has_table_privilege')) {
return { rows: overrides.privileges ?? validPrivileges() };
}
@@ -731,7 +748,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
serverMajor: 16,
currentUser: 'ql3_runtime',
contractName: 'control-core',
contractVersion: 55,
contractVersion: 56,
migrationIds: [
'pg-0001-schema-capability',
'pg-0002-run-core',
@@ -789,6 +806,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
'pg-0054-approval-management-boundary',
'pg-0055-run-attempt-log-retention',
'pg-0056-run-management-boundary',
'pg-0057-run-management-stop-boundary',
],
});
});
@@ -819,10 +837,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
}),
);
assert.equal(report.currentUser, 'ql3_admin');
assert.equal(report.contractVersion, 55);
assert.equal(report.contractVersion, 56);
assert.equal(
report.migrationIds.at(-1),
'pg-0056-run-management-boundary',
'pg-0057-run-management-stop-boundary',
);
});
@@ -835,10 +853,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
}),
);
assert.equal(report.currentUser, 'ql3_automation_manager');
assert.equal(report.contractVersion, 55);
assert.equal(report.contractVersion, 56);
assert.equal(
report.migrationIds.at(-1),
'pg-0056-run-management-boundary',
'pg-0057-run-management-stop-boundary',
);
const widened = automationManagerPrivileges();
@@ -867,10 +885,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
}),
);
assert.equal(report.currentUser, 'ql3_approval_manager');
assert.equal(report.contractVersion, 55);
assert.equal(report.contractVersion, 56);
assert.equal(
report.migrationIds.at(-1),
'pg-0056-run-management-boundary',
'pg-0057-run-management-stop-boundary',
);
const widened = approvalManagerPrivileges();
@@ -901,8 +919,11 @@ test('accepts the isolated least-privilege Run manager role', async () => {
}),
);
assert.equal(report.currentUser, 'ql3_run_manager');
assert.equal(report.contractVersion, 55);
assert.equal(report.migrationIds.at(-1), 'pg-0056-run-management-boundary');
assert.equal(report.contractVersion, 56);
assert.equal(
report.migrationIds.at(-1),
'pg-0057-run-management-stop-boundary',
);
const widened = runManagerPrivileges();
widened.find(({ tableName }) => tableName === 'runs').updateAllowed = true;
@@ -919,6 +940,33 @@ test('accepts the isolated least-privilege Run manager role', async () => {
error.code === 'run_manager_role_invalid' &&
error.facts.includes('table-privileges:runs'),
);
const widenedColumns = postgresqlControlSchemaContract.tables
.find(({ name }) => name === 'runs')
.columns.map((columnName) => ({
columnName,
updateAllowed: [
'cancel_requested_at_ms',
'cancel_reason',
'version',
'event_sequence',
'status',
].includes(columnName),
}));
await assert.rejects(
assertPostgresRunManagerSchemaReady(
queryable({
currentUser: 'ql3_run_manager',
privileges: runManagerPrivileges(),
functionMode: 'run-manager',
runManagerColumnPrivileges: widenedColumns,
}),
),
(error) =>
error instanceof PostgresSchemaReadinessError &&
error.code === 'run_manager_role_invalid' &&
error.facts.includes('column-update-privilege:runs.status'),
);
});
test('accepts isolated Package manager and executor roles', async () => {
@@ -1006,10 +1054,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
}),
);
assert.equal(report.currentUser, 'ql3_worker_ingress');
assert.equal(report.contractVersion, 55);
assert.equal(report.contractVersion, 56);
assert.equal(
report.migrationIds.at(-1),
'pg-0056-run-management-boundary',
'pg-0057-run-management-stop-boundary',
);
});