mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-23 03:18:09 +08:00
feat(ql3): operationalize cancellation rearm
This commit is contained in:
whyour
@@ -117,6 +117,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
|
||||
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
|
||||
'pg-0065-approved-action-manual-recovery',
|
||||
'pg-0066-cancellation-dispatch',
|
||||
'pg-0067-cancellation-dispatch-management',
|
||||
],
|
||||
);
|
||||
for (const migration of postgresqlMainMigrationStream.migrations) {
|
||||
@@ -585,6 +586,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
|
||||
checksum:
|
||||
'b6d7ac81b5f75530df05f8ef05878fa30aa0f4418363973ded89d14ffce151b2',
|
||||
},
|
||||
{
|
||||
id: 'pg-0067-cancellation-dispatch-management',
|
||||
checksum:
|
||||
'e78e24a06dc4c4dbdd859685f28b4bc837a8cfb279eb3512e0a57dc6d27eaaaa',
|
||||
},
|
||||
];
|
||||
assert.deepEqual(
|
||||
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
|
||||
@@ -2329,3 +2335,41 @@ test('advances capability v65 with database-timed fenced cancellation dispatch',
|
||||
/migration_id = 'pg-0065-approved-action-manual-recovery'/,
|
||||
);
|
||||
});
|
||||
|
||||
test('advances capability v66 with least-privilege cancellation diagnostics and rearm', async () => {
|
||||
const migration = migrationById(
|
||||
'pg-0067-cancellation-dispatch-management',
|
||||
);
|
||||
const statements = [];
|
||||
await migration.up({
|
||||
async query(statement) {
|
||||
statements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
const sql = statements.join('\n');
|
||||
assert.match(
|
||||
sql,
|
||||
/DROP CONSTRAINT ql3_run_cancellation_dispatch_result_state_check/,
|
||||
);
|
||||
assert.match(
|
||||
sql,
|
||||
/status IN \('leased', 'retry_wait'\)[\s\S]+identity_mismatch[\s\S]+dispatch_error/,
|
||||
);
|
||||
assert.match(
|
||||
sql,
|
||||
/GRANT SELECT ON "ql3"\."run_cancellation_dispatches" TO ql3_run_manager/,
|
||||
);
|
||||
assert.match(
|
||||
sql,
|
||||
/GRANT UPDATE \(status, version, next_attempt_at_ms, updated_at_ms\) ON "ql3"\."run_cancellation_dispatches" TO ql3_run_manager/,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
sql,
|
||||
/GRANT (?:INSERT|DELETE|TRUNCATE)[^;]+run_cancellation_dispatches[^;]+ql3_run_manager/,
|
||||
);
|
||||
assert.match(sql, /contract_version = 66/);
|
||||
assert.match(sql, /"run_cancellation_dispatch_management":1/);
|
||||
assert.match(sql, /contract_version = 65/);
|
||||
assert.match(sql, /migration_id = 'pg-0066-cancellation-dispatch'/);
|
||||
});
|
||||
|
||||
@@ -527,6 +527,7 @@ function runManagerPrivileges() {
|
||||
'runs',
|
||||
'run_attempts',
|
||||
'run_events',
|
||||
'run_cancellation_dispatches',
|
||||
'security_audit_events',
|
||||
'plugin_package_identity_keyset_ledger',
|
||||
]);
|
||||
@@ -781,19 +782,36 @@ function queryable(overrides = {}) {
|
||||
};
|
||||
}
|
||||
if (text.includes('has_column_privilege')) {
|
||||
assert.match(text, /format\('%I\.%I', \$1::text, 'runs'\)/);
|
||||
const dispatchManagement = text.includes(
|
||||
"format('%I.%I', $1::text, 'run_cancellation_dispatches')",
|
||||
);
|
||||
const tableName = dispatchManagement
|
||||
? 'run_cancellation_dispatches'
|
||||
: 'runs';
|
||||
assert.match(
|
||||
text,
|
||||
new RegExp(
|
||||
`format\\('%I\\.%I', \\$1::text, '${tableName}'\\)`,
|
||||
),
|
||||
);
|
||||
const columns = contract.tables.find(
|
||||
({ name }) => name === 'runs',
|
||||
({ name }) => name === tableName,
|
||||
).columns;
|
||||
const allowed = new Set([
|
||||
'cancel_requested_at_ms',
|
||||
'cancel_reason',
|
||||
'version',
|
||||
'event_sequence',
|
||||
]);
|
||||
const allowed = new Set(
|
||||
dispatchManagement
|
||||
? ['status', 'version', 'next_attempt_at_ms', 'updated_at_ms']
|
||||
: [
|
||||
'cancel_requested_at_ms',
|
||||
'cancel_reason',
|
||||
'version',
|
||||
'event_sequence',
|
||||
],
|
||||
);
|
||||
return {
|
||||
rows:
|
||||
overrides.runManagerColumnPrivileges ??
|
||||
(dispatchManagement
|
||||
? overrides.runManagerDispatchColumnPrivileges
|
||||
: overrides.runManagerColumnPrivileges) ??
|
||||
columns.map((columnName) => ({
|
||||
columnName,
|
||||
updateAllowed: allowed.has(columnName),
|
||||
@@ -817,7 +835,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
serverMajor: 16,
|
||||
currentUser: 'ql3_runtime',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 65,
|
||||
contractVersion: 66,
|
||||
migrationIds: [
|
||||
'pg-0001-schema-capability',
|
||||
'pg-0002-run-core',
|
||||
@@ -885,6 +903,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
|
||||
'pg-0065-approved-action-manual-recovery',
|
||||
'pg-0066-cancellation-dispatch',
|
||||
'pg-0067-cancellation-dispatch-management',
|
||||
],
|
||||
});
|
||||
});
|
||||
@@ -915,10 +934,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_admin');
|
||||
assert.equal(report.contractVersion, 65);
|
||||
assert.equal(report.contractVersion, 66);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0066-cancellation-dispatch',
|
||||
'pg-0067-cancellation-dispatch-management',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -931,10 +950,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_automation_manager');
|
||||
assert.equal(report.contractVersion, 65);
|
||||
assert.equal(report.contractVersion, 66);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0066-cancellation-dispatch',
|
||||
'pg-0067-cancellation-dispatch-management',
|
||||
);
|
||||
|
||||
const widened = automationManagerPrivileges();
|
||||
@@ -963,10 +982,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_approval_manager');
|
||||
assert.equal(report.contractVersion, 65);
|
||||
assert.equal(report.contractVersion, 66);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0066-cancellation-dispatch',
|
||||
'pg-0067-cancellation-dispatch-management',
|
||||
);
|
||||
|
||||
const widened = approvalManagerPrivileges();
|
||||
@@ -997,10 +1016,10 @@ test('accepts the isolated least-privilege Run manager role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_run_manager');
|
||||
assert.equal(report.contractVersion, 65);
|
||||
assert.equal(report.contractVersion, 66);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0066-cancellation-dispatch',
|
||||
'pg-0067-cancellation-dispatch-management',
|
||||
);
|
||||
|
||||
const widened = runManagerPrivileges();
|
||||
@@ -1045,6 +1064,35 @@ test('accepts the isolated least-privilege Run manager role', async () => {
|
||||
error.code === 'run_manager_role_invalid' &&
|
||||
error.facts.includes('column-update-privilege:runs.status'),
|
||||
);
|
||||
|
||||
const widenedDispatchColumns = postgresqlControlSchemaContract.tables
|
||||
.find(({ name }) => name === 'run_cancellation_dispatches')
|
||||
.columns.map((columnName) => ({
|
||||
columnName,
|
||||
updateAllowed: [
|
||||
'status',
|
||||
'version',
|
||||
'next_attempt_at_ms',
|
||||
'updated_at_ms',
|
||||
'lease_token_digest',
|
||||
].includes(columnName),
|
||||
}));
|
||||
await assert.rejects(
|
||||
assertPostgresRunManagerSchemaReady(
|
||||
queryable({
|
||||
currentUser: 'ql3_run_manager',
|
||||
privileges: runManagerPrivileges(),
|
||||
functionMode: 'run-manager',
|
||||
runManagerDispatchColumnPrivileges: widenedDispatchColumns,
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof PostgresSchemaReadinessError &&
|
||||
error.code === 'run_manager_role_invalid' &&
|
||||
error.facts.includes(
|
||||
'column-update-privilege:run_cancellation_dispatches.lease_token_digest',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('accepts isolated Package manager and executor roles', async () => {
|
||||
@@ -1132,10 +1180,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_worker_ingress');
|
||||
assert.equal(report.contractVersion, 65);
|
||||
assert.equal(report.contractVersion, 66);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0066-cancellation-dispatch',
|
||||
'pg-0067-cancellation-dispatch-management',
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
+291
@@ -0,0 +1,291 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
InvalidRunCancellationDispatchManagementError,
|
||||
PostgresRunCancellationDispatchManagementRepository,
|
||||
RunCancellationDispatchManagementConflictError,
|
||||
} = require('@qinglong/cluster-postgres/run-manager');
|
||||
|
||||
const NOW = 1_000_000;
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
requestId: 'request-1',
|
||||
auditEventId: '019f9600-0000-4000-8000-000000000001',
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:run-management-1',
|
||||
authenticatedAtMs: NOW - 1_000,
|
||||
expiresAtMs: NOW + 60_000,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function rearmCommand(overrides = {}) {
|
||||
return command({
|
||||
requestId: 'request-rearm-1',
|
||||
auditEventId: '019f9600-0000-4000-8000-000000000011',
|
||||
mutationId: '019f9600-0000-4000-8000-000000000012',
|
||||
eventId: '019f9600-0000-4000-8000-000000000013',
|
||||
expectedDispatchVersion: 3,
|
||||
expectedLastResult: 'identity_mismatch',
|
||||
retryDelayMs: 5_000,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function runRow() {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
runStatus: 'running',
|
||||
runVersion: 6,
|
||||
eventSequence: 8,
|
||||
cancelRequestedAtMs: NOW - 2_000,
|
||||
cancelReason: 'user',
|
||||
};
|
||||
}
|
||||
|
||||
function dispatchRow(overrides = {}) {
|
||||
return {
|
||||
attemptId: 'attempt-1',
|
||||
dispatchStatus: 'blocked',
|
||||
dispatchVersion: 3,
|
||||
dispatchCount: 1,
|
||||
nextAttemptAtMs: null,
|
||||
leaseExpiresAtMs: null,
|
||||
lastResult: 'identity_mismatch',
|
||||
lastDispatchedAtMs: NOW - 1_500,
|
||||
dispatchCreatedAtMs: NOW - 1_900,
|
||||
dispatchUpdatedAtMs: NOW - 1_500,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const calls = [];
|
||||
const client = {
|
||||
async query(sql, params = []) {
|
||||
const text = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ sql: text, params });
|
||||
if (
|
||||
text === 'BEGIN ISOLATION LEVEL SERIALIZABLE' ||
|
||||
text === 'COMMIT' ||
|
||||
text === 'ROLLBACK' ||
|
||||
text.startsWith('SELECT set_config')
|
||||
) {
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (text.includes('transaction_timestamp()')) {
|
||||
return { rows: [{ nowMs: NOW }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('lock_run_management_policy_fence')) {
|
||||
return {
|
||||
rows: [{ matches: options.authorized !== false }],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."runs" WHERE id = $1 FOR UPDATE')) {
|
||||
return { rows: [runRow()], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."runs" WHERE id = $1')) {
|
||||
return { rows: [runRow()], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."run_events"') &&
|
||||
text.includes('dedupe_key = $2')
|
||||
) {
|
||||
return { rows: options.replay ? [options.replay] : [], rowCount: 0 };
|
||||
}
|
||||
if (
|
||||
text.startsWith('SELECT attempt_id AS "attemptId"') &&
|
||||
!text.includes('dispatchStatus') &&
|
||||
!text.includes('FOR UPDATE')
|
||||
) {
|
||||
return { rows: [{ attemptId: 'attempt-1' }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."run_attempts"')) {
|
||||
return {
|
||||
rows: [{ attemptStatus: options.attemptStatus ?? 'running' }],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."run_cancellation_dispatches"') &&
|
||||
text.includes('FOR UPDATE')
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
dispatchRow({
|
||||
lastResult: options.lastResult ?? 'identity_mismatch',
|
||||
}),
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."run_cancellation_dispatches"')) {
|
||||
return { rows: [dispatchRow()], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."runs"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('UPDATE "ql3"."run_cancellation_dispatches"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('INSERT INTO "ql3"."run_events"')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (text.startsWith('INSERT INTO "ql3"."security_audit_events"')) {
|
||||
return { rows: [{ eventId: params[0] }], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`unexpected query: ${text}`);
|
||||
},
|
||||
release() {
|
||||
calls.push({ sql: 'RELEASE', params: [] });
|
||||
},
|
||||
};
|
||||
const pool = { async connect() { return client; } };
|
||||
return {
|
||||
calls,
|
||||
repository: new PostgresRunCancellationDispatchManagementRepository(pool),
|
||||
};
|
||||
}
|
||||
|
||||
test('inspects one low-sensitive blocked dispatch under run.read authority', async () => {
|
||||
const { calls, repository } = fixture();
|
||||
const result = await repository.inspect(command());
|
||||
assert.equal(result.operatorAction, 'rearm');
|
||||
assert.equal(result.dispatch.status, 'blocked');
|
||||
assert.equal(result.dispatch.lastResult, 'identity_mismatch');
|
||||
const dispatchRead = calls.find(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."run_cancellation_dispatches"'),
|
||||
);
|
||||
assert.equal(dispatchRead.sql.includes('lease_owner'), false);
|
||||
assert.equal(dispatchRead.sql.includes('lease_token_digest'), false);
|
||||
assert.equal(
|
||||
calls.some(
|
||||
({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
|
||||
sql.includes('$3'),
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('rearms an exact blocked dispatch with one event and allowed audit', async () => {
|
||||
const { calls, repository } = fixture();
|
||||
const result = await repository.rearm(rearmCommand());
|
||||
assert.deepEqual(result, {
|
||||
status: 'rearmed',
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
previousDispatchVersion: 3,
|
||||
dispatchVersion: 4,
|
||||
previousResult: 'identity_mismatch',
|
||||
retryDelayMs: 5_000,
|
||||
nextAttemptAtMs: NOW + 5_000,
|
||||
runVersion: 7,
|
||||
eventSequence: 9,
|
||||
});
|
||||
const update = calls.find(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_cancellation_dispatches"'),
|
||||
);
|
||||
assert.deepEqual(update.params, [
|
||||
'run-1',
|
||||
4,
|
||||
NOW + 5_000,
|
||||
NOW,
|
||||
'attempt-1',
|
||||
3,
|
||||
'identity_mismatch',
|
||||
]);
|
||||
const attemptRead = calls.find(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."run_attempts"'),
|
||||
);
|
||||
assert.equal(attemptRead.sql.includes('FOR KEY SHARE'), false);
|
||||
const event = calls.find(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"'),
|
||||
);
|
||||
assert.equal(event.params[0], rearmCommand().eventId);
|
||||
assert.equal(JSON.parse(event.params[6]).previous_result, 'identity_mismatch');
|
||||
assert.ok(
|
||||
calls.findIndex(({ sql }) =>
|
||||
sql.startsWith('UPDATE "ql3"."run_cancellation_dispatches"'),
|
||||
) < calls.findIndex(({ sql }) => sql === 'COMMIT'),
|
||||
);
|
||||
});
|
||||
|
||||
test('exact mutation replay returns the immutable receipt without another update', async () => {
|
||||
const replay = {
|
||||
eventId: rearmCommand().eventId,
|
||||
eventSequence: 9,
|
||||
eventType: 'run.cancel_dispatch_rearmed',
|
||||
actorType: 'user',
|
||||
actorId: 'operator-1',
|
||||
attemptId: 'attempt-1',
|
||||
payload: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-rearm@v1',
|
||||
mutation_id: rearmCommand().mutationId,
|
||||
previous_dispatch_version: 3,
|
||||
dispatch_version: 4,
|
||||
previous_result: 'identity_mismatch',
|
||||
retry_delay_ms: 5_000,
|
||||
next_attempt_at_ms: NOW + 5_000,
|
||||
run_version: 7,
|
||||
},
|
||||
};
|
||||
const { calls, repository } = fixture({ replay });
|
||||
assert.equal((await repository.rearm(rearmCommand())).dispatchVersion, 4);
|
||||
assert.equal(calls.some(({ sql }) => sql.startsWith('UPDATE')), false);
|
||||
});
|
||||
|
||||
test('stale result and authorization changes fail closed before mutation', async () => {
|
||||
const stale = fixture({ lastResult: 'pid_mismatch' });
|
||||
await assert.rejects(
|
||||
stale.repository.rearm(rearmCommand()),
|
||||
(error) =>
|
||||
error instanceof RunCancellationDispatchManagementConflictError &&
|
||||
error.reason === 'dispatch_result_changed',
|
||||
);
|
||||
assert.equal(
|
||||
stale.calls.some(({ sql }) => sql.startsWith('UPDATE')),
|
||||
false,
|
||||
);
|
||||
|
||||
const unauthorized = fixture({ authorized: false });
|
||||
await assert.rejects(
|
||||
unauthorized.repository.inspect(command()),
|
||||
(error) =>
|
||||
error instanceof RunCancellationDispatchManagementConflictError &&
|
||||
error.reason === 'authorization_changed',
|
||||
);
|
||||
assert.equal(
|
||||
unauthorized.calls.some(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."runs"'),
|
||||
),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects malformed management authority before opening PostgreSQL', async () => {
|
||||
let opened = false;
|
||||
const repository = new PostgresRunCancellationDispatchManagementRepository({
|
||||
async connect() {
|
||||
opened = true;
|
||||
throw new Error('must not open');
|
||||
},
|
||||
});
|
||||
assert.throws(
|
||||
() => repository.rearm(rearmCommand({ retryDelayMs: 0 })),
|
||||
InvalidRunCancellationDispatchManagementError,
|
||||
);
|
||||
assert.equal(opened, false);
|
||||
});
|
||||
Reference in New Issue
Block a user