open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-20 16:07:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add strong cluster run stop

Browse Source
This commit is contained in:
whyour
2026-08-12 08:21:28 +08:00
parent c0ab62e64a
commit dd370b2842
22 changed files with 1602 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-cluster-admin/test/runManagement.test.cjs
+112 -22
View File
@@ -67,7 +67,8 @@ function fixture(role = 'operator') {
async query(sql, params = []) {
const text = sql.replace(/\s+/g, ' ').trim();
calls.push({ scope: 'pool', sql: text, params });
if (text.includes('LEFT JOIN LATERAL')) return { rows: [policyRow(role)] };
if (text.includes('LEFT JOIN LATERAL'))
return { rows: [policyRow(role)] };
if (text.startsWith('INSERT INTO "ql3"."security_audit_events"')) {
return { rows: [], rowCount: 1 };
}
@@ -83,40 +84,89 @@ function fixture(role = 'operator') {
text === 'COMMIT' ||
text === 'ROLLBACK' ||
text.startsWith('SELECT set_config')
) return { rows: [], rowCount: 0 };
)
return { rows: [], rowCount: 0 };
if (text.includes('statement_timestamp()')) {
return { rows: [{ nowMs: NOW }], rowCount: 1 };
}
if (text.includes('lock_run_management_policy_fence')) {
return { rows: [{ matches: true }], rowCount: 1 };
}
if (text.includes('FROM "ql3"."runs" WHERE id = $1 FOR UPDATE')) {
return {
rows: [
{
projectId: 'project-1',
runStatus: 'running',
runVersion: 4,
eventSequence: 6,
cancelRequestedAtMs: null,
cancelReason: null,
},
],
rowCount: 1,
};
}
if (text.startsWith('UPDATE "ql3"."runs"')) {
return {
rows: [
{
projectId: 'project-1',
runStatus: 'running',
runVersion: 5,
eventSequence: 7,
cancelRequestedAtMs: NOW,
cancelReason: 'user',
},
],
rowCount: 1,
};
}
if (
text.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
text.includes('RETURNING event_id')
) {
return { rows: [{ eventId: request().auditEventId }], rowCount: 1 };
}
if (text.includes('idempotency_key = $2')) return { rows: [] };
if (text.includes('WHERE run.id = $1')) {
return {
rows: [{
projectId: 'project-1',
taskId: 'task-1',
taskRevision: TASK_REVISION,
taskName: 'Task 1',
taskSnapshotRef: TASK_REVISION,
parentRunId: null,
triggerType: 'task_start',
executionOwner: 'runtime',
inputRef: null,
priority: 1,
runStatus: 'failed',
runVersion: 7,
attemptExecutorType: 'remote_worker',
}],
rows: [
{
projectId: 'project-1',
taskId: 'task-1',
taskRevision: TASK_REVISION,
taskName: 'Task 1',
taskSnapshotRef: TASK_REVISION,
parentRunId: null,
triggerType: 'task_start',
executionOwner: 'runtime',
inputRef: null,
priority: 1,
runStatus: 'failed',
runVersion: 7,
attemptExecutorType: 'remote_worker',
},
],
};
}
if (text.includes('FROM "ql3"."task_definitions"')) {
return { rows: [{ enabled: true }] };
}
if (text.includes('task_execution_revisions')) {
return { rows: [{ sourceContentDigest: SOURCE_DIGEST, contentDigest: EXECUTION_DIGEST }] };
return {
rows: [
{
sourceContentDigest: SOURCE_DIGEST,
contentDigest: EXECUTION_DIGEST,
},
],
};
}
if (text.startsWith('SELECT') && text.includes("trigger_type = 'run_manual_retry'")) {
if (
text.startsWith('SELECT') &&
text.includes("trigger_type = 'run_manual_retry'")
) {
return { rows: [] };
}
if (text.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 };
@@ -143,7 +193,9 @@ test('authorizes run.retry and keeps all generated aggregate identities server-s
assert.equal(result.status, 'accepted');
assert.equal(result.runId, GENERATED[0]);
assert.equal(result.attemptId, GENERATED[1]);
const runInsert = calls.find(({ sql }) => sql.startsWith('INSERT INTO "ql3"."runs"'));
const runInsert = calls.find(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."runs"'),
);
assert.equal(runInsert.params[0], GENERATED[0]);
assert.equal(runInsert.params.includes(GENERATED[2]), false);
assert.equal(
@@ -154,11 +206,49 @@ test('authorizes run.retry and keeps all generated aggregate identities server-s
test('denied policy writes only the caller-supplied failure audit', async () => {
const { calls, service } = fixture('viewer');
await assert.rejects(service.retry(request()), ClusterRunManagementAuthorizationError);
await assert.rejects(
service.retry(request()),
ClusterRunManagementAuthorizationError,
);
const audits = calls.filter(({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
);
assert.equal(audits.length, 1);
assert.equal(audits[0].params[0], request().failureAuditEventId);
assert.equal(calls.some(({ scope }) => scope === 'client'), false);
assert.equal(
calls.some(({ scope }) => scope === 'client'),
false,
);
});
test('authorizes run.stop and commits intent plus allowed audit together', async () => {
const { calls, service } = fixture();
const stopRequest = {
projectId: 'project-1',
runId: 'run-1',
mutationId: '019f9500-0000-4000-8000-000000000021',
requestId: 'request-stop-1',
auditEventId: '019f9500-0000-4000-8000-000000000022',
failureAuditEventId: '019f9500-0000-4000-8000-000000000023',
principal: request().principal,
};
const result = await service.stop(stopRequest);
assert.equal(result.status, 'accepted');
assert.equal(result.cancelRequestedAtMs, NOW);
const event = calls.find(
({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."run_events"') &&
sql.includes('run.cancel_requested'),
);
assert.equal(event.params[0], GENERATED[0]);
const allowedAudit = calls.find(
({ sql }) =>
sql.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
sql.includes("'run.stop'"),
);
assert.equal(allowedAudit.params[0], stopRequest.auditEventId);
assert.ok(
calls.findIndex(({ sql }) => sql.includes("'run.stop'")) <
calls.findIndex(({ sql }) => sql === 'COMMIT'),
);
});
packages/ql3-cluster-admin/test/runManagementClient.test.cjs
+52 -1
View File
@@ -31,6 +31,22 @@ const command = normalizeClusterRunManagementCommand({
},
});
const stopCommand = normalizeClusterRunManagementCommand({
schemaVersion: 1,
operation: 'run.stop',
request: {
projectId: 'project-1',
runId: 'run-1',
requestId: 'request-stop-1',
auditEventId: '019f9400-0000-4000-8000-000000000021',
failureAuditEventId: '019f9400-0000-4000-8000-000000000022',
body: {
schema: 'qinglong/run-cancellation@v1',
mutationId: '019f9400-0000-4000-8000-000000000023',
},
},
});
function response(overrides = {}) {
return {
schemaVersion: 1,
@@ -59,7 +75,10 @@ function response(overrides = {}) {
}
test('validates one low-sensitive retry response against the request fence', () => {
assert.deepEqual(validateClusterRunManagementClientResult(response(), command), response());
assert.deepEqual(
validateClusterRunManagementClientResult(response(), command),
response(),
);
});
test('rejects response target, execution placement and shape drift', () => {
@@ -75,3 +94,35 @@ test('rejects response target, execution placement and shape drift', () => {
);
}
});
test('validates one low-sensitive stop response against the request target', () => {
const value = {
schemaVersion: 1,
operation: 'run.stop',
stop: {
schema: 'qinglong/run-cancellation@v1',
status: 'accepted',
projectId: 'project-1',
runId: 'run-1',
runStatus: 'running',
runVersion: 5,
eventSequence: 7,
cancelRequestedAtMs: 1_000_000,
cancelReason: 'user',
},
};
assert.deepEqual(
validateClusterRunManagementClientResult(value, stopCommand),
value,
);
for (const drift of [
{ ...value, stop: { ...value.stop, projectId: 'project-2' } },
{ ...value, stop: { ...value.stop, runId: 'run-2' } },
{ ...value, operation: 'run.retry' },
]) {
assert.throws(
() => validateClusterRunManagementClientResult(drift, stopCommand),
ClusterPluginPackageManagementClientRequestError,
);
}
});
packages/ql3-cluster-admin/test/runManagementTransport.test.cjs
+85 -2
View File
@@ -65,6 +65,38 @@ function retryResult() {
};
}
function stopCommand(overrides = {}) {
return {
schemaVersion: 1,
operation: 'run.stop',
request: {
projectId: 'project-1',
runId: 'run-1',
requestId: 'request-stop-1',
auditEventId: '019f9300-0000-4000-8000-000000000021',
failureAuditEventId: '019f9300-0000-4000-8000-000000000022',
body: {
schema: 'qinglong/run-cancellation@v1',
mutationId: '019f9300-0000-4000-8000-000000000023',
},
...overrides,
},
};
}
function stopResult() {
return {
status: 'accepted',
projectId: 'project-1',
runId: 'run-1',
runStatus: 'running',
runVersion: 5,
eventSequence: 7,
cancelRequestedAtMs: NOW,
cancelReason: 'user',
};
}
test('routes one exact strong User retry and emits the shared response', async () => {
const calls = [];
const transport = createClusterRunManagementTransport({
@@ -74,6 +106,9 @@ test('routes one exact strong User retry and emits the shared response', async (
calls.push(request);
return retryResult();
},
async stop() {
return stopResult();
},
},
});
const result = await transport.execute(command(), {
@@ -92,6 +127,36 @@ test('routes one exact strong User retry and emits the shared response', async (
});
});
test('routes one exact strong User stop and emits the shared response', async () => {
const calls = [];
const transport = createClusterRunManagementTransport({
now: () => NOW,
service: {
async retry() {
return retryResult();
},
async stop(request) {
calls.push(request);
return stopResult();
},
},
});
const result = await transport.execute(stopCommand(), {
authenticate: async () => principal({ assurance: 'hardware' }),
});
assert.equal(calls.length, 1);
assert.equal(calls[0].runId, 'run-1');
assert.equal(calls[0].mutationId, stopCommand().request.body.mutationId);
assert.deepEqual(result, {
schemaVersion: 1,
operation: 'run.stop',
stop: {
schema: 'qinglong/run-cancellation@v1',
...stopResult(),
},
});
});
test('rejects weak or non-User identity before service authority', async () => {
let called = false;
const transport = createClusterRunManagementTransport({
@@ -101,6 +166,9 @@ test('rejects weak or non-User identity before service authority', async () => {
called = true;
return retryResult();
},
async stop() {
return stopResult();
},
},
});
await assert.rejects(
@@ -121,7 +189,11 @@ test('rejects weak or non-User identity before service authority', async () => {
test('rejects widened commands and ambiguous audit identity', () => {
assert.throws(
() => normalizeClusterRunManagementCommand({ ...command(), principal: principal() }),
() =>
normalizeClusterRunManagementCommand({
...command(),
principal: principal(),
}),
ClusterRunManagementTransportRequestError,
);
assert.throws(
@@ -134,7 +206,18 @@ test('rejects widened commands and ambiguous audit identity', () => {
assert.throws(
() =>
normalizeClusterRunManagementCommand(
command({ body: { ...command().request.body, expectedRunStatus: 'lost' } }),
command({
body: { ...command().request.body, expectedRunStatus: 'lost' },
}),
),
ClusterRunManagementTransportRequestError,
);
assert.throws(
() =>
normalizeClusterRunManagementCommand(
stopCommand({
body: { ...stopCommand().request.body, mutationId: 'weak' },
}),
),
ClusterRunManagementTransportRequestError,
);