mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-23 12:05:27 +08:00
feat(ql3): operationalize cancellation rearm
This commit is contained in:
whyour
@@ -61,7 +61,7 @@ function policyRow(role = 'operator') {
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(role = 'operator') {
|
||||
function fixture(role = 'operator', options = {}) {
|
||||
const calls = [];
|
||||
const pool = {
|
||||
async query(sql, params = []) {
|
||||
@@ -86,13 +86,30 @@ function fixture(role = 'operator') {
|
||||
text.startsWith('SELECT set_config')
|
||||
)
|
||||
return { rows: [], rowCount: 0 };
|
||||
if (text.includes('statement_timestamp()')) {
|
||||
if (
|
||||
text.includes('statement_timestamp()') ||
|
||||
text.includes('transaction_timestamp()')
|
||||
) {
|
||||
return { rows: [{ nowMs: NOW }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('lock_run_management_policy_fence')) {
|
||||
return { rows: [{ matches: true }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('FROM "ql3"."runs" WHERE id = $1 FOR UPDATE')) {
|
||||
if (!text.includes('cancel_reason AS "cancelReason"')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
projectId: 'project-1',
|
||||
runStatus: 'running',
|
||||
runVersion: 6,
|
||||
eventSequence: 8,
|
||||
cancelRequestedAtMs: NOW - 2_000,
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
@@ -123,12 +140,37 @@ function fixture(role = 'operator') {
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
|
||||
text.includes('RETURNING event_id')
|
||||
text.includes('FROM "ql3"."runs" WHERE id = $1') &&
|
||||
!text.includes('FOR UPDATE')
|
||||
) {
|
||||
return { rows: [{ eventId: request().auditEventId }], rowCount: 1 };
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
projectId: 'project-1',
|
||||
runStatus: 'running',
|
||||
runVersion: 6,
|
||||
eventSequence: 8,
|
||||
cancelRequestedAtMs: NOW - 2_000,
|
||||
cancelReason: 'user',
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."run_events"') &&
|
||||
text.includes('dedupe_key = $2')
|
||||
) {
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (
|
||||
text.startsWith('SELECT attempt_id AS "attemptId"') &&
|
||||
text.includes('FROM "ql3"."run_cancellation_dispatches"') &&
|
||||
!text.includes('dispatchStatus') &&
|
||||
!text.includes('FOR UPDATE')
|
||||
) {
|
||||
return { rows: [{ attemptId: 'attempt-1' }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('idempotency_key = $2')) return { rows: [] };
|
||||
if (text.includes('WHERE run.id = $1')) {
|
||||
return {
|
||||
rows: [
|
||||
@@ -150,6 +192,58 @@ function fixture(role = 'operator') {
|
||||
],
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."run_attempts"')) {
|
||||
return { rows: [{ attemptStatus: 'running' }], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."run_cancellation_dispatches"') &&
|
||||
text.includes('FOR UPDATE')
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
attemptId: 'attempt-1',
|
||||
dispatchStatus: 'blocked',
|
||||
dispatchVersion: 3,
|
||||
lastResult: options.lastResult ?? 'identity_mismatch',
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (text.includes('FROM "ql3"."run_cancellation_dispatches"')) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
attemptId: 'attempt-1',
|
||||
dispatchStatus: 'blocked',
|
||||
dispatchVersion: 3,
|
||||
dispatchCount: 1,
|
||||
nextAttemptAtMs: null,
|
||||
leaseExpiresAtMs: null,
|
||||
lastResult: options.lastResult ?? 'identity_mismatch',
|
||||
lastDispatchedAtMs: NOW - 1_500,
|
||||
dispatchCreatedAtMs: NOW - 1_900,
|
||||
dispatchUpdatedAtMs: NOW - 1_500,
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.startsWith(
|
||||
'UPDATE "ql3"."run_cancellation_dispatches"',
|
||||
)
|
||||
) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
if (
|
||||
text.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
|
||||
text.includes('RETURNING event_id')
|
||||
) {
|
||||
return { rows: [{ eventId: request().auditEventId }], rowCount: 1 };
|
||||
}
|
||||
if (text.includes('idempotency_key = $2')) return { rows: [] };
|
||||
if (text.includes('FROM "ql3"."task_definitions"')) {
|
||||
return { rows: [{ enabled: true }] };
|
||||
}
|
||||
@@ -252,3 +346,104 @@ test('authorizes run.stop and commits intent plus allowed audit together', async
|
||||
calls.findIndex(({ sql }) => sql === 'COMMIT'),
|
||||
);
|
||||
});
|
||||
|
||||
test('allows a viewer to inspect only low-sensitive cancellation state', async () => {
|
||||
const { calls, service } = fixture('viewer');
|
||||
const inspectRequest = {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
requestId: 'request-inspect-1',
|
||||
auditEventId: '019f9500-0000-4000-8000-000000000031',
|
||||
failureAuditEventId: '019f9500-0000-4000-8000-000000000032',
|
||||
principal: request().principal,
|
||||
};
|
||||
const result = await service.inspectCancellation(inspectRequest);
|
||||
assert.equal(result.operatorAction, 'rearm');
|
||||
assert.equal(result.dispatch.lastResult, 'identity_mismatch');
|
||||
assert.equal(JSON.stringify(result).includes('leaseOwner'), false);
|
||||
assert.equal(JSON.stringify(result).includes('leaseToken'), false);
|
||||
const audit = calls.find(
|
||||
({ sql, params }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
|
||||
params[2] === 'run.cancellation.inspect',
|
||||
);
|
||||
assert.equal(audit.params[0], inspectRequest.auditEventId);
|
||||
});
|
||||
|
||||
test('authorizes exact cancellation rearm and keeps the event identity server-side', async () => {
|
||||
const { calls, service } = fixture();
|
||||
const rearmRequest = {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
mutationId: '019f9500-0000-4000-8000-000000000041',
|
||||
expectedDispatchVersion: 3,
|
||||
expectedLastResult: 'identity_mismatch',
|
||||
retryDelayMs: 5_000,
|
||||
requestId: 'request-rearm-1',
|
||||
auditEventId: '019f9500-0000-4000-8000-000000000042',
|
||||
failureAuditEventId: '019f9500-0000-4000-8000-000000000043',
|
||||
principal: request().principal,
|
||||
};
|
||||
const result = await service.rearmCancellation(rearmRequest);
|
||||
assert.equal(result.status, 'rearmed');
|
||||
assert.equal(result.dispatchVersion, 4);
|
||||
const event = calls.find(({ sql }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."run_events"'),
|
||||
);
|
||||
assert.equal(event.params[0], GENERATED[0]);
|
||||
assert.equal(event.params.includes(rearmRequest.mutationId), false);
|
||||
const allowedAudit = calls.find(
|
||||
({ sql, params }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
|
||||
params[2] === 'run.cancellation.rearm',
|
||||
);
|
||||
assert.equal(allowedAudit.params[0], rearmRequest.auditEventId);
|
||||
assert.ok(
|
||||
calls.indexOf(allowedAudit) < calls.findIndex(({ sql }) => sql === 'COMMIT'),
|
||||
);
|
||||
});
|
||||
|
||||
test('denies viewer rearm and records stale dispatch conflicts outside the transaction', async () => {
|
||||
const rearmRequest = {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
mutationId: '019f9500-0000-4000-8000-000000000051',
|
||||
expectedDispatchVersion: 3,
|
||||
expectedLastResult: 'identity_mismatch',
|
||||
retryDelayMs: 5_000,
|
||||
requestId: 'request-rearm-conflict-1',
|
||||
auditEventId: '019f9500-0000-4000-8000-000000000052',
|
||||
failureAuditEventId: '019f9500-0000-4000-8000-000000000053',
|
||||
principal: request().principal,
|
||||
};
|
||||
|
||||
const viewer = fixture('viewer');
|
||||
await assert.rejects(
|
||||
viewer.service.rearmCancellation(rearmRequest),
|
||||
ClusterRunManagementAuthorizationError,
|
||||
);
|
||||
assert.equal(
|
||||
viewer.calls.some(({ scope }) => scope === 'client'),
|
||||
false,
|
||||
);
|
||||
|
||||
const stale = fixture('operator', { lastResult: 'pid_mismatch' });
|
||||
await assert.rejects(
|
||||
stale.service.rearmCancellation(rearmRequest),
|
||||
{ code: 'CLUSTER_RUN_MANAGEMENT_CONFLICT' },
|
||||
);
|
||||
const failureAudit = stale.calls.find(
|
||||
({ scope, sql }) =>
|
||||
scope === 'pool' &&
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
|
||||
);
|
||||
assert.equal(failureAudit.params[0], rearmRequest.failureAuditEventId);
|
||||
assert.equal(
|
||||
failureAudit.params.some(
|
||||
(value) =>
|
||||
typeof value === 'string' &&
|
||||
value.includes('dispatch_result_changed'),
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -117,6 +117,40 @@ const stopCommand = normalizeClusterRunManagementCommand({
|
||||
},
|
||||
});
|
||||
|
||||
const inspectCommand = normalizeClusterRunManagementCommand({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.inspect',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
requestId: 'request-inspect-1',
|
||||
auditEventId: '019f9400-0000-4000-8000-000000000031',
|
||||
failureAuditEventId: '019f9400-0000-4000-8000-000000000032',
|
||||
body: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-inspect@v1',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const rearmCommand = normalizeClusterRunManagementCommand({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.rearm',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
requestId: 'request-rearm-1',
|
||||
auditEventId: '019f9400-0000-4000-8000-000000000041',
|
||||
failureAuditEventId: '019f9400-0000-4000-8000-000000000042',
|
||||
body: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-rearm-request@v1',
|
||||
mutationId: '019f9400-0000-4000-8000-000000000043',
|
||||
expectedDispatchVersion: 3,
|
||||
expectedLastResult: 'identity_mismatch',
|
||||
retryDelayMs: 5_000,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
function response(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
@@ -214,3 +248,105 @@ test('validates one low-sensitive stop response against the request target', ()
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('validates a low-sensitive cancellation diagnostic and rejects capability leakage', () => {
|
||||
const value = {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.inspect',
|
||||
diagnostic: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-diagnostic@v1',
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
runStatus: 'running',
|
||||
runVersion: 6,
|
||||
eventSequence: 8,
|
||||
cancelRequestedAtMs: 999_000,
|
||||
cancelReason: 'user',
|
||||
operatorAction: 'rearm',
|
||||
dispatch: {
|
||||
attemptId: 'attempt-1',
|
||||
status: 'blocked',
|
||||
version: 3,
|
||||
dispatchCount: 1,
|
||||
lastResult: 'identity_mismatch',
|
||||
createdAtMs: 999_100,
|
||||
updatedAtMs: 999_200,
|
||||
},
|
||||
},
|
||||
};
|
||||
assert.deepEqual(
|
||||
validateClusterRunManagementClientResult(value, inspectCommand),
|
||||
value,
|
||||
);
|
||||
for (const drift of [
|
||||
{ ...value, diagnostic: { ...value.diagnostic, projectId: 'project-2' } },
|
||||
{ ...value, diagnostic: { ...value.diagnostic, runId: 'run-2' } },
|
||||
{
|
||||
...value,
|
||||
diagnostic: { ...value.diagnostic, operatorAction: 'none' },
|
||||
},
|
||||
{
|
||||
...value,
|
||||
diagnostic: {
|
||||
...value.diagnostic,
|
||||
dispatch: { ...value.diagnostic.dispatch, leaseOwner: 'worker-1' },
|
||||
},
|
||||
},
|
||||
{
|
||||
...value,
|
||||
diagnostic: {
|
||||
...value.diagnostic,
|
||||
dispatch: {
|
||||
...value.diagnostic.dispatch,
|
||||
leaseTokenDigest: 'a'.repeat(64),
|
||||
},
|
||||
},
|
||||
},
|
||||
]) {
|
||||
assert.throws(
|
||||
() => validateClusterRunManagementClientResult(drift, inspectCommand),
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('binds a rearm receipt to the exact dispatch version, result and delay fences', () => {
|
||||
const value = {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.rearm',
|
||||
rearm: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-rearm-receipt@v1',
|
||||
status: 'rearmed',
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
previousDispatchVersion: 3,
|
||||
dispatchVersion: 4,
|
||||
previousResult: 'identity_mismatch',
|
||||
retryDelayMs: 5_000,
|
||||
nextAttemptAtMs: 1_005_000,
|
||||
runVersion: 7,
|
||||
eventSequence: 9,
|
||||
},
|
||||
};
|
||||
assert.deepEqual(
|
||||
validateClusterRunManagementClientResult(value, rearmCommand),
|
||||
value,
|
||||
);
|
||||
for (const rearm of [
|
||||
{ ...value.rearm, previousDispatchVersion: 4 },
|
||||
{ ...value.rearm, dispatchVersion: 5 },
|
||||
{ ...value.rearm, previousResult: 'pid_mismatch' },
|
||||
{ ...value.rearm, retryDelayMs: 6_000 },
|
||||
{ ...value.rearm, runId: 'run-2' },
|
||||
]) {
|
||||
assert.throws(
|
||||
() =>
|
||||
validateClusterRunManagementClientResult(
|
||||
{ ...value, rearm },
|
||||
rearmCommand,
|
||||
),
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -97,6 +97,84 @@ function stopResult() {
|
||||
};
|
||||
}
|
||||
|
||||
function diagnosticResult() {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
runStatus: 'running',
|
||||
runVersion: 6,
|
||||
eventSequence: 8,
|
||||
cancelRequestedAtMs: NOW - 1_000,
|
||||
cancelReason: 'user',
|
||||
operatorAction: 'rearm',
|
||||
dispatch: {
|
||||
attemptId: 'attempt-1',
|
||||
status: 'blocked',
|
||||
version: 3,
|
||||
dispatchCount: 1,
|
||||
lastResult: 'identity_mismatch',
|
||||
createdAtMs: NOW - 900,
|
||||
updatedAtMs: NOW - 800,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function rearmResult() {
|
||||
return {
|
||||
status: 'rearmed',
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
previousDispatchVersion: 3,
|
||||
dispatchVersion: 4,
|
||||
previousResult: 'identity_mismatch',
|
||||
retryDelayMs: 5_000,
|
||||
nextAttemptAtMs: NOW + 5_000,
|
||||
runVersion: 7,
|
||||
eventSequence: 9,
|
||||
};
|
||||
}
|
||||
|
||||
function inspectCommand(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.inspect',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
requestId: 'request-inspect-1',
|
||||
auditEventId: '019f9300-0000-4000-8000-000000000031',
|
||||
failureAuditEventId: '019f9300-0000-4000-8000-000000000032',
|
||||
body: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-inspect@v1',
|
||||
},
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function rearmCommand(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.rearm',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
requestId: 'request-rearm-1',
|
||||
auditEventId: '019f9300-0000-4000-8000-000000000041',
|
||||
failureAuditEventId: '019f9300-0000-4000-8000-000000000042',
|
||||
body: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-rearm-request@v1',
|
||||
mutationId: '019f9300-0000-4000-8000-000000000043',
|
||||
expectedDispatchVersion: 3,
|
||||
expectedLastResult: 'identity_mismatch',
|
||||
retryDelayMs: 5_000,
|
||||
},
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('routes one exact strong User retry and emits the shared response', async () => {
|
||||
const calls = [];
|
||||
const transport = createClusterRunManagementTransport({
|
||||
@@ -109,6 +187,12 @@ test('routes one exact strong User retry and emits the shared response', async (
|
||||
async stop() {
|
||||
return stopResult();
|
||||
},
|
||||
async inspectCancellation() {
|
||||
return diagnosticResult();
|
||||
},
|
||||
async rearmCancellation() {
|
||||
return rearmResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await transport.execute(command(), {
|
||||
@@ -139,6 +223,12 @@ test('routes one exact strong User stop and emits the shared response', async ()
|
||||
calls.push(request);
|
||||
return stopResult();
|
||||
},
|
||||
async inspectCancellation() {
|
||||
return diagnosticResult();
|
||||
},
|
||||
async rearmCancellation() {
|
||||
return rearmResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await transport.execute(stopCommand(), {
|
||||
@@ -169,6 +259,12 @@ test('rejects weak or non-User identity before service authority', async () => {
|
||||
async stop() {
|
||||
return stopResult();
|
||||
},
|
||||
async inspectCancellation() {
|
||||
return diagnosticResult();
|
||||
},
|
||||
async rearmCancellation() {
|
||||
return rearmResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
@@ -187,6 +283,65 @@ test('rejects weak or non-User identity before service authority', async () => {
|
||||
assert.equal(called, false);
|
||||
});
|
||||
|
||||
test('routes bounded cancellation inspection without lease capability data', async () => {
|
||||
const calls = [];
|
||||
const transport = createClusterRunManagementTransport({
|
||||
now: () => NOW,
|
||||
service: {
|
||||
async retry() { return retryResult(); },
|
||||
async stop() { return stopResult(); },
|
||||
async inspectCancellation(request) {
|
||||
calls.push(request);
|
||||
return diagnosticResult();
|
||||
},
|
||||
async rearmCancellation() { return rearmResult(); },
|
||||
},
|
||||
});
|
||||
const result = await transport.execute(inspectCommand(), {
|
||||
authenticate: async () => principal(),
|
||||
});
|
||||
assert.equal(calls[0].runId, 'run-1');
|
||||
assert.deepEqual(result, {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.inspect',
|
||||
diagnostic: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-diagnostic@v1',
|
||||
...diagnosticResult(),
|
||||
},
|
||||
});
|
||||
assert.equal(JSON.stringify(result).includes('leaseOwner'), false);
|
||||
assert.equal(JSON.stringify(result).includes('leaseToken'), false);
|
||||
});
|
||||
|
||||
test('routes an exact blocked cancellation rearm receipt', async () => {
|
||||
const calls = [];
|
||||
const transport = createClusterRunManagementTransport({
|
||||
now: () => NOW,
|
||||
service: {
|
||||
async retry() { return retryResult(); },
|
||||
async stop() { return stopResult(); },
|
||||
async inspectCancellation() { return diagnosticResult(); },
|
||||
async rearmCancellation(request) {
|
||||
calls.push(request);
|
||||
return rearmResult();
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await transport.execute(rearmCommand(), {
|
||||
authenticate: async () => principal({ assurance: 'hardware' }),
|
||||
});
|
||||
assert.equal(calls[0].expectedDispatchVersion, 3);
|
||||
assert.equal(calls[0].expectedLastResult, 'identity_mismatch');
|
||||
assert.deepEqual(result, {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.rearm',
|
||||
rearm: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-rearm-receipt@v1',
|
||||
...rearmResult(),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test('rejects widened commands and ambiguous audit identity', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
|
||||
Reference in New Issue
Block a user