mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add cluster manual run retry authority
This commit is contained in:
whyour
@@ -0,0 +1,334 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
RunManualRetryFenceRejectedError,
|
||||
RunManualRetryRateLimitedError,
|
||||
} = require('@qinglong/runtime-core/run-manual-retry');
|
||||
const {
|
||||
CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT,
|
||||
PostgresRunManualRetryRepository,
|
||||
} = require('@qinglong/cluster-postgres/run-manual-retry');
|
||||
|
||||
const SOURCE_DIGEST = 'a'.repeat(64);
|
||||
const EXECUTION_DIGEST = 'b'.repeat(64);
|
||||
const TASK_REVISION = `qltd:v1:7:${SOURCE_DIGEST}`;
|
||||
const IDS = Object.freeze({
|
||||
mutationId: '019f9200-0000-4000-8000-000000000001',
|
||||
runId: '019f9200-0000-4000-8000-000000000002',
|
||||
attemptId: '019f9200-0000-4000-8000-000000000003',
|
||||
createdEventId: '019f9200-0000-4000-8000-000000000004',
|
||||
queuedEventId: '019f9200-0000-4000-8000-000000000005',
|
||||
auditEventId: '019f9200-0000-4000-8000-000000000006',
|
||||
});
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
mutationId: IDS.mutationId,
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
runId: IDS.runId,
|
||||
attemptId: IDS.attemptId,
|
||||
createdEventId: IDS.createdEventId,
|
||||
queuedEventId: IDS.queuedEventId,
|
||||
auditEventId: IDS.auditEventId,
|
||||
requestId: 'cluster-run-retry-1',
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:mfa-session-1',
|
||||
authenticatedAtMs: 900_000,
|
||||
expiresAtMs: 1_100_000,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function sourceRow(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: TASK_REVISION,
|
||||
taskName: 'Task 1',
|
||||
taskSnapshotRef: TASK_REVISION,
|
||||
parentRunId: null,
|
||||
triggerType: 'task_start',
|
||||
executionOwner: 'runtime',
|
||||
inputRef: 'artifact:input-1',
|
||||
priority: 3,
|
||||
runStatus: 'failed',
|
||||
runVersion: 7,
|
||||
attemptExecutorType: 'remote_worker',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function replayRow(overrides = {}) {
|
||||
return {
|
||||
runId: IDS.runId,
|
||||
projectId: 'project-1',
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: TASK_REVISION,
|
||||
triggerType: 'run_manual_retry',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
triggeredBy: 'operator-1',
|
||||
requestId: IDS.mutationId,
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
createdAtMs: 1_000_000,
|
||||
attemptId: IDS.attemptId,
|
||||
executorType: 'remote_worker',
|
||||
createdActorType: 'user',
|
||||
createdActorId: 'operator-1',
|
||||
createdPayload: {
|
||||
mutation_id: IDS.mutationId,
|
||||
retry_of_run_id: 'source-run-1',
|
||||
source_run_status: 'failed',
|
||||
source_run_version: 7,
|
||||
inherit_retry_policy: false,
|
||||
authentication_id: 'oidc:mfa-session-1',
|
||||
audit_event_id: IDS.auditEventId,
|
||||
execution_revision_digest: EXECUTION_DIGEST,
|
||||
policy_fence: { project_version: 2, binding_version: 3 },
|
||||
},
|
||||
queuedActorType: 'user',
|
||||
queuedActorId: 'operator-1',
|
||||
queuedPayload: {
|
||||
from_status: 'created',
|
||||
to_status: 'queued',
|
||||
version: 2,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const calls = [];
|
||||
let connections = 0;
|
||||
const pool = {
|
||||
async connect() {
|
||||
connections += 1;
|
||||
const connection = connections;
|
||||
return {
|
||||
async query(sql, params = []) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ connection, sql: normalized, params });
|
||||
if (normalized.startsWith('BEGIN')) {
|
||||
if (options.failFirstBegin && connection === 1) {
|
||||
const error = new Error('serialization retry');
|
||||
error.code = '40001';
|
||||
throw error;
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (
|
||||
normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' ||
|
||||
normalized.startsWith('SELECT set_config')
|
||||
)
|
||||
return { rows: [], rowCount: 0 };
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: 1_000_000 }], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."projects"')) {
|
||||
const rows = options.projectRows ?? [
|
||||
{ projectStatus: 'active', projectVersion: 2 },
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('project_role_bindings')) {
|
||||
const rows = options.bindingRows ?? [
|
||||
{
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: 'operator',
|
||||
},
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('idempotency_key = $2')) {
|
||||
const rows = options.replayRows ?? [];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('WHERE run.id = $1')) {
|
||||
const rows = options.sourceRows ?? [sourceRow()];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."task_definitions"')) {
|
||||
const rows = options.taskRows ?? [{ enabled: true }];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('task_execution_revisions')) {
|
||||
const rows = options.executionRows ?? [
|
||||
{
|
||||
sourceContentDigest: SOURCE_DIGEST,
|
||||
contentDigest: EXECUTION_DIGEST,
|
||||
},
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (
|
||||
normalized.includes("trigger_type = 'run_manual_retry'") &&
|
||||
normalized.startsWith('SELECT')
|
||||
) {
|
||||
const rows = options.rateRows ?? [];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.startsWith('INSERT INTO')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() {
|
||||
calls.push({ connection, sql: 'RELEASE', params: [] });
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
repository: new PostgresRunManualRetryRepository(pool),
|
||||
};
|
||||
}
|
||||
|
||||
test('atomically appends a linked queued Run, remote Attempt, events and allowed audit', async () => {
|
||||
const { calls, repository } = fixture();
|
||||
assert.deepEqual(await repository.retryRun(command()), {
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: IDS.runId,
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: TASK_REVISION,
|
||||
attemptId: IDS.attemptId,
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: EXECUTION_DIGEST,
|
||||
createdAtMs: 1_000_000,
|
||||
});
|
||||
const inserts = calls.filter(({ sql }) => sql.startsWith('INSERT INTO'));
|
||||
assert.equal(inserts.length, 5);
|
||||
assert.match(inserts.at(-1).sql, /security_audit_events/);
|
||||
assert.deepEqual(JSON.parse(inserts[2].params[6]), {
|
||||
status: 'created',
|
||||
version: 1,
|
||||
execution_owner: 'runtime',
|
||||
executor_type: 'remote_worker',
|
||||
execution_revision_digest: EXECUTION_DIGEST,
|
||||
source_content_digest: SOURCE_DIGEST,
|
||||
retry_of_run_id: 'source-run-1',
|
||||
source_run_status: 'failed',
|
||||
source_run_version: 7,
|
||||
inherit_retry_policy: false,
|
||||
mutation_id: IDS.mutationId,
|
||||
authentication_id: 'oidc:mfa-session-1',
|
||||
audit_event_id: IDS.auditEventId,
|
||||
policy_fence: { project_version: 2, binding_version: 3 },
|
||||
});
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('returns durable identities for an exact replay without appending again', async () => {
|
||||
const { calls, repository } = fixture({ replayRows: [replayRow()] });
|
||||
const result = await repository.retryRun(
|
||||
command({
|
||||
runId: '019f9200-0000-4000-8000-000000000102',
|
||||
attemptId: '019f9200-0000-4000-8000-000000000103',
|
||||
createdEventId: '019f9200-0000-4000-8000-000000000104',
|
||||
queuedEventId: '019f9200-0000-4000-8000-000000000105',
|
||||
}),
|
||||
);
|
||||
assert.equal(result.status, 'existing');
|
||||
assert.equal(result.runId, IDS.runId);
|
||||
assert.equal(result.attemptId, IDS.attemptId);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.startsWith('INSERT INTO')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects stale authentication and changed authorization inside the transaction', async () => {
|
||||
const stale = fixture();
|
||||
await assert.rejects(
|
||||
stale.repository.retryRun(
|
||||
command({
|
||||
principal: {
|
||||
...command().principal,
|
||||
authenticatedAtMs: 600_000,
|
||||
},
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof RunManualRetryFenceRejectedError &&
|
||||
error.reason === 'authentication_changed',
|
||||
);
|
||||
assert.equal(
|
||||
stale.calls.some(({ sql }) => sql === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
stale.calls.some(({ sql }) => sql.includes('FROM "ql3"."projects"')),
|
||||
false,
|
||||
);
|
||||
|
||||
const changed = fixture({
|
||||
bindingRows: [
|
||||
{ bindingVersion: 4, bindingState: 'active', bindingRole: 'operator' },
|
||||
],
|
||||
});
|
||||
await assert.rejects(
|
||||
changed.repository.retryRun(command()),
|
||||
(error) =>
|
||||
error instanceof RunManualRetryFenceRejectedError &&
|
||||
error.reason === 'authorization_changed',
|
||||
);
|
||||
});
|
||||
|
||||
test('uses the durable Run ledger for the per-project User quota', async () => {
|
||||
const rateRows = Array.from(
|
||||
{ length: CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT },
|
||||
(_, index) => ({ createdAtMs: 999_000 - index }),
|
||||
);
|
||||
const { calls, repository } = fixture({ rateRows });
|
||||
await assert.rejects(
|
||||
repository.retryRun(command()),
|
||||
(error) =>
|
||||
error instanceof RunManualRetryRateLimitedError &&
|
||||
error.retryAfterMs === 58_937,
|
||||
);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.startsWith('INSERT INTO')),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('retries one serializable conflict with a fresh connection', async () => {
|
||||
const { calls, repository } = fixture({ failFirstBegin: true });
|
||||
const result = await repository.retryRun(command());
|
||||
assert.equal(result.status, 'accepted');
|
||||
assert.deepEqual(
|
||||
[...new Set(calls.map(({ connection }) => connection))],
|
||||
[1, 2],
|
||||
);
|
||||
assert.equal(calls.filter(({ sql }) => sql === 'RELEASE').length, 2);
|
||||
});
|
||||
Reference in New Issue
Block a user