mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-21 00:17:47 +08:00
326 lines
10 KiB
JavaScript
326 lines
10 KiB
JavaScript
'use strict';
|
|
|
|
const assert = require('node:assert/strict');
|
|
const { test } = require('node:test');
|
|
|
|
const {
|
|
RunManualRetryFenceRejectedError,
|
|
RunManualRetryRateLimitedError,
|
|
} = require('@qinglong/runtime-core/run-manual-retry');
|
|
const {
|
|
CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT,
|
|
PostgresRunManualRetryRepository,
|
|
} = require('@qinglong/cluster-postgres/run-manual-retry');
|
|
|
|
const SOURCE_DIGEST = 'a'.repeat(64);
|
|
const EXECUTION_DIGEST = 'b'.repeat(64);
|
|
const TASK_REVISION = `qltd:v1:7:${SOURCE_DIGEST}`;
|
|
const IDS = Object.freeze({
|
|
mutationId: '019f9200-0000-4000-8000-000000000001',
|
|
runId: '019f9200-0000-4000-8000-000000000002',
|
|
attemptId: '019f9200-0000-4000-8000-000000000003',
|
|
createdEventId: '019f9200-0000-4000-8000-000000000004',
|
|
queuedEventId: '019f9200-0000-4000-8000-000000000005',
|
|
auditEventId: '019f9200-0000-4000-8000-000000000006',
|
|
});
|
|
|
|
function command(overrides = {}) {
|
|
return {
|
|
projectId: 'project-1',
|
|
sourceRunId: 'source-run-1',
|
|
mutationId: IDS.mutationId,
|
|
expectedRunVersion: 7,
|
|
expectedRunStatus: 'failed',
|
|
runId: IDS.runId,
|
|
attemptId: IDS.attemptId,
|
|
createdEventId: IDS.createdEventId,
|
|
queuedEventId: IDS.queuedEventId,
|
|
auditEventId: IDS.auditEventId,
|
|
requestId: 'cluster-run-retry-1',
|
|
principal: {
|
|
subject: { type: 'user', id: 'operator-1' },
|
|
authenticationId: 'oidc:mfa-session-1',
|
|
authenticatedAtMs: 900_000,
|
|
expiresAtMs: 1_100_000,
|
|
assurance: 'multi_factor',
|
|
},
|
|
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function sourceRow(overrides = {}) {
|
|
return {
|
|
projectId: 'project-1',
|
|
taskId: 'task-1',
|
|
taskRevision: TASK_REVISION,
|
|
taskName: 'Task 1',
|
|
taskSnapshotRef: TASK_REVISION,
|
|
parentRunId: null,
|
|
triggerType: 'task_start',
|
|
executionOwner: 'runtime',
|
|
inputRef: 'artifact:input-1',
|
|
priority: 3,
|
|
runStatus: 'failed',
|
|
runVersion: 7,
|
|
attemptExecutorType: 'remote_worker',
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function replayRow(overrides = {}) {
|
|
return {
|
|
runId: IDS.runId,
|
|
projectId: 'project-1',
|
|
retryOfRunId: 'source-run-1',
|
|
taskId: 'task-1',
|
|
taskRevision: TASK_REVISION,
|
|
triggerType: 'run_manual_retry',
|
|
executionOrigin: 'manual',
|
|
executionOwner: 'runtime',
|
|
triggeredBy: 'operator-1',
|
|
requestId: IDS.mutationId,
|
|
runStatus: 'queued',
|
|
runVersion: 2,
|
|
eventSequence: 2,
|
|
createdAtMs: 1_000_000,
|
|
attemptId: IDS.attemptId,
|
|
executorType: 'remote_worker',
|
|
createdActorType: 'user',
|
|
createdActorId: 'operator-1',
|
|
createdPayload: {
|
|
mutation_id: IDS.mutationId,
|
|
retry_of_run_id: 'source-run-1',
|
|
source_run_status: 'failed',
|
|
source_run_version: 7,
|
|
inherit_retry_policy: false,
|
|
authentication_id: 'oidc:mfa-session-1',
|
|
audit_event_id: IDS.auditEventId,
|
|
execution_revision_digest: EXECUTION_DIGEST,
|
|
policy_fence: { project_version: 2, binding_version: 3 },
|
|
},
|
|
queuedActorType: 'user',
|
|
queuedActorId: 'operator-1',
|
|
queuedPayload: {
|
|
from_status: 'created',
|
|
to_status: 'queued',
|
|
version: 2,
|
|
},
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function fixture(options = {}) {
|
|
const calls = [];
|
|
let connections = 0;
|
|
const pool = {
|
|
async connect() {
|
|
connections += 1;
|
|
const connection = connections;
|
|
return {
|
|
async query(sql, params = []) {
|
|
const normalized = sql.replace(/\s+/g, ' ').trim();
|
|
calls.push({ connection, sql: normalized, params });
|
|
if (normalized.startsWith('BEGIN')) {
|
|
if (options.failFirstBegin && connection === 1) {
|
|
const error = new Error('serialization retry');
|
|
error.code = '40001';
|
|
throw error;
|
|
}
|
|
return { rows: [], rowCount: 0 };
|
|
}
|
|
if (
|
|
normalized === 'COMMIT' ||
|
|
normalized === 'ROLLBACK' ||
|
|
normalized.startsWith('SELECT set_config')
|
|
)
|
|
return { rows: [], rowCount: 0 };
|
|
if (normalized.includes('statement_timestamp()')) {
|
|
return { rows: [{ nowMs: 1_000_000 }], rowCount: 1 };
|
|
}
|
|
if (normalized.includes('lock_run_management_policy_fence')) {
|
|
return {
|
|
rows: [{ matches: options.authorizationMatches ?? true }],
|
|
rowCount: 1,
|
|
};
|
|
}
|
|
if (normalized.includes('idempotency_key = $2')) {
|
|
const rows = options.replayRows ?? [];
|
|
return { rows, rowCount: rows.length };
|
|
}
|
|
if (normalized.includes('WHERE run.id = $1')) {
|
|
const rows = options.sourceRows ?? [sourceRow()];
|
|
return { rows, rowCount: rows.length };
|
|
}
|
|
if (normalized.includes('FROM "ql3"."task_definitions"')) {
|
|
const rows = options.taskRows ?? [{ enabled: true }];
|
|
return { rows, rowCount: rows.length };
|
|
}
|
|
if (normalized.includes('task_execution_revisions')) {
|
|
const rows = options.executionRows ?? [
|
|
{
|
|
sourceContentDigest: SOURCE_DIGEST,
|
|
contentDigest: EXECUTION_DIGEST,
|
|
},
|
|
];
|
|
return { rows, rowCount: rows.length };
|
|
}
|
|
if (
|
|
normalized.includes("trigger_type = 'run_manual_retry'") &&
|
|
normalized.startsWith('SELECT')
|
|
) {
|
|
const rows = options.rateRows ?? [];
|
|
return { rows, rowCount: rows.length };
|
|
}
|
|
if (normalized.startsWith('INSERT INTO')) {
|
|
return { rows: [], rowCount: 1 };
|
|
}
|
|
throw new Error(`Unexpected SQL: ${normalized}`);
|
|
},
|
|
release() {
|
|
calls.push({ connection, sql: 'RELEASE', params: [] });
|
|
},
|
|
};
|
|
},
|
|
};
|
|
return {
|
|
calls,
|
|
repository: new PostgresRunManualRetryRepository(pool),
|
|
};
|
|
}
|
|
|
|
test('atomically appends a linked queued Run, remote Attempt, events and allowed audit', async () => {
|
|
const { calls, repository } = fixture();
|
|
assert.deepEqual(await repository.retryRun(command()), {
|
|
status: 'accepted',
|
|
projectId: 'project-1',
|
|
sourceRunId: 'source-run-1',
|
|
sourceRunStatus: 'failed',
|
|
sourceRunVersion: 7,
|
|
runId: IDS.runId,
|
|
retryOfRunId: 'source-run-1',
|
|
taskId: 'task-1',
|
|
taskRevision: TASK_REVISION,
|
|
attemptId: IDS.attemptId,
|
|
runStatus: 'queued',
|
|
runVersion: 2,
|
|
eventSequence: 2,
|
|
executorType: 'remote_worker',
|
|
executionRevisionDigest: EXECUTION_DIGEST,
|
|
createdAtMs: 1_000_000,
|
|
});
|
|
const inserts = calls.filter(({ sql }) => sql.startsWith('INSERT INTO'));
|
|
assert.equal(inserts.length, 5);
|
|
assert.match(inserts.at(-1).sql, /security_audit_events/);
|
|
assert.deepEqual(JSON.parse(inserts[2].params[6]), {
|
|
status: 'created',
|
|
version: 1,
|
|
execution_owner: 'runtime',
|
|
executor_type: 'remote_worker',
|
|
execution_revision_digest: EXECUTION_DIGEST,
|
|
source_content_digest: SOURCE_DIGEST,
|
|
retry_of_run_id: 'source-run-1',
|
|
source_run_status: 'failed',
|
|
source_run_version: 7,
|
|
inherit_retry_policy: false,
|
|
mutation_id: IDS.mutationId,
|
|
authentication_id: 'oidc:mfa-session-1',
|
|
audit_event_id: IDS.auditEventId,
|
|
policy_fence: { project_version: 2, binding_version: 3 },
|
|
});
|
|
assert.equal(
|
|
calls.some(({ sql }) => sql === 'COMMIT'),
|
|
true,
|
|
);
|
|
});
|
|
|
|
test('returns durable identities for an exact replay without appending again', async () => {
|
|
const { calls, repository } = fixture({
|
|
replayRows: [
|
|
replayRow({ runStatus: 'running', runVersion: 4, eventSequence: 4 }),
|
|
],
|
|
});
|
|
const result = await repository.retryRun(
|
|
command({
|
|
runId: '019f9200-0000-4000-8000-000000000102',
|
|
attemptId: '019f9200-0000-4000-8000-000000000103',
|
|
createdEventId: '019f9200-0000-4000-8000-000000000104',
|
|
queuedEventId: '019f9200-0000-4000-8000-000000000105',
|
|
}),
|
|
);
|
|
assert.equal(result.status, 'existing');
|
|
assert.equal(result.runId, IDS.runId);
|
|
assert.equal(result.attemptId, IDS.attemptId);
|
|
assert.equal(result.runStatus, 'queued');
|
|
assert.equal(
|
|
calls.some(({ sql }) => sql.startsWith('INSERT INTO')),
|
|
false,
|
|
);
|
|
});
|
|
|
|
test('rejects stale authentication and changed authorization inside the transaction', async () => {
|
|
const stale = fixture();
|
|
await assert.rejects(
|
|
stale.repository.retryRun(
|
|
command({
|
|
principal: {
|
|
...command().principal,
|
|
authenticatedAtMs: 600_000,
|
|
},
|
|
}),
|
|
),
|
|
(error) =>
|
|
error instanceof RunManualRetryFenceRejectedError &&
|
|
error.reason === 'authentication_changed',
|
|
);
|
|
assert.equal(
|
|
stale.calls.some(({ sql }) => sql === 'ROLLBACK'),
|
|
true,
|
|
);
|
|
assert.equal(
|
|
stale.calls.some(({ sql }) => sql.includes('lock_run_management_policy_fence')),
|
|
false,
|
|
);
|
|
|
|
const changed = fixture({ authorizationMatches: false });
|
|
await assert.rejects(
|
|
changed.repository.retryRun(command()),
|
|
(error) =>
|
|
error instanceof RunManualRetryFenceRejectedError &&
|
|
error.reason === 'authorization_changed',
|
|
);
|
|
});
|
|
|
|
test('uses the durable Run ledger for the per-project User quota', async () => {
|
|
const rateRows = Array.from(
|
|
{ length: CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT },
|
|
(_, index) => ({ createdAtMs: 999_000 - index }),
|
|
);
|
|
const { calls, repository } = fixture({ rateRows });
|
|
await assert.rejects(
|
|
repository.retryRun(command()),
|
|
(error) =>
|
|
error instanceof RunManualRetryRateLimitedError &&
|
|
error.retryAfterMs === 58_937,
|
|
);
|
|
assert.equal(
|
|
calls.some(({ sql }) => sql.startsWith('INSERT INTO')),
|
|
false,
|
|
);
|
|
assert.equal(
|
|
calls.some(({ sql }) => sql === 'ROLLBACK'),
|
|
true,
|
|
);
|
|
});
|
|
|
|
test('retries one serializable conflict with a fresh connection', async () => {
|
|
const { calls, repository } = fixture({ failFirstBegin: true });
|
|
const result = await repository.retryRun(command());
|
|
assert.equal(result.status, 'accepted');
|
|
assert.deepEqual(
|
|
[...new Set(calls.map(({ connection }) => connection))],
|
|
[1, 2],
|
|
);
|
|
assert.equal(calls.filter(({ sql }) => sql === 'RELEASE').length, 2);
|
|
});
|