mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 02:27:44 +08:00
feat(ql3): add cluster manual run retry authority
This commit is contained in:
whyour
@@ -70,6 +70,11 @@
|
||||
"require": "./dist/task-start/taskStartRepository.js",
|
||||
"default": "./dist/task-start/taskStartRepository.js"
|
||||
},
|
||||
"./run-manual-retry": {
|
||||
"types": "./dist/run-management/runManualRetryRepository.d.ts",
|
||||
"require": "./dist/run-management/runManualRetryRepository.js",
|
||||
"default": "./dist/run-management/runManualRetryRepository.js"
|
||||
},
|
||||
"./approval-manager": {
|
||||
"types": "./dist/approval-management/index.d.ts",
|
||||
"require": "./dist/approval-management/index.js",
|
||||
|
||||
@@ -0,0 +1,660 @@
|
||||
import type { PostgresClient, PostgresPool } from '@qinglong/runtime-core';
|
||||
import {
|
||||
InvalidRunManualRetryError,
|
||||
MAX_RUN_MANUAL_RETRY_AUTHENTICATION_AGE_MS,
|
||||
RUN_MANUAL_RETRY_SOURCE_STATUSES,
|
||||
RunManualRetryFenceRejectedError,
|
||||
RunManualRetryNotFoundError,
|
||||
RunManualRetryRateLimitedError,
|
||||
RunManualRetryUnavailableError,
|
||||
normalizeRunManualRetryCommand,
|
||||
normalizeRunManualRetryResult,
|
||||
type RunManualRetryAllowedRole,
|
||||
type RunManualRetryCommand,
|
||||
type RunManualRetryRepository,
|
||||
type RunManualRetryResult,
|
||||
type RunManualRetrySourceStatus,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
|
||||
import {
|
||||
POSTGRES_DEFINITION_RETRYABLE_SQL_STATES,
|
||||
POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS,
|
||||
configurePostgresDefinitionTransaction,
|
||||
postgresRequiredBoolean,
|
||||
postgresRequiredInteger,
|
||||
postgresRequiredJsonObject,
|
||||
postgresRequiredString,
|
||||
postgresSqlState,
|
||||
rollbackPostgresDefinitionTransaction,
|
||||
} from '../repository/definitionRepositorySupport';
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
export const CLUSTER_RUN_MANUAL_RETRY_RATE_WINDOW_MS = 60_000;
|
||||
export const CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT = 64;
|
||||
|
||||
const ALLOWED_ROLES = new Set<RunManualRetryAllowedRole>([
|
||||
'owner',
|
||||
'admin',
|
||||
'operator',
|
||||
]);
|
||||
const CLUSTER_STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
|
||||
const TASK_REVISION_PATTERN = /^qltd:v1:([1-9]\d*):([0-9a-f]{64})$/;
|
||||
|
||||
interface SourceRun {
|
||||
readonly taskId: string;
|
||||
readonly taskRevision: string;
|
||||
readonly taskName?: string;
|
||||
readonly taskSnapshotRef: string;
|
||||
readonly inputRef?: string;
|
||||
readonly priority: number;
|
||||
}
|
||||
|
||||
interface ExecutionRevision {
|
||||
readonly contentDigest: string;
|
||||
readonly sourceContentDigest: string;
|
||||
}
|
||||
|
||||
function unavailable(options?: ErrorOptions): RunManualRetryUnavailableError {
|
||||
return new RunManualRetryUnavailableError(options);
|
||||
}
|
||||
|
||||
function text(row: Row, key: string): string {
|
||||
return postgresRequiredString(row[key], unavailable);
|
||||
}
|
||||
|
||||
function integer(row: Row, key: string): number {
|
||||
return postgresRequiredInteger(row[key], unavailable);
|
||||
}
|
||||
|
||||
function optionalText(row: Row, key: string): string | undefined {
|
||||
if (row[key] === null) return undefined;
|
||||
return text(row, key);
|
||||
}
|
||||
|
||||
function json(row: Row, key: string): Record<string, unknown> {
|
||||
return postgresRequiredJsonObject(row[key], unavailable);
|
||||
}
|
||||
|
||||
function sourceStatus(value: string): RunManualRetrySourceStatus {
|
||||
if (
|
||||
!RUN_MANUAL_RETRY_SOURCE_STATUSES.includes(
|
||||
value as RunManualRetrySourceStatus,
|
||||
)
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('source_not_terminal');
|
||||
}
|
||||
return value as RunManualRetrySourceStatus;
|
||||
}
|
||||
|
||||
function sameFencePayload(
|
||||
value: unknown,
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
): boolean {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
||||
const fence = value as Record<string, unknown>;
|
||||
return (
|
||||
fence.project_version === command.policyFence.projectVersion &&
|
||||
fence.binding_version === command.policyFence.bindingVersion
|
||||
);
|
||||
}
|
||||
|
||||
async function databaseNow(client: PostgresClient): Promise<number> {
|
||||
const result = await client.query<Row>(`
|
||||
SELECT floor(extract(epoch FROM statement_timestamp()) * 1000)::bigint
|
||||
AS "nowMs"
|
||||
`);
|
||||
if (result.rows.length !== 1) throw unavailable();
|
||||
return integer(result.rows[0]!, 'nowMs');
|
||||
}
|
||||
|
||||
function confirmStrongAuthentication(
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
observedAtMs: number,
|
||||
): void {
|
||||
if (
|
||||
command.principal.subject.type !== 'user' ||
|
||||
!CLUSTER_STRONG_ASSURANCES.has(command.principal.assurance) ||
|
||||
command.principal.authenticatedAtMs > observedAtMs ||
|
||||
command.principal.expiresAtMs <= observedAtMs ||
|
||||
observedAtMs - command.principal.authenticatedAtMs >
|
||||
MAX_RUN_MANUAL_RETRY_AUTHENTICATION_AGE_MS
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('authentication_changed');
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmAuthorization(
|
||||
client: PostgresClient,
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
): Promise<void> {
|
||||
const project = await client.query<Row>(
|
||||
`
|
||||
SELECT status AS "projectStatus", version AS "projectVersion"
|
||||
FROM "ql3"."projects" WHERE id = $1 FOR UPDATE
|
||||
`,
|
||||
[command.projectId],
|
||||
);
|
||||
if (project.rows.length === 0) throw new RunManualRetryNotFoundError();
|
||||
if (project.rows.length !== 1) throw unavailable();
|
||||
// Authorized management mutations take the same Project lock. Keeping this
|
||||
// append-only RoleBinding read lock-free avoids granting UPDATE authority to
|
||||
// the runtime role merely to use PostgreSQL row-lock syntax.
|
||||
const binding = await client.query<Row>(
|
||||
`
|
||||
SELECT version AS "bindingVersion", state AS "bindingState",
|
||||
role AS "bindingRole"
|
||||
FROM "ql3"."project_role_bindings"
|
||||
WHERE project_id = $1 AND subject_type = $2 AND subject_id = $3
|
||||
ORDER BY version DESC LIMIT 1
|
||||
`,
|
||||
[
|
||||
command.projectId,
|
||||
command.principal.subject.type,
|
||||
command.principal.subject.id,
|
||||
],
|
||||
);
|
||||
const currentProject = project.rows[0]!;
|
||||
const currentBinding = binding.rows[0];
|
||||
if (
|
||||
text(currentProject, 'projectStatus') !== 'active' ||
|
||||
integer(currentProject, 'projectVersion') !==
|
||||
command.policyFence.projectVersion ||
|
||||
!currentBinding ||
|
||||
integer(currentBinding, 'bindingVersion') !==
|
||||
command.policyFence.bindingVersion ||
|
||||
text(currentBinding, 'bindingState') !== 'active' ||
|
||||
!ALLOWED_ROLES.has(
|
||||
text(currentBinding, 'bindingRole') as RunManualRetryAllowedRole,
|
||||
)
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('authorization_changed');
|
||||
}
|
||||
}
|
||||
|
||||
async function findReplay(
|
||||
client: PostgresClient,
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
): Promise<Row | undefined> {
|
||||
const result = await client.query<Row>(
|
||||
`
|
||||
SELECT run.id AS "runId", run.project_id AS "projectId",
|
||||
run.retry_of_run_id AS "retryOfRunId",
|
||||
run.task_id AS "taskId", run.task_revision AS "taskRevision",
|
||||
run.trigger_type AS "triggerType",
|
||||
run.execution_origin AS "executionOrigin",
|
||||
run.execution_owner AS "executionOwner",
|
||||
run.triggered_by AS "triggeredBy", run.request_id AS "requestId",
|
||||
run.status AS "runStatus", run.version AS "runVersion",
|
||||
run.event_sequence AS "eventSequence",
|
||||
run.created_at_ms AS "createdAtMs",
|
||||
attempt.id AS "attemptId", attempt.executor_type AS "executorType",
|
||||
created.actor_type AS "createdActorType",
|
||||
created.actor_id AS "createdActorId",
|
||||
created.payload AS "createdPayload",
|
||||
queued.actor_type AS "queuedActorType",
|
||||
queued.actor_id AS "queuedActorId",
|
||||
queued.payload AS "queuedPayload"
|
||||
FROM "ql3"."runs" AS run
|
||||
JOIN "ql3"."run_attempts" AS attempt
|
||||
ON attempt.run_id = run.id AND attempt.attempt = 1
|
||||
JOIN "ql3"."run_events" AS created
|
||||
ON created.run_id = run.id AND created.sequence = 1
|
||||
AND created.type = 'run.created'
|
||||
JOIN "ql3"."run_events" AS queued
|
||||
ON queued.run_id = run.id AND queued.sequence = 2
|
||||
AND queued.type = 'run.queued'
|
||||
WHERE run.project_id = $1 AND run.idempotency_key = $2
|
||||
FOR UPDATE OF run
|
||||
`,
|
||||
[command.projectId, `ql3:run-manual-retry:v1:${command.mutationId}`],
|
||||
);
|
||||
if (result.rows.length > 1) throw unavailable();
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
function replayResult(
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
row: Row,
|
||||
): Readonly<RunManualRetryResult> {
|
||||
const created = json(row, 'createdPayload');
|
||||
const queued = json(row, 'queuedPayload');
|
||||
if (
|
||||
text(row, 'projectId') !== command.projectId ||
|
||||
text(row, 'retryOfRunId') !== command.sourceRunId ||
|
||||
text(row, 'triggerType') !== 'run_manual_retry' ||
|
||||
text(row, 'executionOrigin') !== 'manual' ||
|
||||
text(row, 'executionOwner') !== 'runtime' ||
|
||||
text(row, 'triggeredBy') !== command.principal.subject.id ||
|
||||
text(row, 'requestId') !== command.mutationId ||
|
||||
text(row, 'runStatus') !== 'queued' ||
|
||||
integer(row, 'runVersion') !== 2 ||
|
||||
integer(row, 'eventSequence') !== 2 ||
|
||||
text(row, 'executorType') !== 'remote_worker' ||
|
||||
text(row, 'createdActorType') !== command.principal.subject.type ||
|
||||
text(row, 'createdActorId') !== command.principal.subject.id ||
|
||||
text(row, 'queuedActorType') !== command.principal.subject.type ||
|
||||
text(row, 'queuedActorId') !== command.principal.subject.id ||
|
||||
created.mutation_id !== command.mutationId ||
|
||||
created.retry_of_run_id !== command.sourceRunId ||
|
||||
created.source_run_status !== command.expectedRunStatus ||
|
||||
created.source_run_version !== command.expectedRunVersion ||
|
||||
created.inherit_retry_policy !== false ||
|
||||
created.authentication_id !== command.principal.authenticationId ||
|
||||
created.audit_event_id !== command.auditEventId ||
|
||||
typeof created.execution_revision_digest !== 'string' ||
|
||||
!/^[0-9a-f]{64}$/.test(created.execution_revision_digest) ||
|
||||
!sameFencePayload(created.policy_fence, command) ||
|
||||
queued.from_status !== 'created' ||
|
||||
queued.to_status !== 'queued' ||
|
||||
queued.version !== 2
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('mutation_conflict');
|
||||
}
|
||||
return normalizeRunManualRetryResult({
|
||||
status: 'existing',
|
||||
projectId: command.projectId,
|
||||
sourceRunId: command.sourceRunId,
|
||||
sourceRunStatus: command.expectedRunStatus,
|
||||
sourceRunVersion: command.expectedRunVersion,
|
||||
runId: text(row, 'runId'),
|
||||
retryOfRunId: text(row, 'retryOfRunId'),
|
||||
taskId: text(row, 'taskId'),
|
||||
taskRevision: text(row, 'taskRevision'),
|
||||
attemptId: text(row, 'attemptId'),
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: created.execution_revision_digest,
|
||||
createdAtMs: integer(row, 'createdAtMs'),
|
||||
});
|
||||
}
|
||||
|
||||
async function findSource(
|
||||
client: PostgresClient,
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
): Promise<SourceRun> {
|
||||
const result = await client.query<Row>(
|
||||
`
|
||||
SELECT run.project_id AS "projectId", run.task_id AS "taskId",
|
||||
run.task_revision AS "taskRevision", run.task_name AS "taskName",
|
||||
run.task_snapshot_ref AS "taskSnapshotRef",
|
||||
run.parent_run_id AS "parentRunId",
|
||||
run.trigger_type AS "triggerType",
|
||||
run.execution_owner AS "executionOwner",
|
||||
run.input_ref AS "inputRef", run.priority,
|
||||
run.status AS "runStatus", run.version AS "runVersion",
|
||||
attempt.executor_type AS "attemptExecutorType"
|
||||
FROM "ql3"."runs" AS run
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT executor_type
|
||||
FROM "ql3"."run_attempts"
|
||||
WHERE run_id = run.id
|
||||
ORDER BY attempt DESC
|
||||
LIMIT 1
|
||||
) AS attempt ON true
|
||||
WHERE run.id = $1
|
||||
FOR UPDATE OF run
|
||||
`,
|
||||
[command.sourceRunId],
|
||||
);
|
||||
if (
|
||||
result.rows.length === 0 ||
|
||||
result.rows[0]?.projectId !== command.projectId
|
||||
) {
|
||||
throw new RunManualRetryNotFoundError();
|
||||
}
|
||||
if (result.rows.length !== 1) throw unavailable();
|
||||
const row = result.rows[0]!;
|
||||
const status = sourceStatus(text(row, 'runStatus'));
|
||||
if (
|
||||
status !== command.expectedRunStatus ||
|
||||
integer(row, 'runVersion') !== command.expectedRunVersion
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('source_changed');
|
||||
}
|
||||
const taskRevision = text(row, 'taskRevision');
|
||||
const taskSnapshotRef = optionalText(row, 'taskSnapshotRef');
|
||||
if (
|
||||
text(row, 'executionOwner') !== 'runtime' ||
|
||||
optionalText(row, 'parentRunId') !== undefined ||
|
||||
text(row, 'triggerType') === 'plugin_package_workflow' ||
|
||||
taskSnapshotRef === undefined ||
|
||||
taskSnapshotRef !== taskRevision ||
|
||||
optionalText(row, 'attemptExecutorType') !== 'remote_worker' ||
|
||||
!TASK_REVISION_PATTERN.test(taskRevision)
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('source_not_retryable');
|
||||
}
|
||||
const taskName = optionalText(row, 'taskName');
|
||||
const inputRef = optionalText(row, 'inputRef');
|
||||
return Object.freeze({
|
||||
taskId: text(row, 'taskId'),
|
||||
taskRevision,
|
||||
...(taskName === undefined ? {} : { taskName }),
|
||||
taskSnapshotRef,
|
||||
...(inputRef === undefined ? {} : { inputRef }),
|
||||
priority: integer(row, 'priority'),
|
||||
});
|
||||
}
|
||||
|
||||
async function confirmTaskAndExecution(
|
||||
client: PostgresClient,
|
||||
projectId: string,
|
||||
source: Readonly<SourceRun>,
|
||||
): Promise<ExecutionRevision> {
|
||||
const task = await client.query<Row>(
|
||||
`
|
||||
SELECT revision.enabled
|
||||
FROM "ql3"."task_definitions" AS head
|
||||
JOIN "ql3"."task_definition_revisions" AS revision
|
||||
ON revision.project_id = head.project_id
|
||||
AND revision.task_id = head.task_id
|
||||
AND revision.revision = head.current_revision
|
||||
WHERE head.project_id = $1 AND head.task_id = $2
|
||||
`,
|
||||
[projectId, source.taskId],
|
||||
);
|
||||
if (
|
||||
task.rows.length !== 1 ||
|
||||
!postgresRequiredBoolean(task.rows[0]!.enabled, unavailable)
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('task_disabled');
|
||||
}
|
||||
const execution = await client.query<Row>(
|
||||
`
|
||||
SELECT source_content_digest AS "sourceContentDigest",
|
||||
content_digest AS "contentDigest"
|
||||
FROM "ql3"."task_execution_revisions"
|
||||
WHERE project_id = $1 AND task_id = $2 AND task_revision = $3
|
||||
AND executor_type = 'remote_worker'
|
||||
LIMIT 2
|
||||
`,
|
||||
[projectId, source.taskId, source.taskRevision],
|
||||
);
|
||||
if (execution.rows.length !== 1) {
|
||||
throw new RunManualRetryFenceRejectedError('source_not_retryable');
|
||||
}
|
||||
const match = TASK_REVISION_PATTERN.exec(source.taskRevision);
|
||||
const row = execution.rows[0]!;
|
||||
const sourceContentDigest = text(row, 'sourceContentDigest');
|
||||
const contentDigest = text(row, 'contentDigest');
|
||||
if (
|
||||
!match ||
|
||||
match[2] !== sourceContentDigest ||
|
||||
!/^[0-9a-f]{64}$/.test(contentDigest)
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('source_not_retryable');
|
||||
}
|
||||
return Object.freeze({ contentDigest, sourceContentDigest });
|
||||
}
|
||||
|
||||
async function consumeRateLimit(
|
||||
client: PostgresClient,
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
observedAtMs: number,
|
||||
): Promise<void> {
|
||||
const threshold = Math.max(
|
||||
0,
|
||||
observedAtMs - CLUSTER_RUN_MANUAL_RETRY_RATE_WINDOW_MS,
|
||||
);
|
||||
const result = await client.query<Row>(
|
||||
`
|
||||
SELECT created_at_ms AS "createdAtMs"
|
||||
FROM "ql3"."runs"
|
||||
WHERE project_id = $1 AND trigger_type = 'run_manual_retry'
|
||||
AND execution_origin = 'manual' AND triggered_by = $2
|
||||
AND created_at_ms > $3
|
||||
ORDER BY created_at_ms DESC, id DESC
|
||||
LIMIT $4
|
||||
`,
|
||||
[
|
||||
command.projectId,
|
||||
command.principal.subject.id,
|
||||
threshold,
|
||||
CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT,
|
||||
],
|
||||
);
|
||||
if (result.rows.length < CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT) return;
|
||||
const earliestAtMs = integer(
|
||||
result.rows[result.rows.length - 1]!,
|
||||
'createdAtMs',
|
||||
);
|
||||
throw new RunManualRetryRateLimitedError(
|
||||
Math.max(
|
||||
1,
|
||||
earliestAtMs + CLUSTER_RUN_MANUAL_RETRY_RATE_WINDOW_MS - observedAtMs,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async function insertRetry(
|
||||
client: PostgresClient,
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
source: Readonly<SourceRun>,
|
||||
execution: Readonly<ExecutionRevision>,
|
||||
observedAtMs: number,
|
||||
): Promise<void> {
|
||||
await client.query(
|
||||
`
|
||||
INSERT INTO "ql3"."runs" (
|
||||
id, project_id, task_id, task_revision, task_name,
|
||||
task_snapshot_ref, retry_of_run_id, trigger_type, execution_origin,
|
||||
execution_owner, triggered_by, request_id, status, version,
|
||||
event_sequence, priority, idempotency_key, input_ref,
|
||||
created_at_ms, queued_at_ms
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, 'run_manual_retry', 'manual',
|
||||
'runtime', $8, $9, 'queued', 2, 2, $10, $11, $12, $13, $13
|
||||
)
|
||||
`,
|
||||
[
|
||||
command.runId,
|
||||
command.projectId,
|
||||
source.taskId,
|
||||
source.taskRevision,
|
||||
source.taskName ?? null,
|
||||
source.taskSnapshotRef,
|
||||
command.sourceRunId,
|
||||
command.principal.subject.id,
|
||||
command.mutationId,
|
||||
source.priority,
|
||||
`ql3:run-manual-retry:v1:${command.mutationId}`,
|
||||
source.inputRef ?? null,
|
||||
observedAtMs,
|
||||
],
|
||||
);
|
||||
await client.query(
|
||||
`
|
||||
INSERT INTO "ql3"."run_attempts" (
|
||||
id, run_id, attempt, status, executor_type,
|
||||
callback_sequence, created_at_ms
|
||||
) VALUES ($1, $2, 1, 'claimed', 'remote_worker', 0, $3)
|
||||
`,
|
||||
[command.attemptId, command.runId, observedAtMs],
|
||||
);
|
||||
await client.query(
|
||||
`
|
||||
INSERT INTO "ql3"."run_events" (
|
||||
id, run_id, sequence, type, dedupe_key, actor_type, actor_id,
|
||||
attempt_id, payload, created_at_ms
|
||||
) VALUES ($1, $2, 1, 'run.created', $3, $4, $5, $6, $7::jsonb, $8)
|
||||
`,
|
||||
[
|
||||
command.createdEventId,
|
||||
command.runId,
|
||||
`run-manual-retry-created:${command.mutationId}`,
|
||||
command.principal.subject.type,
|
||||
command.principal.subject.id,
|
||||
command.attemptId,
|
||||
JSON.stringify({
|
||||
status: 'created',
|
||||
version: 1,
|
||||
execution_owner: 'runtime',
|
||||
executor_type: 'remote_worker',
|
||||
execution_revision_digest: execution.contentDigest,
|
||||
source_content_digest: execution.sourceContentDigest,
|
||||
retry_of_run_id: command.sourceRunId,
|
||||
source_run_status: command.expectedRunStatus,
|
||||
source_run_version: command.expectedRunVersion,
|
||||
inherit_retry_policy: false,
|
||||
mutation_id: command.mutationId,
|
||||
authentication_id: command.principal.authenticationId,
|
||||
audit_event_id: command.auditEventId,
|
||||
policy_fence: {
|
||||
project_version: command.policyFence.projectVersion,
|
||||
binding_version: command.policyFence.bindingVersion,
|
||||
},
|
||||
}),
|
||||
observedAtMs,
|
||||
],
|
||||
);
|
||||
await client.query(
|
||||
`
|
||||
INSERT INTO "ql3"."run_events" (
|
||||
id, run_id, sequence, type, dedupe_key, actor_type, actor_id,
|
||||
attempt_id, payload, created_at_ms
|
||||
) VALUES ($1, $2, 2, 'run.queued', $3, $4, $5, $6, $7::jsonb, $8)
|
||||
`,
|
||||
[
|
||||
command.queuedEventId,
|
||||
command.runId,
|
||||
`run-manual-retry-queued:${command.mutationId}`,
|
||||
command.principal.subject.type,
|
||||
command.principal.subject.id,
|
||||
command.attemptId,
|
||||
JSON.stringify({
|
||||
from_status: 'created',
|
||||
to_status: 'queued',
|
||||
version: 2,
|
||||
}),
|
||||
observedAtMs,
|
||||
],
|
||||
);
|
||||
await client.query(
|
||||
`
|
||||
INSERT INTO "ql3"."security_audit_events" (
|
||||
event_id, request_id, operation_id, project_id,
|
||||
subject_type, subject_id, authentication_id, outcome, reasons,
|
||||
project_version, binding_version, occurred_at_ms
|
||||
) VALUES (
|
||||
$1, $2, 'run.retry', $3, $4, $5, $6, 'allowed', $7::jsonb,
|
||||
$8, $9, $10
|
||||
)
|
||||
`,
|
||||
[
|
||||
command.auditEventId,
|
||||
command.requestId,
|
||||
command.projectId,
|
||||
command.principal.subject.type,
|
||||
command.principal.subject.id,
|
||||
command.principal.authenticationId,
|
||||
JSON.stringify(['role_grant', 'strong_authentication']),
|
||||
command.policyFence.projectVersion,
|
||||
command.policyFence.bindingVersion,
|
||||
observedAtMs,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* PostgreSQL authority for one strongly authenticated manual retry. The
|
||||
* Project row lock serializes the existing Run-ledger quota across replicas.
|
||||
*/
|
||||
export class PostgresRunManualRetryRepository
|
||||
implements RunManualRetryRepository
|
||||
{
|
||||
constructor(private readonly pool: PostgresPool) {
|
||||
if (!pool || typeof pool.connect !== 'function') {
|
||||
throw new TypeError('PostgreSQL Run manual retry pool is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async retryRun(
|
||||
value: Readonly<RunManualRetryCommand>,
|
||||
): Promise<Readonly<RunManualRetryResult>> {
|
||||
const command = normalizeRunManualRetryCommand(value);
|
||||
for (
|
||||
let transactionAttempt = 0;
|
||||
transactionAttempt < POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS;
|
||||
transactionAttempt += 1
|
||||
) {
|
||||
let client: PostgresClient;
|
||||
try {
|
||||
client = await this.pool.connect();
|
||||
} catch (error) {
|
||||
throw unavailable({ cause: error });
|
||||
}
|
||||
let began = false;
|
||||
try {
|
||||
await configurePostgresDefinitionTransaction(client);
|
||||
began = true;
|
||||
const observedAtMs = await databaseNow(client);
|
||||
confirmStrongAuthentication(command, observedAtMs);
|
||||
await confirmAuthorization(client, command);
|
||||
|
||||
const replay = await findReplay(client, command);
|
||||
if (replay) {
|
||||
const result = replayResult(command, replay);
|
||||
await client.query('COMMIT');
|
||||
began = false;
|
||||
return result;
|
||||
}
|
||||
|
||||
const source = await findSource(client, command);
|
||||
const execution = await confirmTaskAndExecution(
|
||||
client,
|
||||
command.projectId,
|
||||
source,
|
||||
);
|
||||
await consumeRateLimit(client, command, observedAtMs);
|
||||
await insertRetry(client, command, source, execution, observedAtMs);
|
||||
const result = normalizeRunManualRetryResult({
|
||||
status: 'accepted',
|
||||
projectId: command.projectId,
|
||||
sourceRunId: command.sourceRunId,
|
||||
sourceRunStatus: command.expectedRunStatus,
|
||||
sourceRunVersion: command.expectedRunVersion,
|
||||
runId: command.runId,
|
||||
retryOfRunId: command.sourceRunId,
|
||||
taskId: source.taskId,
|
||||
taskRevision: source.taskRevision,
|
||||
attemptId: command.attemptId,
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: execution.contentDigest,
|
||||
createdAtMs: observedAtMs,
|
||||
});
|
||||
await client.query('COMMIT');
|
||||
began = false;
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (began) await rollbackPostgresDefinitionTransaction(client);
|
||||
const state = postgresSqlState(error);
|
||||
if (
|
||||
(state === '23505' ||
|
||||
POSTGRES_DEFINITION_RETRYABLE_SQL_STATES.has(state ?? '')) &&
|
||||
transactionAttempt + 1 < POSTGRES_DEFINITION_TRANSACTION_ATTEMPTS
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
error instanceof InvalidRunManualRetryError ||
|
||||
error instanceof RunManualRetryNotFoundError ||
|
||||
error instanceof RunManualRetryFenceRejectedError ||
|
||||
error instanceof RunManualRetryRateLimitedError ||
|
||||
error instanceof RunManualRetryUnavailableError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw unavailable({ cause: error });
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
throw unavailable();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
RunManualRetryFenceRejectedError,
|
||||
RunManualRetryRateLimitedError,
|
||||
} = require('@qinglong/runtime-core/run-manual-retry');
|
||||
const {
|
||||
CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT,
|
||||
PostgresRunManualRetryRepository,
|
||||
} = require('@qinglong/cluster-postgres/run-manual-retry');
|
||||
|
||||
const SOURCE_DIGEST = 'a'.repeat(64);
|
||||
const EXECUTION_DIGEST = 'b'.repeat(64);
|
||||
const TASK_REVISION = `qltd:v1:7:${SOURCE_DIGEST}`;
|
||||
const IDS = Object.freeze({
|
||||
mutationId: '019f9200-0000-4000-8000-000000000001',
|
||||
runId: '019f9200-0000-4000-8000-000000000002',
|
||||
attemptId: '019f9200-0000-4000-8000-000000000003',
|
||||
createdEventId: '019f9200-0000-4000-8000-000000000004',
|
||||
queuedEventId: '019f9200-0000-4000-8000-000000000005',
|
||||
auditEventId: '019f9200-0000-4000-8000-000000000006',
|
||||
});
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
mutationId: IDS.mutationId,
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
runId: IDS.runId,
|
||||
attemptId: IDS.attemptId,
|
||||
createdEventId: IDS.createdEventId,
|
||||
queuedEventId: IDS.queuedEventId,
|
||||
auditEventId: IDS.auditEventId,
|
||||
requestId: 'cluster-run-retry-1',
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'oidc:mfa-session-1',
|
||||
authenticatedAtMs: 900_000,
|
||||
expiresAtMs: 1_100_000,
|
||||
assurance: 'multi_factor',
|
||||
},
|
||||
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function sourceRow(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: TASK_REVISION,
|
||||
taskName: 'Task 1',
|
||||
taskSnapshotRef: TASK_REVISION,
|
||||
parentRunId: null,
|
||||
triggerType: 'task_start',
|
||||
executionOwner: 'runtime',
|
||||
inputRef: 'artifact:input-1',
|
||||
priority: 3,
|
||||
runStatus: 'failed',
|
||||
runVersion: 7,
|
||||
attemptExecutorType: 'remote_worker',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function replayRow(overrides = {}) {
|
||||
return {
|
||||
runId: IDS.runId,
|
||||
projectId: 'project-1',
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: TASK_REVISION,
|
||||
triggerType: 'run_manual_retry',
|
||||
executionOrigin: 'manual',
|
||||
executionOwner: 'runtime',
|
||||
triggeredBy: 'operator-1',
|
||||
requestId: IDS.mutationId,
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
createdAtMs: 1_000_000,
|
||||
attemptId: IDS.attemptId,
|
||||
executorType: 'remote_worker',
|
||||
createdActorType: 'user',
|
||||
createdActorId: 'operator-1',
|
||||
createdPayload: {
|
||||
mutation_id: IDS.mutationId,
|
||||
retry_of_run_id: 'source-run-1',
|
||||
source_run_status: 'failed',
|
||||
source_run_version: 7,
|
||||
inherit_retry_policy: false,
|
||||
authentication_id: 'oidc:mfa-session-1',
|
||||
audit_event_id: IDS.auditEventId,
|
||||
execution_revision_digest: EXECUTION_DIGEST,
|
||||
policy_fence: { project_version: 2, binding_version: 3 },
|
||||
},
|
||||
queuedActorType: 'user',
|
||||
queuedActorId: 'operator-1',
|
||||
queuedPayload: {
|
||||
from_status: 'created',
|
||||
to_status: 'queued',
|
||||
version: 2,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(options = {}) {
|
||||
const calls = [];
|
||||
let connections = 0;
|
||||
const pool = {
|
||||
async connect() {
|
||||
connections += 1;
|
||||
const connection = connections;
|
||||
return {
|
||||
async query(sql, params = []) {
|
||||
const normalized = sql.replace(/\s+/g, ' ').trim();
|
||||
calls.push({ connection, sql: normalized, params });
|
||||
if (normalized.startsWith('BEGIN')) {
|
||||
if (options.failFirstBegin && connection === 1) {
|
||||
const error = new Error('serialization retry');
|
||||
error.code = '40001';
|
||||
throw error;
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (
|
||||
normalized === 'COMMIT' ||
|
||||
normalized === 'ROLLBACK' ||
|
||||
normalized.startsWith('SELECT set_config')
|
||||
)
|
||||
return { rows: [], rowCount: 0 };
|
||||
if (normalized.includes('statement_timestamp()')) {
|
||||
return { rows: [{ nowMs: 1_000_000 }], rowCount: 1 };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."projects"')) {
|
||||
const rows = options.projectRows ?? [
|
||||
{ projectStatus: 'active', projectVersion: 2 },
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('project_role_bindings')) {
|
||||
const rows = options.bindingRows ?? [
|
||||
{
|
||||
bindingVersion: 3,
|
||||
bindingState: 'active',
|
||||
bindingRole: 'operator',
|
||||
},
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('idempotency_key = $2')) {
|
||||
const rows = options.replayRows ?? [];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('WHERE run.id = $1')) {
|
||||
const rows = options.sourceRows ?? [sourceRow()];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('FROM "ql3"."task_definitions"')) {
|
||||
const rows = options.taskRows ?? [{ enabled: true }];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.includes('task_execution_revisions')) {
|
||||
const rows = options.executionRows ?? [
|
||||
{
|
||||
sourceContentDigest: SOURCE_DIGEST,
|
||||
contentDigest: EXECUTION_DIGEST,
|
||||
},
|
||||
];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (
|
||||
normalized.includes("trigger_type = 'run_manual_retry'") &&
|
||||
normalized.startsWith('SELECT')
|
||||
) {
|
||||
const rows = options.rateRows ?? [];
|
||||
return { rows, rowCount: rows.length };
|
||||
}
|
||||
if (normalized.startsWith('INSERT INTO')) {
|
||||
return { rows: [], rowCount: 1 };
|
||||
}
|
||||
throw new Error(`Unexpected SQL: ${normalized}`);
|
||||
},
|
||||
release() {
|
||||
calls.push({ connection, sql: 'RELEASE', params: [] });
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
repository: new PostgresRunManualRetryRepository(pool),
|
||||
};
|
||||
}
|
||||
|
||||
test('atomically appends a linked queued Run, remote Attempt, events and allowed audit', async () => {
|
||||
const { calls, repository } = fixture();
|
||||
assert.deepEqual(await repository.retryRun(command()), {
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: IDS.runId,
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: TASK_REVISION,
|
||||
attemptId: IDS.attemptId,
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'remote_worker',
|
||||
executionRevisionDigest: EXECUTION_DIGEST,
|
||||
createdAtMs: 1_000_000,
|
||||
});
|
||||
const inserts = calls.filter(({ sql }) => sql.startsWith('INSERT INTO'));
|
||||
assert.equal(inserts.length, 5);
|
||||
assert.match(inserts.at(-1).sql, /security_audit_events/);
|
||||
assert.deepEqual(JSON.parse(inserts[2].params[6]), {
|
||||
status: 'created',
|
||||
version: 1,
|
||||
execution_owner: 'runtime',
|
||||
executor_type: 'remote_worker',
|
||||
execution_revision_digest: EXECUTION_DIGEST,
|
||||
source_content_digest: SOURCE_DIGEST,
|
||||
retry_of_run_id: 'source-run-1',
|
||||
source_run_status: 'failed',
|
||||
source_run_version: 7,
|
||||
inherit_retry_policy: false,
|
||||
mutation_id: IDS.mutationId,
|
||||
authentication_id: 'oidc:mfa-session-1',
|
||||
audit_event_id: IDS.auditEventId,
|
||||
policy_fence: { project_version: 2, binding_version: 3 },
|
||||
});
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'COMMIT'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('returns durable identities for an exact replay without appending again', async () => {
|
||||
const { calls, repository } = fixture({ replayRows: [replayRow()] });
|
||||
const result = await repository.retryRun(
|
||||
command({
|
||||
runId: '019f9200-0000-4000-8000-000000000102',
|
||||
attemptId: '019f9200-0000-4000-8000-000000000103',
|
||||
createdEventId: '019f9200-0000-4000-8000-000000000104',
|
||||
queuedEventId: '019f9200-0000-4000-8000-000000000105',
|
||||
}),
|
||||
);
|
||||
assert.equal(result.status, 'existing');
|
||||
assert.equal(result.runId, IDS.runId);
|
||||
assert.equal(result.attemptId, IDS.attemptId);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.startsWith('INSERT INTO')),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejects stale authentication and changed authorization inside the transaction', async () => {
|
||||
const stale = fixture();
|
||||
await assert.rejects(
|
||||
stale.repository.retryRun(
|
||||
command({
|
||||
principal: {
|
||||
...command().principal,
|
||||
authenticatedAtMs: 600_000,
|
||||
},
|
||||
}),
|
||||
),
|
||||
(error) =>
|
||||
error instanceof RunManualRetryFenceRejectedError &&
|
||||
error.reason === 'authentication_changed',
|
||||
);
|
||||
assert.equal(
|
||||
stale.calls.some(({ sql }) => sql === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
stale.calls.some(({ sql }) => sql.includes('FROM "ql3"."projects"')),
|
||||
false,
|
||||
);
|
||||
|
||||
const changed = fixture({
|
||||
bindingRows: [
|
||||
{ bindingVersion: 4, bindingState: 'active', bindingRole: 'operator' },
|
||||
],
|
||||
});
|
||||
await assert.rejects(
|
||||
changed.repository.retryRun(command()),
|
||||
(error) =>
|
||||
error instanceof RunManualRetryFenceRejectedError &&
|
||||
error.reason === 'authorization_changed',
|
||||
);
|
||||
});
|
||||
|
||||
test('uses the durable Run ledger for the per-project User quota', async () => {
|
||||
const rateRows = Array.from(
|
||||
{ length: CLUSTER_RUN_MANUAL_RETRY_RATE_LIMIT },
|
||||
(_, index) => ({ createdAtMs: 999_000 - index }),
|
||||
);
|
||||
const { calls, repository } = fixture({ rateRows });
|
||||
await assert.rejects(
|
||||
repository.retryRun(command()),
|
||||
(error) =>
|
||||
error instanceof RunManualRetryRateLimitedError &&
|
||||
error.retryAfterMs === 58_937,
|
||||
);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql.startsWith('INSERT INTO')),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
calls.some(({ sql }) => sql === 'ROLLBACK'),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('retries one serializable conflict with a fresh connection', async () => {
|
||||
const { calls, repository } = fixture({ failFirstBegin: true });
|
||||
const result = await repository.retryRun(command());
|
||||
assert.equal(result.status, 'accepted');
|
||||
assert.deepEqual(
|
||||
[...new Set(calls.map(({ connection }) => connection))],
|
||||
[1, 2],
|
||||
);
|
||||
assert.equal(calls.filter(({ sql }) => sql === 'RELEASE').length, 2);
|
||||
});
|
||||
Reference in New Issue
Block a user