mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add strong local manual run retry
This commit is contained in:
whyour
@@ -276,6 +276,11 @@
|
||||
"require": "./dist/run/clusterRunLostRetry.js",
|
||||
"default": "./dist/run/clusterRunLostRetry.js"
|
||||
},
|
||||
"./run-manual-retry": {
|
||||
"types": "./dist/run/manual-retry/runManualRetry.d.ts",
|
||||
"require": "./dist/run/manual-retry/runManualRetry.js",
|
||||
"default": "./dist/run/manual-retry/runManualRetry.js"
|
||||
},
|
||||
"./cluster-run-lost-retry": {
|
||||
"types": "./dist/run/clusterRunLostRetry.d.ts",
|
||||
"require": "./dist/run/clusterRunLostRetry.js",
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
import {
|
||||
normalizeProjectPolicySubject,
|
||||
type ProjectRole,
|
||||
} from '../../security/project-policy/projectPolicy';
|
||||
import {
|
||||
normalizeSecurityPolicyDecision,
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPolicyFence,
|
||||
type SecurityPrincipal,
|
||||
} from '../../security/security';
|
||||
import type { RunStatus } from '../run';
|
||||
|
||||
export const RUN_MANUAL_RETRY_SCHEMA = 'qinglong/run-manual-retry@v1' as const;
|
||||
export const RUN_MANUAL_RETRY_STATUSES = ['accepted', 'existing'] as const;
|
||||
export const RUN_MANUAL_RETRY_SOURCE_STATUSES = [
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
] as const;
|
||||
export const RUN_MANUAL_RETRY_EXECUTOR_TYPES = [
|
||||
'local_process',
|
||||
'remote_worker',
|
||||
] as const;
|
||||
export const MAX_RUN_MANUAL_RETRY_AUTHENTICATION_AGE_MS = 5 * 60_000;
|
||||
|
||||
export type RunManualRetryStatus = (typeof RUN_MANUAL_RETRY_STATUSES)[number];
|
||||
export type RunManualRetrySourceStatus =
|
||||
(typeof RUN_MANUAL_RETRY_SOURCE_STATUSES)[number];
|
||||
export type RunManualRetryExecutorType =
|
||||
(typeof RUN_MANUAL_RETRY_EXECUTOR_TYPES)[number];
|
||||
export type RunManualRetryAllowedRole = Extract<
|
||||
ProjectRole,
|
||||
'owner' | 'admin' | 'operator'
|
||||
>;
|
||||
|
||||
export interface RunManualRetryRequestBody {
|
||||
readonly schema: typeof RUN_MANUAL_RETRY_SCHEMA;
|
||||
readonly mutationId: string;
|
||||
readonly expectedRunVersion: number;
|
||||
readonly expectedRunStatus: RunManualRetrySourceStatus;
|
||||
}
|
||||
|
||||
export interface RunManualRetryCommand {
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly mutationId: string;
|
||||
readonly expectedRunVersion: number;
|
||||
readonly expectedRunStatus: RunManualRetrySourceStatus;
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly createdEventId: string;
|
||||
readonly queuedEventId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly requestId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
}
|
||||
|
||||
export interface RunManualRetryResult {
|
||||
readonly status: RunManualRetryStatus;
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly sourceRunStatus: RunManualRetrySourceStatus;
|
||||
readonly sourceRunVersion: number;
|
||||
readonly runId: string;
|
||||
readonly retryOfRunId: string;
|
||||
readonly taskId: string;
|
||||
readonly taskRevision: string;
|
||||
readonly attemptId: string;
|
||||
readonly runStatus: 'queued';
|
||||
readonly runVersion: 2;
|
||||
readonly eventSequence: 2;
|
||||
readonly executorType: RunManualRetryExecutorType;
|
||||
readonly executionRevisionDigest: string;
|
||||
readonly createdAtMs: number;
|
||||
}
|
||||
|
||||
export interface RunManualRetryResponseBody extends RunManualRetryResult {
|
||||
readonly schema: typeof RUN_MANUAL_RETRY_SCHEMA;
|
||||
}
|
||||
|
||||
export interface RunManualRetryRepository {
|
||||
retryRun(
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
): Promise<Readonly<RunManualRetryResult>>;
|
||||
}
|
||||
|
||||
export type RunManualRetryFenceReason =
|
||||
| 'authorization_changed'
|
||||
| 'authentication_changed'
|
||||
| 'source_changed'
|
||||
| 'source_not_terminal'
|
||||
| 'source_not_retryable'
|
||||
| 'task_disabled'
|
||||
| 'mutation_conflict';
|
||||
|
||||
export class InvalidRunManualRetryError extends TypeError {
|
||||
readonly code = 'RUN_MANUAL_RETRY_INVALID';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Run manual retry is invalid: ${message}`);
|
||||
this.name = 'InvalidRunManualRetryError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RunManualRetryNotFoundError extends Error {
|
||||
readonly code = 'RUN_MANUAL_RETRY_NOT_FOUND';
|
||||
|
||||
constructor() {
|
||||
super('Run manual retry target does not exist');
|
||||
this.name = 'RunManualRetryNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RunManualRetryFenceRejectedError extends Error {
|
||||
readonly code = 'RUN_MANUAL_RETRY_FENCE_REJECTED';
|
||||
|
||||
constructor(readonly reason: RunManualRetryFenceReason) {
|
||||
super(`Run manual retry fence rejected: ${reason}`);
|
||||
this.name = 'RunManualRetryFenceRejectedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RunManualRetryRateLimitedError extends Error {
|
||||
readonly code = 'RUN_MANUAL_RETRY_RATE_LIMITED';
|
||||
|
||||
constructor(readonly retryAfterMs: number) {
|
||||
if (!Number.isSafeInteger(retryAfterMs) || retryAfterMs < 1) {
|
||||
throw new InvalidRunManualRetryError('retry delay is invalid');
|
||||
}
|
||||
super('Run manual retry rate limit is exhausted');
|
||||
this.name = 'RunManualRetryRateLimitedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RunManualRetryUnavailableError extends Error {
|
||||
readonly code = 'RUN_MANUAL_RETRY_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Run manual retry is unavailable', options);
|
||||
this.name = 'RunManualRetryUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): void {
|
||||
const actual = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
actual.length !== canonical.length ||
|
||||
actual.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
throw new InvalidRunManualRetryError('shape is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function identifier(value: unknown, name: string): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) {
|
||||
throw new InvalidRunManualRetryError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function uuid(value: unknown, name: string): string {
|
||||
if (typeof value !== 'string' || !UUID_PATTERN.test(value)) {
|
||||
throw new InvalidRunManualRetryError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function version(value: unknown, name: string): number {
|
||||
if (
|
||||
typeof value !== 'number' ||
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < 1 ||
|
||||
value > 2_147_483_647
|
||||
) {
|
||||
throw new InvalidRunManualRetryError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function timestamp(value: unknown, name: string): number {
|
||||
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
|
||||
throw new InvalidRunManualRetryError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function sourceStatus(value: unknown): RunManualRetrySourceStatus {
|
||||
if (
|
||||
!RUN_MANUAL_RETRY_SOURCE_STATUSES.includes(
|
||||
value as RunManualRetrySourceStatus,
|
||||
)
|
||||
) {
|
||||
throw new InvalidRunManualRetryError('source Run status is invalid');
|
||||
}
|
||||
return value as RunManualRetrySourceStatus;
|
||||
}
|
||||
|
||||
export function parseRunManualRetryRequestBody(
|
||||
value: unknown,
|
||||
): Readonly<RunManualRetryRequestBody> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new InvalidRunManualRetryError('request body is invalid');
|
||||
}
|
||||
exactKeys(value, [
|
||||
'schema',
|
||||
'mutationId',
|
||||
'expectedRunVersion',
|
||||
'expectedRunStatus',
|
||||
]);
|
||||
const body = value as Record<string, unknown>;
|
||||
if (body.schema !== RUN_MANUAL_RETRY_SCHEMA) {
|
||||
throw new InvalidRunManualRetryError('schema is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
schema: RUN_MANUAL_RETRY_SCHEMA,
|
||||
mutationId: uuid(body.mutationId, 'mutationId'),
|
||||
expectedRunVersion: version(
|
||||
body.expectedRunVersion,
|
||||
'expected Run version',
|
||||
),
|
||||
expectedRunStatus: sourceStatus(body.expectedRunStatus),
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeRunManualRetryCommand(
|
||||
value: RunManualRetryCommand,
|
||||
): Readonly<RunManualRetryCommand> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new InvalidRunManualRetryError('command is invalid');
|
||||
}
|
||||
exactKeys(value, [
|
||||
'projectId',
|
||||
'sourceRunId',
|
||||
'mutationId',
|
||||
'expectedRunVersion',
|
||||
'expectedRunStatus',
|
||||
'runId',
|
||||
'attemptId',
|
||||
'createdEventId',
|
||||
'queuedEventId',
|
||||
'auditEventId',
|
||||
'requestId',
|
||||
'principal',
|
||||
'policyFence',
|
||||
]);
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
let fence: Readonly<SecurityPolicyFence> | null;
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(
|
||||
value.principal,
|
||||
value.principal.authenticatedAtMs,
|
||||
);
|
||||
fence = normalizeSecurityPolicyDecision({
|
||||
effect: 'allow',
|
||||
reasons: ['role_grant'],
|
||||
fence: value.policyFence,
|
||||
}).fence;
|
||||
} catch {
|
||||
throw new InvalidRunManualRetryError('authorization authority is invalid');
|
||||
}
|
||||
if (
|
||||
principal.subject.type !== 'user' ||
|
||||
!['multi_factor', 'hardware', 'local_console'].includes(
|
||||
principal.assurance,
|
||||
) ||
|
||||
!fence ||
|
||||
fence.bindingVersion === null
|
||||
) {
|
||||
throw new InvalidRunManualRetryError(
|
||||
'strong User authorization authority is incomplete',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
projectId: identifier(value.projectId, 'projectId'),
|
||||
sourceRunId: identifier(value.sourceRunId, 'sourceRunId'),
|
||||
mutationId: uuid(value.mutationId, 'mutationId'),
|
||||
expectedRunVersion: version(
|
||||
value.expectedRunVersion,
|
||||
'expected Run version',
|
||||
),
|
||||
expectedRunStatus: sourceStatus(value.expectedRunStatus),
|
||||
runId: uuid(value.runId, 'runId'),
|
||||
attemptId: uuid(value.attemptId, 'attemptId'),
|
||||
createdEventId: uuid(value.createdEventId, 'createdEventId'),
|
||||
queuedEventId: uuid(value.queuedEventId, 'queuedEventId'),
|
||||
auditEventId: uuid(value.auditEventId, 'auditEventId'),
|
||||
requestId: identifier(value.requestId, 'requestId'),
|
||||
principal,
|
||||
policyFence: fence,
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeRunManualRetryResult(
|
||||
value: RunManualRetryResult,
|
||||
): Readonly<RunManualRetryResult> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new InvalidRunManualRetryError('result is invalid');
|
||||
}
|
||||
exactKeys(value, [
|
||||
'status',
|
||||
'projectId',
|
||||
'sourceRunId',
|
||||
'sourceRunStatus',
|
||||
'sourceRunVersion',
|
||||
'runId',
|
||||
'retryOfRunId',
|
||||
'taskId',
|
||||
'taskRevision',
|
||||
'attemptId',
|
||||
'runStatus',
|
||||
'runVersion',
|
||||
'eventSequence',
|
||||
'executorType',
|
||||
'executionRevisionDigest',
|
||||
'createdAtMs',
|
||||
]);
|
||||
if (
|
||||
!RUN_MANUAL_RETRY_STATUSES.includes(value.status) ||
|
||||
value.retryOfRunId !== value.sourceRunId ||
|
||||
value.runId === value.sourceRunId ||
|
||||
value.runStatus !== 'queued' ||
|
||||
value.runVersion !== 2 ||
|
||||
value.eventSequence !== 2 ||
|
||||
!RUN_MANUAL_RETRY_EXECUTOR_TYPES.includes(value.executorType) ||
|
||||
!DIGEST_PATTERN.test(value.executionRevisionDigest)
|
||||
) {
|
||||
throw new InvalidRunManualRetryError('result state is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
status: value.status,
|
||||
projectId: identifier(value.projectId, 'projectId'),
|
||||
sourceRunId: identifier(value.sourceRunId, 'sourceRunId'),
|
||||
sourceRunStatus: sourceStatus(value.sourceRunStatus),
|
||||
sourceRunVersion: version(value.sourceRunVersion, 'source Run version'),
|
||||
runId: uuid(value.runId, 'runId'),
|
||||
retryOfRunId: identifier(value.retryOfRunId, 'retryOfRunId'),
|
||||
taskId: identifier(value.taskId, 'taskId'),
|
||||
taskRevision: identifier(value.taskRevision, 'taskRevision'),
|
||||
attemptId: uuid(value.attemptId, 'attemptId'),
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: value.executorType,
|
||||
executionRevisionDigest: value.executionRevisionDigest,
|
||||
createdAtMs: timestamp(value.createdAtMs, 'createdAtMs'),
|
||||
});
|
||||
}
|
||||
|
||||
export function createRunManualRetryResponseBody(
|
||||
value: RunManualRetryResult,
|
||||
): Readonly<RunManualRetryResponseBody> {
|
||||
return Object.freeze({
|
||||
schema: RUN_MANUAL_RETRY_SCHEMA,
|
||||
...normalizeRunManualRetryResult(value),
|
||||
});
|
||||
}
|
||||
|
||||
export function isRunManualRetrySourceStatus(
|
||||
value: RunStatus,
|
||||
): value is RunManualRetrySourceStatus {
|
||||
return RUN_MANUAL_RETRY_SOURCE_STATUSES.includes(
|
||||
value as RunManualRetrySourceStatus,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
InvalidRunManualRetryError,
|
||||
RUN_MANUAL_RETRY_SCHEMA,
|
||||
RunManualRetryRateLimitedError,
|
||||
createRunManualRetryResponseBody,
|
||||
normalizeRunManualRetryCommand,
|
||||
parseRunManualRetryRequestBody,
|
||||
} = require('../dist/run/manual-retry/runManualRetry.js');
|
||||
|
||||
const IDS = Object.freeze({
|
||||
mutationId: '019f9000-0000-4000-8000-000000000001',
|
||||
runId: '019f9000-0000-4000-8000-000000000002',
|
||||
attemptId: '019f9000-0000-4000-8000-000000000003',
|
||||
createdEventId: '019f9000-0000-4000-8000-000000000004',
|
||||
queuedEventId: '019f9000-0000-4000-8000-000000000005',
|
||||
auditEventId: '019f9000-0000-4000-8000-000000000006',
|
||||
});
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
mutationId: IDS.mutationId,
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
runId: IDS.runId,
|
||||
attemptId: IDS.attemptId,
|
||||
createdEventId: IDS.createdEventId,
|
||||
queuedEventId: IDS.queuedEventId,
|
||||
auditEventId: IDS.auditEventId,
|
||||
requestId: 'run-retry-request-1',
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'local_console:proof-1',
|
||||
authenticatedAtMs: 10_000,
|
||||
expiresAtMs: 20_000,
|
||||
assurance: 'local_console',
|
||||
},
|
||||
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('normalizes one strongly authorized terminal Run retry command', () => {
|
||||
const request = parseRunManualRetryRequestBody({
|
||||
schema: RUN_MANUAL_RETRY_SCHEMA,
|
||||
mutationId: IDS.mutationId,
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'timed_out',
|
||||
});
|
||||
assert.equal(request.expectedRunStatus, 'timed_out');
|
||||
const normalized = normalizeRunManualRetryCommand(command());
|
||||
assert.equal(normalized.principal.assurance, 'local_console');
|
||||
assert.equal(Object.isFrozen(normalized), true);
|
||||
});
|
||||
|
||||
test('rejects reopened states, weak principals and widened commands', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRunManualRetryRequestBody({
|
||||
schema: RUN_MANUAL_RETRY_SCHEMA,
|
||||
mutationId: IDS.mutationId,
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'lost',
|
||||
}),
|
||||
InvalidRunManualRetryError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeRunManualRetryCommand(
|
||||
command({
|
||||
principal: { ...command().principal, assurance: 'single_factor' },
|
||||
}),
|
||||
),
|
||||
/strong User/,
|
||||
);
|
||||
assert.throws(
|
||||
() => normalizeRunManualRetryCommand({ ...command(), hidden: true }),
|
||||
InvalidRunManualRetryError,
|
||||
);
|
||||
});
|
||||
|
||||
test('publishes only the bounded new-Run retry result', () => {
|
||||
assert.deepEqual(
|
||||
createRunManualRetryResponseBody({
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: IDS.runId,
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `v1:${'a'.repeat(64)}`,
|
||||
attemptId: IDS.attemptId,
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'local_process',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: 11_000,
|
||||
}),
|
||||
{
|
||||
schema: RUN_MANUAL_RETRY_SCHEMA,
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: IDS.runId,
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `v1:${'a'.repeat(64)}`,
|
||||
attemptId: IDS.attemptId,
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'local_process',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: 11_000,
|
||||
},
|
||||
);
|
||||
assert.equal(new RunManualRetryRateLimitedError(250).retryAfterMs, 250);
|
||||
assert.throws(() => new RunManualRetryRateLimitedError(0));
|
||||
});
|
||||
Reference in New Issue
Block a user