mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add strong local manual run retry
This commit is contained in:
whyour
@@ -60,6 +60,11 @@
|
||||
"require": "./dist/automation-management/taskDefinitionCommand.js",
|
||||
"default": "./dist/automation-management/taskDefinitionCommand.js"
|
||||
},
|
||||
"./run-retry-command": {
|
||||
"types": "./dist/run-management/runRetryCommand.d.ts",
|
||||
"require": "./dist/run-management/runRetryCommand.js",
|
||||
"default": "./dist/run-management/runRetryCommand.js"
|
||||
},
|
||||
"./plugin-package-workflow-command": {
|
||||
"types": "./dist/plugin-package/pluginPackageWorkflowCommand.d.ts",
|
||||
"require": "./dist/plugin-package/pluginPackageWorkflowCommand.js",
|
||||
@@ -125,6 +130,7 @@
|
||||
"ql3-ai-feature": "dist/ai-management/aiFeatureCli.js",
|
||||
"ql3-secret": "dist/security-management/secretCli.js",
|
||||
"ql3-task": "dist/automation-management/taskDefinitionCli.js",
|
||||
"ql3-run": "dist/run-management/runRetryCli.js",
|
||||
"ql3-workflow": "dist/plugin-package/pluginPackageWorkflowCli.js",
|
||||
"ql3-trigger": "dist/automation-management/triggerCli.js",
|
||||
"ql3-policy": "dist/security-management/projectPolicyCli.js",
|
||||
|
||||
@@ -84,6 +84,12 @@ export const QINGLONG3_PRODUCT_COMMANDS: readonly QingLong3ProductCommandDefinit
|
||||
target: 'automation-management/taskDefinitionCli.js',
|
||||
description: 'manage Task definitions',
|
||||
}),
|
||||
Object.freeze({
|
||||
name: 'run',
|
||||
binary: 'ql3-run',
|
||||
target: 'run-management/runRetryCli.js',
|
||||
description: 'retry terminal Runs under strong local authentication',
|
||||
}),
|
||||
Object.freeze({
|
||||
name: 'trigger',
|
||||
binary: 'ql3-trigger',
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { runLocalRunRetryCommandFile } from './runRetryCommand';
|
||||
|
||||
const USAGE =
|
||||
'Usage: ql3-run retry --command-file /absolute/private-command.json';
|
||||
|
||||
async function main(argv: readonly string[]): Promise<void> {
|
||||
if (argv.length === 1 && (argv[0] === '--help' || argv[0] === '-h')) {
|
||||
process.stdout.write(`${USAGE}\n`);
|
||||
return;
|
||||
}
|
||||
const commandFilePath = argv[2];
|
||||
if (
|
||||
argv.length !== 3 ||
|
||||
argv[0] !== 'retry' ||
|
||||
argv[1] !== '--command-file' ||
|
||||
commandFilePath === undefined
|
||||
) {
|
||||
process.stderr.write(
|
||||
`${JSON.stringify({
|
||||
code: 'LOCAL_RUN_RETRY_CLI_USAGE_INVALID',
|
||||
message: USAGE,
|
||||
})}\n`,
|
||||
);
|
||||
process.exitCode = 64;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await runLocalRunRetryCommandFile(commandFilePath);
|
||||
process.stdout.write(`${JSON.stringify(result)}\n`);
|
||||
} catch (error) {
|
||||
const candidate = error as {
|
||||
readonly code?: unknown;
|
||||
readonly name?: unknown;
|
||||
readonly message?: unknown;
|
||||
readonly retryAfterMs?: unknown;
|
||||
};
|
||||
process.stderr.write(
|
||||
`${JSON.stringify({
|
||||
code:
|
||||
typeof candidate.code === 'string'
|
||||
? candidate.code
|
||||
: 'LOCAL_RUN_RETRY_CLI_FAILED',
|
||||
name: typeof candidate.name === 'string' ? candidate.name : 'Error',
|
||||
message:
|
||||
typeof candidate.message === 'string'
|
||||
? candidate.message
|
||||
: 'Local Run retry command failed',
|
||||
...(Number.isSafeInteger(candidate.retryAfterMs)
|
||||
? { retryAfterMs: candidate.retryAfterMs }
|
||||
: {}),
|
||||
})}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
void main(process.argv.slice(2));
|
||||
@@ -0,0 +1,500 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
PrivateLocalCommandFileError,
|
||||
readPrivateLocalCommandFile,
|
||||
} from '@qinglong/local-command-file';
|
||||
import {
|
||||
AuthenticatedLocalCommandAuthenticationError,
|
||||
establishAuthenticatedLocalCommand,
|
||||
type AuthenticatedLocalCommand,
|
||||
} from '@qinglong/local-owner-console/authenticated-command';
|
||||
import {
|
||||
LocalSqliteAuthenticatedManagementFenceError,
|
||||
type LocalSqliteAuthenticatedUserCredentialFence,
|
||||
} from '@qinglong/local-sqlite/authenticated-management';
|
||||
import {
|
||||
openLocalSqliteRunManagementDatabase,
|
||||
type LocalSqliteRunManagementDatabase,
|
||||
} from '@qinglong/local-sqlite/run-management';
|
||||
import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
|
||||
import {
|
||||
RUN_MANUAL_RETRY_SCHEMA,
|
||||
RunManualRetryFenceRejectedError,
|
||||
RunManualRetryNotFoundError,
|
||||
RunManualRetryRateLimitedError,
|
||||
RunManualRetryUnavailableError,
|
||||
parseRunManualRetryRequestBody,
|
||||
type RunManualRetryResult,
|
||||
type RunManualRetrySourceStatus,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
import {
|
||||
normalizeSecurityAuditRecord,
|
||||
type SecurityAuditRecord,
|
||||
} from '@qinglong/runtime-core/security-audit';
|
||||
|
||||
const MAX_PATH_BYTES = 4_096;
|
||||
const UUID_V4_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
|
||||
export interface LocalRunRetryCommandOptions {
|
||||
readonly deploymentRoot: string;
|
||||
readonly databasePath: string;
|
||||
readonly profile: 'edge' | 'standalone';
|
||||
readonly ownerPepperKeyringDirectory: string;
|
||||
readonly credentialFilePath: string;
|
||||
readonly busyTimeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface LocalRunRetryCommand {
|
||||
readonly schemaVersion: 1;
|
||||
readonly operation: 'run.retry';
|
||||
readonly options: LocalRunRetryCommandOptions;
|
||||
readonly request: Readonly<{
|
||||
projectId: string;
|
||||
sourceRunId: string;
|
||||
mutationId: string;
|
||||
expectedRunVersion: number;
|
||||
expectedRunStatus: RunManualRetrySourceStatus;
|
||||
requestId: string;
|
||||
auditEventId: string;
|
||||
failureAuditEventId: string;
|
||||
occurredAtMs: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface LocalRunRetryCommandResult {
|
||||
readonly schemaVersion: 1;
|
||||
readonly operation: 'run.retry';
|
||||
readonly retry: Readonly<RunManualRetryResult>;
|
||||
}
|
||||
|
||||
export interface LocalRunRetryCommandRunner {
|
||||
run(commandFilePath: string): Promise<Readonly<LocalRunRetryCommandResult>>;
|
||||
}
|
||||
|
||||
export interface LocalRunRetryCommandRunnerDependencies {
|
||||
readonly openDatabase: typeof openLocalSqliteRunManagementDatabase;
|
||||
readonly authenticate: typeof establishAuthenticatedLocalCommand;
|
||||
readonly now: () => number;
|
||||
readonly randomUuid: () => string;
|
||||
}
|
||||
|
||||
export class LocalRunRetryCommandConfigurationError extends TypeError {
|
||||
readonly code = 'LOCAL_RUN_RETRY_COMMAND_CONFIGURATION_INVALID';
|
||||
|
||||
constructor(message: string, options?: ErrorOptions) {
|
||||
super(
|
||||
`Local Run retry command configuration is invalid: ${message}`,
|
||||
options,
|
||||
);
|
||||
this.name = 'LocalRunRetryCommandConfigurationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalRunRetryCommandAuthorizationError extends Error {
|
||||
readonly code = 'LOCAL_RUN_RETRY_COMMAND_AUTHORIZATION_REJECTED';
|
||||
|
||||
constructor() {
|
||||
super('Local Run retry command authorization was rejected');
|
||||
this.name = 'LocalRunRetryCommandAuthorizationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalRunRetryCommandUnavailableError extends Error {
|
||||
readonly code = 'LOCAL_RUN_RETRY_COMMAND_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Local Run retry command is unavailable', options);
|
||||
this.name = 'LocalRunRetryCommandUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
function exactRecord(
|
||||
value: unknown,
|
||||
required: readonly string[],
|
||||
optional: readonly string[],
|
||||
label: string,
|
||||
): asserts value is Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new LocalRunRetryCommandConfigurationError(
|
||||
`${label} must be an object`,
|
||||
);
|
||||
}
|
||||
const keys = Object.keys(value).sort();
|
||||
const allowed = new Set([...required, ...optional]);
|
||||
if (
|
||||
required.some((key) => !keys.includes(key)) ||
|
||||
keys.some((key) => !allowed.has(key))
|
||||
) {
|
||||
throw new LocalRunRetryCommandConfigurationError(
|
||||
`${label} shape is invalid`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function boundedPath(value: unknown, label: string): string {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
Buffer.byteLength(value, 'utf8') > MAX_PATH_BYTES ||
|
||||
value.includes('\0') ||
|
||||
!path.isAbsolute(value) ||
|
||||
path.normalize(value) !== value ||
|
||||
path.parse(value).root === value
|
||||
) {
|
||||
throw new LocalRunRetryCommandConfigurationError(
|
||||
`${label} must be a normalized bounded absolute non-root path`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function descendant(root: string, value: string, label: string): void {
|
||||
const relative = path.relative(root, value);
|
||||
if (
|
||||
relative.length === 0 ||
|
||||
relative === '..' ||
|
||||
relative.startsWith(`..${path.sep}`) ||
|
||||
path.isAbsolute(relative)
|
||||
) {
|
||||
throw new LocalRunRetryCommandConfigurationError(
|
||||
`${label} must be a descendant of deploymentRoot`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function identifier(value: unknown, label: string): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) {
|
||||
throw new LocalRunRetryCommandConfigurationError(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function uuid(value: unknown, label: string): string {
|
||||
if (typeof value !== 'string' || !UUID_V4_PATTERN.test(value)) {
|
||||
throw new LocalRunRetryCommandConfigurationError(`${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function options(value: unknown): Readonly<LocalRunRetryCommandOptions> {
|
||||
exactRecord(
|
||||
value,
|
||||
[
|
||||
'deploymentRoot',
|
||||
'databasePath',
|
||||
'profile',
|
||||
'ownerPepperKeyringDirectory',
|
||||
'credentialFilePath',
|
||||
],
|
||||
['busyTimeoutMs'],
|
||||
'options',
|
||||
);
|
||||
const deploymentRoot = boundedPath(value.deploymentRoot, 'deploymentRoot');
|
||||
const databasePath = boundedPath(value.databasePath, 'databasePath');
|
||||
const ownerPepperKeyringDirectory = boundedPath(
|
||||
value.ownerPepperKeyringDirectory,
|
||||
'ownerPepperKeyringDirectory',
|
||||
);
|
||||
const credentialFilePath = boundedPath(
|
||||
value.credentialFilePath,
|
||||
'credentialFilePath',
|
||||
);
|
||||
for (const [target, label] of [
|
||||
[databasePath, 'databasePath'],
|
||||
[ownerPepperKeyringDirectory, 'ownerPepperKeyringDirectory'],
|
||||
[credentialFilePath, 'credentialFilePath'],
|
||||
] as const) {
|
||||
descendant(deploymentRoot, target, label);
|
||||
}
|
||||
if (
|
||||
new Set([databasePath, ownerPepperKeyringDirectory, credentialFilePath])
|
||||
.size !== 3 ||
|
||||
(value.profile !== 'edge' && value.profile !== 'standalone') ||
|
||||
(value.busyTimeoutMs !== undefined &&
|
||||
(!Number.isSafeInteger(value.busyTimeoutMs) ||
|
||||
(value.busyTimeoutMs as number) < 1 ||
|
||||
(value.busyTimeoutMs as number) > 60_000))
|
||||
) {
|
||||
throw new LocalRunRetryCommandConfigurationError('options are invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
deploymentRoot,
|
||||
databasePath,
|
||||
profile: value.profile,
|
||||
ownerPepperKeyringDirectory,
|
||||
credentialFilePath,
|
||||
...(value.busyTimeoutMs === undefined
|
||||
? {}
|
||||
: { busyTimeoutMs: value.busyTimeoutMs as number }),
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeCommand(value: unknown): Readonly<LocalRunRetryCommand> {
|
||||
exactRecord(
|
||||
value,
|
||||
['schemaVersion', 'operation', 'options', 'request'],
|
||||
[],
|
||||
'command',
|
||||
);
|
||||
if (value.schemaVersion !== 1 || value.operation !== 'run.retry') {
|
||||
throw new LocalRunRetryCommandConfigurationError(
|
||||
'schemaVersion or operation is invalid',
|
||||
);
|
||||
}
|
||||
exactRecord(
|
||||
value.request,
|
||||
[
|
||||
'projectId',
|
||||
'sourceRunId',
|
||||
'mutationId',
|
||||
'expectedRunVersion',
|
||||
'expectedRunStatus',
|
||||
'requestId',
|
||||
'auditEventId',
|
||||
'failureAuditEventId',
|
||||
'occurredAtMs',
|
||||
],
|
||||
[],
|
||||
'request',
|
||||
);
|
||||
let retry;
|
||||
try {
|
||||
retry = parseRunManualRetryRequestBody({
|
||||
schema: RUN_MANUAL_RETRY_SCHEMA,
|
||||
mutationId: value.request.mutationId,
|
||||
expectedRunVersion: value.request.expectedRunVersion,
|
||||
expectedRunStatus: value.request.expectedRunStatus,
|
||||
});
|
||||
} catch (error) {
|
||||
throw new LocalRunRetryCommandConfigurationError(
|
||||
'retry request is invalid',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(value.request.occurredAtMs) ||
|
||||
(value.request.occurredAtMs as number) < 0
|
||||
) {
|
||||
throw new LocalRunRetryCommandConfigurationError(
|
||||
'request occurredAtMs is invalid',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.retry',
|
||||
options: options(value.options),
|
||||
request: Object.freeze({
|
||||
projectId: identifier(value.request.projectId, 'projectId'),
|
||||
sourceRunId: identifier(value.request.sourceRunId, 'sourceRunId'),
|
||||
mutationId: retry.mutationId,
|
||||
expectedRunVersion: retry.expectedRunVersion,
|
||||
expectedRunStatus: retry.expectedRunStatus,
|
||||
requestId: identifier(value.request.requestId, 'requestId'),
|
||||
auditEventId: uuid(value.request.auditEventId, 'auditEventId'),
|
||||
failureAuditEventId: uuid(
|
||||
value.request.failureAuditEventId,
|
||||
'failureAuditEventId',
|
||||
),
|
||||
occurredAtMs: value.request.occurredAtMs as number,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function readCommandFile(
|
||||
commandFilePath: string,
|
||||
): Readonly<LocalRunRetryCommand> {
|
||||
try {
|
||||
return normalizeCommand(readPrivateLocalCommandFile(commandFilePath));
|
||||
} catch (error) {
|
||||
if (error instanceof LocalRunRetryCommandConfigurationError) throw error;
|
||||
if (error instanceof PrivateLocalCommandFileError) {
|
||||
throw new LocalRunRetryCommandConfigurationError(
|
||||
'private command file cannot be read',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
throw new LocalRunRetryCommandConfigurationError(
|
||||
'command file is invalid',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function dependencies(
|
||||
value: LocalRunRetryCommandRunnerDependencies,
|
||||
): LocalRunRetryCommandRunnerDependencies {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).sort().join('\0') !==
|
||||
['authenticate', 'now', 'openDatabase', 'randomUuid'].sort().join('\0') ||
|
||||
typeof value.openDatabase !== 'function' ||
|
||||
typeof value.authenticate !== 'function' ||
|
||||
typeof value.now !== 'function' ||
|
||||
typeof value.randomUuid !== 'function'
|
||||
) {
|
||||
throw new LocalRunRetryCommandConfigurationError(
|
||||
'runner dependencies are invalid',
|
||||
);
|
||||
}
|
||||
return Object.freeze({ ...value });
|
||||
}
|
||||
|
||||
function failureReason(error: unknown): string {
|
||||
if (error instanceof RunManualRetryRateLimitedError) {
|
||||
return 'run_retry_rate_limited';
|
||||
}
|
||||
if (error instanceof RunManualRetryNotFoundError) return 'run_not_found';
|
||||
if (
|
||||
error instanceof LocalRunRetryCommandAuthorizationError ||
|
||||
error instanceof RunManualRetryFenceRejectedError ||
|
||||
error instanceof LocalSqliteAuthenticatedManagementFenceError
|
||||
) {
|
||||
return 'run_retry_fence_rejected';
|
||||
}
|
||||
return 'run_retry_unavailable';
|
||||
}
|
||||
|
||||
function failureAudit(
|
||||
command: Readonly<LocalRunRetryCommand>,
|
||||
authenticated: Readonly<AuthenticatedLocalCommand> | undefined,
|
||||
error: unknown,
|
||||
): Readonly<SecurityAuditRecord> {
|
||||
const unauthenticated = authenticated === undefined;
|
||||
return normalizeSecurityAuditRecord({
|
||||
eventId: command.request.failureAuditEventId,
|
||||
requestId: command.request.requestId,
|
||||
operationId: 'run.retry',
|
||||
projectId: command.request.projectId,
|
||||
subject: authenticated?.principal.subject ?? null,
|
||||
authenticationId: authenticated?.principal.authenticationId ?? null,
|
||||
outcome: unauthenticated ? 'authentication_rejected' : 'denied',
|
||||
reasons: [
|
||||
unauthenticated ? 'local_console_required' : failureReason(error),
|
||||
],
|
||||
fence: null,
|
||||
occurredAtMs: command.request.occurredAtMs,
|
||||
});
|
||||
}
|
||||
|
||||
export function createLocalRunRetryCommandRunner(
|
||||
candidateDependencies: LocalRunRetryCommandRunnerDependencies = {
|
||||
openDatabase: openLocalSqliteRunManagementDatabase,
|
||||
authenticate: establishAuthenticatedLocalCommand,
|
||||
now: Date.now,
|
||||
randomUuid: randomUUID,
|
||||
},
|
||||
): Readonly<LocalRunRetryCommandRunner> {
|
||||
const adapters = dependencies(candidateDependencies);
|
||||
return Object.freeze({
|
||||
async run(commandFilePath: string) {
|
||||
const command = readCommandFile(commandFilePath);
|
||||
const nowMs = adapters.now();
|
||||
if (
|
||||
!Number.isSafeInteger(nowMs) ||
|
||||
nowMs < command.request.occurredAtMs ||
|
||||
nowMs - command.request.occurredAtMs > 5 * 60_000
|
||||
) {
|
||||
throw new LocalRunRetryCommandConfigurationError(
|
||||
'request time is outside the accepted window',
|
||||
);
|
||||
}
|
||||
const database = await adapters.openDatabase({
|
||||
databasePath: command.options.databasePath,
|
||||
profile: command.options.profile,
|
||||
...(command.options.busyTimeoutMs === undefined
|
||||
? {}
|
||||
: { busyTimeoutMs: command.options.busyTimeoutMs }),
|
||||
});
|
||||
let authenticated: Readonly<AuthenticatedLocalCommand> | undefined;
|
||||
try {
|
||||
try {
|
||||
authenticated = await adapters.authenticate(database, {
|
||||
deploymentRoot: command.options.deploymentRoot,
|
||||
databasePath: command.options.databasePath,
|
||||
ownerPepperKeyringDirectory:
|
||||
command.options.ownerPepperKeyringDirectory,
|
||||
credentialFilePath: command.options.credentialFilePath,
|
||||
authenticationNamespace: 'local_run_retry',
|
||||
});
|
||||
database.activateUserCredentialFence(
|
||||
authenticated.databaseFence as Readonly<LocalSqliteAuthenticatedUserCredentialFence>,
|
||||
);
|
||||
const decision = await new ProjectPolicyEngine(
|
||||
database.projectPolicy,
|
||||
).authorize(
|
||||
authenticated.principal,
|
||||
command.request.projectId,
|
||||
'run.retry',
|
||||
);
|
||||
if (
|
||||
decision.effect !== 'allow' ||
|
||||
!decision.fence ||
|
||||
decision.fence.bindingVersion === null
|
||||
) {
|
||||
throw new LocalRunRetryCommandAuthorizationError();
|
||||
}
|
||||
await authenticated.confirm();
|
||||
const result = await database.runManualRetry.retryRun({
|
||||
projectId: command.request.projectId,
|
||||
sourceRunId: command.request.sourceRunId,
|
||||
mutationId: command.request.mutationId,
|
||||
expectedRunVersion: command.request.expectedRunVersion,
|
||||
expectedRunStatus: command.request.expectedRunStatus,
|
||||
runId: adapters.randomUuid(),
|
||||
attemptId: adapters.randomUuid(),
|
||||
createdEventId: adapters.randomUuid(),
|
||||
queuedEventId: adapters.randomUuid(),
|
||||
auditEventId: command.request.auditEventId,
|
||||
requestId: command.request.requestId,
|
||||
principal: authenticated.principal,
|
||||
policyFence: decision.fence,
|
||||
});
|
||||
return Object.freeze({
|
||||
schemaVersion: 1 as const,
|
||||
operation: 'run.retry' as const,
|
||||
retry: result,
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await database.securityAudit.record(
|
||||
failureAudit(command, authenticated, error),
|
||||
);
|
||||
} catch (auditError) {
|
||||
throw new LocalRunRetryCommandUnavailableError({
|
||||
cause: auditError,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
await database.close();
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function runLocalRunRetryCommandFile(
|
||||
commandFilePath: string,
|
||||
): Promise<Readonly<LocalRunRetryCommandResult>> {
|
||||
return createLocalRunRetryCommandRunner().run(commandFilePath);
|
||||
}
|
||||
|
||||
export function isLocalRunRetryCommandError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof LocalRunRetryCommandConfigurationError ||
|
||||
error instanceof LocalRunRetryCommandAuthorizationError ||
|
||||
error instanceof LocalRunRetryCommandUnavailableError ||
|
||||
error instanceof AuthenticatedLocalCommandAuthenticationError ||
|
||||
error instanceof LocalSqliteAuthenticatedManagementFenceError ||
|
||||
error instanceof RunManualRetryNotFoundError ||
|
||||
error instanceof RunManualRetryFenceRejectedError ||
|
||||
error instanceof RunManualRetryRateLimitedError ||
|
||||
error instanceof RunManualRetryUnavailableError
|
||||
);
|
||||
}
|
||||
@@ -32,7 +32,7 @@ function runCli(args) {
|
||||
|
||||
test('catalog maps every product subcommand to an existing same-package binary', () => {
|
||||
assert.equal(manifest.bin.ql3, 'dist/product-cli/cli.js');
|
||||
assert.equal(QINGLONG3_PRODUCT_COMMANDS.length, 20);
|
||||
assert.equal(QINGLONG3_PRODUCT_COMMANDS.length, 21);
|
||||
assert.equal(
|
||||
new Set(QINGLONG3_PRODUCT_COMMANDS.map(({ name }) => name)).size,
|
||||
QINGLONG3_PRODUCT_COMMANDS.length,
|
||||
@@ -60,6 +60,7 @@ test('help and version are bounded installation-derived product facts', () => {
|
||||
const help = qingLong3ProductHelp();
|
||||
assert.match(help, /^Usage: ql3 <command> \[arguments\]/);
|
||||
assert.match(help, /\n task\s+manage Task definitions\n/);
|
||||
assert.match(help, /\n run\s+retry terminal Runs/);
|
||||
assert.match(help, /Root service mutation remains isolated/);
|
||||
assert.equal(help.includes('ql3-service-bridge '), false);
|
||||
assert.equal(loadQingLong3ProductVersion(moduleDirectory), manifest.version);
|
||||
@@ -153,6 +154,14 @@ test('product binary exposes help/version and delegates without a shell', () =>
|
||||
);
|
||||
assert.equal(delegatedHelp.stderr, '');
|
||||
|
||||
const delegatedRunHelp = runCli(['run', '--help']);
|
||||
assert.equal(delegatedRunHelp.status, 0);
|
||||
assert.equal(
|
||||
delegatedRunHelp.stdout.trim(),
|
||||
'Usage: ql3-run retry --command-file /absolute/private-command.json',
|
||||
);
|
||||
assert.equal(delegatedRunHelp.stderr, '');
|
||||
|
||||
const delegatedFailure = runCli(['task']);
|
||||
assert.equal(delegatedFailure.status, 64);
|
||||
assert.equal(delegatedFailure.stdout, '');
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const path = require('node:path');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
RunManualRetryNotFoundError,
|
||||
} = require('@qinglong/runtime-core/run-manual-retry');
|
||||
const {
|
||||
openLocalSqliteRuntimeDatabase,
|
||||
} = require('@qinglong/local-sqlite/runtime');
|
||||
const {
|
||||
runLocalRunRetryCommandFile,
|
||||
} = require('../dist/run-management/runRetryCommand.js');
|
||||
const {
|
||||
auditRows,
|
||||
localManagementFixture,
|
||||
writeCommand,
|
||||
} = require('./localManagementFixture.cjs');
|
||||
|
||||
function uuid(value) {
|
||||
return `019f9200-0000-4000-8000-${value.toString(16).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
async function createFailedSource(value) {
|
||||
const runtime = await openLocalSqliteRuntimeDatabase({
|
||||
databasePath: value.databasePath,
|
||||
profile: 'edge',
|
||||
});
|
||||
let started;
|
||||
try {
|
||||
const task = (
|
||||
await runtime.taskDefinitions.appendTaskDefinitionRevision({
|
||||
projectId: 'default',
|
||||
taskId: 'retry-product-task',
|
||||
expectedRevision: null,
|
||||
mutationId: uuid(1),
|
||||
name: 'Retry product task',
|
||||
kind: 'command',
|
||||
spec: {
|
||||
schema: 'qinglong/command@v1',
|
||||
config: {
|
||||
command: {
|
||||
kind: 'argv',
|
||||
file: '/bin/echo',
|
||||
args: ['retry-product'],
|
||||
},
|
||||
},
|
||||
},
|
||||
labels: {},
|
||||
enabled: true,
|
||||
occurredAtMs: value.now,
|
||||
})
|
||||
).definition;
|
||||
started = await (
|
||||
await runtime.taskStartRepository()
|
||||
).startTask({
|
||||
projectId: 'default',
|
||||
taskId: task.taskId,
|
||||
mutationId: uuid(2),
|
||||
expectedRevision: task.revision,
|
||||
expectedContentDigest: task.contentDigest,
|
||||
runId: uuid(3),
|
||||
attemptId: uuid(4),
|
||||
createdEventId: uuid(5),
|
||||
queuedEventId: uuid(6),
|
||||
subject: { type: 'user', id: 'automation-user' },
|
||||
policyFence: { projectVersion: 1, bindingVersion: 1 },
|
||||
});
|
||||
} finally {
|
||||
await runtime.close();
|
||||
}
|
||||
const database = new DatabaseSync(value.databasePath);
|
||||
try {
|
||||
database
|
||||
.prepare(
|
||||
`UPDATE "Runs"
|
||||
SET "status" = 'failed', "version" = 3, "event_sequence" = 3,
|
||||
"finished_at_ms" = ?, "error_code" = 'PRODUCT_TEST_FAILURE',
|
||||
"error_summary" = 'failed'
|
||||
WHERE "id" = ?`,
|
||||
)
|
||||
.run(value.now + 10, started.runId);
|
||||
database
|
||||
.prepare(
|
||||
`UPDATE "RunAttempts"
|
||||
SET "status" = 'failed', "finished_at_ms" = ?,
|
||||
"error_code" = 'PRODUCT_TEST_FAILURE',
|
||||
"error_summary" = 'failed'
|
||||
WHERE "id" = ?`,
|
||||
)
|
||||
.run(value.now + 10, started.attemptId);
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO "RunEvents" (
|
||||
"id", "run_id", "sequence", "type", "dedupe_key",
|
||||
"actor_type", "actor_id", "attempt_id", "payload",
|
||||
"created_at_ms"
|
||||
) VALUES (?, ?, 3, 'run.failed', 'product-test-failed', 'executor',
|
||||
'product-test', ?, '{}', ?)`,
|
||||
)
|
||||
.run(uuid(7), started.runId, started.attemptId, value.now + 10);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
return started;
|
||||
}
|
||||
|
||||
function request(value, sourceRunId, overrides = {}) {
|
||||
return {
|
||||
projectId: 'default',
|
||||
sourceRunId,
|
||||
mutationId: uuid(10),
|
||||
expectedRunVersion: 3,
|
||||
expectedRunStatus: 'failed',
|
||||
requestId: 'local-run-retry-product-1',
|
||||
auditEventId: uuid(11),
|
||||
failureAuditEventId: uuid(12),
|
||||
occurredAtMs: value.now,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('runs a strongly authenticated manual retry and exactly replays it', async (t) => {
|
||||
const value = await localManagementFixture(t);
|
||||
const source = await createFailedSource(value);
|
||||
const commandFile = writeCommand(
|
||||
value,
|
||||
'run.retry',
|
||||
request(value, source.runId),
|
||||
'run-retry',
|
||||
);
|
||||
|
||||
const accepted = await runLocalRunRetryCommandFile(commandFile);
|
||||
assert.equal(accepted.schemaVersion, 1);
|
||||
assert.equal(accepted.operation, 'run.retry');
|
||||
assert.equal(accepted.retry.status, 'accepted');
|
||||
assert.equal(accepted.retry.retryOfRunId, source.runId);
|
||||
assert.equal(accepted.retry.runStatus, 'queued');
|
||||
|
||||
const replay = await runLocalRunRetryCommandFile(commandFile);
|
||||
assert.equal(replay.retry.status, 'existing');
|
||||
assert.equal(replay.retry.runId, accepted.retry.runId);
|
||||
assert.deepEqual(
|
||||
auditRows(value.databasePath)
|
||||
.filter(({ operationId }) => operationId === 'run.retry')
|
||||
.map((row) => ({ ...row })),
|
||||
[
|
||||
{
|
||||
eventId: uuid(11),
|
||||
operationId: 'run.retry',
|
||||
outcome: 'allowed',
|
||||
reasonsJson: '["role_grant","strong_authentication"]',
|
||||
},
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('audits a missing source without creating Run state', async (t) => {
|
||||
const value = await localManagementFixture(t);
|
||||
const commandFile = writeCommand(
|
||||
value,
|
||||
'run.retry',
|
||||
request(value, 'missing-run', {
|
||||
mutationId: uuid(20),
|
||||
requestId: 'local-run-retry-missing',
|
||||
auditEventId: uuid(21),
|
||||
failureAuditEventId: uuid(22),
|
||||
}),
|
||||
'run-retry-missing',
|
||||
);
|
||||
await assert.rejects(
|
||||
runLocalRunRetryCommandFile(commandFile),
|
||||
RunManualRetryNotFoundError,
|
||||
);
|
||||
assert.deepEqual(
|
||||
auditRows(value.databasePath)
|
||||
.filter(({ operationId }) => operationId === 'run.retry')
|
||||
.map((row) => ({ ...row })),
|
||||
[
|
||||
{
|
||||
eventId: uuid(22),
|
||||
operationId: 'run.retry',
|
||||
outcome: 'denied',
|
||||
reasonsJson: '["run_not_found"]',
|
||||
},
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('binary exposes only the private command-file retry interface', () => {
|
||||
const cli = path.join(
|
||||
__dirname,
|
||||
'..',
|
||||
'dist',
|
||||
'run-management',
|
||||
'runRetryCli.js',
|
||||
);
|
||||
const help = spawnSync(process.execPath, [cli, '--help'], {
|
||||
encoding: 'utf8',
|
||||
});
|
||||
assert.equal(help.status, 0);
|
||||
assert.equal(
|
||||
help.stdout.trim(),
|
||||
'Usage: ql3-run retry --command-file /absolute/private-command.json',
|
||||
);
|
||||
const invalid = spawnSync(process.execPath, [cli, 'retry'], {
|
||||
encoding: 'utf8',
|
||||
});
|
||||
assert.equal(invalid.status, 64);
|
||||
assert.equal(
|
||||
JSON.parse(invalid.stderr).code,
|
||||
'LOCAL_RUN_RETRY_CLI_USAGE_INVALID',
|
||||
);
|
||||
});
|
||||
@@ -255,6 +255,16 @@
|
||||
"require": "./dist/run/runLostRetryRepository.js",
|
||||
"default": "./dist/run/runLostRetryRepository.js"
|
||||
},
|
||||
"./run-manual-retry": {
|
||||
"types": "./dist/run/runManualRetryRepository.d.ts",
|
||||
"require": "./dist/run/runManualRetryRepository.js",
|
||||
"default": "./dist/run/runManualRetryRepository.js"
|
||||
},
|
||||
"./run-management": {
|
||||
"types": "./dist/administration/runManagement.d.ts",
|
||||
"require": "./dist/administration/runManagement.js",
|
||||
"default": "./dist/administration/runManagement.js"
|
||||
},
|
||||
"./trigger-administration": {
|
||||
"types": "./dist/scheduling/triggerAdministration.d.ts",
|
||||
"require": "./dist/scheduling/triggerAdministration.js",
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import type { ApiCredentialRepository } from '@qinglong/runtime-core/api-credential';
|
||||
import type { LocalOwnerPepperRepository } from '@qinglong/runtime-core/local-owner-pepper';
|
||||
import type { ProjectPolicyRepository } from '@qinglong/runtime-core/project-policy';
|
||||
import type { RunManualRetryRepository } from '@qinglong/runtime-core/run-manual-retry';
|
||||
import type { SecurityAuditSink } from '@qinglong/runtime-core/security-audit';
|
||||
|
||||
import { LocalSqliteOperationAuthority } from '../authority/operationAuthority';
|
||||
import { LocalSqliteOwnerPepperRepository } from '../local-owner/ownerPepperRepository';
|
||||
import {
|
||||
EDGE_RUN_MANUAL_RETRY_RATE_LIMIT,
|
||||
LocalSqliteRunManualRetryRepository,
|
||||
STANDALONE_RUN_MANUAL_RETRY_RATE_LIMIT,
|
||||
} from '../run/runManualRetryRepository';
|
||||
import { LocalSqliteApiCredentialRepository } from '../security/apiCredentialRepository';
|
||||
import { LocalSqliteProjectPolicyRepository } from '../security/projectPolicyRepository';
|
||||
import { LocalSqliteSecurityAuthorityStore } from '../security/securityAuthorityStore';
|
||||
import {
|
||||
assertLocalSqliteOptions,
|
||||
assertLocalSqlitePathBoundary,
|
||||
openLocalSqliteClient,
|
||||
type LocalSqliteDatabaseOptions,
|
||||
type LocalSqliteProfile,
|
||||
} from '../storage/config';
|
||||
import {
|
||||
auditLocalSqliteReadiness,
|
||||
type LocalSqliteReadinessEvidence,
|
||||
} from '../readiness/readiness';
|
||||
import {
|
||||
confirmLocalSqliteAuthenticatedUserCredentialFence,
|
||||
LocalSqliteAuthenticatedManagementFenceError,
|
||||
type LocalSqliteAuthenticatedUserCredentialFence,
|
||||
} from './packageManagement';
|
||||
|
||||
export interface LocalSqliteRunManagementDatabase {
|
||||
readonly profile: LocalSqliteProfile;
|
||||
readonly readiness: LocalSqliteReadinessEvidence;
|
||||
readonly apiCredentials: ApiCredentialRepository;
|
||||
readonly ownerPepper: Pick<LocalOwnerPepperRepository, 'resolveKey'>;
|
||||
readonly projectPolicy: ProjectPolicyRepository;
|
||||
readonly runManualRetry: RunManualRetryRepository;
|
||||
readonly securityAudit: SecurityAuditSink;
|
||||
activateUserCredentialFence(
|
||||
fence: Readonly<LocalSqliteAuthenticatedUserCredentialFence>,
|
||||
): void;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
function sameCredentialFence(
|
||||
left: Readonly<LocalSqliteAuthenticatedUserCredentialFence>,
|
||||
right: Readonly<LocalSqliteAuthenticatedUserCredentialFence>,
|
||||
): boolean {
|
||||
return (
|
||||
left.credentialId === right.credentialId &&
|
||||
left.credentialVersion === right.credentialVersion &&
|
||||
left.pepperKeyId === right.pepperKeyId &&
|
||||
left.materialDigest === right.materialDigest &&
|
||||
left.subjectType === right.subjectType &&
|
||||
left.subjectId === right.subjectId &&
|
||||
left.secretDigest === right.secretDigest &&
|
||||
left.notBeforeAtMs === right.notBeforeAtMs &&
|
||||
left.expiresAtMs === right.expiresAtMs
|
||||
);
|
||||
}
|
||||
|
||||
/** Short-lived strong-User authority; it never migrates or starts a timer. */
|
||||
export async function openLocalSqliteRunManagementDatabase(
|
||||
options: LocalSqliteDatabaseOptions,
|
||||
): Promise<LocalSqliteRunManagementDatabase> {
|
||||
assertLocalSqliteOptions(options);
|
||||
assertLocalSqlitePathBoundary(options.databasePath, false);
|
||||
const client = openLocalSqliteClient(options, false);
|
||||
try {
|
||||
const readiness = await auditLocalSqliteReadiness(client);
|
||||
const authority = new LocalSqliteOperationAuthority(client);
|
||||
let activeFence:
|
||||
| Readonly<LocalSqliteAuthenticatedUserCredentialFence>
|
||||
| undefined;
|
||||
const securityAuthority = new LocalSqliteSecurityAuthorityStore(authority);
|
||||
const runManualRetry = new LocalSqliteRunManualRetryRepository(authority, {
|
||||
rateLimit:
|
||||
options.profile === 'edge'
|
||||
? EDGE_RUN_MANUAL_RETRY_RATE_LIMIT
|
||||
: STANDALONE_RUN_MANUAL_RETRY_RATE_LIMIT,
|
||||
beforeMutation(actor) {
|
||||
if (
|
||||
!activeFence ||
|
||||
actor.type !== activeFence.subjectType ||
|
||||
actor.id !== activeFence.subjectId
|
||||
) {
|
||||
throw new LocalSqliteAuthenticatedManagementFenceError();
|
||||
}
|
||||
confirmLocalSqliteAuthenticatedUserCredentialFence(
|
||||
authority,
|
||||
activeFence,
|
||||
);
|
||||
},
|
||||
});
|
||||
let closePromise: Promise<void> | undefined;
|
||||
return Object.freeze({
|
||||
profile: options.profile,
|
||||
readiness,
|
||||
apiCredentials: new LocalSqliteApiCredentialRepository(authority),
|
||||
ownerPepper: new LocalSqliteOwnerPepperRepository(authority),
|
||||
projectPolicy: new LocalSqliteProjectPolicyRepository(authority),
|
||||
runManualRetry,
|
||||
securityAudit: securityAuthority,
|
||||
activateUserCredentialFence(
|
||||
fence: Readonly<LocalSqliteAuthenticatedUserCredentialFence>,
|
||||
) {
|
||||
confirmLocalSqliteAuthenticatedUserCredentialFence(authority, fence);
|
||||
if (activeFence && !sameCredentialFence(activeFence, fence)) {
|
||||
throw new LocalSqliteAuthenticatedManagementFenceError();
|
||||
}
|
||||
activeFence = Object.freeze({ ...fence });
|
||||
},
|
||||
close() {
|
||||
if (closePromise) return closePromise;
|
||||
closePromise = authority.close();
|
||||
return closePromise;
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (client.isOpen) client.close();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export type { LocalSqliteDatabaseOptions, LocalSqliteProfile };
|
||||
export type { LocalSqliteReadinessEvidence } from '../readiness/readiness';
|
||||
@@ -0,0 +1,686 @@
|
||||
import {
|
||||
InvalidRunManualRetryError,
|
||||
MAX_RUN_MANUAL_RETRY_AUTHENTICATION_AGE_MS,
|
||||
RUN_MANUAL_RETRY_SOURCE_STATUSES,
|
||||
RunManualRetryFenceRejectedError,
|
||||
RunManualRetryNotFoundError,
|
||||
RunManualRetryRateLimitedError,
|
||||
RunManualRetryUnavailableError,
|
||||
normalizeRunManualRetryCommand,
|
||||
normalizeRunManualRetryResult,
|
||||
type RunManualRetryAllowedRole,
|
||||
type RunManualRetryCommand,
|
||||
type RunManualRetryRepository,
|
||||
type RunManualRetryResult,
|
||||
type RunManualRetrySourceStatus,
|
||||
} from '@qinglong/runtime-core/run-manual-retry';
|
||||
import {
|
||||
normalizeSecurityAuditRecord,
|
||||
type SecurityAuditRecord,
|
||||
} from '@qinglong/runtime-core/security-audit';
|
||||
import type { SecuritySubject } from '@qinglong/runtime-core/security';
|
||||
|
||||
import { LocalSqliteOperationAuthority } from '../authority/operationAuthority';
|
||||
import { LocalSqliteDispatchDefinitionStore } from '../task-definition/dispatchDefinitionStore';
|
||||
import {
|
||||
insertLocalSecurityAudit,
|
||||
localSecurityAuditFromRow,
|
||||
sameSecurityAuditSemantic,
|
||||
} from '../security/securityPersistence';
|
||||
import {
|
||||
optionalString,
|
||||
requiredInteger,
|
||||
requiredString,
|
||||
type QueryRow,
|
||||
} from './runPersistence';
|
||||
|
||||
export const RUN_MANUAL_RETRY_RATE_WINDOW_MS = 60_000;
|
||||
export const EDGE_RUN_MANUAL_RETRY_RATE_LIMIT = 4;
|
||||
export const STANDALONE_RUN_MANUAL_RETRY_RATE_LIMIT = 16;
|
||||
|
||||
const ALLOWED_ROLES = new Set<RunManualRetryAllowedRole>([
|
||||
'owner',
|
||||
'admin',
|
||||
'operator',
|
||||
]);
|
||||
|
||||
export interface LocalSqliteRunManualRetryRepositoryOptions {
|
||||
readonly rateLimit: number;
|
||||
readonly beforeMutation?: (actor: Readonly<SecuritySubject>) => void;
|
||||
}
|
||||
|
||||
interface SourceRun {
|
||||
readonly taskId: string;
|
||||
readonly taskRevision: string;
|
||||
readonly taskName?: string;
|
||||
readonly taskSnapshotRef: string;
|
||||
readonly inputRef?: string;
|
||||
readonly priority: number;
|
||||
readonly status: string;
|
||||
readonly version: number;
|
||||
readonly attemptExecutorType: string;
|
||||
}
|
||||
|
||||
const AUDIT_SELECT = `
|
||||
"event_id" AS "eventId",
|
||||
"request_id" AS "requestId",
|
||||
"operation_id" AS "operationId",
|
||||
"project_id" AS "auditProjectId",
|
||||
"subject_type" AS "subjectType",
|
||||
"subject_id" AS "subjectId",
|
||||
"authentication_id" AS "authenticationId",
|
||||
"outcome" AS "outcome",
|
||||
"reasons_json" AS "reasonsJson",
|
||||
"fence_project_version" AS "fenceProjectVersion",
|
||||
"fence_binding_version" AS "fenceBindingVersion",
|
||||
"occurred_at_ms" AS "occurredAtMs"`;
|
||||
|
||||
function timestamp(value: unknown, label: string): number {
|
||||
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
|
||||
throw new TypeError(`Local SQLite Run manual retry ${label} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function exactJson(row: QueryRow, key: string): Record<string, unknown> {
|
||||
try {
|
||||
const value = JSON.parse(requiredString(row, key)) as unknown;
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TypeError();
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
} catch {
|
||||
throw new RunManualRetryFenceRejectedError('mutation_conflict');
|
||||
}
|
||||
}
|
||||
|
||||
function sourceStatus(value: string): RunManualRetrySourceStatus {
|
||||
if (
|
||||
!RUN_MANUAL_RETRY_SOURCE_STATUSES.includes(
|
||||
value as RunManualRetrySourceStatus,
|
||||
)
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('source_not_terminal');
|
||||
}
|
||||
return value as RunManualRetrySourceStatus;
|
||||
}
|
||||
|
||||
function rollback(authority: LocalSqliteOperationAuthority): void {
|
||||
if (!authority.client.isTransaction) return;
|
||||
try {
|
||||
authority.client.exec('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the primary transaction failure.
|
||||
}
|
||||
}
|
||||
|
||||
function sameFencePayload(
|
||||
value: unknown,
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
): boolean {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
||||
const fence = value as Record<string, unknown>;
|
||||
return (
|
||||
fence.project_version === command.policyFence.projectVersion &&
|
||||
fence.binding_version === command.policyFence.bindingVersion
|
||||
);
|
||||
}
|
||||
|
||||
export class LocalSqliteRunManualRetryRepository
|
||||
implements RunManualRetryRepository
|
||||
{
|
||||
private readonly beforeMutation: (actor: Readonly<SecuritySubject>) => void;
|
||||
private readonly dispatchDefinitions: LocalSqliteDispatchDefinitionStore;
|
||||
|
||||
constructor(
|
||||
private readonly authority: LocalSqliteOperationAuthority,
|
||||
private readonly options: LocalSqliteRunManualRetryRepositoryOptions,
|
||||
) {
|
||||
if (
|
||||
!(authority instanceof LocalSqliteOperationAuthority) ||
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some(
|
||||
(key) => key !== 'rateLimit' && key !== 'beforeMutation',
|
||||
) ||
|
||||
!Number.isSafeInteger(options.rateLimit) ||
|
||||
options.rateLimit < 1 ||
|
||||
options.rateLimit > 64 ||
|
||||
(options.beforeMutation !== undefined &&
|
||||
typeof options.beforeMutation !== 'function')
|
||||
) {
|
||||
throw new TypeError(
|
||||
'Local SQLite Run manual retry dependencies are invalid',
|
||||
);
|
||||
}
|
||||
this.beforeMutation = options.beforeMutation ?? (() => undefined);
|
||||
this.dispatchDefinitions = new LocalSqliteDispatchDefinitionStore(
|
||||
authority.client,
|
||||
);
|
||||
}
|
||||
|
||||
retryRun(
|
||||
value: Readonly<RunManualRetryCommand>,
|
||||
): Promise<Readonly<RunManualRetryResult>> {
|
||||
let command: Readonly<RunManualRetryCommand>;
|
||||
try {
|
||||
command = normalizeRunManualRetryCommand(value);
|
||||
} catch (error) {
|
||||
return Promise.reject(error);
|
||||
}
|
||||
return this.authority.enqueue(
|
||||
async () => {
|
||||
const client = this.authority.client;
|
||||
try {
|
||||
client.exec('BEGIN IMMEDIATE');
|
||||
const observedAtMs = this.databaseTime();
|
||||
this.confirmStrongAuthentication(command, observedAtMs);
|
||||
try {
|
||||
this.beforeMutation(command.principal.subject);
|
||||
} catch {
|
||||
throw new RunManualRetryFenceRejectedError(
|
||||
'authentication_changed',
|
||||
);
|
||||
}
|
||||
this.confirmAuthorization(command);
|
||||
|
||||
const replay = this.findReplay(command);
|
||||
if (replay) {
|
||||
const result = this.replayResult(command, replay);
|
||||
this.commitAudit(this.audit(command, observedAtMs), true);
|
||||
client.exec('COMMIT');
|
||||
return result;
|
||||
}
|
||||
|
||||
const source = this.findSource(command);
|
||||
const execution = this.dispatchDefinitions.resolveRevision({
|
||||
projectId: command.projectId,
|
||||
taskId: source.taskId,
|
||||
taskRevision: source.taskRevision,
|
||||
});
|
||||
if (
|
||||
!execution ||
|
||||
execution.executorType !== 'local_process' ||
|
||||
source.attemptExecutorType !== execution.executorType
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('source_not_retryable');
|
||||
}
|
||||
this.confirmTaskEnabled(command.projectId, source.taskId);
|
||||
this.consumeRateLimit(command, observedAtMs);
|
||||
this.insertRetry(
|
||||
command,
|
||||
source,
|
||||
execution.contentDigest,
|
||||
observedAtMs,
|
||||
);
|
||||
const result = normalizeRunManualRetryResult({
|
||||
status: 'accepted',
|
||||
projectId: command.projectId,
|
||||
sourceRunId: command.sourceRunId,
|
||||
sourceRunStatus: command.expectedRunStatus,
|
||||
sourceRunVersion: command.expectedRunVersion,
|
||||
runId: command.runId,
|
||||
retryOfRunId: command.sourceRunId,
|
||||
taskId: source.taskId,
|
||||
taskRevision: source.taskRevision,
|
||||
attemptId: command.attemptId,
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'local_process',
|
||||
executionRevisionDigest: execution.contentDigest,
|
||||
createdAtMs: observedAtMs,
|
||||
});
|
||||
this.commitAudit(this.audit(command, observedAtMs), false);
|
||||
client.exec('COMMIT');
|
||||
return result;
|
||||
} catch (error) {
|
||||
rollback(this.authority);
|
||||
if (
|
||||
error instanceof InvalidRunManualRetryError ||
|
||||
error instanceof RunManualRetryNotFoundError ||
|
||||
error instanceof RunManualRetryFenceRejectedError ||
|
||||
error instanceof RunManualRetryRateLimitedError
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
throw new RunManualRetryUnavailableError({ cause: error });
|
||||
}
|
||||
},
|
||||
() => new RunManualRetryUnavailableError(),
|
||||
);
|
||||
}
|
||||
|
||||
private databaseTime(): number {
|
||||
const row = this.authority.client
|
||||
.prepare(
|
||||
`SELECT CAST(unixepoch('subsec') * 1000 AS INTEGER) AS "observedAtMs"`,
|
||||
)
|
||||
.get() as QueryRow | undefined;
|
||||
return timestamp(row?.observedAtMs, 'database clock');
|
||||
}
|
||||
|
||||
private confirmStrongAuthentication(
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
observedAtMs: number,
|
||||
): void {
|
||||
if (
|
||||
command.principal.subject.type !== 'user' ||
|
||||
!['multi_factor', 'hardware', 'local_console'].includes(
|
||||
command.principal.assurance,
|
||||
) ||
|
||||
command.principal.authenticatedAtMs > observedAtMs ||
|
||||
command.principal.expiresAtMs <= observedAtMs ||
|
||||
observedAtMs - command.principal.authenticatedAtMs >
|
||||
MAX_RUN_MANUAL_RETRY_AUTHENTICATION_AGE_MS
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('authentication_changed');
|
||||
}
|
||||
}
|
||||
|
||||
private confirmAuthorization(command: Readonly<RunManualRetryCommand>): void {
|
||||
const row = this.authority.client
|
||||
.prepare(
|
||||
`SELECT project."status" AS "projectStatus",
|
||||
project."version" AS "projectVersion",
|
||||
binding."state" AS "bindingState",
|
||||
binding."version" AS "bindingVersion",
|
||||
binding."role" AS "bindingRole"
|
||||
FROM "QingLong3Projects" AS project
|
||||
JOIN "QingLong3ProjectRoleBindings" AS binding
|
||||
ON binding."project_id" = project."id"
|
||||
AND binding."subject_type" = ?
|
||||
AND binding."subject_id" = ?
|
||||
WHERE project."id" = ?
|
||||
AND binding."version" = (
|
||||
SELECT MAX(latest."version")
|
||||
FROM "QingLong3ProjectRoleBindings" AS latest
|
||||
WHERE latest."project_id" = binding."project_id"
|
||||
AND latest."subject_type" = binding."subject_type"
|
||||
AND latest."subject_id" = binding."subject_id"
|
||||
)`,
|
||||
)
|
||||
.get(
|
||||
command.principal.subject.type,
|
||||
command.principal.subject.id,
|
||||
command.projectId,
|
||||
) as QueryRow | undefined;
|
||||
if (
|
||||
!row ||
|
||||
requiredString(row, 'projectStatus') !== 'active' ||
|
||||
requiredInteger(row, 'projectVersion') !==
|
||||
command.policyFence.projectVersion ||
|
||||
requiredString(row, 'bindingState') !== 'active' ||
|
||||
requiredInteger(row, 'bindingVersion') !==
|
||||
command.policyFence.bindingVersion ||
|
||||
!ALLOWED_ROLES.has(
|
||||
requiredString(row, 'bindingRole') as RunManualRetryAllowedRole,
|
||||
)
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('authorization_changed');
|
||||
}
|
||||
}
|
||||
|
||||
private findReplay(
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
): QueryRow | undefined {
|
||||
return this.authority.client
|
||||
.prepare(
|
||||
`SELECT run."id" AS "runId", run."project_id" AS "projectId",
|
||||
run."retry_of_run_id" AS "retryOfRunId",
|
||||
run."task_id" AS "taskId",
|
||||
run."task_revision" AS "taskRevision",
|
||||
run."trigger_type" AS "triggerType",
|
||||
run."execution_origin" AS "executionOrigin",
|
||||
run."execution_owner" AS "executionOwner",
|
||||
run."triggered_by" AS "triggeredBy",
|
||||
run."request_id" AS "requestId",
|
||||
run."status" AS "runStatus", run."version" AS "runVersion",
|
||||
run."event_sequence" AS "eventSequence",
|
||||
run."created_at_ms" AS "createdAtMs",
|
||||
attempt."id" AS "attemptId",
|
||||
attempt."executor_type" AS "executorType",
|
||||
created."actor_type" AS "createdActorType",
|
||||
created."actor_id" AS "createdActorId",
|
||||
created."payload" AS "createdPayload",
|
||||
queued."actor_type" AS "queuedActorType",
|
||||
queued."actor_id" AS "queuedActorId",
|
||||
queued."payload" AS "queuedPayload"
|
||||
FROM "Runs" AS run
|
||||
JOIN "RunAttempts" AS attempt
|
||||
ON attempt."run_id" = run."id" AND attempt."attempt" = 1
|
||||
JOIN "RunEvents" AS created
|
||||
ON created."run_id" = run."id" AND created."sequence" = 1
|
||||
AND created."type" = 'run.created'
|
||||
JOIN "RunEvents" AS queued
|
||||
ON queued."run_id" = run."id" AND queued."sequence" = 2
|
||||
AND queued."type" = 'run.queued'
|
||||
WHERE run."project_id" = ? AND run."idempotency_key" = ?`,
|
||||
)
|
||||
.get(
|
||||
command.projectId,
|
||||
`ql3:run-manual-retry:v1:${command.mutationId}`,
|
||||
) as QueryRow | undefined;
|
||||
}
|
||||
|
||||
private replayResult(
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
row: QueryRow,
|
||||
): Readonly<RunManualRetryResult> {
|
||||
const created = exactJson(row, 'createdPayload');
|
||||
const queued = exactJson(row, 'queuedPayload');
|
||||
if (
|
||||
requiredString(row, 'projectId') !== command.projectId ||
|
||||
requiredString(row, 'retryOfRunId') !== command.sourceRunId ||
|
||||
requiredString(row, 'triggerType') !== 'run_manual_retry' ||
|
||||
requiredString(row, 'executionOrigin') !== 'manual' ||
|
||||
requiredString(row, 'executionOwner') !== 'runtime' ||
|
||||
requiredString(row, 'triggeredBy') !== command.principal.subject.id ||
|
||||
requiredString(row, 'requestId') !== command.mutationId ||
|
||||
requiredString(row, 'runStatus') !== 'queued' ||
|
||||
requiredInteger(row, 'runVersion') !== 2 ||
|
||||
requiredInteger(row, 'eventSequence') !== 2 ||
|
||||
requiredString(row, 'executorType') !== 'local_process' ||
|
||||
requiredString(row, 'createdActorType') !==
|
||||
command.principal.subject.type ||
|
||||
requiredString(row, 'createdActorId') !== command.principal.subject.id ||
|
||||
requiredString(row, 'queuedActorType') !==
|
||||
command.principal.subject.type ||
|
||||
requiredString(row, 'queuedActorId') !== command.principal.subject.id ||
|
||||
created.mutation_id !== command.mutationId ||
|
||||
created.retry_of_run_id !== command.sourceRunId ||
|
||||
created.source_run_status !== command.expectedRunStatus ||
|
||||
created.source_run_version !== command.expectedRunVersion ||
|
||||
created.inherit_retry_policy !== false ||
|
||||
typeof created.execution_revision_digest !== 'string' ||
|
||||
!/^[0-9a-f]{64}$/.test(created.execution_revision_digest) ||
|
||||
!sameFencePayload(created.policy_fence, command) ||
|
||||
queued.from_status !== 'created' ||
|
||||
queued.to_status !== 'queued' ||
|
||||
queued.version !== 2
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('mutation_conflict');
|
||||
}
|
||||
return normalizeRunManualRetryResult({
|
||||
status: 'existing',
|
||||
projectId: command.projectId,
|
||||
sourceRunId: command.sourceRunId,
|
||||
sourceRunStatus: command.expectedRunStatus,
|
||||
sourceRunVersion: command.expectedRunVersion,
|
||||
runId: requiredString(row, 'runId'),
|
||||
retryOfRunId: requiredString(row, 'retryOfRunId'),
|
||||
taskId: requiredString(row, 'taskId'),
|
||||
taskRevision: requiredString(row, 'taskRevision'),
|
||||
attemptId: requiredString(row, 'attemptId'),
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'local_process',
|
||||
executionRevisionDigest: created.execution_revision_digest,
|
||||
createdAtMs: requiredInteger(row, 'createdAtMs'),
|
||||
});
|
||||
}
|
||||
|
||||
private findSource(command: Readonly<RunManualRetryCommand>): SourceRun {
|
||||
const row = this.authority.client
|
||||
.prepare(
|
||||
`SELECT run."project_id" AS "projectId",
|
||||
run."task_id" AS "taskId",
|
||||
run."task_revision" AS "taskRevision",
|
||||
run."task_name" AS "taskName",
|
||||
run."task_snapshot_ref" AS "taskSnapshotRef",
|
||||
run."parent_run_id" AS "parentRunId",
|
||||
run."trigger_type" AS "triggerType",
|
||||
run."execution_owner" AS "executionOwner",
|
||||
run."input_ref" AS "inputRef",
|
||||
run."priority" AS "priority",
|
||||
run."status" AS "runStatus",
|
||||
run."version" AS "runVersion",
|
||||
attempt."executor_type" AS "attemptExecutorType"
|
||||
FROM "Runs" AS run
|
||||
LEFT JOIN "RunAttempts" AS attempt
|
||||
ON attempt."run_id" = run."id"
|
||||
AND attempt."attempt" = (
|
||||
SELECT MAX(latest."attempt") FROM "RunAttempts" AS latest
|
||||
WHERE latest."run_id" = run."id"
|
||||
)
|
||||
WHERE run."id" = ?`,
|
||||
)
|
||||
.get(command.sourceRunId) as QueryRow | undefined;
|
||||
if (!row || requiredString(row, 'projectId') !== command.projectId) {
|
||||
throw new RunManualRetryNotFoundError();
|
||||
}
|
||||
const status = requiredString(row, 'runStatus');
|
||||
if (!RUN_MANUAL_RETRY_SOURCE_STATUSES.includes(status as never)) {
|
||||
throw new RunManualRetryFenceRejectedError('source_not_terminal');
|
||||
}
|
||||
if (
|
||||
status !== command.expectedRunStatus ||
|
||||
requiredInteger(row, 'runVersion') !== command.expectedRunVersion
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('source_changed');
|
||||
}
|
||||
const taskRevision = requiredString(row, 'taskRevision');
|
||||
const taskSnapshotRef = optionalString(row, 'taskSnapshotRef');
|
||||
if (
|
||||
requiredString(row, 'executionOwner') !== 'runtime' ||
|
||||
optionalString(row, 'parentRunId') !== undefined ||
|
||||
requiredString(row, 'triggerType') === 'plugin_package_workflow' ||
|
||||
taskSnapshotRef === undefined ||
|
||||
taskSnapshotRef !== taskRevision ||
|
||||
optionalString(row, 'attemptExecutorType') === undefined
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('source_not_retryable');
|
||||
}
|
||||
const taskName = optionalString(row, 'taskName');
|
||||
const inputRef = optionalString(row, 'inputRef');
|
||||
return Object.freeze({
|
||||
taskId: requiredString(row, 'taskId'),
|
||||
taskRevision,
|
||||
...(taskName === undefined ? {} : { taskName }),
|
||||
taskSnapshotRef,
|
||||
...(inputRef === undefined ? {} : { inputRef }),
|
||||
priority: requiredInteger(row, 'priority'),
|
||||
status,
|
||||
version: requiredInteger(row, 'runVersion'),
|
||||
attemptExecutorType: requiredString(row, 'attemptExecutorType'),
|
||||
});
|
||||
}
|
||||
|
||||
private confirmTaskEnabled(projectId: string, taskId: string): void {
|
||||
const row = this.authority.client
|
||||
.prepare(
|
||||
`SELECT revision."enabled" AS "enabled"
|
||||
FROM "QingLong3TaskDefinitions" AS head
|
||||
JOIN "QingLong3TaskDefinitionRevisions" AS revision
|
||||
ON revision."project_id" = head."project_id"
|
||||
AND revision."task_id" = head."task_id"
|
||||
AND revision."revision" = head."current_revision"
|
||||
WHERE head."project_id" = ? AND head."task_id" = ?`,
|
||||
)
|
||||
.get(projectId, taskId) as QueryRow | undefined;
|
||||
if (!row || requiredInteger(row, 'enabled') !== 1) {
|
||||
throw new RunManualRetryFenceRejectedError('task_disabled');
|
||||
}
|
||||
}
|
||||
|
||||
private consumeRateLimit(
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
observedAtMs: number,
|
||||
): void {
|
||||
const threshold = Math.max(
|
||||
0,
|
||||
observedAtMs - RUN_MANUAL_RETRY_RATE_WINDOW_MS,
|
||||
);
|
||||
const rows = this.authority.client
|
||||
.prepare(
|
||||
`SELECT "created_at_ms" AS "createdAtMs"
|
||||
FROM "Runs"
|
||||
WHERE "project_id" = ? AND "trigger_type" = 'run_manual_retry'
|
||||
AND "execution_origin" = 'manual' AND "triggered_by" = ?
|
||||
AND "created_at_ms" > ?
|
||||
ORDER BY "created_at_ms" DESC, "id" DESC LIMIT ?`,
|
||||
)
|
||||
.all(
|
||||
command.projectId,
|
||||
command.principal.subject.id,
|
||||
threshold,
|
||||
this.options.rateLimit,
|
||||
) as QueryRow[];
|
||||
if (rows.length < this.options.rateLimit) return;
|
||||
const earliestAtMs = requiredInteger(rows[rows.length - 1]!, 'createdAtMs');
|
||||
throw new RunManualRetryRateLimitedError(
|
||||
Math.max(
|
||||
1,
|
||||
earliestAtMs + RUN_MANUAL_RETRY_RATE_WINDOW_MS - observedAtMs,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private insertRetry(
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
source: Readonly<SourceRun>,
|
||||
executionRevisionDigest: string,
|
||||
observedAtMs: number,
|
||||
): void {
|
||||
const client = this.authority.client;
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO "Runs" (
|
||||
"id", "project_id", "task_id", "task_revision", "task_name",
|
||||
"task_snapshot_ref", "retry_of_run_id", "trigger_type",
|
||||
"execution_origin", "execution_owner", "triggered_by",
|
||||
"request_id", "status", "version", "event_sequence", "priority",
|
||||
"idempotency_key", "input_ref", "created_at_ms", "queued_at_ms"
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, 'run_manual_retry', 'manual',
|
||||
'runtime', ?, ?, 'queued', 2, 2, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run(
|
||||
command.runId,
|
||||
command.projectId,
|
||||
source.taskId,
|
||||
source.taskRevision,
|
||||
source.taskName ?? null,
|
||||
source.taskSnapshotRef,
|
||||
command.sourceRunId,
|
||||
command.principal.subject.id,
|
||||
command.mutationId,
|
||||
source.priority,
|
||||
`ql3:run-manual-retry:v1:${command.mutationId}`,
|
||||
source.inputRef ?? null,
|
||||
observedAtMs,
|
||||
observedAtMs,
|
||||
);
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO "RunAttempts" (
|
||||
"id", "run_id", "attempt", "status", "executor_type",
|
||||
"callback_sequence", "created_at_ms"
|
||||
) VALUES (?, ?, 1, 'claimed', 'local_process', 0, ?)`,
|
||||
)
|
||||
.run(command.attemptId, command.runId, observedAtMs);
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO "RunEvents" (
|
||||
"id", "run_id", "sequence", "type", "dedupe_key",
|
||||
"actor_type", "actor_id", "attempt_id", "payload",
|
||||
"created_at_ms"
|
||||
) VALUES (?, ?, 1, 'run.created', ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run(
|
||||
command.createdEventId,
|
||||
command.runId,
|
||||
`run-manual-retry-created:${command.mutationId}`,
|
||||
command.principal.subject.type,
|
||||
command.principal.subject.id,
|
||||
command.attemptId,
|
||||
JSON.stringify({
|
||||
status: 'created',
|
||||
version: 1,
|
||||
execution_owner: 'runtime',
|
||||
executor_type: 'local_process',
|
||||
execution_revision_digest: executionRevisionDigest,
|
||||
retry_of_run_id: command.sourceRunId,
|
||||
source_run_status: command.expectedRunStatus,
|
||||
source_run_version: command.expectedRunVersion,
|
||||
inherit_retry_policy: false,
|
||||
mutation_id: command.mutationId,
|
||||
policy_fence: {
|
||||
project_version: command.policyFence.projectVersion,
|
||||
binding_version: command.policyFence.bindingVersion,
|
||||
},
|
||||
}),
|
||||
observedAtMs,
|
||||
);
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO "RunEvents" (
|
||||
"id", "run_id", "sequence", "type", "dedupe_key",
|
||||
"actor_type", "actor_id", "attempt_id", "payload",
|
||||
"created_at_ms"
|
||||
) VALUES (?, ?, 2, 'run.queued', ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run(
|
||||
command.queuedEventId,
|
||||
command.runId,
|
||||
`run-manual-retry-queued:${command.mutationId}`,
|
||||
command.principal.subject.type,
|
||||
command.principal.subject.id,
|
||||
command.attemptId,
|
||||
JSON.stringify({
|
||||
from_status: 'created',
|
||||
to_status: 'queued',
|
||||
version: 2,
|
||||
}),
|
||||
observedAtMs,
|
||||
);
|
||||
}
|
||||
|
||||
private audit(
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
observedAtMs: number,
|
||||
) {
|
||||
return normalizeSecurityAuditRecord({
|
||||
eventId: command.auditEventId,
|
||||
requestId: command.requestId,
|
||||
operationId: 'run.retry',
|
||||
projectId: command.projectId,
|
||||
subject: command.principal.subject,
|
||||
authenticationId: command.principal.authenticationId,
|
||||
outcome: 'allowed',
|
||||
reasons: ['role_grant', 'strong_authentication'],
|
||||
fence: command.policyFence,
|
||||
occurredAtMs: observedAtMs,
|
||||
});
|
||||
}
|
||||
|
||||
private commitAudit(
|
||||
audit: Readonly<SecurityAuditRecord>,
|
||||
replay: boolean,
|
||||
): void {
|
||||
const row = this.authority.client
|
||||
.prepare(
|
||||
`SELECT ${AUDIT_SELECT}
|
||||
FROM "QingLong3SecurityAuditEvents" WHERE "event_id" = ?`,
|
||||
)
|
||||
.get(audit.eventId) as QueryRow | undefined;
|
||||
if (replay) {
|
||||
const stored = row ? localSecurityAuditFromRow(row) : null;
|
||||
const storedWithoutTime = stored
|
||||
? Object.freeze({ ...stored, occurredAtMs: audit.occurredAtMs })
|
||||
: null;
|
||||
if (
|
||||
!storedWithoutTime ||
|
||||
!sameSecurityAuditSemantic(storedWithoutTime, audit)
|
||||
) {
|
||||
throw new RunManualRetryFenceRejectedError('mutation_conflict');
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (row) {
|
||||
throw new RunManualRetryFenceRejectedError('mutation_conflict');
|
||||
}
|
||||
insertLocalSecurityAudit(this.authority.client, audit);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
RunManualRetryFenceRejectedError,
|
||||
RunManualRetryRateLimitedError,
|
||||
} = require('@qinglong/runtime-core/run-manual-retry');
|
||||
const {
|
||||
migrateLocalSqlitePath,
|
||||
openLocalSqliteRuntimeDatabase,
|
||||
} = require('../dist');
|
||||
const {
|
||||
LocalSqliteOperationAuthority,
|
||||
} = require('../dist/authority/operationAuthority.js');
|
||||
const {
|
||||
LocalSqliteRunManualRetryRepository,
|
||||
} = require('../dist/run/runManualRetryRepository.js');
|
||||
|
||||
function uuid(value) {
|
||||
return `019f9100-0000-4000-8000-${value.toString(16).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
function definition(index) {
|
||||
return {
|
||||
projectId: 'default',
|
||||
taskId: `retry-task-${index}`,
|
||||
expectedRevision: null,
|
||||
mutationId: uuid(100 + index),
|
||||
name: `Retry Task ${index}`,
|
||||
kind: 'command',
|
||||
spec: {
|
||||
schema: 'qinglong/command@v1',
|
||||
config: {
|
||||
command: { kind: 'argv', file: '/bin/echo', args: [String(index)] },
|
||||
},
|
||||
},
|
||||
labels: {},
|
||||
enabled: true,
|
||||
occurredAtMs: 1_000 + index,
|
||||
};
|
||||
}
|
||||
|
||||
function retryCommand(source, index, overrides = {}) {
|
||||
const now = Date.now();
|
||||
return {
|
||||
projectId: 'default',
|
||||
sourceRunId: source.runId,
|
||||
mutationId: uuid(200 + index),
|
||||
expectedRunVersion: 3,
|
||||
expectedRunStatus: 'failed',
|
||||
runId: uuid(300 + index * 10),
|
||||
attemptId: uuid(301 + index * 10),
|
||||
createdEventId: uuid(302 + index * 10),
|
||||
queuedEventId: uuid(303 + index * 10),
|
||||
auditEventId: uuid(304 + index * 10),
|
||||
requestId: `manual-run-retry-${index}`,
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'user-1' },
|
||||
authenticationId: 'local_console:proof-1',
|
||||
authenticatedAtMs: now - 1_000,
|
||||
expiresAtMs: now + 60_000,
|
||||
assurance: 'local_console',
|
||||
},
|
||||
policyFence: { projectVersion: 1, bindingVersion: 1 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function fixture(t, { rateLimit = 4, beforeMutation } = {}) {
|
||||
const directory = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), 'ql3-manual-run-retry-'),
|
||||
);
|
||||
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
|
||||
const databasePath = path.join(directory, 'qinglong3.sqlite');
|
||||
await migrateLocalSqlitePath({ databasePath, profile: 'edge' });
|
||||
const setup = new DatabaseSync(databasePath);
|
||||
setup
|
||||
.prepare(
|
||||
`INSERT INTO "QingLong3ProjectRoleBindings" (
|
||||
"project_id", "subject_type", "subject_id", "version", "state",
|
||||
"role", "mutation_id", "changed_by_type", "changed_by_id",
|
||||
"created_at_ms"
|
||||
) VALUES ('default', 'user', 'user-1', 1, 'active', 'operator',
|
||||
'grant-run-retry', 'user', 'user-1', 1)`,
|
||||
)
|
||||
.run();
|
||||
setup.close();
|
||||
|
||||
const runtime = await openLocalSqliteRuntimeDatabase({
|
||||
databasePath,
|
||||
profile: 'edge',
|
||||
});
|
||||
const sources = [];
|
||||
for (let index = 1; index <= 2; index += 1) {
|
||||
const task = (
|
||||
await runtime.taskDefinitions.appendTaskDefinitionRevision(
|
||||
definition(index),
|
||||
)
|
||||
).definition;
|
||||
const start = await (
|
||||
await runtime.taskStartRepository()
|
||||
).startTask({
|
||||
projectId: 'default',
|
||||
taskId: task.taskId,
|
||||
mutationId: uuid(400 + index),
|
||||
expectedRevision: task.revision,
|
||||
expectedContentDigest: task.contentDigest,
|
||||
runId: uuid(500 + index * 10),
|
||||
attemptId: uuid(501 + index * 10),
|
||||
createdEventId: uuid(502 + index * 10),
|
||||
queuedEventId: uuid(503 + index * 10),
|
||||
subject: { type: 'user', id: 'user-1' },
|
||||
policyFence: { projectVersion: 1, bindingVersion: 1 },
|
||||
});
|
||||
sources.push({ runId: start.runId, attemptId: start.attemptId, task });
|
||||
}
|
||||
await runtime.close();
|
||||
|
||||
const terminal = new DatabaseSync(databasePath);
|
||||
for (const [index, source] of sources.entries()) {
|
||||
terminal
|
||||
.prepare(
|
||||
`UPDATE "Runs"
|
||||
SET "status" = 'failed', "version" = 3, "event_sequence" = 3,
|
||||
"finished_at_ms" = ?, "error_code" = 'TEST_FAILURE',
|
||||
"error_summary" = 'failed before manual retry'
|
||||
WHERE "id" = ?`,
|
||||
)
|
||||
.run(2_000 + index, source.runId);
|
||||
terminal
|
||||
.prepare(
|
||||
`UPDATE "RunAttempts"
|
||||
SET "status" = 'failed', "finished_at_ms" = ?,
|
||||
"error_code" = 'TEST_FAILURE', "error_summary" = 'failed'
|
||||
WHERE "id" = ?`,
|
||||
)
|
||||
.run(2_000 + index, source.attemptId);
|
||||
terminal
|
||||
.prepare(
|
||||
`INSERT INTO "RunEvents" (
|
||||
"id", "run_id", "sequence", "type", "dedupe_key",
|
||||
"actor_type", "actor_id", "attempt_id", "payload",
|
||||
"created_at_ms"
|
||||
) VALUES (?, ?, 3, 'run.failed', ?, 'executor', 'test', ?, '{}', ?)`,
|
||||
)
|
||||
.run(
|
||||
uuid(600 + index),
|
||||
source.runId,
|
||||
`test-failed:${source.runId}`,
|
||||
source.attemptId,
|
||||
2_000 + index,
|
||||
);
|
||||
}
|
||||
terminal.close();
|
||||
|
||||
const client = new DatabaseSync(databasePath);
|
||||
const authority = new LocalSqliteOperationAuthority(client);
|
||||
const repository = new LocalSqliteRunManualRetryRepository(authority, {
|
||||
rateLimit,
|
||||
...(beforeMutation === undefined ? {} : { beforeMutation }),
|
||||
});
|
||||
t.after(() => authority.close());
|
||||
return { databasePath, repository, sources };
|
||||
}
|
||||
|
||||
test('atomically creates a new linked Run and exactly replays without reopening the source', async (t) => {
|
||||
const { databasePath, repository, sources } = await fixture(t);
|
||||
const command = retryCommand(sources[0], 1);
|
||||
const accepted = await repository.retryRun(command);
|
||||
assert.equal(accepted.status, 'accepted');
|
||||
assert.equal(accepted.retryOfRunId, sources[0].runId);
|
||||
assert.equal(accepted.runStatus, 'queued');
|
||||
assert.equal(accepted.executorType, 'local_process');
|
||||
|
||||
const replay = await repository.retryRun({
|
||||
...command,
|
||||
runId: uuid(901),
|
||||
attemptId: uuid(902),
|
||||
createdEventId: uuid(903),
|
||||
queuedEventId: uuid(904),
|
||||
});
|
||||
assert.equal(replay.status, 'existing');
|
||||
assert.equal(replay.runId, accepted.runId);
|
||||
assert.equal(replay.attemptId, accepted.attemptId);
|
||||
|
||||
const client = new DatabaseSync(databasePath, { readOnly: true });
|
||||
t.after(() => client.close());
|
||||
assert.deepEqual(
|
||||
{
|
||||
...client
|
||||
.prepare(
|
||||
`SELECT "status", "version", "event_sequence" AS "eventSequence",
|
||||
"retry_of_run_id" AS "retryOfRunId",
|
||||
"trigger_type" AS "triggerType"
|
||||
FROM "Runs" WHERE "id" = ?`,
|
||||
)
|
||||
.get(accepted.runId),
|
||||
},
|
||||
{
|
||||
status: 'queued',
|
||||
version: 2,
|
||||
eventSequence: 2,
|
||||
retryOfRunId: sources[0].runId,
|
||||
triggerType: 'run_manual_retry',
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
client
|
||||
.prepare(`SELECT "status" FROM "Runs" WHERE "id" = ?`)
|
||||
.get(sources[0].runId).status,
|
||||
'failed',
|
||||
);
|
||||
assert.equal(
|
||||
client
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS "count" FROM "RunRetryPolicies" WHERE "run_id" = ?`,
|
||||
)
|
||||
.get(accepted.runId).count,
|
||||
0,
|
||||
);
|
||||
assert.equal(
|
||||
client
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS "count" FROM "QingLong3SecurityAuditEvents"
|
||||
WHERE "operation_id" = 'run.retry'`,
|
||||
)
|
||||
.get().count,
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed for changed, non-terminal, disabled and unauthenticated sources', async (t) => {
|
||||
let authenticated = true;
|
||||
const { databasePath, repository, sources } = await fixture(t, {
|
||||
beforeMutation() {
|
||||
if (!authenticated) throw new Error('credential changed');
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
repository.retryRun(retryCommand(sources[0], 1, { expectedRunVersion: 2 })),
|
||||
(error) =>
|
||||
error instanceof RunManualRetryFenceRejectedError &&
|
||||
error.reason === 'source_changed',
|
||||
);
|
||||
const client = new DatabaseSync(databasePath);
|
||||
client
|
||||
.prepare(`UPDATE "Runs" SET "status" = 'lost' WHERE "id" = ?`)
|
||||
.run(sources[0].runId);
|
||||
await assert.rejects(
|
||||
repository.retryRun(retryCommand(sources[0], 2)),
|
||||
(error) =>
|
||||
error instanceof RunManualRetryFenceRejectedError &&
|
||||
error.reason === 'source_not_terminal',
|
||||
);
|
||||
client
|
||||
.prepare(`UPDATE "Runs" SET "status" = 'failed' WHERE "id" = ?`)
|
||||
.run(sources[0].runId);
|
||||
client
|
||||
.prepare(
|
||||
`UPDATE "QingLong3TaskDefinitionRevisions" SET "enabled" = 0
|
||||
WHERE "project_id" = 'default' AND "task_id" = ?`,
|
||||
)
|
||||
.run(sources[0].task.taskId);
|
||||
await assert.rejects(
|
||||
repository.retryRun(retryCommand(sources[0], 3)),
|
||||
(error) =>
|
||||
error instanceof RunManualRetryFenceRejectedError &&
|
||||
error.reason === 'task_disabled',
|
||||
);
|
||||
client.close();
|
||||
authenticated = false;
|
||||
await assert.rejects(
|
||||
repository.retryRun(retryCommand(sources[1], 4)),
|
||||
(error) =>
|
||||
error instanceof RunManualRetryFenceRejectedError &&
|
||||
error.reason === 'authentication_changed',
|
||||
);
|
||||
});
|
||||
|
||||
test('uses the durable Run ledger to enforce a bounded per-User rate', async (t) => {
|
||||
const { repository, sources } = await fixture(t, { rateLimit: 2 });
|
||||
await repository.retryRun(retryCommand(sources[0], 1));
|
||||
await repository.retryRun(retryCommand(sources[0], 2));
|
||||
await assert.rejects(
|
||||
repository.retryRun(retryCommand(sources[1], 3)),
|
||||
(error) =>
|
||||
error instanceof RunManualRetryRateLimitedError &&
|
||||
Number.isSafeInteger(error.retryAfterMs) &&
|
||||
error.retryAfterMs > 0 &&
|
||||
error.retryAfterMs <= 60_000,
|
||||
);
|
||||
});
|
||||
@@ -276,6 +276,11 @@
|
||||
"require": "./dist/run/clusterRunLostRetry.js",
|
||||
"default": "./dist/run/clusterRunLostRetry.js"
|
||||
},
|
||||
"./run-manual-retry": {
|
||||
"types": "./dist/run/manual-retry/runManualRetry.d.ts",
|
||||
"require": "./dist/run/manual-retry/runManualRetry.js",
|
||||
"default": "./dist/run/manual-retry/runManualRetry.js"
|
||||
},
|
||||
"./cluster-run-lost-retry": {
|
||||
"types": "./dist/run/clusterRunLostRetry.d.ts",
|
||||
"require": "./dist/run/clusterRunLostRetry.js",
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
import {
|
||||
normalizeProjectPolicySubject,
|
||||
type ProjectRole,
|
||||
} from '../../security/project-policy/projectPolicy';
|
||||
import {
|
||||
normalizeSecurityPolicyDecision,
|
||||
normalizeSecurityPrincipal,
|
||||
type SecurityPolicyFence,
|
||||
type SecurityPrincipal,
|
||||
} from '../../security/security';
|
||||
import type { RunStatus } from '../run';
|
||||
|
||||
export const RUN_MANUAL_RETRY_SCHEMA = 'qinglong/run-manual-retry@v1' as const;
|
||||
export const RUN_MANUAL_RETRY_STATUSES = ['accepted', 'existing'] as const;
|
||||
export const RUN_MANUAL_RETRY_SOURCE_STATUSES = [
|
||||
'failed',
|
||||
'cancelled',
|
||||
'timed_out',
|
||||
] as const;
|
||||
export const RUN_MANUAL_RETRY_EXECUTOR_TYPES = [
|
||||
'local_process',
|
||||
'remote_worker',
|
||||
] as const;
|
||||
export const MAX_RUN_MANUAL_RETRY_AUTHENTICATION_AGE_MS = 5 * 60_000;
|
||||
|
||||
export type RunManualRetryStatus = (typeof RUN_MANUAL_RETRY_STATUSES)[number];
|
||||
export type RunManualRetrySourceStatus =
|
||||
(typeof RUN_MANUAL_RETRY_SOURCE_STATUSES)[number];
|
||||
export type RunManualRetryExecutorType =
|
||||
(typeof RUN_MANUAL_RETRY_EXECUTOR_TYPES)[number];
|
||||
export type RunManualRetryAllowedRole = Extract<
|
||||
ProjectRole,
|
||||
'owner' | 'admin' | 'operator'
|
||||
>;
|
||||
|
||||
export interface RunManualRetryRequestBody {
|
||||
readonly schema: typeof RUN_MANUAL_RETRY_SCHEMA;
|
||||
readonly mutationId: string;
|
||||
readonly expectedRunVersion: number;
|
||||
readonly expectedRunStatus: RunManualRetrySourceStatus;
|
||||
}
|
||||
|
||||
export interface RunManualRetryCommand {
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly mutationId: string;
|
||||
readonly expectedRunVersion: number;
|
||||
readonly expectedRunStatus: RunManualRetrySourceStatus;
|
||||
readonly runId: string;
|
||||
readonly attemptId: string;
|
||||
readonly createdEventId: string;
|
||||
readonly queuedEventId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly requestId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
}
|
||||
|
||||
export interface RunManualRetryResult {
|
||||
readonly status: RunManualRetryStatus;
|
||||
readonly projectId: string;
|
||||
readonly sourceRunId: string;
|
||||
readonly sourceRunStatus: RunManualRetrySourceStatus;
|
||||
readonly sourceRunVersion: number;
|
||||
readonly runId: string;
|
||||
readonly retryOfRunId: string;
|
||||
readonly taskId: string;
|
||||
readonly taskRevision: string;
|
||||
readonly attemptId: string;
|
||||
readonly runStatus: 'queued';
|
||||
readonly runVersion: 2;
|
||||
readonly eventSequence: 2;
|
||||
readonly executorType: RunManualRetryExecutorType;
|
||||
readonly executionRevisionDigest: string;
|
||||
readonly createdAtMs: number;
|
||||
}
|
||||
|
||||
export interface RunManualRetryResponseBody extends RunManualRetryResult {
|
||||
readonly schema: typeof RUN_MANUAL_RETRY_SCHEMA;
|
||||
}
|
||||
|
||||
export interface RunManualRetryRepository {
|
||||
retryRun(
|
||||
command: Readonly<RunManualRetryCommand>,
|
||||
): Promise<Readonly<RunManualRetryResult>>;
|
||||
}
|
||||
|
||||
export type RunManualRetryFenceReason =
|
||||
| 'authorization_changed'
|
||||
| 'authentication_changed'
|
||||
| 'source_changed'
|
||||
| 'source_not_terminal'
|
||||
| 'source_not_retryable'
|
||||
| 'task_disabled'
|
||||
| 'mutation_conflict';
|
||||
|
||||
export class InvalidRunManualRetryError extends TypeError {
|
||||
readonly code = 'RUN_MANUAL_RETRY_INVALID';
|
||||
|
||||
constructor(message: string) {
|
||||
super(`Run manual retry is invalid: ${message}`);
|
||||
this.name = 'InvalidRunManualRetryError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RunManualRetryNotFoundError extends Error {
|
||||
readonly code = 'RUN_MANUAL_RETRY_NOT_FOUND';
|
||||
|
||||
constructor() {
|
||||
super('Run manual retry target does not exist');
|
||||
this.name = 'RunManualRetryNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RunManualRetryFenceRejectedError extends Error {
|
||||
readonly code = 'RUN_MANUAL_RETRY_FENCE_REJECTED';
|
||||
|
||||
constructor(readonly reason: RunManualRetryFenceReason) {
|
||||
super(`Run manual retry fence rejected: ${reason}`);
|
||||
this.name = 'RunManualRetryFenceRejectedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RunManualRetryRateLimitedError extends Error {
|
||||
readonly code = 'RUN_MANUAL_RETRY_RATE_LIMITED';
|
||||
|
||||
constructor(readonly retryAfterMs: number) {
|
||||
if (!Number.isSafeInteger(retryAfterMs) || retryAfterMs < 1) {
|
||||
throw new InvalidRunManualRetryError('retry delay is invalid');
|
||||
}
|
||||
super('Run manual retry rate limit is exhausted');
|
||||
this.name = 'RunManualRetryRateLimitedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RunManualRetryUnavailableError extends Error {
|
||||
readonly code = 'RUN_MANUAL_RETRY_UNAVAILABLE';
|
||||
|
||||
constructor(options?: ErrorOptions) {
|
||||
super('Run manual retry is unavailable', options);
|
||||
this.name = 'RunManualRetryUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const UUID_PATTERN =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
const DIGEST_PATTERN = /^[0-9a-f]{64}$/;
|
||||
|
||||
function exactKeys(value: object, expected: readonly string[]): void {
|
||||
const actual = Object.keys(value).sort();
|
||||
const canonical = [...expected].sort();
|
||||
if (
|
||||
actual.length !== canonical.length ||
|
||||
actual.some((key, index) => key !== canonical[index])
|
||||
) {
|
||||
throw new InvalidRunManualRetryError('shape is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
function identifier(value: unknown, name: string): string {
|
||||
if (typeof value !== 'string' || !IDENTIFIER_PATTERN.test(value)) {
|
||||
throw new InvalidRunManualRetryError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function uuid(value: unknown, name: string): string {
|
||||
if (typeof value !== 'string' || !UUID_PATTERN.test(value)) {
|
||||
throw new InvalidRunManualRetryError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function version(value: unknown, name: string): number {
|
||||
if (
|
||||
typeof value !== 'number' ||
|
||||
!Number.isSafeInteger(value) ||
|
||||
value < 1 ||
|
||||
value > 2_147_483_647
|
||||
) {
|
||||
throw new InvalidRunManualRetryError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function timestamp(value: unknown, name: string): number {
|
||||
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
|
||||
throw new InvalidRunManualRetryError(`${name} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function sourceStatus(value: unknown): RunManualRetrySourceStatus {
|
||||
if (
|
||||
!RUN_MANUAL_RETRY_SOURCE_STATUSES.includes(
|
||||
value as RunManualRetrySourceStatus,
|
||||
)
|
||||
) {
|
||||
throw new InvalidRunManualRetryError('source Run status is invalid');
|
||||
}
|
||||
return value as RunManualRetrySourceStatus;
|
||||
}
|
||||
|
||||
export function parseRunManualRetryRequestBody(
|
||||
value: unknown,
|
||||
): Readonly<RunManualRetryRequestBody> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new InvalidRunManualRetryError('request body is invalid');
|
||||
}
|
||||
exactKeys(value, [
|
||||
'schema',
|
||||
'mutationId',
|
||||
'expectedRunVersion',
|
||||
'expectedRunStatus',
|
||||
]);
|
||||
const body = value as Record<string, unknown>;
|
||||
if (body.schema !== RUN_MANUAL_RETRY_SCHEMA) {
|
||||
throw new InvalidRunManualRetryError('schema is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
schema: RUN_MANUAL_RETRY_SCHEMA,
|
||||
mutationId: uuid(body.mutationId, 'mutationId'),
|
||||
expectedRunVersion: version(
|
||||
body.expectedRunVersion,
|
||||
'expected Run version',
|
||||
),
|
||||
expectedRunStatus: sourceStatus(body.expectedRunStatus),
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeRunManualRetryCommand(
|
||||
value: RunManualRetryCommand,
|
||||
): Readonly<RunManualRetryCommand> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new InvalidRunManualRetryError('command is invalid');
|
||||
}
|
||||
exactKeys(value, [
|
||||
'projectId',
|
||||
'sourceRunId',
|
||||
'mutationId',
|
||||
'expectedRunVersion',
|
||||
'expectedRunStatus',
|
||||
'runId',
|
||||
'attemptId',
|
||||
'createdEventId',
|
||||
'queuedEventId',
|
||||
'auditEventId',
|
||||
'requestId',
|
||||
'principal',
|
||||
'policyFence',
|
||||
]);
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
let fence: Readonly<SecurityPolicyFence> | null;
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(
|
||||
value.principal,
|
||||
value.principal.authenticatedAtMs,
|
||||
);
|
||||
fence = normalizeSecurityPolicyDecision({
|
||||
effect: 'allow',
|
||||
reasons: ['role_grant'],
|
||||
fence: value.policyFence,
|
||||
}).fence;
|
||||
} catch {
|
||||
throw new InvalidRunManualRetryError('authorization authority is invalid');
|
||||
}
|
||||
if (
|
||||
principal.subject.type !== 'user' ||
|
||||
!['multi_factor', 'hardware', 'local_console'].includes(
|
||||
principal.assurance,
|
||||
) ||
|
||||
!fence ||
|
||||
fence.bindingVersion === null
|
||||
) {
|
||||
throw new InvalidRunManualRetryError(
|
||||
'strong User authorization authority is incomplete',
|
||||
);
|
||||
}
|
||||
return Object.freeze({
|
||||
projectId: identifier(value.projectId, 'projectId'),
|
||||
sourceRunId: identifier(value.sourceRunId, 'sourceRunId'),
|
||||
mutationId: uuid(value.mutationId, 'mutationId'),
|
||||
expectedRunVersion: version(
|
||||
value.expectedRunVersion,
|
||||
'expected Run version',
|
||||
),
|
||||
expectedRunStatus: sourceStatus(value.expectedRunStatus),
|
||||
runId: uuid(value.runId, 'runId'),
|
||||
attemptId: uuid(value.attemptId, 'attemptId'),
|
||||
createdEventId: uuid(value.createdEventId, 'createdEventId'),
|
||||
queuedEventId: uuid(value.queuedEventId, 'queuedEventId'),
|
||||
auditEventId: uuid(value.auditEventId, 'auditEventId'),
|
||||
requestId: identifier(value.requestId, 'requestId'),
|
||||
principal,
|
||||
policyFence: fence,
|
||||
});
|
||||
}
|
||||
|
||||
export function normalizeRunManualRetryResult(
|
||||
value: RunManualRetryResult,
|
||||
): Readonly<RunManualRetryResult> {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new InvalidRunManualRetryError('result is invalid');
|
||||
}
|
||||
exactKeys(value, [
|
||||
'status',
|
||||
'projectId',
|
||||
'sourceRunId',
|
||||
'sourceRunStatus',
|
||||
'sourceRunVersion',
|
||||
'runId',
|
||||
'retryOfRunId',
|
||||
'taskId',
|
||||
'taskRevision',
|
||||
'attemptId',
|
||||
'runStatus',
|
||||
'runVersion',
|
||||
'eventSequence',
|
||||
'executorType',
|
||||
'executionRevisionDigest',
|
||||
'createdAtMs',
|
||||
]);
|
||||
if (
|
||||
!RUN_MANUAL_RETRY_STATUSES.includes(value.status) ||
|
||||
value.retryOfRunId !== value.sourceRunId ||
|
||||
value.runId === value.sourceRunId ||
|
||||
value.runStatus !== 'queued' ||
|
||||
value.runVersion !== 2 ||
|
||||
value.eventSequence !== 2 ||
|
||||
!RUN_MANUAL_RETRY_EXECUTOR_TYPES.includes(value.executorType) ||
|
||||
!DIGEST_PATTERN.test(value.executionRevisionDigest)
|
||||
) {
|
||||
throw new InvalidRunManualRetryError('result state is invalid');
|
||||
}
|
||||
return Object.freeze({
|
||||
status: value.status,
|
||||
projectId: identifier(value.projectId, 'projectId'),
|
||||
sourceRunId: identifier(value.sourceRunId, 'sourceRunId'),
|
||||
sourceRunStatus: sourceStatus(value.sourceRunStatus),
|
||||
sourceRunVersion: version(value.sourceRunVersion, 'source Run version'),
|
||||
runId: uuid(value.runId, 'runId'),
|
||||
retryOfRunId: identifier(value.retryOfRunId, 'retryOfRunId'),
|
||||
taskId: identifier(value.taskId, 'taskId'),
|
||||
taskRevision: identifier(value.taskRevision, 'taskRevision'),
|
||||
attemptId: uuid(value.attemptId, 'attemptId'),
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: value.executorType,
|
||||
executionRevisionDigest: value.executionRevisionDigest,
|
||||
createdAtMs: timestamp(value.createdAtMs, 'createdAtMs'),
|
||||
});
|
||||
}
|
||||
|
||||
export function createRunManualRetryResponseBody(
|
||||
value: RunManualRetryResult,
|
||||
): Readonly<RunManualRetryResponseBody> {
|
||||
return Object.freeze({
|
||||
schema: RUN_MANUAL_RETRY_SCHEMA,
|
||||
...normalizeRunManualRetryResult(value),
|
||||
});
|
||||
}
|
||||
|
||||
export function isRunManualRetrySourceStatus(
|
||||
value: RunStatus,
|
||||
): value is RunManualRetrySourceStatus {
|
||||
return RUN_MANUAL_RETRY_SOURCE_STATUSES.includes(
|
||||
value as RunManualRetrySourceStatus,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
|
||||
const {
|
||||
InvalidRunManualRetryError,
|
||||
RUN_MANUAL_RETRY_SCHEMA,
|
||||
RunManualRetryRateLimitedError,
|
||||
createRunManualRetryResponseBody,
|
||||
normalizeRunManualRetryCommand,
|
||||
parseRunManualRetryRequestBody,
|
||||
} = require('../dist/run/manual-retry/runManualRetry.js');
|
||||
|
||||
const IDS = Object.freeze({
|
||||
mutationId: '019f9000-0000-4000-8000-000000000001',
|
||||
runId: '019f9000-0000-4000-8000-000000000002',
|
||||
attemptId: '019f9000-0000-4000-8000-000000000003',
|
||||
createdEventId: '019f9000-0000-4000-8000-000000000004',
|
||||
queuedEventId: '019f9000-0000-4000-8000-000000000005',
|
||||
auditEventId: '019f9000-0000-4000-8000-000000000006',
|
||||
});
|
||||
|
||||
function command(overrides = {}) {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
mutationId: IDS.mutationId,
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'failed',
|
||||
runId: IDS.runId,
|
||||
attemptId: IDS.attemptId,
|
||||
createdEventId: IDS.createdEventId,
|
||||
queuedEventId: IDS.queuedEventId,
|
||||
auditEventId: IDS.auditEventId,
|
||||
requestId: 'run-retry-request-1',
|
||||
principal: {
|
||||
subject: { type: 'user', id: 'operator-1' },
|
||||
authenticationId: 'local_console:proof-1',
|
||||
authenticatedAtMs: 10_000,
|
||||
expiresAtMs: 20_000,
|
||||
assurance: 'local_console',
|
||||
},
|
||||
policyFence: { projectVersion: 2, bindingVersion: 3 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('normalizes one strongly authorized terminal Run retry command', () => {
|
||||
const request = parseRunManualRetryRequestBody({
|
||||
schema: RUN_MANUAL_RETRY_SCHEMA,
|
||||
mutationId: IDS.mutationId,
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'timed_out',
|
||||
});
|
||||
assert.equal(request.expectedRunStatus, 'timed_out');
|
||||
const normalized = normalizeRunManualRetryCommand(command());
|
||||
assert.equal(normalized.principal.assurance, 'local_console');
|
||||
assert.equal(Object.isFrozen(normalized), true);
|
||||
});
|
||||
|
||||
test('rejects reopened states, weak principals and widened commands', () => {
|
||||
assert.throws(
|
||||
() =>
|
||||
parseRunManualRetryRequestBody({
|
||||
schema: RUN_MANUAL_RETRY_SCHEMA,
|
||||
mutationId: IDS.mutationId,
|
||||
expectedRunVersion: 7,
|
||||
expectedRunStatus: 'lost',
|
||||
}),
|
||||
InvalidRunManualRetryError,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
normalizeRunManualRetryCommand(
|
||||
command({
|
||||
principal: { ...command().principal, assurance: 'single_factor' },
|
||||
}),
|
||||
),
|
||||
/strong User/,
|
||||
);
|
||||
assert.throws(
|
||||
() => normalizeRunManualRetryCommand({ ...command(), hidden: true }),
|
||||
InvalidRunManualRetryError,
|
||||
);
|
||||
});
|
||||
|
||||
test('publishes only the bounded new-Run retry result', () => {
|
||||
assert.deepEqual(
|
||||
createRunManualRetryResponseBody({
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: IDS.runId,
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `v1:${'a'.repeat(64)}`,
|
||||
attemptId: IDS.attemptId,
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'local_process',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: 11_000,
|
||||
}),
|
||||
{
|
||||
schema: RUN_MANUAL_RETRY_SCHEMA,
|
||||
status: 'accepted',
|
||||
projectId: 'project-1',
|
||||
sourceRunId: 'source-run-1',
|
||||
sourceRunStatus: 'failed',
|
||||
sourceRunVersion: 7,
|
||||
runId: IDS.runId,
|
||||
retryOfRunId: 'source-run-1',
|
||||
taskId: 'task-1',
|
||||
taskRevision: `v1:${'a'.repeat(64)}`,
|
||||
attemptId: IDS.attemptId,
|
||||
runStatus: 'queued',
|
||||
runVersion: 2,
|
||||
eventSequence: 2,
|
||||
executorType: 'local_process',
|
||||
executionRevisionDigest: 'b'.repeat(64),
|
||||
createdAtMs: 11_000,
|
||||
},
|
||||
);
|
||||
assert.equal(new RunManualRetryRateLimitedError(250).retryAfterMs, 250);
|
||||
assert.throws(() => new RunManualRetryRateLimitedError(0));
|
||||
});
|
||||
Reference in New Issue
Block a user