open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-22 02:27:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add strong cluster run stop

Browse Source
This commit is contained in:
whyour
2026-08-12 08:21:28 +08:00
parent c0ab62e64a
commit dd370b2842
22 changed files with 1602 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-cluster-admin/src/run-management/runManagement.ts
+147 -4
View File
@@ -1,12 +1,20 @@
import { randomUUID } from 'node:crypto';
import {
PostgresClusterRunCancellationRepository,
PostgresProjectPolicyRepository,
PostgresRunManualRetryRepository,
PostgresSecurityAuditRepository,
} from '@qinglong/cluster-postgres/run-manager';
import type { PostgresPool } from '@qinglong/runtime-core';
import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
import {
ClusterRunCancellationFenceRejectedError,
ClusterRunCancellationNotFoundError,
ClusterRunCancellationUnavailableError,
InvalidClusterRunCancellationError,
type ClusterRunCancellationResult,
} from '@qinglong/runtime-core/run-cancellation';
import {
InvalidRunManualRetryError,
RunManualRetryFenceRejectedError,
@@ -39,10 +47,23 @@ export interface ClusterRunManagementRetryRequest {
readonly principal: Readonly<SecurityPrincipal>;
}
export interface ClusterRunManagementStopRequest {
readonly projectId: string;
readonly runId: string;
readonly mutationId: string;
readonly requestId: string;
readonly auditEventId: string;
readonly failureAuditEventId: string;
readonly principal: Readonly<SecurityPrincipal>;
}
export interface ClusterRunManagementService {
retry(
request: Readonly<ClusterRunManagementRetryRequest>,
): Promise<Readonly<RunManualRetryResult>>;
stop(
request: Readonly<ClusterRunManagementStopRequest>,
): Promise<Readonly<ClusterRunCancellationResult>>;
}
export interface ClusterRunManagementOptions {
@@ -107,7 +128,7 @@ export class ClusterRunManagementUnavailableError extends Error {
}
}
function exactRequest(
function exactRetryRequest(
value: unknown,
): asserts value is Readonly<ClusterRunManagementRetryRequest> {
if (
@@ -133,6 +154,30 @@ function exactRequest(
}
}
function exactStopRequest(
value: unknown,
): asserts value is Readonly<ClusterRunManagementStopRequest> {
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
Object.keys(value).sort().join('\0') !==
[
'auditEventId',
'failureAuditEventId',
'mutationId',
'principal',
'projectId',
'requestId',
'runId',
]
.sort()
.join('\0')
) {
throw new ClusterRunManagementRequestError();
}
}
function validUuid(value: unknown): value is string {
return typeof value === 'string' && UUID_PATTERN.test(value);
}
@@ -144,6 +189,12 @@ function failureReason(error: unknown): string {
if (error instanceof RunManualRetryNotFoundError) return 'run_not_found';
if (error instanceof RunManualRetryRateLimitedError) return 'rate_limited';
if (error instanceof RunManualRetryFenceRejectedError) return error.reason;
if (error instanceof ClusterRunCancellationNotFoundError) {
return 'run_not_found';
}
if (error instanceof ClusterRunCancellationFenceRejectedError) {
return error.reason;
}
return 'management_unavailable';
}
@@ -162,7 +213,8 @@ export function createClusterRunManagementService(
typeof options.pool.query !== 'function' ||
typeof options.pool.connect !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function') ||
(options.randomUuid !== undefined && typeof options.randomUuid !== 'function')
(options.randomUuid !== undefined &&
typeof options.randomUuid !== 'function')
) {
throw new ClusterRunManagementConfigurationError();
}
@@ -172,11 +224,14 @@ export function createClusterRunManagementService(
new PostgresProjectPolicyRepository(options.pool),
);
const retries = new PostgresRunManualRetryRepository(options.pool);
const cancellations = new PostgresClusterRunCancellationRepository(
options.pool,
);
const audit = new PostgresSecurityAuditRepository(options.pool);
return Object.freeze({
async retry(requestValue: Readonly<ClusterRunManagementRetryRequest>) {
exactRequest(requestValue);
exactRetryRequest(requestValue);
const observedAtMs = now();
let principal: Readonly<SecurityPrincipal>;
if (
@@ -250,7 +305,8 @@ export function createClusterRunManagementService(
} catch (auditError) {
throw new ClusterRunManagementUnavailableError({ cause: auditError });
}
if (error instanceof ClusterRunManagementAuthorizationError) throw error;
if (error instanceof ClusterRunManagementAuthorizationError)
throw error;
if (error instanceof InvalidRunManualRetryError) {
throw new ClusterRunManagementRequestError();
}
@@ -269,5 +325,92 @@ export function createClusterRunManagementService(
throw new ClusterRunManagementUnavailableError({ cause: error });
}
},
async stop(requestValue: Readonly<ClusterRunManagementStopRequest>) {
exactStopRequest(requestValue);
const observedAtMs = now();
let principal: Readonly<SecurityPrincipal>;
if (
!Number.isSafeInteger(observedAtMs) ||
observedAtMs < 0 ||
!IDENTIFIER_PATTERN.test(requestValue.projectId) ||
!IDENTIFIER_PATTERN.test(requestValue.runId) ||
!IDENTIFIER_PATTERN.test(requestValue.requestId) ||
!validUuid(requestValue.mutationId) ||
!validUuid(requestValue.auditEventId) ||
!validUuid(requestValue.failureAuditEventId) ||
requestValue.auditEventId === requestValue.failureAuditEventId
) {
throw new ClusterRunManagementRequestError();
}
try {
principal = normalizeSecurityPrincipal(
requestValue.principal,
observedAtMs,
);
} catch {
throw new ClusterRunManagementRequestError();
}
let fence: Readonly<SecurityPolicyFence> | null = null;
try {
const decision = await policy.authorize(
principal,
requestValue.projectId,
'run.stop',
);
fence = decision.fence;
if (
decision.effect !== 'allow' ||
!fence ||
fence.bindingVersion === null
) {
throw new ClusterRunManagementAuthorizationError();
}
return await cancellations.requestUserCancellationAudited({
projectId: requestValue.projectId,
runId: requestValue.runId,
mutationId: requestValue.mutationId,
eventId: createId(),
requestId: requestValue.requestId,
auditEventId: requestValue.auditEventId,
principal,
policyFence: fence,
});
} catch (error) {
try {
await audit.record(
normalizeSecurityAuditRecord({
eventId: requestValue.failureAuditEventId,
requestId: requestValue.requestId,
operationId: 'run.stop',
projectId: requestValue.projectId,
subject: principal.subject,
authenticationId: principal.authenticationId,
outcome: 'denied',
reasons: [failureReason(error)],
fence,
occurredAtMs: observedAtMs,
}),
);
} catch (auditError) {
throw new ClusterRunManagementUnavailableError({ cause: auditError });
}
if (error instanceof ClusterRunManagementAuthorizationError)
throw error;
if (error instanceof InvalidClusterRunCancellationError) {
throw new ClusterRunManagementRequestError();
}
if (error instanceof ClusterRunCancellationNotFoundError) {
throw new ClusterRunManagementTargetUnavailableError();
}
if (error instanceof ClusterRunCancellationFenceRejectedError) {
throw new ClusterRunManagementConflictError();
}
if (error instanceof ClusterRunCancellationUnavailableError) {
throw new ClusterRunManagementUnavailableError({ cause: error });
}
throw new ClusterRunManagementUnavailableError({ cause: error });
}
},
});
}
packages/ql3-cluster-admin/src/run-management/runManagementClient.ts
+71 -21
View File
@@ -2,6 +2,10 @@ import {
RUN_MANUAL_RETRY_SCHEMA,
normalizeRunManualRetryResult,
} from '@qinglong/runtime-core/run-manual-retry';
import {
RUN_CANCELLATION_SCHEMA,
normalizeRunCancellationResult,
} from '@qinglong/runtime-core/run-cancellation';
import {
ClusterPluginPackageManagementClientRequestError,
executeClusterAuthenticatedManagementClient,
@@ -28,7 +32,10 @@ function invalid(): never {
throw new ClusterPluginPackageManagementClientRequestError();
}
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
function exact(
value: unknown,
keys: readonly string[],
): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
const actual = Object.keys(value as object).sort();
const expected = [...keys].sort();
@@ -45,37 +52,80 @@ export function validateClusterRunManagementClientResult(
value: unknown,
command: Readonly<ClusterRunManagementCommand>,
): Readonly<ClusterRunManagementTransportResult> {
const envelope = exact(value, ['schemaVersion', 'operation', 'retry']);
if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
const retry = exact(envelope.retry, [
if (command.operation === 'run.retry') {
const envelope = exact(value, ['schemaVersion', 'operation', 'retry']);
if (
envelope.schemaVersion !== 1 ||
envelope.operation !== command.operation
) {
invalid();
}
const retry = exact(envelope.retry, [
'schema',
'status',
'projectId',
'sourceRunId',
'sourceRunStatus',
'sourceRunVersion',
'runId',
'retryOfRunId',
'taskId',
'taskRevision',
'attemptId',
'runStatus',
'runVersion',
'eventSequence',
'executorType',
'executionRevisionDigest',
'createdAtMs',
]);
if (retry.schema !== RUN_MANUAL_RETRY_SCHEMA) invalid();
try {
const { schema: _schema, ...result } = retry;
const normalized = normalizeRunManualRetryResult(result as never);
if (
normalized.projectId !== command.request.projectId ||
normalized.sourceRunId !== command.request.sourceRunId ||
normalized.sourceRunVersion !==
command.request.body.expectedRunVersion ||
normalized.sourceRunStatus !== command.request.body.expectedRunStatus ||
normalized.executorType !== 'remote_worker'
) {
invalid();
}
} catch {
invalid();
}
return Object.freeze(
envelope as unknown as ClusterRunManagementTransportResult,
);
}
const envelope = exact(value, ['schemaVersion', 'operation', 'stop']);
if (
envelope.schemaVersion !== 1 ||
envelope.operation !== command.operation
) {
invalid();
}
const stop = exact(envelope.stop, [
'schema',
'status',
'projectId',
'sourceRunId',
'sourceRunStatus',
'sourceRunVersion',
'runId',
'retryOfRunId',
'taskId',
'taskRevision',
'attemptId',
'runStatus',
'runVersion',
'eventSequence',
'executorType',
'executionRevisionDigest',
'createdAtMs',
...(Object.hasOwn(envelope.stop as object, 'cancelRequestedAtMs')
? ['cancelRequestedAtMs', 'cancelReason']
: []),
]);
if (retry.schema !== RUN_MANUAL_RETRY_SCHEMA) invalid();
if (stop.schema !== RUN_CANCELLATION_SCHEMA) invalid();
try {
const { schema: _schema, ...result } = retry;
const normalized = normalizeRunManualRetryResult(result as never);
const { schema: _schema, ...result } = stop;
const normalized = normalizeRunCancellationResult(result as never);
if (
normalized.projectId !== command.request.projectId ||
normalized.sourceRunId !== command.request.sourceRunId ||
normalized.sourceRunVersion !== command.request.body.expectedRunVersion ||
normalized.sourceRunStatus !== command.request.body.expectedRunStatus ||
normalized.executorType !== 'remote_worker'
normalized.runId !== command.request.runId
) {
invalid();
}
packages/ql3-cluster-admin/src/run-management/runManagementTransport.ts
+126 -31
View File
@@ -3,6 +3,11 @@ import {
parseRunManualRetryRequestBody,
type RunManualRetryResponseBody,
} from '@qinglong/runtime-core/run-manual-retry';
import {
createRunCancellationResponseBody,
parseRunCancellationRequestBody,
type RunCancellationResponseBody,
} from '@qinglong/runtime-core/run-cancellation';
import {
normalizeSecurityPrincipal,
type SecurityPrincipal,
@@ -14,7 +19,7 @@ const UUID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
export type ClusterRunManagementCommand = Readonly<{
export type ClusterRunManagementRetryCommand = Readonly<{
schemaVersion: 1;
operation: 'run.retry';
request: Readonly<{
@@ -32,12 +37,42 @@ export type ClusterRunManagementCommand = Readonly<{
}>;
}>;
export type ClusterRunManagementTransportResult = Readonly<{
export type ClusterRunManagementStopCommand = Readonly<{
schemaVersion: 1;
operation: 'run.stop';
request: Readonly<{
projectId: string;
runId: string;
requestId: string;
auditEventId: string;
failureAuditEventId: string;
body: Readonly<{
schema: 'qinglong/run-cancellation@v1';
mutationId: string;
}>;
}>;
}>;
export type ClusterRunManagementCommand =
| ClusterRunManagementRetryCommand
| ClusterRunManagementStopCommand;
export type ClusterRunManagementRetryTransportResult = Readonly<{
schemaVersion: 1;
operation: 'run.retry';
retry: Readonly<RunManualRetryResponseBody>;
}>;
export type ClusterRunManagementStopTransportResult = Readonly<{
schemaVersion: 1;
operation: 'run.stop';
stop: Readonly<RunCancellationResponseBody>;
}>;
export type ClusterRunManagementTransportResult =
| ClusterRunManagementRetryTransportResult
| ClusterRunManagementStopTransportResult;
export interface ClusterRunManagementAuthentication {
authenticate(): Promise<Readonly<SecurityPrincipal> | null>;
}
@@ -85,7 +120,10 @@ function invalid(): never {
throw new ClusterRunManagementTransportRequestError();
}
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
function exact(
value: unknown,
keys: readonly string[],
): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
const actual = Object.keys(value as object).sort();
const expected = [...keys].sort();
@@ -112,30 +150,65 @@ export function normalizeClusterRunManagementCommand(
value: unknown,
): Readonly<ClusterRunManagementCommand> {
const envelope = exact(value, ['schemaVersion', 'operation', 'request']);
if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
const request = exact(envelope.request, [
'projectId',
'sourceRunId',
'requestId',
'auditEventId',
'failureAuditEventId',
'body',
]);
let body: ReturnType<typeof parseRunManualRetryRequestBody>;
try {
body = parseRunManualRetryRequestBody(request.body);
} catch {
invalid();
}
if (envelope.schemaVersion !== 1) invalid();
const operation = envelope.operation;
if (operation !== 'run.retry' && operation !== 'run.stop') invalid();
const request = exact(
envelope.request,
operation === 'run.retry'
? [
'projectId',
'sourceRunId',
'requestId',
'auditEventId',
'failureAuditEventId',
'body',
]
: [
'projectId',
'runId',
'requestId',
'auditEventId',
'failureAuditEventId',
'body',
],
);
const auditEventId = uuid(request.auditEventId);
const failureAuditEventId = uuid(request.failureAuditEventId);
if (auditEventId === failureAuditEventId) invalid();
if (operation === 'run.retry') {
let body: ReturnType<typeof parseRunManualRetryRequestBody>;
try {
body = parseRunManualRetryRequestBody(request.body);
} catch {
invalid();
}
return Object.freeze({
schemaVersion: 1,
operation,
request: Object.freeze({
projectId: identifier(request.projectId),
sourceRunId: identifier(request.sourceRunId),
requestId: identifier(request.requestId),
auditEventId,
failureAuditEventId,
body,
}),
});
}
let body: ReturnType<typeof parseRunCancellationRequestBody>;
try {
body = parseRunCancellationRequestBody(request.body);
} catch {
invalid();
}
uuid(body.mutationId);
return Object.freeze({
schemaVersion: 1,
operation: 'run.retry',
operation,
request: Object.freeze({
projectId: identifier(request.projectId),
sourceRunId: identifier(request.sourceRunId),
runId: identifier(request.runId),
requestId: identifier(request.requestId),
auditEventId,
failureAuditEventId,
@@ -144,10 +217,12 @@ export function normalizeClusterRunManagementCommand(
});
}
export function createClusterRunManagementTransport(options: Readonly<{
service: ClusterRunManagementService;
now?: () => number;
}>): Readonly<ClusterRunManagementTransport> {
export function createClusterRunManagementTransport(
options: Readonly<{
service: ClusterRunManagementService;
now?: () => number;
}>,
): Readonly<ClusterRunManagementTransport> {
if (
!options ||
typeof options !== 'object' ||
@@ -155,6 +230,7 @@ export function createClusterRunManagementTransport(options: Readonly<{
Object.keys(options).some((key) => key !== 'service' && key !== 'now') ||
!options.service ||
typeof options.service.retry !== 'function' ||
typeof options.service.stop !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function')
) {
throw new ClusterRunManagementTransportConfigurationError();
@@ -183,7 +259,10 @@ export function createClusterRunManagementTransport(options: Readonly<{
}
let principal: Readonly<SecurityPrincipal>;
try {
principal = normalizeSecurityPrincipal(candidate as SecurityPrincipal, now());
principal = normalizeSecurityPrincipal(
candidate as SecurityPrincipal,
now(),
);
} catch {
throw new ClusterRunManagementTransportAuthenticationError();
}
@@ -193,12 +272,28 @@ export function createClusterRunManagementTransport(options: Readonly<{
) {
throw new ClusterRunManagementTransportAuthenticationError();
}
const result = await options.service.retry({
if (command.operation === 'run.retry') {
const result = await options.service.retry({
projectId: command.request.projectId,
sourceRunId: command.request.sourceRunId,
mutationId: command.request.body.mutationId,
expectedRunVersion: command.request.body.expectedRunVersion,
expectedRunStatus: command.request.body.expectedRunStatus,
requestId: command.request.requestId,
auditEventId: command.request.auditEventId,
failureAuditEventId: command.request.failureAuditEventId,
principal,
});
return Object.freeze({
schemaVersion: 1,
operation: command.operation,
retry: createRunManualRetryResponseBody(result),
});
}
const result = await options.service.stop({
projectId: command.request.projectId,
sourceRunId: command.request.sourceRunId,
runId: command.request.runId,
mutationId: command.request.body.mutationId,
expectedRunVersion: command.request.body.expectedRunVersion,
expectedRunStatus: command.request.body.expectedRunStatus,
requestId: command.request.requestId,
auditEventId: command.request.auditEventId,
failureAuditEventId: command.request.failureAuditEventId,
@@ -206,8 +301,8 @@ export function createClusterRunManagementTransport(options: Readonly<{
});
return Object.freeze({
schemaVersion: 1,
operation: 'run.retry',
retry: createRunManualRetryResponseBody(result),
operation: command.operation,
stop: createRunCancellationResponseBody(result),
});
},
});