feat(ql3): page blocked cancellations

This commit is contained in:
whyour
2026-08-19 23:51:14 +08:00
parent 095683a4cb
commit cf21e984cb
31 changed files with 1466 additions and 42 deletions
@@ -0,0 +1,191 @@
import { randomUUID } from 'node:crypto';
import type { ClusterRunManagementClientResult } from './runManagementClient';
import {
RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA,
normalizeClusterRunManagementCommand,
type ClusterRunManagementCancellationBlockedListCommand,
type ClusterRunManagementCancellationBlockedListTransportResult,
} from './runManagementTransport';
export const RUN_CANCELLATION_BLOCKED_LIST_SCHEMA =
'qinglong/run-cancellation-blocked-list@v1' as const;
const CURSOR_PREFIX = 'v1.';
const CURSOR_PAYLOAD_PATTERN = /^[A-Za-z0-9_-]{1,512}$/;
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
type BlockedCursor = NonNullable<
ClusterRunManagementCancellationBlockedListCommand['request']['body']['after']
>;
export interface RunCancellationBlockedListObservation {
readonly schemaVersion: 1;
readonly schema: typeof RUN_CANCELLATION_BLOCKED_LIST_SCHEMA;
readonly component: 'qinglong3-run-management-client';
readonly event: 'cancellation_blocked_list_observed';
readonly requestId: string;
readonly projectId: string;
readonly snapshotAtMs: number;
readonly observedAtMs: number;
readonly items: readonly Readonly<{
runId: string;
blockedAtMs: number;
}>[];
readonly truncated: boolean;
readonly nextCursor?: string;
}
function invalidCursor(): never {
throw new TypeError('Run cancellation blocked cursor is invalid');
}
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
invalidCursor();
}
const actual = Object.keys(value as object).sort();
const expected = [...keys].sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index])
) {
invalidCursor();
}
return value as Record<string, unknown>;
}
function normalizeCursor(value: unknown): Readonly<BlockedCursor> {
const cursor = exact(value, ['snapshotAtMs', 'blockedAtMs', 'runId']);
if (
typeof cursor.snapshotAtMs !== 'number' ||
!Number.isSafeInteger(cursor.snapshotAtMs) ||
cursor.snapshotAtMs < 0 ||
typeof cursor.blockedAtMs !== 'number' ||
!Number.isSafeInteger(cursor.blockedAtMs) ||
cursor.blockedAtMs < 0 ||
cursor.blockedAtMs > cursor.snapshotAtMs ||
typeof cursor.runId !== 'string' ||
!IDENTIFIER_PATTERN.test(cursor.runId)
) {
invalidCursor();
}
return Object.freeze({
snapshotAtMs: cursor.snapshotAtMs,
blockedAtMs: cursor.blockedAtMs,
runId: cursor.runId,
});
}
export function decodeRunCancellationBlockedCursor(
token: string,
): Readonly<BlockedCursor> {
if (typeof token !== 'string' || !token.startsWith(CURSOR_PREFIX)) {
invalidCursor();
}
const encoded = token.slice(CURSOR_PREFIX.length);
if (!CURSOR_PAYLOAD_PATTERN.test(encoded)) invalidCursor();
let bytes: Buffer;
let parsed: unknown;
try {
bytes = Buffer.from(encoded, 'base64url');
if (
bytes.length < 2 ||
bytes.length > 384 ||
bytes.toString('base64url') !== encoded
) {
invalidCursor();
}
parsed = JSON.parse(bytes.toString('utf8'));
} catch {
invalidCursor();
}
return normalizeCursor(parsed);
}
export function encodeRunCancellationBlockedCursor(
value: Readonly<BlockedCursor>,
): string {
const cursor = normalizeCursor(value);
return `${CURSOR_PREFIX}${Buffer.from(JSON.stringify(cursor)).toString(
'base64url',
)}`;
}
export function createRunCancellationBlockedListCommand(
projectId: string,
cursorToken?: string,
createUuid: () => string = randomUUID,
): Readonly<ClusterRunManagementCancellationBlockedListCommand> {
const requestId = createUuid();
const auditEventId = createUuid();
let failureAuditEventId = createUuid();
for (
let attempts = 0;
failureAuditEventId === auditEventId && attempts < 3;
attempts += 1
) {
failureAuditEventId = createUuid();
}
return normalizeClusterRunManagementCommand({
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
request: {
projectId,
requestId,
auditEventId,
failureAuditEventId,
body: {
schema: RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA,
after:
cursorToken === undefined
? null
: decodeRunCancellationBlockedCursor(cursorToken),
},
},
}) as Readonly<ClusterRunManagementCancellationBlockedListCommand>;
}
export function projectRunCancellationBlockedList(
result: Readonly<ClusterRunManagementClientResult>,
): Readonly<RunCancellationBlockedListObservation> {
if (result.result.operation !== 'run.cancellation.blocked.list') {
throw new TypeError('Run cancellation blocked list requires a list result');
}
const page = (
result.result as ClusterRunManagementCancellationBlockedListTransportResult
).page;
return Object.freeze({
schemaVersion: 1,
schema: RUN_CANCELLATION_BLOCKED_LIST_SCHEMA,
component: 'qinglong3-run-management-client',
event: 'cancellation_blocked_list_observed',
requestId: result.requestId,
projectId: page.projectId,
snapshotAtMs: page.snapshotAtMs,
observedAtMs: page.observedAtMs,
items: page.items,
truncated: page.truncated,
...(page.nextCursor === undefined
? {}
: { nextCursor: encodeRunCancellationBlockedCursor(page.nextCursor) }),
});
}
export function formatRunCancellationBlockedListCard(
observation: Readonly<RunCancellationBlockedListObservation>,
): string {
return [
'QingLong 3.0 / Blocked Cancellations',
`PROJECT ${observation.projectId}`,
`SNAPSHOT ${new Date(observation.snapshotAtMs).toISOString()}`,
`OBSERVED ${new Date(observation.observedAtMs).toISOString()}`,
`ITEMS ${observation.items.length}`,
...observation.items.map(
(item) =>
`BLOCKED ${new Date(item.blockedAtMs).toISOString()} ${item.runId}`,
),
`NEXT_CURSOR ${observation.nextCursor ?? '-'}`,
`REQUEST ${observation.requestId}`,
].join('\n');
}
@@ -11,6 +11,8 @@ import {
RunCancellationDispatchManagementNotFoundError,
RunCancellationDispatchManagementUnavailableError,
type BlockingCancellationDispatchResult,
type RunCancellationDispatchBlockedCursor,
type RunCancellationDispatchBlockedPage,
type RunCancellationDispatchDiagnostic,
type RunCancellationDispatchRearmReceipt,
type RunCancellationDispatchSummary,
@@ -84,6 +86,11 @@ export interface ClusterRunManagementCancellationSummaryRequest {
readonly principal: Readonly<SecurityPrincipal>;
}
export interface ClusterRunManagementCancellationBlockedListRequest
extends ClusterRunManagementCancellationSummaryRequest {
readonly after?: Readonly<RunCancellationDispatchBlockedCursor>;
}
export interface ClusterRunManagementCancellationRearmRequest
extends ClusterRunManagementCancellationInspectRequest {
readonly mutationId: string;
@@ -102,6 +109,9 @@ export interface ClusterRunManagementService {
summarizeCancellation(
request: Readonly<ClusterRunManagementCancellationSummaryRequest>,
): Promise<Readonly<RunCancellationDispatchSummary>>;
listBlockedCancellations(
request: Readonly<ClusterRunManagementCancellationBlockedListRequest>,
): Promise<Readonly<RunCancellationDispatchBlockedPage>>;
inspectCancellation(
request: Readonly<ClusterRunManagementCancellationInspectRequest>,
): Promise<Readonly<RunCancellationDispatchDiagnostic>>;
@@ -267,6 +277,34 @@ function exactCancellationSummaryRequest(
}
}
function exactCancellationBlockedListRequest(
value: unknown,
): asserts value is Readonly<ClusterRunManagementCancellationBlockedListRequest> {
const hasAfter =
value !== null &&
typeof value === 'object' &&
!Array.isArray(value) &&
Object.hasOwn(value, 'after');
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
Object.keys(value).sort().join('\0') !==
[
'auditEventId',
'failureAuditEventId',
'principal',
'projectId',
'requestId',
...(hasAfter ? ['after'] : []),
]
.sort()
.join('\0')
) {
throw new ClusterRunManagementRequestError();
}
}
function exactCancellationRearmRequest(
value: unknown,
): asserts value is Readonly<ClusterRunManagementCancellationRearmRequest> {
@@ -615,6 +653,99 @@ export function createClusterRunManagementService(
throw new ClusterRunManagementUnavailableError({ cause: error });
}
},
async listBlockedCancellations(
requestValue: Readonly<ClusterRunManagementCancellationBlockedListRequest>,
) {
exactCancellationBlockedListRequest(requestValue);
const observedAtMs = now();
let principal: Readonly<SecurityPrincipal>;
const after = requestValue.after;
if (
!Number.isSafeInteger(observedAtMs) ||
observedAtMs < 0 ||
!IDENTIFIER_PATTERN.test(requestValue.projectId) ||
!IDENTIFIER_PATTERN.test(requestValue.requestId) ||
!validUuid(requestValue.auditEventId) ||
!validUuid(requestValue.failureAuditEventId) ||
requestValue.auditEventId === requestValue.failureAuditEventId ||
(after !== undefined &&
(!after ||
typeof after !== 'object' ||
Array.isArray(after) ||
Object.keys(after).sort().join('\0') !==
['blockedAtMs', 'runId', 'snapshotAtMs'].join('\0') ||
!Number.isSafeInteger(after.snapshotAtMs) ||
after.snapshotAtMs < 0 ||
!Number.isSafeInteger(after.blockedAtMs) ||
after.blockedAtMs < 0 ||
after.blockedAtMs > after.snapshotAtMs ||
!IDENTIFIER_PATTERN.test(after.runId)))
) {
throw new ClusterRunManagementRequestError();
}
try {
principal = normalizeSecurityPrincipal(
requestValue.principal,
observedAtMs,
);
} catch {
throw new ClusterRunManagementRequestError();
}
let fence: Readonly<SecurityPolicyFence> | null = null;
try {
const decision = await policy.authorize(
principal,
requestValue.projectId,
'run.read',
);
fence = decision.fence;
if (
decision.effect !== 'allow' ||
!fence ||
fence.bindingVersion === null
) {
throw new ClusterRunManagementAuthorizationError();
}
return await cancellationDispatches.listBlocked({
projectId: requestValue.projectId,
requestId: requestValue.requestId,
auditEventId: requestValue.auditEventId,
principal,
policyFence: fence,
...(after === undefined ? {} : { after }),
});
} catch (error) {
try {
await audit.record(
normalizeSecurityAuditRecord({
eventId: requestValue.failureAuditEventId,
requestId: requestValue.requestId,
operationId: 'run.cancellation.blocked.list',
projectId: requestValue.projectId,
subject: principal.subject,
authenticationId: principal.authenticationId,
outcome: 'denied',
reasons: [failureReason(error)],
fence,
occurredAtMs: observedAtMs,
}),
);
} catch (auditError) {
throw new ClusterRunManagementUnavailableError({ cause: auditError });
}
if (error instanceof ClusterRunManagementAuthorizationError) throw error;
if (error instanceof InvalidRunCancellationDispatchManagementError) {
throw new ClusterRunManagementRequestError();
}
if (error instanceof RunCancellationDispatchManagementConflictError) {
throw new ClusterRunManagementConflictError();
}
if (error instanceof RunCancellationDispatchManagementUnavailableError) {
throw new ClusterRunManagementUnavailableError({ cause: error });
}
throw new ClusterRunManagementUnavailableError({ cause: error });
}
},
async inspectCancellation(
requestValue: Readonly<ClusterRunManagementCancellationInspectRequest>,
) {
@@ -8,6 +8,7 @@ import {
} from '@qinglong/runtime-core/run-cancellation';
import { RUN_STATUSES } from '@qinglong/runtime-core/run';
import {
CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT,
CANCELLATION_DISPATCH_RESULTS,
CANCELLATION_DISPATCH_STATUSES,
} from '@qinglong/runtime-core/cancellation-dispatch';
@@ -21,6 +22,7 @@ import {
} from '../management-support/pluginPackageManagementClient';
import {
RUN_CANCELLATION_DISPATCH_DIAGNOSTIC_SCHEMA,
RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_SCHEMA,
RUN_CANCELLATION_DISPATCH_REARM_RECEIPT_SCHEMA,
RUN_CANCELLATION_DISPATCH_SUMMARY_SCHEMA,
normalizeClusterRunManagementCommand,
@@ -29,6 +31,7 @@ import {
} from './runManagementTransport';
const MANAGEMENT_PATH = '/api/v3/runs/management';
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
export type ClusterRunManagementClientPaths =
ClusterPluginPackageManagementClientPaths;
@@ -231,6 +234,86 @@ export function validateClusterRunManagementClientResult(
envelope as unknown as ClusterRunManagementTransportResult,
);
}
if (command.operation === 'run.cancellation.blocked.list') {
const envelope = exact(value, ['schemaVersion', 'operation', 'page']);
if (
envelope.schemaVersion !== 1 ||
envelope.operation !== command.operation
) {
invalid();
}
const page = exact(envelope.page, [
'schema',
'projectId',
'snapshotAtMs',
'observedAtMs',
'items',
'truncated',
...(Object.hasOwn(envelope.page as object, 'nextCursor')
? ['nextCursor']
: []),
]);
if (
page.schema !== RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_SCHEMA ||
page.projectId !== command.request.projectId ||
!safeInteger(page.snapshotAtMs) ||
!safeInteger(page.observedAtMs) ||
(page.snapshotAtMs as number) > (page.observedAtMs as number) ||
(command.request.body.after === null &&
page.snapshotAtMs !== page.observedAtMs) ||
(command.request.body.after !== null &&
page.snapshotAtMs !== command.request.body.after.snapshotAtMs) ||
!Array.isArray(page.items) ||
page.items.length > CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT ||
typeof page.truncated !== 'boolean' ||
page.truncated !== Object.hasOwn(page, 'nextCursor') ||
(page.truncated &&
page.items.length !== CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT)
) {
invalid();
}
const items = page.items as unknown[];
let previous = command.request.body.after;
for (const value of items) {
const item = exact(value, ['runId', 'blockedAtMs']);
if (
typeof item.runId !== 'string' ||
!IDENTIFIER_PATTERN.test(item.runId) ||
!safeInteger(item.blockedAtMs) ||
(item.blockedAtMs as number) > (page.snapshotAtMs as number) ||
(previous !== null &&
((item.blockedAtMs as number) < previous.blockedAtMs ||
((item.blockedAtMs as number) === previous.blockedAtMs &&
item.runId <= previous.runId)))
) {
invalid();
}
previous = {
snapshotAtMs: page.snapshotAtMs as number,
blockedAtMs: item.blockedAtMs as number,
runId: item.runId,
};
}
if (page.truncated) {
const cursor = exact(page.nextCursor, [
'snapshotAtMs',
'blockedAtMs',
'runId',
]);
const last = previous;
if (
last === null ||
cursor.snapshotAtMs !== page.snapshotAtMs ||
cursor.blockedAtMs !== last.blockedAtMs ||
cursor.runId !== last.runId
) {
invalid();
}
}
return Object.freeze(
envelope as unknown as ClusterRunManagementTransportResult,
);
}
if (command.operation === 'run.cancellation.inspect') {
const envelope = exact(value, [
'schemaVersion',
@@ -5,6 +5,12 @@ import {
executeClusterRunManagementClient,
executeClusterRunManagementCommand,
} from './runManagementClient';
import {
createRunCancellationBlockedListCommand,
decodeRunCancellationBlockedCursor,
formatRunCancellationBlockedListCard,
projectRunCancellationBlockedList,
} from './runCancellationBlockedList';
import {
createRunCancellationStatusCommand,
formatRunCancellationStatusCard,
@@ -14,6 +20,7 @@ import {
const USAGE = [
'Usage: ql3-run-client --config=/absolute/client.json --command=/absolute/command.json --assertion=/absolute/assertion.jwt',
' ql3-run-client status --config=/absolute/client.json --assertion=/absolute/assertion.jwt --project=PROJECT [--format=text|json]',
' ql3-run-client blocked --config=/absolute/client.json --assertion=/absolute/assertion.jwt --project=PROJECT [--cursor=CURSOR] [--format=text|json]',
'',
'Status exit codes: 0=clear, 10=converging, 20=attention_required.',
].join('\n');
@@ -32,18 +39,31 @@ type RunManagementClientArguments =
assertionFile: string;
projectId: string;
format: 'text' | 'json';
}>
| Readonly<{
kind: 'blocked';
configFile: string;
assertionFile: string;
projectId: string;
cursor?: string;
format: 'text' | 'json';
}>;
function argumentsFrom(
argv: readonly string[],
): Readonly<RunManagementClientArguments> | null {
const statusCount = argv.filter((argument) => argument === 'status').length;
if (statusCount > 0) {
if (statusCount !== 1 || argv.length < 4 || argv.length > 5) return null;
const modes = argv.filter(
(argument) => argument === 'status' || argument === 'blocked',
);
if (modes.length > 0) {
if (modes.length !== 1 || argv.length < 4 || argv.length > 6) return null;
const kind = modes[0] as 'status' | 'blocked';
const values = new Map<string, string>();
for (const argument of argv) {
if (argument === 'status') continue;
const match = /^--(config|assertion|project|format)=(.+)$/.exec(argument);
if (argument === kind) continue;
const match = /^--(config|assertion|project|cursor|format)=(.+)$/.exec(
argument,
);
if (!match || values.has(match[1]!)) return null;
values.set(match[1]!, match[2]!);
}
@@ -54,17 +74,28 @@ function argumentsFrom(
!values.get('assertion')!.startsWith('/') ||
!values.has('project') ||
!PROJECT_ID.test(values.get('project')!) ||
(kind === 'status' && values.has('cursor')) ||
(values.has('format') &&
values.get('format') !== 'text' &&
values.get('format') !== 'json')
) {
return null;
}
if (kind === 'blocked' && values.has('cursor')) {
try {
decodeRunCancellationBlockedCursor(values.get('cursor')!);
} catch {
return null;
}
}
return Object.freeze({
kind: 'status',
kind,
configFile: values.get('config')!,
assertionFile: values.get('assertion')!,
projectId: values.get('project')!,
...(kind === 'blocked' && values.has('cursor')
? { cursor: values.get('cursor')! }
: {}),
format: (values.get('format') ?? 'text') as 'text' | 'json',
});
}
@@ -131,6 +162,23 @@ async function run(argv: readonly string[]): Promise<void> {
return;
}
try {
if (paths.kind === 'blocked') {
const result = await executeClusterRunManagementCommand({
configFile: paths.configFile,
assertionFile: paths.assertionFile,
command: createRunCancellationBlockedListCommand(
paths.projectId,
paths.cursor,
),
});
const page = projectRunCancellationBlockedList(result);
process.stdout.write(
paths.format === 'json'
? `${JSON.stringify(page)}\n`
: `${formatRunCancellationBlockedListCard(page)}\n`,
);
return;
}
if (paths.kind === 'status') {
const result = await executeClusterRunManagementCommand({
configFile: paths.configFile,
@@ -26,6 +26,10 @@ export const RUN_CANCELLATION_DISPATCH_SUMMARY_REQUEST_SCHEMA =
'qinglong/run-cancellation-dispatch-summary-request@v1';
export const RUN_CANCELLATION_DISPATCH_SUMMARY_SCHEMA =
'qinglong/run-cancellation-dispatch-summary@v1';
export const RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA =
'qinglong/run-cancellation-dispatch-blocked-list-request@v1';
export const RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_SCHEMA =
'qinglong/run-cancellation-dispatch-blocked-page@v1';
export const RUN_CANCELLATION_DISPATCH_DIAGNOSTIC_SCHEMA =
'qinglong/run-cancellation-dispatch-diagnostic@v1';
export const RUN_CANCELLATION_DISPATCH_REARM_REQUEST_SCHEMA =
@@ -96,6 +100,25 @@ export type ClusterRunManagementCancellationSummaryCommand = Readonly<{
}>;
}>;
export type ClusterRunManagementCancellationBlockedListCommand = Readonly<{
schemaVersion: 1;
operation: 'run.cancellation.blocked.list';
request: Readonly<{
projectId: string;
requestId: string;
auditEventId: string;
failureAuditEventId: string;
body: Readonly<{
schema: typeof RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA;
after: Readonly<{
snapshotAtMs: number;
blockedAtMs: number;
runId: string;
}> | null;
}>;
}>;
}>;
export type ClusterRunManagementCancellationRearmCommand = Readonly<{
schemaVersion: 1;
operation: 'run.cancellation.rearm';
@@ -123,6 +146,7 @@ export type ClusterRunManagementCommand =
| ClusterRunManagementRetryCommand
| ClusterRunManagementStopCommand
| ClusterRunManagementCancellationSummaryCommand
| ClusterRunManagementCancellationBlockedListCommand
| ClusterRunManagementCancellationInspectCommand
| ClusterRunManagementCancellationRearmCommand;
@@ -158,6 +182,19 @@ export type ClusterRunManagementCancellationSummaryTransportResult = Readonly<{
>;
}>;
export type ClusterRunManagementCancellationBlockedListTransportResult =
Readonly<{
schemaVersion: 1;
operation: 'run.cancellation.blocked.list';
page: Readonly<
Awaited<
ReturnType<ClusterRunManagementService['listBlockedCancellations']>
> & {
schema: typeof RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_SCHEMA;
}
>;
}>;
export type ClusterRunManagementCancellationRearmTransportResult = Readonly<{
schemaVersion: 1;
operation: 'run.cancellation.rearm';
@@ -172,6 +209,7 @@ export type ClusterRunManagementTransportResult =
| ClusterRunManagementRetryTransportResult
| ClusterRunManagementStopTransportResult
| ClusterRunManagementCancellationSummaryTransportResult
| ClusterRunManagementCancellationBlockedListTransportResult
| ClusterRunManagementCancellationInspectTransportResult
| ClusterRunManagementCancellationRearmTransportResult;
@@ -258,6 +296,7 @@ export function normalizeClusterRunManagementCommand(
operation !== 'run.retry' &&
operation !== 'run.stop' &&
operation !== 'run.cancellation.summary' &&
operation !== 'run.cancellation.blocked.list' &&
operation !== 'run.cancellation.inspect' &&
operation !== 'run.cancellation.rearm'
) {
@@ -274,7 +313,8 @@ export function normalizeClusterRunManagementCommand(
'failureAuditEventId',
'body',
]
: operation === 'run.cancellation.summary'
: operation === 'run.cancellation.summary' ||
operation === 'run.cancellation.blocked.list'
? [
'projectId',
'requestId',
@@ -333,6 +373,51 @@ export function normalizeClusterRunManagementCommand(
}),
});
}
if (operation === 'run.cancellation.blocked.list') {
const body = exact(request.body, ['schema', 'after']);
if (body.schema !== RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA) {
invalid();
}
let after: ClusterRunManagementCancellationBlockedListCommand['request']['body']['after'] =
null;
if (body.after !== null) {
const cursor = exact(body.after, [
'snapshotAtMs',
'blockedAtMs',
'runId',
]);
if (
typeof cursor.snapshotAtMs !== 'number' ||
!Number.isSafeInteger(cursor.snapshotAtMs) ||
cursor.snapshotAtMs < 0 ||
typeof cursor.blockedAtMs !== 'number' ||
!Number.isSafeInteger(cursor.blockedAtMs) ||
cursor.blockedAtMs < 0 ||
cursor.blockedAtMs > cursor.snapshotAtMs
) {
invalid();
}
after = Object.freeze({
snapshotAtMs: cursor.snapshotAtMs,
blockedAtMs: cursor.blockedAtMs,
runId: identifier(cursor.runId),
});
}
return Object.freeze({
schemaVersion: 1,
operation,
request: Object.freeze({
projectId: identifier(request.projectId),
requestId: identifier(request.requestId),
auditEventId,
failureAuditEventId,
body: Object.freeze({
schema: RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA,
after,
}),
}),
});
}
if (operation === 'run.cancellation.inspect') {
const body = exact(request.body, ['schema']);
if (body.schema !== RUN_CANCELLATION_DISPATCH_INSPECT_REQUEST_SCHEMA) {
@@ -433,6 +518,7 @@ export function createClusterRunManagementTransport(
typeof options.service.retry !== 'function' ||
typeof options.service.stop !== 'function' ||
typeof options.service.summarizeCancellation !== 'function' ||
typeof options.service.listBlockedCancellations !== 'function' ||
typeof options.service.inspectCancellation !== 'function' ||
typeof options.service.rearmCancellation !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function')
@@ -511,6 +597,26 @@ export function createClusterRunManagementTransport(
}),
});
}
if (command.operation === 'run.cancellation.blocked.list') {
const result = await options.service.listBlockedCancellations({
projectId: command.request.projectId,
requestId: command.request.requestId,
auditEventId: command.request.auditEventId,
failureAuditEventId: command.request.failureAuditEventId,
principal,
...(command.request.body.after === null
? {}
: { after: command.request.body.after }),
});
return Object.freeze({
schemaVersion: 1,
operation: command.operation,
page: Object.freeze({
schema: RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_SCHEMA,
...result,
}),
});
}
if (command.operation === 'run.cancellation.inspect') {
const result = await options.service.inspectCancellation({
projectId: command.request.projectId,
@@ -0,0 +1,104 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createRunCancellationBlockedListCommand,
decodeRunCancellationBlockedCursor,
encodeRunCancellationBlockedCursor,
formatRunCancellationBlockedListCard,
projectRunCancellationBlockedList,
} = require('../dist/run-management/runCancellationBlockedList.js');
const UUIDS = [
'019fa000-0000-4000-8000-000000000001',
'019fa000-0000-4000-8000-000000000002',
'019fa000-0000-4000-8000-000000000003',
];
test('round-trips one bounded opaque blocked cursor', () => {
const cursor = {
snapshotAtMs: 1_700_000_000_000,
blockedAtMs: 1_699_999_999_000,
runId: 'run-16',
};
const token = encodeRunCancellationBlockedCursor(cursor);
assert.match(token, /^v1\.[A-Za-z0-9_-]+$/);
assert.deepEqual(decodeRunCancellationBlockedCursor(token), cursor);
for (const invalid of [
'v2.abc',
'v1.***',
`${token}=`,
'v1.e30',
]) {
assert.throws(() => decodeRunCancellationBlockedCursor(invalid));
}
});
test('builds one fixed blocked list command with no caller limit', () => {
let index = 0;
const cursor = encodeRunCancellationBlockedCursor({
snapshotAtMs: 1_700_000_000_000,
blockedAtMs: 1_699_999_999_000,
runId: 'run-16',
});
const command = createRunCancellationBlockedListCommand(
'project-1',
cursor,
() => UUIDS[index++],
);
assert.equal(command.operation, 'run.cancellation.blocked.list');
assert.equal(command.request.projectId, 'project-1');
assert.equal(Object.hasOwn(command.request, 'runId'), false);
assert.equal(Object.hasOwn(command.request.body, 'limit'), false);
assert.deepEqual(command.request.body.after, {
snapshotAtMs: 1_700_000_000_000,
blockedAtMs: 1_699_999_999_000,
runId: 'run-16',
});
});
test('projects a low-sensitive page and renders a deterministic card', () => {
const result = {
schemaVersion: 1,
requestId: 'request-blocked-1',
result: {
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
page: {
schema: 'qinglong/run-cancellation-dispatch-blocked-page@v1',
projectId: 'project-1',
snapshotAtMs: 1_700_000_000_000,
observedAtMs: 1_700_000_000_100,
items: Array.from({ length: 16 }, (_, index) => ({
runId: `run-${index + 1}`,
blockedAtMs: 1_699_999_999_000 + index,
})),
truncated: true,
nextCursor: {
snapshotAtMs: 1_700_000_000_000,
blockedAtMs: 1_699_999_999_015,
runId: 'run-16',
},
},
},
};
const observation = projectRunCancellationBlockedList(result);
assert.equal(observation.schema, 'qinglong/run-cancellation-blocked-list@v1');
assert.equal(observation.items.length, 16);
assert.match(observation.nextCursor, /^v1\./);
const card = formatRunCancellationBlockedListCard(observation);
assert.match(card, /Blocked Cancellations/);
assert.match(card, /run-1/);
assert.match(card, /NEXT_CURSOR\s+v1\./);
for (const forbidden of [
'attemptId',
'lastResult',
'leaseOwner',
'leaseToken',
'\u001b',
]) {
assert.equal(card.includes(forbidden), false);
}
});
@@ -187,6 +187,18 @@ function fixture(role = 'operator', options = {}) {
rowCount: 1,
};
}
if (
text.startsWith(
'SELECT run_id AS "runId", updated_at_ms AS "blockedAtMs"',
)
) {
return {
rows: [
{ runId: 'run-1', blockedAtMs: String(NOW - 1_500) },
],
rowCount: 1,
};
}
if (
text.startsWith('SELECT attempt_id AS "attemptId"') &&
text.includes('FROM "ql3"."run_cancellation_dispatches"') &&
@@ -425,6 +437,35 @@ test('allows a viewer to summarize Project cancellation availability atomically'
);
});
test('allows a viewer to page blocked Run identities under run.read', async () => {
const { calls, service } = fixture('viewer');
const listRequest = {
projectId: 'project-1',
requestId: 'request-blocked-1',
auditEventId: '019f9500-0000-4000-8000-000000000071',
failureAuditEventId: '019f9500-0000-4000-8000-000000000072',
principal: request().principal,
};
const result = await service.listBlockedCancellations(listRequest);
assert.deepEqual(result.items, [
{ runId: 'run-1', blockedAtMs: NOW - 1_500 },
]);
assert.equal(result.snapshotAtMs, NOW);
assert.equal(result.truncated, false);
const list = calls.find(({ sql }) =>
sql.startsWith(
'SELECT run_id AS "runId", updated_at_ms AS "blockedAtMs"',
),
);
assert.deepEqual(list.params, ['project-1', NOW, null, '', 17]);
const audit = calls.find(
({ sql, params }) =>
sql.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
params[2] === 'run.cancellation.blocked.list',
);
assert.equal(audit.params[0], listRequest.auditEventId);
});
test('authorizes exact cancellation rearm and keeps the event identity server-side', async () => {
const { calls, service } = fixture();
const rearmRequest = {
@@ -147,6 +147,21 @@ const summaryCommand = normalizeClusterRunManagementCommand({
},
});
const blockedListCommand = normalizeClusterRunManagementCommand({
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
request: {
projectId: 'project-1',
requestId: 'request-blocked-1',
auditEventId: '019f9400-0000-4000-8000-000000000061',
failureAuditEventId: '019f9400-0000-4000-8000-000000000062',
body: {
schema: 'qinglong/run-cancellation-dispatch-blocked-list-request@v1',
after: null,
},
},
});
const rearmCommand = normalizeClusterRunManagementCommand({
schemaVersion: 1,
operation: 'run.cancellation.rearm',
@@ -412,6 +427,54 @@ test('validates the fixed low-sensitive Project cancellation summary', () => {
}
});
test('validates one snapshot-bound low-sensitive blocked page', () => {
const value = {
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
page: {
schema: 'qinglong/run-cancellation-dispatch-blocked-page@v1',
projectId: 'project-1',
snapshotAtMs: 1_000_000,
observedAtMs: 1_000_000,
items: [
{ runId: 'run-1', blockedAtMs: 999_100 },
{ runId: 'run-2', blockedAtMs: 999_200 },
],
truncated: false,
},
};
assert.deepEqual(
validateClusterRunManagementClientResult(value, blockedListCommand),
value,
);
for (const page of [
{ ...value.page, projectId: 'project-2' },
{ ...value.page, snapshotAtMs: 999_999 },
{
...value.page,
items: [value.page.items[1], value.page.items[0]],
},
{
...value.page,
items: [{ ...value.page.items[0], attemptId: 'attempt-1' }],
},
{
...value.page,
items: [{ ...value.page.items[0], lastResult: 'identity_mismatch' }],
},
{ ...value.page, truncated: true },
]) {
assert.throws(
() =>
validateClusterRunManagementClientResult(
{ ...value, page },
blockedListCommand,
),
ClusterPluginPackageManagementClientRequestError,
);
}
});
test('binds a rearm receipt to the exact dispatch version, result and delay fences', () => {
const value = {
schemaVersion: 1,
@@ -91,11 +91,27 @@ test('status mode calls only the summary operation and emits an alert exit', asy
authorized: request.socket.authorized,
command,
});
const bytes = Buffer.from(
JSON.stringify({
schemaVersion: 1,
requestId: command.request.requestId,
result: {
const managementResult =
command.operation === 'run.cancellation.blocked.list'
? {
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
page: {
schema:
'qinglong/run-cancellation-dispatch-blocked-page@v1',
projectId: 'project-1',
snapshotAtMs: 1_700_000_000_000,
observedAtMs: 1_700_000_000_000,
items: [
{
runId: 'run-1',
blockedAtMs: 1_699_999_999_000,
},
],
truncated: false,
},
}
: {
schemaVersion: 1,
operation: 'run.cancellation.summary',
summary: {
@@ -121,7 +137,12 @@ test('status mode calls only the summary operation and emits an alert exit', asy
},
oldestBlockedAtMs: 1_699_999_999_000,
},
},
};
const bytes = Buffer.from(
JSON.stringify({
schemaVersion: 1,
requestId: command.request.requestId,
result: managementResult,
}),
'utf8',
);
@@ -192,12 +213,37 @@ test('status mode calls only the summary operation and emits an alert exit', asy
schema: 'qinglong/run-cancellation-dispatch-summary-request@v1',
});
assert.equal(Object.hasOwn(requests[0].command.request, 'runId'), false);
const blocked = await runCli([
'blocked',
`--config=${configFile}`,
`--assertion=${assertionFile}`,
'--project=project-1',
'--format=json',
]);
assert.equal(blocked.status, 0, blocked.stderr);
assert.equal(blocked.stderr, '');
const blockedOutput = JSON.parse(blocked.stdout);
assert.equal(
blockedOutput.schema,
'qinglong/run-cancellation-blocked-list@v1',
);
assert.deepEqual(blockedOutput.items, [
{ runId: 'run-1', blockedAtMs: 1_699_999_999_000 },
]);
assert.equal(requests.length, 2);
assert.equal(requests[1].command.operation, 'run.cancellation.blocked.list');
assert.deepEqual(requests[1].command.request.body, {
schema: 'qinglong/run-cancellation-dispatch-blocked-list-request@v1',
after: null,
});
});
test('help documents status routing and invalid projects fail before I/O', async () => {
const help = await runCli(['--help']);
assert.equal(help.status, 0);
assert.match(help.stdout, /ql3-run-client status/);
assert.match(help.stdout, /ql3-run-client blocked/);
assert.match(help.stdout, /0=clear, 10=converging, 20=attention_required/);
const rejected = await runCli([
@@ -214,4 +260,14 @@ test('help documents status routing and invalid projects fail before I/O', async
event: 'usage_invalid',
code: 'QL3_RUN_MANAGEMENT_CLIENT_USAGE_INVALID',
});
const invalidCursor = await runCli([
'blocked',
'--config=/private/client.json',
'--assertion=/private/assertion.jwt',
'--project=project-1',
'--cursor=v1.***',
]);
assert.equal(invalidCursor.status, 64);
assert.equal(invalidCursor.stdout, '');
});
@@ -144,6 +144,16 @@ function summaryResult() {
};
}
function blockedListResult() {
return {
projectId: 'project-1',
snapshotAtMs: NOW,
observedAtMs: NOW,
items: [{ runId: 'run-1', blockedAtMs: NOW - 800 }],
truncated: false,
};
}
function rearmResult() {
return {
status: 'rearmed',
@@ -195,6 +205,24 @@ function summaryCommand(overrides = {}) {
};
}
function blockedListCommand(overrides = {}) {
return {
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
request: {
projectId: 'project-1',
requestId: 'request-blocked-1',
auditEventId: '019f9300-0000-4000-8000-000000000061',
failureAuditEventId: '019f9300-0000-4000-8000-000000000062',
body: {
schema: 'qinglong/run-cancellation-dispatch-blocked-list-request@v1',
after: null,
},
...overrides,
},
};
}
function rearmCommand(overrides = {}) {
return {
schemaVersion: 1,
@@ -232,6 +260,9 @@ test('routes one exact strong User retry and emits the shared response', async (
async summarizeCancellation() {
return summaryResult();
},
async listBlockedCancellations() {
return blockedListResult();
},
async inspectCancellation() {
return diagnosticResult();
},
@@ -271,6 +302,9 @@ test('routes one exact strong User stop and emits the shared response', async ()
async summarizeCancellation() {
return summaryResult();
},
async listBlockedCancellations() {
return blockedListResult();
},
async inspectCancellation() {
return diagnosticResult();
},
@@ -310,6 +344,9 @@ test('rejects weak or non-User identity before service authority', async () => {
async summarizeCancellation() {
return summaryResult();
},
async listBlockedCancellations() {
return blockedListResult();
},
async inspectCancellation() {
return diagnosticResult();
},
@@ -342,6 +379,7 @@ test('routes bounded cancellation inspection without lease capability data', asy
async retry() { return retryResult(); },
async stop() { return stopResult(); },
async summarizeCancellation() { return summaryResult(); },
async listBlockedCancellations() { return blockedListResult(); },
async inspectCancellation(request) {
calls.push(request);
return diagnosticResult();
@@ -376,6 +414,7 @@ test('routes one Project-scoped cancellation summary without Run identity', asyn
calls.push(request);
return summaryResult();
},
async listBlockedCancellations() { return blockedListResult(); },
async inspectCancellation() { return diagnosticResult(); },
async rearmCancellation() { return rearmResult(); },
},
@@ -398,6 +437,47 @@ test('routes one Project-scoped cancellation summary without Run identity', asyn
assert.equal(JSON.stringify(result).includes('leaseOwner'), false);
});
test('routes one fixed Project blocked page without dispatch internals', async () => {
const calls = [];
const transport = createClusterRunManagementTransport({
now: () => NOW,
service: {
async retry() { return retryResult(); },
async stop() { return stopResult(); },
async summarizeCancellation() { return summaryResult(); },
async listBlockedCancellations(request) {
calls.push(request);
return blockedListResult();
},
async inspectCancellation() { return diagnosticResult(); },
async rearmCancellation() { return rearmResult(); },
},
});
const result = await transport.execute(blockedListCommand(), {
authenticate: async () => principal(),
});
assert.equal(calls.length, 1);
assert.equal(calls[0].projectId, 'project-1');
assert.equal(Object.hasOwn(calls[0], 'after'), false);
assert.deepEqual(result, {
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
page: {
schema: 'qinglong/run-cancellation-dispatch-blocked-page@v1',
...blockedListResult(),
},
});
for (const forbidden of [
'attemptId',
'dispatchVersion',
'lastResult',
'leaseOwner',
'leaseToken',
]) {
assert.equal(JSON.stringify(result).includes(forbidden), false);
}
});
test('routes an exact blocked cancellation rearm receipt', async () => {
const calls = [];
const transport = createClusterRunManagementTransport({
@@ -406,6 +486,7 @@ test('routes an exact blocked cancellation rearm receipt', async () => {
async retry() { return retryResult(); },
async stop() { return stopResult(); },
async summarizeCancellation() { return summaryResult(); },
async listBlockedCancellations() { return blockedListResult(); },
async inspectCancellation() { return diagnosticResult(); },
async rearmCancellation(request) {
calls.push(request);