feat(ql3): page blocked cancellations

This commit is contained in:
whyour
2026-08-19 23:51:14 +08:00
parent 095683a4cb
commit cf21e984cb
31 changed files with 1466 additions and 42 deletions
@@ -0,0 +1,191 @@
import { randomUUID } from 'node:crypto';
import type { ClusterRunManagementClientResult } from './runManagementClient';
import {
RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA,
normalizeClusterRunManagementCommand,
type ClusterRunManagementCancellationBlockedListCommand,
type ClusterRunManagementCancellationBlockedListTransportResult,
} from './runManagementTransport';
export const RUN_CANCELLATION_BLOCKED_LIST_SCHEMA =
'qinglong/run-cancellation-blocked-list@v1' as const;
const CURSOR_PREFIX = 'v1.';
const CURSOR_PAYLOAD_PATTERN = /^[A-Za-z0-9_-]{1,512}$/;
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
type BlockedCursor = NonNullable<
ClusterRunManagementCancellationBlockedListCommand['request']['body']['after']
>;
export interface RunCancellationBlockedListObservation {
readonly schemaVersion: 1;
readonly schema: typeof RUN_CANCELLATION_BLOCKED_LIST_SCHEMA;
readonly component: 'qinglong3-run-management-client';
readonly event: 'cancellation_blocked_list_observed';
readonly requestId: string;
readonly projectId: string;
readonly snapshotAtMs: number;
readonly observedAtMs: number;
readonly items: readonly Readonly<{
runId: string;
blockedAtMs: number;
}>[];
readonly truncated: boolean;
readonly nextCursor?: string;
}
function invalidCursor(): never {
throw new TypeError('Run cancellation blocked cursor is invalid');
}
function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
invalidCursor();
}
const actual = Object.keys(value as object).sort();
const expected = [...keys].sort();
if (
actual.length !== expected.length ||
actual.some((key, index) => key !== expected[index])
) {
invalidCursor();
}
return value as Record<string, unknown>;
}
function normalizeCursor(value: unknown): Readonly<BlockedCursor> {
const cursor = exact(value, ['snapshotAtMs', 'blockedAtMs', 'runId']);
if (
typeof cursor.snapshotAtMs !== 'number' ||
!Number.isSafeInteger(cursor.snapshotAtMs) ||
cursor.snapshotAtMs < 0 ||
typeof cursor.blockedAtMs !== 'number' ||
!Number.isSafeInteger(cursor.blockedAtMs) ||
cursor.blockedAtMs < 0 ||
cursor.blockedAtMs > cursor.snapshotAtMs ||
typeof cursor.runId !== 'string' ||
!IDENTIFIER_PATTERN.test(cursor.runId)
) {
invalidCursor();
}
return Object.freeze({
snapshotAtMs: cursor.snapshotAtMs,
blockedAtMs: cursor.blockedAtMs,
runId: cursor.runId,
});
}
export function decodeRunCancellationBlockedCursor(
token: string,
): Readonly<BlockedCursor> {
if (typeof token !== 'string' || !token.startsWith(CURSOR_PREFIX)) {
invalidCursor();
}
const encoded = token.slice(CURSOR_PREFIX.length);
if (!CURSOR_PAYLOAD_PATTERN.test(encoded)) invalidCursor();
let bytes: Buffer;
let parsed: unknown;
try {
bytes = Buffer.from(encoded, 'base64url');
if (
bytes.length < 2 ||
bytes.length > 384 ||
bytes.toString('base64url') !== encoded
) {
invalidCursor();
}
parsed = JSON.parse(bytes.toString('utf8'));
} catch {
invalidCursor();
}
return normalizeCursor(parsed);
}
export function encodeRunCancellationBlockedCursor(
value: Readonly<BlockedCursor>,
): string {
const cursor = normalizeCursor(value);
return `${CURSOR_PREFIX}${Buffer.from(JSON.stringify(cursor)).toString(
'base64url',
)}`;
}
export function createRunCancellationBlockedListCommand(
projectId: string,
cursorToken?: string,
createUuid: () => string = randomUUID,
): Readonly<ClusterRunManagementCancellationBlockedListCommand> {
const requestId = createUuid();
const auditEventId = createUuid();
let failureAuditEventId = createUuid();
for (
let attempts = 0;
failureAuditEventId === auditEventId && attempts < 3;
attempts += 1
) {
failureAuditEventId = createUuid();
}
return normalizeClusterRunManagementCommand({
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
request: {
projectId,
requestId,
auditEventId,
failureAuditEventId,
body: {
schema: RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA,
after:
cursorToken === undefined
? null
: decodeRunCancellationBlockedCursor(cursorToken),
},
},
}) as Readonly<ClusterRunManagementCancellationBlockedListCommand>;
}
export function projectRunCancellationBlockedList(
result: Readonly<ClusterRunManagementClientResult>,
): Readonly<RunCancellationBlockedListObservation> {
if (result.result.operation !== 'run.cancellation.blocked.list') {
throw new TypeError('Run cancellation blocked list requires a list result');
}
const page = (
result.result as ClusterRunManagementCancellationBlockedListTransportResult
).page;
return Object.freeze({
schemaVersion: 1,
schema: RUN_CANCELLATION_BLOCKED_LIST_SCHEMA,
component: 'qinglong3-run-management-client',
event: 'cancellation_blocked_list_observed',
requestId: result.requestId,
projectId: page.projectId,
snapshotAtMs: page.snapshotAtMs,
observedAtMs: page.observedAtMs,
items: page.items,
truncated: page.truncated,
...(page.nextCursor === undefined
? {}
: { nextCursor: encodeRunCancellationBlockedCursor(page.nextCursor) }),
});
}
export function formatRunCancellationBlockedListCard(
observation: Readonly<RunCancellationBlockedListObservation>,
): string {
return [
'QingLong 3.0 / Blocked Cancellations',
`PROJECT ${observation.projectId}`,
`SNAPSHOT ${new Date(observation.snapshotAtMs).toISOString()}`,
`OBSERVED ${new Date(observation.observedAtMs).toISOString()}`,
`ITEMS ${observation.items.length}`,
...observation.items.map(
(item) =>
`BLOCKED ${new Date(item.blockedAtMs).toISOString()} ${item.runId}`,
),
`NEXT_CURSOR ${observation.nextCursor ?? '-'}`,
`REQUEST ${observation.requestId}`,
].join('\n');
}
@@ -11,6 +11,8 @@ import {
RunCancellationDispatchManagementNotFoundError,
RunCancellationDispatchManagementUnavailableError,
type BlockingCancellationDispatchResult,
type RunCancellationDispatchBlockedCursor,
type RunCancellationDispatchBlockedPage,
type RunCancellationDispatchDiagnostic,
type RunCancellationDispatchRearmReceipt,
type RunCancellationDispatchSummary,
@@ -84,6 +86,11 @@ export interface ClusterRunManagementCancellationSummaryRequest {
readonly principal: Readonly<SecurityPrincipal>;
}
export interface ClusterRunManagementCancellationBlockedListRequest
extends ClusterRunManagementCancellationSummaryRequest {
readonly after?: Readonly<RunCancellationDispatchBlockedCursor>;
}
export interface ClusterRunManagementCancellationRearmRequest
extends ClusterRunManagementCancellationInspectRequest {
readonly mutationId: string;
@@ -102,6 +109,9 @@ export interface ClusterRunManagementService {
summarizeCancellation(
request: Readonly<ClusterRunManagementCancellationSummaryRequest>,
): Promise<Readonly<RunCancellationDispatchSummary>>;
listBlockedCancellations(
request: Readonly<ClusterRunManagementCancellationBlockedListRequest>,
): Promise<Readonly<RunCancellationDispatchBlockedPage>>;
inspectCancellation(
request: Readonly<ClusterRunManagementCancellationInspectRequest>,
): Promise<Readonly<RunCancellationDispatchDiagnostic>>;
@@ -267,6 +277,34 @@ function exactCancellationSummaryRequest(
}
}
function exactCancellationBlockedListRequest(
value: unknown,
): asserts value is Readonly<ClusterRunManagementCancellationBlockedListRequest> {
const hasAfter =
value !== null &&
typeof value === 'object' &&
!Array.isArray(value) &&
Object.hasOwn(value, 'after');
if (
!value ||
typeof value !== 'object' ||
Array.isArray(value) ||
Object.keys(value).sort().join('\0') !==
[
'auditEventId',
'failureAuditEventId',
'principal',
'projectId',
'requestId',
...(hasAfter ? ['after'] : []),
]
.sort()
.join('\0')
) {
throw new ClusterRunManagementRequestError();
}
}
function exactCancellationRearmRequest(
value: unknown,
): asserts value is Readonly<ClusterRunManagementCancellationRearmRequest> {
@@ -615,6 +653,99 @@ export function createClusterRunManagementService(
throw new ClusterRunManagementUnavailableError({ cause: error });
}
},
async listBlockedCancellations(
requestValue: Readonly<ClusterRunManagementCancellationBlockedListRequest>,
) {
exactCancellationBlockedListRequest(requestValue);
const observedAtMs = now();
let principal: Readonly<SecurityPrincipal>;
const after = requestValue.after;
if (
!Number.isSafeInteger(observedAtMs) ||
observedAtMs < 0 ||
!IDENTIFIER_PATTERN.test(requestValue.projectId) ||
!IDENTIFIER_PATTERN.test(requestValue.requestId) ||
!validUuid(requestValue.auditEventId) ||
!validUuid(requestValue.failureAuditEventId) ||
requestValue.auditEventId === requestValue.failureAuditEventId ||
(after !== undefined &&
(!after ||
typeof after !== 'object' ||
Array.isArray(after) ||
Object.keys(after).sort().join('\0') !==
['blockedAtMs', 'runId', 'snapshotAtMs'].join('\0') ||
!Number.isSafeInteger(after.snapshotAtMs) ||
after.snapshotAtMs < 0 ||
!Number.isSafeInteger(after.blockedAtMs) ||
after.blockedAtMs < 0 ||
after.blockedAtMs > after.snapshotAtMs ||
!IDENTIFIER_PATTERN.test(after.runId)))
) {
throw new ClusterRunManagementRequestError();
}
try {
principal = normalizeSecurityPrincipal(
requestValue.principal,
observedAtMs,
);
} catch {
throw new ClusterRunManagementRequestError();
}
let fence: Readonly<SecurityPolicyFence> | null = null;
try {
const decision = await policy.authorize(
principal,
requestValue.projectId,
'run.read',
);
fence = decision.fence;
if (
decision.effect !== 'allow' ||
!fence ||
fence.bindingVersion === null
) {
throw new ClusterRunManagementAuthorizationError();
}
return await cancellationDispatches.listBlocked({
projectId: requestValue.projectId,
requestId: requestValue.requestId,
auditEventId: requestValue.auditEventId,
principal,
policyFence: fence,
...(after === undefined ? {} : { after }),
});
} catch (error) {
try {
await audit.record(
normalizeSecurityAuditRecord({
eventId: requestValue.failureAuditEventId,
requestId: requestValue.requestId,
operationId: 'run.cancellation.blocked.list',
projectId: requestValue.projectId,
subject: principal.subject,
authenticationId: principal.authenticationId,
outcome: 'denied',
reasons: [failureReason(error)],
fence,
occurredAtMs: observedAtMs,
}),
);
} catch (auditError) {
throw new ClusterRunManagementUnavailableError({ cause: auditError });
}
if (error instanceof ClusterRunManagementAuthorizationError) throw error;
if (error instanceof InvalidRunCancellationDispatchManagementError) {
throw new ClusterRunManagementRequestError();
}
if (error instanceof RunCancellationDispatchManagementConflictError) {
throw new ClusterRunManagementConflictError();
}
if (error instanceof RunCancellationDispatchManagementUnavailableError) {
throw new ClusterRunManagementUnavailableError({ cause: error });
}
throw new ClusterRunManagementUnavailableError({ cause: error });
}
},
async inspectCancellation(
requestValue: Readonly<ClusterRunManagementCancellationInspectRequest>,
) {
@@ -8,6 +8,7 @@ import {
} from '@qinglong/runtime-core/run-cancellation';
import { RUN_STATUSES } from '@qinglong/runtime-core/run';
import {
CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT,
CANCELLATION_DISPATCH_RESULTS,
CANCELLATION_DISPATCH_STATUSES,
} from '@qinglong/runtime-core/cancellation-dispatch';
@@ -21,6 +22,7 @@ import {
} from '../management-support/pluginPackageManagementClient';
import {
RUN_CANCELLATION_DISPATCH_DIAGNOSTIC_SCHEMA,
RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_SCHEMA,
RUN_CANCELLATION_DISPATCH_REARM_RECEIPT_SCHEMA,
RUN_CANCELLATION_DISPATCH_SUMMARY_SCHEMA,
normalizeClusterRunManagementCommand,
@@ -29,6 +31,7 @@ import {
} from './runManagementTransport';
const MANAGEMENT_PATH = '/api/v3/runs/management';
const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
export type ClusterRunManagementClientPaths =
ClusterPluginPackageManagementClientPaths;
@@ -231,6 +234,86 @@ export function validateClusterRunManagementClientResult(
envelope as unknown as ClusterRunManagementTransportResult,
);
}
if (command.operation === 'run.cancellation.blocked.list') {
const envelope = exact(value, ['schemaVersion', 'operation', 'page']);
if (
envelope.schemaVersion !== 1 ||
envelope.operation !== command.operation
) {
invalid();
}
const page = exact(envelope.page, [
'schema',
'projectId',
'snapshotAtMs',
'observedAtMs',
'items',
'truncated',
...(Object.hasOwn(envelope.page as object, 'nextCursor')
? ['nextCursor']
: []),
]);
if (
page.schema !== RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_SCHEMA ||
page.projectId !== command.request.projectId ||
!safeInteger(page.snapshotAtMs) ||
!safeInteger(page.observedAtMs) ||
(page.snapshotAtMs as number) > (page.observedAtMs as number) ||
(command.request.body.after === null &&
page.snapshotAtMs !== page.observedAtMs) ||
(command.request.body.after !== null &&
page.snapshotAtMs !== command.request.body.after.snapshotAtMs) ||
!Array.isArray(page.items) ||
page.items.length > CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT ||
typeof page.truncated !== 'boolean' ||
page.truncated !== Object.hasOwn(page, 'nextCursor') ||
(page.truncated &&
page.items.length !== CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT)
) {
invalid();
}
const items = page.items as unknown[];
let previous = command.request.body.after;
for (const value of items) {
const item = exact(value, ['runId', 'blockedAtMs']);
if (
typeof item.runId !== 'string' ||
!IDENTIFIER_PATTERN.test(item.runId) ||
!safeInteger(item.blockedAtMs) ||
(item.blockedAtMs as number) > (page.snapshotAtMs as number) ||
(previous !== null &&
((item.blockedAtMs as number) < previous.blockedAtMs ||
((item.blockedAtMs as number) === previous.blockedAtMs &&
item.runId <= previous.runId)))
) {
invalid();
}
previous = {
snapshotAtMs: page.snapshotAtMs as number,
blockedAtMs: item.blockedAtMs as number,
runId: item.runId,
};
}
if (page.truncated) {
const cursor = exact(page.nextCursor, [
'snapshotAtMs',
'blockedAtMs',
'runId',
]);
const last = previous;
if (
last === null ||
cursor.snapshotAtMs !== page.snapshotAtMs ||
cursor.blockedAtMs !== last.blockedAtMs ||
cursor.runId !== last.runId
) {
invalid();
}
}
return Object.freeze(
envelope as unknown as ClusterRunManagementTransportResult,
);
}
if (command.operation === 'run.cancellation.inspect') {
const envelope = exact(value, [
'schemaVersion',
@@ -5,6 +5,12 @@ import {
executeClusterRunManagementClient,
executeClusterRunManagementCommand,
} from './runManagementClient';
import {
createRunCancellationBlockedListCommand,
decodeRunCancellationBlockedCursor,
formatRunCancellationBlockedListCard,
projectRunCancellationBlockedList,
} from './runCancellationBlockedList';
import {
createRunCancellationStatusCommand,
formatRunCancellationStatusCard,
@@ -14,6 +20,7 @@ import {
const USAGE = [
'Usage: ql3-run-client --config=/absolute/client.json --command=/absolute/command.json --assertion=/absolute/assertion.jwt',
' ql3-run-client status --config=/absolute/client.json --assertion=/absolute/assertion.jwt --project=PROJECT [--format=text|json]',
' ql3-run-client blocked --config=/absolute/client.json --assertion=/absolute/assertion.jwt --project=PROJECT [--cursor=CURSOR] [--format=text|json]',
'',
'Status exit codes: 0=clear, 10=converging, 20=attention_required.',
].join('\n');
@@ -32,18 +39,31 @@ type RunManagementClientArguments =
assertionFile: string;
projectId: string;
format: 'text' | 'json';
}>
| Readonly<{
kind: 'blocked';
configFile: string;
assertionFile: string;
projectId: string;
cursor?: string;
format: 'text' | 'json';
}>;
function argumentsFrom(
argv: readonly string[],
): Readonly<RunManagementClientArguments> | null {
const statusCount = argv.filter((argument) => argument === 'status').length;
if (statusCount > 0) {
if (statusCount !== 1 || argv.length < 4 || argv.length > 5) return null;
const modes = argv.filter(
(argument) => argument === 'status' || argument === 'blocked',
);
if (modes.length > 0) {
if (modes.length !== 1 || argv.length < 4 || argv.length > 6) return null;
const kind = modes[0] as 'status' | 'blocked';
const values = new Map<string, string>();
for (const argument of argv) {
if (argument === 'status') continue;
const match = /^--(config|assertion|project|format)=(.+)$/.exec(argument);
if (argument === kind) continue;
const match = /^--(config|assertion|project|cursor|format)=(.+)$/.exec(
argument,
);
if (!match || values.has(match[1]!)) return null;
values.set(match[1]!, match[2]!);
}
@@ -54,17 +74,28 @@ function argumentsFrom(
!values.get('assertion')!.startsWith('/') ||
!values.has('project') ||
!PROJECT_ID.test(values.get('project')!) ||
(kind === 'status' && values.has('cursor')) ||
(values.has('format') &&
values.get('format') !== 'text' &&
values.get('format') !== 'json')
) {
return null;
}
if (kind === 'blocked' && values.has('cursor')) {
try {
decodeRunCancellationBlockedCursor(values.get('cursor')!);
} catch {
return null;
}
}
return Object.freeze({
kind: 'status',
kind,
configFile: values.get('config')!,
assertionFile: values.get('assertion')!,
projectId: values.get('project')!,
...(kind === 'blocked' && values.has('cursor')
? { cursor: values.get('cursor')! }
: {}),
format: (values.get('format') ?? 'text') as 'text' | 'json',
});
}
@@ -131,6 +162,23 @@ async function run(argv: readonly string[]): Promise<void> {
return;
}
try {
if (paths.kind === 'blocked') {
const result = await executeClusterRunManagementCommand({
configFile: paths.configFile,
assertionFile: paths.assertionFile,
command: createRunCancellationBlockedListCommand(
paths.projectId,
paths.cursor,
),
});
const page = projectRunCancellationBlockedList(result);
process.stdout.write(
paths.format === 'json'
? `${JSON.stringify(page)}\n`
: `${formatRunCancellationBlockedListCard(page)}\n`,
);
return;
}
if (paths.kind === 'status') {
const result = await executeClusterRunManagementCommand({
configFile: paths.configFile,
@@ -26,6 +26,10 @@ export const RUN_CANCELLATION_DISPATCH_SUMMARY_REQUEST_SCHEMA =
'qinglong/run-cancellation-dispatch-summary-request@v1';
export const RUN_CANCELLATION_DISPATCH_SUMMARY_SCHEMA =
'qinglong/run-cancellation-dispatch-summary@v1';
export const RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA =
'qinglong/run-cancellation-dispatch-blocked-list-request@v1';
export const RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_SCHEMA =
'qinglong/run-cancellation-dispatch-blocked-page@v1';
export const RUN_CANCELLATION_DISPATCH_DIAGNOSTIC_SCHEMA =
'qinglong/run-cancellation-dispatch-diagnostic@v1';
export const RUN_CANCELLATION_DISPATCH_REARM_REQUEST_SCHEMA =
@@ -96,6 +100,25 @@ export type ClusterRunManagementCancellationSummaryCommand = Readonly<{
}>;
}>;
export type ClusterRunManagementCancellationBlockedListCommand = Readonly<{
schemaVersion: 1;
operation: 'run.cancellation.blocked.list';
request: Readonly<{
projectId: string;
requestId: string;
auditEventId: string;
failureAuditEventId: string;
body: Readonly<{
schema: typeof RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA;
after: Readonly<{
snapshotAtMs: number;
blockedAtMs: number;
runId: string;
}> | null;
}>;
}>;
}>;
export type ClusterRunManagementCancellationRearmCommand = Readonly<{
schemaVersion: 1;
operation: 'run.cancellation.rearm';
@@ -123,6 +146,7 @@ export type ClusterRunManagementCommand =
| ClusterRunManagementRetryCommand
| ClusterRunManagementStopCommand
| ClusterRunManagementCancellationSummaryCommand
| ClusterRunManagementCancellationBlockedListCommand
| ClusterRunManagementCancellationInspectCommand
| ClusterRunManagementCancellationRearmCommand;
@@ -158,6 +182,19 @@ export type ClusterRunManagementCancellationSummaryTransportResult = Readonly<{
>;
}>;
export type ClusterRunManagementCancellationBlockedListTransportResult =
Readonly<{
schemaVersion: 1;
operation: 'run.cancellation.blocked.list';
page: Readonly<
Awaited<
ReturnType<ClusterRunManagementService['listBlockedCancellations']>
> & {
schema: typeof RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_SCHEMA;
}
>;
}>;
export type ClusterRunManagementCancellationRearmTransportResult = Readonly<{
schemaVersion: 1;
operation: 'run.cancellation.rearm';
@@ -172,6 +209,7 @@ export type ClusterRunManagementTransportResult =
| ClusterRunManagementRetryTransportResult
| ClusterRunManagementStopTransportResult
| ClusterRunManagementCancellationSummaryTransportResult
| ClusterRunManagementCancellationBlockedListTransportResult
| ClusterRunManagementCancellationInspectTransportResult
| ClusterRunManagementCancellationRearmTransportResult;
@@ -258,6 +296,7 @@ export function normalizeClusterRunManagementCommand(
operation !== 'run.retry' &&
operation !== 'run.stop' &&
operation !== 'run.cancellation.summary' &&
operation !== 'run.cancellation.blocked.list' &&
operation !== 'run.cancellation.inspect' &&
operation !== 'run.cancellation.rearm'
) {
@@ -274,7 +313,8 @@ export function normalizeClusterRunManagementCommand(
'failureAuditEventId',
'body',
]
: operation === 'run.cancellation.summary'
: operation === 'run.cancellation.summary' ||
operation === 'run.cancellation.blocked.list'
? [
'projectId',
'requestId',
@@ -333,6 +373,51 @@ export function normalizeClusterRunManagementCommand(
}),
});
}
if (operation === 'run.cancellation.blocked.list') {
const body = exact(request.body, ['schema', 'after']);
if (body.schema !== RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA) {
invalid();
}
let after: ClusterRunManagementCancellationBlockedListCommand['request']['body']['after'] =
null;
if (body.after !== null) {
const cursor = exact(body.after, [
'snapshotAtMs',
'blockedAtMs',
'runId',
]);
if (
typeof cursor.snapshotAtMs !== 'number' ||
!Number.isSafeInteger(cursor.snapshotAtMs) ||
cursor.snapshotAtMs < 0 ||
typeof cursor.blockedAtMs !== 'number' ||
!Number.isSafeInteger(cursor.blockedAtMs) ||
cursor.blockedAtMs < 0 ||
cursor.blockedAtMs > cursor.snapshotAtMs
) {
invalid();
}
after = Object.freeze({
snapshotAtMs: cursor.snapshotAtMs,
blockedAtMs: cursor.blockedAtMs,
runId: identifier(cursor.runId),
});
}
return Object.freeze({
schemaVersion: 1,
operation,
request: Object.freeze({
projectId: identifier(request.projectId),
requestId: identifier(request.requestId),
auditEventId,
failureAuditEventId,
body: Object.freeze({
schema: RUN_CANCELLATION_DISPATCH_BLOCKED_LIST_REQUEST_SCHEMA,
after,
}),
}),
});
}
if (operation === 'run.cancellation.inspect') {
const body = exact(request.body, ['schema']);
if (body.schema !== RUN_CANCELLATION_DISPATCH_INSPECT_REQUEST_SCHEMA) {
@@ -433,6 +518,7 @@ export function createClusterRunManagementTransport(
typeof options.service.retry !== 'function' ||
typeof options.service.stop !== 'function' ||
typeof options.service.summarizeCancellation !== 'function' ||
typeof options.service.listBlockedCancellations !== 'function' ||
typeof options.service.inspectCancellation !== 'function' ||
typeof options.service.rearmCancellation !== 'function' ||
(options.now !== undefined && typeof options.now !== 'function')
@@ -511,6 +597,26 @@ export function createClusterRunManagementTransport(
}),
});
}
if (command.operation === 'run.cancellation.blocked.list') {
const result = await options.service.listBlockedCancellations({
projectId: command.request.projectId,
requestId: command.request.requestId,
auditEventId: command.request.auditEventId,
failureAuditEventId: command.request.failureAuditEventId,
principal,
...(command.request.body.after === null
? {}
: { after: command.request.body.after }),
});
return Object.freeze({
schemaVersion: 1,
operation: command.operation,
page: Object.freeze({
schema: RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_SCHEMA,
...result,
}),
});
}
if (command.operation === 'run.cancellation.inspect') {
const result = await options.service.inspectCancellation({
projectId: command.request.projectId,
@@ -0,0 +1,104 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
createRunCancellationBlockedListCommand,
decodeRunCancellationBlockedCursor,
encodeRunCancellationBlockedCursor,
formatRunCancellationBlockedListCard,
projectRunCancellationBlockedList,
} = require('../dist/run-management/runCancellationBlockedList.js');
const UUIDS = [
'019fa000-0000-4000-8000-000000000001',
'019fa000-0000-4000-8000-000000000002',
'019fa000-0000-4000-8000-000000000003',
];
test('round-trips one bounded opaque blocked cursor', () => {
const cursor = {
snapshotAtMs: 1_700_000_000_000,
blockedAtMs: 1_699_999_999_000,
runId: 'run-16',
};
const token = encodeRunCancellationBlockedCursor(cursor);
assert.match(token, /^v1\.[A-Za-z0-9_-]+$/);
assert.deepEqual(decodeRunCancellationBlockedCursor(token), cursor);
for (const invalid of [
'v2.abc',
'v1.***',
`${token}=`,
'v1.e30',
]) {
assert.throws(() => decodeRunCancellationBlockedCursor(invalid));
}
});
test('builds one fixed blocked list command with no caller limit', () => {
let index = 0;
const cursor = encodeRunCancellationBlockedCursor({
snapshotAtMs: 1_700_000_000_000,
blockedAtMs: 1_699_999_999_000,
runId: 'run-16',
});
const command = createRunCancellationBlockedListCommand(
'project-1',
cursor,
() => UUIDS[index++],
);
assert.equal(command.operation, 'run.cancellation.blocked.list');
assert.equal(command.request.projectId, 'project-1');
assert.equal(Object.hasOwn(command.request, 'runId'), false);
assert.equal(Object.hasOwn(command.request.body, 'limit'), false);
assert.deepEqual(command.request.body.after, {
snapshotAtMs: 1_700_000_000_000,
blockedAtMs: 1_699_999_999_000,
runId: 'run-16',
});
});
test('projects a low-sensitive page and renders a deterministic card', () => {
const result = {
schemaVersion: 1,
requestId: 'request-blocked-1',
result: {
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
page: {
schema: 'qinglong/run-cancellation-dispatch-blocked-page@v1',
projectId: 'project-1',
snapshotAtMs: 1_700_000_000_000,
observedAtMs: 1_700_000_000_100,
items: Array.from({ length: 16 }, (_, index) => ({
runId: `run-${index + 1}`,
blockedAtMs: 1_699_999_999_000 + index,
})),
truncated: true,
nextCursor: {
snapshotAtMs: 1_700_000_000_000,
blockedAtMs: 1_699_999_999_015,
runId: 'run-16',
},
},
},
};
const observation = projectRunCancellationBlockedList(result);
assert.equal(observation.schema, 'qinglong/run-cancellation-blocked-list@v1');
assert.equal(observation.items.length, 16);
assert.match(observation.nextCursor, /^v1\./);
const card = formatRunCancellationBlockedListCard(observation);
assert.match(card, /Blocked Cancellations/);
assert.match(card, /run-1/);
assert.match(card, /NEXT_CURSOR\s+v1\./);
for (const forbidden of [
'attemptId',
'lastResult',
'leaseOwner',
'leaseToken',
'\u001b',
]) {
assert.equal(card.includes(forbidden), false);
}
});
@@ -187,6 +187,18 @@ function fixture(role = 'operator', options = {}) {
rowCount: 1,
};
}
if (
text.startsWith(
'SELECT run_id AS "runId", updated_at_ms AS "blockedAtMs"',
)
) {
return {
rows: [
{ runId: 'run-1', blockedAtMs: String(NOW - 1_500) },
],
rowCount: 1,
};
}
if (
text.startsWith('SELECT attempt_id AS "attemptId"') &&
text.includes('FROM "ql3"."run_cancellation_dispatches"') &&
@@ -425,6 +437,35 @@ test('allows a viewer to summarize Project cancellation availability atomically'
);
});
test('allows a viewer to page blocked Run identities under run.read', async () => {
const { calls, service } = fixture('viewer');
const listRequest = {
projectId: 'project-1',
requestId: 'request-blocked-1',
auditEventId: '019f9500-0000-4000-8000-000000000071',
failureAuditEventId: '019f9500-0000-4000-8000-000000000072',
principal: request().principal,
};
const result = await service.listBlockedCancellations(listRequest);
assert.deepEqual(result.items, [
{ runId: 'run-1', blockedAtMs: NOW - 1_500 },
]);
assert.equal(result.snapshotAtMs, NOW);
assert.equal(result.truncated, false);
const list = calls.find(({ sql }) =>
sql.startsWith(
'SELECT run_id AS "runId", updated_at_ms AS "blockedAtMs"',
),
);
assert.deepEqual(list.params, ['project-1', NOW, null, '', 17]);
const audit = calls.find(
({ sql, params }) =>
sql.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
params[2] === 'run.cancellation.blocked.list',
);
assert.equal(audit.params[0], listRequest.auditEventId);
});
test('authorizes exact cancellation rearm and keeps the event identity server-side', async () => {
const { calls, service } = fixture();
const rearmRequest = {
@@ -147,6 +147,21 @@ const summaryCommand = normalizeClusterRunManagementCommand({
},
});
const blockedListCommand = normalizeClusterRunManagementCommand({
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
request: {
projectId: 'project-1',
requestId: 'request-blocked-1',
auditEventId: '019f9400-0000-4000-8000-000000000061',
failureAuditEventId: '019f9400-0000-4000-8000-000000000062',
body: {
schema: 'qinglong/run-cancellation-dispatch-blocked-list-request@v1',
after: null,
},
},
});
const rearmCommand = normalizeClusterRunManagementCommand({
schemaVersion: 1,
operation: 'run.cancellation.rearm',
@@ -412,6 +427,54 @@ test('validates the fixed low-sensitive Project cancellation summary', () => {
}
});
test('validates one snapshot-bound low-sensitive blocked page', () => {
const value = {
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
page: {
schema: 'qinglong/run-cancellation-dispatch-blocked-page@v1',
projectId: 'project-1',
snapshotAtMs: 1_000_000,
observedAtMs: 1_000_000,
items: [
{ runId: 'run-1', blockedAtMs: 999_100 },
{ runId: 'run-2', blockedAtMs: 999_200 },
],
truncated: false,
},
};
assert.deepEqual(
validateClusterRunManagementClientResult(value, blockedListCommand),
value,
);
for (const page of [
{ ...value.page, projectId: 'project-2' },
{ ...value.page, snapshotAtMs: 999_999 },
{
...value.page,
items: [value.page.items[1], value.page.items[0]],
},
{
...value.page,
items: [{ ...value.page.items[0], attemptId: 'attempt-1' }],
},
{
...value.page,
items: [{ ...value.page.items[0], lastResult: 'identity_mismatch' }],
},
{ ...value.page, truncated: true },
]) {
assert.throws(
() =>
validateClusterRunManagementClientResult(
{ ...value, page },
blockedListCommand,
),
ClusterPluginPackageManagementClientRequestError,
);
}
});
test('binds a rearm receipt to the exact dispatch version, result and delay fences', () => {
const value = {
schemaVersion: 1,
@@ -91,11 +91,27 @@ test('status mode calls only the summary operation and emits an alert exit', asy
authorized: request.socket.authorized,
command,
});
const bytes = Buffer.from(
JSON.stringify({
schemaVersion: 1,
requestId: command.request.requestId,
result: {
const managementResult =
command.operation === 'run.cancellation.blocked.list'
? {
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
page: {
schema:
'qinglong/run-cancellation-dispatch-blocked-page@v1',
projectId: 'project-1',
snapshotAtMs: 1_700_000_000_000,
observedAtMs: 1_700_000_000_000,
items: [
{
runId: 'run-1',
blockedAtMs: 1_699_999_999_000,
},
],
truncated: false,
},
}
: {
schemaVersion: 1,
operation: 'run.cancellation.summary',
summary: {
@@ -121,7 +137,12 @@ test('status mode calls only the summary operation and emits an alert exit', asy
},
oldestBlockedAtMs: 1_699_999_999_000,
},
},
};
const bytes = Buffer.from(
JSON.stringify({
schemaVersion: 1,
requestId: command.request.requestId,
result: managementResult,
}),
'utf8',
);
@@ -192,12 +213,37 @@ test('status mode calls only the summary operation and emits an alert exit', asy
schema: 'qinglong/run-cancellation-dispatch-summary-request@v1',
});
assert.equal(Object.hasOwn(requests[0].command.request, 'runId'), false);
const blocked = await runCli([
'blocked',
`--config=${configFile}`,
`--assertion=${assertionFile}`,
'--project=project-1',
'--format=json',
]);
assert.equal(blocked.status, 0, blocked.stderr);
assert.equal(blocked.stderr, '');
const blockedOutput = JSON.parse(blocked.stdout);
assert.equal(
blockedOutput.schema,
'qinglong/run-cancellation-blocked-list@v1',
);
assert.deepEqual(blockedOutput.items, [
{ runId: 'run-1', blockedAtMs: 1_699_999_999_000 },
]);
assert.equal(requests.length, 2);
assert.equal(requests[1].command.operation, 'run.cancellation.blocked.list');
assert.deepEqual(requests[1].command.request.body, {
schema: 'qinglong/run-cancellation-dispatch-blocked-list-request@v1',
after: null,
});
});
test('help documents status routing and invalid projects fail before I/O', async () => {
const help = await runCli(['--help']);
assert.equal(help.status, 0);
assert.match(help.stdout, /ql3-run-client status/);
assert.match(help.stdout, /ql3-run-client blocked/);
assert.match(help.stdout, /0=clear, 10=converging, 20=attention_required/);
const rejected = await runCli([
@@ -214,4 +260,14 @@ test('help documents status routing and invalid projects fail before I/O', async
event: 'usage_invalid',
code: 'QL3_RUN_MANAGEMENT_CLIENT_USAGE_INVALID',
});
const invalidCursor = await runCli([
'blocked',
'--config=/private/client.json',
'--assertion=/private/assertion.jwt',
'--project=project-1',
'--cursor=v1.***',
]);
assert.equal(invalidCursor.status, 64);
assert.equal(invalidCursor.stdout, '');
});
@@ -144,6 +144,16 @@ function summaryResult() {
};
}
function blockedListResult() {
return {
projectId: 'project-1',
snapshotAtMs: NOW,
observedAtMs: NOW,
items: [{ runId: 'run-1', blockedAtMs: NOW - 800 }],
truncated: false,
};
}
function rearmResult() {
return {
status: 'rearmed',
@@ -195,6 +205,24 @@ function summaryCommand(overrides = {}) {
};
}
function blockedListCommand(overrides = {}) {
return {
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
request: {
projectId: 'project-1',
requestId: 'request-blocked-1',
auditEventId: '019f9300-0000-4000-8000-000000000061',
failureAuditEventId: '019f9300-0000-4000-8000-000000000062',
body: {
schema: 'qinglong/run-cancellation-dispatch-blocked-list-request@v1',
after: null,
},
...overrides,
},
};
}
function rearmCommand(overrides = {}) {
return {
schemaVersion: 1,
@@ -232,6 +260,9 @@ test('routes one exact strong User retry and emits the shared response', async (
async summarizeCancellation() {
return summaryResult();
},
async listBlockedCancellations() {
return blockedListResult();
},
async inspectCancellation() {
return diagnosticResult();
},
@@ -271,6 +302,9 @@ test('routes one exact strong User stop and emits the shared response', async ()
async summarizeCancellation() {
return summaryResult();
},
async listBlockedCancellations() {
return blockedListResult();
},
async inspectCancellation() {
return diagnosticResult();
},
@@ -310,6 +344,9 @@ test('rejects weak or non-User identity before service authority', async () => {
async summarizeCancellation() {
return summaryResult();
},
async listBlockedCancellations() {
return blockedListResult();
},
async inspectCancellation() {
return diagnosticResult();
},
@@ -342,6 +379,7 @@ test('routes bounded cancellation inspection without lease capability data', asy
async retry() { return retryResult(); },
async stop() { return stopResult(); },
async summarizeCancellation() { return summaryResult(); },
async listBlockedCancellations() { return blockedListResult(); },
async inspectCancellation(request) {
calls.push(request);
return diagnosticResult();
@@ -376,6 +414,7 @@ test('routes one Project-scoped cancellation summary without Run identity', asyn
calls.push(request);
return summaryResult();
},
async listBlockedCancellations() { return blockedListResult(); },
async inspectCancellation() { return diagnosticResult(); },
async rearmCancellation() { return rearmResult(); },
},
@@ -398,6 +437,47 @@ test('routes one Project-scoped cancellation summary without Run identity', asyn
assert.equal(JSON.stringify(result).includes('leaseOwner'), false);
});
test('routes one fixed Project blocked page without dispatch internals', async () => {
const calls = [];
const transport = createClusterRunManagementTransport({
now: () => NOW,
service: {
async retry() { return retryResult(); },
async stop() { return stopResult(); },
async summarizeCancellation() { return summaryResult(); },
async listBlockedCancellations(request) {
calls.push(request);
return blockedListResult();
},
async inspectCancellation() { return diagnosticResult(); },
async rearmCancellation() { return rearmResult(); },
},
});
const result = await transport.execute(blockedListCommand(), {
authenticate: async () => principal(),
});
assert.equal(calls.length, 1);
assert.equal(calls[0].projectId, 'project-1');
assert.equal(Object.hasOwn(calls[0], 'after'), false);
assert.deepEqual(result, {
schemaVersion: 1,
operation: 'run.cancellation.blocked.list',
page: {
schema: 'qinglong/run-cancellation-dispatch-blocked-page@v1',
...blockedListResult(),
},
});
for (const forbidden of [
'attemptId',
'dispatchVersion',
'lastResult',
'leaseOwner',
'leaseToken',
]) {
assert.equal(JSON.stringify(result).includes(forbidden), false);
}
});
test('routes an exact blocked cancellation rearm receipt', async () => {
const calls = [];
const transport = createClusterRunManagementTransport({
@@ -406,6 +486,7 @@ test('routes an exact blocked cancellation rearm receipt', async () => {
async retry() { return retryResult(); },
async stop() { return stopResult(); },
async summarizeCancellation() { return summaryResult(); },
async listBlockedCancellations() { return blockedListResult(); },
async inspectCancellation() { return diagnosticResult(); },
async rearmCancellation(request) {
calls.push(request);
@@ -5,11 +5,15 @@ export {
RunCancellationDispatchManagementConflictError,
RunCancellationDispatchManagementNotFoundError,
RunCancellationDispatchManagementUnavailableError,
RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT,
type BlockingCancellationDispatchResult,
type PostgresRunCancellationDispatchBlockedListCommand,
type PostgresRunCancellationDispatchInspectCommand,
type PostgresRunCancellationDispatchRearmCommand,
type PostgresRunCancellationDispatchSummaryCommand,
type RunCancellationDispatchDiagnostic,
type RunCancellationDispatchBlockedCursor,
type RunCancellationDispatchBlockedPage,
type RunCancellationDispatchRearmReceipt,
type RunCancellationDispatchSummary,
} from '../run-management/runCancellationDispatchManagementRepository';
@@ -343,5 +343,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest =
checksum:
'e78e24a06dc4c4dbdd859685f28b4bc837a8cfb279eb3512e0a57dc6d27eaaaa',
}),
Object.freeze({
id: 'pg-0068-cancellation-dispatch-project-keyset',
checksum:
'2fcac38386581189db63faacff325356f11c4529a8db9cef6be1a1ca706aaf10',
}),
]),
});
@@ -70,6 +70,7 @@ import { pg0064PluginPackageSecretBindingTransitionApprovalPlansMigration } from
import { pg0065ApprovedActionManualRecoveryMigration } from '../approved-action/pg-0065-approved-action-manual-recovery';
import { pg0066CancellationDispatchMigration } from '../run/migrations/pg-0066-cancellation-dispatch';
import { pg0067CancellationDispatchManagementMigration } from '../run-management/pg-0067-cancellation-dispatch-management';
import { pg0068CancellationDispatchProjectKeysetMigration } from '../run-management/pg-0068-cancellation-dispatch-project-keyset';
export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
Object.freeze({
@@ -145,5 +146,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMi
pg0065ApprovedActionManualRecoveryMigration,
pg0066CancellationDispatchMigration,
pg0067CancellationDispatchManagementMigration,
pg0068CancellationDispatchProjectKeysetMigration,
]),
});
@@ -0,0 +1,22 @@
import { CAPABILITIES_V66 } from './pg-0067-cancellation-dispatch-management';
import { definePostgresSqlMigration } from '../migrations/sqlMigration';
export const CAPABILITIES_V67 = CAPABILITIES_V66.replace(
'"run_cancellation_dispatch_management":1,',
'"run_cancellation_dispatch_blocked_list":1,"run_cancellation_dispatch_management":1,',
);
export const pg0068CancellationDispatchProjectKeysetMigration =
definePostgresSqlMigration({
id: 'pg-0068-cancellation-dispatch-project-keyset',
statements: [
`ALTER TABLE "ql3"."run_cancellation_dispatches" ADD COLUMN project_id varchar(128)`,
`UPDATE "ql3"."run_cancellation_dispatches" AS dispatch SET project_id = run.project_id FROM "ql3"."runs" AS run WHERE run.id = dispatch.run_id`,
`ALTER TABLE "ql3"."run_cancellation_dispatches" ALTER COLUMN project_id SET NOT NULL`,
`CREATE UNIQUE INDEX ql3_runs_project_id_uidx ON "ql3"."runs" (project_id, id)`,
`ALTER TABLE "ql3"."run_cancellation_dispatches" DROP CONSTRAINT ql3_run_cancellation_dispatch_run_fk`,
`ALTER TABLE "ql3"."run_cancellation_dispatches" ADD CONSTRAINT ql3_run_cancellation_dispatch_run_fk FOREIGN KEY (project_id, run_id) REFERENCES "ql3"."runs" (project_id, id) ON DELETE CASCADE ON UPDATE RESTRICT`,
`CREATE INDEX ql3_run_cancellation_dispatch_project_blocked_idx ON "ql3"."run_cancellation_dispatches" (project_id, updated_at_ms, run_id) WHERE status = 'blocked'`,
`DO $ql3$ BEGIN UPDATE "ql3"."schema_capabilities" SET contract_version = 67, migration_id = 'pg-0068-cancellation-dispatch-project-keyset', capabilities = '${CAPABILITIES_V67}'::jsonb, updated_at_ms = floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint WHERE contract_name = 'control-core' AND contract_version = 66 AND migration_id = 'pg-0067-cancellation-dispatch-management' AND capabilities = '${CAPABILITIES_V66}'::jsonb; IF NOT FOUND THEN RAISE EXCEPTION 'control-core capability is not at version 66' USING ERRCODE = 'check_violation'; END IF; END $ql3$`,
],
});
@@ -7,6 +7,7 @@ import {
} from '@qinglong/runtime-core';
import {
CANCELLATION_DISPATCH_BLOCKING_RESULTS,
CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT,
CANCELLATION_DISPATCH_RESULTS,
CANCELLATION_DISPATCH_STATUSES,
MAX_CANCELLATION_DISPATCH_RETRY_DELAY_MS,
@@ -83,6 +84,27 @@ export type RunCancellationDispatchSummary = Readonly<{
oldestBlockedAtMs?: number;
}>;
export const RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT =
CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT;
export type RunCancellationDispatchBlockedCursor = Readonly<{
snapshotAtMs: number;
blockedAtMs: number;
runId: string;
}>;
export type RunCancellationDispatchBlockedPage = Readonly<{
projectId: string;
snapshotAtMs: number;
observedAtMs: number;
items: readonly Readonly<{
runId: string;
blockedAtMs: number;
}>[];
truncated: boolean;
nextCursor?: Readonly<RunCancellationDispatchBlockedCursor>;
}>;
interface ProjectManagementAuthority {
readonly projectId: string;
readonly requestId: string;
@@ -98,6 +120,11 @@ interface ManagementAuthority extends ProjectManagementAuthority {
export interface PostgresRunCancellationDispatchSummaryCommand
extends ProjectManagementAuthority {}
export interface PostgresRunCancellationDispatchBlockedListCommand
extends ProjectManagementAuthority {
readonly after?: Readonly<RunCancellationDispatchBlockedCursor>;
}
export interface PostgresRunCancellationDispatchInspectCommand
extends ManagementAuthority {}
@@ -347,6 +374,48 @@ function normalizeSummaryCommand(
});
}
function normalizeBlockedCursor(
value: unknown,
): Readonly<RunCancellationDispatchBlockedCursor> {
const cursor = exact(value, ['snapshotAtMs', 'blockedAtMs', 'runId']);
const snapshotAtMs = boundedInteger(cursor.snapshotAtMs, 0);
const blockedAtMs = boundedInteger(cursor.blockedAtMs, 0, snapshotAtMs);
return Object.freeze({
snapshotAtMs,
blockedAtMs,
runId: identifier(cursor.runId),
});
}
function normalizeBlockedListCommand(
value: Readonly<PostgresRunCancellationDispatchBlockedListCommand>,
): Readonly<PostgresRunCancellationDispatchBlockedListCommand> {
const hasAfter =
value !== null &&
typeof value === 'object' &&
!Array.isArray(value) &&
Object.hasOwn(value, 'after');
const input = exact(value, [
'projectId',
'requestId',
'auditEventId',
'principal',
'policyFence',
...(hasAfter ? ['after'] : []),
]);
const authority = normalizeSummaryCommand({
projectId: input.projectId as string,
requestId: input.requestId as string,
auditEventId: input.auditEventId as string,
principal: input.principal as SecurityPrincipal,
policyFence: input.policyFence as SecurityPolicyFence,
});
return Object.freeze({
...authority,
...(hasAfter ? { after: normalizeBlockedCursor(input.after) } : {}),
});
}
function normalizeRearmCommand(
value: Readonly<PostgresRunCancellationDispatchRearmCommand>,
): Readonly<PostgresRunCancellationDispatchRearmCommand> {
@@ -474,6 +543,7 @@ async function recordAllowedAudit(
command: Readonly<ProjectManagementAuthority>,
operationId:
| 'run.cancellation.summary'
| 'run.cancellation.blocked.list'
| 'run.cancellation.inspect'
| 'run.cancellation.rearm',
observedAtMs: number,
@@ -603,6 +673,76 @@ function summaryProjection(
});
}
function storedIdentifier(row: Row, key: string): string {
const value = text(row, key);
if (!IDENTIFIER_PATTERN.test(value)) {
throw new TypeError(
`PostgreSQL cancellation management ${key} is invalid`,
);
}
return value;
}
function blockedPageProjection(
command: Readonly<PostgresRunCancellationDispatchBlockedListCommand>,
observedAtMs: number,
snapshotAtMs: number,
rows: readonly Row[],
): Readonly<RunCancellationDispatchBlockedPage> {
if (rows.length > RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT + 1) {
throw new TypeError(
'PostgreSQL cancellation management blocked page is invalid',
);
}
const projected = rows.map((row) =>
Object.freeze({
runId: storedIdentifier(row, 'runId'),
blockedAtMs: integer(row, 'blockedAtMs'),
}),
);
let previous = command.after;
for (const item of projected) {
if (
item.blockedAtMs > snapshotAtMs ||
(previous !== undefined &&
(item.blockedAtMs < previous.blockedAtMs ||
(item.blockedAtMs === previous.blockedAtMs &&
item.runId <= previous.runId)))
) {
throw new TypeError(
'PostgreSQL cancellation management blocked cursor order is invalid',
);
}
previous = Object.freeze({
snapshotAtMs,
blockedAtMs: item.blockedAtMs,
runId: item.runId,
});
}
const truncated =
projected.length > RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT;
const items = Object.freeze(
projected.slice(0, RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT),
);
const last = items.at(-1);
return Object.freeze({
projectId: command.projectId,
snapshotAtMs,
observedAtMs,
items,
truncated,
...(truncated && last
? {
nextCursor: Object.freeze({
snapshotAtMs,
blockedAtMs: last.blockedAtMs,
runId: last.runId,
}),
}
: {}),
});
}
function runStatus(row: Row): RunStatus {
const value = text(row, 'runStatus') as RunStatus;
if (!RUN_STATUSES.includes(value)) {
@@ -818,6 +958,53 @@ export class PostgresRunCancellationDispatchManagementRepository {
});
}
listBlocked(
value: Readonly<PostgresRunCancellationDispatchBlockedListCommand>,
): Promise<Readonly<RunCancellationDispatchBlockedPage>> {
const command = normalizeBlockedListCommand(value);
return this.transaction(async (client) => {
const observedAtMs = await databaseNow(client);
const snapshotAtMs = command.after?.snapshotAtMs ?? observedAtMs;
if (snapshotAtMs > observedAtMs) {
throw new InvalidRunCancellationDispatchManagementError();
}
const authorized = Object.freeze({
...command,
principal: strongPrincipal(command.principal, observedAtMs),
});
await confirmAuthorization(client, authorized);
const result = await client.query<Row>(
`SELECT run_id AS "runId", updated_at_ms AS "blockedAtMs"
FROM "ql3"."run_cancellation_dispatches"
WHERE project_id = $1 AND status = 'blocked'
AND updated_at_ms <= $2
AND ($3::bigint IS NULL OR
(updated_at_ms, run_id) > ($3::bigint, $4::varchar))
ORDER BY updated_at_ms ASC, run_id ASC
LIMIT $5`,
[
command.projectId,
snapshotAtMs,
command.after?.blockedAtMs ?? null,
command.after?.runId ?? '',
RUN_CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT + 1,
],
);
await recordAllowedAudit(
client,
authorized,
'run.cancellation.blocked.list',
observedAtMs,
);
return blockedPageProjection(
command,
observedAtMs,
snapshotAtMs,
result.rows,
);
});
}
inspect(
value: Readonly<PostgresRunCancellationDispatchInspectCommand>,
): Promise<Readonly<RunCancellationDispatchDiagnostic>> {
@@ -180,7 +180,7 @@ export class PostgresCancellationDispatchRepository
return this.transaction(async (client) => {
const nowMs = await databaseNow(client);
const run = await client.query<Row>(
`SELECT execution_owner AS "executionOwner", status,
`SELECT project_id AS "projectId", execution_owner AS "executionOwner", status,
cancel_requested_at_ms AS "cancelRequestedAtMs"
FROM "ql3"."runs" WHERE id = $1 FOR UPDATE`,
[command.runId],
@@ -222,11 +222,17 @@ export class PostgresCancellationDispatchRepository
if (dispatchResult.rows.length === 0) {
dispatchResult = await client.query<Row>(
`INSERT INTO "ql3"."run_cancellation_dispatches" (
run_id, attempt_id, status, version, dispatch_count,
project_id, run_id, attempt_id, status, version, dispatch_count,
next_attempt_at_ms, created_at_ms, updated_at_ms
) VALUES ($1, $2, 'pending', 0, 0, $3, $4, $4)
) VALUES ($5, $1, $2, 'pending', 0, 0, $3, $4, $4)
RETURNING ${DISPATCH_COLUMNS}`,
[command.runId, command.attemptId, command.requestedAtMs, nowMs],
[
command.runId,
command.attemptId,
command.requestedAtMs,
nowMs,
text(runRow, 'projectId'),
],
);
}
if (dispatchResult.rows.length !== 1) {
@@ -3592,6 +3592,7 @@ export const runs = ql3Schema.table(
uniqueIndex('ql3_runs_project_idempotency_uidx')
.on(table.projectId, table.idempotencyKey)
.where(sql`${table.idempotencyKey} is not null`),
uniqueIndex('ql3_runs_project_id_uidx').on(table.projectId, table.id),
index('ql3_runs_project_created_idx').on(
table.projectId,
table.createdAtMs,
@@ -4834,6 +4835,7 @@ export const runAttempts = ql3Schema.table(
export const runCancellationDispatches = ql3Schema.table(
'run_cancellation_dispatches',
{
projectId: varchar('project_id', { length: 128 }).notNull(),
runId: varchar('run_id', { length: 36 }).primaryKey(),
attemptId: varchar('attempt_id', { length: 36 }).notNull(),
status: varchar('status', { length: 32 }).notNull(),
@@ -4851,8 +4853,8 @@ export const runCancellationDispatches = ql3Schema.table(
(table) => [
foreignKey({
name: 'ql3_run_cancellation_dispatch_run_fk',
columns: [table.runId],
foreignColumns: [runs.id],
columns: [table.projectId, table.runId],
foreignColumns: [runs.projectId, runs.id],
})
.onDelete('cascade')
.onUpdate('restrict'),
@@ -4897,6 +4899,9 @@ export const runCancellationDispatches = ql3Schema.table(
index('ql3_run_cancellation_dispatch_lease_expiry_idx')
.on(table.leaseExpiresAtMs, table.runId)
.where(sql`${table.status} = 'leased'`),
index('ql3_run_cancellation_dispatch_project_blocked_idx')
.on(table.projectId, table.updatedAtMs, table.runId)
.where(sql`${table.status} = 'blocked'`),
],
);
@@ -21,13 +21,14 @@ export interface PostgresSchemaContractTrigger {
export interface PostgresSchemaContract {
readonly schema: 'ql3';
readonly contractName: 'control-core';
readonly contractVersion: 66;
readonly migrationId: 'pg-0067-cancellation-dispatch-management';
readonly contractVersion: 67;
readonly migrationId: 'pg-0068-cancellation-dispatch-project-keyset';
readonly minimumServerMajor: 16;
readonly maximumServerMajor: 18;
readonly capabilities: Readonly<{
run_core: 1;
run_cancellation_dispatch: 1;
run_cancellation_dispatch_blocked_list: 1;
run_cancellation_dispatch_management: 1;
run_attempt_log_retention: 1;
run_management_boundary: 1;
@@ -120,8 +121,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
Object.freeze({
schema: 'ql3',
contractName: 'control-core',
contractVersion: 66,
migrationId: 'pg-0067-cancellation-dispatch-management',
contractVersion: 67,
migrationId: 'pg-0068-cancellation-dispatch-project-keyset',
minimumServerMajor: 16,
maximumServerMajor: 18,
capabilities: Object.freeze({
@@ -170,6 +171,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
project_tool_definition_snapshot: 1,
run_core: 1,
run_cancellation_dispatch: 1,
run_cancellation_dispatch_blocked_list: 1,
run_cancellation_dispatch_management: 1,
run_attempt_log_retention: 1,
run_management_boundary: 1,
@@ -1294,6 +1296,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
'error_summary',
]),
table('run_cancellation_dispatches', [
'project_id',
'run_id',
'attempt_id',
'status',
@@ -1718,6 +1721,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
'ql3_api_credential_mutations_actor_idx',
'runs_pkey',
'ql3_runs_project_idempotency_uidx',
'ql3_runs_project_id_uidx',
'ql3_runs_project_created_idx',
'ql3_runs_task_created_idx',
'ql3_runs_dispatch_candidates_idx',
@@ -1796,6 +1800,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
'run_cancellation_dispatches_pkey',
'ql3_run_cancellation_dispatch_due_idx',
'ql3_run_cancellation_dispatch_lease_expiry_idx',
'ql3_run_cancellation_dispatch_project_blocked_idx',
'run_attempt_log_retention_controls_pkey',
'ql3_run_log_retention_control_artifact_key',
'ql3_run_log_retention_retry_idx',
@@ -118,6 +118,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
'pg-0065-approved-action-manual-recovery',
'pg-0066-cancellation-dispatch',
'pg-0067-cancellation-dispatch-management',
'pg-0068-cancellation-dispatch-project-keyset',
],
);
for (const migration of postgresqlMainMigrationStream.migrations) {
@@ -591,6 +592,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
checksum:
'e78e24a06dc4c4dbdd859685f28b4bc837a8cfb279eb3512e0a57dc6d27eaaaa',
},
{
id: 'pg-0068-cancellation-dispatch-project-keyset',
checksum:
'2fcac38386581189db63faacff325356f11c4529a8db9cef6be1a1ca706aaf10',
},
];
assert.deepEqual(
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
@@ -2373,3 +2379,45 @@ test('advances capability v66 with least-privilege cancellation diagnostics and
assert.match(sql, /contract_version = 65/);
assert.match(sql, /migration_id = 'pg-0066-cancellation-dispatch'/);
});
test('advances capability v67 with a Project-scoped blocked keyset', async () => {
const migration = migrationById(
'pg-0068-cancellation-dispatch-project-keyset',
);
const statements = [];
await migration.up({
async query(statement) {
statements.push(statement);
return { rows: [] };
},
});
const sql = statements.join('\n');
assert.match(
sql,
/ADD COLUMN project_id varchar\(128\)/,
);
assert.match(
sql,
/SET project_id = run\.project_id FROM "ql3"\."runs" AS run/,
);
assert.match(sql, /ALTER COLUMN project_id SET NOT NULL/);
assert.match(
sql,
/CREATE UNIQUE INDEX ql3_runs_project_id_uidx ON "ql3"\."runs" \(project_id, id\)/,
);
assert.match(
sql,
/FOREIGN KEY \(project_id, run_id\) REFERENCES "ql3"\."runs" \(project_id, id\)/,
);
assert.match(
sql,
/CREATE INDEX ql3_run_cancellation_dispatch_project_blocked_idx[\s\S]+\(project_id, updated_at_ms, run_id\) WHERE status = 'blocked'/,
);
assert.match(sql, /contract_version = 67/);
assert.match(sql, /"run_cancellation_dispatch_blocked_list":1/);
assert.match(sql, /contract_version = 66/);
assert.match(
sql,
/migration_id = 'pg-0067-cancellation-dispatch-management'/,
);
});
@@ -835,7 +835,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
serverMajor: 16,
currentUser: 'ql3_runtime',
contractName: 'control-core',
contractVersion: 66,
contractVersion: 67,
migrationIds: [
'pg-0001-schema-capability',
'pg-0002-run-core',
@@ -904,6 +904,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
'pg-0065-approved-action-manual-recovery',
'pg-0066-cancellation-dispatch',
'pg-0067-cancellation-dispatch-management',
'pg-0068-cancellation-dispatch-project-keyset',
],
});
});
@@ -934,10 +935,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
}),
);
assert.equal(report.currentUser, 'ql3_admin');
assert.equal(report.contractVersion, 66);
assert.equal(report.contractVersion, 67);
assert.equal(
report.migrationIds.at(-1),
'pg-0067-cancellation-dispatch-management',
'pg-0068-cancellation-dispatch-project-keyset',
);
});
@@ -950,10 +951,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
}),
);
assert.equal(report.currentUser, 'ql3_automation_manager');
assert.equal(report.contractVersion, 66);
assert.equal(report.contractVersion, 67);
assert.equal(
report.migrationIds.at(-1),
'pg-0067-cancellation-dispatch-management',
'pg-0068-cancellation-dispatch-project-keyset',
);
const widened = automationManagerPrivileges();
@@ -982,10 +983,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
}),
);
assert.equal(report.currentUser, 'ql3_approval_manager');
assert.equal(report.contractVersion, 66);
assert.equal(report.contractVersion, 67);
assert.equal(
report.migrationIds.at(-1),
'pg-0067-cancellation-dispatch-management',
'pg-0068-cancellation-dispatch-project-keyset',
);
const widened = approvalManagerPrivileges();
@@ -1016,10 +1017,10 @@ test('accepts the isolated least-privilege Run manager role', async () => {
}),
);
assert.equal(report.currentUser, 'ql3_run_manager');
assert.equal(report.contractVersion, 66);
assert.equal(report.contractVersion, 67);
assert.equal(
report.migrationIds.at(-1),
'pg-0067-cancellation-dispatch-management',
'pg-0068-cancellation-dispatch-project-keyset',
);
const widened = runManagerPrivileges();
@@ -1180,10 +1181,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
}),
);
assert.equal(report.currentUser, 'ql3_worker_ingress');
assert.equal(report.contractVersion, 66);
assert.equal(report.contractVersion, 67);
assert.equal(
report.migrationIds.at(-1),
'pg-0067-cancellation-dispatch-management',
'pg-0068-cancellation-dispatch-project-keyset',
);
});
@@ -47,6 +47,16 @@ function summaryCommand(overrides = {}) {
return { ...authority, ...overrides };
}
function blockedListCommand(after) {
return {
...summaryCommand({
requestId: 'request-blocked-1',
auditEventId: '019f9600-0000-4000-8000-000000000021',
}),
...(after === undefined ? {} : { after }),
};
}
function runRow() {
return {
projectId: 'project-1',
@@ -133,6 +143,13 @@ function fixture(options = {}) {
rowCount: 1,
};
}
if (
text.startsWith(
'SELECT run_id AS "runId", updated_at_ms AS "blockedAtMs"',
)
) {
return { rows: options.blockedRows ?? [], rowCount: 0 };
}
if (
text.startsWith('SELECT attempt_id AS "attemptId"') &&
!text.includes('dispatchStatus') &&
@@ -296,6 +313,70 @@ test('derives clear and converging assessments from fixed status counts', async
assert.equal(result.operatorAction, 'wait');
});
test('lists one fixed oldest-first blocked page with a snapshot cursor', async () => {
const blockedRows = Array.from({ length: 17 }, (_, index) => ({
runId: `run-${String(index + 1).padStart(2, '0')}`,
blockedAtMs: String(NOW - 100 + index),
}));
const { calls, repository } = fixture({ blockedRows });
const result = await repository.listBlocked(blockedListCommand());
assert.equal(result.projectId, 'project-1');
assert.equal(result.snapshotAtMs, NOW);
assert.equal(result.observedAtMs, NOW);
assert.equal(result.items.length, 16);
assert.equal(result.items[0].runId, 'run-01');
assert.equal(result.items[15].runId, 'run-16');
assert.equal(result.truncated, true);
assert.deepEqual(result.nextCursor, {
snapshotAtMs: NOW,
blockedAtMs: NOW - 85,
runId: 'run-16',
});
const read = calls.find(({ sql }) =>
sql.startsWith(
'SELECT run_id AS "runId", updated_at_ms AS "blockedAtMs"',
),
);
assert.deepEqual(read.params, ['project-1', NOW, null, '', 17]);
assert.match(
read.sql,
/project_id = \$1 AND status = 'blocked'[\s\S]+ORDER BY updated_at_ms ASC, run_id ASC/,
);
const audit = calls.find(
({ sql, params }) =>
sql.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
params[2] === 'run.cancellation.blocked.list',
);
assert.equal(audit.params[0], blockedListCommand().auditEventId);
});
test('continues only inside the original blocked snapshot', async () => {
const after = {
snapshotAtMs: NOW - 50,
blockedAtMs: NOW - 80,
runId: 'run-03',
};
const { calls, repository } = fixture({
blockedRows: [{ runId: 'run-04', blockedAtMs: String(NOW - 79) }],
});
const result = await repository.listBlocked(blockedListCommand(after));
assert.equal(result.snapshotAtMs, NOW - 50);
assert.equal(result.truncated, false);
assert.equal(Object.hasOwn(result, 'nextCursor'), false);
const read = calls.find(({ sql }) =>
sql.startsWith(
'SELECT run_id AS "runId", updated_at_ms AS "blockedAtMs"',
),
);
assert.deepEqual(read.params, [
'project-1',
NOW - 50,
NOW - 80,
'run-03',
17,
]);
});
test('rearms an exact blocked dispatch with one event and allowed audit', async () => {
const { calls, repository } = fixture();
const result = await repository.rearm(rearmCommand());
@@ -49,6 +49,7 @@ const CANCELLATION_DISPATCH_RETRY_HISTORY_RESULTS = new Set<
export const MAX_CANCELLATION_DISPATCH_LEASE_MS = 5 * 60_000;
export const MAX_CANCELLATION_DISPATCH_RETRY_DELAY_MS = 24 * 60 * 60_000;
export const CANCELLATION_DISPATCH_BLOCKED_PAGE_LIMIT = 16;
export interface CancellationDispatchRecord {
readonly runId: string;