mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 10:32:40 +08:00
feat(ql3): expose cancellation availability summary
This commit is contained in:
whyour
@@ -13,6 +13,7 @@ import {
|
||||
type BlockingCancellationDispatchResult,
|
||||
type RunCancellationDispatchDiagnostic,
|
||||
type RunCancellationDispatchRearmReceipt,
|
||||
type RunCancellationDispatchSummary,
|
||||
} from '@qinglong/cluster-postgres/run-manager';
|
||||
import type { PostgresPool } from '@qinglong/runtime-core';
|
||||
import { CANCELLATION_DISPATCH_BLOCKING_RESULTS } from '@qinglong/runtime-core/cancellation-dispatch';
|
||||
@@ -75,6 +76,14 @@ export interface ClusterRunManagementCancellationInspectRequest {
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
}
|
||||
|
||||
export interface ClusterRunManagementCancellationSummaryRequest {
|
||||
readonly projectId: string;
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly failureAuditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
}
|
||||
|
||||
export interface ClusterRunManagementCancellationRearmRequest
|
||||
extends ClusterRunManagementCancellationInspectRequest {
|
||||
readonly mutationId: string;
|
||||
@@ -90,6 +99,9 @@ export interface ClusterRunManagementService {
|
||||
stop(
|
||||
request: Readonly<ClusterRunManagementStopRequest>,
|
||||
): Promise<Readonly<ClusterRunCancellationResult>>;
|
||||
summarizeCancellation(
|
||||
request: Readonly<ClusterRunManagementCancellationSummaryRequest>,
|
||||
): Promise<Readonly<RunCancellationDispatchSummary>>;
|
||||
inspectCancellation(
|
||||
request: Readonly<ClusterRunManagementCancellationInspectRequest>,
|
||||
): Promise<Readonly<RunCancellationDispatchDiagnostic>>;
|
||||
@@ -233,6 +245,28 @@ function exactCancellationInspectRequest(
|
||||
}
|
||||
}
|
||||
|
||||
function exactCancellationSummaryRequest(
|
||||
value: unknown,
|
||||
): asserts value is Readonly<ClusterRunManagementCancellationSummaryRequest> {
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
Object.keys(value).sort().join('\0') !==
|
||||
[
|
||||
'auditEventId',
|
||||
'failureAuditEventId',
|
||||
'principal',
|
||||
'projectId',
|
||||
'requestId',
|
||||
]
|
||||
.sort()
|
||||
.join('\0')
|
||||
) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
}
|
||||
|
||||
function exactCancellationRearmRequest(
|
||||
value: unknown,
|
||||
): asserts value is Readonly<ClusterRunManagementCancellationRearmRequest> {
|
||||
@@ -502,6 +536,85 @@ export function createClusterRunManagementService(
|
||||
throw new ClusterRunManagementUnavailableError({ cause: error });
|
||||
}
|
||||
},
|
||||
async summarizeCancellation(
|
||||
requestValue: Readonly<ClusterRunManagementCancellationSummaryRequest>,
|
||||
) {
|
||||
exactCancellationSummaryRequest(requestValue);
|
||||
const observedAtMs = now();
|
||||
let principal: Readonly<SecurityPrincipal>;
|
||||
if (
|
||||
!Number.isSafeInteger(observedAtMs) ||
|
||||
observedAtMs < 0 ||
|
||||
!IDENTIFIER_PATTERN.test(requestValue.projectId) ||
|
||||
!IDENTIFIER_PATTERN.test(requestValue.requestId) ||
|
||||
!validUuid(requestValue.auditEventId) ||
|
||||
!validUuid(requestValue.failureAuditEventId) ||
|
||||
requestValue.auditEventId === requestValue.failureAuditEventId
|
||||
) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
try {
|
||||
principal = normalizeSecurityPrincipal(
|
||||
requestValue.principal,
|
||||
observedAtMs,
|
||||
);
|
||||
} catch {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
let fence: Readonly<SecurityPolicyFence> | null = null;
|
||||
try {
|
||||
const decision = await policy.authorize(
|
||||
principal,
|
||||
requestValue.projectId,
|
||||
'run.read',
|
||||
);
|
||||
fence = decision.fence;
|
||||
if (
|
||||
decision.effect !== 'allow' ||
|
||||
!fence ||
|
||||
fence.bindingVersion === null
|
||||
) {
|
||||
throw new ClusterRunManagementAuthorizationError();
|
||||
}
|
||||
return await cancellationDispatches.summary({
|
||||
projectId: requestValue.projectId,
|
||||
requestId: requestValue.requestId,
|
||||
auditEventId: requestValue.auditEventId,
|
||||
principal,
|
||||
policyFence: fence,
|
||||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
await audit.record(
|
||||
normalizeSecurityAuditRecord({
|
||||
eventId: requestValue.failureAuditEventId,
|
||||
requestId: requestValue.requestId,
|
||||
operationId: 'run.cancellation.summary',
|
||||
projectId: requestValue.projectId,
|
||||
subject: principal.subject,
|
||||
authenticationId: principal.authenticationId,
|
||||
outcome: 'denied',
|
||||
reasons: [failureReason(error)],
|
||||
fence,
|
||||
occurredAtMs: observedAtMs,
|
||||
}),
|
||||
);
|
||||
} catch (auditError) {
|
||||
throw new ClusterRunManagementUnavailableError({ cause: auditError });
|
||||
}
|
||||
if (error instanceof ClusterRunManagementAuthorizationError) throw error;
|
||||
if (error instanceof InvalidRunCancellationDispatchManagementError) {
|
||||
throw new ClusterRunManagementRequestError();
|
||||
}
|
||||
if (error instanceof RunCancellationDispatchManagementConflictError) {
|
||||
throw new ClusterRunManagementConflictError();
|
||||
}
|
||||
if (error instanceof RunCancellationDispatchManagementUnavailableError) {
|
||||
throw new ClusterRunManagementUnavailableError({ cause: error });
|
||||
}
|
||||
throw new ClusterRunManagementUnavailableError({ cause: error });
|
||||
}
|
||||
},
|
||||
async inspectCancellation(
|
||||
requestValue: Readonly<ClusterRunManagementCancellationInspectRequest>,
|
||||
) {
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
import {
|
||||
RUN_CANCELLATION_DISPATCH_DIAGNOSTIC_SCHEMA,
|
||||
RUN_CANCELLATION_DISPATCH_REARM_RECEIPT_SCHEMA,
|
||||
RUN_CANCELLATION_DISPATCH_SUMMARY_SCHEMA,
|
||||
normalizeClusterRunManagementCommand,
|
||||
type ClusterRunManagementCommand,
|
||||
type ClusterRunManagementTransportResult,
|
||||
@@ -115,6 +116,118 @@ export function validateClusterRunManagementClientResult(
|
||||
envelope as unknown as ClusterRunManagementTransportResult,
|
||||
);
|
||||
}
|
||||
if (command.operation === 'run.cancellation.summary') {
|
||||
const envelope = exact(value, ['schemaVersion', 'operation', 'summary']);
|
||||
if (
|
||||
envelope.schemaVersion !== 1 ||
|
||||
envelope.operation !== command.operation
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
const summary = exact(envelope.summary, [
|
||||
'schema',
|
||||
'projectId',
|
||||
'observedAtMs',
|
||||
'assessment',
|
||||
'operatorAction',
|
||||
'dispatches',
|
||||
'signals',
|
||||
'blockingResults',
|
||||
...(Object.hasOwn(envelope.summary as object, 'oldestBlockedAtMs')
|
||||
? ['oldestBlockedAtMs']
|
||||
: []),
|
||||
]);
|
||||
const dispatches = exact(summary.dispatches, [
|
||||
'total',
|
||||
'pending',
|
||||
'leased',
|
||||
'retryWait',
|
||||
'dispatched',
|
||||
'blocked',
|
||||
]);
|
||||
const signals = exact(summary.signals, ['due', 'expiredLease']);
|
||||
const blockingResults = exact(summary.blockingResults, [
|
||||
'identityMismatch',
|
||||
'pidMismatch',
|
||||
'unsupported',
|
||||
'invalid',
|
||||
]);
|
||||
const dispatchCounts = [
|
||||
dispatches.total,
|
||||
dispatches.pending,
|
||||
dispatches.leased,
|
||||
dispatches.retryWait,
|
||||
dispatches.dispatched,
|
||||
dispatches.blocked,
|
||||
];
|
||||
const blockingCounts = [
|
||||
blockingResults.identityMismatch,
|
||||
blockingResults.pidMismatch,
|
||||
blockingResults.unsupported,
|
||||
blockingResults.invalid,
|
||||
];
|
||||
if (
|
||||
summary.schema !== RUN_CANCELLATION_DISPATCH_SUMMARY_SCHEMA ||
|
||||
summary.projectId !== command.request.projectId ||
|
||||
!safeInteger(summary.observedAtMs) ||
|
||||
!['clear', 'converging', 'attention_required'].includes(
|
||||
summary.assessment as string,
|
||||
) ||
|
||||
!['none', 'wait', 'inspect'].includes(summary.operatorAction as string) ||
|
||||
dispatchCounts.some((count) => !safeInteger(count)) ||
|
||||
!safeInteger(signals.due) ||
|
||||
!safeInteger(signals.expiredLease) ||
|
||||
blockingCounts.some((count) => !safeInteger(count)) ||
|
||||
dispatches.total !==
|
||||
(dispatches.pending as number) +
|
||||
(dispatches.leased as number) +
|
||||
(dispatches.retryWait as number) +
|
||||
(dispatches.dispatched as number) +
|
||||
(dispatches.blocked as number) ||
|
||||
dispatches.blocked !==
|
||||
(blockingResults.identityMismatch as number) +
|
||||
(blockingResults.pidMismatch as number) +
|
||||
(blockingResults.unsupported as number) +
|
||||
(blockingResults.invalid as number) ||
|
||||
(signals.due as number) >
|
||||
(dispatches.pending as number) + (dispatches.retryWait as number) ||
|
||||
(signals.expiredLease as number) > (dispatches.leased as number) ||
|
||||
(Object.hasOwn(summary, 'oldestBlockedAtMs') &&
|
||||
(!safeInteger(summary.oldestBlockedAtMs) ||
|
||||
(summary.oldestBlockedAtMs as number) >
|
||||
(summary.observedAtMs as number))) ||
|
||||
((dispatches.blocked as number) === 0) !==
|
||||
!Object.hasOwn(summary, 'oldestBlockedAtMs')
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
const active =
|
||||
(dispatches.pending as number) +
|
||||
(dispatches.leased as number) +
|
||||
(dispatches.retryWait as number) +
|
||||
(dispatches.blocked as number);
|
||||
const expectedAssessment =
|
||||
(dispatches.blocked as number) > 0
|
||||
? 'attention_required'
|
||||
: active > 0
|
||||
? 'converging'
|
||||
: 'clear';
|
||||
const expectedOperatorAction =
|
||||
(dispatches.blocked as number) > 0
|
||||
? 'inspect'
|
||||
: active > 0
|
||||
? 'wait'
|
||||
: 'none';
|
||||
if (
|
||||
summary.assessment !== expectedAssessment ||
|
||||
summary.operatorAction !== expectedOperatorAction
|
||||
) {
|
||||
invalid();
|
||||
}
|
||||
return Object.freeze(
|
||||
envelope as unknown as ClusterRunManagementTransportResult,
|
||||
);
|
||||
}
|
||||
if (command.operation === 'run.cancellation.inspect') {
|
||||
const envelope = exact(value, [
|
||||
'schemaVersion',
|
||||
|
||||
@@ -22,6 +22,10 @@ const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
|
||||
|
||||
export const RUN_CANCELLATION_DISPATCH_INSPECT_REQUEST_SCHEMA =
|
||||
'qinglong/run-cancellation-dispatch-inspect@v1';
|
||||
export const RUN_CANCELLATION_DISPATCH_SUMMARY_REQUEST_SCHEMA =
|
||||
'qinglong/run-cancellation-dispatch-summary-request@v1';
|
||||
export const RUN_CANCELLATION_DISPATCH_SUMMARY_SCHEMA =
|
||||
'qinglong/run-cancellation-dispatch-summary@v1';
|
||||
export const RUN_CANCELLATION_DISPATCH_DIAGNOSTIC_SCHEMA =
|
||||
'qinglong/run-cancellation-dispatch-diagnostic@v1';
|
||||
export const RUN_CANCELLATION_DISPATCH_REARM_REQUEST_SCHEMA =
|
||||
@@ -78,6 +82,20 @@ export type ClusterRunManagementCancellationInspectCommand = Readonly<{
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export type ClusterRunManagementCancellationSummaryCommand = Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'run.cancellation.summary';
|
||||
request: Readonly<{
|
||||
projectId: string;
|
||||
requestId: string;
|
||||
auditEventId: string;
|
||||
failureAuditEventId: string;
|
||||
body: Readonly<{
|
||||
schema: typeof RUN_CANCELLATION_DISPATCH_SUMMARY_REQUEST_SCHEMA;
|
||||
}>;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
export type ClusterRunManagementCancellationRearmCommand = Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'run.cancellation.rearm';
|
||||
@@ -104,6 +122,7 @@ export type ClusterRunManagementCancellationRearmCommand = Readonly<{
|
||||
export type ClusterRunManagementCommand =
|
||||
| ClusterRunManagementRetryCommand
|
||||
| ClusterRunManagementStopCommand
|
||||
| ClusterRunManagementCancellationSummaryCommand
|
||||
| ClusterRunManagementCancellationInspectCommand
|
||||
| ClusterRunManagementCancellationRearmCommand;
|
||||
|
||||
@@ -129,6 +148,16 @@ export type ClusterRunManagementCancellationInspectTransportResult = Readonly<{
|
||||
>;
|
||||
}>;
|
||||
|
||||
export type ClusterRunManagementCancellationSummaryTransportResult = Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'run.cancellation.summary';
|
||||
summary: Readonly<
|
||||
Awaited<ReturnType<ClusterRunManagementService['summarizeCancellation']>> & {
|
||||
schema: typeof RUN_CANCELLATION_DISPATCH_SUMMARY_SCHEMA;
|
||||
}
|
||||
>;
|
||||
}>;
|
||||
|
||||
export type ClusterRunManagementCancellationRearmTransportResult = Readonly<{
|
||||
schemaVersion: 1;
|
||||
operation: 'run.cancellation.rearm';
|
||||
@@ -142,6 +171,7 @@ export type ClusterRunManagementCancellationRearmTransportResult = Readonly<{
|
||||
export type ClusterRunManagementTransportResult =
|
||||
| ClusterRunManagementRetryTransportResult
|
||||
| ClusterRunManagementStopTransportResult
|
||||
| ClusterRunManagementCancellationSummaryTransportResult
|
||||
| ClusterRunManagementCancellationInspectTransportResult
|
||||
| ClusterRunManagementCancellationRearmTransportResult;
|
||||
|
||||
@@ -227,6 +257,7 @@ export function normalizeClusterRunManagementCommand(
|
||||
if (
|
||||
operation !== 'run.retry' &&
|
||||
operation !== 'run.stop' &&
|
||||
operation !== 'run.cancellation.summary' &&
|
||||
operation !== 'run.cancellation.inspect' &&
|
||||
operation !== 'run.cancellation.rearm'
|
||||
) {
|
||||
@@ -243,7 +274,15 @@ export function normalizeClusterRunManagementCommand(
|
||||
'failureAuditEventId',
|
||||
'body',
|
||||
]
|
||||
: [
|
||||
: operation === 'run.cancellation.summary'
|
||||
? [
|
||||
'projectId',
|
||||
'requestId',
|
||||
'auditEventId',
|
||||
'failureAuditEventId',
|
||||
'body',
|
||||
]
|
||||
: [
|
||||
'projectId',
|
||||
'runId',
|
||||
'requestId',
|
||||
@@ -275,6 +314,25 @@ export function normalizeClusterRunManagementCommand(
|
||||
}),
|
||||
});
|
||||
}
|
||||
if (operation === 'run.cancellation.summary') {
|
||||
const body = exact(request.body, ['schema']);
|
||||
if (body.schema !== RUN_CANCELLATION_DISPATCH_SUMMARY_REQUEST_SCHEMA) {
|
||||
invalid();
|
||||
}
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation,
|
||||
request: Object.freeze({
|
||||
projectId: identifier(request.projectId),
|
||||
requestId: identifier(request.requestId),
|
||||
auditEventId,
|
||||
failureAuditEventId,
|
||||
body: Object.freeze({
|
||||
schema: RUN_CANCELLATION_DISPATCH_SUMMARY_REQUEST_SCHEMA,
|
||||
}),
|
||||
}),
|
||||
});
|
||||
}
|
||||
if (operation === 'run.cancellation.inspect') {
|
||||
const body = exact(request.body, ['schema']);
|
||||
if (body.schema !== RUN_CANCELLATION_DISPATCH_INSPECT_REQUEST_SCHEMA) {
|
||||
@@ -374,6 +432,7 @@ export function createClusterRunManagementTransport(
|
||||
!options.service ||
|
||||
typeof options.service.retry !== 'function' ||
|
||||
typeof options.service.stop !== 'function' ||
|
||||
typeof options.service.summarizeCancellation !== 'function' ||
|
||||
typeof options.service.inspectCancellation !== 'function' ||
|
||||
typeof options.service.rearmCancellation !== 'function' ||
|
||||
(options.now !== undefined && typeof options.now !== 'function')
|
||||
@@ -435,6 +494,23 @@ export function createClusterRunManagementTransport(
|
||||
retry: createRunManualRetryResponseBody(result),
|
||||
});
|
||||
}
|
||||
if (command.operation === 'run.cancellation.summary') {
|
||||
const result = await options.service.summarizeCancellation({
|
||||
projectId: command.request.projectId,
|
||||
requestId: command.request.requestId,
|
||||
auditEventId: command.request.auditEventId,
|
||||
failureAuditEventId: command.request.failureAuditEventId,
|
||||
principal,
|
||||
});
|
||||
return Object.freeze({
|
||||
schemaVersion: 1,
|
||||
operation: command.operation,
|
||||
summary: Object.freeze({
|
||||
schema: RUN_CANCELLATION_DISPATCH_SUMMARY_SCHEMA,
|
||||
...result,
|
||||
}),
|
||||
});
|
||||
}
|
||||
if (command.operation === 'run.cancellation.inspect') {
|
||||
const result = await options.service.inspectCancellation({
|
||||
projectId: command.request.projectId,
|
||||
|
||||
@@ -163,6 +163,30 @@ function fixture(role = 'operator', options = {}) {
|
||||
) {
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."run_cancellation_dispatches" AS dispatch')
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
{
|
||||
total: '5',
|
||||
pending: '1',
|
||||
leased: '1',
|
||||
retryWait: '1',
|
||||
dispatched: '1',
|
||||
blocked: '1',
|
||||
due: '1',
|
||||
expiredLease: '1',
|
||||
identityMismatch: '1',
|
||||
pidMismatch: '0',
|
||||
unsupported: '0',
|
||||
invalid: '0',
|
||||
oldestBlockedAtMs: String(NOW - 1_500),
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.startsWith('SELECT attempt_id AS "attemptId"') &&
|
||||
text.includes('FROM "ql3"."run_cancellation_dispatches"') &&
|
||||
@@ -370,6 +394,37 @@ test('allows a viewer to inspect only low-sensitive cancellation state', async (
|
||||
assert.equal(audit.params[0], inspectRequest.auditEventId);
|
||||
});
|
||||
|
||||
test('allows a viewer to summarize Project cancellation availability atomically', async () => {
|
||||
const { calls, service } = fixture('viewer');
|
||||
const summaryRequest = {
|
||||
projectId: 'project-1',
|
||||
requestId: 'request-summary-1',
|
||||
auditEventId: '019f9500-0000-4000-8000-000000000061',
|
||||
failureAuditEventId: '019f9500-0000-4000-8000-000000000062',
|
||||
principal: request().principal,
|
||||
};
|
||||
const result = await service.summarizeCancellation(summaryRequest);
|
||||
assert.equal(result.assessment, 'attention_required');
|
||||
assert.equal(result.operatorAction, 'inspect');
|
||||
assert.equal(result.dispatches.blocked, 1);
|
||||
assert.equal(result.blockingResults.identityMismatch, 1);
|
||||
assert.equal(Object.hasOwn(result, 'runId'), false);
|
||||
assert.equal(JSON.stringify(result).includes('attemptId'), false);
|
||||
const aggregate = calls.find(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."run_cancellation_dispatches" AS dispatch'),
|
||||
);
|
||||
assert.deepEqual(aggregate.params, ['project-1', NOW]);
|
||||
const audit = calls.find(
|
||||
({ sql, params }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
|
||||
params[2] === 'run.cancellation.summary',
|
||||
);
|
||||
assert.equal(audit.params[0], summaryRequest.auditEventId);
|
||||
assert.ok(
|
||||
calls.indexOf(audit) < calls.findIndex(({ sql }) => sql === 'COMMIT'),
|
||||
);
|
||||
});
|
||||
|
||||
test('authorizes exact cancellation rearm and keeps the event identity server-side', async () => {
|
||||
const { calls, service } = fixture();
|
||||
const rearmRequest = {
|
||||
|
||||
@@ -132,6 +132,20 @@ const inspectCommand = normalizeClusterRunManagementCommand({
|
||||
},
|
||||
});
|
||||
|
||||
const summaryCommand = normalizeClusterRunManagementCommand({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.summary',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
requestId: 'request-summary-1',
|
||||
auditEventId: '019f9400-0000-4000-8000-000000000051',
|
||||
failureAuditEventId: '019f9400-0000-4000-8000-000000000052',
|
||||
body: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-summary-request@v1',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const rearmCommand = normalizeClusterRunManagementCommand({
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.rearm',
|
||||
@@ -310,6 +324,67 @@ test('validates a low-sensitive cancellation diagnostic and rejects capability l
|
||||
}
|
||||
});
|
||||
|
||||
test('validates the fixed low-sensitive Project cancellation summary', () => {
|
||||
const value = {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.summary',
|
||||
summary: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-summary@v1',
|
||||
projectId: 'project-1',
|
||||
observedAtMs: 1_000_000,
|
||||
assessment: 'attention_required',
|
||||
operatorAction: 'inspect',
|
||||
dispatches: {
|
||||
total: 5,
|
||||
pending: 1,
|
||||
leased: 1,
|
||||
retryWait: 1,
|
||||
dispatched: 1,
|
||||
blocked: 1,
|
||||
},
|
||||
signals: { due: 1, expiredLease: 1 },
|
||||
blockingResults: {
|
||||
identityMismatch: 1,
|
||||
pidMismatch: 0,
|
||||
unsupported: 0,
|
||||
invalid: 0,
|
||||
},
|
||||
oldestBlockedAtMs: 999_200,
|
||||
},
|
||||
};
|
||||
assert.deepEqual(
|
||||
validateClusterRunManagementClientResult(value, summaryCommand),
|
||||
value,
|
||||
);
|
||||
for (const summary of [
|
||||
{ ...value.summary, projectId: 'project-2' },
|
||||
{ ...value.summary, assessment: 'clear' },
|
||||
{ ...value.summary, operatorAction: 'wait' },
|
||||
{
|
||||
...value.summary,
|
||||
dispatches: { ...value.summary.dispatches, total: 6 },
|
||||
},
|
||||
{
|
||||
...value.summary,
|
||||
blockingResults: {
|
||||
...value.summary.blockingResults,
|
||||
identityMismatch: 0,
|
||||
},
|
||||
},
|
||||
{ ...value.summary, oldestBlockedAtMs: 1_000_001 },
|
||||
{ ...value.summary, runId: 'run-1' },
|
||||
]) {
|
||||
assert.throws(
|
||||
() =>
|
||||
validateClusterRunManagementClientResult(
|
||||
{ ...value, summary },
|
||||
summaryCommand,
|
||||
),
|
||||
ClusterPluginPackageManagementClientRequestError,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('binds a rearm receipt to the exact dispatch version, result and delay fences', () => {
|
||||
const value = {
|
||||
schemaVersion: 1,
|
||||
|
||||
@@ -119,6 +119,31 @@ function diagnosticResult() {
|
||||
};
|
||||
}
|
||||
|
||||
function summaryResult() {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
observedAtMs: NOW,
|
||||
assessment: 'attention_required',
|
||||
operatorAction: 'inspect',
|
||||
dispatches: {
|
||||
total: 5,
|
||||
pending: 1,
|
||||
leased: 1,
|
||||
retryWait: 1,
|
||||
dispatched: 1,
|
||||
blocked: 1,
|
||||
},
|
||||
signals: { due: 1, expiredLease: 1 },
|
||||
blockingResults: {
|
||||
identityMismatch: 1,
|
||||
pidMismatch: 0,
|
||||
unsupported: 0,
|
||||
invalid: 0,
|
||||
},
|
||||
oldestBlockedAtMs: NOW - 800,
|
||||
};
|
||||
}
|
||||
|
||||
function rearmResult() {
|
||||
return {
|
||||
status: 'rearmed',
|
||||
@@ -153,6 +178,23 @@ function inspectCommand(overrides = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function summaryCommand(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.summary',
|
||||
request: {
|
||||
projectId: 'project-1',
|
||||
requestId: 'request-summary-1',
|
||||
auditEventId: '019f9300-0000-4000-8000-000000000051',
|
||||
failureAuditEventId: '019f9300-0000-4000-8000-000000000052',
|
||||
body: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-summary-request@v1',
|
||||
},
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function rearmCommand(overrides = {}) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
@@ -187,6 +229,9 @@ test('routes one exact strong User retry and emits the shared response', async (
|
||||
async stop() {
|
||||
return stopResult();
|
||||
},
|
||||
async summarizeCancellation() {
|
||||
return summaryResult();
|
||||
},
|
||||
async inspectCancellation() {
|
||||
return diagnosticResult();
|
||||
},
|
||||
@@ -223,6 +268,9 @@ test('routes one exact strong User stop and emits the shared response', async ()
|
||||
calls.push(request);
|
||||
return stopResult();
|
||||
},
|
||||
async summarizeCancellation() {
|
||||
return summaryResult();
|
||||
},
|
||||
async inspectCancellation() {
|
||||
return diagnosticResult();
|
||||
},
|
||||
@@ -259,6 +307,9 @@ test('rejects weak or non-User identity before service authority', async () => {
|
||||
async stop() {
|
||||
return stopResult();
|
||||
},
|
||||
async summarizeCancellation() {
|
||||
return summaryResult();
|
||||
},
|
||||
async inspectCancellation() {
|
||||
return diagnosticResult();
|
||||
},
|
||||
@@ -290,6 +341,7 @@ test('routes bounded cancellation inspection without lease capability data', asy
|
||||
service: {
|
||||
async retry() { return retryResult(); },
|
||||
async stop() { return stopResult(); },
|
||||
async summarizeCancellation() { return summaryResult(); },
|
||||
async inspectCancellation(request) {
|
||||
calls.push(request);
|
||||
return diagnosticResult();
|
||||
@@ -313,6 +365,39 @@ test('routes bounded cancellation inspection without lease capability data', asy
|
||||
assert.equal(JSON.stringify(result).includes('leaseToken'), false);
|
||||
});
|
||||
|
||||
test('routes one Project-scoped cancellation summary without Run identity', async () => {
|
||||
const calls = [];
|
||||
const transport = createClusterRunManagementTransport({
|
||||
now: () => NOW,
|
||||
service: {
|
||||
async retry() { return retryResult(); },
|
||||
async stop() { return stopResult(); },
|
||||
async summarizeCancellation(request) {
|
||||
calls.push(request);
|
||||
return summaryResult();
|
||||
},
|
||||
async inspectCancellation() { return diagnosticResult(); },
|
||||
async rearmCancellation() { return rearmResult(); },
|
||||
},
|
||||
});
|
||||
const result = await transport.execute(summaryCommand(), {
|
||||
authenticate: async () => principal(),
|
||||
});
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].projectId, 'project-1');
|
||||
assert.equal(Object.hasOwn(calls[0], 'runId'), false);
|
||||
assert.deepEqual(result, {
|
||||
schemaVersion: 1,
|
||||
operation: 'run.cancellation.summary',
|
||||
summary: {
|
||||
schema: 'qinglong/run-cancellation-dispatch-summary@v1',
|
||||
...summaryResult(),
|
||||
},
|
||||
});
|
||||
assert.equal(JSON.stringify(result).includes('attemptId'), false);
|
||||
assert.equal(JSON.stringify(result).includes('leaseOwner'), false);
|
||||
});
|
||||
|
||||
test('routes an exact blocked cancellation rearm receipt', async () => {
|
||||
const calls = [];
|
||||
const transport = createClusterRunManagementTransport({
|
||||
@@ -320,6 +405,7 @@ test('routes an exact blocked cancellation rearm receipt', async () => {
|
||||
service: {
|
||||
async retry() { return retryResult(); },
|
||||
async stop() { return stopResult(); },
|
||||
async summarizeCancellation() { return summaryResult(); },
|
||||
async inspectCancellation() { return diagnosticResult(); },
|
||||
async rearmCancellation(request) {
|
||||
calls.push(request);
|
||||
|
||||
@@ -8,8 +8,10 @@ export {
|
||||
type BlockingCancellationDispatchResult,
|
||||
type PostgresRunCancellationDispatchInspectCommand,
|
||||
type PostgresRunCancellationDispatchRearmCommand,
|
||||
type PostgresRunCancellationDispatchSummaryCommand,
|
||||
type RunCancellationDispatchDiagnostic,
|
||||
type RunCancellationDispatchRearmReceipt,
|
||||
type RunCancellationDispatchSummary,
|
||||
} from '../run-management/runCancellationDispatchManagementRepository';
|
||||
export {
|
||||
PostgresClusterRunCancellationRepository,
|
||||
|
||||
+215
-5
@@ -57,15 +57,47 @@ export type RunCancellationDispatchRearmReceipt = Readonly<{
|
||||
eventSequence: number;
|
||||
}>;
|
||||
|
||||
interface ManagementAuthority {
|
||||
export type RunCancellationDispatchSummary = Readonly<{
|
||||
projectId: string;
|
||||
observedAtMs: number;
|
||||
assessment: 'clear' | 'converging' | 'attention_required';
|
||||
operatorAction: 'none' | 'wait' | 'inspect';
|
||||
dispatches: Readonly<{
|
||||
total: number;
|
||||
pending: number;
|
||||
leased: number;
|
||||
retryWait: number;
|
||||
dispatched: number;
|
||||
blocked: number;
|
||||
}>;
|
||||
signals: Readonly<{
|
||||
due: number;
|
||||
expiredLease: number;
|
||||
}>;
|
||||
blockingResults: Readonly<{
|
||||
identityMismatch: number;
|
||||
pidMismatch: number;
|
||||
unsupported: number;
|
||||
invalid: number;
|
||||
}>;
|
||||
oldestBlockedAtMs?: number;
|
||||
}>;
|
||||
|
||||
interface ProjectManagementAuthority {
|
||||
readonly projectId: string;
|
||||
readonly runId: string;
|
||||
readonly requestId: string;
|
||||
readonly auditEventId: string;
|
||||
readonly principal: Readonly<SecurityPrincipal>;
|
||||
readonly policyFence: Readonly<SecurityPolicyFence>;
|
||||
}
|
||||
|
||||
interface ManagementAuthority extends ProjectManagementAuthority {
|
||||
readonly runId: string;
|
||||
}
|
||||
|
||||
export interface PostgresRunCancellationDispatchSummaryCommand
|
||||
extends ProjectManagementAuthority {}
|
||||
|
||||
export interface PostgresRunCancellationDispatchInspectCommand
|
||||
extends ManagementAuthority {}
|
||||
|
||||
@@ -282,6 +314,39 @@ function normalizeInspectCommand(
|
||||
return normalizeAuthority(value, []);
|
||||
}
|
||||
|
||||
function normalizeSummaryCommand(
|
||||
value: Readonly<PostgresRunCancellationDispatchSummaryCommand>,
|
||||
): Readonly<PostgresRunCancellationDispatchSummaryCommand> {
|
||||
const input = exact(value, [
|
||||
'projectId',
|
||||
'requestId',
|
||||
'auditEventId',
|
||||
'principal',
|
||||
'policyFence',
|
||||
]);
|
||||
const principal = exact(input.principal, [
|
||||
'subject',
|
||||
'authenticationId',
|
||||
'authenticatedAtMs',
|
||||
'expiresAtMs',
|
||||
'assurance',
|
||||
]) as unknown as SecurityPrincipal;
|
||||
const fence = exact(input.policyFence, [
|
||||
'projectVersion',
|
||||
'bindingVersion',
|
||||
]);
|
||||
return Object.freeze({
|
||||
projectId: identifier(input.projectId),
|
||||
requestId: identifier(input.requestId),
|
||||
auditEventId: uuid(input.auditEventId),
|
||||
principal,
|
||||
policyFence: Object.freeze({
|
||||
projectVersion: boundedInteger(fence.projectVersion, 1, 2_147_483_647),
|
||||
bindingVersion: boundedInteger(fence.bindingVersion, 1, 2_147_483_647),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeRearmCommand(
|
||||
value: Readonly<PostgresRunCancellationDispatchRearmCommand>,
|
||||
): Readonly<PostgresRunCancellationDispatchRearmCommand> {
|
||||
@@ -383,7 +448,7 @@ function strongPrincipal(
|
||||
|
||||
async function confirmAuthorization(
|
||||
client: PostgresClient,
|
||||
command: Readonly<ManagementAuthority>,
|
||||
command: Readonly<ProjectManagementAuthority>,
|
||||
): Promise<void> {
|
||||
const result = await client.query<Row>(
|
||||
`SELECT "ql3"."lock_run_management_policy_fence"(
|
||||
@@ -406,8 +471,11 @@ async function confirmAuthorization(
|
||||
|
||||
async function recordAllowedAudit(
|
||||
client: PostgresClient,
|
||||
command: Readonly<ManagementAuthority>,
|
||||
operationId: 'run.cancellation.inspect' | 'run.cancellation.rearm',
|
||||
command: Readonly<ProjectManagementAuthority>,
|
||||
operationId:
|
||||
| 'run.cancellation.summary'
|
||||
| 'run.cancellation.inspect'
|
||||
| 'run.cancellation.rearm',
|
||||
observedAtMs: number,
|
||||
): Promise<void> {
|
||||
const inserted = await client.query<Row>(
|
||||
@@ -463,6 +531,78 @@ async function recordAllowedAudit(
|
||||
}
|
||||
}
|
||||
|
||||
function summaryProjection(
|
||||
projectId: string,
|
||||
observedAtMs: number,
|
||||
row: Row,
|
||||
): Readonly<RunCancellationDispatchSummary> {
|
||||
const dispatches = Object.freeze({
|
||||
total: integer(row, 'total'),
|
||||
pending: integer(row, 'pending'),
|
||||
leased: integer(row, 'leased'),
|
||||
retryWait: integer(row, 'retryWait'),
|
||||
dispatched: integer(row, 'dispatched'),
|
||||
blocked: integer(row, 'blocked'),
|
||||
});
|
||||
const signals = Object.freeze({
|
||||
due: integer(row, 'due'),
|
||||
expiredLease: integer(row, 'expiredLease'),
|
||||
});
|
||||
const blockingResults = Object.freeze({
|
||||
identityMismatch: integer(row, 'identityMismatch'),
|
||||
pidMismatch: integer(row, 'pidMismatch'),
|
||||
unsupported: integer(row, 'unsupported'),
|
||||
invalid: integer(row, 'invalid'),
|
||||
});
|
||||
if (
|
||||
dispatches.total !==
|
||||
dispatches.pending +
|
||||
dispatches.leased +
|
||||
dispatches.retryWait +
|
||||
dispatches.dispatched +
|
||||
dispatches.blocked ||
|
||||
dispatches.blocked !==
|
||||
blockingResults.identityMismatch +
|
||||
blockingResults.pidMismatch +
|
||||
blockingResults.unsupported +
|
||||
blockingResults.invalid ||
|
||||
signals.due > dispatches.pending + dispatches.retryWait ||
|
||||
signals.expiredLease > dispatches.leased
|
||||
) {
|
||||
throw new TypeError('PostgreSQL cancellation management summary is invalid');
|
||||
}
|
||||
const oldestBlockedAtMs = optionalInteger(row, 'oldestBlockedAtMs');
|
||||
if (
|
||||
(dispatches.blocked === 0) !== (oldestBlockedAtMs === undefined) ||
|
||||
(oldestBlockedAtMs !== undefined && oldestBlockedAtMs > observedAtMs)
|
||||
) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL cancellation management blocked summary is invalid',
|
||||
);
|
||||
}
|
||||
const active =
|
||||
dispatches.pending +
|
||||
dispatches.leased +
|
||||
dispatches.retryWait +
|
||||
dispatches.blocked;
|
||||
return Object.freeze({
|
||||
projectId,
|
||||
observedAtMs,
|
||||
assessment:
|
||||
dispatches.blocked > 0
|
||||
? 'attention_required'
|
||||
: active > 0
|
||||
? 'converging'
|
||||
: 'clear',
|
||||
operatorAction:
|
||||
dispatches.blocked > 0 ? 'inspect' : active > 0 ? 'wait' : 'none',
|
||||
dispatches,
|
||||
signals,
|
||||
blockingResults,
|
||||
...(oldestBlockedAtMs === undefined ? {} : { oldestBlockedAtMs }),
|
||||
});
|
||||
}
|
||||
|
||||
function runStatus(row: Row): RunStatus {
|
||||
const value = text(row, 'runStatus') as RunStatus;
|
||||
if (!RUN_STATUSES.includes(value)) {
|
||||
@@ -608,6 +748,76 @@ export class PostgresRunCancellationDispatchManagementRepository {
|
||||
}
|
||||
}
|
||||
|
||||
summary(
|
||||
value: Readonly<PostgresRunCancellationDispatchSummaryCommand>,
|
||||
): Promise<Readonly<RunCancellationDispatchSummary>> {
|
||||
const command = normalizeSummaryCommand(value);
|
||||
return this.transaction(async (client) => {
|
||||
const observedAtMs = await databaseNow(client);
|
||||
const authorized = Object.freeze({
|
||||
...command,
|
||||
principal: strongPrincipal(command.principal, observedAtMs),
|
||||
});
|
||||
await confirmAuthorization(client, authorized);
|
||||
const result = await client.query<Row>(
|
||||
`SELECT count(*)::bigint AS total,
|
||||
count(*) FILTER (WHERE dispatch.status = 'pending')::bigint
|
||||
AS pending,
|
||||
count(*) FILTER (WHERE dispatch.status = 'leased')::bigint
|
||||
AS leased,
|
||||
count(*) FILTER (WHERE dispatch.status = 'retry_wait')::bigint
|
||||
AS "retryWait",
|
||||
count(*) FILTER (WHERE dispatch.status = 'dispatched')::bigint
|
||||
AS dispatched,
|
||||
count(*) FILTER (WHERE dispatch.status = 'blocked')::bigint
|
||||
AS blocked,
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status IN ('pending', 'retry_wait')
|
||||
AND dispatch.next_attempt_at_ms <= $2
|
||||
)::bigint AS due,
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status = 'leased'
|
||||
AND dispatch.lease_expires_at_ms <= $2
|
||||
)::bigint AS "expiredLease",
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status = 'blocked'
|
||||
AND dispatch.last_result = 'identity_mismatch'
|
||||
)::bigint AS "identityMismatch",
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status = 'blocked'
|
||||
AND dispatch.last_result = 'pid_mismatch'
|
||||
)::bigint AS "pidMismatch",
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status = 'blocked'
|
||||
AND dispatch.last_result = 'unsupported'
|
||||
)::bigint AS unsupported,
|
||||
count(*) FILTER (
|
||||
WHERE dispatch.status = 'blocked'
|
||||
AND dispatch.last_result = 'invalid'
|
||||
)::bigint AS invalid,
|
||||
min(dispatch.updated_at_ms) FILTER (
|
||||
WHERE dispatch.status = 'blocked'
|
||||
) AS "oldestBlockedAtMs"
|
||||
FROM "ql3"."run_cancellation_dispatches" AS dispatch
|
||||
JOIN "ql3"."runs" AS run ON run.id = dispatch.run_id
|
||||
WHERE run.project_id = $1`,
|
||||
[command.projectId, observedAtMs],
|
||||
);
|
||||
if (result.rows.length !== 1) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL cancellation management summary row is invalid',
|
||||
);
|
||||
}
|
||||
await recordAllowedAudit(
|
||||
client,
|
||||
authorized,
|
||||
'run.cancellation.summary',
|
||||
observedAtMs,
|
||||
);
|
||||
return summaryProjection(command.projectId, observedAtMs, result.rows[0]!);
|
||||
});
|
||||
}
|
||||
|
||||
inspect(
|
||||
value: Readonly<PostgresRunCancellationDispatchInspectCommand>,
|
||||
): Promise<Readonly<RunCancellationDispatchDiagnostic>> {
|
||||
|
||||
+117
@@ -42,6 +42,11 @@ function rearmCommand(overrides = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
function summaryCommand(overrides = {}) {
|
||||
const { runId: _runId, ...authority } = command();
|
||||
return { ...authority, ...overrides };
|
||||
}
|
||||
|
||||
function runRow() {
|
||||
return {
|
||||
projectId: 'project-1',
|
||||
@@ -104,6 +109,30 @@ function fixture(options = {}) {
|
||||
) {
|
||||
return { rows: options.replay ? [options.replay] : [], rowCount: 0 };
|
||||
}
|
||||
if (
|
||||
text.includes('FROM "ql3"."run_cancellation_dispatches" AS dispatch')
|
||||
) {
|
||||
return {
|
||||
rows: [
|
||||
options.summary ?? {
|
||||
total: '5',
|
||||
pending: '1',
|
||||
leased: '1',
|
||||
retryWait: '1',
|
||||
dispatched: '1',
|
||||
blocked: '1',
|
||||
due: '1',
|
||||
expiredLease: '1',
|
||||
identityMismatch: '1',
|
||||
pidMismatch: '0',
|
||||
unsupported: '0',
|
||||
invalid: '0',
|
||||
oldestBlockedAtMs: String(NOW - 1_500),
|
||||
},
|
||||
],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (
|
||||
text.startsWith('SELECT attempt_id AS "attemptId"') &&
|
||||
!text.includes('dispatchStatus') &&
|
||||
@@ -179,6 +208,94 @@ test('inspects one low-sensitive blocked dispatch under run.read authority', asy
|
||||
);
|
||||
});
|
||||
|
||||
test('summarizes one Project without exposing Run, Attempt or lease identity', async () => {
|
||||
const { calls, repository } = fixture();
|
||||
const result = await repository.summary(summaryCommand());
|
||||
assert.deepEqual(result, {
|
||||
projectId: 'project-1',
|
||||
observedAtMs: NOW,
|
||||
assessment: 'attention_required',
|
||||
operatorAction: 'inspect',
|
||||
dispatches: {
|
||||
total: 5,
|
||||
pending: 1,
|
||||
leased: 1,
|
||||
retryWait: 1,
|
||||
dispatched: 1,
|
||||
blocked: 1,
|
||||
},
|
||||
signals: { due: 1, expiredLease: 1 },
|
||||
blockingResults: {
|
||||
identityMismatch: 1,
|
||||
pidMismatch: 0,
|
||||
unsupported: 0,
|
||||
invalid: 0,
|
||||
},
|
||||
oldestBlockedAtMs: NOW - 1_500,
|
||||
});
|
||||
const read = calls.find(({ sql }) =>
|
||||
sql.includes('FROM "ql3"."run_cancellation_dispatches" AS dispatch'),
|
||||
);
|
||||
assert.deepEqual(read.params, ['project-1', NOW]);
|
||||
for (const forbidden of [
|
||||
'attempt_id AS',
|
||||
'run.id AS',
|
||||
'lease_owner',
|
||||
'lease_token',
|
||||
'lease_token_digest',
|
||||
]) {
|
||||
assert.equal(read.sql.includes(forbidden), false);
|
||||
}
|
||||
const audit = calls.find(
|
||||
({ sql, params }) =>
|
||||
sql.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
|
||||
params[2] === 'run.cancellation.summary',
|
||||
);
|
||||
assert.equal(audit.params[0], summaryCommand().auditEventId);
|
||||
});
|
||||
|
||||
test('derives clear and converging assessments from fixed status counts', async () => {
|
||||
const clear = fixture({
|
||||
summary: {
|
||||
total: '1',
|
||||
pending: '0',
|
||||
leased: '0',
|
||||
retryWait: '0',
|
||||
dispatched: '1',
|
||||
blocked: '0',
|
||||
due: '0',
|
||||
expiredLease: '0',
|
||||
identityMismatch: '0',
|
||||
pidMismatch: '0',
|
||||
unsupported: '0',
|
||||
invalid: '0',
|
||||
oldestBlockedAtMs: null,
|
||||
},
|
||||
});
|
||||
assert.equal((await clear.repository.summary(summaryCommand())).assessment, 'clear');
|
||||
|
||||
const converging = fixture({
|
||||
summary: {
|
||||
total: '1',
|
||||
pending: '0',
|
||||
leased: '0',
|
||||
retryWait: '1',
|
||||
dispatched: '0',
|
||||
blocked: '0',
|
||||
due: '1',
|
||||
expiredLease: '0',
|
||||
identityMismatch: '0',
|
||||
pidMismatch: '0',
|
||||
unsupported: '0',
|
||||
invalid: '0',
|
||||
oldestBlockedAtMs: null,
|
||||
},
|
||||
});
|
||||
const result = await converging.repository.summary(summaryCommand());
|
||||
assert.equal(result.assessment, 'converging');
|
||||
assert.equal(result.operatorAction, 'wait');
|
||||
});
|
||||
|
||||
test('rearms an exact blocked dispatch with one event and allowed audit', async () => {
|
||||
const { calls, repository } = fixture();
|
||||
const result = await repository.rearm(rearmCommand());
|
||||
|
||||
Reference in New Issue
Block a user