feat(ql3): operationalize cancellation rearm

packages/ql3-cluster-admin/src/run-management/runManagement.ts

@@ -1,12 +1,21 @@
 import { randomUUID } from 'node:crypto';
 
 import {
+  InvalidRunCancellationDispatchManagementError,
   PostgresClusterRunCancellationRepository,
   PostgresProjectPolicyRepository,
+  PostgresRunCancellationDispatchManagementRepository,
   PostgresRunManualRetryRepository,
   PostgresSecurityAuditRepository,
+  RunCancellationDispatchManagementConflictError,
+  RunCancellationDispatchManagementNotFoundError,
+  RunCancellationDispatchManagementUnavailableError,
+  type BlockingCancellationDispatchResult,
+  type RunCancellationDispatchDiagnostic,
+  type RunCancellationDispatchRearmReceipt,
 } from '@qinglong/cluster-postgres/run-manager';
 import type { PostgresPool } from '@qinglong/runtime-core';
+import { CANCELLATION_DISPATCH_BLOCKING_RESULTS } from '@qinglong/runtime-core/cancellation-dispatch';
 import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
 import {
   ClusterRunCancellationFenceRejectedError,
@@ -57,6 +66,23 @@ export interface ClusterRunManagementStopRequest {
   readonly principal: Readonly<SecurityPrincipal>;
 }
 
+export interface ClusterRunManagementCancellationInspectRequest {
+  readonly projectId: string;
+  readonly runId: string;
+  readonly requestId: string;
+  readonly auditEventId: string;
+  readonly failureAuditEventId: string;
+  readonly principal: Readonly<SecurityPrincipal>;
+}
+
+export interface ClusterRunManagementCancellationRearmRequest
+  extends ClusterRunManagementCancellationInspectRequest {
+  readonly mutationId: string;
+  readonly expectedDispatchVersion: number;
+  readonly expectedLastResult: BlockingCancellationDispatchResult;
+  readonly retryDelayMs: number;
+}
+
 export interface ClusterRunManagementService {
   retry(
     request: Readonly<ClusterRunManagementRetryRequest>,
@@ -64,6 +90,12 @@ export interface ClusterRunManagementService {
   stop(
     request: Readonly<ClusterRunManagementStopRequest>,
   ): Promise<Readonly<ClusterRunCancellationResult>>;
+  inspectCancellation(
+    request: Readonly<ClusterRunManagementCancellationInspectRequest>,
+  ): Promise<Readonly<RunCancellationDispatchDiagnostic>>;
+  rearmCancellation(
+    request: Readonly<ClusterRunManagementCancellationRearmRequest>,
+  ): Promise<Readonly<RunCancellationDispatchRearmReceipt>>;
 }
 
 export interface ClusterRunManagementOptions {
@@ -178,6 +210,56 @@ function exactStopRequest(
   }
 }
 
+function exactCancellationInspectRequest(
+  value: unknown,
+): asserts value is Readonly<ClusterRunManagementCancellationInspectRequest> {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    Object.keys(value).sort().join('\0') !==
+      [
+        'auditEventId',
+        'failureAuditEventId',
+        'principal',
+        'projectId',
+        'requestId',
+        'runId',
+      ]
+        .sort()
+        .join('\0')
+  ) {
+    throw new ClusterRunManagementRequestError();
+  }
+}
+
+function exactCancellationRearmRequest(
+  value: unknown,
+): asserts value is Readonly<ClusterRunManagementCancellationRearmRequest> {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    Object.keys(value).sort().join('\0') !==
+      [
+        'auditEventId',
+        'expectedDispatchVersion',
+        'expectedLastResult',
+        'failureAuditEventId',
+        'mutationId',
+        'principal',
+        'projectId',
+        'requestId',
+        'retryDelayMs',
+        'runId',
+      ]
+        .sort()
+        .join('\0')
+  ) {
+    throw new ClusterRunManagementRequestError();
+  }
+}
+
 function validUuid(value: unknown): value is string {
   return typeof value === 'string' && UUID_PATTERN.test(value);
 }
@@ -195,6 +277,12 @@ function failureReason(error: unknown): string {
   if (error instanceof ClusterRunCancellationFenceRejectedError) {
     return error.reason;
   }
+  if (error instanceof RunCancellationDispatchManagementNotFoundError) {
+    return 'run_not_found';
+  }
+  if (error instanceof RunCancellationDispatchManagementConflictError) {
+    return error.reason;
+  }
   return 'management_unavailable';
 }
 
@@ -227,6 +315,8 @@ export function createClusterRunManagementService(
   const cancellations = new PostgresClusterRunCancellationRepository(
     options.pool,
   );
+  const cancellationDispatches =
+    new PostgresRunCancellationDispatchManagementRepository(options.pool);
   const audit = new PostgresSecurityAuditRepository(options.pool);
 
   return Object.freeze({
@@ -412,5 +502,188 @@ export function createClusterRunManagementService(
         throw new ClusterRunManagementUnavailableError({ cause: error });
       }
     },
+    async inspectCancellation(
+      requestValue: Readonly<ClusterRunManagementCancellationInspectRequest>,
+    ) {
+      exactCancellationInspectRequest(requestValue);
+      const observedAtMs = now();
+      let principal: Readonly<SecurityPrincipal>;
+      if (
+        !Number.isSafeInteger(observedAtMs) ||
+        observedAtMs < 0 ||
+        !IDENTIFIER_PATTERN.test(requestValue.projectId) ||
+        !IDENTIFIER_PATTERN.test(requestValue.runId) ||
+        !IDENTIFIER_PATTERN.test(requestValue.requestId) ||
+        !validUuid(requestValue.auditEventId) ||
+        !validUuid(requestValue.failureAuditEventId) ||
+        requestValue.auditEventId === requestValue.failureAuditEventId
+      ) {
+        throw new ClusterRunManagementRequestError();
+      }
+      try {
+        principal = normalizeSecurityPrincipal(
+          requestValue.principal,
+          observedAtMs,
+        );
+      } catch {
+        throw new ClusterRunManagementRequestError();
+      }
+      let fence: Readonly<SecurityPolicyFence> | null = null;
+      try {
+        const decision = await policy.authorize(
+          principal,
+          requestValue.projectId,
+          'run.read',
+        );
+        fence = decision.fence;
+        if (
+          decision.effect !== 'allow' ||
+          !fence ||
+          fence.bindingVersion === null
+        ) {
+          throw new ClusterRunManagementAuthorizationError();
+        }
+        return await cancellationDispatches.inspect({
+          projectId: requestValue.projectId,
+          runId: requestValue.runId,
+          requestId: requestValue.requestId,
+          auditEventId: requestValue.auditEventId,
+          principal,
+          policyFence: fence,
+        });
+      } catch (error) {
+        try {
+          await audit.record(
+            normalizeSecurityAuditRecord({
+              eventId: requestValue.failureAuditEventId,
+              requestId: requestValue.requestId,
+              operationId: 'run.cancellation.inspect',
+              projectId: requestValue.projectId,
+              subject: principal.subject,
+              authenticationId: principal.authenticationId,
+              outcome: 'denied',
+              reasons: [failureReason(error)],
+              fence,
+              occurredAtMs: observedAtMs,
+            }),
+          );
+        } catch (auditError) {
+          throw new ClusterRunManagementUnavailableError({ cause: auditError });
+        }
+        if (error instanceof ClusterRunManagementAuthorizationError) throw error;
+        if (error instanceof InvalidRunCancellationDispatchManagementError) {
+          throw new ClusterRunManagementRequestError();
+        }
+        if (error instanceof RunCancellationDispatchManagementNotFoundError) {
+          throw new ClusterRunManagementTargetUnavailableError();
+        }
+        if (error instanceof RunCancellationDispatchManagementConflictError) {
+          throw new ClusterRunManagementConflictError();
+        }
+        if (error instanceof RunCancellationDispatchManagementUnavailableError) {
+          throw new ClusterRunManagementUnavailableError({ cause: error });
+        }
+        throw new ClusterRunManagementUnavailableError({ cause: error });
+      }
+    },
+    async rearmCancellation(
+      requestValue: Readonly<ClusterRunManagementCancellationRearmRequest>,
+    ) {
+      exactCancellationRearmRequest(requestValue);
+      const observedAtMs = now();
+      let principal: Readonly<SecurityPrincipal>;
+      if (
+        !Number.isSafeInteger(observedAtMs) ||
+        observedAtMs < 0 ||
+        !IDENTIFIER_PATTERN.test(requestValue.projectId) ||
+        !IDENTIFIER_PATTERN.test(requestValue.runId) ||
+        !IDENTIFIER_PATTERN.test(requestValue.requestId) ||
+        !validUuid(requestValue.mutationId) ||
+        !validUuid(requestValue.auditEventId) ||
+        !validUuid(requestValue.failureAuditEventId) ||
+        requestValue.auditEventId === requestValue.failureAuditEventId ||
+        !Number.isSafeInteger(requestValue.expectedDispatchVersion) ||
+        requestValue.expectedDispatchVersion < 1 ||
+        requestValue.expectedDispatchVersion >= 2_147_483_647 ||
+        !CANCELLATION_DISPATCH_BLOCKING_RESULTS.includes(
+          requestValue.expectedLastResult,
+        ) ||
+        !Number.isSafeInteger(requestValue.retryDelayMs) ||
+        requestValue.retryDelayMs < 1_000 ||
+        requestValue.retryDelayMs > 24 * 60 * 60_000
+      ) {
+        throw new ClusterRunManagementRequestError();
+      }
+      try {
+        principal = normalizeSecurityPrincipal(
+          requestValue.principal,
+          observedAtMs,
+        );
+      } catch {
+        throw new ClusterRunManagementRequestError();
+      }
+      let fence: Readonly<SecurityPolicyFence> | null = null;
+      try {
+        const decision = await policy.authorize(
+          principal,
+          requestValue.projectId,
+          'run.stop',
+        );
+        fence = decision.fence;
+        if (
+          decision.effect !== 'allow' ||
+          !fence ||
+          fence.bindingVersion === null
+        ) {
+          throw new ClusterRunManagementAuthorizationError();
+        }
+        return await cancellationDispatches.rearm({
+          projectId: requestValue.projectId,
+          runId: requestValue.runId,
+          requestId: requestValue.requestId,
+          auditEventId: requestValue.auditEventId,
+          principal,
+          policyFence: fence,
+          mutationId: requestValue.mutationId,
+          eventId: createId(),
+          expectedDispatchVersion: requestValue.expectedDispatchVersion,
+          expectedLastResult: requestValue.expectedLastResult,
+          retryDelayMs: requestValue.retryDelayMs,
+        });
+      } catch (error) {
+        try {
+          await audit.record(
+            normalizeSecurityAuditRecord({
+              eventId: requestValue.failureAuditEventId,
+              requestId: requestValue.requestId,
+              operationId: 'run.cancellation.rearm',
+              projectId: requestValue.projectId,
+              subject: principal.subject,
+              authenticationId: principal.authenticationId,
+              outcome: 'denied',
+              reasons: [failureReason(error)],
+              fence,
+              occurredAtMs: observedAtMs,
+            }),
+          );
+        } catch (auditError) {
+          throw new ClusterRunManagementUnavailableError({ cause: auditError });
+        }
+        if (error instanceof ClusterRunManagementAuthorizationError) throw error;
+        if (error instanceof InvalidRunCancellationDispatchManagementError) {
+          throw new ClusterRunManagementRequestError();
+        }
+        if (error instanceof RunCancellationDispatchManagementNotFoundError) {
+          throw new ClusterRunManagementTargetUnavailableError();
+        }
+        if (error instanceof RunCancellationDispatchManagementConflictError) {
+          throw new ClusterRunManagementConflictError();
+        }
+        if (error instanceof RunCancellationDispatchManagementUnavailableError) {
+          throw new ClusterRunManagementUnavailableError({ cause: error });
+        }
+        throw new ClusterRunManagementUnavailableError({ cause: error });
+      }
+    },
   });
 }

packages/ql3-cluster-admin/src/run-management/runManagementClient.ts

@@ -6,6 +6,11 @@ import {
   RUN_CANCELLATION_SCHEMA,
   normalizeRunCancellationResult,
 } from '@qinglong/runtime-core/run-cancellation';
+import { RUN_STATUSES } from '@qinglong/runtime-core/run';
+import {
+  CANCELLATION_DISPATCH_RESULTS,
+  CANCELLATION_DISPATCH_STATUSES,
+} from '@qinglong/runtime-core/cancellation-dispatch';
 import {
   ClusterPluginPackageManagementClientRequestError,
   executeClusterAuthenticatedManagementClient,
@@ -14,6 +19,8 @@ import {
   type ClusterPluginPackageManagementClientPaths,
 } from '../management-support/pluginPackageManagementClient';
 import {
+  RUN_CANCELLATION_DISPATCH_DIAGNOSTIC_SCHEMA,
+  RUN_CANCELLATION_DISPATCH_REARM_RECEIPT_SCHEMA,
   normalizeClusterRunManagementCommand,
   type ClusterRunManagementCommand,
   type ClusterRunManagementTransportResult,
@@ -48,6 +55,14 @@ function exact(
   return value as Record<string, unknown>;
 }
 
+function safeInteger(value: unknown, minimum = 0): value is number {
+  return (
+    typeof value === 'number' &&
+    Number.isSafeInteger(value) &&
+    value >= minimum
+  );
+}
+
 export function validateClusterRunManagementClientResult(
   value: unknown,
   command: Readonly<ClusterRunManagementCommand>,
@@ -100,6 +115,151 @@ export function validateClusterRunManagementClientResult(
       envelope as unknown as ClusterRunManagementTransportResult,
     );
   }
+  if (command.operation === 'run.cancellation.inspect') {
+    const envelope = exact(value, [
+      'schemaVersion',
+      'operation',
+      'diagnostic',
+    ]);
+    if (
+      envelope.schemaVersion !== 1 ||
+      envelope.operation !== command.operation
+    ) {
+      invalid();
+    }
+    const diagnostic = exact(envelope.diagnostic, [
+      'schema',
+      'projectId',
+      'runId',
+      'runStatus',
+      'runVersion',
+      'eventSequence',
+      ...(Object.hasOwn(envelope.diagnostic as object, 'cancelRequestedAtMs')
+        ? ['cancelRequestedAtMs', 'cancelReason']
+        : []),
+      'operatorAction',
+      'dispatch',
+    ]);
+    if (
+      diagnostic.schema !== RUN_CANCELLATION_DISPATCH_DIAGNOSTIC_SCHEMA ||
+      diagnostic.projectId !== command.request.projectId ||
+      diagnostic.runId !== command.request.runId ||
+      !RUN_STATUSES.includes(diagnostic.runStatus as never) ||
+      !safeInteger(diagnostic.runVersion, 1) ||
+      !safeInteger(diagnostic.eventSequence) ||
+      !['none', 'wait', 'rearm'].includes(diagnostic.operatorAction as string) ||
+      (Object.hasOwn(diagnostic, 'cancelRequestedAtMs') &&
+        (!safeInteger(diagnostic.cancelRequestedAtMs) ||
+          !['user', 'policy', 'shutdown', 'reconcile', 'timeout'].includes(
+            diagnostic.cancelReason as string,
+          )))
+    ) {
+      invalid();
+    }
+    let dispatchStatus: string | null = null;
+    if (diagnostic.dispatch !== null) {
+      const dispatch = exact(diagnostic.dispatch, [
+        'attemptId',
+        'status',
+        'version',
+        'dispatchCount',
+        ...(Object.hasOwn(diagnostic.dispatch as object, 'nextAttemptAtMs')
+          ? ['nextAttemptAtMs']
+          : []),
+        ...(Object.hasOwn(diagnostic.dispatch as object, 'leaseExpiresAtMs')
+          ? ['leaseExpiresAtMs']
+          : []),
+        ...(Object.hasOwn(diagnostic.dispatch as object, 'lastResult')
+          ? ['lastResult']
+          : []),
+        ...(Object.hasOwn(
+          diagnostic.dispatch as object,
+          'lastDispatchedAtMs',
+        )
+          ? ['lastDispatchedAtMs']
+          : []),
+        'createdAtMs',
+        'updatedAtMs',
+      ]);
+      if (
+        typeof dispatch.attemptId !== 'string' ||
+        dispatch.attemptId.length < 1 ||
+        !CANCELLATION_DISPATCH_STATUSES.includes(dispatch.status as never) ||
+        !safeInteger(dispatch.version) ||
+        !safeInteger(dispatch.dispatchCount) ||
+        !safeInteger(dispatch.createdAtMs) ||
+        !safeInteger(dispatch.updatedAtMs) ||
+        (Object.hasOwn(dispatch, 'nextAttemptAtMs') &&
+          !safeInteger(dispatch.nextAttemptAtMs)) ||
+        (Object.hasOwn(dispatch, 'leaseExpiresAtMs') &&
+          !safeInteger(dispatch.leaseExpiresAtMs)) ||
+        (Object.hasOwn(dispatch, 'lastDispatchedAtMs') &&
+          !safeInteger(dispatch.lastDispatchedAtMs)) ||
+        (Object.hasOwn(dispatch, 'lastResult') &&
+          !CANCELLATION_DISPATCH_RESULTS.includes(dispatch.lastResult as never))
+      ) {
+        invalid();
+      }
+      dispatchStatus = dispatch.status as string;
+    }
+    const expectedOperatorAction =
+      dispatchStatus === 'blocked'
+        ? 'rearm'
+        : dispatchStatus !== null && dispatchStatus !== 'dispatched'
+          ? 'wait'
+          : dispatchStatus === null &&
+              Object.hasOwn(diagnostic, 'cancelRequestedAtMs')
+            ? 'wait'
+            : 'none';
+    if (diagnostic.operatorAction !== expectedOperatorAction) invalid();
+    return Object.freeze(
+      envelope as unknown as ClusterRunManagementTransportResult,
+    );
+  }
+  if (command.operation === 'run.cancellation.rearm') {
+    const envelope = exact(value, ['schemaVersion', 'operation', 'rearm']);
+    if (
+      envelope.schemaVersion !== 1 ||
+      envelope.operation !== command.operation
+    ) {
+      invalid();
+    }
+    const rearm = exact(envelope.rearm, [
+      'schema',
+      'status',
+      'projectId',
+      'runId',
+      'attemptId',
+      'previousDispatchVersion',
+      'dispatchVersion',
+      'previousResult',
+      'retryDelayMs',
+      'nextAttemptAtMs',
+      'runVersion',
+      'eventSequence',
+    ]);
+    if (
+      rearm.schema !== RUN_CANCELLATION_DISPATCH_REARM_RECEIPT_SCHEMA ||
+      rearm.status !== 'rearmed' ||
+      rearm.projectId !== command.request.projectId ||
+      rearm.runId !== command.request.runId ||
+      typeof rearm.attemptId !== 'string' ||
+      rearm.attemptId.length < 1 ||
+      rearm.previousDispatchVersion !==
+        command.request.body.expectedDispatchVersion ||
+      rearm.dispatchVersion !== rearm.previousDispatchVersion + 1 ||
+      rearm.previousResult !== command.request.body.expectedLastResult ||
+      rearm.retryDelayMs !== command.request.body.retryDelayMs ||
+      !safeInteger(rearm.nextAttemptAtMs) ||
+      !safeInteger(rearm.runVersion, 1) ||
+      !safeInteger(rearm.eventSequence, 1)
+    ) {
+      invalid();
+    }
+    return Object.freeze(
+      envelope as unknown as ClusterRunManagementTransportResult,
+    );
+  }
   const envelope = exact(value, ['schemaVersion', 'operation', 'stop']);
   if (
     envelope.schemaVersion !== 1 ||

packages/ql3-cluster-admin/src/run-management/runManagementTransport.ts

@@ -12,6 +12,7 @@ import {
   normalizeSecurityPrincipal,
   type SecurityPrincipal,
 } from '@qinglong/runtime-core/security';
+import { CANCELLATION_DISPATCH_BLOCKING_RESULTS } from '@qinglong/runtime-core/cancellation-dispatch';
 import type { ClusterRunManagementService } from './runManagement';
 
 const IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
@@ -19,6 +20,15 @@ const UUID_PATTERN =
   /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
 const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);
 
+export const RUN_CANCELLATION_DISPATCH_INSPECT_REQUEST_SCHEMA =
+  'qinglong/run-cancellation-dispatch-inspect@v1';
+export const RUN_CANCELLATION_DISPATCH_DIAGNOSTIC_SCHEMA =
+  'qinglong/run-cancellation-dispatch-diagnostic@v1';
+export const RUN_CANCELLATION_DISPATCH_REARM_REQUEST_SCHEMA =
+  'qinglong/run-cancellation-dispatch-rearm-request@v1';
+export const RUN_CANCELLATION_DISPATCH_REARM_RECEIPT_SCHEMA =
+  'qinglong/run-cancellation-dispatch-rearm-receipt@v1';
+
 export type ClusterRunManagementRetryCommand = Readonly<{
   schemaVersion: 1;
   operation: 'run.retry';
@@ -53,9 +63,49 @@ export type ClusterRunManagementStopCommand = Readonly<{
   }>;
 }>;
 
+export type ClusterRunManagementCancellationInspectCommand = Readonly<{
+  schemaVersion: 1;
+  operation: 'run.cancellation.inspect';
+  request: Readonly<{
+    projectId: string;
+    runId: string;
+    requestId: string;
+    auditEventId: string;
+    failureAuditEventId: string;
+    body: Readonly<{
+      schema: typeof RUN_CANCELLATION_DISPATCH_INSPECT_REQUEST_SCHEMA;
+    }>;
+  }>;
+}>;
+
+export type ClusterRunManagementCancellationRearmCommand = Readonly<{
+  schemaVersion: 1;
+  operation: 'run.cancellation.rearm';
+  request: Readonly<{
+    projectId: string;
+    runId: string;
+    requestId: string;
+    auditEventId: string;
+    failureAuditEventId: string;
+    body: Readonly<{
+      schema: typeof RUN_CANCELLATION_DISPATCH_REARM_REQUEST_SCHEMA;
+      mutationId: string;
+      expectedDispatchVersion: number;
+      expectedLastResult:
+        | 'identity_mismatch'
+        | 'pid_mismatch'
+        | 'unsupported'
+        | 'invalid';
+      retryDelayMs: number;
+    }>;
+  }>;
+}>;
+
 export type ClusterRunManagementCommand =
   | ClusterRunManagementRetryCommand
-  | ClusterRunManagementStopCommand;
+  | ClusterRunManagementStopCommand
+  | ClusterRunManagementCancellationInspectCommand
+  | ClusterRunManagementCancellationRearmCommand;
 
 export type ClusterRunManagementRetryTransportResult = Readonly<{
   schemaVersion: 1;
@@ -69,9 +119,31 @@ export type ClusterRunManagementStopTransportResult = Readonly<{
   stop: Readonly<RunCancellationResponseBody>;
 }>;
 
+export type ClusterRunManagementCancellationInspectTransportResult = Readonly<{
+  schemaVersion: 1;
+  operation: 'run.cancellation.inspect';
+  diagnostic: Readonly<
+    Awaited<ReturnType<ClusterRunManagementService['inspectCancellation']>> & {
+      schema: typeof RUN_CANCELLATION_DISPATCH_DIAGNOSTIC_SCHEMA;
+    }
+  >;
+}>;
+
+export type ClusterRunManagementCancellationRearmTransportResult = Readonly<{
+  schemaVersion: 1;
+  operation: 'run.cancellation.rearm';
+  rearm: Readonly<
+    Awaited<ReturnType<ClusterRunManagementService['rearmCancellation']>> & {
+      schema: typeof RUN_CANCELLATION_DISPATCH_REARM_RECEIPT_SCHEMA;
+    }
+  >;
+}>;
+
 export type ClusterRunManagementTransportResult =
   | ClusterRunManagementRetryTransportResult
-  | ClusterRunManagementStopTransportResult;
+  | ClusterRunManagementStopTransportResult
+  | ClusterRunManagementCancellationInspectTransportResult
+  | ClusterRunManagementCancellationRearmTransportResult;
 
 export interface ClusterRunManagementAuthentication {
   authenticate(): Promise<Readonly<SecurityPrincipal> | null>;
@@ -152,7 +224,14 @@ export function normalizeClusterRunManagementCommand(
   const envelope = exact(value, ['schemaVersion', 'operation', 'request']);
   if (envelope.schemaVersion !== 1) invalid();
   const operation = envelope.operation;
-  if (operation !== 'run.retry' && operation !== 'run.stop') invalid();
+  if (
+    operation !== 'run.retry' &&
+    operation !== 'run.stop' &&
+    operation !== 'run.cancellation.inspect' &&
+    operation !== 'run.cancellation.rearm'
+  ) {
+    invalid();
+  }
   const request = exact(
     envelope.request,
     operation === 'run.retry'
@@ -196,6 +275,70 @@ export function normalizeClusterRunManagementCommand(
       }),
     });
   }
+  if (operation === 'run.cancellation.inspect') {
+    const body = exact(request.body, ['schema']);
+    if (body.schema !== RUN_CANCELLATION_DISPATCH_INSPECT_REQUEST_SCHEMA) {
+      invalid();
+    }
+    return Object.freeze({
+      schemaVersion: 1,
+      operation,
+      request: Object.freeze({
+        projectId: identifier(request.projectId),
+        runId: identifier(request.runId),
+        requestId: identifier(request.requestId),
+        auditEventId,
+        failureAuditEventId,
+        body: Object.freeze({
+          schema: RUN_CANCELLATION_DISPATCH_INSPECT_REQUEST_SCHEMA,
+        }),
+      }),
+    });
+  }
+  if (operation === 'run.cancellation.rearm') {
+    const body = exact(request.body, [
+      'schema',
+      'mutationId',
+      'expectedDispatchVersion',
+      'expectedLastResult',
+      'retryDelayMs',
+    ]);
+    if (
+      body.schema !== RUN_CANCELLATION_DISPATCH_REARM_REQUEST_SCHEMA ||
+      !CANCELLATION_DISPATCH_BLOCKING_RESULTS.includes(
+        body.expectedLastResult as never,
+      ) ||
+      typeof body.expectedDispatchVersion !== 'number' ||
+      !Number.isSafeInteger(body.expectedDispatchVersion) ||
+      body.expectedDispatchVersion < 1 ||
+      body.expectedDispatchVersion >= 2_147_483_647 ||
+      typeof body.retryDelayMs !== 'number' ||
+      !Number.isSafeInteger(body.retryDelayMs) ||
+      body.retryDelayMs < 1_000 ||
+      body.retryDelayMs > 24 * 60 * 60_000
+    ) {
+      invalid();
+    }
+    return Object.freeze({
+      schemaVersion: 1,
+      operation,
+      request: Object.freeze({
+        projectId: identifier(request.projectId),
+        runId: identifier(request.runId),
+        requestId: identifier(request.requestId),
+        auditEventId,
+        failureAuditEventId,
+        body: Object.freeze({
+          schema: RUN_CANCELLATION_DISPATCH_REARM_REQUEST_SCHEMA,
+          mutationId: uuid(body.mutationId),
+          expectedDispatchVersion: body.expectedDispatchVersion,
+          expectedLastResult: body.expectedLastResult as
+            ClusterRunManagementCancellationRearmCommand['request']['body']['expectedLastResult'],
+          retryDelayMs: body.retryDelayMs,
+        }),
+      }),
+    });
+  }
   let body: ReturnType<typeof parseRunCancellationRequestBody>;
   try {
     body = parseRunCancellationRequestBody(request.body);
@@ -231,6 +374,8 @@ export function createClusterRunManagementTransport(
     !options.service ||
     typeof options.service.retry !== 'function' ||
     typeof options.service.stop !== 'function' ||
+    typeof options.service.inspectCancellation !== 'function' ||
+    typeof options.service.rearmCancellation !== 'function' ||
     (options.now !== undefined && typeof options.now !== 'function')
   ) {
     throw new ClusterRunManagementTransportConfigurationError();
@@ -290,6 +435,47 @@ export function createClusterRunManagementTransport(
           retry: createRunManualRetryResponseBody(result),
         });
       }
+      if (command.operation === 'run.cancellation.inspect') {
+        const result = await options.service.inspectCancellation({
+          projectId: command.request.projectId,
+          runId: command.request.runId,
+          requestId: command.request.requestId,
+          auditEventId: command.request.auditEventId,
+          failureAuditEventId: command.request.failureAuditEventId,
+          principal,
+        });
+        return Object.freeze({
+          schemaVersion: 1,
+          operation: command.operation,
+          diagnostic: Object.freeze({
+            schema: RUN_CANCELLATION_DISPATCH_DIAGNOSTIC_SCHEMA,
+            ...result,
+          }),
+        });
+      }
+      if (command.operation === 'run.cancellation.rearm') {
+        const result = await options.service.rearmCancellation({
+          projectId: command.request.projectId,
+          runId: command.request.runId,
+          requestId: command.request.requestId,
+          auditEventId: command.request.auditEventId,
+          failureAuditEventId: command.request.failureAuditEventId,
+          principal,
+          mutationId: command.request.body.mutationId,
+          expectedDispatchVersion:
+            command.request.body.expectedDispatchVersion,
+          expectedLastResult: command.request.body.expectedLastResult,
+          retryDelayMs: command.request.body.retryDelayMs,
+        });
+        return Object.freeze({
+          schemaVersion: 1,
+          operation: command.operation,
+          rearm: Object.freeze({
+            schema: RUN_CANCELLATION_DISPATCH_REARM_RECEIPT_SCHEMA,
+            ...result,
+          }),
+        });
+      }
       const result = await options.service.stop({
         projectId: command.request.projectId,
         runId: command.request.runId,