feat(ql3): add strong cluster run stop

2026-09-22 10:32:40 +08:00 · 2026-08-12 08:21:28 +08:00
parent c0ab62e64a
commit dd370b2842
22 changed files with 1602 additions and 321 deletions
@@ -1,12 +1,20 @@
 import { randomUUID } from 'node:crypto';

 import {
+  PostgresClusterRunCancellationRepository,
  PostgresProjectPolicyRepository,
  PostgresRunManualRetryRepository,
  PostgresSecurityAuditRepository,
 } from '@qinglong/cluster-postgres/run-manager';
 import type { PostgresPool } from '@qinglong/runtime-core';
 import { ProjectPolicyEngine } from '@qinglong/runtime-core/project-policy';
+import {
+  ClusterRunCancellationFenceRejectedError,
+  ClusterRunCancellationNotFoundError,
+  ClusterRunCancellationUnavailableError,
+  InvalidClusterRunCancellationError,
+  type ClusterRunCancellationResult,
+} from '@qinglong/runtime-core/run-cancellation';
 import {
  InvalidRunManualRetryError,
  RunManualRetryFenceRejectedError,
@@ -39,10 +47,23 @@ export interface ClusterRunManagementRetryRequest {
  readonly principal: Readonly<SecurityPrincipal>;
 }

+export interface ClusterRunManagementStopRequest {
+  readonly projectId: string;
+  readonly runId: string;
+  readonly mutationId: string;
+  readonly requestId: string;
+  readonly auditEventId: string;
+  readonly failureAuditEventId: string;
+  readonly principal: Readonly<SecurityPrincipal>;
+}
+
 export interface ClusterRunManagementService {
  retry(
    request: Readonly<ClusterRunManagementRetryRequest>,
  ): Promise<Readonly<RunManualRetryResult>>;
+  stop(
+    request: Readonly<ClusterRunManagementStopRequest>,
+  ): Promise<Readonly<ClusterRunCancellationResult>>;
 }

 export interface ClusterRunManagementOptions {
@@ -107,7 +128,7 @@ export class ClusterRunManagementUnavailableError extends Error {
  }
 }

-function exactRequest(
+function exactRetryRequest(
  value: unknown,
 ): asserts value is Readonly<ClusterRunManagementRetryRequest> {
  if (
@@ -133,6 +154,30 @@ function exactRequest(
  }
 }

+function exactStopRequest(
+  value: unknown,
+): asserts value is Readonly<ClusterRunManagementStopRequest> {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    Object.keys(value).sort().join('\0') !==
+      [
+        'auditEventId',
+        'failureAuditEventId',
+        'mutationId',
+        'principal',
+        'projectId',
+        'requestId',
+        'runId',
+      ]
+        .sort()
+        .join('\0')
+  ) {
+    throw new ClusterRunManagementRequestError();
+  }
+}
+
 function validUuid(value: unknown): value is string {
  return typeof value === 'string' && UUID_PATTERN.test(value);
 }
@@ -144,6 +189,12 @@ function failureReason(error: unknown): string {
  if (error instanceof RunManualRetryNotFoundError) return 'run_not_found';
  if (error instanceof RunManualRetryRateLimitedError) return 'rate_limited';
  if (error instanceof RunManualRetryFenceRejectedError) return error.reason;
+  if (error instanceof ClusterRunCancellationNotFoundError) {
+    return 'run_not_found';
+  }
+  if (error instanceof ClusterRunCancellationFenceRejectedError) {
+    return error.reason;
+  }
  return 'management_unavailable';
 }

@@ -162,7 +213,8 @@ export function createClusterRunManagementService(
    typeof options.pool.query !== 'function' ||
    typeof options.pool.connect !== 'function' ||
    (options.now !== undefined && typeof options.now !== 'function') ||
-    (options.randomUuid !== undefined && typeof options.randomUuid !== 'function')
+    (options.randomUuid !== undefined &&
+      typeof options.randomUuid !== 'function')
  ) {
    throw new ClusterRunManagementConfigurationError();
  }
@@ -172,11 +224,14 @@ export function createClusterRunManagementService(
    new PostgresProjectPolicyRepository(options.pool),
  );
  const retries = new PostgresRunManualRetryRepository(options.pool);
+  const cancellations = new PostgresClusterRunCancellationRepository(
+    options.pool,
+  );
  const audit = new PostgresSecurityAuditRepository(options.pool);

  return Object.freeze({
    async retry(requestValue: Readonly<ClusterRunManagementRetryRequest>) {
-      exactRequest(requestValue);
+      exactRetryRequest(requestValue);
      const observedAtMs = now();
      let principal: Readonly<SecurityPrincipal>;
      if (
@@ -250,7 +305,8 @@ export function createClusterRunManagementService(
        } catch (auditError) {
          throw new ClusterRunManagementUnavailableError({ cause: auditError });
        }
-        if (error instanceof ClusterRunManagementAuthorizationError) throw error;
+        if (error instanceof ClusterRunManagementAuthorizationError)
+          throw error;
        if (error instanceof InvalidRunManualRetryError) {
          throw new ClusterRunManagementRequestError();
        }
@@ -269,5 +325,92 @@ export function createClusterRunManagementService(
        throw new ClusterRunManagementUnavailableError({ cause: error });
      }
    },
+    async stop(requestValue: Readonly<ClusterRunManagementStopRequest>) {
+      exactStopRequest(requestValue);
+      const observedAtMs = now();
+      let principal: Readonly<SecurityPrincipal>;
+      if (
+        !Number.isSafeInteger(observedAtMs) ||
+        observedAtMs < 0 ||
+        !IDENTIFIER_PATTERN.test(requestValue.projectId) ||
+        !IDENTIFIER_PATTERN.test(requestValue.runId) ||
+        !IDENTIFIER_PATTERN.test(requestValue.requestId) ||
+        !validUuid(requestValue.mutationId) ||
+        !validUuid(requestValue.auditEventId) ||
+        !validUuid(requestValue.failureAuditEventId) ||
+        requestValue.auditEventId === requestValue.failureAuditEventId
+      ) {
+        throw new ClusterRunManagementRequestError();
+      }
+      try {
+        principal = normalizeSecurityPrincipal(
+          requestValue.principal,
+          observedAtMs,
+        );
+      } catch {
+        throw new ClusterRunManagementRequestError();
+      }
+
+      let fence: Readonly<SecurityPolicyFence> | null = null;
+      try {
+        const decision = await policy.authorize(
+          principal,
+          requestValue.projectId,
+          'run.stop',
+        );
+        fence = decision.fence;
+        if (
+          decision.effect !== 'allow' ||
+          !fence ||
+          fence.bindingVersion === null
+        ) {
+          throw new ClusterRunManagementAuthorizationError();
+        }
+        return await cancellations.requestUserCancellationAudited({
+          projectId: requestValue.projectId,
+          runId: requestValue.runId,
+          mutationId: requestValue.mutationId,
+          eventId: createId(),
+          requestId: requestValue.requestId,
+          auditEventId: requestValue.auditEventId,
+          principal,
+          policyFence: fence,
+        });
+      } catch (error) {
+        try {
+          await audit.record(
+            normalizeSecurityAuditRecord({
+              eventId: requestValue.failureAuditEventId,
+              requestId: requestValue.requestId,
+              operationId: 'run.stop',
+              projectId: requestValue.projectId,
+              subject: principal.subject,
+              authenticationId: principal.authenticationId,
+              outcome: 'denied',
+              reasons: [failureReason(error)],
+              fence,
+              occurredAtMs: observedAtMs,
+            }),
+          );
+        } catch (auditError) {
+          throw new ClusterRunManagementUnavailableError({ cause: auditError });
+        }
+        if (error instanceof ClusterRunManagementAuthorizationError)
+          throw error;
+        if (error instanceof InvalidClusterRunCancellationError) {
+          throw new ClusterRunManagementRequestError();
+        }
+        if (error instanceof ClusterRunCancellationNotFoundError) {
+          throw new ClusterRunManagementTargetUnavailableError();
+        }
+        if (error instanceof ClusterRunCancellationFenceRejectedError) {
+          throw new ClusterRunManagementConflictError();
+        }
+        if (error instanceof ClusterRunCancellationUnavailableError) {
+          throw new ClusterRunManagementUnavailableError({ cause: error });
+        }
+        throw new ClusterRunManagementUnavailableError({ cause: error });
+      }
+    },
  });
 }
@@ -2,6 +2,10 @@ import {
  RUN_MANUAL_RETRY_SCHEMA,
  normalizeRunManualRetryResult,
 } from '@qinglong/runtime-core/run-manual-retry';
+import {
+  RUN_CANCELLATION_SCHEMA,
+  normalizeRunCancellationResult,
+} from '@qinglong/runtime-core/run-cancellation';
 import {
  ClusterPluginPackageManagementClientRequestError,
  executeClusterAuthenticatedManagementClient,
@@ -28,7 +32,10 @@ function invalid(): never {
  throw new ClusterPluginPackageManagementClientRequestError();
 }

-function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
+function exact(
+  value: unknown,
+  keys: readonly string[],
+): Record<string, unknown> {
  if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
  const actual = Object.keys(value as object).sort();
  const expected = [...keys].sort();
@@ -45,37 +52,80 @@ export function validateClusterRunManagementClientResult(
  value: unknown,
  command: Readonly<ClusterRunManagementCommand>,
 ): Readonly<ClusterRunManagementTransportResult> {
-  const envelope = exact(value, ['schemaVersion', 'operation', 'retry']);
-  if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
-  const retry = exact(envelope.retry, [
+  if (command.operation === 'run.retry') {
+    const envelope = exact(value, ['schemaVersion', 'operation', 'retry']);
+    if (
+      envelope.schemaVersion !== 1 ||
+      envelope.operation !== command.operation
+    ) {
+      invalid();
+    }
+    const retry = exact(envelope.retry, [
+      'schema',
+      'status',
+      'projectId',
+      'sourceRunId',
+      'sourceRunStatus',
+      'sourceRunVersion',
+      'runId',
+      'retryOfRunId',
+      'taskId',
+      'taskRevision',
+      'attemptId',
+      'runStatus',
+      'runVersion',
+      'eventSequence',
+      'executorType',
+      'executionRevisionDigest',
+      'createdAtMs',
+    ]);
+    if (retry.schema !== RUN_MANUAL_RETRY_SCHEMA) invalid();
+    try {
+      const { schema: _schema, ...result } = retry;
+      const normalized = normalizeRunManualRetryResult(result as never);
+      if (
+        normalized.projectId !== command.request.projectId ||
+        normalized.sourceRunId !== command.request.sourceRunId ||
+        normalized.sourceRunVersion !==
+          command.request.body.expectedRunVersion ||
+        normalized.sourceRunStatus !== command.request.body.expectedRunStatus ||
+        normalized.executorType !== 'remote_worker'
+      ) {
+        invalid();
+      }
+    } catch {
+      invalid();
+    }
+    return Object.freeze(
+      envelope as unknown as ClusterRunManagementTransportResult,
+    );
+  }
+  const envelope = exact(value, ['schemaVersion', 'operation', 'stop']);
+  if (
+    envelope.schemaVersion !== 1 ||
+    envelope.operation !== command.operation
+  ) {
+    invalid();
+  }
+  const stop = exact(envelope.stop, [
    'schema',
    'status',
    'projectId',
-    'sourceRunId',
-    'sourceRunStatus',
-    'sourceRunVersion',
    'runId',
-    'retryOfRunId',
-    'taskId',
-    'taskRevision',
-    'attemptId',
    'runStatus',
    'runVersion',
    'eventSequence',
-    'executorType',
-    'executionRevisionDigest',
-    'createdAtMs',
+    ...(Object.hasOwn(envelope.stop as object, 'cancelRequestedAtMs')
+      ? ['cancelRequestedAtMs', 'cancelReason']
+      : []),
  ]);
-  if (retry.schema !== RUN_MANUAL_RETRY_SCHEMA) invalid();
+  if (stop.schema !== RUN_CANCELLATION_SCHEMA) invalid();
  try {
-    const { schema: _schema, ...result } = retry;
-    const normalized = normalizeRunManualRetryResult(result as never);
+    const { schema: _schema, ...result } = stop;
+    const normalized = normalizeRunCancellationResult(result as never);
    if (
      normalized.projectId !== command.request.projectId ||
-      normalized.sourceRunId !== command.request.sourceRunId ||
-      normalized.sourceRunVersion !== command.request.body.expectedRunVersion ||
-      normalized.sourceRunStatus !== command.request.body.expectedRunStatus ||
-      normalized.executorType !== 'remote_worker'
+      normalized.runId !== command.request.runId
    ) {
      invalid();
    }
@@ -3,6 +3,11 @@ import {
  parseRunManualRetryRequestBody,
  type RunManualRetryResponseBody,
 } from '@qinglong/runtime-core/run-manual-retry';
+import {
+  createRunCancellationResponseBody,
+  parseRunCancellationRequestBody,
+  type RunCancellationResponseBody,
+} from '@qinglong/runtime-core/run-cancellation';
 import {
  normalizeSecurityPrincipal,
  type SecurityPrincipal,
@@ -14,7 +19,7 @@ const UUID_PATTERN =
  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
 const STRONG_ASSURANCES = new Set(['multi_factor', 'hardware']);

-export type ClusterRunManagementCommand = Readonly<{
+export type ClusterRunManagementRetryCommand = Readonly<{
  schemaVersion: 1;
  operation: 'run.retry';
  request: Readonly<{
@@ -32,12 +37,42 @@ export type ClusterRunManagementCommand = Readonly<{
  }>;
 }>;

-export type ClusterRunManagementTransportResult = Readonly<{
+export type ClusterRunManagementStopCommand = Readonly<{
+  schemaVersion: 1;
+  operation: 'run.stop';
+  request: Readonly<{
+    projectId: string;
+    runId: string;
+    requestId: string;
+    auditEventId: string;
+    failureAuditEventId: string;
+    body: Readonly<{
+      schema: 'qinglong/run-cancellation@v1';
+      mutationId: string;
+    }>;
+  }>;
+}>;
+
+export type ClusterRunManagementCommand =
+  | ClusterRunManagementRetryCommand
+  | ClusterRunManagementStopCommand;
+
+export type ClusterRunManagementRetryTransportResult = Readonly<{
  schemaVersion: 1;
  operation: 'run.retry';
  retry: Readonly<RunManualRetryResponseBody>;
 }>;

+export type ClusterRunManagementStopTransportResult = Readonly<{
+  schemaVersion: 1;
+  operation: 'run.stop';
+  stop: Readonly<RunCancellationResponseBody>;
+}>;
+
+export type ClusterRunManagementTransportResult =
+  | ClusterRunManagementRetryTransportResult
+  | ClusterRunManagementStopTransportResult;
+
 export interface ClusterRunManagementAuthentication {
  authenticate(): Promise<Readonly<SecurityPrincipal> | null>;
 }
@@ -85,7 +120,10 @@ function invalid(): never {
  throw new ClusterRunManagementTransportRequestError();
 }

-function exact(value: unknown, keys: readonly string[]): Record<string, unknown> {
+function exact(
+  value: unknown,
+  keys: readonly string[],
+): Record<string, unknown> {
  if (!value || typeof value !== 'object' || Array.isArray(value)) invalid();
  const actual = Object.keys(value as object).sort();
  const expected = [...keys].sort();
@@ -112,30 +150,65 @@ export function normalizeClusterRunManagementCommand(
  value: unknown,
 ): Readonly<ClusterRunManagementCommand> {
  const envelope = exact(value, ['schemaVersion', 'operation', 'request']);
-  if (envelope.schemaVersion !== 1 || envelope.operation !== 'run.retry') invalid();
-  const request = exact(envelope.request, [
-    'projectId',
-    'sourceRunId',
-    'requestId',
-    'auditEventId',
-    'failureAuditEventId',
-    'body',
-  ]);
-  let body: ReturnType<typeof parseRunManualRetryRequestBody>;
-  try {
-    body = parseRunManualRetryRequestBody(request.body);
-  } catch {
-    invalid();
-  }
+  if (envelope.schemaVersion !== 1) invalid();
+  const operation = envelope.operation;
+  if (operation !== 'run.retry' && operation !== 'run.stop') invalid();
+  const request = exact(
+    envelope.request,
+    operation === 'run.retry'
+      ? [
+          'projectId',
+          'sourceRunId',
+          'requestId',
+          'auditEventId',
+          'failureAuditEventId',
+          'body',
+        ]
+      : [
+          'projectId',
+          'runId',
+          'requestId',
+          'auditEventId',
+          'failureAuditEventId',
+          'body',
+        ],
+  );
  const auditEventId = uuid(request.auditEventId);
  const failureAuditEventId = uuid(request.failureAuditEventId);
  if (auditEventId === failureAuditEventId) invalid();
+  if (operation === 'run.retry') {
+    let body: ReturnType<typeof parseRunManualRetryRequestBody>;
+    try {
+      body = parseRunManualRetryRequestBody(request.body);
+    } catch {
+      invalid();
+    }
+    return Object.freeze({
+      schemaVersion: 1,
+      operation,
+      request: Object.freeze({
+        projectId: identifier(request.projectId),
+        sourceRunId: identifier(request.sourceRunId),
+        requestId: identifier(request.requestId),
+        auditEventId,
+        failureAuditEventId,
+        body,
+      }),
+    });
+  }
+  let body: ReturnType<typeof parseRunCancellationRequestBody>;
+  try {
+    body = parseRunCancellationRequestBody(request.body);
+  } catch {
+    invalid();
+  }
+  uuid(body.mutationId);
  return Object.freeze({
    schemaVersion: 1,
-    operation: 'run.retry',
+    operation,
    request: Object.freeze({
      projectId: identifier(request.projectId),
-      sourceRunId: identifier(request.sourceRunId),
+      runId: identifier(request.runId),
      requestId: identifier(request.requestId),
      auditEventId,
      failureAuditEventId,
@@ -144,10 +217,12 @@ export function normalizeClusterRunManagementCommand(
  });
 }

-export function createClusterRunManagementTransport(options: Readonly<{
-  service: ClusterRunManagementService;
-  now?: () => number;
-}>): Readonly<ClusterRunManagementTransport> {
+export function createClusterRunManagementTransport(
+  options: Readonly<{
+    service: ClusterRunManagementService;
+    now?: () => number;
+  }>,
+): Readonly<ClusterRunManagementTransport> {
  if (
    !options ||
    typeof options !== 'object' ||
@@ -155,6 +230,7 @@ export function createClusterRunManagementTransport(options: Readonly<{
    Object.keys(options).some((key) => key !== 'service' && key !== 'now') ||
    !options.service ||
    typeof options.service.retry !== 'function' ||
+    typeof options.service.stop !== 'function' ||
    (options.now !== undefined && typeof options.now !== 'function')
  ) {
    throw new ClusterRunManagementTransportConfigurationError();
@@ -183,7 +259,10 @@ export function createClusterRunManagementTransport(options: Readonly<{
      }
      let principal: Readonly<SecurityPrincipal>;
      try {
-        principal = normalizeSecurityPrincipal(candidate as SecurityPrincipal, now());
+        principal = normalizeSecurityPrincipal(
+          candidate as SecurityPrincipal,
+          now(),
+        );
      } catch {
        throw new ClusterRunManagementTransportAuthenticationError();
      }
@@ -193,12 +272,28 @@ export function createClusterRunManagementTransport(options: Readonly<{
      ) {
        throw new ClusterRunManagementTransportAuthenticationError();
      }
-      const result = await options.service.retry({
+      if (command.operation === 'run.retry') {
+        const result = await options.service.retry({
+          projectId: command.request.projectId,
+          sourceRunId: command.request.sourceRunId,
+          mutationId: command.request.body.mutationId,
+          expectedRunVersion: command.request.body.expectedRunVersion,
+          expectedRunStatus: command.request.body.expectedRunStatus,
+          requestId: command.request.requestId,
+          auditEventId: command.request.auditEventId,
+          failureAuditEventId: command.request.failureAuditEventId,
+          principal,
+        });
+        return Object.freeze({
+          schemaVersion: 1,
+          operation: command.operation,
+          retry: createRunManualRetryResponseBody(result),
+        });
+      }
+      const result = await options.service.stop({
        projectId: command.request.projectId,
-        sourceRunId: command.request.sourceRunId,
+        runId: command.request.runId,
        mutationId: command.request.body.mutationId,
-        expectedRunVersion: command.request.body.expectedRunVersion,
-        expectedRunStatus: command.request.body.expectedRunStatus,
        requestId: command.request.requestId,
        auditEventId: command.request.auditEventId,
        failureAuditEventId: command.request.failureAuditEventId,
@@ -206,8 +301,8 @@ export function createClusterRunManagementTransport(options: Readonly<{
      });
      return Object.freeze({
        schemaVersion: 1,
-        operation: 'run.retry',
-        retry: createRunManualRetryResponseBody(result),
+        operation: command.operation,
+        stop: createRunCancellationResponseBody(result),
      });
    },
  });
@@ -67,7 +67,8 @@ function fixture(role = 'operator') {
    async query(sql, params = []) {
      const text = sql.replace(/\s+/g, ' ').trim();
      calls.push({ scope: 'pool', sql: text, params });
-      if (text.includes('LEFT JOIN LATERAL')) return { rows: [policyRow(role)] };
+      if (text.includes('LEFT JOIN LATERAL'))
+        return { rows: [policyRow(role)] };
      if (text.startsWith('INSERT INTO "ql3"."security_audit_events"')) {
        return { rows: [], rowCount: 1 };
      }
@@ -83,40 +84,89 @@ function fixture(role = 'operator') {
            text === 'COMMIT' ||
            text === 'ROLLBACK' ||
            text.startsWith('SELECT set_config')
-          ) return { rows: [], rowCount: 0 };
+          )
+            return { rows: [], rowCount: 0 };
          if (text.includes('statement_timestamp()')) {
            return { rows: [{ nowMs: NOW }], rowCount: 1 };
          }
          if (text.includes('lock_run_management_policy_fence')) {
            return { rows: [{ matches: true }], rowCount: 1 };
          }
+          if (text.includes('FROM "ql3"."runs" WHERE id = $1 FOR UPDATE')) {
+            return {
+              rows: [
+                {
+                  projectId: 'project-1',
+                  runStatus: 'running',
+                  runVersion: 4,
+                  eventSequence: 6,
+                  cancelRequestedAtMs: null,
+                  cancelReason: null,
+                },
+              ],
+              rowCount: 1,
+            };
+          }
+          if (text.startsWith('UPDATE "ql3"."runs"')) {
+            return {
+              rows: [
+                {
+                  projectId: 'project-1',
+                  runStatus: 'running',
+                  runVersion: 5,
+                  eventSequence: 7,
+                  cancelRequestedAtMs: NOW,
+                  cancelReason: 'user',
+                },
+              ],
+              rowCount: 1,
+            };
+          }
+          if (
+            text.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
+            text.includes('RETURNING event_id')
+          ) {
+            return { rows: [{ eventId: request().auditEventId }], rowCount: 1 };
+          }
          if (text.includes('idempotency_key = $2')) return { rows: [] };
          if (text.includes('WHERE run.id = $1')) {
            return {
-              rows: [{
-                projectId: 'project-1',
-                taskId: 'task-1',
-                taskRevision: TASK_REVISION,
-                taskName: 'Task 1',
-                taskSnapshotRef: TASK_REVISION,
-                parentRunId: null,
-                triggerType: 'task_start',
-                executionOwner: 'runtime',
-                inputRef: null,
-                priority: 1,
-                runStatus: 'failed',
-                runVersion: 7,
-                attemptExecutorType: 'remote_worker',
-              }],
+              rows: [
+                {
+                  projectId: 'project-1',
+                  taskId: 'task-1',
+                  taskRevision: TASK_REVISION,
+                  taskName: 'Task 1',
+                  taskSnapshotRef: TASK_REVISION,
+                  parentRunId: null,
+                  triggerType: 'task_start',
+                  executionOwner: 'runtime',
+                  inputRef: null,
+                  priority: 1,
+                  runStatus: 'failed',
+                  runVersion: 7,
+                  attemptExecutorType: 'remote_worker',
+                },
+              ],
            };
          }
          if (text.includes('FROM "ql3"."task_definitions"')) {
            return { rows: [{ enabled: true }] };
          }
          if (text.includes('task_execution_revisions')) {
-            return { rows: [{ sourceContentDigest: SOURCE_DIGEST, contentDigest: EXECUTION_DIGEST }] };
+            return {
+              rows: [
+                {
+                  sourceContentDigest: SOURCE_DIGEST,
+                  contentDigest: EXECUTION_DIGEST,
+                },
+              ],
+            };
          }
-          if (text.startsWith('SELECT') && text.includes("trigger_type = 'run_manual_retry'")) {
+          if (
+            text.startsWith('SELECT') &&
+            text.includes("trigger_type = 'run_manual_retry'")
+          ) {
            return { rows: [] };
          }
          if (text.startsWith('INSERT INTO')) return { rows: [], rowCount: 1 };
@@ -143,7 +193,9 @@ test('authorizes run.retry and keeps all generated aggregate identities server-s
  assert.equal(result.status, 'accepted');
  assert.equal(result.runId, GENERATED[0]);
  assert.equal(result.attemptId, GENERATED[1]);
-  const runInsert = calls.find(({ sql }) => sql.startsWith('INSERT INTO "ql3"."runs"'));
+  const runInsert = calls.find(({ sql }) =>
+    sql.startsWith('INSERT INTO "ql3"."runs"'),
+  );
  assert.equal(runInsert.params[0], GENERATED[0]);
  assert.equal(runInsert.params.includes(GENERATED[2]), false);
  assert.equal(
@@ -154,11 +206,49 @@ test('authorizes run.retry and keeps all generated aggregate identities server-s

 test('denied policy writes only the caller-supplied failure audit', async () => {
  const { calls, service } = fixture('viewer');
-  await assert.rejects(service.retry(request()), ClusterRunManagementAuthorizationError);
+  await assert.rejects(
+    service.retry(request()),
+    ClusterRunManagementAuthorizationError,
+  );
  const audits = calls.filter(({ sql }) =>
    sql.startsWith('INSERT INTO "ql3"."security_audit_events"'),
  );
  assert.equal(audits.length, 1);
  assert.equal(audits[0].params[0], request().failureAuditEventId);
-  assert.equal(calls.some(({ scope }) => scope === 'client'), false);
+  assert.equal(
+    calls.some(({ scope }) => scope === 'client'),
+    false,
+  );
+});
+
+test('authorizes run.stop and commits intent plus allowed audit together', async () => {
+  const { calls, service } = fixture();
+  const stopRequest = {
+    projectId: 'project-1',
+    runId: 'run-1',
+    mutationId: '019f9500-0000-4000-8000-000000000021',
+    requestId: 'request-stop-1',
+    auditEventId: '019f9500-0000-4000-8000-000000000022',
+    failureAuditEventId: '019f9500-0000-4000-8000-000000000023',
+    principal: request().principal,
+  };
+  const result = await service.stop(stopRequest);
+  assert.equal(result.status, 'accepted');
+  assert.equal(result.cancelRequestedAtMs, NOW);
+  const event = calls.find(
+    ({ sql }) =>
+      sql.startsWith('INSERT INTO "ql3"."run_events"') &&
+      sql.includes('run.cancel_requested'),
+  );
+  assert.equal(event.params[0], GENERATED[0]);
+  const allowedAudit = calls.find(
+    ({ sql }) =>
+      sql.startsWith('INSERT INTO "ql3"."security_audit_events"') &&
+      sql.includes("'run.stop'"),
+  );
+  assert.equal(allowedAudit.params[0], stopRequest.auditEventId);
+  assert.ok(
+    calls.findIndex(({ sql }) => sql.includes("'run.stop'")) <
+      calls.findIndex(({ sql }) => sql === 'COMMIT'),
+  );
 });
@@ -31,6 +31,22 @@ const command = normalizeClusterRunManagementCommand({
  },
 });

+const stopCommand = normalizeClusterRunManagementCommand({
+  schemaVersion: 1,
+  operation: 'run.stop',
+  request: {
+    projectId: 'project-1',
+    runId: 'run-1',
+    requestId: 'request-stop-1',
+    auditEventId: '019f9400-0000-4000-8000-000000000021',
+    failureAuditEventId: '019f9400-0000-4000-8000-000000000022',
+    body: {
+      schema: 'qinglong/run-cancellation@v1',
+      mutationId: '019f9400-0000-4000-8000-000000000023',
+    },
+  },
+});
+
 function response(overrides = {}) {
  return {
    schemaVersion: 1,
@@ -59,7 +75,10 @@ function response(overrides = {}) {
 }

 test('validates one low-sensitive retry response against the request fence', () => {
-  assert.deepEqual(validateClusterRunManagementClientResult(response(), command), response());
+  assert.deepEqual(
+    validateClusterRunManagementClientResult(response(), command),
+    response(),
+  );
 });

 test('rejects response target, execution placement and shape drift', () => {
@@ -75,3 +94,35 @@ test('rejects response target, execution placement and shape drift', () => {
    );
  }
 });
+
+test('validates one low-sensitive stop response against the request target', () => {
+  const value = {
+    schemaVersion: 1,
+    operation: 'run.stop',
+    stop: {
+      schema: 'qinglong/run-cancellation@v1',
+      status: 'accepted',
+      projectId: 'project-1',
+      runId: 'run-1',
+      runStatus: 'running',
+      runVersion: 5,
+      eventSequence: 7,
+      cancelRequestedAtMs: 1_000_000,
+      cancelReason: 'user',
+    },
+  };
+  assert.deepEqual(
+    validateClusterRunManagementClientResult(value, stopCommand),
+    value,
+  );
+  for (const drift of [
+    { ...value, stop: { ...value.stop, projectId: 'project-2' } },
+    { ...value, stop: { ...value.stop, runId: 'run-2' } },
+    { ...value, operation: 'run.retry' },
+  ]) {
+    assert.throws(
+      () => validateClusterRunManagementClientResult(drift, stopCommand),
+      ClusterPluginPackageManagementClientRequestError,
+    );
+  }
+});
@@ -65,6 +65,38 @@ function retryResult() {
  };
 }

+function stopCommand(overrides = {}) {
+  return {
+    schemaVersion: 1,
+    operation: 'run.stop',
+    request: {
+      projectId: 'project-1',
+      runId: 'run-1',
+      requestId: 'request-stop-1',
+      auditEventId: '019f9300-0000-4000-8000-000000000021',
+      failureAuditEventId: '019f9300-0000-4000-8000-000000000022',
+      body: {
+        schema: 'qinglong/run-cancellation@v1',
+        mutationId: '019f9300-0000-4000-8000-000000000023',
+      },
+      ...overrides,
+    },
+  };
+}
+
+function stopResult() {
+  return {
+    status: 'accepted',
+    projectId: 'project-1',
+    runId: 'run-1',
+    runStatus: 'running',
+    runVersion: 5,
+    eventSequence: 7,
+    cancelRequestedAtMs: NOW,
+    cancelReason: 'user',
+  };
+}
+
 test('routes one exact strong User retry and emits the shared response', async () => {
  const calls = [];
  const transport = createClusterRunManagementTransport({
@@ -74,6 +106,9 @@ test('routes one exact strong User retry and emits the shared response', async (
        calls.push(request);
        return retryResult();
      },
+      async stop() {
+        return stopResult();
+      },
    },
  });
  const result = await transport.execute(command(), {
@@ -92,6 +127,36 @@ test('routes one exact strong User retry and emits the shared response', async (
  });
 });

+test('routes one exact strong User stop and emits the shared response', async () => {
+  const calls = [];
+  const transport = createClusterRunManagementTransport({
+    now: () => NOW,
+    service: {
+      async retry() {
+        return retryResult();
+      },
+      async stop(request) {
+        calls.push(request);
+        return stopResult();
+      },
+    },
+  });
+  const result = await transport.execute(stopCommand(), {
+    authenticate: async () => principal({ assurance: 'hardware' }),
+  });
+  assert.equal(calls.length, 1);
+  assert.equal(calls[0].runId, 'run-1');
+  assert.equal(calls[0].mutationId, stopCommand().request.body.mutationId);
+  assert.deepEqual(result, {
+    schemaVersion: 1,
+    operation: 'run.stop',
+    stop: {
+      schema: 'qinglong/run-cancellation@v1',
+      ...stopResult(),
+    },
+  });
+});
+
 test('rejects weak or non-User identity before service authority', async () => {
  let called = false;
  const transport = createClusterRunManagementTransport({
@@ -101,6 +166,9 @@ test('rejects weak or non-User identity before service authority', async () => {
        called = true;
        return retryResult();
      },
+      async stop() {
+        return stopResult();
+      },
    },
  });
  await assert.rejects(
@@ -121,7 +189,11 @@ test('rejects weak or non-User identity before service authority', async () => {

 test('rejects widened commands and ambiguous audit identity', () => {
  assert.throws(
-    () => normalizeClusterRunManagementCommand({ ...command(), principal: principal() }),
+    () =>
+      normalizeClusterRunManagementCommand({
+        ...command(),
+        principal: principal(),
+      }),
    ClusterRunManagementTransportRequestError,
  );
  assert.throws(
@@ -134,7 +206,18 @@ test('rejects widened commands and ambiguous audit identity', () => {
  assert.throws(
    () =>
      normalizeClusterRunManagementCommand(
-        command({ body: { ...command().request.body, expectedRunStatus: 'lost' } }),
+        command({
+          body: { ...command().request.body, expectedRunStatus: 'lost' },
+        }),
+      ),
+    ClusterRunManagementTransportRequestError,
+  );
+  assert.throws(
+    () =>
+      normalizeClusterRunManagementCommand(
+        stopCommand({
+          body: { ...stopCommand().request.body, mutationId: 'weak' },
+        }),
      ),
    ClusterRunManagementTransportRequestError,
  );