mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 10:32:40 +08:00
feat(ql3): gate remote cancellation delivery
This commit is contained in:
@@ -175,6 +175,11 @@
|
||||
"require": "./dist/remote-execution/remoteWorkerLeaseControlService.js",
|
||||
"default": "./dist/remote-execution/remoteWorkerLeaseControlService.js"
|
||||
},
|
||||
"./cancellation-dispatch-control": {
|
||||
"types": "./dist/remote-execution/remoteWorkerCancellationDispatchControl.d.ts",
|
||||
"require": "./dist/remote-execution/remoteWorkerCancellationDispatchControl.js",
|
||||
"default": "./dist/remote-execution/remoteWorkerCancellationDispatchControl.js"
|
||||
},
|
||||
"./workflow-scheduler": {
|
||||
"types": "./dist/scheduling/workflowScheduler.d.ts",
|
||||
"require": "./dist/scheduling/workflowScheduler.js",
|
||||
|
||||
@@ -903,6 +903,9 @@ export async function bootstrapClusterControlRuntime(
|
||||
workerRuntime: createClusterWorkerRuntimePort(
|
||||
database.pool,
|
||||
options.workerRuntime,
|
||||
{
|
||||
cancellationDispatchOwnerId: recoveryRuntime.ownerId,
|
||||
},
|
||||
),
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -75,6 +75,7 @@ import {
|
||||
createClusterControlCopilotFailureDiagnosisCancellationRoute,
|
||||
type ClusterCopilotFailureDiagnosisCancellationCapability,
|
||||
} from '../copilot/failure-diagnosis/failureDiagnosisCancellationRoute';
|
||||
import type { ClusterRemoteWorkerCancellationDispatchObservation } from '../remote-execution/remoteWorkerCancellationDispatchControl';
|
||||
|
||||
export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
|
||||
'task.get',
|
||||
@@ -150,6 +151,12 @@ export interface ProductionClusterWorkerIngressOptions {
|
||||
readonly artifactStore: ClusterRemoteWorkerArtifactStore;
|
||||
readonly secretProvider?: RemoteWorkerSecretValueProvider;
|
||||
readonly onDiagnostic?: (error: unknown) => void | Promise<void>;
|
||||
readonly onCancellationDispatch?: (
|
||||
observation: ClusterRemoteWorkerCancellationDispatchObservation,
|
||||
) => void | Promise<void>;
|
||||
readonly onCancellationDispatchDiagnostic?: (
|
||||
error: unknown,
|
||||
) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface ProductionClusterControlApplicationOptions
|
||||
@@ -439,6 +446,28 @@ export function startProductionClusterControlApplication(
|
||||
...(workerIngress.secretProvider === undefined
|
||||
? {}
|
||||
: { secretProvider: workerIngress.secretProvider }),
|
||||
...(
|
||||
workerIngress.onCancellationDispatch === undefined &&
|
||||
workerIngress.onCancellationDispatchDiagnostic === undefined
|
||||
? {}
|
||||
: {
|
||||
cancellationDispatch: {
|
||||
...(workerIngress.onCancellationDispatch === undefined
|
||||
? {}
|
||||
: {
|
||||
onObservation:
|
||||
workerIngress.onCancellationDispatch,
|
||||
}),
|
||||
...(workerIngress.onCancellationDispatchDiagnostic ===
|
||||
undefined
|
||||
? {}
|
||||
: {
|
||||
onDiagnostic:
|
||||
workerIngress.onCancellationDispatchDiagnostic,
|
||||
}),
|
||||
},
|
||||
}
|
||||
),
|
||||
},
|
||||
}),
|
||||
...database,
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
} from '../worker-ingress/workerIngressConfig';
|
||||
import type { ClusterWorkerArtifactBinding } from '../artifact/workerArtifactBinding';
|
||||
import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/remote-secret-delivery';
|
||||
import type { ClusterRemoteWorkerCancellationDispatchObservation } from '../remote-execution/remoteWorkerCancellationDispatchControl';
|
||||
|
||||
export type ClusterControlProcessSignal = 'SIGINT' | 'SIGTERM';
|
||||
|
||||
@@ -34,10 +35,12 @@ export interface ClusterControlProcessEvent {
|
||||
readonly stopResult?: ClusterControlStopResult;
|
||||
readonly address?: Readonly<{ host: string; port: number }>;
|
||||
readonly activation?: ClusterControlActivationAudit;
|
||||
readonly cancellationDispatch?: ClusterRemoteWorkerCancellationDispatchObservation;
|
||||
readonly diagnostic?: Readonly<{
|
||||
scope:
|
||||
| 'scheduler'
|
||||
| 'cancellation-convergence'
|
||||
| 'cancellation-dispatch'
|
||||
| 'log-retention'
|
||||
| 'database'
|
||||
| 'worker-ingress';
|
||||
@@ -333,6 +336,32 @@ export async function runProductionClusterControlProcess(
|
||||
),
|
||||
).catch(() => undefined);
|
||||
},
|
||||
onCancellationDispatch(observation) {
|
||||
void Promise.resolve(
|
||||
options.emit(
|
||||
event(replicaId, {
|
||||
level:
|
||||
observation.status === 'blocked' ? 'error' : 'info',
|
||||
event: 'cancellation_dispatch',
|
||||
cancellationDispatch: observation,
|
||||
}),
|
||||
),
|
||||
).catch(() => undefined);
|
||||
},
|
||||
onCancellationDispatchDiagnostic(error: unknown) {
|
||||
void Promise.resolve(
|
||||
options.emit(
|
||||
event(replicaId, {
|
||||
level: 'error',
|
||||
event: 'runtime_diagnostic',
|
||||
diagnostic: diagnosticFact(
|
||||
'cancellation-dispatch',
|
||||
error,
|
||||
),
|
||||
}),
|
||||
),
|
||||
).catch(() => undefined);
|
||||
},
|
||||
},
|
||||
}),
|
||||
audit(record) {
|
||||
|
||||
+238
@@ -0,0 +1,238 @@
|
||||
// Remote execution owns cancellation delivery to the Worker that already holds
|
||||
// the exact RunDispatchLease. This layer adds no timer, queue, or connection.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
MAX_CANCELLATION_DISPATCH_LEASE_MS,
|
||||
type CancellationDispatchRepository,
|
||||
} from '@qinglong/runtime-core/cancellation-dispatch';
|
||||
import {
|
||||
RemoteWorkerLeaseControlUnavailableError,
|
||||
type RemoteWorkerLeaseControlCommand,
|
||||
type RemoteWorkerLeaseControlResult,
|
||||
} from '@qinglong/runtime-core/remote-worker-lease-control';
|
||||
|
||||
export type ClusterRemoteWorkerCancellationDispatchObservation = Readonly<{
|
||||
readonly status:
|
||||
| 'dispatched'
|
||||
| 'already_dispatched'
|
||||
| 'untracked'
|
||||
| 'deferred'
|
||||
| 'blocked';
|
||||
}>;
|
||||
|
||||
export interface ClusterRemoteWorkerCancellationDispatchControlOptions {
|
||||
readonly ownerId: string;
|
||||
readonly leaseDurationMs?: number;
|
||||
readonly createLeaseToken?: () => string;
|
||||
readonly createEventId?: () => string;
|
||||
readonly onObservation?: (
|
||||
observation: ClusterRemoteWorkerCancellationDispatchObservation,
|
||||
) => void | Promise<void>;
|
||||
readonly onDiagnostic?: (error: unknown) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export class ClusterRemoteWorkerCancellationDispatchError extends Error {
|
||||
readonly code = 'CLUSTER_REMOTE_CANCELLATION_DISPATCH_FAILED';
|
||||
|
||||
constructor(
|
||||
readonly reason:
|
||||
| 'invalid_configuration'
|
||||
| 'claim_failed'
|
||||
| 'result_failed'
|
||||
| 'delivery_deferred'
|
||||
| 'delivery_blocked',
|
||||
options?: ErrorOptions,
|
||||
) {
|
||||
super(`Cluster Remote Worker cancellation dispatch failed: ${reason}`, options);
|
||||
this.name = 'ClusterRemoteWorkerCancellationDispatchError';
|
||||
}
|
||||
}
|
||||
|
||||
const OWNER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u;
|
||||
const OPTION_KEYS = new Set([
|
||||
'createEventId',
|
||||
'createLeaseToken',
|
||||
'leaseDurationMs',
|
||||
'onDiagnostic',
|
||||
'onObservation',
|
||||
'ownerId',
|
||||
]);
|
||||
|
||||
function invalidConfiguration(): never {
|
||||
throw new ClusterRemoteWorkerCancellationDispatchError(
|
||||
'invalid_configuration',
|
||||
);
|
||||
}
|
||||
|
||||
function capability(factory: () => string, name: string): string {
|
||||
let value: unknown;
|
||||
try {
|
||||
value = factory();
|
||||
} catch (error) {
|
||||
throw new ClusterRemoteWorkerCancellationDispatchError(
|
||||
'claim_failed',
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
const maximum = name === 'eventId' ? 36 : 128;
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
value.length < 1 ||
|
||||
value.length > maximum ||
|
||||
/[\u0000-\u001f\u007f]/u.test(value)
|
||||
) {
|
||||
throw new ClusterRemoteWorkerCancellationDispatchError('claim_failed');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts the existing caller-driven Worker lease-control tick into the only
|
||||
* Cluster cancellation delivery path. A stop response is released only after
|
||||
* its durable CancellationDispatch is settled, while Workflow-scoped timeout
|
||||
* stops remain valid without forging a Run cancellation record.
|
||||
*/
|
||||
export class ClusterRemoteWorkerCancellationDispatchControl {
|
||||
private readonly ownerId: string;
|
||||
private readonly leaseDurationMs: number;
|
||||
private readonly createLeaseToken: () => string;
|
||||
private readonly createEventId: () => string;
|
||||
private readonly onObservation?: ClusterRemoteWorkerCancellationDispatchControlOptions['onObservation'];
|
||||
private readonly onDiagnostic?: ClusterRemoteWorkerCancellationDispatchControlOptions['onDiagnostic'];
|
||||
|
||||
constructor(
|
||||
private readonly leaseControl: Readonly<{
|
||||
control(
|
||||
command: RemoteWorkerLeaseControlCommand,
|
||||
): Promise<Readonly<RemoteWorkerLeaseControlResult>>;
|
||||
}>,
|
||||
private readonly dispatches: CancellationDispatchRepository,
|
||||
options: ClusterRemoteWorkerCancellationDispatchControlOptions,
|
||||
) {
|
||||
if (
|
||||
typeof leaseControl?.control !== 'function' ||
|
||||
typeof dispatches?.claim !== 'function' ||
|
||||
typeof dispatches?.recordResult !== 'function' ||
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).some((key) => !OPTION_KEYS.has(key)) ||
|
||||
!OWNER_PATTERN.test(options.ownerId ?? '') ||
|
||||
(options.leaseDurationMs !== undefined &&
|
||||
(!Number.isSafeInteger(options.leaseDurationMs) ||
|
||||
options.leaseDurationMs < 1 ||
|
||||
options.leaseDurationMs > MAX_CANCELLATION_DISPATCH_LEASE_MS)) ||
|
||||
(options.createLeaseToken !== undefined &&
|
||||
typeof options.createLeaseToken !== 'function') ||
|
||||
(options.createEventId !== undefined &&
|
||||
typeof options.createEventId !== 'function') ||
|
||||
(options.onObservation !== undefined &&
|
||||
typeof options.onObservation !== 'function') ||
|
||||
(options.onDiagnostic !== undefined &&
|
||||
typeof options.onDiagnostic !== 'function')
|
||||
) {
|
||||
invalidConfiguration();
|
||||
}
|
||||
this.ownerId = options.ownerId;
|
||||
this.leaseDurationMs = options.leaseDurationMs ?? 30_000;
|
||||
this.createLeaseToken = options.createLeaseToken ?? randomUUID;
|
||||
this.createEventId = options.createEventId ?? randomUUID;
|
||||
this.onObservation = options.onObservation;
|
||||
this.onDiagnostic = options.onDiagnostic;
|
||||
}
|
||||
|
||||
async control(
|
||||
command: RemoteWorkerLeaseControlCommand,
|
||||
): Promise<Readonly<RemoteWorkerLeaseControlResult>> {
|
||||
const result = await this.leaseControl.control(command);
|
||||
if (result.status !== 'stop_requested') return result;
|
||||
|
||||
let claim: Awaited<ReturnType<CancellationDispatchRepository['claim']>>;
|
||||
try {
|
||||
claim = await this.dispatches.claim({
|
||||
runId: result.runId,
|
||||
attemptId: result.attemptId,
|
||||
requestedAtMs: result.stop!.requestedAtMs,
|
||||
owner: this.ownerId,
|
||||
leaseToken: capability(this.createLeaseToken, 'leaseToken'),
|
||||
leaseDurationMs: this.leaseDurationMs,
|
||||
});
|
||||
} catch (error) {
|
||||
return this.unavailable('claim_failed', error);
|
||||
}
|
||||
|
||||
if (claim.status === 'not_eligible') {
|
||||
// Workflow Task timeout is represented by its own event and does not set
|
||||
// Run.cancel_requested_at_ms. The already-fenced Worker stop must remain
|
||||
// deliverable without inventing a Run-level cancellation fact.
|
||||
this.observe('untracked');
|
||||
return result;
|
||||
}
|
||||
if (claim.status === 'dispatched') {
|
||||
this.observe('already_dispatched');
|
||||
return result;
|
||||
}
|
||||
if (claim.status === 'leased' || claim.status === 'not_due') {
|
||||
this.observe('deferred');
|
||||
return this.unavailable('delivery_deferred');
|
||||
}
|
||||
if (claim.status === 'blocked') {
|
||||
this.observe('blocked');
|
||||
return this.unavailable('delivery_blocked');
|
||||
}
|
||||
if (claim.status !== 'claimed') {
|
||||
return this.unavailable('claim_failed');
|
||||
}
|
||||
|
||||
try {
|
||||
const settled = await this.dispatches.recordResult({
|
||||
runId: result.runId,
|
||||
attemptId: result.attemptId,
|
||||
owner: this.ownerId,
|
||||
leaseToken: claim.leaseToken,
|
||||
expectedVersion: claim.dispatch.version,
|
||||
result: 'termination_requested',
|
||||
eventId: capability(this.createEventId, 'eventId'),
|
||||
});
|
||||
if (
|
||||
settled.dispatch.status !== 'dispatched' ||
|
||||
settled.dispatch.lastResult !== 'termination_requested' ||
|
||||
settled.event.type !== 'run.cancel_dispatched'
|
||||
) {
|
||||
return this.unavailable('result_failed');
|
||||
}
|
||||
} catch (error) {
|
||||
return this.unavailable('result_failed', error);
|
||||
}
|
||||
this.observe('dispatched');
|
||||
return result;
|
||||
}
|
||||
|
||||
private unavailable(
|
||||
reason: Exclude<
|
||||
ClusterRemoteWorkerCancellationDispatchError['reason'],
|
||||
'invalid_configuration'
|
||||
>,
|
||||
cause?: unknown,
|
||||
): never {
|
||||
const error = new ClusterRemoteWorkerCancellationDispatchError(reason, {
|
||||
...(cause === undefined ? {} : { cause }),
|
||||
});
|
||||
this.diagnostic(error);
|
||||
throw new RemoteWorkerLeaseControlUnavailableError({ cause: error });
|
||||
}
|
||||
|
||||
private observe(
|
||||
status: ClusterRemoteWorkerCancellationDispatchObservation['status'],
|
||||
): void {
|
||||
if (!this.onObservation) return;
|
||||
void Promise.resolve(
|
||||
this.onObservation(Object.freeze({ status })),
|
||||
).catch(() => undefined);
|
||||
}
|
||||
|
||||
private diagnostic(error: unknown): void {
|
||||
if (!this.onDiagnostic) return;
|
||||
void Promise.resolve(this.onDiagnostic(error)).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/rem
|
||||
import type { RunAttemptLogRangeReader } from '@qinglong/runtime-core/run-attempt-log-read';
|
||||
import {
|
||||
PostgresClusterDispatchSource,
|
||||
PostgresCancellationDispatchRepository,
|
||||
PostgresRemoteRunActivationRepository,
|
||||
PostgresRemoteWorkerCompletionRepository,
|
||||
PostgresRemoteWorkerLeaseControlRepository,
|
||||
@@ -21,11 +22,23 @@ import {
|
||||
type ClusterRemoteWorkerArtifactStore,
|
||||
} from './remoteWorkerCompletionService';
|
||||
import { ClusterRemoteWorkerLeaseControlService } from './remoteWorkerLeaseControlService';
|
||||
import {
|
||||
ClusterRemoteWorkerCancellationDispatchControl,
|
||||
type ClusterRemoteWorkerCancellationDispatchControlOptions,
|
||||
} from './remoteWorkerCancellationDispatchControl';
|
||||
import type { WorkerIngressPipelineOptions } from '../worker-ingress/workerIngressPipeline';
|
||||
|
||||
export interface ClusterWorkerRuntimeDependencies {
|
||||
readonly artifactStore: ClusterRemoteWorkerArtifactStore;
|
||||
readonly secretProvider?: RemoteWorkerSecretValueProvider;
|
||||
readonly cancellationDispatch?: Readonly<{
|
||||
readonly onObservation?: ClusterRemoteWorkerCancellationDispatchControlOptions['onObservation'];
|
||||
readonly onDiagnostic?: ClusterRemoteWorkerCancellationDispatchControlOptions['onDiagnostic'];
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface ClusterWorkerRuntimePortOptions {
|
||||
readonly cancellationDispatchOwnerId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -48,6 +61,7 @@ export interface ClusterWorkerRuntimePort {
|
||||
export function createClusterWorkerRuntimePort(
|
||||
pool: PostgresPool,
|
||||
dependencies: ClusterWorkerRuntimeDependencies,
|
||||
options: ClusterWorkerRuntimePortOptions,
|
||||
): Readonly<ClusterWorkerRuntimePort> {
|
||||
if (!pool || typeof pool.query !== 'function') {
|
||||
throw new TypeError('Cluster Worker runtime Pool is invalid');
|
||||
@@ -59,6 +73,15 @@ export function createClusterWorkerRuntimePort(
|
||||
) {
|
||||
throw new TypeError('Cluster Worker runtime dependencies are invalid');
|
||||
}
|
||||
if (
|
||||
!options ||
|
||||
typeof options !== 'object' ||
|
||||
Array.isArray(options) ||
|
||||
Object.keys(options).length !== 1 ||
|
||||
typeof options.cancellationDispatchOwnerId !== 'string'
|
||||
) {
|
||||
throw new TypeError('Cluster Worker runtime options are invalid');
|
||||
}
|
||||
|
||||
const workerSessions = new PostgresWorkerSessionRepository(pool);
|
||||
const completionRepository = new PostgresRemoteWorkerCompletionRepository(
|
||||
@@ -92,8 +115,25 @@ export function createClusterWorkerRuntimePort(
|
||||
completionRepository,
|
||||
dependencies.artifactStore,
|
||||
),
|
||||
leaseControl: new ClusterRemoteWorkerLeaseControlService(
|
||||
new PostgresRemoteWorkerLeaseControlRepository(pool),
|
||||
leaseControl: new ClusterRemoteWorkerCancellationDispatchControl(
|
||||
new ClusterRemoteWorkerLeaseControlService(
|
||||
new PostgresRemoteWorkerLeaseControlRepository(pool),
|
||||
),
|
||||
new PostgresCancellationDispatchRepository(pool),
|
||||
{
|
||||
ownerId: options.cancellationDispatchOwnerId,
|
||||
...(dependencies.cancellationDispatch?.onObservation === undefined
|
||||
? {}
|
||||
: {
|
||||
onObservation:
|
||||
dependencies.cancellationDispatch.onObservation,
|
||||
}),
|
||||
...(dependencies.cancellationDispatch?.onDiagnostic === undefined
|
||||
? {}
|
||||
: {
|
||||
onDiagnostic: dependencies.cancellationDispatch.onDiagnostic,
|
||||
}),
|
||||
},
|
||||
),
|
||||
...(readLogRange === undefined
|
||||
? {}
|
||||
|
||||
@@ -206,6 +206,12 @@ test('starts the optional Worker listener and closes its lazy Artifact binding',
|
||||
code: 'S3Unavailable',
|
||||
}),
|
||||
);
|
||||
options.workerIngress.onCancellationDispatch({ status: 'dispatched' });
|
||||
options.workerIngress.onCancellationDispatchDiagnostic(
|
||||
Object.assign(new Error('must-not-be-logged'), {
|
||||
code: 'CANCEL_DISPATCH_UNAVAILABLE',
|
||||
}),
|
||||
);
|
||||
return {
|
||||
status: 'active',
|
||||
address: { host: '0.0.0.0', port: 5800 },
|
||||
@@ -248,6 +254,25 @@ test('starts the optional Worker listener and closes its lazy Artifact binding',
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
facts.some(
|
||||
(fact) =>
|
||||
fact.event === 'cancellation_dispatch' &&
|
||||
fact.level === 'info' &&
|
||||
fact.cancellationDispatch.status === 'dispatched',
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
facts.some(
|
||||
(fact) =>
|
||||
fact.event === 'runtime_diagnostic' &&
|
||||
fact.diagnostic.scope === 'cancellation-dispatch' &&
|
||||
fact.diagnostic.code === 'CANCEL_DISPATCH_UNAVAILABLE' &&
|
||||
JSON.stringify(fact).includes('must-not-be-logged') === false,
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('creates the configured mounted Secret provider before Worker activation', async () => {
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
ClusterRemoteWorkerCancellationDispatchControl,
|
||||
ClusterRemoteWorkerCancellationDispatchError,
|
||||
} = require('@qinglong/cluster-control/cancellation-dispatch-control');
|
||||
const {
|
||||
RemoteWorkerLeaseControlUnavailableError,
|
||||
} = require('@qinglong/runtime-core/remote-worker-lease-control');
|
||||
|
||||
const COMMAND = Object.freeze({
|
||||
workerId: 'worker-1',
|
||||
workerSessionId: '018f0000-0000-7000-8000-000000000001',
|
||||
workerGeneration: 2,
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
leaseGeneration: 3,
|
||||
leaseToken: 'worker_generated_lease_capability_0000000000000001',
|
||||
expectedLeaseVersion: 4,
|
||||
});
|
||||
const STOP = Object.freeze({
|
||||
status: 'stop_requested',
|
||||
projectId: 'project-1',
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
offerId: 'offer-1',
|
||||
leaseGeneration: 3,
|
||||
leaseVersion: 5,
|
||||
renewedAtMs: 10_000,
|
||||
expiresAtMs: 40_000,
|
||||
stop: Object.freeze({ reason: 'user', requestedAtMs: 9_000 }),
|
||||
});
|
||||
|
||||
function leasedDispatch() {
|
||||
return Object.freeze({
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
status: 'leased',
|
||||
version: 1,
|
||||
dispatchCount: 1,
|
||||
leaseOwner: 'replica-1',
|
||||
leaseTokenDigest: 'a'.repeat(64),
|
||||
leaseExpiresAtMs: 40_000,
|
||||
createdAtMs: 10_000,
|
||||
updatedAtMs: 10_000,
|
||||
});
|
||||
}
|
||||
|
||||
function service(dispatches, overrides = {}) {
|
||||
return new ClusterRemoteWorkerCancellationDispatchControl(
|
||||
{
|
||||
async control() {
|
||||
return overrides.result ?? STOP;
|
||||
},
|
||||
},
|
||||
dispatches,
|
||||
{
|
||||
ownerId: 'replica-1',
|
||||
leaseDurationMs: 30_000,
|
||||
createLeaseToken: () => 'cancel-token-1',
|
||||
createEventId: () => '018f0000-0000-7000-8000-000000000011',
|
||||
...(overrides.onObservation === undefined
|
||||
? {}
|
||||
: { onObservation: overrides.onObservation }),
|
||||
...(overrides.onDiagnostic === undefined
|
||||
? {}
|
||||
: { onDiagnostic: overrides.onDiagnostic }),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test('bypasses dispatch storage when lease control only renews', async () => {
|
||||
let calls = 0;
|
||||
const renewed = Object.freeze({
|
||||
...STOP,
|
||||
status: 'renewed',
|
||||
stop: undefined,
|
||||
});
|
||||
const control = service(
|
||||
{
|
||||
async claim() {
|
||||
calls += 1;
|
||||
throw new Error('must not claim');
|
||||
},
|
||||
async recordResult() {
|
||||
calls += 1;
|
||||
throw new Error('must not record');
|
||||
},
|
||||
},
|
||||
{ result: renewed },
|
||||
);
|
||||
assert.equal(await control.control(COMMAND), renewed);
|
||||
assert.equal(calls, 0);
|
||||
});
|
||||
|
||||
test('settles one durable dispatch before releasing a Worker stop', async () => {
|
||||
const observed = [];
|
||||
let claimCommand;
|
||||
let resultCommand;
|
||||
const claimed = leasedDispatch();
|
||||
const control = service(
|
||||
{
|
||||
async claim(value) {
|
||||
claimCommand = value;
|
||||
return { status: 'claimed', dispatch: claimed, leaseToken: 'cancel-token-1' };
|
||||
},
|
||||
async recordResult(value) {
|
||||
resultCommand = value;
|
||||
return {
|
||||
dispatch: {
|
||||
...claimed,
|
||||
status: 'dispatched',
|
||||
version: 2,
|
||||
leaseOwner: undefined,
|
||||
leaseTokenDigest: undefined,
|
||||
leaseExpiresAtMs: undefined,
|
||||
lastResult: 'termination_requested',
|
||||
lastDispatchedAtMs: 10_001,
|
||||
updatedAtMs: 10_001,
|
||||
},
|
||||
event: { type: 'run.cancel_dispatched' },
|
||||
};
|
||||
},
|
||||
},
|
||||
{ onObservation: (value) => observed.push(value) },
|
||||
);
|
||||
|
||||
assert.equal(await control.control(COMMAND), STOP);
|
||||
assert.deepEqual(claimCommand, {
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
requestedAtMs: 9_000,
|
||||
owner: 'replica-1',
|
||||
leaseToken: 'cancel-token-1',
|
||||
leaseDurationMs: 30_000,
|
||||
});
|
||||
assert.deepEqual(resultCommand, {
|
||||
runId: 'run-1',
|
||||
attemptId: 'attempt-1',
|
||||
owner: 'replica-1',
|
||||
leaseToken: 'cancel-token-1',
|
||||
expectedVersion: 1,
|
||||
result: 'termination_requested',
|
||||
eventId: '018f0000-0000-7000-8000-000000000011',
|
||||
});
|
||||
assert.deepEqual(observed, [{ status: 'dispatched' }]);
|
||||
});
|
||||
|
||||
test('releases an already-dispatched stop without a second result event', async () => {
|
||||
let results = 0;
|
||||
const observed = [];
|
||||
const control = service(
|
||||
{
|
||||
async claim() {
|
||||
return {
|
||||
status: 'dispatched',
|
||||
dispatch: { ...leasedDispatch(), status: 'dispatched' },
|
||||
};
|
||||
},
|
||||
async recordResult() {
|
||||
results += 1;
|
||||
throw new Error('must not record');
|
||||
},
|
||||
},
|
||||
{ onObservation: (value) => observed.push(value) },
|
||||
);
|
||||
assert.equal(await control.control(COMMAND), STOP);
|
||||
assert.equal(results, 0);
|
||||
assert.deepEqual(observed, [{ status: 'already_dispatched' }]);
|
||||
});
|
||||
|
||||
test('keeps a foreign live dispatch from releasing a duplicate stop', async () => {
|
||||
const diagnostics = [];
|
||||
const observed = [];
|
||||
const control = service(
|
||||
{
|
||||
async claim() {
|
||||
return { status: 'leased', dispatch: leasedDispatch() };
|
||||
},
|
||||
async recordResult() {
|
||||
throw new Error('must not record');
|
||||
},
|
||||
},
|
||||
{
|
||||
onObservation: (value) => observed.push(value),
|
||||
onDiagnostic: (error) => diagnostics.push(error),
|
||||
},
|
||||
);
|
||||
await assert.rejects(
|
||||
control.control(COMMAND),
|
||||
(error) =>
|
||||
error instanceof RemoteWorkerLeaseControlUnavailableError &&
|
||||
error.cause instanceof ClusterRemoteWorkerCancellationDispatchError &&
|
||||
error.cause.reason === 'delivery_deferred',
|
||||
);
|
||||
assert.deepEqual(observed, [{ status: 'deferred' }]);
|
||||
assert.equal(diagnostics[0].reason, 'delivery_deferred');
|
||||
});
|
||||
|
||||
test('fails closed and reports a durable blocked dispatch', async () => {
|
||||
const diagnostics = [];
|
||||
const observed = [];
|
||||
const control = service(
|
||||
{
|
||||
async claim() {
|
||||
return { status: 'blocked', dispatch: leasedDispatch() };
|
||||
},
|
||||
async recordResult() {
|
||||
throw new Error('must not record');
|
||||
},
|
||||
},
|
||||
{
|
||||
onObservation: (value) => observed.push(value),
|
||||
onDiagnostic: (error) => diagnostics.push(error),
|
||||
},
|
||||
);
|
||||
await assert.rejects(
|
||||
control.control(COMMAND),
|
||||
(error) => error.cause?.reason === 'delivery_blocked',
|
||||
);
|
||||
assert.deepEqual(observed, [{ status: 'blocked' }]);
|
||||
assert.equal(diagnostics[0].code, 'CLUSTER_REMOTE_CANCELLATION_DISPATCH_FAILED');
|
||||
});
|
||||
|
||||
test('preserves Workflow-scoped timeout stops without forging Run cancellation', async () => {
|
||||
let results = 0;
|
||||
const observed = [];
|
||||
const control = service(
|
||||
{
|
||||
async claim() {
|
||||
return { status: 'not_eligible' };
|
||||
},
|
||||
async recordResult() {
|
||||
results += 1;
|
||||
},
|
||||
},
|
||||
{ onObservation: (value) => observed.push(value) },
|
||||
);
|
||||
assert.equal(await control.control(COMMAND), STOP);
|
||||
assert.equal(results, 0);
|
||||
assert.deepEqual(observed, [{ status: 'untracked' }]);
|
||||
});
|
||||
|
||||
test('does not release a stop when durable result settlement fails', async () => {
|
||||
const diagnostics = [];
|
||||
const control = service(
|
||||
{
|
||||
async claim() {
|
||||
return {
|
||||
status: 'claimed',
|
||||
dispatch: leasedDispatch(),
|
||||
leaseToken: 'cancel-token-1',
|
||||
};
|
||||
},
|
||||
async recordResult() {
|
||||
throw new Error('database unavailable');
|
||||
},
|
||||
},
|
||||
{ onDiagnostic: (error) => diagnostics.push(error) },
|
||||
);
|
||||
await assert.rejects(
|
||||
control.control(COMMAND),
|
||||
(error) => error.cause?.reason === 'result_failed',
|
||||
);
|
||||
assert.equal(diagnostics[0].reason, 'result_failed');
|
||||
});
|
||||
|
||||
test('rejects widened or unbounded production configuration', () => {
|
||||
const repository = { claim() {}, recordResult() {} };
|
||||
const leaseControl = { control() {} };
|
||||
assert.throws(
|
||||
() =>
|
||||
new ClusterRemoteWorkerCancellationDispatchControl(
|
||||
leaseControl,
|
||||
repository,
|
||||
{ ownerId: '', extra: true },
|
||||
),
|
||||
/invalid_configuration/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
new ClusterRemoteWorkerCancellationDispatchControl(
|
||||
leaseControl,
|
||||
repository,
|
||||
{ ownerId: 'replica-1', leaseDurationMs: 0 },
|
||||
),
|
||||
/invalid_configuration/,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user