feat(ql3): gate remote cancellation delivery

This commit is contained in:
whyour
2026-08-19 06:55:36 +08:00
parent 1809fbb8d3
commit 0b5f3bcb39
14 changed files with 776 additions and 25 deletions
@@ -175,6 +175,11 @@
"require": "./dist/remote-execution/remoteWorkerLeaseControlService.js",
"default": "./dist/remote-execution/remoteWorkerLeaseControlService.js"
},
"./cancellation-dispatch-control": {
"types": "./dist/remote-execution/remoteWorkerCancellationDispatchControl.d.ts",
"require": "./dist/remote-execution/remoteWorkerCancellationDispatchControl.js",
"default": "./dist/remote-execution/remoteWorkerCancellationDispatchControl.js"
},
"./workflow-scheduler": {
"types": "./dist/scheduling/workflowScheduler.d.ts",
"require": "./dist/scheduling/workflowScheduler.js",
@@ -903,6 +903,9 @@ export async function bootstrapClusterControlRuntime(
workerRuntime: createClusterWorkerRuntimePort(
database.pool,
options.workerRuntime,
{
cancellationDispatchOwnerId: recoveryRuntime.ownerId,
},
),
}),
});
@@ -75,6 +75,7 @@ import {
createClusterControlCopilotFailureDiagnosisCancellationRoute,
type ClusterCopilotFailureDiagnosisCancellationCapability,
} from '../copilot/failure-diagnosis/failureDiagnosisCancellationRoute';
import type { ClusterRemoteWorkerCancellationDispatchObservation } from '../remote-execution/remoteWorkerCancellationDispatchControl';
export const PRODUCTION_CLUSTER_CONTROL_ROUTE_OPERATIONS = Object.freeze([
'task.get',
@@ -150,6 +151,12 @@ export interface ProductionClusterWorkerIngressOptions {
readonly artifactStore: ClusterRemoteWorkerArtifactStore;
readonly secretProvider?: RemoteWorkerSecretValueProvider;
readonly onDiagnostic?: (error: unknown) => void | Promise<void>;
readonly onCancellationDispatch?: (
observation: ClusterRemoteWorkerCancellationDispatchObservation,
) => void | Promise<void>;
readonly onCancellationDispatchDiagnostic?: (
error: unknown,
) => void | Promise<void>;
}
export interface ProductionClusterControlApplicationOptions
@@ -439,6 +446,28 @@ export function startProductionClusterControlApplication(
...(workerIngress.secretProvider === undefined
? {}
: { secretProvider: workerIngress.secretProvider }),
...(
workerIngress.onCancellationDispatch === undefined &&
workerIngress.onCancellationDispatchDiagnostic === undefined
? {}
: {
cancellationDispatch: {
...(workerIngress.onCancellationDispatch === undefined
? {}
: {
onObservation:
workerIngress.onCancellationDispatch,
}),
...(workerIngress.onCancellationDispatchDiagnostic ===
undefined
? {}
: {
onDiagnostic:
workerIngress.onCancellationDispatchDiagnostic,
}),
},
}
),
},
}),
...database,
@@ -21,6 +21,7 @@ import {
} from '../worker-ingress/workerIngressConfig';
import type { ClusterWorkerArtifactBinding } from '../artifact/workerArtifactBinding';
import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/remote-secret-delivery';
import type { ClusterRemoteWorkerCancellationDispatchObservation } from '../remote-execution/remoteWorkerCancellationDispatchControl';
export type ClusterControlProcessSignal = 'SIGINT' | 'SIGTERM';
@@ -34,10 +35,12 @@ export interface ClusterControlProcessEvent {
readonly stopResult?: ClusterControlStopResult;
readonly address?: Readonly<{ host: string; port: number }>;
readonly activation?: ClusterControlActivationAudit;
readonly cancellationDispatch?: ClusterRemoteWorkerCancellationDispatchObservation;
readonly diagnostic?: Readonly<{
scope:
| 'scheduler'
| 'cancellation-convergence'
| 'cancellation-dispatch'
| 'log-retention'
| 'database'
| 'worker-ingress';
@@ -333,6 +336,32 @@ export async function runProductionClusterControlProcess(
),
).catch(() => undefined);
},
onCancellationDispatch(observation) {
void Promise.resolve(
options.emit(
event(replicaId, {
level:
observation.status === 'blocked' ? 'error' : 'info',
event: 'cancellation_dispatch',
cancellationDispatch: observation,
}),
),
).catch(() => undefined);
},
onCancellationDispatchDiagnostic(error: unknown) {
void Promise.resolve(
options.emit(
event(replicaId, {
level: 'error',
event: 'runtime_diagnostic',
diagnostic: diagnosticFact(
'cancellation-dispatch',
error,
),
}),
),
).catch(() => undefined);
},
},
}),
audit(record) {
@@ -0,0 +1,238 @@
// Remote execution owns cancellation delivery to the Worker that already holds
// the exact RunDispatchLease. This layer adds no timer, queue, or connection.
import { randomUUID } from 'node:crypto';
import {
MAX_CANCELLATION_DISPATCH_LEASE_MS,
type CancellationDispatchRepository,
} from '@qinglong/runtime-core/cancellation-dispatch';
import {
RemoteWorkerLeaseControlUnavailableError,
type RemoteWorkerLeaseControlCommand,
type RemoteWorkerLeaseControlResult,
} from '@qinglong/runtime-core/remote-worker-lease-control';
export type ClusterRemoteWorkerCancellationDispatchObservation = Readonly<{
readonly status:
| 'dispatched'
| 'already_dispatched'
| 'untracked'
| 'deferred'
| 'blocked';
}>;
export interface ClusterRemoteWorkerCancellationDispatchControlOptions {
readonly ownerId: string;
readonly leaseDurationMs?: number;
readonly createLeaseToken?: () => string;
readonly createEventId?: () => string;
readonly onObservation?: (
observation: ClusterRemoteWorkerCancellationDispatchObservation,
) => void | Promise<void>;
readonly onDiagnostic?: (error: unknown) => void | Promise<void>;
}
export class ClusterRemoteWorkerCancellationDispatchError extends Error {
readonly code = 'CLUSTER_REMOTE_CANCELLATION_DISPATCH_FAILED';
constructor(
readonly reason:
| 'invalid_configuration'
| 'claim_failed'
| 'result_failed'
| 'delivery_deferred'
| 'delivery_blocked',
options?: ErrorOptions,
) {
super(`Cluster Remote Worker cancellation dispatch failed: ${reason}`, options);
this.name = 'ClusterRemoteWorkerCancellationDispatchError';
}
}
const OWNER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u;
const OPTION_KEYS = new Set([
'createEventId',
'createLeaseToken',
'leaseDurationMs',
'onDiagnostic',
'onObservation',
'ownerId',
]);
function invalidConfiguration(): never {
throw new ClusterRemoteWorkerCancellationDispatchError(
'invalid_configuration',
);
}
function capability(factory: () => string, name: string): string {
let value: unknown;
try {
value = factory();
} catch (error) {
throw new ClusterRemoteWorkerCancellationDispatchError(
'claim_failed',
{ cause: error },
);
}
const maximum = name === 'eventId' ? 36 : 128;
if (
typeof value !== 'string' ||
value.length < 1 ||
value.length > maximum ||
/[\u0000-\u001f\u007f]/u.test(value)
) {
throw new ClusterRemoteWorkerCancellationDispatchError('claim_failed');
}
return value;
}
/**
* Converts the existing caller-driven Worker lease-control tick into the only
* Cluster cancellation delivery path. A stop response is released only after
* its durable CancellationDispatch is settled, while Workflow-scoped timeout
* stops remain valid without forging a Run cancellation record.
*/
export class ClusterRemoteWorkerCancellationDispatchControl {
private readonly ownerId: string;
private readonly leaseDurationMs: number;
private readonly createLeaseToken: () => string;
private readonly createEventId: () => string;
private readonly onObservation?: ClusterRemoteWorkerCancellationDispatchControlOptions['onObservation'];
private readonly onDiagnostic?: ClusterRemoteWorkerCancellationDispatchControlOptions['onDiagnostic'];
constructor(
private readonly leaseControl: Readonly<{
control(
command: RemoteWorkerLeaseControlCommand,
): Promise<Readonly<RemoteWorkerLeaseControlResult>>;
}>,
private readonly dispatches: CancellationDispatchRepository,
options: ClusterRemoteWorkerCancellationDispatchControlOptions,
) {
if (
typeof leaseControl?.control !== 'function' ||
typeof dispatches?.claim !== 'function' ||
typeof dispatches?.recordResult !== 'function' ||
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
Object.keys(options).some((key) => !OPTION_KEYS.has(key)) ||
!OWNER_PATTERN.test(options.ownerId ?? '') ||
(options.leaseDurationMs !== undefined &&
(!Number.isSafeInteger(options.leaseDurationMs) ||
options.leaseDurationMs < 1 ||
options.leaseDurationMs > MAX_CANCELLATION_DISPATCH_LEASE_MS)) ||
(options.createLeaseToken !== undefined &&
typeof options.createLeaseToken !== 'function') ||
(options.createEventId !== undefined &&
typeof options.createEventId !== 'function') ||
(options.onObservation !== undefined &&
typeof options.onObservation !== 'function') ||
(options.onDiagnostic !== undefined &&
typeof options.onDiagnostic !== 'function')
) {
invalidConfiguration();
}
this.ownerId = options.ownerId;
this.leaseDurationMs = options.leaseDurationMs ?? 30_000;
this.createLeaseToken = options.createLeaseToken ?? randomUUID;
this.createEventId = options.createEventId ?? randomUUID;
this.onObservation = options.onObservation;
this.onDiagnostic = options.onDiagnostic;
}
async control(
command: RemoteWorkerLeaseControlCommand,
): Promise<Readonly<RemoteWorkerLeaseControlResult>> {
const result = await this.leaseControl.control(command);
if (result.status !== 'stop_requested') return result;
let claim: Awaited<ReturnType<CancellationDispatchRepository['claim']>>;
try {
claim = await this.dispatches.claim({
runId: result.runId,
attemptId: result.attemptId,
requestedAtMs: result.stop!.requestedAtMs,
owner: this.ownerId,
leaseToken: capability(this.createLeaseToken, 'leaseToken'),
leaseDurationMs: this.leaseDurationMs,
});
} catch (error) {
return this.unavailable('claim_failed', error);
}
if (claim.status === 'not_eligible') {
// Workflow Task timeout is represented by its own event and does not set
// Run.cancel_requested_at_ms. The already-fenced Worker stop must remain
// deliverable without inventing a Run-level cancellation fact.
this.observe('untracked');
return result;
}
if (claim.status === 'dispatched') {
this.observe('already_dispatched');
return result;
}
if (claim.status === 'leased' || claim.status === 'not_due') {
this.observe('deferred');
return this.unavailable('delivery_deferred');
}
if (claim.status === 'blocked') {
this.observe('blocked');
return this.unavailable('delivery_blocked');
}
if (claim.status !== 'claimed') {
return this.unavailable('claim_failed');
}
try {
const settled = await this.dispatches.recordResult({
runId: result.runId,
attemptId: result.attemptId,
owner: this.ownerId,
leaseToken: claim.leaseToken,
expectedVersion: claim.dispatch.version,
result: 'termination_requested',
eventId: capability(this.createEventId, 'eventId'),
});
if (
settled.dispatch.status !== 'dispatched' ||
settled.dispatch.lastResult !== 'termination_requested' ||
settled.event.type !== 'run.cancel_dispatched'
) {
return this.unavailable('result_failed');
}
} catch (error) {
return this.unavailable('result_failed', error);
}
this.observe('dispatched');
return result;
}
private unavailable(
reason: Exclude<
ClusterRemoteWorkerCancellationDispatchError['reason'],
'invalid_configuration'
>,
cause?: unknown,
): never {
const error = new ClusterRemoteWorkerCancellationDispatchError(reason, {
...(cause === undefined ? {} : { cause }),
});
this.diagnostic(error);
throw new RemoteWorkerLeaseControlUnavailableError({ cause: error });
}
private observe(
status: ClusterRemoteWorkerCancellationDispatchObservation['status'],
): void {
if (!this.onObservation) return;
void Promise.resolve(
this.onObservation(Object.freeze({ status })),
).catch(() => undefined);
}
private diagnostic(error: unknown): void {
if (!this.onDiagnostic) return;
void Promise.resolve(this.onDiagnostic(error)).catch(() => undefined);
}
}
@@ -4,6 +4,7 @@ import type { RemoteWorkerSecretValueProvider } from '@qinglong/runtime-core/rem
import type { RunAttemptLogRangeReader } from '@qinglong/runtime-core/run-attempt-log-read';
import {
PostgresClusterDispatchSource,
PostgresCancellationDispatchRepository,
PostgresRemoteRunActivationRepository,
PostgresRemoteWorkerCompletionRepository,
PostgresRemoteWorkerLeaseControlRepository,
@@ -21,11 +22,23 @@ import {
type ClusterRemoteWorkerArtifactStore,
} from './remoteWorkerCompletionService';
import { ClusterRemoteWorkerLeaseControlService } from './remoteWorkerLeaseControlService';
import {
ClusterRemoteWorkerCancellationDispatchControl,
type ClusterRemoteWorkerCancellationDispatchControlOptions,
} from './remoteWorkerCancellationDispatchControl';
import type { WorkerIngressPipelineOptions } from '../worker-ingress/workerIngressPipeline';
export interface ClusterWorkerRuntimeDependencies {
readonly artifactStore: ClusterRemoteWorkerArtifactStore;
readonly secretProvider?: RemoteWorkerSecretValueProvider;
readonly cancellationDispatch?: Readonly<{
readonly onObservation?: ClusterRemoteWorkerCancellationDispatchControlOptions['onObservation'];
readonly onDiagnostic?: ClusterRemoteWorkerCancellationDispatchControlOptions['onDiagnostic'];
}>;
}
export interface ClusterWorkerRuntimePortOptions {
readonly cancellationDispatchOwnerId: string;
}
/**
@@ -48,6 +61,7 @@ export interface ClusterWorkerRuntimePort {
export function createClusterWorkerRuntimePort(
pool: PostgresPool,
dependencies: ClusterWorkerRuntimeDependencies,
options: ClusterWorkerRuntimePortOptions,
): Readonly<ClusterWorkerRuntimePort> {
if (!pool || typeof pool.query !== 'function') {
throw new TypeError('Cluster Worker runtime Pool is invalid');
@@ -59,6 +73,15 @@ export function createClusterWorkerRuntimePort(
) {
throw new TypeError('Cluster Worker runtime dependencies are invalid');
}
if (
!options ||
typeof options !== 'object' ||
Array.isArray(options) ||
Object.keys(options).length !== 1 ||
typeof options.cancellationDispatchOwnerId !== 'string'
) {
throw new TypeError('Cluster Worker runtime options are invalid');
}
const workerSessions = new PostgresWorkerSessionRepository(pool);
const completionRepository = new PostgresRemoteWorkerCompletionRepository(
@@ -92,8 +115,25 @@ export function createClusterWorkerRuntimePort(
completionRepository,
dependencies.artifactStore,
),
leaseControl: new ClusterRemoteWorkerLeaseControlService(
new PostgresRemoteWorkerLeaseControlRepository(pool),
leaseControl: new ClusterRemoteWorkerCancellationDispatchControl(
new ClusterRemoteWorkerLeaseControlService(
new PostgresRemoteWorkerLeaseControlRepository(pool),
),
new PostgresCancellationDispatchRepository(pool),
{
ownerId: options.cancellationDispatchOwnerId,
...(dependencies.cancellationDispatch?.onObservation === undefined
? {}
: {
onObservation:
dependencies.cancellationDispatch.onObservation,
}),
...(dependencies.cancellationDispatch?.onDiagnostic === undefined
? {}
: {
onDiagnostic: dependencies.cancellationDispatch.onDiagnostic,
}),
},
),
...(readLogRange === undefined
? {}
@@ -206,6 +206,12 @@ test('starts the optional Worker listener and closes its lazy Artifact binding',
code: 'S3Unavailable',
}),
);
options.workerIngress.onCancellationDispatch({ status: 'dispatched' });
options.workerIngress.onCancellationDispatchDiagnostic(
Object.assign(new Error('must-not-be-logged'), {
code: 'CANCEL_DISPATCH_UNAVAILABLE',
}),
);
return {
status: 'active',
address: { host: '0.0.0.0', port: 5800 },
@@ -248,6 +254,25 @@ test('starts the optional Worker listener and closes its lazy Artifact binding',
),
true,
);
assert.equal(
facts.some(
(fact) =>
fact.event === 'cancellation_dispatch' &&
fact.level === 'info' &&
fact.cancellationDispatch.status === 'dispatched',
),
true,
);
assert.equal(
facts.some(
(fact) =>
fact.event === 'runtime_diagnostic' &&
fact.diagnostic.scope === 'cancellation-dispatch' &&
fact.diagnostic.code === 'CANCEL_DISPATCH_UNAVAILABLE' &&
JSON.stringify(fact).includes('must-not-be-logged') === false,
),
true,
);
});
test('creates the configured mounted Secret provider before Worker activation', async () => {
@@ -0,0 +1,293 @@
'use strict';
const assert = require('node:assert/strict');
const { test } = require('node:test');
const {
ClusterRemoteWorkerCancellationDispatchControl,
ClusterRemoteWorkerCancellationDispatchError,
} = require('@qinglong/cluster-control/cancellation-dispatch-control');
const {
RemoteWorkerLeaseControlUnavailableError,
} = require('@qinglong/runtime-core/remote-worker-lease-control');
const COMMAND = Object.freeze({
workerId: 'worker-1',
workerSessionId: '018f0000-0000-7000-8000-000000000001',
workerGeneration: 2,
projectId: 'project-1',
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
leaseGeneration: 3,
leaseToken: 'worker_generated_lease_capability_0000000000000001',
expectedLeaseVersion: 4,
});
const STOP = Object.freeze({
status: 'stop_requested',
projectId: 'project-1',
runId: 'run-1',
attemptId: 'attempt-1',
offerId: 'offer-1',
leaseGeneration: 3,
leaseVersion: 5,
renewedAtMs: 10_000,
expiresAtMs: 40_000,
stop: Object.freeze({ reason: 'user', requestedAtMs: 9_000 }),
});
function leasedDispatch() {
return Object.freeze({
runId: 'run-1',
attemptId: 'attempt-1',
status: 'leased',
version: 1,
dispatchCount: 1,
leaseOwner: 'replica-1',
leaseTokenDigest: 'a'.repeat(64),
leaseExpiresAtMs: 40_000,
createdAtMs: 10_000,
updatedAtMs: 10_000,
});
}
function service(dispatches, overrides = {}) {
return new ClusterRemoteWorkerCancellationDispatchControl(
{
async control() {
return overrides.result ?? STOP;
},
},
dispatches,
{
ownerId: 'replica-1',
leaseDurationMs: 30_000,
createLeaseToken: () => 'cancel-token-1',
createEventId: () => '018f0000-0000-7000-8000-000000000011',
...(overrides.onObservation === undefined
? {}
: { onObservation: overrides.onObservation }),
...(overrides.onDiagnostic === undefined
? {}
: { onDiagnostic: overrides.onDiagnostic }),
},
);
}
test('bypasses dispatch storage when lease control only renews', async () => {
let calls = 0;
const renewed = Object.freeze({
...STOP,
status: 'renewed',
stop: undefined,
});
const control = service(
{
async claim() {
calls += 1;
throw new Error('must not claim');
},
async recordResult() {
calls += 1;
throw new Error('must not record');
},
},
{ result: renewed },
);
assert.equal(await control.control(COMMAND), renewed);
assert.equal(calls, 0);
});
test('settles one durable dispatch before releasing a Worker stop', async () => {
const observed = [];
let claimCommand;
let resultCommand;
const claimed = leasedDispatch();
const control = service(
{
async claim(value) {
claimCommand = value;
return { status: 'claimed', dispatch: claimed, leaseToken: 'cancel-token-1' };
},
async recordResult(value) {
resultCommand = value;
return {
dispatch: {
...claimed,
status: 'dispatched',
version: 2,
leaseOwner: undefined,
leaseTokenDigest: undefined,
leaseExpiresAtMs: undefined,
lastResult: 'termination_requested',
lastDispatchedAtMs: 10_001,
updatedAtMs: 10_001,
},
event: { type: 'run.cancel_dispatched' },
};
},
},
{ onObservation: (value) => observed.push(value) },
);
assert.equal(await control.control(COMMAND), STOP);
assert.deepEqual(claimCommand, {
runId: 'run-1',
attemptId: 'attempt-1',
requestedAtMs: 9_000,
owner: 'replica-1',
leaseToken: 'cancel-token-1',
leaseDurationMs: 30_000,
});
assert.deepEqual(resultCommand, {
runId: 'run-1',
attemptId: 'attempt-1',
owner: 'replica-1',
leaseToken: 'cancel-token-1',
expectedVersion: 1,
result: 'termination_requested',
eventId: '018f0000-0000-7000-8000-000000000011',
});
assert.deepEqual(observed, [{ status: 'dispatched' }]);
});
test('releases an already-dispatched stop without a second result event', async () => {
let results = 0;
const observed = [];
const control = service(
{
async claim() {
return {
status: 'dispatched',
dispatch: { ...leasedDispatch(), status: 'dispatched' },
};
},
async recordResult() {
results += 1;
throw new Error('must not record');
},
},
{ onObservation: (value) => observed.push(value) },
);
assert.equal(await control.control(COMMAND), STOP);
assert.equal(results, 0);
assert.deepEqual(observed, [{ status: 'already_dispatched' }]);
});
test('keeps a foreign live dispatch from releasing a duplicate stop', async () => {
const diagnostics = [];
const observed = [];
const control = service(
{
async claim() {
return { status: 'leased', dispatch: leasedDispatch() };
},
async recordResult() {
throw new Error('must not record');
},
},
{
onObservation: (value) => observed.push(value),
onDiagnostic: (error) => diagnostics.push(error),
},
);
await assert.rejects(
control.control(COMMAND),
(error) =>
error instanceof RemoteWorkerLeaseControlUnavailableError &&
error.cause instanceof ClusterRemoteWorkerCancellationDispatchError &&
error.cause.reason === 'delivery_deferred',
);
assert.deepEqual(observed, [{ status: 'deferred' }]);
assert.equal(diagnostics[0].reason, 'delivery_deferred');
});
test('fails closed and reports a durable blocked dispatch', async () => {
const diagnostics = [];
const observed = [];
const control = service(
{
async claim() {
return { status: 'blocked', dispatch: leasedDispatch() };
},
async recordResult() {
throw new Error('must not record');
},
},
{
onObservation: (value) => observed.push(value),
onDiagnostic: (error) => diagnostics.push(error),
},
);
await assert.rejects(
control.control(COMMAND),
(error) => error.cause?.reason === 'delivery_blocked',
);
assert.deepEqual(observed, [{ status: 'blocked' }]);
assert.equal(diagnostics[0].code, 'CLUSTER_REMOTE_CANCELLATION_DISPATCH_FAILED');
});
test('preserves Workflow-scoped timeout stops without forging Run cancellation', async () => {
let results = 0;
const observed = [];
const control = service(
{
async claim() {
return { status: 'not_eligible' };
},
async recordResult() {
results += 1;
},
},
{ onObservation: (value) => observed.push(value) },
);
assert.equal(await control.control(COMMAND), STOP);
assert.equal(results, 0);
assert.deepEqual(observed, [{ status: 'untracked' }]);
});
test('does not release a stop when durable result settlement fails', async () => {
const diagnostics = [];
const control = service(
{
async claim() {
return {
status: 'claimed',
dispatch: leasedDispatch(),
leaseToken: 'cancel-token-1',
};
},
async recordResult() {
throw new Error('database unavailable');
},
},
{ onDiagnostic: (error) => diagnostics.push(error) },
);
await assert.rejects(
control.control(COMMAND),
(error) => error.cause?.reason === 'result_failed',
);
assert.equal(diagnostics[0].reason, 'result_failed');
});
test('rejects widened or unbounded production configuration', () => {
const repository = { claim() {}, recordResult() {} };
const leaseControl = { control() {} };
assert.throws(
() =>
new ClusterRemoteWorkerCancellationDispatchControl(
leaseControl,
repository,
{ ownerId: '', extra: true },
),
/invalid_configuration/,
);
assert.throws(
() =>
new ClusterRemoteWorkerCancellationDispatchControl(
leaseControl,
repository,
{ ownerId: 'replica-1', leaseDurationMs: 0 },
),
/invalid_configuration/,
);
});