open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-20 16:07:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add postgres cancellation dispatch

Browse Source
This commit is contained in:
whyour
2026-08-19 06:24:11 +08:00
parent 36035ac43e
commit 1809fbb8d3
29 changed files with 2265 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
back/runtime/adapters/legacy-sequelize/cancellationDispatchRepository.ts
+84 -35
View File
@@ -1,3 +1,4 @@
import { createHash } from 'crypto';
import {
DataTypes,
Model,
@@ -55,6 +56,19 @@ const BLOCKING_RESULTS: readonly CancellationDispatchResult[] = [
'unsupported',
'invalid',
];
const MAX_LEASE_DURATION_MS = 5 * 60_000;
const MAX_RETRY_DELAY_MS = 24 * 60 * 60_000;
export interface LegacySequelizeCancellationDispatchRepositoryOptions {
clock?: () => number;
}
function digestLeaseToken(value: string): string {
return createHash('sha256')
.update('qinglong.cancellation-dispatch-lease.v1\0', 'utf8')
.update(value, 'utf8')
.digest('hex');
}
interface CancellationDispatchRow {
runId: string;
@@ -287,18 +301,13 @@ function assertClaim(command: ClaimCancellationDispatchCommand): void {
assertId('owner', command.owner, 128);
assertId('leaseToken', command.leaseToken, 128);
assertTimestamp('requestedAtMs', command.requestedAtMs);
assertTimestamp('nowMs', command.nowMs);
if (
!Number.isSafeInteger(command.leaseDurationMs) ||
command.leaseDurationMs < 1
command.leaseDurationMs < 1 ||
command.leaseDurationMs > MAX_LEASE_DURATION_MS
) {
throw new InvalidCancellationDispatchCommandError(
'leaseDurationMs must be a positive safe integer',
);
}
if (!Number.isSafeInteger(command.nowMs + command.leaseDurationMs)) {
throw new InvalidCancellationDispatchCommandError(
'lease expiry exceeds the supported range',
`leaseDurationMs must be between 1 and ${MAX_LEASE_DURATION_MS}`,
);
}
}
@@ -311,7 +320,6 @@ function assertRecordResult(
assertId('owner', command.owner, 128);
assertId('leaseToken', command.leaseToken, 128);
assertId('eventId', command.eventId);
assertTimestamp('atMs', command.atMs);
if (
!Number.isSafeInteger(command.expectedVersion) ||
command.expectedVersion < 1
@@ -327,17 +335,18 @@ function assertRecordResult(
}
if (RETRYABLE_RESULTS.includes(command.result)) {
if (
command.nextAttemptAtMs === undefined ||
!Number.isSafeInteger(command.nextAttemptAtMs) ||
command.nextAttemptAtMs <= command.atMs
command.retryDelayMs === undefined ||
!Number.isSafeInteger(command.retryDelayMs) ||
command.retryDelayMs < 1 ||
command.retryDelayMs > MAX_RETRY_DELAY_MS
) {
throw new InvalidCancellationDispatchCommandError(
'retryable results require nextAttemptAtMs greater than atMs',
`retryable results require retryDelayMs between 1 and ${MAX_RETRY_DELAY_MS}`,
);
}
} else if (command.nextAttemptAtMs !== undefined) {
} else if (command.retryDelayMs !== undefined) {
throw new InvalidCancellationDispatchCommandError(
'terminal results must not include nextAttemptAtMs',
'terminal results must not include retryDelayMs',
);
}
}
@@ -417,7 +426,9 @@ function rowToDispatch(
? {}
: { nextAttemptAtMs: Number(row.nextAttemptAtMs) }),
...(row.leaseOwner === null ? {} : { leaseOwner: row.leaseOwner }),
...(row.leaseToken === null ? {} : { leaseToken: row.leaseToken }),
...(row.leaseToken === null
? {}
: { leaseTokenDigest: digestLeaseToken(row.leaseToken) }),
...(row.leaseExpiresAtMs === null
? {}
: { leaseExpiresAtMs: Number(row.leaseExpiresAtMs) }),
@@ -447,12 +458,15 @@ function withoutScheduleAndLease(
dispatch: CancellationDispatchRecord,
): Omit<
CancellationDispatchRecord,
'nextAttemptAtMs' | 'leaseOwner' | 'leaseToken' | 'leaseExpiresAtMs'
| 'nextAttemptAtMs'
| 'leaseOwner'
| 'leaseTokenDigest'
| 'leaseExpiresAtMs'
> {
const {
nextAttemptAtMs: _nextAttemptAtMs,
leaseOwner: _leaseOwner,
leaseToken: _leaseToken,
leaseTokenDigest: _leaseTokenDigest,
leaseExpiresAtMs: _leaseExpiresAtMs,
...rest
} = dispatch;
@@ -466,12 +480,17 @@ export class LegacySequelizeCancellationDispatchRepository
private readonly run: ModelStatic<CancellationDispatchRunInstance>;
private readonly attempt: ModelStatic<CancellationDispatchAttemptInstance>;
private readonly event: ModelStatic<CancellationDispatchEventInstance>;
private readonly clock: () => number;
constructor(private readonly database: Sequelize) {
constructor(
private readonly database: Sequelize,
options: LegacySequelizeCancellationDispatchRepositoryOptions = {},
) {
this.dispatch = defineDispatchModel(database);
this.run = defineRunModel(database);
this.attempt = defineAttemptModel(database);
this.event = defineEventModel(database);
this.clock = options.clock ?? Date.now;
}
async findByRunId(runId: string): Promise<CancellationDispatchRecord | null> {
@@ -489,6 +508,7 @@ export class LegacySequelizeCancellationDispatchRepository
return this.database.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
const nowMs = this.now();
const [run, attempt] = await Promise.all([
this.run.findByPk(command.runId, { raw: true, transaction }),
this.attempt.findByPk(command.attemptId, { raw: true, transaction }),
@@ -530,8 +550,8 @@ export class LegacySequelizeCancellationDispatchRepository
leaseExpiresAtMs: null,
lastResult: null,
lastDispatchedAtMs: null,
createdAtMs: command.nowMs,
updatedAtMs: command.nowMs,
createdAtMs: nowMs,
updatedAtMs: nowMs,
},
{ transaction },
);
@@ -556,21 +576,26 @@ export class LegacySequelizeCancellationDispatchRepository
if (
dispatch.status === 'leased' &&
dispatch.leaseExpiresAtMs !== undefined &&
dispatch.leaseExpiresAtMs > command.nowMs
dispatch.leaseExpiresAtMs > nowMs
) {
return { status: 'leased', dispatch };
}
if (
dispatch.status !== 'leased' &&
dispatch.nextAttemptAtMs !== undefined &&
dispatch.nextAttemptAtMs > command.nowMs
dispatch.nextAttemptAtMs > nowMs
) {
return { status: 'not_due', dispatch };
}
const nextVersion = dispatch.version + 1;
const nextCount = dispatch.dispatchCount + 1;
const leaseExpiresAtMs = command.nowMs + command.leaseDurationMs;
const leaseExpiresAtMs = nowMs + command.leaseDurationMs;
if (!Number.isSafeInteger(leaseExpiresAtMs)) {
throw new CancellationDispatchRepositoryError(
new Error('Cancellation dispatch lease expiry overflowed'),
);
}
const [affected] = await this.dispatch.update(
{
status: 'leased',
@@ -580,7 +605,7 @@ export class LegacySequelizeCancellationDispatchRepository
leaseOwner: command.owner,
leaseToken: command.leaseToken,
leaseExpiresAtMs,
updatedAtMs: command.nowMs,
updatedAtMs: nowMs,
},
{
where: { runId: command.runId, version: dispatch.version },
@@ -592,15 +617,16 @@ export class LegacySequelizeCancellationDispatchRepository
}
return {
status: 'claimed',
leaseToken: command.leaseToken,
dispatch: {
...withoutScheduleAndLease(dispatch),
status: 'leased',
version: nextVersion,
dispatchCount: nextCount,
leaseOwner: command.owner,
leaseToken: command.leaseToken,
leaseTokenDigest: digestLeaseToken(command.leaseToken),
leaseExpiresAtMs,
updatedAtMs: command.nowMs,
updatedAtMs: nowMs,
},
};
},
@@ -614,6 +640,7 @@ export class LegacySequelizeCancellationDispatchRepository
return this.database.transaction(
{ type: Transaction.TYPES.IMMEDIATE },
async (transaction) => {
const atMs = this.now();
const row = (await this.dispatch.findByPk(command.runId, {
raw: true,
transaction,
@@ -644,6 +671,18 @@ export class LegacySequelizeCancellationDispatchRepository
].includes(command.result);
const nextVersion = row.version + 1;
const nextSequence = Number(run.eventSequence) + 1;
const nextAttemptAtMs =
command.retryDelayMs === undefined
? undefined
: atMs + command.retryDelayMs;
if (
nextAttemptAtMs !== undefined &&
!Number.isSafeInteger(nextAttemptAtMs)
) {
throw new CancellationDispatchRepositoryError(
new Error('Cancellation dispatch retry deadline overflowed'),
);
}
const [runAffected] = await this.run.update(
{ version: Number(run.version) + 1, eventSequence: nextSequence },
{ where: { id: command.runId, version: run.version }, transaction },
@@ -655,15 +694,15 @@ export class LegacySequelizeCancellationDispatchRepository
{
status: state.status,
version: nextVersion,
nextAttemptAtMs: command.nextAttemptAtMs ?? null,
nextAttemptAtMs: nextAttemptAtMs ?? null,
leaseOwner: null,
leaseToken: null,
leaseExpiresAtMs: null,
lastResult: command.result,
lastDispatchedAtMs: controllerInvoked
? command.atMs
? atMs
: row.lastDispatchedAtMs,
updatedAtMs: command.atMs,
updatedAtMs: atMs,
},
{
where: {
@@ -694,7 +733,7 @@ export class LegacySequelizeCancellationDispatchRepository
dispatch_count: row.dispatchCount,
result: command.result,
},
createdAtMs: command.atMs,
createdAtMs: atMs,
};
await this.event.create(
{
@@ -716,20 +755,30 @@ export class LegacySequelizeCancellationDispatchRepository
...withoutScheduleAndLease(rowToDispatch(row)),
status: state.status,
version: nextVersion,
...(command.nextAttemptAtMs === undefined
...(nextAttemptAtMs === undefined
? {}
: { nextAttemptAtMs: command.nextAttemptAtMs }),
: { nextAttemptAtMs }),
lastResult: command.result,
...(controllerInvoked
? { lastDispatchedAtMs: command.atMs }
? { lastDispatchedAtMs: atMs }
: row.lastDispatchedAtMs === null
? {}
: { lastDispatchedAtMs: Number(row.lastDispatchedAtMs) }),
updatedAtMs: command.atMs,
updatedAtMs: atMs,
},
event,
};
},
);
}
private now(): number {
const nowMs = this.clock();
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
throw new CancellationDispatchRepositoryError(
new Error('Cancellation dispatch repository clock is invalid'),
);
}
return nowMs;
}
}
back/runtime/application/primaryCancellationDispatcher.ts
+34 -24
View File
@@ -35,7 +35,6 @@ export interface PrimaryCancellationDispatcherOptions {
leaseDurationMs?: number;
retryBaseMs?: number;
retryMaxMs?: number;
clock?: () => number;
createId?: () => string;
}
@@ -55,7 +54,6 @@ export class PrimaryCancellationDispatcher {
private readonly leaseDurationMs: number;
private readonly retryBaseMs: number;
private readonly retryMaxMs: number;
private readonly clock: () => number;
private readonly createId: () => string;
constructor(
@@ -71,7 +69,6 @@ export class PrimaryCancellationDispatcher {
this.leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
this.retryBaseMs = options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS;
this.retryMaxMs = options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS;
this.clock = options.clock ?? Date.now;
this.createId = options.createId ?? uuidV7;
assertPositiveInteger('leaseDurationMs', this.leaseDurationMs);
assertPositiveInteger('retryBaseMs', this.retryBaseMs);
@@ -130,7 +127,6 @@ export class PrimaryCancellationDispatcher {
}
const attempt = candidate.attempts[0];
const claimedAtMs = this.now();
let claim;
try {
claim = await this.dispatches.claim({
@@ -139,7 +135,6 @@ export class PrimaryCancellationDispatcher {
requestedAtMs: candidate.requestedAtMs,
owner: this.owner,
leaseToken: this.createId(),
nowMs: claimedAtMs,
leaseDurationMs: this.leaseDurationMs,
});
} catch {
@@ -165,6 +160,11 @@ export class PrimaryCancellationDispatcher {
summary.blocked += 1;
continue;
}
if (claim.status !== 'claimed') {
summary.failed += 1;
summary.pending += 1;
continue;
}
summary.claimed += 1;
await this.dispatchClaimed(
@@ -172,6 +172,7 @@ export class PrimaryCancellationDispatcher {
candidate.requestedAtMs,
attempt,
claim.dispatch,
claim.leaseToken,
summary,
);
}
@@ -186,15 +187,28 @@ export class PrimaryCancellationDispatcher {
Awaited<ReturnType<CancellationDispatchRepository['claim']>>,
{ status: 'claimed' }
>['dispatch'],
leaseToken: string,
summary: PrimaryCancellationDispatchSummary,
): Promise<void> {
const controller = this.controllers.get(attempt.executorType);
if (!controller) {
await this.record(attempt, dispatch, 'controller_missing', summary);
await this.record(
attempt,
dispatch,
leaseToken,
'controller_missing',
summary,
);
return;
}
if (!attempt.executorHandle) {
await this.record(attempt, dispatch, 'handle_missing', summary);
await this.record(
attempt,
dispatch,
leaseToken,
'handle_missing',
summary,
);
return;
}
@@ -207,7 +221,7 @@ export class PrimaryCancellationDispatcher {
requestedAtMs,
},
});
await this.record(attempt, dispatch, result.status, summary);
await this.record(attempt, dispatch, leaseToken, result.status, summary);
if (result.status === 'termination_requested') {
summary.terminationRequested += 1;
} else if (result.status === 'already_exited') {
@@ -217,7 +231,13 @@ export class PrimaryCancellationDispatcher {
}
} catch {
summary.failed += 1;
await this.record(attempt, dispatch, 'dispatch_error', summary);
await this.record(
attempt,
dispatch,
leaseToken,
'dispatch_error',
summary,
);
}
}
@@ -227,10 +247,10 @@ export class PrimaryCancellationDispatcher {
Awaited<ReturnType<CancellationDispatchRepository['claim']>>,
{ status: 'claimed' }
>['dispatch'],
leaseToken: string,
result: CancellationDispatchResult,
summary: PrimaryCancellationDispatchSummary,
): Promise<void> {
const atMs = this.now();
const retryable = [
'controller_missing',
'handle_missing',
@@ -241,12 +261,11 @@ export class PrimaryCancellationDispatcher {
runId: dispatch.runId,
attemptId: attempt.attemptId,
owner: this.owner,
leaseToken: dispatch.leaseToken!,
leaseToken,
expectedVersion: dispatch.version,
result,
atMs,
...(retryable
? { nextAttemptAtMs: this.nextRetryAt(atMs, dispatch.dispatchCount) }
? { retryDelayMs: this.nextRetryDelay(dispatch.dispatchCount) }
: {}),
eventId: this.createId(),
});
@@ -257,17 +276,8 @@ export class PrimaryCancellationDispatcher {
}
}
private nextRetryAt(atMs: number, dispatchCount: number): number {
private nextRetryDelay(dispatchCount: number): number {
const exponent = Math.max(0, Math.min(dispatchCount - 1, 30));
const delay = Math.min(this.retryMaxMs, this.retryBaseMs * 2 ** exponent);
return Math.min(Number.MAX_SAFE_INTEGER, atMs + delay);
}
private now(): number {
const nowMs = this.clock();
if (!Number.isSafeInteger(nowMs) || nowMs < 0) {
throw new RangeError('clock must return a non-negative safe integer');
}
return nowMs;
return Math.min(this.retryMaxMs, this.retryBaseMs * 2 ** exponent);
}
}
back/runtime/domain/cancellationDispatch.ts
+1 -1
View File
@@ -32,7 +32,7 @@ export interface CancellationDispatchRecord {
dispatchCount: number;
nextAttemptAtMs?: number;
leaseOwner?: string;
leaseToken?: string;
leaseTokenDigest?: string;
leaseExpiresAtMs?: number;
lastResult?: CancellationDispatchResult;
lastDispatchedAtMs?: number;
back/runtime/ports/cancellationDispatchRepository.ts
+6 -4
View File
@@ -10,12 +10,15 @@ export interface ClaimCancellationDispatchCommand {
requestedAtMs: number;
owner: string;
leaseToken: string;
nowMs: number;
leaseDurationMs: number;
}
export type ClaimCancellationDispatchResult =
| { status: 'claimed'; dispatch: CancellationDispatchRecord }
| {
status: 'claimed';
dispatch: CancellationDispatchRecord;
leaseToken: string;
}
| { status: 'not_eligible' }
| {
status: 'not_due' | 'leased' | 'dispatched' | 'blocked';
@@ -29,8 +32,7 @@ export interface RecordCancellationDispatchResultCommand {
leaseToken: string;
expectedVersion: number;
result: CancellationDispatchResult;
atMs: number;
nextAttemptAtMs?: number;
retryDelayMs?: number;
eventId: string;
}