mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 02:27:44 +08:00
feat(ql3): add postgres cancellation dispatch
This commit is contained in:
whyour
@@ -85,6 +85,11 @@
|
||||
"require": "./dist/entrypoints/runManager.js",
|
||||
"default": "./dist/entrypoints/runManager.js"
|
||||
},
|
||||
"./cancellation-dispatch": {
|
||||
"types": "./dist/run/cancellationDispatchRepository.d.ts",
|
||||
"require": "./dist/run/cancellationDispatchRepository.js",
|
||||
"default": "./dist/run/cancellationDispatchRepository.js"
|
||||
},
|
||||
"./approval-manager": {
|
||||
"types": "./dist/approval-management/index.d.ts",
|
||||
"require": "./dist/approval-management/index.js",
|
||||
|
||||
@@ -54,6 +54,7 @@ export { PostgresToolResultKeyCatalogReader } from '../tool-execution/toolResult
|
||||
export { PostgresToolResultRekeyReader } from '../tool-execution/toolResultRekeyRepository';
|
||||
|
||||
export * from '../run/runRepository';
|
||||
export * from '../run/cancellationDispatchRepository';
|
||||
export * from '../run/runAttemptLogRetentionClaimRepository';
|
||||
export * from '../security/projectPolicyRepository';
|
||||
export * from '../security/apiCredentialRepository';
|
||||
|
||||
@@ -333,5 +333,10 @@ export const postgresqlMainMigrationManifest: MigrationStreamManifest =
|
||||
checksum:
|
||||
'95387c5b40659490dbcb7626ecd15bacf6412360752bef88873bde57c43e0185',
|
||||
}),
|
||||
Object.freeze({
|
||||
id: 'pg-0066-cancellation-dispatch',
|
||||
checksum:
|
||||
'b6d7ac81b5f75530df05f8ef05878fa30aa0f4418363973ded89d14ffce151b2',
|
||||
}),
|
||||
]),
|
||||
});
|
||||
|
||||
@@ -68,6 +68,7 @@ import { pg0062PluginPackageSecretBindingTargetGuardMigration } from './pg-0062-
|
||||
import { pg0063PluginPackageSecretBindingTransitionReceiptsMigration } from './pg-0063-plugin-package-secret-binding-transition-receipts';
|
||||
import { pg0064PluginPackageSecretBindingTransitionApprovalPlansMigration } from './pg-0064-plugin-package-secret-binding-transition-approval-plans';
|
||||
import { pg0065ApprovedActionManualRecoveryMigration } from '../approved-action/pg-0065-approved-action-manual-recovery';
|
||||
import { pg0066CancellationDispatchMigration } from '../run/migrations/pg-0066-cancellation-dispatch';
|
||||
|
||||
export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMigrationContext> =
|
||||
Object.freeze({
|
||||
@@ -141,5 +142,6 @@ export const postgresqlMainMigrationStream: MigrationStreamDefinition<PostgresMi
|
||||
pg0063PluginPackageSecretBindingTransitionReceiptsMigration,
|
||||
pg0064PluginPackageSecretBindingTransitionApprovalPlansMigration,
|
||||
pg0065ApprovedActionManualRecoveryMigration,
|
||||
pg0066CancellationDispatchMigration,
|
||||
]),
|
||||
});
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
import type { PostgresClient, PostgresPool } from '@qinglong/runtime-core';
|
||||
import {
|
||||
CancellationDispatchBindingConflictError,
|
||||
CancellationDispatchError,
|
||||
CancellationDispatchFenceRejectedError,
|
||||
CancellationDispatchRepositoryError,
|
||||
cancellationDispatchResultState,
|
||||
digestCancellationDispatchLeaseToken,
|
||||
normalizeCancellationDispatchRecord,
|
||||
normalizeCancellationDispatchRunId,
|
||||
normalizeClaimCancellationDispatchCommand,
|
||||
normalizeRecordCancellationDispatchResultCommand,
|
||||
type CancellationDispatchRecord,
|
||||
type CancellationDispatchRepository,
|
||||
type ClaimCancellationDispatchCommand,
|
||||
type ClaimCancellationDispatchResult,
|
||||
type RecordCancellationDispatchResult,
|
||||
type RecordCancellationDispatchResultCommand,
|
||||
} from '@qinglong/runtime-core/cancellation-dispatch';
|
||||
import type { RunEventRecord } from '@qinglong/runtime-core/run';
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
const ACTIVE_RUN_STATUSES = new Set([
|
||||
'created',
|
||||
'queued',
|
||||
'dispatching',
|
||||
'running',
|
||||
'waiting_approval',
|
||||
'retry_wait',
|
||||
'lost',
|
||||
]);
|
||||
const ACTIVE_ATTEMPT_STATUSES = new Set(['claimed', 'starting', 'running']);
|
||||
const CONTROLLER_NOT_INVOKED_RESULTS = new Set([
|
||||
'controller_missing',
|
||||
'handle_missing',
|
||||
]);
|
||||
|
||||
function text(row: Row, key: string): string {
|
||||
const value = row[key];
|
||||
if (typeof value !== 'string' || value.length < 1) {
|
||||
throw new TypeError(`PostgreSQL cancellation dispatch ${key} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function integer(row: Row, key: string): number {
|
||||
const raw = row[key];
|
||||
const value =
|
||||
typeof raw === 'string' && /^(0|[1-9]\d*)$/u.test(raw)
|
||||
? Number(raw)
|
||||
: raw;
|
||||
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
|
||||
throw new TypeError(`PostgreSQL cancellation dispatch ${key} is invalid`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function optionalText(row: Row, key: string): string | undefined {
|
||||
return row[key] === null || row[key] === undefined
|
||||
? undefined
|
||||
: text(row, key);
|
||||
}
|
||||
|
||||
function optionalInteger(row: Row, key: string): number | undefined {
|
||||
return row[key] === null || row[key] === undefined
|
||||
? undefined
|
||||
: integer(row, key);
|
||||
}
|
||||
|
||||
function dispatchFromRow(row: Row): Readonly<CancellationDispatchRecord> {
|
||||
const nextAttemptAtMs = optionalInteger(row, 'nextAttemptAtMs');
|
||||
const leaseOwner = optionalText(row, 'leaseOwner');
|
||||
const leaseTokenDigest = optionalText(row, 'leaseTokenDigest');
|
||||
const leaseExpiresAtMs = optionalInteger(row, 'leaseExpiresAtMs');
|
||||
const lastResult = optionalText(row, 'lastResult');
|
||||
const lastDispatchedAtMs = optionalInteger(row, 'lastDispatchedAtMs');
|
||||
return normalizeCancellationDispatchRecord({
|
||||
runId: text(row, 'runId'),
|
||||
attemptId: text(row, 'attemptId'),
|
||||
status: text(row, 'status') as CancellationDispatchRecord['status'],
|
||||
version: integer(row, 'version'),
|
||||
dispatchCount: integer(row, 'dispatchCount'),
|
||||
...(nextAttemptAtMs === undefined ? {} : { nextAttemptAtMs }),
|
||||
...(leaseOwner === undefined ? {} : { leaseOwner }),
|
||||
...(leaseTokenDigest === undefined ? {} : { leaseTokenDigest }),
|
||||
...(leaseExpiresAtMs === undefined ? {} : { leaseExpiresAtMs }),
|
||||
...(lastResult === undefined
|
||||
? {}
|
||||
: {
|
||||
lastResult:
|
||||
lastResult as NonNullable<CancellationDispatchRecord['lastResult']>,
|
||||
}),
|
||||
...(lastDispatchedAtMs === undefined ? {} : { lastDispatchedAtMs }),
|
||||
createdAtMs: integer(row, 'createdAtMs'),
|
||||
updatedAtMs: integer(row, 'updatedAtMs'),
|
||||
});
|
||||
}
|
||||
|
||||
const DISPATCH_COLUMNS = `
|
||||
run_id AS "runId", attempt_id AS "attemptId", status,
|
||||
version, dispatch_count AS "dispatchCount",
|
||||
next_attempt_at_ms AS "nextAttemptAtMs", lease_owner AS "leaseOwner",
|
||||
lease_token_digest AS "leaseTokenDigest",
|
||||
lease_expires_at_ms AS "leaseExpiresAtMs", last_result AS "lastResult",
|
||||
last_dispatched_at_ms AS "lastDispatchedAtMs",
|
||||
created_at_ms AS "createdAtMs", updated_at_ms AS "updatedAtMs"
|
||||
`;
|
||||
|
||||
async function begin(client: PostgresClient): Promise<void> {
|
||||
await client.query('BEGIN');
|
||||
await client.query(`SELECT set_config('statement_timeout', $1, true)`, [
|
||||
'5000ms',
|
||||
]);
|
||||
await client.query(`SELECT set_config('lock_timeout', $1, true)`, ['1000ms']);
|
||||
await client.query(
|
||||
`SELECT set_config('idle_in_transaction_session_timeout', $1, true)`,
|
||||
['10000ms'],
|
||||
);
|
||||
}
|
||||
|
||||
async function databaseNow(client: PostgresClient): Promise<number> {
|
||||
const result = await client.query<Row>(`
|
||||
SELECT floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint
|
||||
AS "nowMs"
|
||||
`);
|
||||
if (result.rows.length !== 1) {
|
||||
throw new TypeError('PostgreSQL cancellation dispatch clock is invalid');
|
||||
}
|
||||
return integer(result.rows[0]!, 'nowMs');
|
||||
}
|
||||
|
||||
async function rollback(client: PostgresClient): Promise<void> {
|
||||
try {
|
||||
await client.query('ROLLBACK');
|
||||
} catch {
|
||||
// Preserve the transaction failure.
|
||||
}
|
||||
}
|
||||
|
||||
function repositoryFailure(error: unknown): never {
|
||||
if (error instanceof CancellationDispatchError) throw error;
|
||||
throw new CancellationDispatchRepositoryError(error);
|
||||
}
|
||||
|
||||
export class PostgresCancellationDispatchRepository
|
||||
implements CancellationDispatchRepository
|
||||
{
|
||||
constructor(private readonly pool: PostgresPool) {
|
||||
if (!pool || typeof pool.connect !== 'function') {
|
||||
throw new TypeError('PostgreSQL cancellation dispatch pool is invalid');
|
||||
}
|
||||
}
|
||||
|
||||
async findByRunId(
|
||||
runId: string,
|
||||
): Promise<Readonly<CancellationDispatchRecord> | null> {
|
||||
try {
|
||||
const normalizedRunId = normalizeCancellationDispatchRunId(runId);
|
||||
const result = await this.pool.query<Row>(
|
||||
`SELECT ${DISPATCH_COLUMNS}
|
||||
FROM "ql3"."run_cancellation_dispatches"
|
||||
WHERE run_id = $1`,
|
||||
[normalizedRunId],
|
||||
);
|
||||
if (result.rows.length === 0) return null;
|
||||
if (result.rows.length !== 1) {
|
||||
throw new TypeError('PostgreSQL cancellation dispatch is not unique');
|
||||
}
|
||||
return dispatchFromRow(result.rows[0]!);
|
||||
} catch (error) {
|
||||
return repositoryFailure(error);
|
||||
}
|
||||
}
|
||||
|
||||
async claim(
|
||||
value: Readonly<ClaimCancellationDispatchCommand>,
|
||||
): Promise<ClaimCancellationDispatchResult> {
|
||||
const command = normalizeClaimCancellationDispatchCommand(value);
|
||||
return this.transaction(async (client) => {
|
||||
const nowMs = await databaseNow(client);
|
||||
const run = await client.query<Row>(
|
||||
`SELECT execution_owner AS "executionOwner", status,
|
||||
cancel_requested_at_ms AS "cancelRequestedAtMs"
|
||||
FROM "ql3"."runs" WHERE id = $1 FOR UPDATE`,
|
||||
[command.runId],
|
||||
);
|
||||
if (run.rows.length > 1) {
|
||||
throw new TypeError('PostgreSQL cancellation dispatch Run is invalid');
|
||||
}
|
||||
const attempt = await client.query<Row>(
|
||||
`SELECT run_id AS "runId", status
|
||||
FROM "ql3"."run_attempts" WHERE id = $1 FOR UPDATE`,
|
||||
[command.attemptId],
|
||||
);
|
||||
if (attempt.rows.length > 1) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL cancellation dispatch Attempt is invalid',
|
||||
);
|
||||
}
|
||||
const runRow = run.rows[0];
|
||||
const attemptRow = attempt.rows[0];
|
||||
if (
|
||||
!runRow ||
|
||||
!attemptRow ||
|
||||
runRow.executionOwner !== 'runtime' ||
|
||||
!ACTIVE_RUN_STATUSES.has(runRow.status as string) ||
|
||||
optionalInteger(runRow, 'cancelRequestedAtMs') !==
|
||||
command.requestedAtMs ||
|
||||
attemptRow.runId !== command.runId ||
|
||||
!ACTIVE_ATTEMPT_STATUSES.has(attemptRow.status as string)
|
||||
) {
|
||||
return Object.freeze({ status: 'not_eligible' as const });
|
||||
}
|
||||
|
||||
let dispatchResult = await client.query<Row>(
|
||||
`SELECT ${DISPATCH_COLUMNS}
|
||||
FROM "ql3"."run_cancellation_dispatches"
|
||||
WHERE run_id = $1 FOR UPDATE`,
|
||||
[command.runId],
|
||||
);
|
||||
if (dispatchResult.rows.length === 0) {
|
||||
dispatchResult = await client.query<Row>(
|
||||
`INSERT INTO "ql3"."run_cancellation_dispatches" (
|
||||
run_id, attempt_id, status, version, dispatch_count,
|
||||
next_attempt_at_ms, created_at_ms, updated_at_ms
|
||||
) VALUES ($1, $2, 'pending', 0, 0, $3, $4, $4)
|
||||
RETURNING ${DISPATCH_COLUMNS}`,
|
||||
[command.runId, command.attemptId, command.requestedAtMs, nowMs],
|
||||
);
|
||||
}
|
||||
if (dispatchResult.rows.length !== 1) {
|
||||
throw new TypeError('PostgreSQL cancellation dispatch is invalid');
|
||||
}
|
||||
const current = dispatchFromRow(dispatchResult.rows[0]!);
|
||||
if (current.attemptId !== command.attemptId) {
|
||||
throw new CancellationDispatchBindingConflictError(
|
||||
command.runId,
|
||||
command.attemptId,
|
||||
);
|
||||
}
|
||||
if (current.status === 'dispatched' || current.status === 'blocked') {
|
||||
return Object.freeze({ status: current.status, dispatch: current });
|
||||
}
|
||||
if (
|
||||
current.status === 'leased' &&
|
||||
current.leaseExpiresAtMs! > nowMs
|
||||
) {
|
||||
return Object.freeze({ status: 'leased' as const, dispatch: current });
|
||||
}
|
||||
if (
|
||||
current.status !== 'leased' &&
|
||||
current.nextAttemptAtMs! > nowMs
|
||||
) {
|
||||
return Object.freeze({ status: 'not_due' as const, dispatch: current });
|
||||
}
|
||||
if (
|
||||
current.version >= 2_147_483_647 ||
|
||||
current.dispatchCount >= 2_147_483_647
|
||||
) {
|
||||
throw new TypeError('PostgreSQL cancellation dispatch counter overflowed');
|
||||
}
|
||||
const leaseExpiresAtMs = nowMs + command.leaseDurationMs;
|
||||
if (!Number.isSafeInteger(leaseExpiresAtMs)) {
|
||||
throw new TypeError('PostgreSQL cancellation dispatch lease overflowed');
|
||||
}
|
||||
const leaseTokenDigest = digestCancellationDispatchLeaseToken(
|
||||
command.leaseToken,
|
||||
);
|
||||
const claimed = await client.query<Row>(
|
||||
`UPDATE "ql3"."run_cancellation_dispatches"
|
||||
SET status = 'leased', version = version + 1,
|
||||
dispatch_count = dispatch_count + 1,
|
||||
next_attempt_at_ms = NULL, lease_owner = $3,
|
||||
lease_token_digest = $4, lease_expires_at_ms = $5,
|
||||
updated_at_ms = $6
|
||||
WHERE run_id = $1 AND attempt_id = $2 AND version = $7
|
||||
RETURNING ${DISPATCH_COLUMNS}`,
|
||||
[
|
||||
command.runId,
|
||||
command.attemptId,
|
||||
command.owner,
|
||||
leaseTokenDigest,
|
||||
leaseExpiresAtMs,
|
||||
nowMs,
|
||||
current.version,
|
||||
],
|
||||
);
|
||||
if (claimed.rows.length !== 1) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
return Object.freeze({
|
||||
status: 'claimed' as const,
|
||||
dispatch: dispatchFromRow(claimed.rows[0]!),
|
||||
leaseToken: command.leaseToken,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async recordResult(
|
||||
value: Readonly<RecordCancellationDispatchResultCommand>,
|
||||
): Promise<Readonly<RecordCancellationDispatchResult>> {
|
||||
const command = normalizeRecordCancellationDispatchResultCommand(value);
|
||||
return this.transaction(async (client) => {
|
||||
const atMs = await databaseNow(client);
|
||||
const run = await client.query<Row>(
|
||||
`SELECT version, event_sequence AS "eventSequence"
|
||||
FROM "ql3"."runs" WHERE id = $1 FOR UPDATE`,
|
||||
[command.runId],
|
||||
);
|
||||
if (run.rows.length !== 1) {
|
||||
throw new TypeError(
|
||||
'PostgreSQL cancellation dispatch Run disappeared',
|
||||
);
|
||||
}
|
||||
const dispatchResult = await client.query<Row>(
|
||||
`SELECT ${DISPATCH_COLUMNS}
|
||||
FROM "ql3"."run_cancellation_dispatches"
|
||||
WHERE run_id = $1 FOR UPDATE`,
|
||||
[command.runId],
|
||||
);
|
||||
if (dispatchResult.rows.length !== 1) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
const current = dispatchFromRow(dispatchResult.rows[0]!);
|
||||
if (
|
||||
current.attemptId !== command.attemptId ||
|
||||
current.status !== 'leased' ||
|
||||
current.version !== command.expectedVersion ||
|
||||
current.leaseOwner !== command.owner ||
|
||||
current.leaseTokenDigest !==
|
||||
digestCancellationDispatchLeaseToken(command.leaseToken)
|
||||
) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
const runVersion = integer(run.rows[0]!, 'version');
|
||||
const eventSequence = integer(run.rows[0]!, 'eventSequence');
|
||||
if (
|
||||
current.version >= 2_147_483_647 ||
|
||||
runVersion >= 2_147_483_647 ||
|
||||
eventSequence >= 2_147_483_647
|
||||
) {
|
||||
throw new TypeError('PostgreSQL cancellation dispatch counter overflowed');
|
||||
}
|
||||
const nextAttemptAtMs =
|
||||
command.retryDelayMs === undefined
|
||||
? undefined
|
||||
: atMs + command.retryDelayMs;
|
||||
if (
|
||||
nextAttemptAtMs !== undefined &&
|
||||
!Number.isSafeInteger(nextAttemptAtMs)
|
||||
) {
|
||||
throw new TypeError('PostgreSQL cancellation dispatch retry overflowed');
|
||||
}
|
||||
const state = cancellationDispatchResultState(command.result);
|
||||
const nextSequence = eventSequence + 1;
|
||||
const runUpdated = await client.query(
|
||||
`UPDATE "ql3"."runs"
|
||||
SET version = version + 1, event_sequence = $2
|
||||
WHERE id = $1 AND version = $3`,
|
||||
[command.runId, nextSequence, runVersion],
|
||||
);
|
||||
if (runUpdated.rowCount !== 1) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
const dispatchUpdated = await client.query<Row>(
|
||||
`UPDATE "ql3"."run_cancellation_dispatches"
|
||||
SET status = $6, version = version + 1,
|
||||
next_attempt_at_ms = $7, lease_owner = NULL,
|
||||
lease_token_digest = NULL, lease_expires_at_ms = NULL,
|
||||
last_result = $8,
|
||||
last_dispatched_at_ms = CASE
|
||||
WHEN $9::boolean THEN last_dispatched_at_ms ELSE $10 END,
|
||||
updated_at_ms = $10
|
||||
WHERE run_id = $1 AND attempt_id = $2 AND status = 'leased'
|
||||
AND version = $3 AND lease_owner = $4
|
||||
AND lease_token_digest = $5
|
||||
RETURNING ${DISPATCH_COLUMNS}`,
|
||||
[
|
||||
command.runId,
|
||||
command.attemptId,
|
||||
command.expectedVersion,
|
||||
command.owner,
|
||||
current.leaseTokenDigest,
|
||||
state.status,
|
||||
nextAttemptAtMs ?? null,
|
||||
command.result,
|
||||
CONTROLLER_NOT_INVOKED_RESULTS.has(command.result),
|
||||
atMs,
|
||||
],
|
||||
);
|
||||
if (dispatchUpdated.rows.length !== 1) {
|
||||
throw new CancellationDispatchFenceRejectedError(command.runId);
|
||||
}
|
||||
const event: Readonly<RunEventRecord> = Object.freeze({
|
||||
id: command.eventId,
|
||||
runId: command.runId,
|
||||
sequence: nextSequence,
|
||||
type: state.eventType,
|
||||
dedupeKey: `cancel-dispatch:${command.attemptId}:${current.dispatchCount}`,
|
||||
actorType: 'worker',
|
||||
actorId: command.owner,
|
||||
attemptId: command.attemptId,
|
||||
payload: Object.freeze({
|
||||
attempt_id: command.attemptId,
|
||||
dispatch_count: current.dispatchCount,
|
||||
result: command.result,
|
||||
}),
|
||||
createdAtMs: atMs,
|
||||
});
|
||||
await client.query(
|
||||
`INSERT INTO "ql3"."run_events" (
|
||||
id, run_id, sequence, type, dedupe_key, actor_type, actor_id,
|
||||
attempt_id, step_run_id, payload, created_at_ms
|
||||
) VALUES ($1, $2, $3, $4, $5, 'worker', $6, $7, NULL, $8::jsonb, $9)`,
|
||||
[
|
||||
event.id,
|
||||
event.runId,
|
||||
event.sequence,
|
||||
event.type,
|
||||
event.dedupeKey,
|
||||
event.actorId,
|
||||
event.attemptId,
|
||||
JSON.stringify(event.payload),
|
||||
event.createdAtMs,
|
||||
],
|
||||
);
|
||||
return Object.freeze({
|
||||
dispatch: dispatchFromRow(dispatchUpdated.rows[0]!),
|
||||
event,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async transaction<T>(
|
||||
operation: (client: PostgresClient) => Promise<T>,
|
||||
): Promise<T> {
|
||||
let client: PostgresClient | undefined;
|
||||
try {
|
||||
client = await this.pool.connect();
|
||||
await begin(client);
|
||||
const result = await operation(client);
|
||||
await client.query('COMMIT');
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (client) await rollback(client);
|
||||
return repositoryFailure(error);
|
||||
} finally {
|
||||
client?.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import { CAPABILITIES_V64 } from '../../approved-action/pg-0065-approved-action-manual-recovery';
|
||||
import { definePostgresSqlMigration } from '../../migrations/sqlMigration';
|
||||
|
||||
export const POSTGRESQL_CANCELLATION_DISPATCH_TABLE =
|
||||
'run_cancellation_dispatches';
|
||||
|
||||
export const CAPABILITIES_V65 = CAPABILITIES_V64.replace(
|
||||
'"run_core":1,',
|
||||
'"run_cancellation_dispatch":1,"run_core":1,',
|
||||
);
|
||||
|
||||
export const pg0066CancellationDispatchMigration =
|
||||
definePostgresSqlMigration({
|
||||
id: 'pg-0066-cancellation-dispatch',
|
||||
statements: [
|
||||
`CREATE UNIQUE INDEX ql3_run_attempts_run_id_uidx ON "ql3"."run_attempts" (run_id, id)`,
|
||||
`
|
||||
CREATE TABLE "ql3"."${POSTGRESQL_CANCELLATION_DISPATCH_TABLE}" (
|
||||
run_id varchar(36) PRIMARY KEY,
|
||||
attempt_id varchar(36) NOT NULL,
|
||||
status varchar(32) NOT NULL,
|
||||
version integer NOT NULL,
|
||||
dispatch_count integer NOT NULL,
|
||||
next_attempt_at_ms bigint,
|
||||
lease_owner varchar(128),
|
||||
lease_token_digest char(64),
|
||||
lease_expires_at_ms bigint,
|
||||
last_result varchar(32),
|
||||
last_dispatched_at_ms bigint,
|
||||
created_at_ms bigint NOT NULL,
|
||||
updated_at_ms bigint NOT NULL,
|
||||
CONSTRAINT ql3_run_cancellation_dispatch_run_fk
|
||||
FOREIGN KEY (run_id) REFERENCES "ql3"."runs" (id)
|
||||
ON DELETE CASCADE ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_run_cancellation_dispatch_attempt_fk
|
||||
FOREIGN KEY (run_id, attempt_id)
|
||||
REFERENCES "ql3"."run_attempts" (run_id, id)
|
||||
ON DELETE CASCADE ON UPDATE RESTRICT,
|
||||
CONSTRAINT ql3_run_cancellation_dispatch_status_check CHECK (
|
||||
status IN ('pending', 'leased', 'retry_wait', 'dispatched', 'blocked')
|
||||
),
|
||||
CONSTRAINT ql3_run_cancellation_dispatch_result_check CHECK (
|
||||
last_result IS NULL OR last_result IN (
|
||||
'termination_requested', 'already_exited', 'identity_mismatch',
|
||||
'pid_mismatch', 'unsupported', 'invalid', 'controller_missing',
|
||||
'handle_missing', 'dispatch_error'
|
||||
)
|
||||
),
|
||||
CONSTRAINT ql3_run_cancellation_dispatch_counter_check CHECK (
|
||||
version BETWEEN 0 AND 2147483647 AND
|
||||
dispatch_count BETWEEN 0 AND 2147483647 AND
|
||||
version >= dispatch_count AND
|
||||
((status = 'pending' AND version = 0 AND dispatch_count = 0) OR
|
||||
(status <> 'pending' AND dispatch_count >= 1))
|
||||
),
|
||||
CONSTRAINT ql3_run_cancellation_dispatch_time_check CHECK (
|
||||
(next_attempt_at_ms IS NULL OR next_attempt_at_ms >= 0) AND
|
||||
(lease_expires_at_ms IS NULL OR lease_expires_at_ms >= 0) AND
|
||||
(last_dispatched_at_ms IS NULL OR last_dispatched_at_ms >= 0) AND
|
||||
created_at_ms >= 0 AND updated_at_ms >= created_at_ms
|
||||
),
|
||||
CONSTRAINT ql3_run_cancellation_dispatch_lease_digest_check CHECK (
|
||||
lease_token_digest IS NULL OR lease_token_digest ~ '^[0-9a-f]{64}$'
|
||||
),
|
||||
CONSTRAINT ql3_run_cancellation_dispatch_shape_check CHECK (
|
||||
(status = 'leased' AND next_attempt_at_ms IS NULL AND
|
||||
lease_owner IS NOT NULL AND
|
||||
octet_length(lease_owner) BETWEEN 1 AND 128 AND
|
||||
lease_owner !~ '[[:cntrl:]]' AND
|
||||
lease_token_digest IS NOT NULL AND lease_expires_at_ms IS NOT NULL) OR
|
||||
(status IN ('pending', 'retry_wait') AND next_attempt_at_ms IS NOT NULL AND
|
||||
lease_owner IS NULL AND lease_token_digest IS NULL AND
|
||||
lease_expires_at_ms IS NULL) OR
|
||||
(status IN ('dispatched', 'blocked') AND next_attempt_at_ms IS NULL AND
|
||||
lease_owner IS NULL AND lease_token_digest IS NULL AND
|
||||
lease_expires_at_ms IS NULL AND last_result IS NOT NULL)
|
||||
),
|
||||
CONSTRAINT ql3_run_cancellation_dispatch_result_state_check CHECK (
|
||||
(status = 'pending' AND last_result IS NULL) OR
|
||||
(status IN ('leased', 'retry_wait') AND
|
||||
(last_result IS NULL OR last_result IN (
|
||||
'controller_missing', 'handle_missing', 'dispatch_error'
|
||||
))) OR
|
||||
(status = 'dispatched' AND last_result IN (
|
||||
'termination_requested', 'already_exited'
|
||||
)) OR
|
||||
(status = 'blocked' AND last_result IN (
|
||||
'identity_mismatch', 'pid_mismatch', 'unsupported', 'invalid'
|
||||
))
|
||||
)
|
||||
)
|
||||
`.trim(),
|
||||
`CREATE INDEX ql3_run_cancellation_dispatch_due_idx ON "ql3"."${POSTGRESQL_CANCELLATION_DISPATCH_TABLE}" (next_attempt_at_ms, run_id) WHERE status IN ('pending', 'retry_wait')`,
|
||||
`CREATE INDEX ql3_run_cancellation_dispatch_lease_expiry_idx ON "ql3"."${POSTGRESQL_CANCELLATION_DISPATCH_TABLE}" (lease_expires_at_ms, run_id) WHERE status = 'leased'`,
|
||||
`REVOKE ALL ON "ql3"."${POSTGRESQL_CANCELLATION_DISPATCH_TABLE}" FROM PUBLIC, ql3_runtime, ql3_admin, ql3_package_manager, ql3_package_executor, ql3_worker_ingress, ql3_worker_credential_manager, ql3_worker_credential_executor, ql3_automation_manager, ql3_approval_manager, ql3_run_manager`,
|
||||
`GRANT SELECT, INSERT, UPDATE ON "ql3"."${POSTGRESQL_CANCELLATION_DISPATCH_TABLE}" TO ql3_runtime`,
|
||||
`DO $ql3$ BEGIN UPDATE "ql3"."schema_capabilities" SET contract_version = 65, migration_id = 'pg-0066-cancellation-dispatch', capabilities = '${CAPABILITIES_V65}'::jsonb, updated_at_ms = floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint WHERE contract_name = 'control-core' AND contract_version = 64 AND migration_id = 'pg-0065-approved-action-manual-recovery' AND capabilities = '${CAPABILITIES_V64}'::jsonb; IF NOT FOUND THEN RAISE EXCEPTION 'control-core capability is not at version 64' USING ERRCODE = 'check_violation'; END IF; END $ql3$`,
|
||||
],
|
||||
});
|
||||
@@ -4810,6 +4810,7 @@ export const runAttempts = ql3Schema.table(
|
||||
table.runId,
|
||||
table.attempt,
|
||||
),
|
||||
uniqueIndex('ql3_run_attempts_run_id_uidx').on(table.runId, table.id),
|
||||
index('ql3_run_attempts_dispatch_candidates_idx').on(
|
||||
table.status,
|
||||
table.runId,
|
||||
@@ -4830,6 +4831,75 @@ export const runAttempts = ql3Schema.table(
|
||||
],
|
||||
);
|
||||
|
||||
export const runCancellationDispatches = ql3Schema.table(
|
||||
'run_cancellation_dispatches',
|
||||
{
|
||||
runId: varchar('run_id', { length: 36 }).primaryKey(),
|
||||
attemptId: varchar('attempt_id', { length: 36 }).notNull(),
|
||||
status: varchar('status', { length: 32 }).notNull(),
|
||||
version: integer('version').notNull(),
|
||||
dispatchCount: integer('dispatch_count').notNull(),
|
||||
nextAttemptAtMs: bigint('next_attempt_at_ms', { mode: 'number' }),
|
||||
leaseOwner: varchar('lease_owner', { length: 128 }),
|
||||
leaseTokenDigest: char('lease_token_digest', { length: 64 }),
|
||||
leaseExpiresAtMs: bigint('lease_expires_at_ms', { mode: 'number' }),
|
||||
lastResult: varchar('last_result', { length: 32 }),
|
||||
lastDispatchedAtMs: bigint('last_dispatched_at_ms', { mode: 'number' }),
|
||||
createdAtMs: bigint('created_at_ms', { mode: 'number' }).notNull(),
|
||||
updatedAtMs: bigint('updated_at_ms', { mode: 'number' }).notNull(),
|
||||
},
|
||||
(table) => [
|
||||
foreignKey({
|
||||
name: 'ql3_run_cancellation_dispatch_run_fk',
|
||||
columns: [table.runId],
|
||||
foreignColumns: [runs.id],
|
||||
})
|
||||
.onDelete('cascade')
|
||||
.onUpdate('restrict'),
|
||||
foreignKey({
|
||||
name: 'ql3_run_cancellation_dispatch_attempt_fk',
|
||||
columns: [table.runId, table.attemptId],
|
||||
foreignColumns: [runAttempts.runId, runAttempts.id],
|
||||
})
|
||||
.onDelete('cascade')
|
||||
.onUpdate('restrict'),
|
||||
check(
|
||||
'ql3_run_cancellation_dispatch_status_check',
|
||||
sql`${table.status} in ('pending', 'leased', 'retry_wait', 'dispatched', 'blocked')`,
|
||||
),
|
||||
check(
|
||||
'ql3_run_cancellation_dispatch_result_check',
|
||||
sql`${table.lastResult} is null or ${table.lastResult} in ('termination_requested', 'already_exited', 'identity_mismatch', 'pid_mismatch', 'unsupported', 'invalid', 'controller_missing', 'handle_missing', 'dispatch_error')`,
|
||||
),
|
||||
check(
|
||||
'ql3_run_cancellation_dispatch_counter_check',
|
||||
sql`${table.version} between 0 and 2147483647 and ${table.dispatchCount} between 0 and 2147483647 and ${table.version} >= ${table.dispatchCount} and ((${table.status} = 'pending' and ${table.version} = 0 and ${table.dispatchCount} = 0) or (${table.status} <> 'pending' and ${table.dispatchCount} >= 1))`,
|
||||
),
|
||||
check(
|
||||
'ql3_run_cancellation_dispatch_time_check',
|
||||
sql`(${table.nextAttemptAtMs} is null or ${table.nextAttemptAtMs} >= 0) and (${table.leaseExpiresAtMs} is null or ${table.leaseExpiresAtMs} >= 0) and (${table.lastDispatchedAtMs} is null or ${table.lastDispatchedAtMs} >= 0) and ${table.createdAtMs} >= 0 and ${table.updatedAtMs} >= ${table.createdAtMs}`,
|
||||
),
|
||||
check(
|
||||
'ql3_run_cancellation_dispatch_lease_digest_check',
|
||||
sql`${table.leaseTokenDigest} is null or ${table.leaseTokenDigest} ~ '^[0-9a-f]{64}$'`,
|
||||
),
|
||||
check(
|
||||
'ql3_run_cancellation_dispatch_shape_check',
|
||||
sql`(${table.status} = 'leased' and ${table.nextAttemptAtMs} is null and ${table.leaseOwner} is not null and octet_length(${table.leaseOwner}) between 1 and 128 and ${table.leaseOwner} !~ '[[:cntrl:]]' and ${table.leaseTokenDigest} is not null and ${table.leaseExpiresAtMs} is not null) or (${table.status} in ('pending', 'retry_wait') and ${table.nextAttemptAtMs} is not null and ${table.leaseOwner} is null and ${table.leaseTokenDigest} is null and ${table.leaseExpiresAtMs} is null) or (${table.status} in ('dispatched', 'blocked') and ${table.nextAttemptAtMs} is null and ${table.leaseOwner} is null and ${table.leaseTokenDigest} is null and ${table.leaseExpiresAtMs} is null and ${table.lastResult} is not null)`,
|
||||
),
|
||||
check(
|
||||
'ql3_run_cancellation_dispatch_result_state_check',
|
||||
sql`(${table.status} = 'pending' and ${table.lastResult} is null) or (${table.status} in ('leased', 'retry_wait') and (${table.lastResult} is null or ${table.lastResult} in ('controller_missing', 'handle_missing', 'dispatch_error'))) or (${table.status} = 'dispatched' and ${table.lastResult} in ('termination_requested', 'already_exited')) or (${table.status} = 'blocked' and ${table.lastResult} in ('identity_mismatch', 'pid_mismatch', 'unsupported', 'invalid'))`,
|
||||
),
|
||||
index('ql3_run_cancellation_dispatch_due_idx')
|
||||
.on(table.nextAttemptAtMs, table.runId)
|
||||
.where(sql`${table.status} in ('pending', 'retry_wait')`),
|
||||
index('ql3_run_cancellation_dispatch_lease_expiry_idx')
|
||||
.on(table.leaseExpiresAtMs, table.runId)
|
||||
.where(sql`${table.status} = 'leased'`),
|
||||
],
|
||||
);
|
||||
|
||||
export const runAttemptLogRetentionControls = ql3Schema.table(
|
||||
'run_attempt_log_retention_controls',
|
||||
{
|
||||
@@ -6211,6 +6281,7 @@ export const ql3PostgresTables = [
|
||||
toolExecutionResultRekeyHeads,
|
||||
toolResultKeyRetirementReceipts,
|
||||
runAttempts,
|
||||
runCancellationDispatches,
|
||||
runAttemptLogRetentionControls,
|
||||
runAttemptLogArtifactTombstones,
|
||||
workerSessions,
|
||||
|
||||
@@ -21,12 +21,13 @@ export interface PostgresSchemaContractTrigger {
|
||||
export interface PostgresSchemaContract {
|
||||
readonly schema: 'ql3';
|
||||
readonly contractName: 'control-core';
|
||||
readonly contractVersion: 64;
|
||||
readonly migrationId: 'pg-0065-approved-action-manual-recovery';
|
||||
readonly contractVersion: 65;
|
||||
readonly migrationId: 'pg-0066-cancellation-dispatch';
|
||||
readonly minimumServerMajor: 16;
|
||||
readonly maximumServerMajor: 18;
|
||||
readonly capabilities: Readonly<{
|
||||
run_core: 1;
|
||||
run_cancellation_dispatch: 1;
|
||||
run_attempt_log_retention: 1;
|
||||
run_management_boundary: 1;
|
||||
run_management_stop: 1;
|
||||
@@ -118,8 +119,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
Object.freeze({
|
||||
schema: 'ql3',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 64,
|
||||
migrationId: 'pg-0065-approved-action-manual-recovery',
|
||||
contractVersion: 65,
|
||||
migrationId: 'pg-0066-cancellation-dispatch',
|
||||
minimumServerMajor: 16,
|
||||
maximumServerMajor: 18,
|
||||
capabilities: Object.freeze({
|
||||
@@ -167,6 +168,7 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
project_policy: 1,
|
||||
project_tool_definition_snapshot: 1,
|
||||
run_core: 1,
|
||||
run_cancellation_dispatch: 1,
|
||||
run_attempt_log_retention: 1,
|
||||
run_management_boundary: 1,
|
||||
run_management_stop: 1,
|
||||
@@ -1289,6 +1291,21 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
'error_code',
|
||||
'error_summary',
|
||||
]),
|
||||
table('run_cancellation_dispatches', [
|
||||
'run_id',
|
||||
'attempt_id',
|
||||
'status',
|
||||
'version',
|
||||
'dispatch_count',
|
||||
'next_attempt_at_ms',
|
||||
'lease_owner',
|
||||
'lease_token_digest',
|
||||
'lease_expires_at_ms',
|
||||
'last_result',
|
||||
'last_dispatched_at_ms',
|
||||
'created_at_ms',
|
||||
'updated_at_ms',
|
||||
]),
|
||||
table('run_attempt_log_retention_controls', [
|
||||
'attempt_id',
|
||||
'project_id',
|
||||
@@ -1770,9 +1787,13 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
'ql3_result_retirement_catalog_idx',
|
||||
'run_attempts_pkey',
|
||||
'ql3_run_attempts_run_attempt_uidx',
|
||||
'ql3_run_attempts_run_id_uidx',
|
||||
'ql3_run_attempts_dispatch_candidates_idx',
|
||||
'ql3_run_attempts_recovery_idx',
|
||||
'ql3_run_attempts_lease_idx',
|
||||
'run_cancellation_dispatches_pkey',
|
||||
'ql3_run_cancellation_dispatch_due_idx',
|
||||
'ql3_run_cancellation_dispatch_lease_expiry_idx',
|
||||
'run_attempt_log_retention_controls_pkey',
|
||||
'ql3_run_log_retention_control_artifact_key',
|
||||
'ql3_run_log_retention_retry_idx',
|
||||
@@ -2106,6 +2127,13 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
'ql3_runs_cancel_reason_check',
|
||||
'ql3_run_attempts_attempt_check',
|
||||
'ql3_run_attempts_status_check',
|
||||
'ql3_run_cancellation_dispatch_status_check',
|
||||
'ql3_run_cancellation_dispatch_result_check',
|
||||
'ql3_run_cancellation_dispatch_counter_check',
|
||||
'ql3_run_cancellation_dispatch_time_check',
|
||||
'ql3_run_cancellation_dispatch_lease_digest_check',
|
||||
'ql3_run_cancellation_dispatch_shape_check',
|
||||
'ql3_run_cancellation_dispatch_result_state_check',
|
||||
'ql3_run_attempts_pid_check',
|
||||
'ql3_run_attempts_lease_expiry_check',
|
||||
'ql3_run_attempts_worker_session_id_check',
|
||||
@@ -2445,6 +2473,8 @@ export const postgresqlControlSchemaContract: PostgresSchemaContract =
|
||||
'ql3_result_rekey_head_overlay_fk',
|
||||
'ql3_result_retirement_catalog_fk',
|
||||
'ql3_run_attempts_run_fk',
|
||||
'ql3_run_cancellation_dispatch_run_fk',
|
||||
'ql3_run_cancellation_dispatch_attempt_fk',
|
||||
'ql3_run_attempts_step_run_fk',
|
||||
'ql3_run_log_retention_control_attempt_fk',
|
||||
'ql3_run_log_retention_control_run_fk',
|
||||
|
||||
@@ -588,6 +588,12 @@ const REQUIRED_RUNTIME_PRIVILEGES = Object.freeze({
|
||||
update: true,
|
||||
delete: false,
|
||||
}),
|
||||
run_cancellation_dispatches: Object.freeze({
|
||||
select: true,
|
||||
insert: true,
|
||||
update: true,
|
||||
delete: false,
|
||||
}),
|
||||
run_attempt_log_retention_controls: Object.freeze({
|
||||
select: true,
|
||||
insert: true,
|
||||
@@ -1131,6 +1137,12 @@ const REQUIRED_ADMIN_PRIVILEGES = Object.freeze({
|
||||
update: false,
|
||||
delete: false,
|
||||
}),
|
||||
run_cancellation_dispatches: Object.freeze({
|
||||
select: false,
|
||||
insert: false,
|
||||
update: false,
|
||||
delete: false,
|
||||
}),
|
||||
run_attempt_log_retention_controls: Object.freeze({
|
||||
select: false,
|
||||
insert: false,
|
||||
|
||||
@@ -33,6 +33,7 @@ const {
|
||||
PostgresClusterControlRecoveryResolutionRepository,
|
||||
PostgresClusterControlRecoverySource,
|
||||
PostgresClusterRunCancellationConvergenceRepository,
|
||||
PostgresCancellationDispatchRepository,
|
||||
PostgresClusterScheduleRepository,
|
||||
PostgresProjectPolicyRepository,
|
||||
PostgresRunRepository,
|
||||
@@ -41,6 +42,11 @@ const {
|
||||
PostgresWorkerSessionRepository,
|
||||
PostgresRemoteWorkerAttestationEvidenceProvider,
|
||||
} = require('../dist/entrypoints/runtime');
|
||||
const {
|
||||
CancellationDispatchBindingConflictError,
|
||||
CancellationDispatchFenceRejectedError,
|
||||
digestCancellationDispatchLeaseToken,
|
||||
} = require('@qinglong/runtime-core/cancellation-dispatch');
|
||||
const {
|
||||
PostgresTaskDefinitionRepository,
|
||||
PostgresTriggerRepository,
|
||||
@@ -789,6 +795,251 @@ if (!migrationConnectionString) {
|
||||
},
|
||||
});
|
||||
|
||||
test('PostgreSQL cancellation dispatch fences replicas with database time and atomic events', async () => {
|
||||
const runId = '019f7300-0000-7000-8000-000000000901';
|
||||
const attemptId = '019f7300-0000-7000-8000-000000000902';
|
||||
const secondAttemptId = '019f7300-0000-7000-8000-000000000903';
|
||||
const duplicateEventId = '019f7300-0000-7000-8000-000000000904';
|
||||
const retryEventId = '019f7300-0000-7000-8000-000000000905';
|
||||
const terminalEventId = '019f7300-0000-7000-8000-000000000906';
|
||||
const requestedAtMs = 1_750_000_000_100;
|
||||
const migrationDatabase = await open('migration');
|
||||
let firstDatabase;
|
||||
let secondDatabase;
|
||||
try {
|
||||
await runPostgresMigrations({ pool: migrationDatabase.pool });
|
||||
await migrationDatabase.pool.query(
|
||||
'TRUNCATE TABLE "ql3"."run_events", "ql3"."run_retry_policies", "ql3"."run_attempts", "ql3"."runs" CASCADE',
|
||||
);
|
||||
const before = await migrationDatabase.pool.query(
|
||||
`SELECT floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint
|
||||
AS "nowMs"`,
|
||||
);
|
||||
await migrationDatabase.pool.query(
|
||||
`INSERT INTO "ql3"."runs" (
|
||||
id, project_id, task_id, task_revision, trigger_type,
|
||||
execution_origin, execution_owner, status, version,
|
||||
event_sequence, created_at_ms, started_at_ms,
|
||||
cancel_requested_at_ms, cancel_reason
|
||||
) VALUES (
|
||||
$1, 'default', 'cancellation-integration', 'v1', 'manual',
|
||||
'api', 'runtime', 'running', 2, 0, $2, $2, $3, 'user'
|
||||
)`,
|
||||
[runId, requestedAtMs - 100, requestedAtMs],
|
||||
);
|
||||
await migrationDatabase.pool.query(
|
||||
`INSERT INTO "ql3"."run_attempts" (
|
||||
id, run_id, attempt, status, executor_type, callback_sequence,
|
||||
created_at_ms
|
||||
) VALUES ($1, $2, 1, 'running', 'local_process', 0, $3)`,
|
||||
[attemptId, runId, requestedAtMs - 50],
|
||||
);
|
||||
|
||||
[firstDatabase, secondDatabase] = await Promise.all([
|
||||
open('runtime'),
|
||||
open('runtime'),
|
||||
]);
|
||||
const firstRepository = new PostgresCancellationDispatchRepository(
|
||||
firstDatabase.pool,
|
||||
);
|
||||
const secondRepository = new PostgresCancellationDispatchRepository(
|
||||
secondDatabase.pool,
|
||||
);
|
||||
const candidate = {
|
||||
runId,
|
||||
attemptId,
|
||||
requestedAtMs,
|
||||
leaseDurationMs: 10_000,
|
||||
};
|
||||
const [firstClaim, secondClaim] = await Promise.all([
|
||||
firstRepository.claim({
|
||||
...candidate,
|
||||
owner: 'primary-a',
|
||||
leaseToken: 'lease-a',
|
||||
}),
|
||||
secondRepository.claim({
|
||||
...candidate,
|
||||
owner: 'primary-b',
|
||||
leaseToken: 'lease-b',
|
||||
}),
|
||||
]);
|
||||
const claimed = [firstClaim, secondClaim].find(
|
||||
(result) => result.status === 'claimed',
|
||||
);
|
||||
const competing = [firstClaim, secondClaim].find(
|
||||
(result) => result.status !== 'claimed',
|
||||
);
|
||||
assert.equal(claimed?.status, 'claimed');
|
||||
assert.equal(competing?.status, 'leased');
|
||||
assert.equal(claimed.dispatch.version, 1);
|
||||
assert.equal(claimed.dispatch.dispatchCount, 1);
|
||||
assert.equal(claimed.dispatch.createdAtMs >= Number(before.rows[0].nowMs), true);
|
||||
const rawLeaseToken = claimed.leaseToken;
|
||||
const stored = await migrationDatabase.pool.query(
|
||||
`SELECT lease_token_digest AS "leaseTokenDigest",
|
||||
lease_owner AS "leaseOwner", version, dispatch_count
|
||||
AS "dispatchCount"
|
||||
FROM "ql3"."run_cancellation_dispatches" WHERE run_id = $1`,
|
||||
[runId],
|
||||
);
|
||||
assert.equal(
|
||||
stored.rows[0].leaseTokenDigest,
|
||||
digestCancellationDispatchLeaseToken(rawLeaseToken),
|
||||
);
|
||||
assert.notEqual(stored.rows[0].leaseTokenDigest, rawLeaseToken);
|
||||
|
||||
await migrationDatabase.pool.query(
|
||||
`UPDATE "ql3"."run_cancellation_dispatches"
|
||||
SET lease_expires_at_ms = 0 WHERE run_id = $1`,
|
||||
[runId],
|
||||
);
|
||||
const takeover = await secondRepository.claim({
|
||||
...candidate,
|
||||
owner: 'primary-takeover',
|
||||
leaseToken: 'lease-takeover',
|
||||
});
|
||||
assert.equal(takeover.status, 'claimed');
|
||||
assert.equal(takeover.dispatch.version, 2);
|
||||
assert.equal(takeover.dispatch.dispatchCount, 2);
|
||||
await assert.rejects(
|
||||
firstRepository.recordResult({
|
||||
runId,
|
||||
attemptId,
|
||||
owner: claimed.dispatch.leaseOwner,
|
||||
leaseToken: rawLeaseToken,
|
||||
expectedVersion: claimed.dispatch.version,
|
||||
result: 'already_exited',
|
||||
eventId: terminalEventId,
|
||||
}),
|
||||
CancellationDispatchFenceRejectedError,
|
||||
);
|
||||
|
||||
await migrationDatabase.pool.query(
|
||||
`INSERT INTO "ql3"."run_events" (
|
||||
id, run_id, sequence, type, dedupe_key, actor_type, payload,
|
||||
created_at_ms
|
||||
) VALUES ($1, $2, 99, 'fixture.event', 'fixture-event', 'system',
|
||||
'{}'::jsonb, $3)`,
|
||||
[duplicateEventId, runId, requestedAtMs],
|
||||
);
|
||||
await assert.rejects(
|
||||
secondRepository.recordResult({
|
||||
runId,
|
||||
attemptId,
|
||||
owner: 'primary-takeover',
|
||||
leaseToken: 'lease-takeover',
|
||||
expectedVersion: takeover.dispatch.version,
|
||||
result: 'dispatch_error',
|
||||
retryDelayMs: 1_000,
|
||||
eventId: duplicateEventId,
|
||||
}),
|
||||
);
|
||||
const rolledBack = await migrationDatabase.pool.query(
|
||||
`SELECT dispatch.status, dispatch.version, run.version AS "runVersion",
|
||||
run.event_sequence AS "eventSequence"
|
||||
FROM "ql3"."run_cancellation_dispatches" dispatch
|
||||
JOIN "ql3"."runs" run ON run.id = dispatch.run_id
|
||||
WHERE dispatch.run_id = $1`,
|
||||
[runId],
|
||||
);
|
||||
assert.deepEqual(rolledBack.rows, [
|
||||
{ status: 'leased', version: 2, runVersion: 2, eventSequence: 0 },
|
||||
]);
|
||||
|
||||
const retry = await secondRepository.recordResult({
|
||||
runId,
|
||||
attemptId,
|
||||
owner: 'primary-takeover',
|
||||
leaseToken: 'lease-takeover',
|
||||
expectedVersion: takeover.dispatch.version,
|
||||
result: 'dispatch_error',
|
||||
retryDelayMs: 60_000,
|
||||
eventId: retryEventId,
|
||||
});
|
||||
assert.equal(retry.dispatch.status, 'retry_wait');
|
||||
assert.equal(retry.event.type, 'run.cancel_dispatch_failed');
|
||||
assert.equal(
|
||||
(await firstRepository.claim({
|
||||
...candidate,
|
||||
owner: 'primary-a',
|
||||
leaseToken: 'lease-a-retry',
|
||||
})).status,
|
||||
'not_due',
|
||||
);
|
||||
await migrationDatabase.pool.query(
|
||||
`UPDATE "ql3"."run_cancellation_dispatches"
|
||||
SET next_attempt_at_ms = 0 WHERE run_id = $1`,
|
||||
[runId],
|
||||
);
|
||||
const finalLease = await firstRepository.claim({
|
||||
...candidate,
|
||||
owner: 'primary-final',
|
||||
leaseToken: 'lease-final',
|
||||
});
|
||||
assert.equal(finalLease.status, 'claimed');
|
||||
assert.equal(finalLease.dispatch.dispatchCount, 3);
|
||||
|
||||
await migrationDatabase.pool.query(
|
||||
`INSERT INTO "ql3"."run_attempts" (
|
||||
id, run_id, attempt, status, executor_type, callback_sequence,
|
||||
created_at_ms
|
||||
) VALUES ($1, $2, 2, 'running', 'local_process', 0, $3)`,
|
||||
[secondAttemptId, runId, requestedAtMs],
|
||||
);
|
||||
await assert.rejects(
|
||||
secondRepository.claim({
|
||||
...candidate,
|
||||
attemptId: secondAttemptId,
|
||||
owner: 'primary-conflict',
|
||||
leaseToken: 'lease-conflict',
|
||||
}),
|
||||
CancellationDispatchBindingConflictError,
|
||||
);
|
||||
|
||||
const terminal = await firstRepository.recordResult({
|
||||
runId,
|
||||
attemptId,
|
||||
owner: 'primary-final',
|
||||
leaseToken: 'lease-final',
|
||||
expectedVersion: finalLease.dispatch.version,
|
||||
result: 'already_exited',
|
||||
eventId: terminalEventId,
|
||||
});
|
||||
assert.equal(terminal.dispatch.status, 'dispatched');
|
||||
assert.equal(terminal.event.sequence, 2);
|
||||
assert.deepEqual(terminal.event.payload, {
|
||||
attempt_id: attemptId,
|
||||
dispatch_count: 3,
|
||||
result: 'already_exited',
|
||||
});
|
||||
const durable = await migrationDatabase.pool.query(
|
||||
`SELECT dispatch.status, dispatch.lease_token_digest AS "leaseDigest",
|
||||
dispatch.dispatch_count AS "dispatchCount",
|
||||
run.version AS "runVersion",
|
||||
run.event_sequence AS "eventSequence"
|
||||
FROM "ql3"."run_cancellation_dispatches" dispatch
|
||||
JOIN "ql3"."runs" run ON run.id = dispatch.run_id
|
||||
WHERE dispatch.run_id = $1`,
|
||||
[runId],
|
||||
);
|
||||
assert.deepEqual(durable.rows, [
|
||||
{
|
||||
status: 'dispatched',
|
||||
leaseDigest: null,
|
||||
dispatchCount: 3,
|
||||
runVersion: 4,
|
||||
eventSequence: 2,
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
await Promise.allSettled([
|
||||
firstDatabase?.close(),
|
||||
secondDatabase?.close(),
|
||||
]);
|
||||
await migrationDatabase.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('PostgreSQL Task Start atomically persists and exactly replays one Run aggregate', async () => {
|
||||
const projectId = 'task-start-integration';
|
||||
const taskId = 'task-start-command';
|
||||
|
||||
@@ -116,6 +116,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
|
||||
'pg-0063-plugin-package-secret-binding-transition-receipts',
|
||||
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
|
||||
'pg-0065-approved-action-manual-recovery',
|
||||
'pg-0066-cancellation-dispatch',
|
||||
],
|
||||
);
|
||||
for (const migration of postgresqlMainMigrationStream.migrations) {
|
||||
@@ -579,6 +580,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
|
||||
checksum:
|
||||
'95387c5b40659490dbcb7626ecd15bacf6412360752bef88873bde57c43e0185',
|
||||
},
|
||||
{
|
||||
id: 'pg-0066-cancellation-dispatch',
|
||||
checksum:
|
||||
'b6d7ac81b5f75530df05f8ef05878fa30aa0f4418363973ded89d14ffce151b2',
|
||||
},
|
||||
];
|
||||
assert.deepEqual(
|
||||
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
|
||||
@@ -2282,3 +2288,44 @@ test('advances capability v64 with atomic least-privilege manual recovery', asyn
|
||||
/migration_id = 'pg-0064-plugin-package-secret-binding-transition-approval-plans'/,
|
||||
);
|
||||
});
|
||||
|
||||
test('advances capability v65 with database-timed fenced cancellation dispatch', async () => {
|
||||
const migration = migrationById('pg-0066-cancellation-dispatch');
|
||||
const statements = [];
|
||||
await migration.up({
|
||||
async query(statement) {
|
||||
statements.push(statement);
|
||||
return { rows: [] };
|
||||
},
|
||||
});
|
||||
const sql = statements.join('\n');
|
||||
assert.match(
|
||||
sql,
|
||||
/CREATE UNIQUE INDEX ql3_run_attempts_run_id_uidx ON "ql3"\."run_attempts" \(run_id, id\)/,
|
||||
);
|
||||
assert.match(
|
||||
sql,
|
||||
/CREATE TABLE "ql3"\."run_cancellation_dispatches"/,
|
||||
);
|
||||
assert.match(
|
||||
sql,
|
||||
/FOREIGN KEY \(run_id, attempt_id\)[\s\S]+REFERENCES "ql3"\."run_attempts" \(run_id, id\)/,
|
||||
);
|
||||
assert.match(sql, /lease_token_digest char\(64\)/);
|
||||
assert.doesNotMatch(sql, /lease_token varchar/);
|
||||
assert.match(
|
||||
sql,
|
||||
/GRANT SELECT, INSERT, UPDATE ON "ql3"\."run_cancellation_dispatches" TO ql3_runtime/,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
sql,
|
||||
/GRANT (?:SELECT|INSERT|UPDATE|DELETE)[^;]+run_cancellation_dispatches[^;]+ql3_admin/,
|
||||
);
|
||||
assert.match(sql, /contract_version = 65/);
|
||||
assert.match(sql, /"run_cancellation_dispatch":1/);
|
||||
assert.match(sql, /contract_version = 64/);
|
||||
assert.match(
|
||||
sql,
|
||||
/migration_id = 'pg-0065-approved-action-manual-recovery'/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -61,6 +61,7 @@ function validPrivileges() {
|
||||
tool_invocation_input_artifacts: [true, true, false, false],
|
||||
tool_invocation_preview_artifacts: [true, true, false, false],
|
||||
run_attempts: [true, true, true, false],
|
||||
run_cancellation_dispatches: [true, true, true, false],
|
||||
run_attempt_log_retention_controls: [true, true, true, true],
|
||||
run_attempt_log_artifact_tombstones: [true, true, false, false],
|
||||
worker_sessions: [true, true, true, false],
|
||||
@@ -194,6 +195,7 @@ function validAdminPrivileges() {
|
||||
tool_invocation_input_artifacts: [false, false, false, false],
|
||||
tool_invocation_preview_artifacts: [false, false, false, false],
|
||||
run_attempts: [false, false, false, false],
|
||||
run_cancellation_dispatches: [false, false, false, false],
|
||||
run_attempt_log_retention_controls: [false, false, false, false],
|
||||
run_attempt_log_artifact_tombstones: [false, false, false, false],
|
||||
worker_sessions: [false, false, false, false],
|
||||
@@ -815,7 +817,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
serverMajor: 16,
|
||||
currentUser: 'ql3_runtime',
|
||||
contractName: 'control-core',
|
||||
contractVersion: 64,
|
||||
contractVersion: 65,
|
||||
migrationIds: [
|
||||
'pg-0001-schema-capability',
|
||||
'pg-0002-run-core',
|
||||
@@ -882,6 +884,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
|
||||
'pg-0063-plugin-package-secret-binding-transition-receipts',
|
||||
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
|
||||
'pg-0065-approved-action-manual-recovery',
|
||||
'pg-0066-cancellation-dispatch',
|
||||
],
|
||||
});
|
||||
});
|
||||
@@ -912,10 +915,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_admin');
|
||||
assert.equal(report.contractVersion, 64);
|
||||
assert.equal(report.contractVersion, 65);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0065-approved-action-manual-recovery',
|
||||
'pg-0066-cancellation-dispatch',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -928,10 +931,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_automation_manager');
|
||||
assert.equal(report.contractVersion, 64);
|
||||
assert.equal(report.contractVersion, 65);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0065-approved-action-manual-recovery',
|
||||
'pg-0066-cancellation-dispatch',
|
||||
);
|
||||
|
||||
const widened = automationManagerPrivileges();
|
||||
@@ -960,10 +963,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_approval_manager');
|
||||
assert.equal(report.contractVersion, 64);
|
||||
assert.equal(report.contractVersion, 65);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0065-approved-action-manual-recovery',
|
||||
'pg-0066-cancellation-dispatch',
|
||||
);
|
||||
|
||||
const widened = approvalManagerPrivileges();
|
||||
@@ -994,10 +997,10 @@ test('accepts the isolated least-privilege Run manager role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_run_manager');
|
||||
assert.equal(report.contractVersion, 64);
|
||||
assert.equal(report.contractVersion, 65);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0065-approved-action-manual-recovery',
|
||||
'pg-0066-cancellation-dispatch',
|
||||
);
|
||||
|
||||
const widened = runManagerPrivileges();
|
||||
@@ -1129,10 +1132,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
|
||||
}),
|
||||
);
|
||||
assert.equal(report.currentUser, 'ql3_worker_ingress');
|
||||
assert.equal(report.contractVersion, 64);
|
||||
assert.equal(report.contractVersion, 65);
|
||||
assert.equal(
|
||||
report.migrationIds.at(-1),
|
||||
'pg-0065-approved-action-manual-recovery',
|
||||
'pg-0066-cancellation-dispatch',
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user