open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-09-22 10:32:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ql3): add postgres cancellation dispatch

Browse Source
This commit is contained in:
whyour
2026-08-19 06:24:11 +08:00
parent 36035ac43e
commit 1809fbb8d3
29 changed files with 2265 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ql3-cluster-postgres/test/postgres.integration.test.cjs
+251
View File
@@ -33,6 +33,7 @@ const {
PostgresClusterControlRecoveryResolutionRepository,
PostgresClusterControlRecoverySource,
PostgresClusterRunCancellationConvergenceRepository,
PostgresCancellationDispatchRepository,
PostgresClusterScheduleRepository,
PostgresProjectPolicyRepository,
PostgresRunRepository,
@@ -41,6 +42,11 @@ const {
PostgresWorkerSessionRepository,
PostgresRemoteWorkerAttestationEvidenceProvider,
} = require('../dist/entrypoints/runtime');
const {
CancellationDispatchBindingConflictError,
CancellationDispatchFenceRejectedError,
digestCancellationDispatchLeaseToken,
} = require('@qinglong/runtime-core/cancellation-dispatch');
const {
PostgresTaskDefinitionRepository,
PostgresTriggerRepository,
@@ -789,6 +795,251 @@ if (!migrationConnectionString) {
},
});
test('PostgreSQL cancellation dispatch fences replicas with database time and atomic events', async () => {
const runId = '019f7300-0000-7000-8000-000000000901';
const attemptId = '019f7300-0000-7000-8000-000000000902';
const secondAttemptId = '019f7300-0000-7000-8000-000000000903';
const duplicateEventId = '019f7300-0000-7000-8000-000000000904';
const retryEventId = '019f7300-0000-7000-8000-000000000905';
const terminalEventId = '019f7300-0000-7000-8000-000000000906';
const requestedAtMs = 1_750_000_000_100;
const migrationDatabase = await open('migration');
let firstDatabase;
let secondDatabase;
try {
await runPostgresMigrations({ pool: migrationDatabase.pool });
await migrationDatabase.pool.query(
'TRUNCATE TABLE "ql3"."run_events", "ql3"."run_retry_policies", "ql3"."run_attempts", "ql3"."runs" CASCADE',
);
const before = await migrationDatabase.pool.query(
`SELECT floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint
AS "nowMs"`,
);
await migrationDatabase.pool.query(
`INSERT INTO "ql3"."runs" (
id, project_id, task_id, task_revision, trigger_type,
execution_origin, execution_owner, status, version,
event_sequence, created_at_ms, started_at_ms,
cancel_requested_at_ms, cancel_reason
) VALUES (
$1, 'default', 'cancellation-integration', 'v1', 'manual',
'api', 'runtime', 'running', 2, 0, $2, $2, $3, 'user'
)`,
[runId, requestedAtMs - 100, requestedAtMs],
);
await migrationDatabase.pool.query(
`INSERT INTO "ql3"."run_attempts" (
id, run_id, attempt, status, executor_type, callback_sequence,
created_at_ms
) VALUES ($1, $2, 1, 'running', 'local_process', 0, $3)`,
[attemptId, runId, requestedAtMs - 50],
);
[firstDatabase, secondDatabase] = await Promise.all([
open('runtime'),
open('runtime'),
]);
const firstRepository = new PostgresCancellationDispatchRepository(
firstDatabase.pool,
);
const secondRepository = new PostgresCancellationDispatchRepository(
secondDatabase.pool,
);
const candidate = {
runId,
attemptId,
requestedAtMs,
leaseDurationMs: 10_000,
};
const [firstClaim, secondClaim] = await Promise.all([
firstRepository.claim({
...candidate,
owner: 'primary-a',
leaseToken: 'lease-a',
}),
secondRepository.claim({
...candidate,
owner: 'primary-b',
leaseToken: 'lease-b',
}),
]);
const claimed = [firstClaim, secondClaim].find(
(result) => result.status === 'claimed',
);
const competing = [firstClaim, secondClaim].find(
(result) => result.status !== 'claimed',
);
assert.equal(claimed?.status, 'claimed');
assert.equal(competing?.status, 'leased');
assert.equal(claimed.dispatch.version, 1);
assert.equal(claimed.dispatch.dispatchCount, 1);
assert.equal(claimed.dispatch.createdAtMs >= Number(before.rows[0].nowMs), true);
const rawLeaseToken = claimed.leaseToken;
const stored = await migrationDatabase.pool.query(
`SELECT lease_token_digest AS "leaseTokenDigest",
lease_owner AS "leaseOwner", version, dispatch_count
AS "dispatchCount"
FROM "ql3"."run_cancellation_dispatches" WHERE run_id = $1`,
[runId],
);
assert.equal(
stored.rows[0].leaseTokenDigest,
digestCancellationDispatchLeaseToken(rawLeaseToken),
);
assert.notEqual(stored.rows[0].leaseTokenDigest, rawLeaseToken);
await migrationDatabase.pool.query(
`UPDATE "ql3"."run_cancellation_dispatches"
SET lease_expires_at_ms = 0 WHERE run_id = $1`,
[runId],
);
const takeover = await secondRepository.claim({
...candidate,
owner: 'primary-takeover',
leaseToken: 'lease-takeover',
});
assert.equal(takeover.status, 'claimed');
assert.equal(takeover.dispatch.version, 2);
assert.equal(takeover.dispatch.dispatchCount, 2);
await assert.rejects(
firstRepository.recordResult({
runId,
attemptId,
owner: claimed.dispatch.leaseOwner,
leaseToken: rawLeaseToken,
expectedVersion: claimed.dispatch.version,
result: 'already_exited',
eventId: terminalEventId,
}),
CancellationDispatchFenceRejectedError,
);
await migrationDatabase.pool.query(
`INSERT INTO "ql3"."run_events" (
id, run_id, sequence, type, dedupe_key, actor_type, payload,
created_at_ms
) VALUES ($1, $2, 99, 'fixture.event', 'fixture-event', 'system',
'{}'::jsonb, $3)`,
[duplicateEventId, runId, requestedAtMs],
);
await assert.rejects(
secondRepository.recordResult({
runId,
attemptId,
owner: 'primary-takeover',
leaseToken: 'lease-takeover',
expectedVersion: takeover.dispatch.version,
result: 'dispatch_error',
retryDelayMs: 1_000,
eventId: duplicateEventId,
}),
);
const rolledBack = await migrationDatabase.pool.query(
`SELECT dispatch.status, dispatch.version, run.version AS "runVersion",
run.event_sequence AS "eventSequence"
FROM "ql3"."run_cancellation_dispatches" dispatch
JOIN "ql3"."runs" run ON run.id = dispatch.run_id
WHERE dispatch.run_id = $1`,
[runId],
);
assert.deepEqual(rolledBack.rows, [
{ status: 'leased', version: 2, runVersion: 2, eventSequence: 0 },
]);
const retry = await secondRepository.recordResult({
runId,
attemptId,
owner: 'primary-takeover',
leaseToken: 'lease-takeover',
expectedVersion: takeover.dispatch.version,
result: 'dispatch_error',
retryDelayMs: 60_000,
eventId: retryEventId,
});
assert.equal(retry.dispatch.status, 'retry_wait');
assert.equal(retry.event.type, 'run.cancel_dispatch_failed');
assert.equal(
(await firstRepository.claim({
...candidate,
owner: 'primary-a',
leaseToken: 'lease-a-retry',
})).status,
'not_due',
);
await migrationDatabase.pool.query(
`UPDATE "ql3"."run_cancellation_dispatches"
SET next_attempt_at_ms = 0 WHERE run_id = $1`,
[runId],
);
const finalLease = await firstRepository.claim({
...candidate,
owner: 'primary-final',
leaseToken: 'lease-final',
});
assert.equal(finalLease.status, 'claimed');
assert.equal(finalLease.dispatch.dispatchCount, 3);
await migrationDatabase.pool.query(
`INSERT INTO "ql3"."run_attempts" (
id, run_id, attempt, status, executor_type, callback_sequence,
created_at_ms
) VALUES ($1, $2, 2, 'running', 'local_process', 0, $3)`,
[secondAttemptId, runId, requestedAtMs],
);
await assert.rejects(
secondRepository.claim({
...candidate,
attemptId: secondAttemptId,
owner: 'primary-conflict',
leaseToken: 'lease-conflict',
}),
CancellationDispatchBindingConflictError,
);
const terminal = await firstRepository.recordResult({
runId,
attemptId,
owner: 'primary-final',
leaseToken: 'lease-final',
expectedVersion: finalLease.dispatch.version,
result: 'already_exited',
eventId: terminalEventId,
});
assert.equal(terminal.dispatch.status, 'dispatched');
assert.equal(terminal.event.sequence, 2);
assert.deepEqual(terminal.event.payload, {
attempt_id: attemptId,
dispatch_count: 3,
result: 'already_exited',
});
const durable = await migrationDatabase.pool.query(
`SELECT dispatch.status, dispatch.lease_token_digest AS "leaseDigest",
dispatch.dispatch_count AS "dispatchCount",
run.version AS "runVersion",
run.event_sequence AS "eventSequence"
FROM "ql3"."run_cancellation_dispatches" dispatch
JOIN "ql3"."runs" run ON run.id = dispatch.run_id
WHERE dispatch.run_id = $1`,
[runId],
);
assert.deepEqual(durable.rows, [
{
status: 'dispatched',
leaseDigest: null,
dispatchCount: 3,
runVersion: 4,
eventSequence: 2,
},
]);
} finally {
await Promise.allSettled([
firstDatabase?.close(),
secondDatabase?.close(),
]);
await migrationDatabase.close();
}
});
test('PostgreSQL Task Start atomically persists and exactly replays one Run aggregate', async () => {
const projectId = 'task-start-integration';
const taskId = 'task-start-command';
packages/ql3-cluster-postgres/test/postgresqlMigrationDefinitions.test.cjs
+47
View File
@@ -116,6 +116,7 @@ test('defines the immutable PostgreSQL capability and Run core stream', async ()
'pg-0063-plugin-package-secret-binding-transition-receipts',
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
'pg-0066-cancellation-dispatch',
],
);
for (const migration of postgresqlMainMigrationStream.migrations) {
@@ -579,6 +580,11 @@ test('freezes every published PostgreSQL migration checksum', () => {
checksum:
'95387c5b40659490dbcb7626ecd15bacf6412360752bef88873bde57c43e0185',
},
{
id: 'pg-0066-cancellation-dispatch',
checksum:
'b6d7ac81b5f75530df05f8ef05878fa30aa0f4418363973ded89d14ffce151b2',
},
];
assert.deepEqual(
postgresqlMainMigrationStream.migrations.map(({ id, checksum }) => ({
@@ -2282,3 +2288,44 @@ test('advances capability v64 with atomic least-privilege manual recovery', asyn
/migration_id = 'pg-0064-plugin-package-secret-binding-transition-approval-plans'/,
);
});
test('advances capability v65 with database-timed fenced cancellation dispatch', async () => {
const migration = migrationById('pg-0066-cancellation-dispatch');
const statements = [];
await migration.up({
async query(statement) {
statements.push(statement);
return { rows: [] };
},
});
const sql = statements.join('\n');
assert.match(
sql,
/CREATE UNIQUE INDEX ql3_run_attempts_run_id_uidx ON "ql3"\."run_attempts" \(run_id, id\)/,
);
assert.match(
sql,
/CREATE TABLE "ql3"\."run_cancellation_dispatches"/,
);
assert.match(
sql,
/FOREIGN KEY \(run_id, attempt_id\)[\s\S]+REFERENCES "ql3"\."run_attempts" \(run_id, id\)/,
);
assert.match(sql, /lease_token_digest char\(64\)/);
assert.doesNotMatch(sql, /lease_token varchar/);
assert.match(
sql,
/GRANT SELECT, INSERT, UPDATE ON "ql3"\."run_cancellation_dispatches" TO ql3_runtime/,
);
assert.doesNotMatch(
sql,
/GRANT (?:SELECT|INSERT|UPDATE|DELETE)[^;]+run_cancellation_dispatches[^;]+ql3_admin/,
);
assert.match(sql, /contract_version = 65/);
assert.match(sql, /"run_cancellation_dispatch":1/);
assert.match(sql, /contract_version = 64/);
assert.match(
sql,
/migration_id = 'pg-0065-approved-action-manual-recovery'/,
);
});
packages/ql3-cluster-postgres/test/postgresqlSchemaReadiness.test.cjs
+14 -11
View File
@@ -61,6 +61,7 @@ function validPrivileges() {
tool_invocation_input_artifacts: [true, true, false, false],
tool_invocation_preview_artifacts: [true, true, false, false],
run_attempts: [true, true, true, false],
run_cancellation_dispatches: [true, true, true, false],
run_attempt_log_retention_controls: [true, true, true, true],
run_attempt_log_artifact_tombstones: [true, true, false, false],
worker_sessions: [true, true, true, false],
@@ -194,6 +195,7 @@ function validAdminPrivileges() {
tool_invocation_input_artifacts: [false, false, false, false],
tool_invocation_preview_artifacts: [false, false, false, false],
run_attempts: [false, false, false, false],
run_cancellation_dispatches: [false, false, false, false],
run_attempt_log_retention_controls: [false, false, false, false],
run_attempt_log_artifact_tombstones: [false, false, false, false],
worker_sessions: [false, false, false, false],
@@ -815,7 +817,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
serverMajor: 16,
currentUser: 'ql3_runtime',
contractName: 'control-core',
contractVersion: 64,
contractVersion: 65,
migrationIds: [
'pg-0001-schema-capability',
'pg-0002-run-core',
@@ -882,6 +884,7 @@ test('accepts the exact PostgreSQL control schema and least-privilege runtime ro
'pg-0063-plugin-package-secret-binding-transition-receipts',
'pg-0064-plugin-package-secret-binding-transition-approval-plans',
'pg-0065-approved-action-manual-recovery',
'pg-0066-cancellation-dispatch',
],
});
});
@@ -912,10 +915,10 @@ test('accepts the exact schema and isolated least-privilege admin role', async (
}),
);
assert.equal(report.currentUser, 'ql3_admin');
assert.equal(report.contractVersion, 64);
assert.equal(report.contractVersion, 65);
assert.equal(
report.migrationIds.at(-1),
'pg-0065-approved-action-manual-recovery',
'pg-0066-cancellation-dispatch',
);
});
@@ -928,10 +931,10 @@ test('accepts the isolated least-privilege automation manager role', async () =>
}),
);
assert.equal(report.currentUser, 'ql3_automation_manager');
assert.equal(report.contractVersion, 64);
assert.equal(report.contractVersion, 65);
assert.equal(
report.migrationIds.at(-1),
'pg-0065-approved-action-manual-recovery',
'pg-0066-cancellation-dispatch',
);
const widened = automationManagerPrivileges();
@@ -960,10 +963,10 @@ test('accepts the isolated least-privilege human Approval manager role', async (
}),
);
assert.equal(report.currentUser, 'ql3_approval_manager');
assert.equal(report.contractVersion, 64);
assert.equal(report.contractVersion, 65);
assert.equal(
report.migrationIds.at(-1),
'pg-0065-approved-action-manual-recovery',
'pg-0066-cancellation-dispatch',
);
const widened = approvalManagerPrivileges();
@@ -994,10 +997,10 @@ test('accepts the isolated least-privilege Run manager role', async () => {
}),
);
assert.equal(report.currentUser, 'ql3_run_manager');
assert.equal(report.contractVersion, 64);
assert.equal(report.contractVersion, 65);
assert.equal(
report.migrationIds.at(-1),
'pg-0065-approved-action-manual-recovery',
'pg-0066-cancellation-dispatch',
);
const widened = runManagerPrivileges();
@@ -1129,10 +1132,10 @@ test('accepts the exact schema and isolated Worker ingress role', async () => {
}),
);
assert.equal(report.currentUser, 'ql3_worker_ingress');
assert.equal(report.contractVersion, 64);
assert.equal(report.contractVersion, 65);
assert.equal(
report.migrationIds.at(-1),
'pg-0065-approved-action-manual-recovery',
'pg-0066-cancellation-dispatch',
);
});