mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-20 16:07:11 +08:00
feat(ql3): add postgres cancellation dispatch
This commit is contained in:
whyour
@@ -0,0 +1,280 @@
|
||||
const assert = require('node:assert/strict');
|
||||
|
||||
const {
|
||||
createPostgresDatabaseOpener,
|
||||
PostgresCancellationDispatchRepository,
|
||||
} = require('../packages/ql3-cluster-postgres/dist/entrypoints/runtime.js');
|
||||
const {
|
||||
CancellationDispatchFenceRejectedError,
|
||||
digestCancellationDispatchLeaseToken,
|
||||
} = require('../packages/ql3-runtime-core/dist/run/cancellation-dispatch/cancellationDispatch.js');
|
||||
|
||||
const FIXTURE = Object.freeze({
|
||||
runId: 'ha-cancel-run-d363',
|
||||
attemptId: 'ha-cancel-attempt-d363',
|
||||
requestedAtMs: 1_750_000_000_100,
|
||||
retryEventId: 'ha-cancel-retry-event-d363',
|
||||
terminalEventId: 'ha-cancel-terminal-event-d363',
|
||||
settledEventId: 'ha-cancel-settled-event-d363',
|
||||
});
|
||||
|
||||
async function openRuntime(connectionString, applicationName) {
|
||||
return createPostgresDatabaseOpener({
|
||||
role: 'runtime',
|
||||
connection: { connectionString, tls: { mode: 'disable' } },
|
||||
pool: { maxConnections: 2, applicationName },
|
||||
onPoolError(error) {
|
||||
throw error;
|
||||
},
|
||||
})();
|
||||
}
|
||||
|
||||
async function cancellationDispatchFacts(pool) {
|
||||
const result = await pool.query(
|
||||
`SELECT dispatch.status, dispatch.version,
|
||||
dispatch.dispatch_count AS "dispatchCount",
|
||||
dispatch.lease_token_digest AS "leaseTokenDigest",
|
||||
dispatch.last_result AS "lastResult",
|
||||
run.version AS "runVersion",
|
||||
run.event_sequence AS "eventSequence",
|
||||
count(event.id)::integer AS "eventCount"
|
||||
FROM "ql3"."run_cancellation_dispatches" dispatch
|
||||
JOIN "ql3"."runs" run ON run.id = dispatch.run_id
|
||||
LEFT JOIN "ql3"."run_events" event ON event.run_id = run.id
|
||||
WHERE dispatch.run_id = $1
|
||||
GROUP BY dispatch.run_id, run.id`,
|
||||
[FIXTURE.runId],
|
||||
);
|
||||
assert.equal(result.rows.length, 1);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async function persistCancellationDispatchHaFixture(options) {
|
||||
const { migrationPool, runtimeConnectionString } = options;
|
||||
const beforeClock = await migrationPool.query(
|
||||
`SELECT floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint
|
||||
AS "nowMs"`,
|
||||
);
|
||||
await migrationPool.query(
|
||||
`INSERT INTO "ql3"."runs" (
|
||||
id, project_id, task_id, task_revision, trigger_type,
|
||||
execution_origin, execution_owner, status, version, event_sequence,
|
||||
created_at_ms, started_at_ms, cancel_requested_at_ms, cancel_reason
|
||||
) VALUES (
|
||||
$1, 'default', 'ha-cancel-task', 'v1', 'manual', 'api', 'runtime',
|
||||
'running', 2, 0, $2, $2, $3, 'user'
|
||||
)`,
|
||||
[FIXTURE.runId, FIXTURE.requestedAtMs - 100, FIXTURE.requestedAtMs],
|
||||
);
|
||||
await migrationPool.query(
|
||||
`INSERT INTO "ql3"."run_attempts" (
|
||||
id, run_id, attempt, status, executor_type, callback_sequence,
|
||||
created_at_ms
|
||||
) VALUES ($1, $2, 1, 'running', 'local_process', 0, $3)`,
|
||||
[FIXTURE.attemptId, FIXTURE.runId, FIXTURE.requestedAtMs - 50],
|
||||
);
|
||||
|
||||
const [firstDatabase, secondDatabase] = await Promise.all([
|
||||
openRuntime(runtimeConnectionString, 'ql3-ha-cancel-primary-a'),
|
||||
openRuntime(runtimeConnectionString, 'ql3-ha-cancel-primary-b'),
|
||||
]);
|
||||
try {
|
||||
const first = new PostgresCancellationDispatchRepository(
|
||||
firstDatabase.pool,
|
||||
);
|
||||
const second = new PostgresCancellationDispatchRepository(
|
||||
secondDatabase.pool,
|
||||
);
|
||||
const candidate = {
|
||||
runId: FIXTURE.runId,
|
||||
attemptId: FIXTURE.attemptId,
|
||||
requestedAtMs: FIXTURE.requestedAtMs,
|
||||
leaseDurationMs: 30_000,
|
||||
};
|
||||
const claims = await Promise.all([
|
||||
first.claim({
|
||||
...candidate,
|
||||
owner: 'ha-cancel-primary-a',
|
||||
leaseToken: 'ha-cancel-lease-a',
|
||||
}),
|
||||
second.claim({
|
||||
...candidate,
|
||||
owner: 'ha-cancel-primary-b',
|
||||
leaseToken: 'ha-cancel-lease-b',
|
||||
}),
|
||||
]);
|
||||
const claimed = claims.find((result) => result.status === 'claimed');
|
||||
const competing = claims.find((result) => result.status !== 'claimed');
|
||||
assert.equal(claimed?.status, 'claimed');
|
||||
assert.equal(competing?.status, 'leased');
|
||||
assert.equal(
|
||||
claimed.dispatch.createdAtMs >= Number(beforeClock.rows[0].nowMs),
|
||||
true,
|
||||
);
|
||||
const storedLease = await migrationPool.query(
|
||||
`SELECT lease_token_digest AS "leaseTokenDigest"
|
||||
FROM "ql3"."run_cancellation_dispatches" WHERE run_id = $1`,
|
||||
[FIXTURE.runId],
|
||||
);
|
||||
assert.equal(
|
||||
storedLease.rows[0].leaseTokenDigest,
|
||||
digestCancellationDispatchLeaseToken(claimed.leaseToken),
|
||||
);
|
||||
assert.notEqual(storedLease.rows[0].leaseTokenDigest, claimed.leaseToken);
|
||||
|
||||
await migrationPool.query(
|
||||
`UPDATE "ql3"."run_cancellation_dispatches"
|
||||
SET lease_expires_at_ms = 0 WHERE run_id = $1`,
|
||||
[FIXTURE.runId],
|
||||
);
|
||||
const takeover = await second.claim({
|
||||
...candidate,
|
||||
owner: 'ha-cancel-takeover',
|
||||
leaseToken: 'ha-cancel-takeover-token',
|
||||
});
|
||||
assert.equal(takeover.status, 'claimed');
|
||||
assert.equal(takeover.dispatch.version, 2);
|
||||
await assert.rejects(
|
||||
first.recordResult({
|
||||
runId: FIXTURE.runId,
|
||||
attemptId: FIXTURE.attemptId,
|
||||
owner: claimed.dispatch.leaseOwner,
|
||||
leaseToken: claimed.leaseToken,
|
||||
expectedVersion: claimed.dispatch.version,
|
||||
result: 'already_exited',
|
||||
eventId: FIXTURE.terminalEventId,
|
||||
}),
|
||||
CancellationDispatchFenceRejectedError,
|
||||
);
|
||||
|
||||
const retry = await second.recordResult({
|
||||
runId: FIXTURE.runId,
|
||||
attemptId: FIXTURE.attemptId,
|
||||
owner: 'ha-cancel-takeover',
|
||||
leaseToken: 'ha-cancel-takeover-token',
|
||||
expectedVersion: takeover.dispatch.version,
|
||||
result: 'dispatch_error',
|
||||
retryDelayMs: 60_000,
|
||||
eventId: FIXTURE.retryEventId,
|
||||
});
|
||||
assert.equal(retry.dispatch.status, 'retry_wait');
|
||||
assert.equal(
|
||||
(await first.claim({
|
||||
...candidate,
|
||||
owner: 'ha-cancel-early',
|
||||
leaseToken: 'ha-cancel-early-token',
|
||||
})).status,
|
||||
'not_due',
|
||||
);
|
||||
await migrationPool.query(
|
||||
`UPDATE "ql3"."run_cancellation_dispatches"
|
||||
SET next_attempt_at_ms = 0 WHERE run_id = $1`,
|
||||
[FIXTURE.runId],
|
||||
);
|
||||
const finalLease = await first.claim({
|
||||
...candidate,
|
||||
owner: 'ha-cancel-final',
|
||||
leaseToken: 'ha-cancel-final-token',
|
||||
});
|
||||
assert.equal(finalLease.status, 'claimed');
|
||||
assert.equal(finalLease.dispatch.dispatchCount, 3);
|
||||
const terminal = await first.recordResult({
|
||||
runId: FIXTURE.runId,
|
||||
attemptId: FIXTURE.attemptId,
|
||||
owner: 'ha-cancel-final',
|
||||
leaseToken: 'ha-cancel-final-token',
|
||||
expectedVersion: finalLease.dispatch.version,
|
||||
result: 'already_exited',
|
||||
eventId: FIXTURE.terminalEventId,
|
||||
});
|
||||
assert.equal(terminal.dispatch.status, 'dispatched');
|
||||
assert.equal(terminal.event.sequence, 2);
|
||||
await migrationPool.query(
|
||||
`WITH observed AS (
|
||||
SELECT floor(extract(epoch FROM transaction_timestamp()) * 1000)::bigint
|
||||
AS at_ms
|
||||
), closed_attempt AS (
|
||||
UPDATE "ql3"."run_attempts"
|
||||
SET status = 'cancelled', finished_at_ms = observed.at_ms
|
||||
FROM observed
|
||||
WHERE id = $2 AND run_id = $1
|
||||
RETURNING id
|
||||
), closed_run AS (
|
||||
UPDATE "ql3"."runs"
|
||||
SET status = 'cancelled', version = version + 1,
|
||||
event_sequence = event_sequence + 1,
|
||||
finished_at_ms = observed.at_ms
|
||||
FROM observed
|
||||
WHERE id = $1
|
||||
AND EXISTS (SELECT 1 FROM closed_attempt)
|
||||
RETURNING id, event_sequence, finished_at_ms
|
||||
)
|
||||
INSERT INTO "ql3"."run_events" (
|
||||
id, run_id, sequence, type, dedupe_key, actor_type, actor_id,
|
||||
attempt_id, payload, created_at_ms
|
||||
)
|
||||
SELECT $3, id, event_sequence, 'run.cancelled',
|
||||
'ha-cancel-fixture-settled', 'system', 'ha-contract', $2,
|
||||
'{"reason":"fixture_settled"}'::jsonb, finished_at_ms
|
||||
FROM closed_run`,
|
||||
[FIXTURE.runId, FIXTURE.attemptId, FIXTURE.settledEventId],
|
||||
);
|
||||
const beforePromotion = await cancellationDispatchFacts(migrationPool);
|
||||
assert.deepEqual(beforePromotion, {
|
||||
status: 'dispatched',
|
||||
version: 5,
|
||||
dispatchCount: 3,
|
||||
leaseTokenDigest: null,
|
||||
lastResult: 'already_exited',
|
||||
runVersion: 5,
|
||||
eventSequence: 3,
|
||||
eventCount: 3,
|
||||
});
|
||||
return {
|
||||
fixture: FIXTURE,
|
||||
beforePromotion,
|
||||
databaseTimed: true,
|
||||
crossPoolClaimExactlyOnce: true,
|
||||
rawLeaseTokenNeverStored: true,
|
||||
expiredLeaseTakenOver: true,
|
||||
staleLeaseFenced: true,
|
||||
retryDeferredUntilDue: true,
|
||||
replicatedBeforePromotion: false,
|
||||
survivedPromotion: false,
|
||||
};
|
||||
} finally {
|
||||
await Promise.allSettled([
|
||||
firstDatabase.close(),
|
||||
secondDatabase.close(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyPromotedCancellationDispatchHaFixture(options) {
|
||||
const { promotedPool, runtimeConnectionString, evidence } = options;
|
||||
const afterPromotion = await cancellationDispatchFacts(promotedPool);
|
||||
assert.deepEqual(afterPromotion, evidence.beforePromotion);
|
||||
const runtimeDatabase = await openRuntime(
|
||||
runtimeConnectionString,
|
||||
'ql3-ha-cancel-promoted',
|
||||
);
|
||||
try {
|
||||
const dispatch = await new PostgresCancellationDispatchRepository(
|
||||
runtimeDatabase.pool,
|
||||
).findByRunId(FIXTURE.runId);
|
||||
assert.equal(dispatch?.status, 'dispatched');
|
||||
assert.equal(dispatch?.dispatchCount, 3);
|
||||
assert.equal(dispatch?.leaseTokenDigest, undefined);
|
||||
} finally {
|
||||
await runtimeDatabase.close();
|
||||
}
|
||||
evidence.afterPromotion = afterPromotion;
|
||||
evidence.survivedPromotion = true;
|
||||
return evidence;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
cancellationDispatchFacts,
|
||||
persistCancellationDispatchHaFixture,
|
||||
verifyPromotedCancellationDispatchHaFixture,
|
||||
};
|
||||
@@ -374,6 +374,11 @@ const {
|
||||
persistNonEmptyToolResultRetirement,
|
||||
verifyPromotedNonEmptyToolResult,
|
||||
} = require('./ql3-postgres-ha-tool-result-fixture.cjs');
|
||||
const {
|
||||
cancellationDispatchFacts,
|
||||
persistCancellationDispatchHaFixture,
|
||||
verifyPromotedCancellationDispatchHaFixture,
|
||||
} = require('./ql3-postgres-ha-cancellation-dispatch-fixture.cjs');
|
||||
const {
|
||||
activateInstall,
|
||||
pluginPackageTaskReconciliationFixture,
|
||||
@@ -11399,6 +11404,7 @@ async function main(argv = process.argv.slice(2)) {
|
||||
let runAttemptLogRetentionEvidence;
|
||||
let runAttemptLogRetention;
|
||||
let manualRunRetry;
|
||||
let cancellationDispatch;
|
||||
const startedAt = performance.now();
|
||||
const timeline = [];
|
||||
let report;
|
||||
@@ -11772,6 +11778,18 @@ async function main(argv = process.argv.slice(2)) {
|
||||
state: 'durable_identity_keyset_ledger_verified',
|
||||
atMs: Number((performance.now() - startedAt).toFixed(3)),
|
||||
});
|
||||
cancellationDispatch = await persistCancellationDispatchHaFixture({
|
||||
migrationPool: primaryDatabase.pool,
|
||||
runtimeConnectionString: databaseUrl(
|
||||
RUNTIME_USER,
|
||||
RUNTIME_PASSWORD,
|
||||
primaryPort,
|
||||
),
|
||||
});
|
||||
timeline.push({
|
||||
state: 'cancellation_dispatch_fenced_on_primary',
|
||||
atMs: Number((performance.now() - startedAt).toFixed(3)),
|
||||
});
|
||||
await primaryDatabase.pool.query(
|
||||
`CREATE ROLE ${REPLICATION_USER} WITH REPLICATION LOGIN`,
|
||||
);
|
||||
@@ -12409,6 +12427,14 @@ async function main(argv = process.argv.slice(2)) {
|
||||
);
|
||||
return marker.rows[0]?.count === 1 ? marker.rows[0] : null;
|
||||
}, 'pre-promotion marker WAL replay');
|
||||
await waitFor(async () => {
|
||||
const facts = await cancellationDispatchFacts(standbyDatabase.pool);
|
||||
return JSON.stringify(facts) ===
|
||||
JSON.stringify(cancellationDispatch.beforePromotion)
|
||||
? facts
|
||||
: null;
|
||||
}, 'cancellation dispatch WAL replay');
|
||||
cancellationDispatch.replicatedBeforePromotion = true;
|
||||
const projectToolSnapshotProjectId = 'ha-tool-snapshot';
|
||||
await primaryDatabase.pool.query(
|
||||
`INSERT INTO "ql3"."projects" (
|
||||
@@ -12993,6 +13019,15 @@ async function main(argv = process.argv.slice(2)) {
|
||||
modelInvocationFeaturePromotion.afterPromotion =
|
||||
promotedModelInvocationFeature;
|
||||
modelInvocationFeaturePromotion.survivedPromotion = true;
|
||||
await verifyPromotedCancellationDispatchHaFixture({
|
||||
promotedPool: promotedDatabase.pool,
|
||||
runtimeConnectionString: databaseUrl(
|
||||
RUNTIME_USER,
|
||||
RUNTIME_PASSWORD,
|
||||
standbyPort,
|
||||
),
|
||||
evidence: cancellationDispatch,
|
||||
});
|
||||
const promotedCopilotFailureDiagnosisAdmission =
|
||||
await copilotFailureDiagnosisAdmissionFacts(
|
||||
promotedDatabase.pool,
|
||||
@@ -13788,8 +13823,10 @@ async function main(argv = process.argv.slice(2)) {
|
||||
]);
|
||||
const sideEffects = await promotedDatabase.pool.query(
|
||||
`SELECT
|
||||
(SELECT count(*)::integer FROM "ql3"."runs") AS runs,
|
||||
(SELECT count(*)::integer FROM "ql3"."run_events") AS "runEvents",
|
||||
(SELECT count(*)::integer FROM "ql3"."runs"
|
||||
WHERE id <> $3) AS runs,
|
||||
(SELECT count(*)::integer FROM "ql3"."run_events"
|
||||
WHERE run_id <> $3) AS "runEvents",
|
||||
(SELECT count(*)::integer
|
||||
FROM "ql3"."runs"
|
||||
WHERE id IN ($1, $2)) AS "diagnosisReadRuns",
|
||||
@@ -13801,6 +13838,7 @@ async function main(argv = process.argv.slice(2)) {
|
||||
[
|
||||
copilotFailureDiagnosisRead.sourceRunId,
|
||||
copilotFailureDiagnosisRead.runId,
|
||||
cancellationDispatch.fixture.runId,
|
||||
],
|
||||
);
|
||||
assert.deepEqual(sideEffects.rows, [
|
||||
@@ -13879,6 +13917,7 @@ async function main(argv = process.argv.slice(2)) {
|
||||
unexpectedDomainSideEffects: 0,
|
||||
},
|
||||
manualRunRetry: manualRunRetry.report,
|
||||
cancellationDispatch,
|
||||
transactionWindows: {
|
||||
ambiguousCommit: {
|
||||
clientObservedFailure: ambiguousCommitClientRejected,
|
||||
@@ -13934,6 +13973,16 @@ async function main(argv = process.argv.slice(2)) {
|
||||
runAttemptLogRetention,
|
||||
timeline,
|
||||
gates: {
|
||||
cancellationDispatchUsesDatabaseTimeAndExactFences:
|
||||
cancellationDispatch.databaseTimed &&
|
||||
cancellationDispatch.crossPoolClaimExactlyOnce &&
|
||||
cancellationDispatch.rawLeaseTokenNeverStored &&
|
||||
cancellationDispatch.expiredLeaseTakenOver &&
|
||||
cancellationDispatch.staleLeaseFenced &&
|
||||
cancellationDispatch.retryDeferredUntilDue,
|
||||
cancellationDispatchReplicatesAndSurvivesPromotion:
|
||||
cancellationDispatch.replicatedBeforePromotion &&
|
||||
cancellationDispatch.survivedPromotion,
|
||||
runAttemptLogRetentionLeaseTakeoverAndTombstoneConverge:
|
||||
runAttemptLogRetention.replicatedBeforePromotion &&
|
||||
runAttemptLogRetention.stalePrimarySettlementFenced &&
|
||||
|
||||
Reference in New Issue
Block a user