feat(ql3): add postgres cancellation dispatch

test/back/cancellationDispatchRepository.test.cjs

@@ -61,9 +61,15 @@ async function createRepository() {
     logger: { info() {} },
   });
   databases.push(database);
+  let nowMs = 1_750_000_000_100;
   return {
     database,
-    repository: new LegacySequelizeCancellationDispatchRepository(database),
+    repository: new LegacySequelizeCancellationDispatchRepository(database, {
+      clock: () => nowMs,
+    }),
+    setNow(value) {
+      nowMs = value;
+    },
   };
 }
 
@@ -113,7 +119,6 @@ function claim(candidate, overrides = {}) {
     ...candidate,
     owner: 'worker-a',
     leaseToken: 'lease-a',
-    nowMs: candidate.requestedAtMs,
     leaseDurationMs: 50,
     ...overrides,
   };
@@ -124,7 +129,7 @@ afterEach(async () => {
 });
 
 test('fences two workers and lets a second worker recover an expired lease', async () => {
-  const { database, repository } = await createRepository();
+  const { database, repository, setNow } = await createRepository();
   const candidate = await insertCandidate(database);
 
   const first = await repository.claim(claim(candidate));
@@ -133,21 +138,21 @@ test('fences two workers and lets a second worker recover an expired lease', asy
   assert.equal(first.dispatch.dispatchCount, 1);
   assert.equal(first.dispatch.leaseOwner, 'worker-a');
 
+  setNow(candidate.requestedAtMs + 25);
   const competing = await repository.claim(
     claim(candidate, {
       owner: 'worker-b',
       leaseToken: 'lease-b',
-      nowMs: candidate.requestedAtMs + 25,
     }),
   );
   assert.equal(competing.status, 'leased');
   assert.equal(competing.dispatch.leaseOwner, 'worker-a');
 
+  setNow(candidate.requestedAtMs + 50);
   const recovered = await repository.claim(
     claim(candidate, {
       owner: 'worker-b',
       leaseToken: 'lease-b',
-      nowMs: candidate.requestedAtMs + 50,
     }),
   );
   assert.equal(recovered.status, 'claimed');
@@ -155,6 +160,7 @@ test('fences two workers and lets a second worker recover an expired lease', asy
   assert.equal(recovered.dispatch.dispatchCount, 2);
   assert.equal(recovered.dispatch.leaseOwner, 'worker-b');
 
+  setNow(candidate.requestedAtMs + 51);
   await assert.rejects(
     repository.recordResult({
       runId: candidate.runId,
@@ -163,12 +169,12 @@ test('fences two workers and lets a second worker recover an expired lease', asy
       leaseToken: 'lease-a',
       expectedVersion: 1,
       result: 'termination_requested',
-      atMs: candidate.requestedAtMs + 51,
       eventId: nextId(),
     }),
     CancellationDispatchFenceRejectedError,
   );
 
+  setNow(candidate.requestedAtMs + 52);
   const recorded = await repository.recordResult({
     runId: candidate.runId,
     attemptId: candidate.attemptId,
@@ -176,7 +182,6 @@ test('fences two workers and lets a second worker recover an expired lease', asy
     leaseToken: 'lease-b',
     expectedVersion: recovered.dispatch.version,
     result: 'termination_requested',
-    atMs: candidate.requestedAtMs + 52,
     eventId: nextId(),
   });
   assert.equal(recorded.dispatch.status, 'dispatched');
@@ -188,22 +193,23 @@ test('fences two workers and lets a second worker recover an expired lease', asy
     result: 'termination_requested',
   });
 
+  setNow(candidate.requestedAtMs + 100);
   const terminal = await repository.claim(
     claim(candidate, {
       owner: 'worker-c',
       leaseToken: 'lease-c',
-      nowMs: candidate.requestedAtMs + 100,
     }),
   );
   assert.equal(terminal.status, 'dispatched');
 });
 
 test('persists retry backoff and only reclaims when it becomes due', async () => {
-  const { database, repository } = await createRepository();
+  const { database, repository, setNow } = await createRepository();
   const candidate = await insertCandidate(database);
   const leased = await repository.claim(claim(candidate));
   const retryAtMs = candidate.requestedAtMs + 1_000;
 
+  setNow(candidate.requestedAtMs + 1);
   const failed = await repository.recordResult({
     runId: candidate.runId,
     attemptId: candidate.attemptId,
@@ -211,33 +217,33 @@ test('persists retry backoff and only reclaims when it becomes due', async () =>
     leaseToken: 'lease-a',
     expectedVersion: leased.dispatch.version,
     result: 'dispatch_error',
-    atMs: candidate.requestedAtMs + 1,
-    nextAttemptAtMs: retryAtMs,
+    retryDelayMs: retryAtMs - (candidate.requestedAtMs + 1),
     eventId: nextId(),
   });
   assert.equal(failed.dispatch.status, 'retry_wait');
   assert.equal(failed.dispatch.nextAttemptAtMs, retryAtMs);
   assert.equal(failed.event.type, 'run.cancel_dispatch_failed');
 
+  setNow(retryAtMs - 1);
   const early = await repository.claim(
     claim(candidate, {
       owner: 'worker-b',
       leaseToken: 'lease-b',
-      nowMs: retryAtMs - 1,
     }),
   );
   assert.equal(early.status, 'not_due');
 
+  setNow(retryAtMs);
   const retry = await repository.claim(
     claim(candidate, {
       owner: 'worker-b',
       leaseToken: 'lease-b',
-      nowMs: retryAtMs,
     }),
   );
   assert.equal(retry.status, 'claimed');
   assert.equal(retry.dispatch.dispatchCount, 2);
 
+  setNow(retryAtMs + 1);
   const missingController = await repository.recordResult({
     runId: candidate.runId,
     attemptId: candidate.attemptId,
@@ -245,8 +251,7 @@ test('persists retry backoff and only reclaims when it becomes due', async () =>
     leaseToken: 'lease-b',
     expectedVersion: retry.dispatch.version,
     result: 'controller_missing',
-    atMs: retryAtMs + 1,
-    nextAttemptAtMs: retryAtMs + 2_000,
+    retryDelayMs: 1_999,
     eventId: nextId(),
   });
   assert.equal(missingController.dispatch.status, 'retry_wait');
@@ -257,7 +262,7 @@ test('persists retry backoff and only reclaims when it becomes due', async () =>
 });
 
 test('fails closed for stale candidates and conflicting Attempt bindings', async () => {
-  const { database, repository } = await createRepository();
+  const { database, repository, setNow } = await createRepository();
   const stale = await insertCandidate(database, { runStatus: 'succeeded' });
   assert.deepEqual(await repository.claim(claim(stale)), {
     status: 'not_eligible',
@@ -278,6 +283,7 @@ test('fails closed for stale candidates and conflicting Attempt bindings', async
       created_at_ms: candidate.requestedAtMs + 1,
     },
   ]);
+  setNow(candidate.requestedAtMs + 50);
   await assert.rejects(
     repository.claim(
       claim(
@@ -285,7 +291,6 @@ test('fails closed for stale candidates and conflicting Attempt bindings', async
         {
           owner: 'worker-b',
           leaseToken: 'lease-b',
-          nowMs: candidate.requestedAtMs + 50,
         },
       ),
     ),
@@ -294,7 +299,7 @@ test('fails closed for stale candidates and conflicting Attempt bindings', async
 });
 
 test('rolls back dispatch state and Run version when event append fails', async () => {
-  const { database, repository } = await createRepository();
+  const { database, repository, setNow } = await createRepository();
   const candidate = await insertCandidate(database);
   const leased = await repository.claim(claim(candidate));
   const duplicateEventId = nextId();
@@ -311,6 +316,7 @@ test('rolls back dispatch state and Run version when event append fails', async
     },
   ]);
 
+  setNow(candidate.requestedAtMs + 1);
   await assert.rejects(
     repository.recordResult({
       runId: candidate.runId,
@@ -319,7 +325,6 @@ test('rolls back dispatch state and Run version when event append fails', async
       leaseToken: 'lease-a',
       expectedVersion: leased.dispatch.version,
       result: 'already_exited',
-      atMs: candidate.requestedAtMs + 1,
       eventId: duplicateEventId,
     }),
   );
@@ -336,6 +341,7 @@ test('rolls back dispatch state and Run version when event append fails', async
   );
   assert.deepEqual(run, { version: 2, event_sequence: 0 });
 
+  setNow(candidate.requestedAtMs + 2);
   const recovered = await repository.recordResult({
     runId: candidate.runId,
     attemptId: candidate.attemptId,
@@ -343,7 +349,6 @@ test('rolls back dispatch state and Run version when event append fails', async
     leaseToken: 'lease-a',
     expectedVersion: leased.dispatch.version,
     result: 'already_exited',
-    atMs: candidate.requestedAtMs + 2,
     eventId: nextId(),
   });
   assert.equal(recovered.dispatch.status, 'dispatched');
@@ -439,7 +444,6 @@ test('fails closed instead of reclaiming a corrupt persisted lease', async () =>
       claim(candidate, {
         owner: 'worker-b',
         leaseToken: 'lease-b',
-        nowMs: candidate.requestedAtMs,
       }),
     ),
     CancellationDispatchRepositoryError,

test/back/primaryCancellationDispatcher.test.cjs

@@ -39,6 +39,7 @@ function fakeDispatchRepository(overrides = {}) {
       if (overrides.claim) return overrides.claim(command);
       return {
         status: 'claimed',
+        leaseToken: command.leaseToken,
         dispatch: {
           runId: command.runId,
           attemptId: command.attemptId,
@@ -46,10 +47,10 @@ function fakeDispatchRepository(overrides = {}) {
           version: 1,
           dispatchCount: 1,
           leaseOwner: command.owner,
-          leaseToken: command.leaseToken,
-          leaseExpiresAtMs: command.nowMs + command.leaseDurationMs,
-          createdAtMs: command.nowMs,
-          updatedAtMs: command.nowMs,
+          leaseTokenDigest: 'a'.repeat(64),
+          leaseExpiresAtMs: NOW_MS + command.leaseDurationMs,
+          createdAtMs: NOW_MS,
+          updatedAtMs: NOW_MS,
         },
       };
     },
@@ -60,12 +61,12 @@ function fakeDispatchRepository(overrides = {}) {
         dispatch: {
           runId: command.runId,
           attemptId: command.attemptId,
-          status: command.nextAttemptAtMs ? 'retry_wait' : 'dispatched',
+          status: command.retryDelayMs ? 'retry_wait' : 'dispatched',
           version: command.expectedVersion + 1,
           dispatchCount: 1,
           lastResult: command.result,
           createdAtMs: NOW_MS,
-          updatedAtMs: command.atMs,
+          updatedAtMs: NOW_MS,
         },
         event: {
           id: command.eventId,
@@ -74,7 +75,7 @@ function fakeDispatchRepository(overrides = {}) {
           type: 'fixture',
           actorType: 'worker',
           payload: {},
-          createdAtMs: command.atMs,
+          createdAtMs: NOW_MS,
         },
       };
     },
@@ -88,7 +89,6 @@ function dispatcherOptions(overrides = {}) {
     leaseDurationMs: 100,
     retryBaseMs: 1_000,
     retryMaxMs: 8_000,
-    clock: () => NOW_MS,
     createId: () => `019f71d0-0000-7000-8000-${String(++id).padStart(12, '0')}`,
     ...overrides,
   };
@@ -149,7 +149,7 @@ test('leases before signalling and durably classifies every controller result',
             version: 3,
             dispatchCount: 2,
             leaseOwner: 'worker-b',
-            leaseToken: 'other-lease',
+            leaseTokenDigest: 'b'.repeat(64),
             leaseExpiresAtMs: NOW_MS + 1_000,
             createdAtMs: NOW_MS - 5_000,
             updatedAtMs: NOW_MS - 100,
@@ -158,6 +158,7 @@ test('leases before signalling and durably classifies every controller result',
       }
       return {
         status: 'claimed',
+        leaseToken: command.leaseToken,
         dispatch: {
           runId: command.runId,
           attemptId: command.attemptId,
@@ -165,10 +166,10 @@ test('leases before signalling and durably classifies every controller result',
           version: 1,
           dispatchCount: 1,
           leaseOwner: command.owner,
-          leaseToken: command.leaseToken,
-          leaseExpiresAtMs: command.nowMs + command.leaseDurationMs,
-          createdAtMs: command.nowMs,
-          updatedAtMs: command.nowMs,
+          leaseTokenDigest: 'a'.repeat(64),
+          leaseExpiresAtMs: NOW_MS + command.leaseDurationMs,
+          createdAtMs: NOW_MS,
+          updatedAtMs: NOW_MS,
         },
       };
     },
@@ -243,8 +244,8 @@ test('leases before signalling and durably classifies every controller result',
   );
   assert.equal(
     dispatches.results.find((result) => result.result === 'dispatch_error')
-      .nextAttemptAtMs,
-    NOW_MS + 1_000,
+      .retryDelayMs,
+    1_000,
   );
 });
 

test/back/ql3PackageBoundaryAudit.test.cjs

@@ -299,10 +299,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', (
       rootSourceFileRoles: runtimeCore.rootSourceFileRoles,
     },
     {
-      sourceFiles: 168,
+      sourceFiles: 169,
       rootSourceFiles: 1,
       rootSourceLines: 160,
-      nestedSourceFiles: 167,
+      nestedSourceFiles: 168,
       rootSourceFileRoles: { 'index.ts': 'public_export' },
     },
   );
@@ -421,10 +421,10 @@ test('current QL3 workspace has exactly eighteen reviewed package boundaries', (
       rootSourceFileRoles: clusterPostgres.rootSourceFileRoles,
     },
     {
-      sourceFiles: 168,
+      sourceFiles: 170,
       rootSourceFiles: 1,
       rootSourceLines: 126,
-      nestedSourceFiles: 167,
+      nestedSourceFiles: 169,
       rootSourceFileRoles: { 'index.ts': 'public_export' },
     },
   );