feat(ql3): establish 3.0 incubation baseline

2026-09-21 18:08:20 +08:00 · 2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
@@ -0,0 +1,409 @@
+require('ts-node/register/transpile-only');
+
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+const {
+  PrimaryCancellationDispatcher,
+} = require('../../back/runtime/application/primaryCancellationDispatcher');
+
+const NOW_MS = 1_750_000_000_100;
+
+function candidate(runId, overrides = {}) {
+  return {
+    runId,
+    requestedAtMs: NOW_MS - 10,
+    reason: 'user',
+    attempts: [
+      {
+        attemptId: `${runId}-attempt`,
+        executorType: 'local_process',
+        executorHandle: `handle:${runId}`,
+        pid: 4001,
+      },
+    ],
+    ...overrides,
+  };
+}
+
+function fakeDispatchRepository(overrides = {}) {
+  const claims = [];
+  const results = [];
+  return {
+    claims,
+    results,
+    async findByRunId() {
+      return null;
+    },
+    async claim(command) {
+      claims.push(command);
+      if (overrides.claim) return overrides.claim(command);
+      return {
+        status: 'claimed',
+        dispatch: {
+          runId: command.runId,
+          attemptId: command.attemptId,
+          status: 'leased',
+          version: 1,
+          dispatchCount: 1,
+          leaseOwner: command.owner,
+          leaseToken: command.leaseToken,
+          leaseExpiresAtMs: command.nowMs + command.leaseDurationMs,
+          createdAtMs: command.nowMs,
+          updatedAtMs: command.nowMs,
+        },
+      };
+    },
+    async recordResult(command) {
+      results.push(command);
+      if (overrides.recordResult) return overrides.recordResult(command);
+      return {
+        dispatch: {
+          runId: command.runId,
+          attemptId: command.attemptId,
+          status: command.nextAttemptAtMs ? 'retry_wait' : 'dispatched',
+          version: command.expectedVersion + 1,
+          dispatchCount: 1,
+          lastResult: command.result,
+          createdAtMs: NOW_MS,
+          updatedAtMs: command.atMs,
+        },
+        event: {
+          id: command.eventId,
+          runId: command.runId,
+          sequence: 1,
+          type: 'fixture',
+          actorType: 'worker',
+          payload: {},
+          createdAtMs: command.atMs,
+        },
+      };
+    },
+  };
+}
+
+function dispatcherOptions(overrides = {}) {
+  let id = 0;
+  return {
+    owner: 'worker-a',
+    leaseDurationMs: 100,
+    retryBaseMs: 1_000,
+    retryMaxMs: 8_000,
+    clock: () => NOW_MS,
+    createId: () => `019f71d0-0000-7000-8000-${String(++id).padStart(12, '0')}`,
+    ...overrides,
+  };
+}
+
+test('leases before signalling and durably classifies every controller result', async () => {
+  const stopped = [];
+  const source = {
+    async listCandidates(options) {
+      assert.deepEqual(options, { limit: 8 });
+      return {
+        candidates: [
+          candidate('run-1'),
+          candidate('run-2'),
+          candidate('run-3', { attempts: [] }),
+          candidate('run-4', {
+            attempts: [
+              {
+                attemptId: 'attempt-4a',
+                executorType: 'local_process',
+                executorHandle: 'handle:4a',
+              },
+              {
+                attemptId: 'attempt-4b',
+                executorType: 'local_process',
+                executorHandle: 'handle:4b',
+              },
+            ],
+          }),
+          candidate('run-5', {
+            attempts: [
+              {
+                attemptId: 'attempt-5',
+                executorType: 'remote_worker',
+                executorHandle: 'remote:5',
+              },
+            ],
+          }),
+          candidate('run-6'),
+          candidate('run-7'),
+          candidate('run-8'),
+        ],
+        truncated: true,
+        unsafeAttemptOverflow: false,
+        nextCursor: { requestedAtMs: NOW_MS - 10, runId: 'run-8' },
+      };
+    },
+  };
+  const dispatches = fakeDispatchRepository({
+    claim(command) {
+      if (command.runId === 'run-8') {
+        return {
+          status: 'leased',
+          dispatch: {
+            runId: command.runId,
+            attemptId: command.attemptId,
+            status: 'leased',
+            version: 3,
+            dispatchCount: 2,
+            leaseOwner: 'worker-b',
+            leaseToken: 'other-lease',
+            leaseExpiresAtMs: NOW_MS + 1_000,
+            createdAtMs: NOW_MS - 5_000,
+            updatedAtMs: NOW_MS - 100,
+          },
+        };
+      }
+      return {
+        status: 'claimed',
+        dispatch: {
+          runId: command.runId,
+          attemptId: command.attemptId,
+          status: 'leased',
+          version: 1,
+          dispatchCount: 1,
+          leaseOwner: command.owner,
+          leaseToken: command.leaseToken,
+          leaseExpiresAtMs: command.nowMs + command.leaseDurationMs,
+          createdAtMs: command.nowMs,
+          updatedAtMs: command.nowMs,
+        },
+      };
+    },
+  });
+  let calls = 0;
+  const controller = {
+    executorType: 'local_process',
+    async stop(input) {
+      calls += 1;
+      stopped.push(input);
+      if (calls === 1) {
+        return {
+          status: 'termination_requested',
+          termSignalSent: true,
+          killSignalSent: false,
+        };
+      }
+      if (calls === 2) {
+        return {
+          status: 'already_exited',
+          termSignalSent: false,
+          killSignalSent: false,
+        };
+      }
+      if (calls === 3) throw new Error('transient stop failure');
+      return {
+        status: 'identity_mismatch',
+        termSignalSent: false,
+        killSignalSent: false,
+      };
+    },
+  };
+  const dispatcher = new PrimaryCancellationDispatcher(
+    source,
+    dispatches,
+    [controller],
+    dispatcherOptions(),
+  );
+
+  const summary = await dispatcher.dispatchBatch({ limit: 8 });
+  assert.deepEqual(summary, {
+    scanned: 8,
+    claimed: 5,
+    terminationRequested: 1,
+    alreadyExited: 1,
+    pending: 5,
+    ambiguous: 1,
+    blocked: 1,
+    deferred: 1,
+    alreadyResolved: 0,
+    notEligible: 0,
+    failed: 1,
+    truncated: true,
+    unsafeAttemptOverflow: false,
+    nextCursor: { requestedAtMs: NOW_MS - 10, runId: 'run-8' },
+  });
+  assert.equal(stopped.length, 4);
+  assert.deepEqual(stopped[0], {
+    durableHandle: 'handle:run-1',
+    expectedPid: 4001,
+    reason: { kind: 'user', requestedAtMs: NOW_MS - 10 },
+  });
+  assert.deepEqual(
+    dispatches.results.map((result) => result.result),
+    [
+      'termination_requested',
+      'already_exited',
+      'controller_missing',
+      'dispatch_error',
+      'identity_mismatch',
+    ],
+  );
+  assert.equal(
+    dispatches.results.find((result) => result.result === 'dispatch_error')
+      .nextAttemptAtMs,
+    NOW_MS + 1_000,
+  );
+});
+
+test('fails closed on unsafe overflow, duplicate controllers, and corrupt reasons', async () => {
+  let stopCalls = 0;
+  const controller = {
+    executorType: 'local_process',
+    async stop() {
+      stopCalls += 1;
+      throw new Error('must not run');
+    },
+  };
+  const dispatches = fakeDispatchRepository();
+  assert.throws(
+    () =>
+      new PrimaryCancellationDispatcher(
+        { async listCandidates() {} },
+        dispatches,
+        [controller, controller],
+        dispatcherOptions(),
+      ),
+    /Duplicate persisted Executor controller/,
+  );
+
+  const overflow = new PrimaryCancellationDispatcher(
+    {
+      async listCandidates() {
+        return {
+          candidates: [],
+          truncated: true,
+          unsafeAttemptOverflow: true,
+        };
+      },
+    },
+    dispatches,
+    [controller],
+    dispatcherOptions(),
+  );
+  const overflowSummary = await overflow.dispatchBatch();
+  assert.equal(overflowSummary.unsafeAttemptOverflow, true);
+
+  const corrupt = new PrimaryCancellationDispatcher(
+    {
+      async listCandidates() {
+        return {
+          candidates: [candidate('run-corrupt', { reason: 'raw-corruption' })],
+          truncated: false,
+          unsafeAttemptOverflow: false,
+        };
+      },
+    },
+    dispatches,
+    [controller],
+    dispatcherOptions(),
+  );
+  const corruptSummary = await corrupt.dispatchBatch();
+  assert.equal(corruptSummary.pending, 1);
+  assert.equal(stopCalls, 0);
+  assert.equal(dispatches.claims.length, 0);
+});
+
+test('reports persisted terminal, deferred, stale, and failed claims without signalling', async () => {
+  const statuses = new Map([
+    ['run-not-due', 'not_due'],
+    ['run-dispatched', 'dispatched'],
+    ['run-blocked', 'blocked'],
+    ['run-stale', 'not_eligible'],
+  ]);
+  const dispatches = fakeDispatchRepository({
+    claim(command) {
+      if (command.runId === 'run-failed') throw new Error('database busy');
+      const status = statuses.get(command.runId);
+      if (status === 'not_eligible') return { status };
+      return {
+        status,
+        dispatch: {
+          runId: command.runId,
+          attemptId: command.attemptId,
+          status: status === 'not_due' ? 'retry_wait' : status,
+          version: 2,
+          dispatchCount: 1,
+          createdAtMs: NOW_MS - 1_000,
+          updatedAtMs: NOW_MS - 500,
+        },
+      };
+    },
+  });
+  let stopCalls = 0;
+  const dispatcher = new PrimaryCancellationDispatcher(
+    {
+      async listCandidates() {
+        return {
+          candidates: [
+            candidate('run-not-due'),
+            candidate('run-dispatched'),
+            candidate('run-blocked'),
+            candidate('run-stale'),
+            candidate('run-failed'),
+          ],
+          truncated: false,
+          unsafeAttemptOverflow: false,
+        };
+      },
+    },
+    dispatches,
+    [
+      {
+        executorType: 'local_process',
+        async stop() {
+          stopCalls += 1;
+          throw new Error('must not run');
+        },
+      },
+    ],
+    dispatcherOptions(),
+  );
+  const summary = await dispatcher.dispatchBatch();
+  assert.equal(summary.deferred, 1);
+  assert.equal(summary.alreadyResolved, 2);
+  assert.equal(summary.blocked, 1);
+  assert.equal(summary.notEligible, 1);
+  assert.equal(summary.failed, 1);
+  assert.equal(summary.pending, 2);
+  assert.equal(stopCalls, 0);
+});
+
+test('counts result persistence failure while retaining the signal outcome', async () => {
+  const dispatches = fakeDispatchRepository({
+    recordResult() {
+      throw new Error('commit failed');
+    },
+  });
+  const dispatcher = new PrimaryCancellationDispatcher(
+    {
+      async listCandidates() {
+        return {
+          candidates: [candidate('run-signal-before-commit')],
+          truncated: false,
+          unsafeAttemptOverflow: false,
+        };
+      },
+    },
+    dispatches,
+    [
+      {
+        executorType: 'local_process',
+        async stop() {
+          return {
+            status: 'termination_requested',
+            termSignalSent: true,
+            killSignalSent: false,
+          };
+        },
+      },
+    ],
+    dispatcherOptions(),
+  );
+  const summary = await dispatcher.dispatchBatch();
+  assert.equal(summary.terminationRequested, 1);
+  assert.equal(summary.failed, 1);
+  assert.equal(summary.pending, 1);
+});