feat(ql3): establish 3.0 incubation baseline

test/back/workerExecutionLeaseLossCoordinator.test.cjs

@@ -0,0 +1,251 @@
+require('ts-node/register/transpile-only');
+
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+const {
+  WorkerExecutionLeaseLossCoordinator,
+} = require('../../back/runtime/application/workerExecutionLeaseLossCoordinator');
+const {
+  cloneWorkerExecutionOfferJournalRecord,
+  createWorkerExecutionOfferJournalRecord,
+} = require('../../back/runtime/domain/workerExecutionOffer');
+const {
+  createWorkerExecutionCompletionReceiptAuthentication,
+} = require('../../back/runtime/domain/workerExecutionCompletionReceiptAuthentication');
+const {
+  createExecutionSpecDigest,
+  createRunDispatchOfferId,
+} = require('../../back/runtime/domain/runDispatchOffer');
+
+const START = 1_760_900_000_000;
+const RUN_ID = '019f8300-0000-7000-8000-000000000001';
+const ATTEMPT_ID = '019f8300-0000-7000-8000-000000000002';
+const SESSION_ID = '019f8300-0000-7000-8000-000000000003';
+const RECEIPT_TOKEN = 'receipt_capability_abcdefghijklmnopqrstuvwxyz012345';
+
+function offer() {
+  const lease = {
+    attemptId: ATTEMPT_ID,
+    runId: RUN_ID,
+    status: 'leased',
+    version: 3,
+    leaseGeneration: 2,
+    workerId: 'worker-edge',
+    workerSessionId: SESSION_ID,
+    workerGeneration: 4,
+    leaseToken: 'lease_capability_abcdefghijklmnopqrstuvwxyz0123456',
+    acquiredAtMs: START,
+    renewedAtMs: START + 10,
+    expiresAtMs: START + 1_000,
+    updatedAtMs: START + 10,
+  };
+  const executionSpec = {
+    runId: RUN_ID,
+    attemptId: ATTEMPT_ID,
+    projectId: 'default',
+    taskId: 'task-1',
+    taskRevision: 'revision-1',
+    command: { kind: 'argv', file: '/bin/true', args: [] },
+    environmentPolicy: 'isolated',
+    terminationGraceMs: 1_000,
+  };
+  return {
+    offerId: createRunDispatchOfferId(lease),
+    executionSpecDigest: createExecutionSpecDigest(executionSpec),
+    deliveryKind: 'new_claim',
+    candidate: {
+      runId: RUN_ID,
+      attemptId: ATTEMPT_ID,
+      projectId: 'default',
+      taskId: 'task-1',
+      taskRevision: 'revision-1',
+      executorType: 'remote_worker',
+      priority: 0,
+      queuedAtMs: START,
+      attemptCreatedAtMs: START,
+    },
+    worker: {
+      id: 'worker-edge',
+      sessionId: SESSION_ID,
+      generation: 4,
+    },
+    lease,
+    executionSpec,
+  };
+}
+
+function record(state = 'running_acknowledged') {
+  const initial = createWorkerExecutionOfferJournalRecord(offer(), START);
+  const authentication = createWorkerExecutionCompletionReceiptAuthentication({
+    token: RECEIPT_TOKEN,
+    callbackSequence: 1,
+  });
+  const ownsExecution =
+    state === 'started' ||
+    state === 'running_acknowledged' ||
+    state === 'recovery_required';
+  return cloneWorkerExecutionOfferJournalRecord({
+    ...initial,
+    revision: 5,
+    state,
+    updatedAtMs: START + 20,
+    ...(state === 'launching' || ownsExecution
+      ? {
+          completionReceiptCallbackSequence: authentication.callbackSequence,
+          completionReceiptTokenDigest: authentication.tokenDigest,
+        }
+      : {}),
+    ...(ownsExecution
+      ? {
+          executorHandle: 'ql3lp1.durable-handle',
+          executorStartedAtMs: START + 15,
+        }
+      : {}),
+    ...(state === 'recovery_required'
+      ? { recoveryReason: 'launch_outcome_unknown' }
+      : {}),
+    ...(state === 'completion_acknowledged'
+      ? {
+          completionReceiptCallbackSequence: authentication.callbackSequence,
+          completionReceiptTokenDigest: authentication.tokenDigest,
+          completionAcknowledgedAtMs: START + 20,
+        }
+      : {}),
+  });
+}
+
+class MemoryJournal {
+  constructor(value) {
+    this.value = cloneWorkerExecutionOfferJournalRecord(value);
+    this.replaceCalls = 0;
+  }
+
+  async read(offerId) {
+    if (this.value?.offer.offerId !== offerId) return undefined;
+    return cloneWorkerExecutionOfferJournalRecord(this.value);
+  }
+
+  async replace(value, expectedRevision) {
+    this.replaceCalls += 1;
+    assert.equal(this.value.revision, expectedRevision);
+    this.value = cloneWorkerExecutionOfferJournalRecord(value);
+  }
+}
+
+function loss(overrides = {}) {
+  return {
+    lease: { ...offer().lease, ...overrides.lease },
+    reason: overrides.reason || 'lease_expired',
+  };
+}
+
+function fixture(options = {}) {
+  const journal = new MemoryJournal(options.record || record());
+  const calls = [];
+  const controller = {
+    executorType: 'local_process',
+    async stop(command) {
+      calls.push(command);
+      if (options.barrier) await options.barrier;
+      if (options.stopError) throw options.stopError;
+      return {
+        status: options.stopStatus || 'termination_requested',
+        termSignalSent: options.stopStatus !== 'already_exited',
+        killSignalSent: false,
+      };
+    },
+  };
+  const coordinator = new WorkerExecutionLeaseLossCoordinator(
+    journal,
+    controller,
+    { clock: { now: () => START + 2_000 } },
+  );
+  return { calls, coordinator, journal };
+}
+
+test('stops only the durable execution bound to the exact lost lease', async () => {
+  const context = fixture();
+  const result = await context.coordinator.reconcile(loss());
+  assert.equal(result.status, 'stop_acknowledged');
+  assert.equal(result.stopStatus, 'termination_requested');
+  assert.deepEqual(context.calls, [
+    {
+      durableHandle: 'ql3lp1.durable-handle',
+      reason: { kind: 'reconcile', requestedAtMs: START + 2_000 },
+    },
+  ]);
+  assert.equal(context.journal.value.state, 'recovery_required');
+  assert.equal(
+    context.journal.value.recoveryReason,
+    'lease_lost_local_execution_stopped',
+  );
+  assert.equal(context.journal.value.revision, 6);
+});
+
+test('persists an unverified outcome without claiming that a mismatched identity stopped', async () => {
+  const context = fixture({ stopStatus: 'identity_mismatch' });
+  const result = await context.coordinator.reconcile(
+    loss({ reason: 'worker_session_replaced' }),
+  );
+  assert.equal(result.status, 'stop_unverified');
+  assert.equal(result.stopStatus, 'identity_mismatch');
+  assert.equal(
+    context.journal.value.recoveryReason,
+    'lease_lost_local_execution_unverified',
+  );
+});
+
+test('fails closed when launch ownership has no durable handle', async () => {
+  const context = fixture({ record: record('launching') });
+  const result = await context.coordinator.reconcile(loss());
+  assert.equal(result.status, 'stop_unverified');
+  assert.deepEqual(context.calls, []);
+  assert.equal(
+    context.journal.value.recoveryReason,
+    'lease_lost_local_execution_unverified',
+  );
+});
+
+test('never stops an execution after local completion is acknowledged', async () => {
+  const context = fixture({ record: record('completion_acknowledged') });
+  const result = await context.coordinator.reconcile(loss());
+  assert.equal(result.status, 'already_completed');
+  assert.deepEqual(context.calls, []);
+  assert.equal(context.journal.replaceCalls, 0);
+});
+
+test('treats a persisted loss outcome as idempotent', async () => {
+  const current = record('recovery_required');
+  current.recoveryReason = 'lease_lost_local_execution_stopped';
+  const context = fixture({ record: current });
+  const result = await context.coordinator.reconcile(loss());
+  assert.equal(result.status, 'already_stopped');
+  assert.deepEqual(context.calls, []);
+  assert.equal(context.journal.replaceCalls, 0);
+});
+
+test('coalesces concurrent loss callbacks for one offer', async () => {
+  let release;
+  const barrier = new Promise((resolve) => {
+    release = resolve;
+  });
+  const context = fixture({ barrier });
+  const first = context.coordinator.reconcile(loss());
+  const second = context.coordinator.reconcile(
+    loss({ reason: 'worker_unavailable' }),
+  );
+  release();
+  assert.strictEqual(await first, await second);
+  assert.equal(context.calls.length, 1);
+  assert.equal(context.journal.replaceCalls, 1);
+});
+
+test('does not persist a stopped claim when process control fails', async () => {
+  const context = fixture({ stopError: new Error('controller unavailable') });
+  await assert.rejects(
+    context.coordinator.reconcile(loss()),
+    /controller unavailable/,
+  );
+  assert.equal(context.journal.value.state, 'running_acknowledged');
+  assert.equal(context.journal.replaceCalls, 0);
+});