qinglong/test/back/workerExecutionOfferReceiver.test.cjs

require('ts-node/register/transpile-only');

const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const {
  WorkerExecutionOfferFileJournal,
} = require('../../back/runtime/adapters/fs/workerExecutionOfferFileJournal');
const {
  WorkerExecutionOfferReceiver,
  WorkerExecutionOfferExpiredError,
  WorkerExecutionOfferTargetError,
} = require('../../back/runtime/application/workerExecutionOfferReceiver');
const {
  WorkerExecutionOfferConflictError,
  createWorkerExecutionOfferJournalRecord,
} = require('../../back/runtime/domain/workerExecutionOffer');
const {
  createWorkerExecutionCompletionReceiptAuthentication,
} = require('../../back/runtime/domain/workerExecutionCompletionReceiptAuthentication');
const {
  createExecutionSpecDigest,
  createRunDispatchOfferId,
} = require('../../back/runtime/domain/runDispatchOffer');

const START = 1_760_400_000_000;
const SESSION = '019f7e00-0000-7000-8000-000000000001';
const REPLACEMENT_SESSION = '019f7e00-0000-7000-8000-000000000002';
const COMPLETION_TOKEN =
  'worker_completion_capability_abcdefghijklmnopqrstuvwxyz01';

function offer(overrides = {}) {
  const lease = {
    attemptId: 'attempt-1',
    runId: 'run-1',
    status: 'leased',
    version: 0,
    leaseGeneration: 1,
    workerId: 'worker-edge',
    workerSessionId: SESSION,
    workerGeneration: 1,
    leaseToken: 'lease_token_abcdefghijklmnopqrstuvwxyz0123456789',
    acquiredAtMs: START,
    renewedAtMs: START,
    expiresAtMs: START + 60_000,
    updatedAtMs: START,
    ...overrides.lease,
  };
  const executionSpec = {
    runId: lease.runId,
    attemptId: lease.attemptId,
    projectId: 'default',
    taskId: 'task-1',
    taskRevision: 'revision-1',
    command: { kind: 'argv', file: '/bin/true', args: [] },
    environmentPolicy: 'isolated',
    terminationGraceMs: 1_000,
    ...overrides.executionSpec,
  };
  const candidate = {
    runId: lease.runId,
    attemptId: lease.attemptId,
    projectId: executionSpec.projectId,
    taskId: executionSpec.taskId,
    taskRevision: executionSpec.taskRevision,
    executorType: 'remote_worker',
    priority: 0,
    queuedAtMs: START,
    attemptCreatedAtMs: START,
    ...overrides.candidate,
  };
  return {
    offerId: createRunDispatchOfferId(lease),
    executionSpecDigest: createExecutionSpecDigest(executionSpec),
    deliveryKind: 'new_claim',
    candidate,
    worker: {
      id: lease.workerId,
      sessionId: lease.workerSessionId,
      generation: lease.workerGeneration,
    },
    lease,
    executionSpec,
    ...overrides.offer,
  };
}

function session(overrides = {}) {
  return {
    id: 'worker-edge',
    sessionId: SESSION,
    generation: 1,
    status: 'online',
    version: 0,
    capabilities: {
      architecture: 'arm64',
      operatingSystem: 'linux',
      executors: ['remote_worker'],
      runtimes: [{ name: 'node', version: '24.14.0' }],
      labels: {},
      capacity: { cpuCores: 1, memoryBytes: 256 * 1024 * 1024 },
      features: ['direct_file_log'],
    },
    capabilitiesHash: 'a'.repeat(64),
    maxConcurrentRuns: 2,
    availableSlots: 2,
    registeredAtMs: START,
    lastHeartbeatAtMs: START,
    leaseExpiresAtMs: START + 120_000,
    updatedAtMs: START,
    ...overrides,
  };
}

function leaseTracker() {
  const tracked = new Map();
  return {
    track(lease) {
      tracked.set(lease.attemptId, { ...lease });
    },
    untrack(attemptId) {
      const value = tracked.get(attemptId);
      tracked.delete(attemptId);
      return value;
    },
    leases() {
      return [...tracked.values()].map((lease) => ({ ...lease }));
    },
  };
}

async function fixture(t, overrides = {}) {
  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-offer-receiver-'));
  const calls = [];
  const journal = new WorkerExecutionOfferFileJournal(root, {
    maximumEntries: 8,
  });
  await journal.acquireOwnership();
  const tracker = overrides.tracker || leaseTracker();
  const activation = {
    async acknowledgeStarting(command) {
      calls.push(['starting', command]);
      return {
        status: 'applied',
        run: {},
        attempt: {},
        lease: { ...overrides.startingLease, ...commandToLease(command) },
        events: [],
      };
    },
    async acknowledgeRunning(command) {
      calls.push(['running', command]);
      return {
        status: 'applied',
        run: {},
        attempt: {},
        lease: commandToLease(command),
        events: [],
      };
    },
    async failStart(command) {
      calls.push(['fail', command]);
      return {
        status: 'applied',
        run: {},
        attempt: {},
        lease: {
          ...commandToLease(command),
          status: 'completed',
          completedAtMs: START + 10,
          expiresAtMs: START + 60_000,
        },
        events: [],
      };
    },
    ...overrides.activation,
  };
  const executor = {
    type: 'local_process',
    capabilities() {
      return {};
    },
    async start(spec) {
      calls.push(['executor.start', spec.attemptId]);
      return {
        id: 'local-handle-1',
        durableHandle: 'local-process:durable-handle-1',
        executorType: 'local_process',
        runId: spec.runId,
        attemptId: spec.attemptId,
        startedAtMs: START + 5,
        completion: new Promise(() => undefined),
      };
    },
    async stop() {
      calls.push(['executor.stop']);
      return {
        status: 'termination_requested',
        termSignalSent: true,
        killSignalSent: false,
      };
    },
    async inspect() {
      return { status: 'running' };
    },
    ...overrides.executor,
  };
  const contexts = {
    async prepare(delivered) {
      calls.push(['context.prepare', delivered.offerId]);
      return {
        context: {
          environment: {},
          output: { async write() {} },
          completionCallback: {
            token: COMPLETION_TOKEN,
            callbackSequence: 1,
          },
        },
        logArtifactId: 'artifact-1',
      };
    },
    ...overrides.contexts,
  };
  let current = overrides.session || session();
  const receiver = new WorkerExecutionOfferReceiver(
    journal,
    activation,
    executor,
    tracker,
    contexts,
    {
      currentSession: () => current,
      clock: { now: () => START + 1 },
    },
  );
  t.after(async () => {
    await journal.releaseOwnership();
    await fs.rm(root, { recursive: true, force: true });
  });
  return {
    root,
    journal,
    tracker,
    calls,
    receiver,
    setSession(value) {
      current = value;
    },
  };
}

function commandToLease(command) {
  return {
    attemptId: command.attemptId,
    runId: command.runId,
    status: 'leased',
    version: command.expectedLeaseVersion,
    leaseGeneration: command.leaseGeneration,
    workerId: command.workerId,
    workerSessionId: command.workerSessionId,
    workerGeneration: command.workerGeneration,
    leaseToken: command.leaseToken,
    acquiredAtMs: START,
    renewedAtMs: START,
    expiresAtMs: START + 60_000,
    updatedAtMs: START,
  };
}

test('persists every ACK/start boundary and deduplicates a replay', async (t) => {
  const context = await fixture(t);
  const delivered = offer();
  const result = await context.receiver.receive(delivered);
  assert.deepEqual(result, {
    status: 'running',
    offerId: delivered.offerId,
    executorHandle: 'local-process:durable-handle-1',
  });
  assert.deepEqual(
    context.calls.map(([name]) => name),
    ['starting', 'context.prepare', 'executor.start', 'running'],
  );
  const persisted = await context.journal.read(delivered.offerId);
  assert.equal(persisted.state, 'running_acknowledged');
  assert.equal(persisted.executorHandle, 'local-process:durable-handle-1');
  assert.deepEqual(
    {
      callbackSequence: persisted.completionReceiptCallbackSequence,
      tokenDigest: persisted.completionReceiptTokenDigest,
    },
    createWorkerExecutionCompletionReceiptAuthentication({
      token: COMPLETION_TOKEN,
      callbackSequence: 1,
    }),
  );
  assert.doesNotMatch(JSON.stringify(persisted), new RegExp(COMPLETION_TOKEN));

  assert.equal(
    (await context.receiver.receive(delivered)).status,
    'already_running',
  );
  assert.equal(
    context.calls.filter(([name]) => name === 'executor.start').length,
    1,
  );
});

test('rejects same offer authority with a different canonical spec', async (t) => {
  const context = await fixture(t);
  const delivered = offer();
  await context.receiver.receive(delivered);
  const conflicting = offer({
    executionSpec: {
      command: { kind: 'argv', file: '/bin/false', args: [] },
    },
  });
  await assert.rejects(
    context.receiver.receive(conflicting),
    WorkerExecutionOfferConflictError,
  );
  assert.equal(
    context.calls.filter(([name]) => name === 'executor.start').length,
    1,
  );
});

test('rejects expired, replaced-session and draining new offers before journaling', async (t) => {
  const context = await fixture(t);
  const expired = offer({ lease: { expiresAtMs: START + 1 } });
  await assert.rejects(
    context.receiver.receive(expired),
    WorkerExecutionOfferExpiredError,
  );
  context.setSession(
    session({ sessionId: '019f7e00-0000-7000-8000-000000000002' }),
  );
  await assert.rejects(
    context.receiver.receive(offer()),
    WorkerExecutionOfferTargetError,
  );
  context.setSession(session({ status: 'draining' }));
  await assert.rejects(
    context.receiver.receive(offer()),
    WorkerExecutionOfferTargetError,
  );
  assert.equal((await context.journal.list()).records.length, 0);
});

test('allows lease recovery while draining but still starts only once', async (t) => {
  const context = await fixture(t, {
    session: session({ status: 'draining' }),
  });
  const delivered = offer({ offer: { deliveryKind: 'lease_recovery' } });
  assert.equal((await context.receiver.receive(delivered)).status, 'running');
  assert.equal(
    context.calls.filter(([name]) => name === 'executor.start').length,
    1,
  );
});

test('turns a restart-visible launching record into recovery_required without spawning', async (t) => {
  const context = await fixture(t);
  const delivered = offer();
  const accepted = createWorkerExecutionOfferJournalRecord(delivered, START);
  await context.journal.create(accepted);
  const starting = {
    ...accepted,
    revision: 1,
    state: 'starting_acknowledged',
    updatedAtMs: START + 1,
  };
  await context.journal.replace(starting, 0);
  const launching = {
    ...starting,
    revision: 2,
    state: 'launching',
    updatedAtMs: START + 2,
  };
  await context.journal.replace(launching, 1);

  assert.deepEqual(await context.receiver.receive(delivered), {
    status: 'recovery_required',
    offerId: delivered.offerId,
    reason: 'launch_outcome_unknown',
  });
  assert.equal(context.calls.length, 0);
  assert.equal(
    (await context.journal.read(delivered.offerId)).state,
    'recovery_required',
  );
});

test('replays running ACK from a durable started record without a second spawn', async (t) => {
  const context = await fixture(t);
  const delivered = offer();
  const accepted = createWorkerExecutionOfferJournalRecord(delivered, START);
  await context.journal.create(accepted);
  const starting = {
    ...accepted,
    revision: 1,
    state: 'starting_acknowledged',
    updatedAtMs: START + 1,
  };
  await context.journal.replace(starting, 0);
  const launching = {
    ...starting,
    revision: 2,
    state: 'launching',
    updatedAtMs: START + 2,
  };
  await context.journal.replace(launching, 1);
  const started = {
    ...launching,
    revision: 3,
    state: 'started',
    updatedAtMs: START + 3,
    executorHandle: 'local-process:durable-handle-1',
    executorStartedAtMs: START + 2,
  };
  await context.journal.replace(started, 2);

  assert.equal((await context.receiver.receive(delivered)).status, 'running');
  assert.deepEqual(
    context.calls.map(([name]) => name),
    ['running'],
  );
});

test('durably reports a rejected executor start and replays no side effects', async (t) => {
  const context = await fixture(t, {
    executor: {
      async start() {
        context.calls.push(['executor.start']);
        throw new Error('spawn rejected');
      },
    },
  });
  const delivered = offer();
  assert.equal(
    (await context.receiver.receive(delivered)).status,
    'start_failed',
  );
  assert.deepEqual(
    context.calls.map(([name]) => name),
    ['starting', 'context.prepare', 'executor.start', 'fail'],
  );
  assert.equal(
    (await context.journal.read(delivered.offerId)).state,
    'start_failure_acknowledged',
  );
  assert.equal(
    (await context.receiver.receive(delivered)).status,
    'already_failed',
  );
  assert.equal(
    context.calls.filter(([name]) => name === 'executor.start').length,
    1,
  );
});

test('rejects an invalid completion capability before the spawn barrier', async (t) => {
  const context = await fixture(t, {
    contexts: {
      async prepare() {
        context.calls.push(['context.prepare']);
        return {
          context: {
            environment: {},
            output: { async write() {} },
            completionCallback: {
              token: 'too-short',
              callbackSequence: 1,
            },
          },
        };
      },
    },
  });
  const delivered = offer();
  assert.equal(
    (await context.receiver.receive(delivered)).status,
    'start_failed',
  );
  assert.deepEqual(
    context.calls.map(([name]) => name),
    ['starting', 'context.prepare', 'fail'],
  );
  const persisted = await context.journal.read(delivered.offerId);
  assert.equal(persisted.state, 'start_failure_acknowledged');
  assert.equal(persisted.completionReceiptTokenDigest, undefined);
});

test('coalesces concurrent duplicate delivery into one journal/start operation', async (t) => {
  let releaseStart;
  const startGate = new Promise((resolve) => {
    releaseStart = resolve;
  });
  const context = await fixture(t, {
    executor: {
      async start(spec) {
        context.calls.push(['executor.start', spec.attemptId]);
        await startGate;
        return {
          id: 'local-handle-concurrent',
          executorType: 'local_process',
          runId: spec.runId,
          attemptId: spec.attemptId,
          startedAtMs: START + 5,
          completion: new Promise(() => undefined),
        };
      },
    },
  });
  const delivered = offer();
  const first = context.receiver.receive(delivered);
  const second = context.receiver.receive(delivered);
  assert.strictEqual(first, second);
  releaseStart();
  assert.equal((await first).status, 'running');
  assert.equal(
    context.calls.filter(([name]) => name === 'executor.start').length,
    1,
  );
});

test('does not spawn when the control plane already owns a running execution', async (t) => {
  const context = await fixture(t, {
    activation: {
      async acknowledgeStarting(command) {
        context.calls.push(['starting', command]);
        return {
          status: 'already_running',
          run: {},
          attempt: {},
          lease: commandToLease(command),
          events: [],
        };
      },
    },
  });
  const delivered = offer();
  assert.deepEqual(await context.receiver.receive(delivered), {
    status: 'recovery_required',
    offerId: delivered.offerId,
    reason: 'control_plane_already_running',
  });
  assert.deepEqual(
    context.calls.map(([name]) => name),
    ['starting'],
  );
});

test('returns an already completed result for a durable terminal redelivery', async (t) => {
  const context = await fixture(t);
  const delivered = offer();
  const initial = createWorkerExecutionOfferJournalRecord(delivered, START);
  const authentication = createWorkerExecutionCompletionReceiptAuthentication({
    token: COMPLETION_TOKEN,
    callbackSequence: 1,
  });
  await context.journal.create(initial);
  await context.journal.replace(
    {
      ...initial,
      revision: 1,
      state: 'completion_acknowledged',
      updatedAtMs: START + 1,
      completionReceiptCallbackSequence: authentication.callbackSequence,
      completionReceiptTokenDigest: authentication.tokenDigest,
      completionAcknowledgedAtMs: START + 1,
    },
    0,
  );
  context.setSession(
    session({ sessionId: REPLACEMENT_SESSION, generation: 2 }),
  );

  assert.deepEqual(await context.receiver.receive(delivered), {
    status: 'already_completed',
    offerId: delivered.offerId,
  });
  assert.deepEqual(context.calls, []);
});

test('never restarts a durable execution after lease-loss stop acknowledgement', async (t) => {
  const context = await fixture(t);
  const delivered = offer();
  const initial = createWorkerExecutionOfferJournalRecord(delivered, START);
  const authentication = createWorkerExecutionCompletionReceiptAuthentication({
    token: COMPLETION_TOKEN,
    callbackSequence: 1,
  });
  await context.journal.create(initial);
  await context.journal.replace(
    {
      ...initial,
      revision: 1,
      state: 'recovery_required',
      updatedAtMs: START + 1,
      executorHandle: 'local-process:durable-handle-1',
      executorStartedAtMs: START + 1,
      completionReceiptCallbackSequence: authentication.callbackSequence,
      completionReceiptTokenDigest: authentication.tokenDigest,
      recoveryReason: 'lease_lost_local_execution_stopped',
    },
    0,
  );

  assert.deepEqual(await context.receiver.receive(delivered), {
    status: 'recovery_required',
    offerId: delivered.offerId,
    reason: 'lease_lost_local_execution_stopped',
  });
  assert.deepEqual(context.calls, []);
});