feat(ql3): establish 3.0 incubation baseline

2026-09-20 16:07:11 +08:00 · 2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
@@ -0,0 +1,282 @@
+require('ts-node/register/transpile-only');
+
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const { afterEach, test } = require('node:test');
+const { Sequelize } = require('sequelize');
+const {
+  defineSchemaMigrationModel,
+} = require('../../back/data/schemaMigration');
+const { runSchemaMigration } = require('../../back/migrations/0002-run-schema');
+const {
+  runCancellationRequestMigration,
+} = require('../../back/migrations/0004-run-cancellation-request');
+const {
+  runAttemptDeadlineMigration,
+} = require('../../back/migrations/0006-run-attempt-deadline');
+const { runMigrations } = require('../../back/migrations/runner');
+const {
+  CompletionReceiptFileStore,
+} = require('../../back/runtime/adapters/fs/completionReceiptFileStore');
+const {
+  LegacySequelizePrimaryRunRecoverySource,
+} = require('../../back/runtime/adapters/legacy-sequelize/primaryRunRecoverySource');
+const {
+  LegacySequelizeRunRepository,
+} = require('../../back/runtime/adapters/legacy-sequelize/runRepository');
+const {
+  PrimaryCompletionReceiptConsumer,
+} = require('../../back/runtime/application/primaryCompletionReceiptConsumer');
+const {
+  hashPrimaryCompletionToken,
+  PrimaryRunCompletionService,
+} = require('../../back/runtime/application/primaryRunCompletionService');
+const {
+  PrimaryRunStartupReconciler,
+} = require('../../back/runtime/application/primaryRunStartupReconciler');
+
+const BASE_TIME = 1_750_300_000_000;
+const TOKEN = 'c'.repeat(43);
+const databases = [];
+const roots = [];
+let idSequence = 1_800;
+
+function nextId() {
+  idSequence += 1;
+  return `019f7400-0000-7000-8000-${String(idSequence).padStart(12, '0')}`;
+}
+
+async function createRuntime() {
+  const database = new Sequelize({
+    dialect: 'sqlite',
+    storage: ':memory:',
+    logging: false,
+  });
+  await runMigrations({
+    database,
+    migrationModel: defineSchemaMigrationModel(database),
+    migrations: [
+      runSchemaMigration,
+      runCancellationRequestMigration,
+      runAttemptDeadlineMigration,
+    ],
+    logger: { info() {} },
+  });
+  databases.push(database);
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql3-startup-receipt-'));
+  roots.push(root);
+  const repository = new LegacySequelizeRunRepository(database);
+  const store = new CompletionReceiptFileStore(root);
+  const consumer = new PrimaryCompletionReceiptConsumer(
+    store,
+    new PrimaryRunCompletionService(repository, nextId),
+  );
+  return { database, repository, store, consumer };
+}
+
+function aggregate() {
+  const run = {
+    id: nextId(),
+    projectId: 'default',
+    taskId: 'receipt-recovery',
+    taskRevision: 'revision-1',
+    triggerType: 'manual',
+    executionOrigin: 'manual',
+    executionOwner: 'runtime',
+    status: 'running',
+    version: 0,
+    eventSequence: 0,
+    priority: 0,
+    createdAtMs: BASE_TIME,
+    startedAtMs: BASE_TIME + 1,
+  };
+  const attempt = {
+    id: nextId(),
+    runId: run.id,
+    attempt: 1,
+    status: 'running',
+    executorType: 'local_process',
+    executorHandle: `durable:${run.id}`,
+    pid: 4321,
+    callbackTokenHash: hashPrimaryCompletionToken(TOKEN),
+    callbackSequence: 0,
+    createdAtMs: BASE_TIME,
+    startedAtMs: BASE_TIME + 1,
+  };
+  return { run, attempt };
+}
+
+function receipt(run, attempt) {
+  return {
+    schemaVersion: 1,
+    runId: run.id,
+    attemptId: attempt.id,
+    callbackSequence: 1,
+    token: TOKEN,
+    startedAtMs: BASE_TIME + 1,
+    finishedAtMs: BASE_TIME + 20,
+    exitCode: 0,
+  };
+}
+
+async function seed(repository, run, attempt) {
+  await repository.transaction(async (transaction) => {
+    await transaction.insertRun(run);
+    await transaction.insertAttempt(attempt);
+  });
+}
+
+afterEach(async () => {
+  await Promise.all(databases.splice(0).map((database) => database.close()));
+  await Promise.all(
+    roots
+      .splice(0)
+      .map((root) => fs.rm(root, { recursive: true, force: true })),
+  );
+});
+
+test('startup consumes a receipt before consulting durable identity', async () => {
+  const { database, repository, store, consumer } = await createRuntime();
+  const { run, attempt } = aggregate();
+  await seed(repository, run, attempt);
+  await store.publish(receipt(run, attempt));
+  let inspections = 0;
+  const registrations = [];
+  const reconciler = new PrimaryRunStartupReconciler(
+    repository,
+    new LegacySequelizePrimaryRunRecoverySource(database),
+    [
+      {
+        executorType: 'local_process',
+        async inspect() {
+          inspections += 1;
+          return { status: 'exited', identityPid: 4321 };
+        },
+      },
+    ],
+    {
+      completionReceipts: consumer,
+      completionReceiptJournal: {
+        async register(command) {
+          registrations.push(command);
+        },
+      },
+      createEventId: nextId,
+    },
+  );
+
+  const summary = await reconciler.reconcileBatch();
+  assert.equal(summary.completedFromReceipt, 1);
+  assert.equal(summary.markedLost, 0);
+  assert.equal(inspections, 0);
+  assert.deepEqual(registrations, [
+    {
+      runId: run.id,
+      attemptId: attempt.id,
+      registeredAtMs: attempt.createdAtMs,
+    },
+  ]);
+  assert.equal((await repository.findRunById(run.id)).status, 'succeeded');
+  assert.equal(
+    (await repository.findAttemptById(attempt.id)).status,
+    'succeeded',
+  );
+  assert.equal(await store.read(attempt.id), undefined);
+});
+
+test('startup rechecks a receipt after identity reports process exit', async () => {
+  const { database, repository, store, consumer } = await createRuntime();
+  const { run, attempt } = aggregate();
+  await seed(repository, run, attempt);
+  let inspections = 0;
+  const reconciler = new PrimaryRunStartupReconciler(
+    repository,
+    new LegacySequelizePrimaryRunRecoverySource(database),
+    [
+      {
+        executorType: 'local_process',
+        async inspect() {
+          inspections += 1;
+          await store.publish(receipt(run, attempt));
+          return { status: 'exited', identityPid: 4321 };
+        },
+      },
+    ],
+    { completionReceipts: consumer, createEventId: nextId },
+  );
+
+  const summary = await reconciler.reconcileBatch();
+  assert.equal(summary.completedFromReceipt, 1);
+  assert.equal(summary.markedLost, 0);
+  assert.equal(inspections, 1);
+  assert.equal((await repository.findRunById(run.id)).status, 'succeeded');
+  assert.equal(await store.read(attempt.id), undefined);
+});
+
+test('startup waits once for a late receipt after observing process exit', async () => {
+  const { database, repository, store, consumer } = await createRuntime();
+  const { run, attempt } = aggregate();
+  await seed(repository, run, attempt);
+  const waits = [];
+  const reconciler = new PrimaryRunStartupReconciler(
+    repository,
+    new LegacySequelizePrimaryRunRecoverySource(database),
+    [
+      {
+        executorType: 'local_process',
+        async inspect() {
+          return { status: 'exited', identityPid: 4321 };
+        },
+      },
+    ],
+    {
+      completionReceipts: consumer,
+      createEventId: nextId,
+      receiptPublishGraceMs: 50,
+      async wait(delayMs) {
+        waits.push(delayMs);
+        await store.publish(receipt(run, attempt));
+      },
+    },
+  );
+
+  const summary = await reconciler.reconcileBatch();
+  assert.deepEqual(waits, [50]);
+  assert.equal(summary.publishGraceWaits, 1);
+  assert.equal(summary.completedFromReceipt, 1);
+  assert.equal(summary.markedLost, 0);
+  assert.equal((await repository.findRunById(run.id)).status, 'succeeded');
+});
+
+test('startup quarantines an unauthorized receipt and still verifies the process', async () => {
+  const { database, repository, store, consumer } = await createRuntime();
+  const { run, attempt } = aggregate();
+  await seed(repository, run, attempt);
+  await store.publish({
+    ...receipt(run, attempt),
+    token: 'd'.repeat(43),
+  });
+  const reconciler = new PrimaryRunStartupReconciler(
+    repository,
+    new LegacySequelizePrimaryRunRecoverySource(database),
+    [
+      {
+        executorType: 'local_process',
+        async inspect() {
+          return { status: 'running', identityPid: 4321 };
+        },
+      },
+    ],
+    { completionReceipts: consumer, createEventId: nextId },
+  );
+
+  const summary = await reconciler.reconcileBatch();
+  assert.equal(summary.quarantinedReceipts, 1);
+  assert.equal(summary.completedFromReceipt, 0);
+  assert.equal(summary.verifiedRunning, 1);
+  assert.equal(summary.markedLost, 0);
+  assert.equal((await repository.findRunById(run.id)).status, 'running');
+  assert.equal(await store.read(attempt.id), undefined);
+});