feat(ql3): establish 3.0 incubation baseline

2026-09-20 16:07:11 +08:00 · 2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
@@ -0,0 +1,244 @@
+'use strict';
+
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+
+const {
+  PostgresClusterRunLostRetryRepository,
+} = require('../dist/entrypoints/runtime');
+
+function runRow(status) {
+  return {
+    id: 'run-1',
+    projectId: 'project-1',
+    taskId: 'task-1',
+    taskRevision: 'revision-1',
+    taskName: null,
+    taskSnapshotRef: null,
+    legacyCronId: null,
+    parentRunId: null,
+    retryOfRunId: null,
+    triggerId: null,
+    triggerType: 'manual',
+    executionOrigin: 'manual',
+    executionOwner: 'runtime',
+    triggeredBy: null,
+    requestId: null,
+    scheduledForMs: null,
+    status,
+    version: 4,
+    eventSequence: 4,
+    priority: 0,
+    idempotencyKey: null,
+    inputRef: null,
+    outputRef: null,
+    createdAtMs: '100',
+    queuedAtMs: '110',
+    startedAtMs: '150',
+    finishedAtMs: null,
+    cancelRequestedAtMs: null,
+    cancelReason: null,
+    errorCode: 'CLUSTER_RECOVERY_EXECUTION_NOT_RUNNING',
+    errorSummary: 'lost',
+  };
+}
+
+function attemptRow() {
+  return {
+    id: 'attempt-1',
+    runId: 'run-1',
+    stepRunId: null,
+    attempt: 1,
+    status: 'lost',
+    executorType: 'remote_worker',
+    workerId: 'worker-1',
+    workerSessionId: null,
+    workerGeneration: null,
+    executorHandle: null,
+    pid: null,
+    logArtifactId: null,
+    leaseToken: null,
+    leaseTokenDigest: null,
+    leaseGeneration: null,
+    leaseVersion: null,
+    leaseExpiresAtMs: null,
+    offerId: null,
+    deadlineAtMs: null,
+    callbackTokenHash: null,
+    callbackSequence: 0,
+    createdAtMs: '120',
+    startedAtMs: '150',
+    finishedAtMs: '200',
+    exitCode: null,
+    errorCode: 'CLUSTER_RECOVERY_EXECUTION_NOT_RUNNING',
+    errorSummary: 'lost',
+  };
+}
+
+function policyRow(nextAttemptAtMs = null) {
+  return {
+    runId: 'run-1',
+    maxAttempts: 3,
+    retryOnLost: true,
+    safety: 'idempotent',
+    backoffBaseMs: '1000',
+    backoffMaxMs: '8000',
+    nextAttemptAtMs,
+    version: 0,
+    createdAtMs: '100',
+    updatedAtMs: '100',
+  };
+}
+
+function harness(status = 'lost') {
+  const calls = [];
+  let id = 0;
+  const client = {
+    async query(text, values = []) {
+      calls.push({ text, values });
+      if (text.includes('pg_advisory_xact_lock')) {
+        return { rows: [{ locked: true }], rowCount: 1 };
+      }
+      if (
+        text.includes('FROM "ql3"."runs" AS run') &&
+        text.includes('FOR UPDATE')
+      ) {
+        return { rows: [{ runId: 'run-1' }], rowCount: 1 };
+      }
+      if (
+        text.startsWith('SELECT') &&
+        text.includes('FROM "ql3"."runs" WHERE')
+      ) {
+        return { rows: [runRow(status)], rowCount: 1 };
+      }
+      if (
+        text.startsWith('SELECT') &&
+        text.includes('FROM "ql3"."run_attempts"')
+      ) {
+        return { rows: [attemptRow()], rowCount: 1 };
+      }
+      if (
+        text.startsWith('SELECT') &&
+        text.includes('FROM "ql3"."run_retry_policies" WHERE') &&
+        !text.includes('FOR UPDATE')
+      ) {
+        return {
+          rows: [policyRow(status === 'retry_wait' ? '250' : null)],
+          rowCount: 1,
+        };
+      }
+      if (
+        text.includes('FROM "ql3"."run_retry_policies"') &&
+        text.includes('FOR UPDATE')
+      ) {
+        return { rows: [{ runId: 'run-1' }], rowCount: 1 };
+      }
+      if (text.includes('AS "observedAtMs"')) {
+        return { rows: [{ observedAtMs: '300' }], rowCount: 1 };
+      }
+      if (text.startsWith('UPDATE "ql3"."runs"')) {
+        return { rows: [{ id: 'run-1' }], rowCount: 1 };
+      }
+      if (text.startsWith('UPDATE "ql3"."run_retry_policies"')) {
+        return { rows: [{ run_id: 'run-1' }], rowCount: 1 };
+      }
+      if (text.startsWith('INSERT INTO "ql3"."run_attempts"')) {
+        return { rows: [], rowCount: 1 };
+      }
+      if (text.startsWith('INSERT INTO "ql3"."run_events"')) {
+        return { rows: [], rowCount: 1 };
+      }
+      return { rows: [], rowCount: 0 };
+    },
+    release() {},
+  };
+  return {
+    calls,
+    repository: new PostgresClusterRunLostRetryRepository(
+      {
+        async query(text, values) {
+          calls.push({ text, values });
+          assert.match(text, /JOIN LATERAL/);
+          return {
+            rows: [{ runId: 'run-1', attemptId: 'attempt-1' }],
+            rowCount: 1,
+          };
+        },
+        async connect() {
+          return client;
+        },
+      },
+      () => `00000000-0000-4000-8000-${String(++id).padStart(12, '0')}`,
+    ),
+  };
+}
+
+test('schedules one safe lost Run in a bounded atomic page', async () => {
+  const database = harness('lost');
+  assert.deepEqual(await database.repository.reconcilePage({ limit: 2 }), {
+    scanned: 1,
+    scheduled: 1,
+    requeued: 0,
+    failed: 0,
+    raced: 0,
+    hasMore: false,
+  });
+  assert.equal(
+    database.calls.filter(({ text }) =>
+      text.startsWith('UPDATE "ql3"."runs"'),
+    ).length,
+    1,
+  );
+  assert.equal(
+    database.calls.filter(({ text }) =>
+      text.startsWith('UPDATE "ql3"."run_retry_policies"'),
+    ).length,
+    1,
+  );
+  assert.equal(
+    database.calls.filter(({ text }) =>
+      text.startsWith('INSERT INTO "ql3"."run_events"'),
+    ).length,
+    1,
+  );
+  assert.equal(database.calls.at(-1).text, 'COMMIT');
+});
+
+test('creates exactly one fresh Attempt when retry_wait is due', async () => {
+  const database = harness('retry_wait');
+  assert.deepEqual(await database.repository.reconcilePage({ limit: 1 }), {
+    scanned: 1,
+    scheduled: 0,
+    requeued: 1,
+    failed: 0,
+    raced: 0,
+    hasMore: false,
+  });
+  assert.equal(
+    database.calls.filter(({ text }) =>
+      text.startsWith('UPDATE "ql3"."runs"'),
+    ).length,
+    2,
+  );
+  assert.equal(
+    database.calls.filter(({ text }) =>
+      text.startsWith('INSERT INTO "ql3"."run_attempts"'),
+    ).length,
+    1,
+  );
+  assert.equal(
+    database.calls.filter(({ text }) =>
+      text.startsWith('INSERT INTO "ql3"."run_events"'),
+    ).length,
+    2,
+  );
+});
+
+test('rejects widened pages before querying PostgreSQL', async () => {
+  const database = harness('lost');
+  await assert.rejects(
+    database.repository.reconcilePage({ limit: 65 }),
+    /page size/,
+  );
+  assert.equal(database.calls.length, 0);
+});