feat(ql3): establish 3.0 incubation baseline

2026-09-21 18:08:20 +08:00 · 2026-08-12 00:25:26 +08:00
parent 4bf92dcfeb
commit c699c32461
2817 changed files with 779642 additions and 653 deletions
@@ -0,0 +1,443 @@
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+const {
+  resolveClusterScheduleDecision,
+} = require('@qinglong/runtime-core/cluster-scheduler');
+const {
+  createTaskDefinitionRecord,
+} = require('@qinglong/runtime-core/task-definition');
+const {
+  createBuiltInTaskSpecSemanticRegistry,
+} = require('@qinglong/runtime-core/task-spec-semantic');
+const {
+  compileClusterCommandTaskDefinition,
+} = require('@qinglong/runtime-core/cluster-execution-revision');
+const {
+  createBuiltInTriggerSpecSemanticRegistry,
+  createTriggerRecord,
+} = require('@qinglong/runtime-core/trigger');
+const {
+  PostgresClusterScheduleRepository,
+  PostgresClusterScheduleUnavailableError,
+} = require('../dist/scheduling/clusterScheduleRepository');
+
+function nextMinute(schedule, afterMs) {
+  if (schedule.expression !== '* * * * *' || schedule.timezone !== 'UTC') {
+    throw new Error('unsupported test schedule');
+  }
+  return Math.floor(afterMs / 60_000 + 1) * 60_000;
+}
+
+const CLAIM_TOKEN = '019f7700-0000-7000-8000-000000000001';
+const TASK_COMMAND = Object.freeze({
+  projectId: 'default',
+  taskId: 'task-00001',
+  expectedRevision: null,
+  mutationId: '019f7700-0000-7000-8000-000000000010',
+  name: 'Scheduled Task',
+  kind: 'command',
+  spec: Object.freeze({
+    schema: 'qinglong/command@v1',
+    config: Object.freeze({
+      command: Object.freeze({
+        kind: 'argv',
+        file: '/bin/echo',
+        args: Object.freeze(['scheduled']),
+      }),
+    }),
+  }),
+  labels: Object.freeze({}),
+  enabled: true,
+  occurredAtMs: 101,
+});
+const taskSemantics = createBuiltInTaskSpecSemanticRegistry();
+const TASK = createTaskDefinitionRecord(
+  Object.freeze({
+    ...TASK_COMMAND,
+    spec: taskSemantics.normalize({
+      projectId: TASK_COMMAND.projectId,
+      taskId: TASK_COMMAND.taskId,
+      kind: TASK_COMMAND.kind,
+      spec: TASK_COMMAND.spec,
+    }),
+  }),
+  TASK_COMMAND.occurredAtMs,
+);
+const EXECUTION = compileClusterCommandTaskDefinition(TASK, taskSemantics);
+const TRIGGER_COMMAND = Object.freeze({
+  projectId: TASK.projectId,
+  triggerId: 'trigger-00001',
+  expectedRevision: null,
+  mutationId: '019f7700-0000-7000-8000-000000000011',
+  taskId: TASK.taskId,
+  taskRevision: TASK.revision,
+  taskContentDigest: TASK.contentDigest,
+  spec: Object.freeze({
+    schema: 'qinglong/cron@v1',
+    config: Object.freeze({
+      expression: '* * * * *',
+      timezone: 'UTC',
+      misfirePolicy: 'skip',
+    }),
+  }),
+  enabled: true,
+  occurredAtMs: 201,
+});
+const triggerSemantics = createBuiltInTriggerSpecSemanticRegistry();
+const TRIGGER = createTriggerRecord(
+  Object.freeze({
+    ...TRIGGER_COMMAND,
+    spec: triggerSemantics.normalize({
+      projectId: TRIGGER_COMMAND.projectId,
+      triggerId: TRIGGER_COMMAND.triggerId,
+      taskId: TRIGGER_COMMAND.taskId,
+      taskRevision: TRIGGER_COMMAND.taskRevision,
+      spec: TRIGGER_COMMAND.spec,
+    }),
+  }),
+  TRIGGER_COMMAND.occurredAtMs,
+);
+
+function claim(overrides = {}) {
+  return {
+    projectId: TRIGGER.projectId,
+    triggerId: TRIGGER.triggerId,
+    triggerRevision: TRIGGER.revision,
+    triggerContentDigest: TRIGGER.contentDigest,
+    triggerUpdatedAtMs: TRIGGER.updatedAtMs,
+    taskId: TRIGGER.taskId,
+    taskRevision: TRIGGER.taskRevision,
+    taskContentDigest: TRIGGER.taskContentDigest,
+    expression: TRIGGER.spec.config.expression,
+    timezone: TRIGGER.spec.config.timezone,
+    misfirePolicy: TRIGGER.spec.config.misfirePolicy,
+    stateVersion: 1,
+    nextFireAtMs: 60_000,
+    claimOwner: 'scheduler-a',
+    claimToken: CLAIM_TOKEN,
+    claimVersion: 1,
+    claimAcquiredAtMs: 61_000,
+    claimExpiresAtMs: 91_000,
+    ...overrides,
+  };
+}
+
+function claimRow(overrides = {}) {
+  const value = claim(overrides);
+  return {
+    projectId: value.projectId,
+    triggerId: value.triggerId,
+    triggerRevision: value.triggerRevision,
+    triggerContentDigest: value.triggerContentDigest,
+    triggerUpdatedAtMs: String(value.triggerUpdatedAtMs),
+    taskId: value.taskId,
+    taskRevision: value.taskRevision,
+    taskContentDigest: value.taskContentDigest,
+    specJson: TRIGGER.spec,
+    taskName: TASK.name,
+    stateVersion: value.stateVersion,
+    nextFireAtMs:
+      value.nextFireAtMs === null ? null : String(value.nextFireAtMs),
+    claimOwner: value.claimOwner,
+    claimToken: value.claimToken,
+    claimVersion: value.claimVersion,
+    claimAcquiredAtMs: String(value.claimAcquiredAtMs),
+    claimExpiresAtMs: String(value.claimExpiresAtMs),
+    commitObservedAtMs: String(overrides.commitObservedAtMs ?? 62_000),
+  };
+}
+
+function executionRow(overrides = {}) {
+  return {
+    projectId: EXECUTION.projectId,
+    taskId: EXECUTION.taskId,
+    sourceRevision: EXECUTION.sourceRevision,
+    taskRevision: EXECUTION.taskRevision,
+    sourceContentDigest: EXECUTION.sourceContentDigest,
+    executorType: EXECUTION.executorType,
+    planSchema: EXECUTION.planSchema,
+    planJson: {
+      command: EXECUTION.command,
+      environment: EXECUTION.environment,
+      ...(EXECUTION.workingDirectory === undefined
+        ? {}
+        : { workingDirectory: EXECUTION.workingDirectory }),
+      ...(EXECUTION.timeoutMs === undefined
+        ? {}
+        : { timeoutMs: EXECUTION.timeoutMs }),
+      ...(EXECUTION.placement === undefined
+        ? {}
+        : { placement: EXECUTION.placement }),
+    },
+    contentDigest: EXECUTION.contentDigest,
+    createdAtMs: String(EXECUTION.createdAtMs),
+    ...overrides,
+  };
+}
+
+function fixture(options = {}) {
+  const queries = [];
+  let commitFailuresRemaining = options.failCommitOnce ? 1 : 0;
+  const client = {
+    async query(text, values) {
+      queries.push({ text, values });
+      if (text === 'COMMIT' && commitFailuresRemaining > 0) {
+        commitFailuresRemaining -= 1;
+        const error = new Error('injected committed response loss');
+        error.code = '40001';
+        throw error;
+      }
+      if (text.includes('FOR UPDATE OF schedule')) {
+        return {
+          rows: options.missingClaim
+            ? []
+            : [
+                claimRow({
+                  ...options.claimOverrides,
+                  commitObservedAtMs: options.commitObservedAtMs,
+                }),
+              ],
+        };
+      }
+      if (text.includes('FROM "ql3"."task_execution_revisions"')) {
+        return {
+          rows: options.missingExecution
+            ? []
+            : [executionRow(options.executionOverrides)],
+        };
+      }
+      if (
+        options.failAt &&
+        text.includes(`INSERT INTO "ql3"."${options.failAt}"`)
+      ) {
+        throw new Error('injected write failure');
+      }
+      if (text.includes('UPDATE "ql3"."trigger_schedules"')) {
+        return { rows: [], rowCount: options.advanceRaced ? 0 : 1 };
+      }
+      return { rows: [], rowCount: 1 };
+    },
+    release() {
+      queries.push({ text: 'RELEASE' });
+    },
+  };
+  return {
+    queries,
+    pool: {
+      async query(text, values) {
+        queries.push({ text, values });
+        return {
+          rows: options.noDue ? [] : [claimRow(options.claimOverrides)],
+        };
+      },
+      async connect() {
+        return client;
+      },
+    },
+  };
+}
+
+function admissionCommand(claimed = claim()) {
+  return {
+    claim: claimed,
+    decision: resolveClusterScheduleDecision(claimed, 5_000, nextMinute),
+    runId: '019f7700-0000-7000-8000-000000000020',
+    attemptId: '019f7700-0000-7000-8000-000000000021',
+    createdEventId: '019f7700-0000-7000-8000-000000000022',
+    queuedEventId: '019f7700-0000-7000-8000-000000000023',
+  };
+}
+
+test('claims one due schedule with ordered SKIP LOCKED lease takeover', async () => {
+  const db = fixture();
+  const repository = new PostgresClusterScheduleRepository(db.pool);
+  assert.deepEqual(
+    await repository.claimNextClusterSchedule({
+      ownerId: 'scheduler-a',
+      claimToken: CLAIM_TOKEN,
+      leaseMs: 30_000,
+    }),
+    claim(),
+  );
+  assert.equal(db.queries.length, 1);
+  assert.match(db.queries[0].text, /FOR UPDATE OF schedule SKIP LOCKED/);
+  assert.match(db.queries[0].text, /clock_timestamp\(\)/);
+  assert.match(
+    db.queries[0].text,
+    /schedule\.claim_expires_at_ms <= observation\.observed_at_ms/,
+  );
+  assert.match(db.queries[0].text, /NULLS FIRST/);
+  assert.match(
+    db.queries[0].text,
+    /task_head\.current_revision = task\.revision/,
+  );
+  assert.deepEqual(db.queries[0].values, ['scheduler-a', CLAIM_TOKEN, 30_000]);
+
+  const empty = fixture({ noDue: true });
+  assert.equal(
+    await new PostgresClusterScheduleRepository(
+      empty.pool,
+    ).claimNextClusterSchedule({
+      ownerId: 'scheduler-a',
+      claimToken: CLAIM_TOKEN,
+      leaseMs: 30_000,
+    }),
+    null,
+  );
+});
+
+test('admits Run, Attempt and two events before advancing the exact claim', async () => {
+  const db = fixture();
+  const result = await new PostgresClusterScheduleRepository(
+    db.pool,
+  ).commitClusterScheduleDecision(admissionCommand());
+  assert.deepEqual(result, {
+    status: 'admitted',
+    disposition: 'admit',
+    runId: '019f7700-0000-7000-8000-000000000020',
+    attemptId: '019f7700-0000-7000-8000-000000000021',
+  });
+  const sql = db.queries.map(({ text }) => text);
+  assert.match(
+    sql.find((text) => text.includes('FOR UPDATE OF schedule')),
+    /task_head\.current_revision = task\.revision/,
+  );
+  const run = sql.findIndex((text) =>
+    text.includes('INSERT INTO "ql3"."runs"'),
+  );
+  const attempt = sql.findIndex((text) =>
+    text.includes('INSERT INTO "ql3"."run_attempts"'),
+  );
+  const events = sql.filter((text) =>
+    text.includes('INSERT INTO "ql3"."run_events"'),
+  );
+  const advance = sql.findIndex((text) =>
+    text.includes('UPDATE "ql3"."trigger_schedules"'),
+  );
+  assert.ok(
+    run > 0 && attempt > run && events.length === 2 && advance > attempt,
+  );
+  assert.match(sql[attempt], /'remote_worker'/);
+  assert.match(sql[advance], /claim_token = \$10::uuid/);
+  assert.equal(
+    db.queries
+      .find(({ text }) => text.includes('INSERT INTO "ql3"."runs"'))
+      .values.at(-1),
+    62_000,
+  );
+  assert.equal(sql.includes('COMMIT'), true);
+  assert.equal(sql.at(-1), 'RELEASE');
+});
+
+test('does not retry after COMMIT was sent and its response is lost', async () => {
+  const db = fixture({ failCommitOnce: true });
+  await assert.rejects(
+    new PostgresClusterScheduleRepository(
+      db.pool,
+    ).commitClusterScheduleDecision(admissionCommand()),
+    PostgresClusterScheduleUnavailableError,
+  );
+  const sql = db.queries.map(({ text }) => text);
+  assert.equal(sql.filter((text) => text === 'COMMIT').length, 1);
+  assert.equal(sql.includes('ROLLBACK'), false);
+});
+
+test('returns raced without writes when the durable claim fence changed', async () => {
+  const db = fixture({ claimOverrides: { stateVersion: 2 } });
+  assert.deepEqual(
+    await new PostgresClusterScheduleRepository(
+      db.pool,
+    ).commitClusterScheduleDecision(admissionCommand()),
+    { status: 'raced' },
+  );
+  const sql = db.queries.map(({ text }) => text);
+  assert.equal(
+    sql.some((text) => text.includes('INSERT INTO')),
+    false,
+  );
+  assert.equal(sql.includes('ROLLBACK'), true);
+});
+
+test('uses the database commit clock for expiry and rejects clock regression', async () => {
+  const expired = fixture({ commitObservedAtMs: 91_000 });
+  assert.deepEqual(
+    await new PostgresClusterScheduleRepository(
+      expired.pool,
+    ).commitClusterScheduleDecision(admissionCommand()),
+    { status: 'raced' },
+  );
+  assert.equal(
+    expired.queries.some(({ text }) => text.includes('INSERT INTO')),
+    false,
+  );
+
+  const backwards = fixture({ commitObservedAtMs: 60_999 });
+  await assert.rejects(
+    new PostgresClusterScheduleRepository(
+      backwards.pool,
+    ).commitClusterScheduleDecision(admissionCommand()),
+    /clock moved backwards/,
+  );
+  assert.equal(
+    backwards.queries.some(({ text }) => text === 'ROLLBACK'),
+    true,
+  );
+});
+
+test('advances a skipped occurrence without creating a Run', async () => {
+  const claimed = claim({
+    claimAcquiredAtMs: 900_000,
+    claimExpiresAtMs: 930_000,
+  });
+  const db = fixture({
+    claimOverrides: {
+      claimAcquiredAtMs: 900_000,
+      claimExpiresAtMs: 930_000,
+    },
+    commitObservedAtMs: 900_001,
+  });
+  const decision = resolveClusterScheduleDecision(claimed, 0, nextMinute);
+  assert.equal(decision.disposition, 'skip');
+  assert.deepEqual(
+    await new PostgresClusterScheduleRepository(
+      db.pool,
+    ).commitClusterScheduleDecision({ claim: claimed, decision }),
+    { status: 'advanced', disposition: 'skip' },
+  );
+  assert.equal(
+    db.queries.some(({ text }) => text.includes('INSERT INTO "ql3"."runs"')),
+    false,
+  );
+});
+
+test('rolls back partial admission and rejects corrupt execution revisions', async () => {
+  const failed = fixture({ failAt: 'run_attempts' });
+  await assert.rejects(
+    new PostgresClusterScheduleRepository(
+      failed.pool,
+    ).commitClusterScheduleDecision(admissionCommand()),
+    PostgresClusterScheduleUnavailableError,
+  );
+  assert.equal(
+    failed.queries.some(({ text }) => text === 'ROLLBACK'),
+    true,
+  );
+  assert.equal(
+    failed.queries.some(({ text }) => text === 'COMMIT'),
+    false,
+  );
+
+  const corrupt = fixture({
+    executionOverrides: { contentDigest: 'f'.repeat(64) },
+  });
+  await assert.rejects(
+    new PostgresClusterScheduleRepository(
+      corrupt.pool,
+    ).commitClusterScheduleDecision(admissionCommand()),
+    PostgresClusterScheduleUnavailableError,
+  );
+  assert.equal(
+    corrupt.queries.some(({ text }) => text === 'ROLLBACK'),
+    true,
+  );
+});