refactor: migrate cron scheduling to node-cron (#3079)

* refactor: migrate cron scheduling to node-cron

* feat: support annually midnight and minutely cron macros

* fix: harden cron recovery and system scheduler compatibility

test/back/cron-macros.test.cjs

@@ -0,0 +1,51 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { createCronJob } = require('../../back/shared/cronScheduler');
+const { isValidCronSchedule } = require('../../back/shared/cronSchedule');
+
+// node-cron caches timezone formatters; model each deployment in a fresh process.
+if (!process.env.QL_MACRO_TEST_TZ) {
+  for (const tz of ['UTC', 'Asia/Shanghai']) {
+    test(`macro execution in ${tz}`, () => {
+      const result = require('node:child_process').spawnSync(process.execPath,
+        ['-r', 'ts-node/register/transpile-only', '--test', __filename],
+        { env: { ...process.env, TZ: tz, QL_MACRO_TEST_TZ: tz }, encoding: 'utf8' });
+      assert.equal(result.status, 0, result.stdout + result.stderr);
+    });
+  }
+}
+for (const tz of process.env.QL_MACRO_TEST_TZ ? [process.env.QL_MACRO_TEST_TZ] : []) {
+  for (const [macro, start, next] of [
+    ['@annually', [2026, 11, 31, 23, 59, 59], [[2027, 0, 1, 0, 0, 0], [2028, 0, 1, 0, 0, 0]]],
+    ['@midnight', [2026, 8, 27, 23, 59, 59], [[2026, 8, 28, 0, 0, 0], [2026, 8, 29, 0, 0, 0]]],
+    ['@minutely', [2026, 8, 27, 12, 34, 59], [[2026, 8, 27, 12, 35, 0], [2026, 8, 27, 12, 36, 0]]],
+  ]) {
+    test(`${macro} fires at the expected local times in ${tz}`, async (t) => {
+      const previousTZ = process.env.TZ;
+      process.env.TZ = tz;
+      t.after(() => { if (previousTZ === undefined) delete process.env.TZ; else process.env.TZ = previousTZ; });
+      t.mock.timers.enable({ apis: ['Date', 'setTimeout', 'setInterval'], now: new Date(...start) });
+      const expected = next.map((parts) => new Date(...parts).getTime());
+      const actual = [];
+      assert.equal(isValidCronSchedule(` ${macro} `), true);
+      const job = createCronJob(macro, (date) => actual.push(date.getTime()), {
+        name: macro, logger: { warn() {}, error() {} },
+      });
+      t.after(() => job.cancel());
+      for (const time of expected) {
+        t.mock.timers.tick(time - Date.now() - 1);
+        await new Promise(setImmediate);
+        assert.equal(actual.includes(time), false, 'must not fire early');
+        t.mock.timers.tick(1);
+        await new Promise(setImmediate);
+      }
+      assert.deepEqual(actual, expected);
+    });
+  }
+}
+
+test('new aliases do not enable unknown macros or raw parser extensions', () => {
+  for (const schedule of ['@annuallyx', '@midnightx', '@minutelyx', '@secondly', '@reboot', 'H * * * *', '0 /5 * * * *']) {
+    assert.equal(isValidCronSchedule(schedule), false, schedule);
+  }
+});

test/back/cron-schedule-compatibility.test.cjs

@@ -1,41 +1,69 @@
 const test = require('node:test');
 const assert = require('node:assert/strict');
-const nodeSchedule = require('node-schedule');
 const { isValidCronSchedule } = require('../../back/shared/cronSchedule');
+const { createCronJob } = require('../../back/shared/cronScheduler');
+const legacy = require('../fixtures/legacy-cron.json');
+const logger = { warn() {}, error() {} };
+const flush = () => new Promise(setImmediate);
 
-test('validation agrees with the actual scheduler across legacy cron syntax', () => {
-  const candidates = new Set([
-    '*', '0', '?', '0 0', '0 0 *', '0 0 * *',
-    '@yearly', '@annually', '@monthly', '@weekly', '@daily', '@midnight',
-    '@hourly', '@secondly', '@minutely', '@weekdays', '@weekends', '@reboot',
-    '0 0 1 1 * 2027', '0 0 0 L * *', '0 0 0 * * 5L',
-    '0 0 0 * * MON#2', '0 0 12 LW * *', '0 0 12 15W * *',
-    '0 0 0 ? * MON', '0 0/30 * * * ?', '0 /5 * * * *',
-  ]);
-  const fields = [
-    ['*', '?', '/5', '*/5', '0/5', '0', '01', '1-5', '1,3', '5-1', 'H', 'H/5', 'H(0-10)', 'L'],
-    ['*', '?', '/5', '*/5', '0/5', '0', '01', '1-5', '1,3', '5-1', 'H', 'H/5', 'H(0-10)', 'L'],
-    ['*', '?', '/2', '*/2', '0/2', '0', '01', '1-5', '1,3', '23-2', 'H', 'L'],
-    ['*', '?', '/2', '*/2', '1/2', '1', '01', '1-5', '1,3', 'L', 'L-1', 'LW', '15W', 'H'],
-    ['*', '?', '/2', '*/2', '1/2', '1', '01', '1-5', '1,3', 'JAN', 'jan', 'JAN-MAR', 'DEC-FEB', 'H'],
-    ['*', '?', '/2', '*/2', '0/2', '0', '7', '01', '1-5', '1,3', 'MON', 'mon', 'MON-FRI', 'FRI-MON', '5L', 'L', 'MON#2', '1#5', 'H'],
-  ];
-  fields.forEach((options, index) => options.forEach((field) => {
-    const values = ['0', '0', '0', '*', '*', '*'];
-    values[index] = field;
-    candidates.add(values.join(' '));
-  }));
-  for (const value of [...candidates]) {
-    if (value.split(' ').length === 6) {
-      candidates.add(` ${value} `);
-      candidates.add(value.split(' ').slice(1).join(' '));
-    }
+test('validation preserves legacy syntax except the explicitly added macros', () => {
+  const addedMacros = new Set(['@annually', '@midnight', '@minutely']);
+  assert.ok(legacy.validation.length > 250);
+  for (const { schedule, accepted } of legacy.validation) {
+    assert.equal(isValidCronSchedule(schedule), accepted || addedMacros.has(schedule), schedule);
   }
-  for (const value of candidates) {
-    const job = nodeSchedule.scheduleJob(value, () => {});
-    const accepted = Boolean(job);
-    job?.cancel();
-    assert.equal(isValidCronSchedule(value), accepted, value);
-  }
-  assert.ok(candidates.size > 250);
+});
+
+for (const sample of legacy.times) {
+  test(`calendar compatibility: ${sample.schedule} from ${sample.currentDate}`, async (t) => {
+    const previousTZ = process.env.TZ;
+    process.env.TZ = 'UTC';
+    t.after(() => { if (previousTZ === undefined) delete process.env.TZ; else process.env.TZ = previousTZ; });
+    t.mock.timers.enable({ apis: ['Date', 'setTimeout', 'setInterval'], now: new Date(sample.currentDate) });
+    const actual = [];
+    const job = createCronJob(sample.schedule, (date) => actual.push(date.toISOString()), { name: 'calendar', logger });
+    t.after(() => job.cancel());
+    for (const expected of sample.next) {
+      t.mock.timers.tick(new Date(expected).getTime() - Date.now());
+      await flush();
+    }
+    assert.deepEqual(actual, sample.next);
+  });
+}
+
+test('late callbacks catch up exactly once and cancellation destroys native tasks', async (t) => {
+  t.mock.timers.enable({ apis: ['Date', 'setTimeout', 'setInterval'], now: new Date('2026-09-27T00:00:00Z') });
+  const native = require('node-cron');
+  const before = native.getTasks().size;
+  const dates = [], warnings = [];
+  const job = createCronJob('* * * * * *', (date) => dates.push(date.getTime()), {
+    name: 'late', logger: { warn: (...args) => warnings.push(args), error() {} },
+  });
+  t.after(() => job.cancel());
+  t.mock.timers.tick(8000);
+  await flush();
+  assert.equal(dates.length, 8);
+  assert.equal(new Set(dates).size, 8);
+  assert.ok(warnings.length > 0);
+  job.cancel();
+  t.mock.timers.tick(5000);
+  await flush();
+  assert.equal(dates.length, 8);
+  assert.equal(native.getTasks().size, before);
+});
+
+test('callback rejection is logged and future executions continue', async (t) => {
+  t.mock.timers.enable({ apis: ['Date', 'setTimeout', 'setInterval'], now: new Date('2026-09-27T00:00:00Z') });
+  const errors = [];
+  let count = 0;
+  const job = createCronJob('* * * * * *', async () => { count++; throw Error('callback failed'); }, {
+    name: 'failure', logger: { warn() {}, error: (...args) => errors.push(args) },
+  });
+  t.after(() => job.cancel());
+  t.mock.timers.tick(1000);
+  await flush();
+  t.mock.timers.tick(1000);
+  await flush();
+  assert.equal(count, 2);
+  assert.equal(errors.length, 2);
 });

test/back/cron-scheduler-routing.test.cjs

@@ -25,6 +25,14 @@ test('system and node modes route extended cron once and keep portable rules in
     const rows = [
       { id: 'plain', schedule: '*/5 0-23 * * 1,3' },
       { id: 'spaces', schedule: '  0\t0 * * *  ' },
+      { id: 'numeric-step', schedule: '0/5 * * * *' },
+      { id: 'sunday-seven', schedule: '0 0 * * 7' },
+      { id: 'day-step', schedule: '0 0 */2 * *' },
+      { id: 'restricted-days', schedule: '0 0 1-31 * 1' },
+      { id: 'restricted-week', schedule: '0 0 1 * 0-6' },
+      { id: 'annually', schedule: '@annually' },
+      { id: 'midnight', schedule: '@midnight' },
+      { id: 'minutely', schedule: '@minutely' },
       { id: 'seconds', schedule: '0 0 * * * *' },
       { id: 'question', schedule: '0 0 * * ?' },
       { id: 'last', schedule: '0 0 L * *' },
@@ -43,7 +51,7 @@ test('system and node modes route extended cron once and keep portable rules in
     }
     await service.setCrontab({ data: rows, total: rows.length });
     assert.equal(installs, mode === 'system' ? 1 : 0);
-    for (const id of ['seconds', 'question', 'last', 'nth', 'named', 'macro', 'short', 'extra', 'once', 'boot']) {
+    for (const id of ['numeric-step', 'sunday-seven', 'day-step', 'restricted-days', 'restricted-week', 'annually', 'midnight', 'minutely', 'seconds', 'question', 'last', 'nth', 'named', 'macro', 'short', 'extra', 'once', 'boot']) {
       const line = file.split('\n').find((line) => line.endsWith(`task ${id}.js`));
       assert.ok(line.startsWith('# '), `${mode}: ${id} cannot also run from system crontab`);
     }

test/back/legacy-cron-recovery.test.cjs

@@ -4,6 +4,8 @@ const { format } = require('node:util');
 const load = require('../helpers/load-security-module.cjs');
 const { SchedulerReadiness } = require('../../back/shared/schedulerReadiness');
 
+const { getTasks } = require('node-cron');
+
 const legacySchedules = [
   '* * * * *',
   '*/5  * * * *',
@@ -54,8 +56,11 @@ test('legacy main and extra schedules restore real jobs and healthy readiness',
   assert.equal(stacks.size, crons.length);
   for (const jobs of stacks.values()) {
     assert.equal(jobs.length, 2);
-    assert.ok(jobs.every((job) => job.nextInvocation()));
+    assert.ok(jobs.every((job) => typeof job.cancel === 'function'));
   }
+  const nativeJobs = [...getTasks().values()];
+  assert.equal(nativeJobs.length, crons.length * 2);
+  assert.ok(nativeJobs.every((job) => job.getStatus() !== 'stopped'));
   const previousJobs = [...stacks.values()].flat();
   for (const schedule of ['@bogus', '0 /5 * * * ?', '0 70 * * * ?']) {
     for (const invalid of [
@@ -64,7 +69,8 @@ test('legacy main and extra schedules restore real jobs and healthy readiness',
     ]) {
       await assert.rejects(register([invalid]), (error) => error.code === 3);
       assert.deepEqual([...stacks.values()].flat(), previousJobs);
-      assert.ok(previousJobs.every((job) => job.nextInvocation()));
+      assert.deepEqual([...getTasks().values()], nativeJobs);
+      assert.ok(nativeJobs.every((job) => job.getStatus() !== 'stopped'));
     }
   }
 });

test/back/node-cron-migration.test.cjs

@@ -0,0 +1,65 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const load = require('../helpers/load-security-module.cjs');
+const { createCronJob } = require('../../back/shared/cronScheduler');
+const logger = { info() {}, warn() {}, error() {} };
+
+test('native calendar skips nonexistent spring time and duplicate fall hour', async (t) => {
+  const previous = process.env.TZ;
+  process.env.TZ = 'America/New_York';
+  t.after(() => { if (previous === undefined) delete process.env.TZ; else process.env.TZ = previous; });
+  t.mock.timers.enable({ apis: ['Date', 'setTimeout', 'setInterval'], now: new Date('2026-03-08T05:00:00Z') });
+  const springRuns = [];
+  const spring = createCronJob('0 30 2 * * *', (date) => springRuns.push(date.toISOString()), { name: 'spring', logger });
+  t.after(() => spring.cancel());
+  t.mock.timers.tick(new Date('2026-03-09T06:30:00Z') - Date.now());
+  await new Promise(setImmediate);
+  assert.deepEqual(springRuns, ['2026-03-09T06:30:00.000Z']);
+  spring.cancel();
+  t.mock.timers.setTime(new Date('2026-11-01T05:30:00Z').getTime());
+  const fallRuns = [];
+  const fall = createCronJob('0 30 1 * * *', (date) => fallRuns.push(date.toISOString()), { name: 'fall', logger });
+  t.after(() => fall.cancel());
+  t.mock.timers.tick(new Date('2026-11-02T06:30:00Z') - Date.now());
+  await new Promise(setImmediate);
+  assert.deepEqual(fallRuns, ['2026-11-02T06:30:00.000Z']);
+});
+
+test('construction failure cleans staged jobs and preserves the running snapshot', async () => {
+  const cancelled = [];
+  const stacks = new Map([['old', [{ cancel: () => cancelled.push('old') }]]]);
+  let count = 0;
+  const { addCron } = load('back/schedule/addCron.ts', {
+    './data': { scheduleStacks: stacks },
+    '../shared/cronScheduler': { createCronJob: () => {
+      if (++count === 2) throw Error('construction failed');
+      return { start() { throw Error('must not start'); }, cancel: () => cancelled.push('staged') };
+    } },
+    '../shared/runCron': {}, '../loaders/logger': logger, '../shared/i18n': { tf: require('node:util').format },
+  });
+  await assert.rejects(new Promise((resolve, reject) => addCron({ request: {
+    replace: true, crons: [{ id: 'new', schedule: '* * * * *', extra_schedules: [{ schedule: '0 * * * *' }] }],
+  } }, (error) => error ? reject(error) : resolve())), /construction failed/);
+  assert.deepEqual([...stacks.keys()], ['old']);
+  assert.deepEqual(cancelled, ['staged']);
+});
+
+test('invalid subscription schedules are isolated and immediate runs without a schedule survive', async () => {
+  const warnings = [], runs = [];
+  const Service = load('back/services/schedule.ts', {
+    typedi: { Service: () => (x) => x, Inject: () => () => {} },
+    '../shared/pLimit': {},
+  }).default;
+  const service = new Service({ ...logger, warn: (...args) => warnings.push(args) });
+  service.runTask = async (...args) => { runs.push(args); };
+  const item = { id: 123, name: 'subscription', command: 'true', runOrigin: 'subscription' };
+  await service.createCronTask({ ...item, schedule: 'not a cron' });
+  assert.equal(warnings.length, 1);
+  assert.equal(service.scheduleStacks.size, 0);
+  await service.createCronTask(item, {}, true);
+  assert.equal(runs.length, 1);
+  await service.createCronTask({ ...item, schedule: '* * * * *' });
+  assert.equal(service.scheduleStacks.size, 1);
+  await service.cancelCronTask(item);
+  assert.equal(service.scheduleStacks.size, 0);
+});

test/back/scheduler-bulk-recovery.test.cjs

@@ -0,0 +1,40 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const grpc = require('@grpc/grpc-js');
+const load = require('../helpers/load-security-module.cjs');
+const { SchedulerReadiness } = require('../../back/shared/schedulerReadiness');
+
+for (const [name, rows, delay, succeeds] of [
+  ['large distinct snapshot', Array.from({length:10000}, (_, i) => ({id:String(i), schedule:`${i%60} ${Math.floor(i/60)%60} ${Math.floor(i/3600)} * * *`})), 10000, true],
+  ['extra schedules', [{id:'extra', schedule:'* * * * *', extra_schedules:Array.from({length:2000},()=>({schedule:'0 * * * *'}))}], 10000, true],
+  ['unresponsive small write', [{id:'small', schedule:'* * * * *'}], 30000, false],
+  ['unresponsive large write remains bounded', Array.from({length:30000},()=>({schedule:'* * * * *'})), 180000, false],
+]) {
+  test(`recovery deadline: ${name}`, async (t) => {
+    t.mock.timers.enable({apis:['Date','setTimeout'], now:0});
+    let calls=0, deadline;
+    const fake={waitForReady:(_,cb)=>cb(), addCron:(_req,_meta,options,cb)=>{
+      calls++; deadline=options.deadline;
+      const budget=deadline-Date.now();
+      setTimeout(()=>cb(delay<=budget ? null : Object.assign(Error('deadline'),{code:grpc.status.DEADLINE_EXCEEDED}), {}), Math.min(delay,budget));
+    }};
+    const client=load('back/schedule/client.ts', {
+      '../protos/cron':{CronClient:class{constructor(){return fake}}},
+      '../config':{grpcPort:5500},
+      '../config/grpcCerts':{getGrpcCerts:()=>({caCert:'ca',clientKey:'key',clientCert:'cert'})},
+      '@grpc/grpc-js':{...grpc,credentials:{createSsl:()=>({})}},
+    }).default;
+    const readiness=new SchedulerReadiness(async()=>{}, 60000);
+    t.after(()=>clearTimeout(readiness.retry));
+    client.readiness=readiness;
+    readiness.configure(()=>client.addCron(rows,true));
+    const recovering=readiness.recover();
+    await new Promise(setImmediate);
+    assert.equal(calls,1);
+    assert.ok(deadline<=120000);
+    t.mock.timers.tick(Math.min(delay,deadline));
+    assert.equal(await recovering,succeeds);
+    assert.equal(calls,1,'uncertain writes are not replayed inline');
+    assert.equal(await readiness.check(),succeeds);
+  });
+}

test/back/scheduler-reconciliation.test.cjs

@@ -41,8 +41,10 @@ test('recovery reconciles a surviving scheduler after missed deletes and disable
   );
   const { addCron } = load('back/schedule/addCron.ts', {
     './data': { scheduleStacks: stacks },
-    'node-schedule': {
-      scheduleJob: (id) => ({ cancel: () => cancelled.push(id) }),
+    '../shared/cronScheduler': {
+      createCronJob: (_schedule, _callback, options) => ({
+        start() {}, cancel: () => cancelled.push(options.name.split(':')[0]),
+      }),
     },
     '../shared/runCron': {},
     '../loaders/logger': { info() {}, warn() {} },