fix: invalidate queued cron revisions and enforce task process timeouts

test/back/execution-lifecycle.test.cjs

@@ -36,7 +36,7 @@ test(
       },
       '../loaders/logger': logger,
       '../data/cron': {
-        CrontabModel: { findOne: async () => null },
+        CrontabModel: { findOne: async () => ({ isDisabled: 0 }) },
         CrontabStatus: {},
       },
       '../data/runningInstance': {

test/back/queued-cron-invalidation.test.cjs

@@ -0,0 +1,98 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const load = require('../helpers/load-security-module.cjs');
+const tick = () => new Promise(setImmediate);
+function gate() { let release; const promise = new Promise(r => release = r); return { promise, release }; }
+
+async function fixture() {
+  const logger = { info() {}, error() {}, warn() {} }, notices = [];
+  const limit = load('back/shared/pLimit.ts', {
+    '../data/system': { AuthDataType: { systemConfig: 'systemConfig' }, SystemModel: { sync: async () => {}, findOne: async () => ({ info: { cronConcurrency: 1 } }) } },
+    '../loaders/logger': logger,
+    '../services/notify': class {},
+    '../shared/i18n': { t: s => s, tf: s => s },
+    '../config': { grpcPort: 1 },
+    '../protos/api': { ApiClient: class { systemNotify(value, cb) { notices.push(value); cb(); } } },
+    '../config/grpcCerts': { getGrpcCerts: () => null },
+  }).default;
+  await tick();
+  const spawned = [], records = new Map(), jobs = new Map();
+  let readGate, killGate;
+  const { runCron } = load('back/shared/runCron.ts', {
+    'cross-spawn': { spawn: command => { spawned.push(command); return {}; } },
+    './pLimit': limit,
+    '../loaders/logger': logger,
+    '../data/cron': { CrontabModel: { findOne: async ({ where }) => { const row = records.get(String(where.id)); if (readGate) await readGate.promise; return row; }, update: async () => {} }, CrontabStatus: { running: 0, queued: 3, idle: 1 } },
+    '../data/runningInstance': { RunningInstanceModel: { update: async () => {} }, InstanceStatus: { running: 0, stopped: 2 } },
+    '../config/util': { killTask: async () => { if (killGate) await killGate.promise; } },
+    './childProcess': { observeChildProcess: () => ({ completed: Promise.resolve({ code: 0 }) }), asError: e => e },
+  });
+  const { addCron } = load('back/schedule/addCron.ts', {
+    '../shared/cronScheduler': { createCronJob: (schedule, callback) => ({ start() {}, cancel() {}, fire: callback }) },
+    '../shared/cronSchedule': { isValidCronSchedule: x => x !== 'invalid' },
+    './data': { scheduleStacks: jobs }, '../shared/runCron': { runCron }, '../loaders/logger': logger,
+    '../shared/i18n': { tf: s => s },
+  });
+  const { delCron } = load('back/schedule/delCron.ts', { './data': { scheduleStacks: jobs }, '../loaders/logger': logger });
+  function add(id = '1', command = 'old', replace = false) {
+    records.set(id, { isDisabled: 0, allow_multiple_instances: 1 });
+    addCron({ request: { crons: [{ id, command, schedule: '* * * * * *' }], replace } }, err => assert.ifError(err));
+    return jobs.get(id)[0];
+  }
+  function remove(id = '1') { delCron({ request: { ids: [id] } }, err => assert.ifError(err)); }
+  async function block() { const g = gate(); const done = limit.runWithCronLimit({ id: 'blocker' }, async () => { await g.promise; }); await tick(); return { release: async () => { g.release(); await done; } }; }
+  return { limit, spawned, records, jobs, add, remove, block, notices, setReadGate: g => readGate = g, setKillGate: g => killGate = g };
+}
+
+for (const operation of ['disable', 'delete', 'update', 'replace', 'disable-enable']) {
+  test(`queued scheduled execution is invalidated by ${operation}`, async () => {
+    const f = await fixture(), old = f.add(), blocker = await f.block();
+    const queued = old.fire();
+    if (operation === 'disable' || operation === 'disable-enable') { f.records.get('1').isDisabled = 1; f.remove(); }
+    if (operation === 'delete') { f.records.delete('1'); f.remove(); }
+    if (operation === 'update') f.add('1', 'new');
+    if (operation === 'replace') f.add('1', 'new', true);
+    if (operation === 'disable-enable') f.add('1', 'old');
+    const next = f.jobs.get('1')?.[0].fire();
+    await blocker.release(); await queued; await next;
+    assert.deepEqual(f.spawned, ['update', 'replace'].includes(operation) ? ['new'] : operation === 'disable-enable' ? ['old'] : []);
+    assert.equal(f.limit.cronLimitPendingCount, 0);
+    assert.equal(f.limit.cronLimitActiveCount, 0);
+  });
+}
+
+test('stale callbacks cannot re-enqueue after cancellation', async () => {
+  const f = await fixture(), old = f.add(); f.remove(); await old.fire();
+  assert.deepEqual(f.spawned, []);
+});
+
+test('invalidation during an asynchronous database read prevents spawning', async () => {
+  const f = await fixture(), old = f.add(), pending = gate(); f.setReadGate(pending);
+  const run = old.fire(); await tick(); f.remove(); pending.release(); await run;
+  assert.deepEqual(f.spawned, []);
+});
+
+test('invalidation while replacing a running instance prevents the old spawn', async () => {
+  const f = await fixture(), old = f.add(), pending = gate();
+  f.records.set('1', { isDisabled: 0, allow_multiple_instances: 0, pid: 10, status: 0 }); f.setKillGate(pending);
+  const run = old.fire(); await tick(); f.remove(); pending.release(); await run;
+  assert.deepEqual(f.spawned, []);
+});
+
+test('database removal or disable is respected before scheduler RPC arrives', async () => {
+  for (const disabled of [true, false]) {
+    const f = await fixture(), old = f.add(), blocker = await f.block(); const run = old.fire();
+    if (disabled) f.records.get('1').isDisabled = 1; else f.records.delete('1');
+    await blocker.release(); await run; assert.deepEqual(f.spawned, []);
+  }
+});
+
+test('five stale runs do not suppress the new revision or erase its repeat accounting', async () => {
+  const f = await fixture(), old = f.add(), blocker = await f.block();
+  const pending = Array.from({ length: 5 }, () => old.fire());
+  const current = f.add('1', 'new'); pending.push(...Array.from({ length: 5 }, () => current.fire()));
+  assert.equal(f.notices.length, 0);
+  await blocker.release(); await Promise.all(pending);
+  assert.deepEqual(f.spawned, Array(5).fill('new'));
+  await current.fire(); assert.equal(f.spawned.length, 6);
+});

test/back/task-shell-lifecycle.test.cjs

@@ -12,6 +12,7 @@ function extract(file, name) {
   return text.slice(start, text.indexOf('\n}', start) + 2);
 }
 const helpers = [
+  fs.readFileSync('shell/task-timeout.sh', 'utf8'),
   ...['handle_task_start', 'handle_task_end', 'run_task_before', 'run_task_after', 'get_env_array', 'clear_env'].map(n => extract('shell/share.sh', n)),
   ...['run_shell_script', 'define_program', 'format_params'].map(n => extract('shell/task.sh', n)),
 ].join('\n');
@@ -24,7 +25,7 @@ function fixture(t) {
   write('env.sh', 'export QA_PANEL="alpha&beta&gamma"\n');
   write('before.sh', 'export QA_BEFORE=ready\nqa_function() { printf "HOOK_FUNCTION\\n"; }\n');
   write('after.sh', 'printf "AFTER:%s:%s\\n" "$QA_PANEL" "$QA_BEFORE" >> "$QA_ROOT/events"\n');
-  const run = (args) => {
+  const run = (args, timeout = '') => {
     const r = spawnSync('/bin/bash', ['-c', helpers + `
       dir_scripts=$QA_ROOT; dir_shell=$QA_ROOT; dir_dep=$QA_ROOT
       file_env=$QA_ROOT/env.sh; file_task_before=$QA_ROOT/before.sh; file_task_after=$QA_ROOT/after.sh
@@ -40,7 +41,7 @@ function fixture(t) {
       format_params "$@"; define_program "${'${task_shell_params[@]}'}"
       . "$QA_TASK_SOURCE"
       printf 'WRAPPER_FINISHED\\n' >> "$QA_ROOT/events"
-    `, 'fixture', ...args], { cwd: root, env: { ...process.env, QA_ROOT: root, QA_TASK_SOURCE: taskSource }, encoding: 'utf8', timeout: 10000 });
+    `, 'fixture', ...args], { cwd: root, env: { ...process.env, command_timeout_time: timeout, QA_ROOT: root, QA_TASK_SOURCE: taskSource }, encoding: 'utf8', timeout: 10000 });
     assert.equal(r.status, 0, r.stdout + r.stderr);
     return { stdout: r.stdout, events: fs.readFileSync(path.join(root, 'events'), 'utf8').trim().split('\n') };
   };
@@ -117,3 +118,77 @@ for (const mode of ['desi', 'conc']) {
     assert.equal(r.events.filter(x => x.startsWith('AFTER:')).length, 1);
   });
 }
+
+for (const [label, args] of [
+  ['sourced shell', ['probe.sh', '--', 'space value']],
+  ['explicit bash', ['bash', 'probe.sh', 'space value']],
+]) {
+  test(`${label} timeout stops descendants and finalizes once with 124`, t => {
+    const f = fixture(t);
+    f.write('before.sh', 'export QA_BEFORE=ready\nprivate_value=private\nprivate_array=(one "two words")\nqa_function() { printf "HOOK_FUNCTION\\n"; }\n');
+    f.write('probe.sh', `[[ "$QA_PANEL" == 'alpha&beta&gamma' && "$QA_BEFORE" == ready && "$1" == 'space value' ]] || exit 91\n${label === 'sourced shell' ? '[[ "$private_value" = private && "${private_array[1]}" = "two words" ]] || exit 92\nqa_function\n' : ''}trap 'echo EXIT_TRAP' EXIT\nsleep 3\necho SHOULD_NOT_RUN\n`);
+    const before = Date.now();
+    const r = f.run(args, '0.2s');
+    assert.ok(Date.now() - before < 2500, r.stdout);
+    assert.doesNotMatch(r.stdout, /SHOULD_NOT_RUN/);
+    assert.match(r.stdout, /EXIT_TRAP/);
+    assert.deepEqual(r.events, ['STATUS:0:', 'AFTER:alpha&beta&gamma:ready', 'STATUS:1:124', 'STAT:124', 'WRAPPER_FINISHED']);
+  });
+}
+
+test('timeout escalates when a shell and its children ignore TERM', t => {
+  const f = fixture(t);
+  f.write('probe.sh', 'trap "" TERM\nsleep 3\necho SHOULD_NOT_RUN\n');
+  const before = Date.now();
+  const r = f.run(['probe.sh'], '0.1');
+  assert.ok(Date.now() - before < 2500);
+  assert.doesNotMatch(r.stdout, /SHOULD_NOT_RUN/);
+  assert.ok(r.events.includes('STATUS:1:124'));
+});
+
+test('completed timed tasks keep their exit code and do not wait for the timer', t => {
+  const f = fixture(t);
+  f.write('probe.sh', 'exit 7\n');
+  const before = Date.now();
+  const r = f.run(['probe.sh'], '1h');
+  assert.ok(Date.now() - before < 1500);
+  assert.ok(r.events.includes('STATUS:1:7'));
+});
+
+test('zero disables the deadline and invalid durations never start the script', t => {
+  const f = fixture(t);
+  f.write('probe.sh', 'echo DID_RUN\n');
+  assert.match(f.run(['probe.sh'], '0s').stdout, /DID_RUN/);
+  f.write('events', '');
+  const r = f.run(['probe.sh'], 'wrong');
+  assert.doesNotMatch(r.stdout, /DID_RUN/);
+  assert.ok(r.events.includes('STATUS:1:125'));
+});
+
+for (const [runtime, filename, body] of [
+  ['python3', 'timed.py', 'import time\nprint("STARTED", flush=True)\ntime.sleep(3)\nprint("SHOULD_NOT_RUN", flush=True)\n'],
+  ['node', 'timed.cjs', 'console.log("STARTED"); setTimeout(() => console.log("SHOULD_NOT_RUN"), 3000);\n'],
+]) {
+  test(`${runtime} timeout stops execution and reports 124`, t => {
+    const f = fixture(t);
+    f.write(filename, body);
+    const r = f.run([runtime, filename], '0.5s');
+    assert.match(r.stdout, /STARTED/);
+    assert.doesNotMatch(r.stdout, /SHOULD_NOT_RUN/);
+    assert.ok(r.events.includes('STATUS:1:124'));
+    assert.equal(r.events.filter(x => x.startsWith('AFTER:')).length, 1);
+  });
+}
+
+for (const mode of ['desi', 'conc']) {
+  test(`timed shell ${mode} preserves selected accounts and terminates every worker`, t => {
+    const f = fixture(t);
+    f.write('probe.sh', 'echo "ACCOUNT=$QA_PANEL"\nsleep 3\necho SHOULD_NOT_RUN\n');
+    const r = f.run(['probe.sh', mode, 'QA_PANEL', '2-3'], '0.2s');
+    assert.match(r.stdout, mode === 'desi' ? /ACCOUNT=beta&gamma/ : /ACCOUNT=beta\nACCOUNT=gamma/);
+    assert.doesNotMatch(r.stdout, /SHOULD_NOT_RUN/);
+    assert.equal(r.events.filter(x => x.startsWith('AFTER:')).length, 1);
+    // Concurrent mode retains its existing aggregate wait status behavior.
+    assert.ok(r.events.includes(`STATUS:1:${mode === 'desi' ? 124 : 0}`));
+  });
+}