qinglong/test/back/task-timeout-processes.test.cjs

const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawn } = require('node:child_process');
const psTree = require('ps-tree');
const test = require('node:test');

const bash = process.env.QL_TEST_BASH || '/bin/bash';
const helper = path.resolve(__dirname, '../../shell/task-timeout.sh');
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const tree = (pid) =>
  new Promise((resolve, reject) =>
    psTree(pid, (error, children) =>
      error ? reject(error) : resolve(children),
    ),
  );
async function until(predicate, message) {
  const deadline = Date.now() + 5000;
  while (!(await predicate())) {
    assert.ok(Date.now() < deadline, message);
    await delay(20);
  }
}
function alive(pid) {
  try {
    process.kill(pid, 0);
    if (process.platform === 'linux') {
      try {
        const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8');
        if (
          ['Z', 'X', 'x'].includes(
            stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0],
          )
        )
          return false;
      } catch (error) {
        if (error.code === 'ENOENT') return false;
        throw error;
      }
    }
    return true;
  } catch (error) {
    if (error.code === 'ESRCH') return false;
    throw error;
  }
}

for (const mode of ['timeout', 'manual']) {
  test(
    `${mode} cleans up TERM-resistant children, grandchildren and the watchdog`,
    { timeout: 12000, skip: process.platform === 'win32' },
    async (t) => {
      const root = fs.mkdtempSync(
        path.join(os.tmpdir(), 'ql-timeout-processes-'),
      );
      const known = new Set();
      let child;
      t.after(async () => {
        if (child) {
          for (const p of await tree(child.pid).catch(() => []))
            known.add(Number(p.PID));
        }
        for (const pid of known) {
          if (pid > 1) {
            try {
              process.kill(pid, 'SIGKILL');
            } catch {}
          }
        }
        fs.rmSync(root, { recursive: true, force: true });
      });
      fs.writeFileSync(
        path.join(root, 'child.sh'),
        `
      trap '' TERM
      printf '%s' "$$" > "$QA_ROOT/child"
      sleep 30 &
      printf '%s' "$!" > "$QA_ROOT/grandchild"
      wait
    `,
      );
      child = spawn(
        bash,
        [
          '-c',
          `
      . "$QA_HELPER"
      task() {
        trap '' TERM
        /bin/sh -c 'printf "%s" "$PPID"' > "$QA_ROOT/worker"
        "$QA_BASH" "$QA_ROOT/child.sh" &
        wait
      }
      run_task_command task
      exit $?
    `,
        ],
        {
          detached: true,
          env: {
            ...process.env,
            QA_ROOT: root,
            QA_HELPER: helper,
            QA_BASH: bash,
            command_timeout_time: mode === 'timeout' ? '1s' : '1h',
          },
          stdio: ['ignore', 'pipe', 'pipe'],
        },
      );
      known.add(child.pid);
      let result,
        stderr = '';
      child.stdout.resume();
      child.stderr.on('data', (chunk) => (stderr += chunk));
      child.on('exit', (code, signal) => {
        result = { code, signal };
      });
      await until(
        () =>
          ['worker', 'child', 'grandchild'].every((name) => {
            const file = path.join(root, name);
            return (
              fs.existsSync(file) && Number(fs.readFileSync(file, 'utf8')) > 1
            );
          }),
        'task process tree did not initialize',
      );
      const descendants = await tree(child.pid);
      for (const p of descendants) known.add(Number(p.PID));
      for (const name of ['worker', 'child', 'grandchild'])
        known.add(Number(fs.readFileSync(path.join(root, name), 'utf8')));
      assert.ok(
        known.size >= 6,
        'must observe the wrapper, watchdog and task descendants',
      );
      if (mode === 'manual') {
        const worker = Number(
          fs.readFileSync(path.join(root, 'worker'), 'utf8'),
        );
        const wrapper = descendants.find((p) => Number(p.PID) === worker);
        assert.ok(wrapper, 'worker must belong to this test process tree');
        process.kill(Number(wrapper.PPID), 'SIGTERM');
      }
      await until(() => result, 'task wrapper did not exit');
      assert.equal(result.code, mode === 'timeout' ? 124 : 143, stderr);
      await until(
        () => [...known].every((pid) => !alive(pid)),
        'task or watchdog process survived cleanup',
      );
    },
  );
}