fix: validate ARMv6 upgrades and publish rolling development bundles

test/back/mirror-status.test.cjs

@@ -50,6 +50,30 @@ test('mirror verification reads matching refs without changing either repository
   assert.equal(git('-C', source, 'status', '--porcelain'), '');
 });
 
+test('mirror verification selects GNU timeout when the default is another implementation', (t) => {
+  const { tmp, source, mirror } = fixture(t);
+  git('-C', source, 'push', mirror, 'develop');
+  const realTimeout = ['timeout', 'gnutimeout', 'gtimeout'].find((name) => {
+    const result = spawnSync(name, ['--version'], { encoding: 'utf8' });
+    return result.status === 0 && result.stdout.includes('(GNU coreutils)');
+  });
+  assert.ok(realTimeout, 'The mirror script requires GNU timeout');
+  const bin = path.join(tmp, 'bin');
+  fs.mkdirSync(bin);
+  fs.symlinkSync(execFileSync('which', [realTimeout], { encoding: 'utf8' }).trim(), path.join(bin, 'gnutimeout'));
+  fs.symlinkSync(execFileSync('which', ['git'], { encoding: 'utf8' }).trim(), path.join(bin, 'git'));
+  fs.symlinkSync(execFileSync('which', ['sort'], { encoding: 'utf8' }).trim(), path.join(bin, 'sort'));
+  fs.writeFileSync(path.join(bin, 'timeout'), '#!/bin/sh\nif [ "$1" = --version ]; then echo "timeout (uutils coreutils)"; exit 0; fi\nexit 98\n', { mode: 0o755 });
+  const args = [script, source, mirror, 'develop'];
+  const result = spawnSync('/bin/bash', args, options({ PATH: bin }));
+  assert.equal(result.status, 0, result.stderr + result.stdout);
+  assert.match(result.stdout, /Mirror matches the latest source refs/);
+  fs.unlinkSync(path.join(bin, 'gnutimeout'));
+  const unsupported = spawnSync('/bin/bash', args, options({ PATH: bin }));
+  assert.equal(unsupported.status, 2, unsupported.stderr + unsupported.stdout);
+  assert.match(unsupported.stderr, /GNU timeout is required/);
+});
+
 test('mirror polling follows a source that advances while waiting', async (t) => {
   const { source, mirror } = fixture(t);
   git('-C', source, 'push', mirror, 'develop');

test/back/node-runtime.test.cjs

@@ -0,0 +1,72 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const {
+  serverRuntimeArgs,
+} = require('../../cli/src/internal/maintenance/nodeRuntime.cjs');
+
+test('server compatibility flags target ARMv6 Node 24 while retaining WebAssembly and the existing Node', () => {
+  for (const [arch, armVersion, version, enabled] of [
+    ['arm', '6', '24.18.1', true],
+    ['arm', 6, '24.18.1', true],
+    ['arm', 6, '20.18.1', false],
+    ['arm', 6, '22.20.0', false],
+    ['arm', 7, '24.18.1', false],
+    ['arm64', undefined, '24.18.1', false],
+    ['x64', undefined, '24.18.1', false],
+  ]) {
+    const flags = serverRuntimeArgs({
+      arch,
+      config: { variables: { arm_version: armVersion } },
+      versions: { node: version },
+    });
+    assert.deepEqual(
+      flags,
+      enabled ? ['--no-opt', '--no-sparkplug', '--no-maglev'] : [],
+    );
+    assert.ok(!flags.includes('--jitless'));
+  }
+});
+
+test('the normal PM2 application uses the same server arguments as prestart', () => {
+  const app = require('../../ecosystem.config.js').apps.find(
+    (x) => x.name === 'qinglong',
+  );
+  const flags = serverRuntimeArgs();
+  assert.deepEqual(app.node_args || [], flags);
+  if (!flags.length) assert.equal(app.node_args, undefined);
+});
+
+test('ARMv6 server flags retain real HTTP, fetch and undici parsing', () => {
+  const { spawnSync } = require('node:child_process');
+  const flags = serverRuntimeArgs({
+    arch: 'arm',
+    config: { variables: { arm_version: 6 } },
+    versions: { node: '24.18.1' },
+  });
+  const result = spawnSync(
+    process.execPath,
+    [
+      ...flags,
+      '-e',
+      `
+    const http=require('http'),assert=require('assert/strict');
+    assert.equal(typeof WebAssembly,'object');
+    const server=http.createServer((req,res)=>res.end('parser-ok'));
+    server.listen(0,'127.0.0.1',async()=>{
+      try {
+        const url='http://127.0.0.1:'+server.address().port;
+        assert.equal(await (await fetch(url)).text(),'parser-ok');
+        assert.equal(await (await require('undici').request(url)).body.text(),'parser-ok');
+      } catch(e){console.error(e);process.exitCode=1;}
+      finally{server.closeAllConnections();server.close();}
+    });
+  `,
+    ],
+    {
+      encoding: 'utf8',
+      timeout: 10000,
+      env: { ...process.env, NODE_OPTIONS: '' },
+    },
+  );
+  assert.equal(result.status, 0, result.stderr || result.error?.message);
+});

test/back/platform-bundle.test.cjs

@@ -7,6 +7,67 @@ const tar = require('tar');
 const bundle = require('../../cli/src/internal/maintenance/platformBundle.cjs');
 const artifacts = require('../../shell/upgrade.cjs');
 
+test('dependency probes allow slow ARMv6 startup while preserving cancellation and normal Node execution', async () => {
+  const vm = require('node:vm');
+  const filename = require.resolve(
+    '../../cli/src/internal/maintenance/platformBundle.cjs',
+  );
+  const text = await fs.readFile(filename, 'utf8');
+  for (const [arch, armVersion, version, timeout] of [
+    ['arm', 6, '20.18.1', 300000],
+    ['arm', 6, '24.18.1', 300000],
+    ['arm', 7, '24.18.1', 30000],
+    ['x64', undefined, '24.18.1', 30000],
+  ]) {
+    let launch;
+    const m = { exports: {} };
+    const controller = new AbortController();
+    const localRequire = (name) => {
+      if (name === 'node:child_process')
+        return {
+          execFile(program, args, options, callback) {
+            launch = { program, args, options };
+            callback(null, '', '');
+          },
+        };
+      return require(require.resolve(name, {
+        paths: [path.dirname(filename)],
+      }));
+    };
+    vm.runInNewContext(
+      text,
+      {
+        module: m,
+        exports: m.exports,
+        require: localRequire,
+        process: {
+          arch,
+          config: { variables: { arm_version: armVersion } },
+          versions: { node: version },
+          execPath: process.execPath,
+          env: { NODE_OPTIONS: '--jitless', NODE_PATH: '/outside' },
+        },
+      },
+      { filename },
+    );
+    await m.exports.smokeDependencies('/staged/payload', controller.signal);
+    assert.equal(launch.program, process.execPath);
+    assert.equal(launch.args.at(-2), '-e');
+    assert.deepEqual(
+      Array.from(launch.args.slice(0, -2)),
+      arch === 'arm' && armVersion === 6 && version.startsWith('24.')
+        ? ['--no-opt', '--no-sparkplug', '--no-maglev']
+        : [],
+    );
+    assert.ok(!launch.args.includes('--jitless'));
+    assert.equal(launch.options.cwd, '/staged/payload');
+    assert.equal(launch.options.timeout, timeout);
+    assert.equal(launch.options.signal, controller.signal);
+    assert.equal(launch.options.env.NODE_OPTIONS, '');
+    assert.equal(launch.options.env.NODE_PATH, '');
+  }
+});
+
 for (const failure of ['prestart', 'repair'])
   test(`failed ${failure} preserves the installed dependencies and previous ready pointer`, async (t) => {
     const f = await fixture(t);
@@ -395,6 +456,7 @@ test('native target compatibility distinguishes ARMv6/ARMv7 and libc and retains
   bundle.assertCompatible(manifest, target, '20.20.2');
   const glibcTarget = {
     ...target,
+    arch: 'armv7',
     libc: 'glibc',
     libcVersion: '2.31',
     endianness: 'LE',
@@ -405,7 +467,7 @@ test('native target compatibility distinguishes ARMv6/ARMv7 and libc and retains
     target: { ...glibcTarget, libcMinimum: '2.31' },
   };
   bundle.assertCompatible(glibcManifest, glibcTarget, '20.18.1');
-  assert.equal(bundle.targetKey(glibcTarget), 'linux-armv6-glibc');
+  assert.equal(bundle.targetKey(glibcTarget), 'linux-armv7-glibc');
   assert.throws(
     () =>
       bundle.assertCompatible(glibcManifest, {
@@ -529,10 +591,10 @@ test('publishing requires every platform and validates all immutable archive has
     );
   }
   const index = releaseIndex(f.directory, sourceCommit, repository);
-  assert.equal(Object.keys(index.packages).length, 13);
+  assert.equal(Object.keys(index.packages).length, 12);
   const armv6 = path.join(
     f.directory,
-    `qinglong-${sourceCommit}-linux-armv6-glibc.tar.gz`,
+    `qinglong-${sourceCommit}-linux-armv6-musl.tar.gz`,
   );
   const armv6Bytes = await fs.readFile(armv6);
   await fs.rm(armv6);

test/back/primary-apm.test.cjs

@@ -2,11 +2,12 @@ const test = require('node:test');
 const assert = require('node:assert/strict');
 const vm = require('node:vm');
 const fs = require('node:fs');
+const configRequire = require('node:module').createRequire(require('node:path').resolve('ecosystem.config.js'));
 const source = fs.readFileSync('ecosystem.config.js', 'utf8');
 const appSource = fs.readFileSync('back/app.ts', 'utf8');
 const ts = require('typescript');
 const compiled = ts.transpileModule(appSource.slice(0, appSource.indexOf('\nconst app = new Application();')) + '\nmodule.exports = Application;', {compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2022,esModuleInterop:true}}).outputText;
-function config(env) {const module={exports:{}};vm.runInNewContext(source,{module,process:{env}});return module.exports.apps[0];}
+function config(env) {const module={exports:{}};vm.runInNewContext(source,{module,process:{env},require:configRequire});return module.exports.apps[0];}
 test('container APM policy keeps worker isolation and can restore all inherited probes', () => {
   for (const primary of [undefined, 'true']) for (const workers of [undefined, 'true']) {
     const env={QL_CONTAINER:'true',QL_PRIMARY_APM:primary,QL_WORKER_APM:workers};

test/back/publish-upgrade-bundles.test.cjs

@@ -69,6 +69,15 @@ function fixture(t) {
       const route = args[1];
       if (route.endsWith('/git/ref/heads/master')) return state.head;
       if (route.endsWith('/git/ref/heads/develop')) return state.developHead;
+      if (route.endsWith('/releases?per_page=100'))
+        return JSON.stringify([
+          Array.from(state.releases.keys(), (tag_name) => ({
+            tag_name,
+            prerelease: true,
+            draft: false,
+            ...state.releaseMetadata?.[tag_name],
+          })),
+        ]);
       if (route.includes('/releases/tags/')) {
         const tag = decodeURIComponent(route.split('/releases/tags/')[1]);
         if (state.lookupFailure === tag)
@@ -118,13 +127,23 @@ function fixture(t) {
           digest: `sha256:${sha256(bytes)}`,
         });
       }
+      if (tag === 'upgrade-develop' && state.corruptChannel)
+        release.get('latest.json').digest = 'sha256:wrong';
       if (files.length > 1) {
         if (state.corruptUploadedBatch)
           release.get(path.basename(files[0])).digest = 'sha256:wrong';
         if (state.changeHeadAfterUpload) state.head = 'b'.repeat(40);
+        if (state.changeDevelopHeadAfterUpload)
+          state.developHead = 'd'.repeat(40);
       }
       return '';
     }
+    if (args[0] === 'release' && args[1] === 'delete') {
+      if (state.failDelete) throw new Error('Delete unavailable');
+      assert.ok(state.releases.has(args[2]));
+      state.releases.delete(args[2]);
+      return '';
+    }
     if (args[0] === 'release' && args[1] === 'edit') return '';
     throw new Error(`Unexpected gh call: ${JSON.stringify(args)}`);
   };
@@ -143,10 +162,10 @@ test('publishes every platform and installer before advancing version and legacy
   assert.equal(index.version, '2.24.0');
   assert.equal(index.buildId, '100-1');
   assert.equal(index.sourceCommit, f.options.commit);
-  assert.equal(Object.keys(index.packages).length, 13);
+  assert.equal(Object.keys(index.packages).length, 12);
   assert.deepEqual(index, f.index('upgrade-v2.24.0'));
   const tag = 'upgrade-build-v2.24.0-100-1';
-  assert.equal(f.state.releases.get(tag).size, 28);
+  assert.equal(f.state.releases.get(tag).size, 26);
   for (const entry of [...Object.values(index.packages), index.installer]) {
     const url = new URL(entry.url),
       filename = path.basename(url.pathname);
@@ -230,7 +249,7 @@ for (const fault of ['missing', 'corrupt', 'wrong-version'])
     const f = fixture(t);
     const archive = path.join(
       f.directory,
-      `qinglong-${f.options.commit}-linux-armv6-glibc.tar.gz`,
+      `qinglong-${f.options.commit}-linux-armv6-musl.tar.gz`,
     );
     if (fault === 'missing') fs.rmSync(archive);
     else if (fault === 'corrupt') fs.appendFileSync(archive, 'corrupt');
@@ -288,7 +307,7 @@ test('an explicitly rerun earlier workflow may replace the same version and comm
 
 test('non-master branches, invalid build IDs and API failures cannot overwrite published versions', (t) => {
   const f = fixture(t);
-  for (const branch of ['develop', 'v2.24.0', 'feature/test'])
+  for (const branch of ['debian-dev', 'v2.24.0', 'feature/test'])
     assert.throws(() => f.publish({ branch }), /Only master/);
   for (const buildId of [undefined, '100', '100-1/other'])
     assert.throws(() => f.publish({ buildId }), /Invalid bundle build ID/);
@@ -316,7 +335,7 @@ test('existing master clients can migrate from indexes without version/build met
   assert.equal(f.index().buildId, '100-2');
 });
 
-test('explicit development publication shares musl/glibc packages without changing version or latest production assets', (t) => {
+test('automatic development publication shares musl/glibc packages without changing version or latest production assets', (t) => {
   const f = fixture(t);
   f.publish();
   const production = f.index();
@@ -325,18 +344,17 @@ test('explicit development publication shares musl/glibc packages without changi
     f.publish({
       branch: 'develop',
       commit: f.state.developHead,
-      allowDevelopment: true,
     }),
     true,
   );
   const development = f.index('upgrade-develop');
   assert.equal(development.channel, 'develop');
   assert.equal(development.sourceCommit, f.state.developHead);
-  assert.equal(Object.keys(development.packages).length, 13);
+  assert.equal(Object.keys(development.packages).length, 12);
   assert.ok(development.packages['linux-x64-musl']);
   assert.ok(development.packages['linux-x64-glibc']);
   assert.ok(development.packages['linux-armv6-musl']);
-  assert.ok(development.packages['linux-armv6-glibc']);
+  assert.equal(development.packages['linux-armv6-glibc'], undefined);
   for (const entry of [
     ...Object.values(development.packages),
     development.installer,
@@ -344,44 +362,30 @@ test('explicit development publication shares musl/glibc packages without changi
     assert.ok(entry.url.includes('/upgrade-build-develop-v2.24.0-100-1/'));
   assert.deepEqual(f.index(), production);
   assert.deepEqual(f.index('upgrade-v2.24.0'), production);
-  assert.equal(f.state.releases.get('upgrade-build-v2.24.0-100-1').size, 28);
+  assert.equal(f.state.releases.get('upgrade-build-v2.24.0-100-1').size, 26);
   assert.equal(
     f.state.releases.get('upgrade-build-develop-v2.24.0-100-1').size,
-    28,
+    26,
   );
 });
 
-test('development builds require explicit boolean opt-in and cannot publish obsolete development source', (t) => {
+test('development builds cannot publish obsolete development source', (t) => {
   const f = fixture(t);
-  for (const allowDevelopment of [false, undefined, 'true'])
-    assert.throws(
-      () => f.publish({ branch: 'develop', allowDevelopment }),
-      /explicitly requested/,
-    );
-  assert.throws(
-    () => f.publish({ branch: 'feature/test', allowDevelopment: true }),
-    /Only master/,
-  );
-  assert.equal(f.state.calls.length, 0);
   f.write(f.state.developHead, 'development');
   const obsolete = f.state.developHead;
   f.state.developHead = 'd'.repeat(40);
-  assert.equal(
-    f.publish({ branch: 'develop', commit: obsolete, allowDevelopment: true }),
-    false,
-  );
+  assert.equal(f.publish({ branch: 'develop', commit: obsolete }), false);
   assert.equal(f.state.releases.size, 0);
 });
 
 for (const fault of ['failBatchUpload', 'corruptUploadedBatch'])
-  test(`failed manual development publication preserves both channels (${fault})`, (t) => {
+  test(`failed automatic development publication preserves both channels (${fault})`, (t) => {
     const f = fixture(t);
     f.publish();
     const production = f.index();
     const options = {
       branch: 'develop',
       commit: f.state.developHead,
-      allowDevelopment: true,
     };
     f.write(options.commit, 'first development');
     f.publish(options);
@@ -396,3 +400,98 @@ for (const fault of ['failBatchUpload', 'corruptUploadedBatch'])
     assert.deepEqual(f.index(), production);
     assert.deepEqual(f.index('upgrade-v2.24.0'), production);
   });
+
+for (const fault of [
+  null,
+  'failBatchUpload',
+  'corruptUploadedBatch',
+  'corruptChannel',
+])
+  test(`development retention deletes old batches only after verified publication (${
+    fault || 'success'
+  })`, (t) => {
+    const f = fixture(t);
+    f.publish();
+    const production = f.index();
+    const options = { branch: 'develop', commit: f.state.developHead };
+    f.write(options.commit, 'development');
+    f.publish(options);
+    const oldTag = 'upgrade-build-develop-v2.24.0-100-1';
+    // Never prune production, channel aliases, legacy/unclassified releases,
+    // drafts or a non-prerelease even when its name resembles a dev build.
+    const protectedTags = [
+      'upgrade-86c37f426a5ba9945d6af1910a7885a26f77e4bc',
+      'v2.24.0',
+      'upgrade-build-develop-v2.24.0-10-1',
+      'upgrade-build-develop-v2.24.0-11-1',
+    ];
+    for (const tag of protectedTags) f.state.releases.set(tag, new Map());
+    f.state.releaseMetadata = {
+      [protectedTags[2]]: { draft: true },
+      [protectedTags[3]]: { prerelease: false },
+    };
+    f.write(options.commit, 'next development');
+    if (fault) f.state[fault] = true;
+    if (fault)
+      assert.throws(
+        () => f.publish({ ...options, buildId: '101-1' }),
+        /Upload interrupted|verification failed/,
+      );
+    else {
+      assert.equal(f.publish({ ...options, buildId: '101-1' }), true);
+      assert.equal(f.index('upgrade-develop').buildId, '101-1');
+      const deletion = f.state.calls.findIndex(
+        (a) => a[0] === 'release' && a[1] === 'delete',
+      );
+      const aliasEdit = f.state.calls.findLastIndex(
+        (a) => a[0] === 'release' && a[1] === 'edit',
+      );
+      assert.ok(deletion > aliasEdit);
+      assert.ok(
+        f.state.calls
+          .slice(aliasEdit + 1, deletion)
+          .some(
+            (a) =>
+              a[0] === 'api' && a[1].includes('/releases/tags/upgrade-develop'),
+          ),
+      );
+      assert.ok(f.state.releases.has('upgrade-build-develop-v2.24.0-101-1'));
+    }
+    assert.equal(f.state.releases.has(oldTag), Boolean(fault));
+    for (const tag of protectedTags) assert.ok(f.state.releases.has(tag));
+    assert.deepEqual(f.index(), production);
+    assert.deepEqual(f.index('upgrade-v2.24.0'), production);
+  });
+
+test('failed development cleanup is retryable without overwriting immutable assets', (t) => {
+  const f = fixture(t);
+  const options = { branch: 'develop', commit: f.state.developHead };
+  f.write(options.commit, 'development');
+  f.publish(options);
+  f.state.failDelete = true;
+  assert.throws(
+    () => f.publish({ ...options, buildId: '101-1' }),
+    /Delete unavailable/,
+  );
+  assert.equal(f.index('upgrade-develop').buildId, '101-1');
+  assert.ok(f.state.releases.has('upgrade-build-develop-v2.24.0-100-1'));
+  f.state.failDelete = false;
+  f.publish({ ...options, buildId: '101-1' });
+  assert.ok(!f.state.releases.has('upgrade-build-develop-v2.24.0-100-1'));
+  assert.ok(f.state.releases.has('upgrade-build-develop-v2.24.0-101-1'));
+});
+
+test('a newer development commit during upload preserves the published channel and its old package', (t) => {
+  const f = fixture(t);
+  const options = { branch: 'develop', commit: f.state.developHead };
+  f.write(options.commit, 'development');
+  f.publish(options);
+  const before = f.index('upgrade-develop');
+  f.state.changeDevelopHeadAfterUpload = true;
+  assert.equal(f.publish({ ...options, buildId: '101-1' }), false);
+  assert.deepEqual(f.index('upgrade-develop'), before);
+  assert.ok(f.state.releases.has('upgrade-build-develop-v2.24.0-100-1'));
+  assert.ok(
+    !f.state.calls.some((a) => a[0] === 'release' && a[1] === 'delete'),
+  );
+});

test/back/start-certificate-preparation.test.cjs

@@ -6,7 +6,9 @@ const path = require('node:path');
 const { spawnSync } = require('node:child_process');
 
 const startup = fs.readFileSync('shell/start-simplify.sh', 'utf8');
-const preparation = startup.slice(
+const runtimeLoader = fs.readFileSync('shell/share.sh', 'utf8')
+  .match(/^load_node_runtime_args\(\) \{[\s\S]*?^\}/m)[0];
+const preparation = `${runtimeLoader}\ndir_root="$QL_DIR"\n` + startup.slice(
   startup.indexOf('\nfix_config\n'),
   startup.indexOf('log_with_style "INFO" "$(t \'🔄'),
 );

test/back/static-mirror.test.cjs

@@ -6,6 +6,61 @@ const os = require('node:os');
 const { execFileSync, spawnSync } = require('node:child_process');
 const { writeManifest } = require('../helpers/upgrade-fixture.cjs');
 
+test('both static publication steps supply a token as the HTTPS password and reject missing credentials', () => {
+  const workflow = require('js-yaml').load(
+    fs.readFileSync('.github/workflows/build-docker-image.yml', 'utf8'),
+  );
+  for (const name of [
+    'copy to static repo',
+    'publish verified upgrade download sizes',
+  ]) {
+    const step = workflow.jobs['build-static'].steps.find(
+      (step) => step.name === name,
+    );
+    assert.equal(step.env.STATIC_TOKEN, '${{ secrets.API_TOKEN }}');
+    const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-static-auth-'));
+    try {
+      // Capture the real workflow command's argv, without contacting any remote.
+      for (const program of ['bash', 'node']) {
+        fs.writeFileSync(
+          path.join(tmp, program),
+          '#!/bin/sh\nprintf "%s\\n" "$@" > "$QA_ARGS"\n',
+          { mode: 0o755 },
+        );
+      }
+      const argsFile = path.join(tmp, 'args');
+      const env = {
+        PATH: tmp,
+        QA_ARGS: argsFile,
+        STATIC_TOKEN: 'fixture-secret',
+        GITHUB_REPO: 'github.com/fixture/static',
+        GITHUB_BRANCH: 'develop',
+        GITHUB_REF_TYPE: 'branch',
+        GITHUB_SHA: 'a'.repeat(40),
+      };
+      const result = spawnSync('/bin/bash', ['-euc', step.run], {
+        env,
+        encoding: 'utf8',
+      });
+      assert.equal(result.status, 0, result.stderr);
+      const remote = fs.readFileSync(argsFile, 'utf8').split('\n')[1];
+      const url = new URL(remote);
+      assert.equal(url.username, 'x-access-token');
+      assert.equal(url.password, env.STATIC_TOKEN);
+      fs.unlinkSync(argsFile);
+      assert.notEqual(
+        spawnSync('/bin/bash', ['-euc', step.run], {
+          env: { ...env, STATIC_TOKEN: '' },
+        }).status,
+        0,
+      );
+      assert.equal(fs.existsSync(argsFile), false);
+    } finally {
+      fs.rmSync(tmp, { recursive: true, force: true });
+    }
+  }
+});
+
 test('static mirror receives complete orphan snapshots of each requested branch and reports rejection', (t) => {
   const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-static-mirror-'));
   t.after(() => fs.rmSync(tmp, { recursive: true, force: true }));

test/back/upgrade-bundle-workflow.test.cjs

@@ -7,6 +7,14 @@ const yaml = require('js-yaml');
 const workflow = yaml.load(
   fs.readFileSync('.github/workflows/build-docker-image.yml', 'utf8'),
 );
+
+test('shared server runtime changes trigger the image and upgrade verification workflow', () => {
+  const paths = workflow.on.push.paths;
+  assert.ok(
+    paths.indexOf('cli/src/internal/maintenance/nodeRuntime.cjs') >
+      paths.indexOf('!cli/**'),
+  );
+});
 // These conditions use GitHub's boolean operators and format(), which have the
 // same semantics for the boolean/string contexts exercised here.
 const evaluate = (expression, github, inputs = {}) => {
@@ -25,25 +33,14 @@ const evaluate = (expression, github, inputs = {}) => {
       );
 };
 
-test('both bundle jobs require master or an explicit development dispatch', () => {
-  assert.equal(
-    workflow.on.workflow_dispatch.inputs.build_development_bundles.default,
-    false,
-  );
+test('both channels automatically build bundles, with static-only and Git-tag runs excluded', () => {
   const cases = [
-    ['push', 'refs/heads/develop', {}, false],
-    ['push', 'refs/heads/develop', { build_development_bundles: true }, false],
-    ['workflow_dispatch', 'refs/heads/develop', {}, false],
+    ['push', 'refs/heads/develop', {}, true],
+    ['workflow_dispatch', 'refs/heads/develop', {}, true],
     [
       'workflow_dispatch',
       'refs/heads/develop',
-      { build_development_bundles: true },
-      true,
-    ],
-    [
-      'workflow_dispatch',
-      'refs/heads/develop',
-      { build_development_bundles: true, static_sync_only: true },
+      { static_sync_only: true },
       false,
     ],
     ['push', 'refs/heads/master', {}, true],
@@ -55,12 +52,7 @@ test('both bundle jobs require master or an explicit development dispatch', () =
       false,
     ],
     ['push', 'refs/tags/v2.24.0', {}, false],
-    [
-      'workflow_dispatch',
-      'refs/tags/v2.24.0',
-      { build_development_bundles: true },
-      false,
-    ],
+    ['workflow_dispatch', 'refs/tags/v2.24.0', {}, false],
   ];
   for (const name of ['build-upgrade-bundles', 'publish-upgrade-bundles'])
     for (const [event_name, ref, inputs, expected] of cases)
@@ -84,26 +76,53 @@ test('both bundle jobs require master or an explicit development dispatch', () =
   );
 });
 
-test('manual development releases serialize separately while normal develop CI remains independent', () => {
+test('development image and bundle publication serialize separately from production and static-only runs', () => {
   const group = (event_name, ref, inputs = {}, run_id = '100') =>
     evaluate(workflow.concurrency.group, { event_name, ref, run_id }, inputs);
   const master = group('push', 'refs/heads/master');
-  const development = group('workflow_dispatch', 'refs/heads/develop', {
-    build_development_bundles: true,
-  });
+  const development = group('push', 'refs/heads/develop');
   assert.equal(
     development,
-    group(
-      'workflow_dispatch',
-      'refs/heads/develop',
-      { build_development_bundles: true },
-      '101',
-    ),
+    group('workflow_dispatch', 'refs/heads/develop', {}, '101'),
   );
   assert.equal(master, group('push', 'refs/heads/master', {}, '101'));
   assert.notEqual(master, development);
-  const push = group('push', 'refs/heads/develop');
-  assert.notEqual(push, development);
-  assert.notEqual(push, group('push', 'refs/heads/develop', {}, '101'));
+  assert.notEqual(
+    development,
+    group('workflow_dispatch', 'refs/heads/develop', {
+      static_sync_only: true,
+    }),
+  );
+  assert.notEqual(
+    group('push', 'refs/tags/v2.24.0'),
+    group('push', 'refs/tags/v2.24.0', {}, '101'),
+  );
   assert.equal(workflow.concurrency['cancel-in-progress'], false);
 });
+
+test('bundle targets exactly match the source image platforms and publisher requirements', () => {
+  const targets =
+    workflow.jobs['build-upgrade-bundles'].strategy.matrix.include;
+  for (const [job, libc] of [
+    ['build-alpine', 'musl'],
+    ['build-debian', 'glibc'],
+  ]) {
+    const platforms = workflow.jobs[job].steps
+      .find((step) => step.with?.platforms)
+      .with.platforms.split(',');
+    assert.deepEqual(
+      targets
+        .filter((t) => t.libc === libc)
+        .map((t) => t.platform)
+        .sort(),
+      platforms.sort(),
+    );
+  }
+  const { platforms } = require('../../scripts/publish-upgrade-bundles.cjs');
+  assert.deepEqual(
+    targets.map((t) => `linux-${t.name}`).sort(),
+    [...platforms].sort(),
+  );
+  assert.ok(platforms.includes('linux-armv6-musl'));
+  assert.ok(!platforms.includes('linux-armv6-glibc'));
+});

test/back/upgrade-readiness.test.cjs

@@ -0,0 +1,162 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const vm = require('node:vm');
+const { EventEmitter } = require('node:events');
+const ts = require('typescript');
+const load = require('../helpers/load-security-module.cjs');
+const { SchedulerReadiness } = load('back/shared/schedulerReadiness.ts');
+const { outputText } = ts.transpileModule(
+  fs.readFileSync('back/bootstrap/upgradeProbe.ts', 'utf8'),
+  { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020, esModuleInterop: true } },
+);
+
+// Run the real probe entry point, including its lifecycle and IPC reporting.
+// Only external services and the clock are replaced; no test exports are needed.
+async function runProbe({ readiness, arch = 'x64', armVersion, supervised = true }) {
+  let now = 0;
+  const events = [], waits = [], messages = [];
+  const build = { sourceCommit: 'a'.repeat(40) };
+  let finish;
+  const completed = new Promise((resolve) => { finish = resolve; });
+  const listener = { address: () => ({ port: 12345 }) };
+  const server = {
+    initialize: async () => { events.push('http'); return listener; },
+    shutdown: async () => { events.push('http-close'); },
+  };
+  const grpc = {
+    initialize: async () => { events.push('grpc'); return 12346; },
+    getServer: () => ({ forceShutdown: () => events.push('grpc-close') }),
+  };
+  class GrpcServerService {}
+  class HttpServerService {}
+  const modules = {
+    'reflect-metadata': {},
+    dns: { setDefaultResultOrder: () => {} },
+    '../config': {},
+    '../data': { sequelize: {
+      transaction: async (action) => action({}),
+      query: async () => [[{ value: 42 }]],
+      close: async () => { events.push('db-close'); },
+    } },
+    '../loaders/db': async () => { events.push('db'); },
+    typedi: { Container: { get: (service) => service === GrpcServerService ? grpc : server } },
+    '../services/grpc': { GrpcServerService },
+    '../services/http': { HttpServerService },
+    '../loaders/httpApplication': { createHttpApplication: async () => ({}) },
+    '../loaders/server': async () => {},
+    '../loaders/express': () => {},
+    '../loaders/app': {},
+    '../loaders/depInjector': () => {},
+    '../schedule/client': { readiness: {
+      configure: (restore) => readiness.configure(restore),
+      recover: async () => {
+        const ready = await readiness.recover();
+        if (ready) events.push('ready');
+        return ready;
+      },
+    } },
+    http: { get: (url, respond) => {
+      events.push('health');
+      const request = new EventEmitter();
+      request.setTimeout = () => {};
+      queueMicrotask(() => {
+        const response = new EventEmitter();
+        response.statusCode = 200;
+        respond(response);
+        response.emit('data', JSON.stringify({ code: 200, data: { status: 'ok', services: { grpc: true, http: true }, build } }));
+        response.emit('end');
+      });
+      return request;
+    } },
+  };
+  vm.runInNewContext(outputText, {
+    exports: {},
+    require: (name) => {
+      assert.ok(Object.hasOwn(modules, name), `Unexpected probe dependency: ${name}`);
+      return modules[name];
+    },
+    process: {
+      env: { QL_UPGRADE_PROBE: supervised ? '1' : '0' }, arch,
+      config: { variables: { arm_version: armVersion } }, pid: 123,
+      once: () => {}, memoryUsage: () => ({}),
+      send: (message) => { messages.push(message); events.push('report'); },
+      exit: finish,
+    },
+    Date: { now: () => now },
+    setTimeout: (callback, ms) => {
+      assert.ok(ms > 0 && ms <= 250);
+      waits.push(ms);
+      now += ms;
+      queueMicrotask(callback);
+    },
+    console: { error: () => {} },
+  }, { filename: 'upgradeProbe.js' });
+  const exitCode = await completed;
+  return { exitCode, events, waits, messages, elapsed: now, build };
+}
+
+test('upgrade probe survives cold scheduler recovery before checking HTTP and reporting readiness', async () => {
+  let attempts = 0, restored = false;
+  const readiness = new SchedulerReadiness(async () => {
+    if (++attempts < 3) throw new Error('Failed to connect before the deadline');
+  });
+  // The probe must configure and finish restoration before reporting readiness.
+  const configure = readiness.configure.bind(readiness);
+  readiness.configure = (restore) => configure(async () => { await restore(); restored = true; });
+  const result = await runProbe({ readiness });
+  assert.equal(result.exitCode, 0);
+  assert.equal(restored, true);
+  assert.equal(await readiness.check(), true);
+  assert.deepEqual(result.waits, [250, 250]);
+  assert.deepEqual(result.events, ['db', 'grpc', 'ready', 'http', 'health', 'report', 'http-close', 'grpc-close', 'db-close']);
+  assert.equal(result.messages.length, 1);
+  assert.equal(result.messages[0].ok, true);
+  assert.equal(result.messages[0].build.sourceCommit, result.build.sourceCommit);
+});
+
+for (const [arch, armVersion, timeout] of [['x64', undefined, 10000], ['arm', 7, 10000], ['arm', '6', 60000]]) {
+  test(`unavailable ${arch}/v${armVersion} scheduler fails at its startup budget and cleans up`, async () => {
+    let attempts = 0;
+    const result = await runProbe({ arch, armVersion, readiness: {
+      configure: () => {}, recover: async () => { attempts++; return false; },
+    } });
+    assert.equal(result.exitCode, 1);
+    assert.equal(result.elapsed, timeout);
+    assert.equal(attempts, timeout / 250);
+    assert.deepEqual(result.events, ['db', 'grpc', 'http-close', 'grpc-close', 'db-close', 'report']);
+    assert.equal(result.messages.length, 1);
+    assert.equal(result.messages[0].ok, false);
+    assert.match(result.messages[0].message, /Upgrade scheduler readiness failed/);
+    await new Promise((resolve) => setImmediate(resolve));
+    assert.equal(attempts, timeout / 250, 'no retries after failure');
+  });
+}
+
+test('unexpected scheduler recovery errors fail prestart without retries or a success report', async () => {
+  let attempts = 0;
+  const result = await runProbe({ readiness: {
+    configure: () => {}, recover: async () => { attempts++; throw new Error('broken scheduler configuration'); },
+  } });
+  assert.equal(attempts, 1);
+  assert.equal(result.exitCode, 1);
+  assert.deepEqual(result.waits, []);
+  assert.deepEqual(result.events, ['db', 'grpc', 'http-close', 'grpc-close', 'db-close', 'report']);
+  assert.equal(result.messages.length, 1);
+  assert.equal(result.messages[0].ok, false);
+  assert.match(result.messages[0].message, /broken scheduler configuration/);
+});
+
+test('an already ready scheduler reaches health without waiting', async () => {
+  const result = await runProbe({ readiness: { configure: () => {}, recover: async () => true } });
+  assert.equal(result.exitCode, 0);
+  assert.deepEqual(result.waits, []);
+  assert.equal(result.messages[0].ok, true);
+});
+
+test('the upgrade probe still requires its isolated supervisor', async () => {
+  const result = await runProbe({ supervised: false });
+  assert.equal(result.exitCode, 1);
+  assert.deepEqual(result.events, ['report']);
+  assert.match(result.messages[0].message, /requires its isolated supervisor/);
+});