fix: keep temporary Node binaries out of image layers

.github/workflows/release-readiness.yml

@@ -133,7 +133,7 @@ jobs:
           docker run --rm --network none --entrypoint node \
             qinglong-readiness:${{ matrix.image }} \
             -e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.4.0",sequelize:"6.37.8"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); require("sqlite3"); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")'
-      - name: Load ARMv6 image for SQLite and Node regression checks
+      - name: Export ARMv6 image layers for Node regression checks
         if: startsWith(matrix.image, 'alpine')
         uses: docker/build-push-action@v7
         with:
@@ -144,23 +144,14 @@ jobs:
             QL_URL=${{ steps.source.outputs.url }}
             QL_BRANCH=${{ steps.source.outputs.branch }}
             SOURCE_COMMIT=${{ steps.source.outputs.sha }}
-          tags: qinglong-readiness:${{ matrix.image }}-armv6
-          load: true
+          outputs: type=docker,dest=${{ runner.temp }}/node-layers-arm.tar
           push: false
           provenance: false
           sbom: false
-      - name: Verify ARMv6 SQLite on an ARM1176 CPU without networking
-        if: startsWith(matrix.image, 'alpine')
-        run: |
-          docker run --rm --network none --platform linux/arm/v6 \
-            -e QEMU_CPU=arm1176 --entrypoint node \
-            qinglong-readiness:${{ matrix.image }}-armv6 --jitless \
-            -e 'const assert=require("node:assert/strict"); const sqlite3=require("sqlite3"); const db=new sqlite3.Database(":memory:"); db.get("SELECT 42 AS n",(error,row)=>{assert.ifError(error); assert.equal(row.n,42); db.close()});'
       - name: Check Node binaries in every serialized image layer
         if: startsWith(matrix.image, 'alpine')
         run: |
           docker save qinglong-readiness:${{ matrix.image }} -o "$RUNNER_TEMP/node-layers-amd64.tar"
-          docker save qinglong-readiness:${{ matrix.image }}-armv6 -o "$RUNNER_TEMP/node-layers-arm.tar"
           python3 scripts/verify-image-node-layers.py "$RUNNER_TEMP/node-layers-amd64.tar" amd64
           python3 scripts/verify-image-node-layers.py "$RUNNER_TEMP/node-layers-arm.tar" arm
           rm "$RUNNER_TEMP/node-layers-amd64.tar" "$RUNNER_TEMP/node-layers-arm.tar"

back/shared/runCron.ts

@@ -4,6 +4,7 @@ import taskLimit from './pLimit';
 import Logger from '../loaders/logger';
 import { ICron } from '../protos/cron';
 import { CrontabModel, CrontabStatus } from '../data/cron';
+import { killTask } from '../config/util';
 import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
 import dayjs from 'dayjs';
 import { observeChildProcess, asError } from './childProcess';
@@ -16,7 +17,7 @@ export function runCron(
   const execute = async () => {
     try {
       if (!isCurrent()) return;
-      // A single-instance trigger must not interrupt work already in progress.
+      // Check if the cron is already running and stop it (only if multiple instances are not allowed)
       try {
         const existingCron = await CrontabModel.findOne({
           where: { id: Number(cron.id) },
@@ -29,93 +30,35 @@ export function runCron(
         const allowSingleInstances =
           existingCron?.allow_multiple_instances === 0;
 
-        if (allowSingleInstances) {
-          // A queued manual run is an active reservation, even if its stored PID
-          // belongs to a previous execution.
-          if (
-            existingCron.status === CrontabStatus.queued &&
-            existingCron.pid
-          ) {
-            Logger.info(
-              '[schedule][Skip queued single-instance task] Task ID: %s, PID: %s',
-              cron.id,
-              existingCron.pid,
-            );
-            return;
-          }
-          if (
-            existingCron.status === CrontabStatus.running &&
-            existingCron.pid
-          ) {
-            let missing = false;
-            try {
-              process.kill(existingCron.pid, 0);
-            } catch (error) {
-              const code = (error as NodeJS.ErrnoException).code;
-              if (code === 'ESRCH') missing = true;
-              else if (code !== 'EPERM') throw error;
-            }
-            if (!missing) {
-              Logger.info(
-                '[schedule][Skip active single-instance task] Task ID: %s, PID: %s',
-                cron.id,
-                existingCron.pid,
-              );
-              return;
-            }
-
-            // Capture old instance IDs before releasing the stale cron state so
-            // a concurrent replacement cannot have its instance marked stopped.
-            const staleInstances = await RunningInstanceModel.findAll({
-              attributes: ['id'],
+        if (
+          allowSingleInstances &&
+          existingCron &&
+          existingCron.pid &&
+          (existingCron.status === CrontabStatus.running ||
+            existingCron.status === CrontabStatus.queued)
+        ) {
+          Logger.info(
+            t('[schedule][停止已运行任务] 任务ID: %s, PID: %s'),
+            cron.id,
+            existingCron.pid,
+          );
+          await killTask(existingCron.pid, true);
+          // Mark old running instances as stopped
+          const stoppedAt = dayjs().unix();
+          await RunningInstanceModel.update(
+            { status: InstanceStatus.stopped, finished_at: stoppedAt },
+            {
               where: {
                 cron_id: Number(cron.id),
-                pid: existingCron.pid,
                 status: InstanceStatus.running,
               },
-            });
-            if (!isCurrent()) return;
-            const [recovered] = await CrontabModel.update(
-              {
-                status: CrontabStatus.idle,
-                pid: null,
-                queued_token: null,
-              } as any,
-              {
-                where: {
-                  id: Number(cron.id),
-                  pid: existingCron.pid,
-                  status: CrontabStatus.running,
-                  ...(existingCron.log_path !== undefined
-                    ? { log_path: existingCron.log_path }
-                    : {}),
-                  ...(existingCron.queued_token !== undefined
-                    ? { queued_token: existingCron.queued_token }
-                    : {}),
-                  ...(existingCron.last_execution_time !== undefined
-                    ? { last_execution_time: existingCron.last_execution_time }
-                    : {}),
-                  ...(existingCron.isDisabled !== undefined
-                    ? { isDisabled: existingCron.isDisabled }
-                    : {}),
-                  allow_multiple_instances: 0,
-                },
-              },
-            );
-            if (recovered !== 1) return;
-            const staleIds = staleInstances.map((instance) => instance.id!);
-            if (staleIds.length) {
-              await RunningInstanceModel.update(
-                { status: InstanceStatus.stopped, finished_at: dayjs().unix() },
-                { where: { id: staleIds, status: InstanceStatus.running } },
-              );
-            }
-            Logger.info(
-              '[schedule][Recovered stale single-instance task] Task ID: %s, PID: %s',
-              cron.id,
-              existingCron.pid,
-            );
-          }
+            },
+          );
+          // Update the status to idle after killing
+          await CrontabModel.update(
+            { status: CrontabStatus.idle, pid: undefined },
+            { where: { id: Number(cron.id) } },
+          );
         }
       } catch (error) {
         Logger.error(
@@ -126,7 +69,7 @@ export function runCron(
         throw error;
       }
 
-      // Invalidation can happen during the asynchronous database check.
+      // In particular, invalidation can happen while stopping a previous run.
       if (!isCurrent()) return;
       Logger.info(
         t('[schedule][开始执行任务] 参数 %s'),

docker/Dockerfile

@@ -3,7 +3,6 @@
 FROM --platform=$BUILDPLATFORM node:18-alpine3.18 AS builder
 
 ARG TARGETARCH
-ARG TARGETVARIANT
 ENV NPM_CONFIG_PREFIX=/opt/node-global
 ENV PATH=/opt/node-global/bin:${PATH}
 
@@ -18,16 +17,10 @@ RUN set -x \
     ppc64le) NODE_ARCH=ppc64 ;; \
     *) NODE_ARCH="${TARGETARCH}" ;; \
   esac \
-  && case "${TARGETARCH}/${TARGETVARIANT:-}" in \
-    arm/v6) NODE_LIBC=musl-armv6 ;; \
-    arm/v7) NODE_LIBC=musl-armv7 ;; \
-    arm/*) echo "Unsupported ARM variant: ${TARGETVARIANT:-unknown}" >&2; exit 1 ;; \
-    *) NODE_LIBC=musl ;; \
-  esac \
   && npm_config_target_platform=linux \
     npm_config_target_arch="${NODE_ARCH}" \
-    npm_config_target_libc="${NODE_LIBC}" \
-    pnpm install --prod --frozen-lockfile --side-effects-cache=false
+    npm_config_target_libc=musl \
+    pnpm install --prod --frozen-lockfile
 
 # Pin amd64 to upstream's musl runtime after native URL parser crashes with APK Node.
 FROM --platform=linux/amd64 node:24.18.1-alpine3.24@sha256:f70403e87646dc51b45295f4b8b70cdad0b63d2297c4c9899119b03f7af7a6b3 AS node-runtime-amd64

docker/Dockerfile.310

@@ -3,7 +3,6 @@
 FROM --platform=$BUILDPLATFORM node:18-alpine3.18 AS builder
 
 ARG TARGETARCH
-ARG TARGETVARIANT
 ENV NPM_CONFIG_PREFIX=/opt/node-global
 ENV PATH=/opt/node-global/bin:${PATH}
 
@@ -18,16 +17,10 @@ RUN set -x \
     ppc64le) NODE_ARCH=ppc64 ;; \
     *) NODE_ARCH="${TARGETARCH}" ;; \
   esac \
-  && case "${TARGETARCH}/${TARGETVARIANT:-}" in \
-    arm/v6) NODE_LIBC=musl-armv6 ;; \
-    arm/v7) NODE_LIBC=musl-armv7 ;; \
-    arm/*) echo "Unsupported ARM variant: ${TARGETVARIANT:-unknown}" >&2; exit 1 ;; \
-    *) NODE_LIBC=musl ;; \
-  esac \
   && npm_config_target_platform=linux \
     npm_config_target_arch="${NODE_ARCH}" \
-    npm_config_target_libc="${NODE_LIBC}" \
-    pnpm install --prod --frozen-lockfile --side-effects-cache=false
+    npm_config_target_libc=musl \
+    pnpm install --prod --frozen-lockfile
 
 # Pin amd64 to upstream's musl runtime after native URL parser crashes with APK Node.
 FROM --platform=linux/amd64 node:24.18.1-alpine3.24@sha256:f70403e87646dc51b45295f4b8b70cdad0b63d2297c4c9899119b03f7af7a6b3 AS node-runtime-amd64

docker/Dockerfile.upgrade-bundle

@@ -6,11 +6,25 @@ ENV QL_BUNDLE_BASE_IMAGE=${BASE_IMAGE}
 ARG TARGETARCH
 ARG TARGETVARIANT
 USER root
-RUN ulimit -c 0; \
-    if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
+# The generic "arm" sqlite release is ARMv7. ARMv6 must compile its own addon.
+RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
+      apk add --no-cache --virtual .ql-bundle-build build-base binutils python3 nodejs-dev && \
+      cd "$(readlink -f /ql/node_modules/sqlite3)" && \
+      CFLAGS='-march=armv6 -mfpu=vfp -mfloat-abi=hard' \
+      CXXFLAGS='-march=armv6 -mfpu=vfp -mfloat-abi=hard' \
+      npm_config_nodedir=/usr npm_config_build_from_source=true QEMU_CPU=arm1176 \
+        node -e 'require("node:child_process").execFileSync(process.execPath,[require.resolve("@mapbox/node-pre-gyp/bin/node-pre-gyp"),"rebuild","--napi_build_version=6"],{stdio:"inherit"})' && \
+      readelf -A lib/binding/napi-v6-linux-arm/node_sqlite3.node | grep -E 'Tag_CPU_arch: v6($|K)' && \
+      rm -rf build && \
+      apk del .ql-bundle-build; \
+    fi
+RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
       export QEMU_CPU=arm1176; \
-      set -- --jitless; \
-    fi; \
-    node "$@" /ql/scripts/build-upgrade-bundle.cjs /ql /out
+      node --jitless /ql/scripts/build-upgrade-bundle.cjs /ql /out && \
+      for archive in /out/*.tar.gz; do \
+        node /ql/shell/install-bundle.cjs --root /tmp/ql-bundle-offline-smoke --file "$archive" --install && \
+        rm -rf /tmp/ql-bundle-offline-smoke || exit 1; \
+      done; \
+    else node /ql/scripts/build-upgrade-bundle.cjs /ql /out; fi
 FROM scratch
 COPY --from=bundle /out/ /

package.json

@@ -111,7 +111,7 @@
       }
     },
     "overrides": {
-      "sqlite3": "npm:@whyour/sqlite3@1.1.4",
+      "sqlite3": "npm:@whyour/sqlite3@1.1.2",
       "@codemirror/state": "6.5.4",
       "@codemirror/view": "6.39.16",
       "proxy-addr": "2.0.8",
@@ -159,7 +159,7 @@
     "request-ip": "3.3.0",
     "sequelize": "6.37.8",
     "sockjs": "^0.3.24",
-    "sqlite3": "npm:@whyour/sqlite3@1.1.4",
+    "sqlite3": "npm:@whyour/sqlite3@1.1.2",
     "toad-scheduler": "^3.0.1",
     "typedi": "^0.10.0",
     "undici": "^7.9.0",

pnpm-lock.yaml

@@ -1,7 +1,7 @@
 lockfileVersion: '6.0'
 
 overrides:
-  sqlite3: npm:@whyour/sqlite3@1.1.4
+  sqlite3: npm:@whyour/sqlite3@1.1.2
   '@codemirror/state': 6.5.4
   '@codemirror/view': 6.39.16
   proxy-addr: 2.0.8
@@ -120,13 +120,13 @@ dependencies:
     version: 3.3.0
   sequelize:
     specifier: 6.37.8
-    version: 6.37.8(@whyour/sqlite3@1.1.4)
+    version: 6.37.8(@whyour/sqlite3@1.1.2)
   sockjs:
     specifier: ^0.3.24
     version: 0.3.24
   sqlite3:
-    specifier: npm:@whyour/sqlite3@1.1.4
-    version: /@whyour/sqlite3@1.1.4
+    specifier: npm:@whyour/sqlite3@1.1.2
+    version: /@whyour/sqlite3@1.1.2
   tar:
     specifier: 7.5.20
     version: 7.5.20
@@ -3193,7 +3193,7 @@ packages:
     resolution: {integrity: sha512-Ngs9jhElXN7efS9WvvCC/p6rXMbihna8eNLVBc421Zf+VcFd+pR4DOcS6yA9V22EKtAy4DEj7LtvEEIm0bb80A==}
     engines: {node: '>= 18'}
     dependencies:
-      sqlite3: /@whyour/sqlite3@1.1.4
+      sqlite3: /@whyour/sqlite3@1.1.2
     transitivePeerDependencies:
       - encoding
       - supports-color
@@ -5611,15 +5611,14 @@ packages:
       - supports-color
     dev: true
 
-  /@whyour/sqlite3@1.1.4:
-    resolution: {integrity: sha512-MlWzqOdpZ0KPuBx/FYRrHZ8p3S0eCbaf2d8duDzWmfko/aky25Xoss67qDjFgbRjnwj9z0i0KNdNtQI02dnWKA==}
+  /@whyour/sqlite3@1.1.2:
+    resolution: {integrity: sha512-OY24/OJMqLHIGolo4GCFW4lrJKAoOB+ZqERXb7u7DKZQz7QR3f0GALE99BZWQBSZEVVZ/StVeAb2fCkMfTtuDQ==}
     requiresBuild: true
     peerDependenciesMeta:
       node-gyp:
         optional: true
     dependencies:
       '@mapbox/node-pre-gyp': 1.0.11
-      detect-libc: 1.0.3
       node-addon-api: 4.3.0
       tar: 6.2.1
     optionalDependencies:
@@ -7602,6 +7601,7 @@ packages:
     resolution: {integrity: sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==}
     engines: {node: '>=0.10'}
     hasBin: true
+    dev: true
 
   /detect-libc@2.1.2:
     resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==}
@@ -13903,7 +13903,7 @@ packages:
     engines: {node: '>= 10.0.0'}
     dev: false
 
-  /sequelize@6.37.8(@whyour/sqlite3@1.1.4):
+  /sequelize@6.37.8(@whyour/sqlite3@1.1.2):
     resolution: {integrity: sha512-HJ0IQFqcTsTiqbEgiuioYFMSD00TP6Cz7zoTti+zVVBwVe9fEhev9cH6WnM3XU31+ABS356durAb99ZuOthnKw==}
     engines: {node: '>=10.0.0'}
     peerDependencies:
@@ -13948,7 +13948,7 @@ packages:
       retry-as-promised: 7.0.4
       semver: 7.7.4
       sequelize-pool: 7.1.0
-      sqlite3: /@whyour/sqlite3@1.1.4
+      sqlite3: /@whyour/sqlite3@1.1.2
       toposort-class: 1.0.1
       uuid: 8.3.2
       validator: 13.12.0

test/back/single-instance-skip.test.cjs

@@ -1,192 +0,0 @@
-const test = require('node:test');
-const assert = require('node:assert/strict');
-const { spawnSync } = require('node:child_process');
-const load = require('../helpers/load-security-module.cjs');
-
-function fixture(row, { claim = true, onSpawn = () => {} } = {}) {
-  const spawned = [],
-    cronWrites = [],
-    instanceWrites = [],
-    instanceReads = [];
-  const { runCron } = load('back/shared/runCron.ts', {
-    './i18n': { t: (s) => s },
-    'cross-spawn': {
-      spawn: (cmd) => {
-        spawned.push(cmd);
-        onSpawn(row);
-        return { pid: process.pid };
-      },
-    },
-    './pLimit': {
-      runWithCronLimit: (_cron, execute) => execute(),
-      removeQueuedCron() {},
-    },
-    '../loaders/logger': { info() {}, error() {} },
-    '../data/cron': {
-      CrontabModel: {
-        findOne: async () => row,
-        update: async (...args) => {
-          cronWrites.push(args);
-          if (!claim) return [0];
-          Object.assign(row, args[0]);
-          return [1];
-        },
-      },
-      CrontabStatus: { running: 0, idle: 1, queued: 3 },
-    },
-    '../data/runningInstance': {
-      RunningInstanceModel: {
-        findAll: async (options) => {
-          instanceReads.push(options);
-          return [{ id: 7 }];
-        },
-        update: async (...args) => instanceWrites.push(args),
-      },
-      InstanceStatus: { running: 0, stopped: 2 },
-    },
-    dayjs: () => ({ unix: () => 123 }),
-    './childProcess': {
-      observeChildProcess: () => ({ completed: Promise.resolve({ code: 0 }) }),
-      asError: (e) => e,
-    },
-  });
-  return {
-    run: () => runCron('task example.py', { id: '1' }),
-    spawned,
-    cronWrites,
-    instanceWrites,
-    instanceReads,
-  };
-}
-
-function exitedPid() {
-  const child = spawnSync(process.execPath, ['-e', '']);
-  assert.equal(child.status, 0);
-  assert.throws(() => process.kill(child.pid, 0), { code: 'ESRCH' });
-  return child.pid;
-}
-
-test('running single-instance task stays untouched while its process is alive', async () => {
-  const f = fixture({
-    allow_multiple_instances: 0,
-    pid: process.pid,
-    status: 0,
-  });
-  await f.run();
-  assert.deepEqual(f.spawned, []);
-  assert.deepEqual(f.cronWrites, []);
-  assert.deepEqual(f.instanceWrites, []);
-});
-
-test('queued single-instance task keeps its reservation even with an exited PID', async () => {
-  const f = fixture({
-    allow_multiple_instances: 0,
-    pid: exitedPid(),
-    status: 3,
-  });
-  await f.run();
-  assert.deepEqual(f.spawned, []);
-  assert.deepEqual(f.cronWrites, []);
-  assert.deepEqual(f.instanceWrites, []);
-});
-
-test('exited running task is recovered and the next trigger sees the new active run', async () => {
-  const oldPid = exitedPid();
-  const f = fixture(
-    {
-      allow_multiple_instances: 0,
-      pid: oldPid,
-      status: 0,
-      log_path: 'old.log',
-      queued_token: 'old-token',
-      last_execution_time: 10,
-      isDisabled: 0,
-    },
-    {
-      onSpawn: (row) => {
-        row.status = 0;
-        row.pid = process.pid;
-      },
-    },
-  );
-  await f.run();
-  await f.run();
-  assert.equal(f.spawned.length, 1);
-  assert.deepEqual(f.instanceReads, [
-    {
-      attributes: ['id'],
-      where: { cron_id: 1, pid: oldPid, status: 0 },
-    },
-  ]);
-  assert.deepEqual(f.cronWrites, [
-    [
-      { status: 1, pid: null, queued_token: null },
-      {
-        where: {
-          id: 1,
-          pid: oldPid,
-          status: 0,
-          log_path: 'old.log',
-          queued_token: 'old-token',
-          last_execution_time: 10,
-          isDisabled: 0,
-          allow_multiple_instances: 0,
-        },
-      },
-    ],
-  ]);
-  assert.deepEqual(f.instanceWrites, [
-    [{ status: 2, finished_at: 123 }, { where: { id: [7], status: 0 } }],
-  ]);
-});
-
-test('a lost stale-state claim does not spawn or change running instances', async () => {
-  const f = fixture(
-    { allow_multiple_instances: 0, pid: exitedPid(), status: 0 },
-    { claim: false },
-  );
-  await f.run();
-  assert.equal(f.cronWrites.length, 1);
-  assert.deepEqual(f.spawned, []);
-  assert.deepEqual(f.instanceWrites, []);
-});
-
-test('EPERM means an existing process must still block a new run', async () => {
-  const pid = exitedPid();
-  const originalKill = process.kill;
-  process.kill = (target, signal) => {
-    if (target === pid && signal === 0)
-      throw Object.assign(new Error('denied'), { code: 'EPERM' });
-    return originalKill(target, signal);
-  };
-  try {
-    const f = fixture({ allow_multiple_instances: 0, pid, status: 0 });
-    await f.run();
-    assert.deepEqual(f.spawned, []);
-    assert.deepEqual(f.cronWrites, []);
-  } finally {
-    process.kill = originalKill;
-  }
-});
-
-test('single-instance still starts idle tasks with a stale stored PID', async () => {
-  const f = fixture({
-    allow_multiple_instances: 0,
-    pid: exitedPid(),
-    status: 1,
-  });
-  await f.run();
-  assert.equal(f.spawned.length, 1);
-  assert.deepEqual(f.cronWrites, []);
-});
-
-test('multi-instance keeps its existing concurrent execution behavior', async () => {
-  const f = fixture({
-    allow_multiple_instances: 1,
-    pid: process.pid,
-    status: 0,
-  });
-  await f.run();
-  assert.equal(f.spawned.length, 1);
-  assert.deepEqual(f.cronWrites, []);
-});

test/back/stop-failure-feedback.test.cjs

@@ -4,6 +4,75 @@ const load = require('../helpers/load-security-module.cjs');
 
 const logger = { info() {}, error() {} };
 
+function cronFixture(killTask) {
+  const events = [];
+  const { runCron } = load('back/shared/runCron.ts', {
+    'cross-spawn': {
+      spawn: () => {
+        events.push('spawn');
+        return {};
+      },
+    },
+    './childProcess': {
+      observeChildProcess: () => ({ completed: Promise.resolve({ code: 0 }) }),
+      asError: (e) => e,
+    },
+    './pLimit': {
+      runWithCronLimit: async (_, fn) => fn(),
+      removeQueuedCron: () => events.push('release'),
+    },
+    '../loaders/logger': logger,
+    '../config/util': { killTask },
+    '../data/cron': {
+      CrontabModel: {
+        findOne: async () => ({
+          pid: 100,
+          status: 0,
+          allow_multiple_instances: 0,
+        }),
+        update: async () => events.push('idle'),
+      },
+      CrontabStatus: { running: 0, idle: 1, queued: 3 },
+    },
+    '../data/runningInstance': {
+      RunningInstanceModel: { update: async () => events.push('stopped') },
+      InstanceStatus: { running: 0, stopped: 2 },
+    },
+  });
+  return { run: () => runCron('ignored', { id: '1' }), events };
+}
+
+test('single-instance replacement does not spawn or finalize when termination fails', async () => {
+  const f = cronFixture(async () => {
+    throw Error('EPERM');
+  });
+  await f.run();
+  assert.deepEqual(f.events, ['release']);
+});
+
+test('single-instance replacement waits for verified exit before spawning', async () => {
+  let finish, entered, verified;
+  const pending = new Promise((resolve) => {
+    entered = resolve;
+  });
+  const gate = new Promise((resolve) => {
+    finish = resolve;
+  });
+  const f = cronFixture(async (pid, wait) => {
+    assert.equal(pid, 100);
+    verified = wait;
+    entered();
+    await gate;
+  });
+  const running = f.run();
+  await pending;
+  assert.deepEqual(f.events, []);
+  finish();
+  await running;
+  assert.equal(verified, true);
+  assert.deepEqual(f.events, ['stopped', 'idle', 'spawn', 'release']);
+});
+
 function subscriptionFixture(killTask) {
   const updates = [];
   const Service = load('back/services/subscription.ts', {