fix: skip new scheduled triggers while a single-instance task is active (#3101)

* fix: skip active single-instance scheduled executions

* fix: recover stale single-instance scheduled tasks

* test: remove obsolete single-instance replacement assertions

---------

Co-authored-by: Patrick Gu <username@users.noreply.github.com>
Co-authored-by: whyour <imwhyour@gmail.com>

back/shared/runCron.ts

@@ -4,7 +4,6 @@ import taskLimit from './pLimit';
 import Logger from '../loaders/logger';
 import { ICron } from '../protos/cron';
 import { CrontabModel, CrontabStatus } from '../data/cron';
-import { killTask } from '../config/util';
 import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
 import dayjs from 'dayjs';
 import { observeChildProcess, asError } from './childProcess';
@@ -17,7 +16,7 @@ export function runCron(
   const execute = async () => {
     try {
       if (!isCurrent()) return;
-      // Check if the cron is already running and stop it (only if multiple instances are not allowed)
+      // A single-instance trigger must not interrupt work already in progress.
       try {
         const existingCron = await CrontabModel.findOne({
           where: { id: Number(cron.id) },
@@ -30,35 +29,93 @@ export function runCron(
         const allowSingleInstances =
           existingCron?.allow_multiple_instances === 0;
 
-        if (
-          allowSingleInstances &&
-          existingCron &&
-          existingCron.pid &&
-          (existingCron.status === CrontabStatus.running ||
-            existingCron.status === CrontabStatus.queued)
-        ) {
-          Logger.info(
-            t('[schedule][停止已运行任务] 任务ID: %s, PID: %s'),
-            cron.id,
-            existingCron.pid,
-          );
-          await killTask(existingCron.pid, true);
-          // Mark old running instances as stopped
-          const stoppedAt = dayjs().unix();
-          await RunningInstanceModel.update(
-            { status: InstanceStatus.stopped, finished_at: stoppedAt },
-            {
+        if (allowSingleInstances) {
+          // A queued manual run is an active reservation, even if its stored PID
+          // belongs to a previous execution.
+          if (
+            existingCron.status === CrontabStatus.queued &&
+            existingCron.pid
+          ) {
+            Logger.info(
+              '[schedule][Skip queued single-instance task] Task ID: %s, PID: %s',
+              cron.id,
+              existingCron.pid,
+            );
+            return;
+          }
+          if (
+            existingCron.status === CrontabStatus.running &&
+            existingCron.pid
+          ) {
+            let missing = false;
+            try {
+              process.kill(existingCron.pid, 0);
+            } catch (error) {
+              const code = (error as NodeJS.ErrnoException).code;
+              if (code === 'ESRCH') missing = true;
+              else if (code !== 'EPERM') throw error;
+            }
+            if (!missing) {
+              Logger.info(
+                '[schedule][Skip active single-instance task] Task ID: %s, PID: %s',
+                cron.id,
+                existingCron.pid,
+              );
+              return;
+            }
+
+            // Capture old instance IDs before releasing the stale cron state so
+            // a concurrent replacement cannot have its instance marked stopped.
+            const staleInstances = await RunningInstanceModel.findAll({
+              attributes: ['id'],
               where: {
                 cron_id: Number(cron.id),
+                pid: existingCron.pid,
                 status: InstanceStatus.running,
               },
-            },
-          );
-          // Update the status to idle after killing
-          await CrontabModel.update(
-            { status: CrontabStatus.idle, pid: undefined },
-            { where: { id: Number(cron.id) } },
-          );
+            });
+            if (!isCurrent()) return;
+            const [recovered] = await CrontabModel.update(
+              {
+                status: CrontabStatus.idle,
+                pid: null,
+                queued_token: null,
+              } as any,
+              {
+                where: {
+                  id: Number(cron.id),
+                  pid: existingCron.pid,
+                  status: CrontabStatus.running,
+                  ...(existingCron.log_path !== undefined
+                    ? { log_path: existingCron.log_path }
+                    : {}),
+                  ...(existingCron.queued_token !== undefined
+                    ? { queued_token: existingCron.queued_token }
+                    : {}),
+                  ...(existingCron.last_execution_time !== undefined
+                    ? { last_execution_time: existingCron.last_execution_time }
+                    : {}),
+                  ...(existingCron.isDisabled !== undefined
+                    ? { isDisabled: existingCron.isDisabled }
+                    : {}),
+                  allow_multiple_instances: 0,
+                },
+              },
+            );
+            if (recovered !== 1) return;
+            const staleIds = staleInstances.map((instance) => instance.id!);
+            if (staleIds.length) {
+              await RunningInstanceModel.update(
+                { status: InstanceStatus.stopped, finished_at: dayjs().unix() },
+                { where: { id: staleIds, status: InstanceStatus.running } },
+              );
+            }
+            Logger.info(
+              '[schedule][Recovered stale single-instance task] Task ID: %s, PID: %s',
+              cron.id,
+              existingCron.pid,
+            );
+          }
         }
       } catch (error) {
         Logger.error(
@@ -69,7 +126,7 @@ export function runCron(
         throw error;
       }
 
-      // In particular, invalidation can happen while stopping a previous run.
+      // Invalidation can happen during the asynchronous database check.
       if (!isCurrent()) return;
       Logger.info(
         t('[schedule][开始执行任务] 参数 %s'),