fix: 修复 develop 调度锁、运行恢复与 Alpine 缓存并发 (#3070)

* fix: repair develop scheduler locks and runtime recovery

* test: wait for child reaping after verified termination

* fix: abort cron mutations when scheduler deletion fails

back/config/util.ts

@@ -510,23 +510,45 @@ export async function killTask(pid: number, waitForExit = false) {
     }
   };
   for (const target of pids) signal(target, 'SIGTERM');
-  const alive = (target: number) => {
+  const alive = async (target: number) => {
     try {
       process.kill(target, 0);
-      return true;
     } catch (error: any) {
       if (error.code === 'ESRCH') return false;
       throw error;
     }
+    if (process.platform === 'linux') {
+      try {
+        const stat = await fs.readFile(`/proc/${target}/stat`, 'utf8');
+        // The command field may contain spaces and parentheses. Zombies have
+        // exited even while their parent has not reaped the PID yet.
+        const state = stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0];
+        if (['Z', 'X', 'x'].includes(state)) return false;
+      } catch {
+        // /proc may be unavailable, or the process may have just exited.
+        // Retain the portable signal probe rather than assuming it is dead.
+        try {
+          process.kill(target, 0);
+        } catch (error: any) {
+          if (error.code === 'ESRCH') return false;
+          throw error;
+        }
+      }
+    }
+    return true;
   };
   const wait = async () => {
     const deadline = Date.now() + 1000;
-    while (pids.some(alive) && Date.now() < deadline) {
+    let remaining = pids;
+    while (true) {
+      const states = await Promise.all(remaining.map(alive));
+      remaining = remaining.filter((_, index) => states[index]);
+      if (!remaining.length || Date.now() >= deadline) return remaining;
       await new Promise((resolve) => setTimeout(resolve, 20));
     }
-    return pids.filter(alive);
   };
   let remaining = await wait();
+  if (!remaining.length) return;
   for (const target of remaining) signal(target, 'SIGKILL');
   remaining = await wait();
   if (remaining.length)

back/schedule/api.ts

@@ -38,6 +38,7 @@ import {
   RunCronsRequest,
 } from '../protos/api';
 import { NotificationInfo } from '../data/notify';
+import { Model } from 'sequelize';
 
 Container.set('logger', LoggerInstance);
 
@@ -247,13 +248,16 @@ export const systemNotify = async (
 
 const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
   if (!data) return undefined;
+  // create() returns a Sequelize instance; spreading it omits attribute getters.
+  const cron = data instanceof Model ? (data.get({ plain: true }) as CronItem) : data;
   return {
-    ...data,
-    sub_id: data.sub_id ?? undefined,
-    extra_schedules: data.extra_schedules ?? [],
-    pid: data.pid ?? undefined,
-    task_before: data.task_before ?? undefined,
-    task_after: data.task_after ?? undefined,
+    ...cron,
+    labels: cron.labels ?? [],
+    sub_id: cron.sub_id ?? undefined,
+    extra_schedules: cron.extra_schedules ?? [],
+    pid: cron.pid ?? undefined,
+    task_before: cron.task_before ?? undefined,
+    task_after: cron.task_after ?? undefined,
   };
 };
 

back/schedule/client.ts

@@ -74,7 +74,9 @@ class Client {
         { deadline: Date.now() + 5000 },
         (err, res) => {
           if (err) {
-            if (err.code === status.UNAVAILABLE) {
+            if (err.code === status.UNAVAILABLE || err.code === status.DEADLINE_EXCEEDED) {
+              // A timed-out write may already have reached the scheduler.
+              // Reconcile its state from the DB instead of replaying the RPC.
               this.readiness.invalidate();
               Object.assign(err, { status: 503 });
             }
@@ -91,7 +93,7 @@ class Client {
     return new Promise((resolve, reject) => {
       this.client.delCron({ ids: request }, new Metadata(), { deadline: Date.now() + 5000 }, (err, res) => {
         if (err) {
-          if (err.code === status.UNAVAILABLE) {
+          if (err.code === status.UNAVAILABLE || err.code === status.DEADLINE_EXCEEDED) {
             this.readiness.invalidate();
             Object.assign(err, { status: 503 });
           }

back/services/cron.ts

@@ -154,20 +154,14 @@ export default class CronService {
       const tab = new Crontab({ ...doc, ...payload });
       tab.saved = false;
       tab.log_name = await this.getLogName(tab);
-      const newDoc = await this.updateDb(tab);
-
       if (doc.isDisabled === 1 || isDemoEnv()) {
-        return newDoc;
+        return await this.updateDb(tab);
       }
 
-      try {
-        await cronClient.delCron([String(newDoc.id)]);
-      } catch (error: any) {
-        this.logger.warn(
-          '[crontab] Failed to unregister cron job in scheduler:',
-          error?.message || error,
-        );
-      }
+      // Keep the DB snapshot unchanged if deletion has an uncertain outcome.
+      // Recovery uses that snapshot after this mutation releases its lock.
+      await cronClient.delCron([String(doc.id)]);
+      const newDoc = await this.updateDb(tab);
 
       if (this.shouldUseCronClient(newDoc)) {
         try {
@@ -305,15 +299,8 @@ export default class CronService {
 
   public async remove(ids: number[]) {
     return withSchedulerMutation(async () => {
+      await cronClient.delCron(ids.map(String));
       await CrontabModel.destroy({ where: { id: ids } });
-      try {
-        await cronClient.delCron(ids.map(String));
-      } catch (error: any) {
-        this.logger.warn(
-          '[crontab] Failed to unregister cron job in scheduler:',
-          error?.message || error,
-        );
-      }
       await this.setCrontab();
     });
   }
@@ -853,15 +840,8 @@ export default class CronService {
 
   public async disabled(ids: number[]) {
     return withSchedulerMutation(async () => {
+      await cronClient.delCron(ids.map(String));
       await CrontabModel.update({ isDisabled: 1 }, { where: { id: ids } });
-      try {
-        await cronClient.delCron(ids.map(String));
-      } catch (error: any) {
-        this.logger.warn(
-          '[crontab] Failed to unregister cron job in scheduler:',
-          error?.message || error,
-        );
-      }
       await this.setCrontab();
     });
   }

back/shared/schedulerMutationLock.ts

@@ -9,7 +9,9 @@ export async function withSchedulerMutation<T>(
 ): Promise<T> {
   let release: () => Promise<void>;
   try {
-    release = await lockfile.lock(config.crontabFile, {
+    // proper-lockfile indexes held locks by target, not lockfilePath. Keep
+    // this identity separate from nested writeFileWithLock(crontabFile).
+    release = await lockfile.lock(`${config.crontabFile}.scheduler`, {
       realpath: false,
       lockfilePath: `${config.crontabFile}.scheduler.lock`,
       stale: 30000,