fix: 修复任务生命周期与调度就绪，优化执行和构建开销 (#3069)

* fix: harden task lifecycle and scheduler readiness

* fix: confine log writes to the configured log directory

* fix: verify complete build artifacts and untracked inputs

* fix: reconcile scheduler state and make stop win startup races

* fix: isolate cron generations and serialize scheduler recovery

.dockerignore

@@ -0,0 +1,13 @@
+.git
+.gitnexus
+.claude
+.codex
+.env*
+node_modules
+data
+audit
+docs
+test
+src/.umi*
+**/__pycache__
+**/*.log

.github/workflows/build-docker-image.yml

@@ -15,6 +15,9 @@ permissions:
   contents: read
 
 jobs:
+  validate:
+    uses: ./.github/workflows/validate.yml
+
   code_gitlab:
     runs-on: ubuntu-latest
     steps:
@@ -63,6 +66,7 @@ jobs:
           git push --force --tags gitee 2>&1 || echo "::warning::Gitee tags push failed"
 
   build-static:
+    needs: validate
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
@@ -72,6 +76,7 @@ jobs:
 
       - uses: actions/setup-node@v6
         with:
+          node-version: "20"
           cache: "pnpm"
           cache-dependency-path: pnpm-lock.yaml
 
@@ -80,6 +85,14 @@ jobs:
           pnpm install --frozen-lockfile
           pnpm build:front
           pnpm build:back
+          pnpm build:info
+
+      - uses: actions/upload-artifact@v6
+        with:
+          name: qinglong-static-${{ github.sha }}
+          path: static/
+          if-no-files-found: error
+          include-hidden-files: true
 
       - name: copy to static repo
         env:
@@ -151,11 +164,16 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v6
+      - uses: actions/download-artifact@v7
+        with:
+          name: qinglong-static-${{ github.sha }}
+          path: static/
       - uses: pnpm/action-setup@v6
         with:
           version: "8.3.1"
       - uses: actions/setup-node@v6
         with:
+          node-version: "20"
           cache: "pnpm"
           cache-dependency-path: pnpm-lock.yaml
 
@@ -208,7 +226,7 @@ jobs:
         uses: docker/build-push-action@v7
         with:
           build-args: |
-            MAINTAINER=${{ github.repository_owner }}
+            QL_MAINTAINER=${{ github.repository_owner }}
             QL_BRANCH=${{ github.ref_name }}
             SOURCE_COMMIT=${{ github.sha }}
           network: host
@@ -232,11 +250,16 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v6
+      - uses: actions/download-artifact@v7
+        with:
+          name: qinglong-static-${{ github.sha }}
+          path: static/
       - uses: pnpm/action-setup@v6
         with:
           version: "8.3.1"
       - uses: actions/setup-node@v6
         with:
+          node-version: "20"
           cache: "pnpm"
           cache-dependency-path: pnpm-lock.yaml
 
@@ -288,7 +311,7 @@ jobs:
         uses: docker/build-push-action@v7
         with:
           build-args: |
-            MAINTAINER=${{ github.repository_owner }}
+            QL_MAINTAINER=${{ github.repository_owner }}
             QL_BRANCH=${{ github.ref_name }}
             SOURCE_COMMIT=${{ github.sha }}
           network: host
@@ -313,11 +336,16 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v6
+      - uses: actions/download-artifact@v7
+        with:
+          name: qinglong-static-${{ github.sha }}
+          path: static/
       - uses: pnpm/action-setup@v6
         with:
           version: "8.3.1"
       - uses: actions/setup-node@v6
         with:
+          node-version: "20"
           cache: "pnpm"
           cache-dependency-path: pnpm-lock.yaml
 
@@ -355,7 +383,7 @@ jobs:
         uses: docker/build-push-action@v7
         with:
           build-args: |
-            MAINTAINER=${{ github.repository_owner }}
+            QL_MAINTAINER=${{ github.repository_owner }}
             QL_BRANCH=${{ github.ref_name }}
             SOURCE_COMMIT=${{ github.sha }}
           network: host
@@ -380,11 +408,16 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v6
+      - uses: actions/download-artifact@v7
+        with:
+          name: qinglong-static-${{ github.sha }}
+          path: static/
       - uses: pnpm/action-setup@v6
         with:
           version: "8.3.1"
       - uses: actions/setup-node@v6
         with:
+          node-version: "20"
           cache: "pnpm"
           cache-dependency-path: pnpm-lock.yaml
 
@@ -422,7 +455,7 @@ jobs:
         uses: docker/build-push-action@v7
         with:
           build-args: |
-            MAINTAINER=${{ github.repository_owner }}
+            QL_MAINTAINER=${{ github.repository_owner }}
             QL_BRANCH=${{ github.ref_name }}
             SOURCE_COMMIT=${{ github.sha }}
           network: host
@@ -451,6 +484,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
+          node-version: "20"
           cache: "pnpm"
 
       - name: build front and back

.github/workflows/validate.yml

@@ -0,0 +1,31 @@
+name: Validate
+
+on:
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  regression:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: pnpm/action-setup@v6
+        with:
+          version: '8.3.1'
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '20'
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      - run: cp .env.example .env
+      - run: pnpm test
+      - run: pnpm build:back
+      - run: pnpm benchmark:execution -- --runs=10 --output=execution-baseline.json
+      - uses: actions/upload-artifact@v6
+        with:
+          name: execution-baseline
+          path: execution-baseline.json

back/api/cron.ts

@@ -10,12 +10,26 @@ import {
   InstanceStatus,
 } from '../data/runningInstance';
 import { t } from '../shared/i18n';
+import cronClient from '../schedule/client';
 
 const route = Router();
 
 export default (app: Router) => {
   app.use('/crons', route);
 
+  route.use(async (req, res, next) => {
+    // Keep stop/status callbacks available even when the scheduler is down.
+    if (['POST', 'PUT', 'DELETE'].includes(req.method) &&
+      ['/', '/run', '/enable', '/disable', '/views/enable', '/views/disable'].includes(req.path)) {
+      try {
+        await cronClient.readiness.ensureReady();
+      } catch (error) {
+        return next(error);
+      }
+    }
+    return next();
+  });
+
   route.get(
     '/views',
     async (req: Request, res: Response, next: NextFunction) => {

back/api/health.ts

@@ -11,8 +11,8 @@ export default (app: Router) => {
     try {
       const healthService = Container.get(HealthService);
       const health = await healthService.check();
-      res.status(200).send({
-        code: 200,
+      res.status(health.status === 'ok' ? 200 : 503).send({
+        code: health.status === 'ok' ? 200 : 503,
         data: health,
       });
     } catch (err: any) {

back/api/system.ts

@@ -273,8 +273,11 @@ export default (app: Router) => {
             },
             onEnd: async (cp, endTime, diff) => {
               // Close the stream after task completion
-              await logStreamManager.closeStream(await handleLogPath(logPath));
-              res.end();
+              try {
+                await logStreamManager.closeStream(await handleLogPath(logPath));
+              } finally {
+                res.end();
+              }
             },
             onError: async (message: string) => {
               res.write(message);

back/app.ts

@@ -11,6 +11,7 @@ import { monitoringMiddleware } from './middlewares/monitoring';
 import { errStack } from './config/util';
 import { type GrpcServerService } from './services/grpc';
 import { type HttpServerService } from './services/http';
+import cronClient from './schedule/client';
 
 interface WorkerMetadata {
   id: number;
@@ -79,6 +80,11 @@ class Application {
           );
           // If gRPC worker died, restart it and wait for it to be ready
           if (metadata.serviceType === 'grpc') {
+            try {
+              this.httpWorker?.send('scheduler-unavailable');
+            } catch (error) {
+              Logger.warn('Unable to notify HTTP worker of scheduler exit');
+            }
             const newGrpcWorker = this.forkWorker('grpc');
             this.waitForWorkerReady(newGrpcWorker, 30000)
               .then(() => {
@@ -132,7 +138,14 @@ class Application {
   }
 
   private forkWorker(serviceType: string): Worker {
-    const worker = cluster.fork({ SERVICE_TYPE: serviceType });
+    const workerEnv: NodeJS.ProcessEnv = { SERVICE_TYPE: serviceType };
+    // PM2's fork launcher is inherited by our own cluster workers. Their APM
+    // messages go to this primary, not PM2, and duplicate its sampling work.
+    // Keep primary monitoring and allow restoring the inherited worker APM.
+    if (process.env.pm_id !== undefined && process.env.QL_WORKER_APM !== 'true') {
+      workerEnv.pmx = 'false';
+    }
+    const worker = cluster.fork(workerEnv);
 
     this.workerMetadataMap.set(worker.id, {
       id: worker.id,
@@ -264,17 +277,9 @@ class Application {
     process.on('message', async (msg) => {
       if (msg === 'shutdown') {
         this.gracefulShutdown(serviceType);
-      } else if (msg === 'reregister-crons' && serviceType === 'http') {
-        // Re-register cron jobs when gRPC worker restarts
-        try {
-          Logger.info('[boot] Received reregister-crons message, re-registering cron jobs...');
-          const CronService = (await import('./services/cron')).default;
-          const cronService = Container.get(CronService);
-          await cronService.autosave_crontab();
-          Logger.info('[boot] Cron jobs re-registered successfully');
-        } catch (error) {
-          Logger.error(`[boot] Failed to re-register cron jobs:\n${errStack(error)}`);
-        }
+      } else if (serviceType === 'http' &&
+        (msg === 'reregister-crons' || msg === 'scheduler-unavailable')) {
+        cronClient.readiness.invalidate();
       }
     });
 

back/config/util.ts

@@ -12,6 +12,7 @@ import { DependenceTypes } from '../data/dependence';
 import { FormData } from 'undici';
 import os from 'os';
 import { maybeSudo, isInContainer } from './container';
+import { resolveFileAccess } from '../shared/fileAccess';
 
 export * from './share';
 
@@ -144,7 +145,8 @@ export async function handleLogPath(
   logPath: string,
   data: string = '',
 ): Promise<string> {
-  const absolutePath = path.resolve(config.logPath, logPath);
+  const absolutePath = resolveFileAccess(config.logPath, [logPath]);
+  if (!absolutePath) throw new Error('Log path is outside the log directory');
   const logFileExist = await fileExist(absolutePath);
   if (!logFileExist) {
     await createFile(absolutePath, data);
@@ -487,18 +489,48 @@ export function psTree(pid: number): Promise<number[]> {
   });
 }
 
-export async function killTask(pid: number) {
-  const pids = await psTree(pid);
-
-  if (pids.length) {
-    try {
-      [pid, ...pids].reverse().forEach((x) => {
-        process.kill(x, 15);
-      });
-    } catch (error) { }
-  } else {
-    process.kill(pid, 2);
+export async function killTask(pid: number, waitForExit = false) {
+  const descendants = await psTree(pid);
+  if (!waitForExit) {
+    if (descendants.length) {
+      try {
+        [pid, ...descendants]
+          .reverse()
+          .forEach((target) => process.kill(target, 15));
+      } catch {}
+    } else process.kill(pid, 2);
+    return;
   }
+  const pids = [...descendants.reverse(), pid];
+  const signal = (target: number, sig: NodeJS.Signals) => {
+    try {
+      process.kill(target, sig);
+    } catch (error: any) {
+      if (error.code !== 'ESRCH') throw error;
+    }
+  };
+  for (const target of pids) signal(target, 'SIGTERM');
+  const alive = (target: number) => {
+    try {
+      process.kill(target, 0);
+      return true;
+    } catch (error: any) {
+      if (error.code === 'ESRCH') return false;
+      throw error;
+    }
+  };
+  const wait = async () => {
+    const deadline = Date.now() + 1000;
+    while (pids.some(alive) && Date.now() < deadline) {
+      await new Promise((resolve) => setTimeout(resolve, 20));
+    }
+    return pids.filter(alive);
+  };
+  let remaining = await wait();
+  for (const target of remaining) signal(target, 'SIGKILL');
+  remaining = await wait();
+  if (remaining.length)
+    throw new Error(`Task processes did not exit: ${remaining.join(', ')}`);
 }
 
 export async function getPid(cmd: string) {

back/data/cron.ts

@@ -13,6 +13,7 @@ export class Crontab {
   pid?: number;
   isDisabled?: 1 | 0;
   log_path?: string;
+  queued_token?: string | null;
   isPinned?: 1 | 0;
   labels?: string[];
   last_running_time?: number;
@@ -83,6 +84,7 @@ export const CrontabModel = sequelize.define<CronInstance>('Crontab', {
   isDisabled: DataTypes.NUMBER,
   isPinned: DataTypes.NUMBER,
   log_path: DataTypes.STRING,
+  queued_token: DataTypes.STRING,
   labels: DataTypes.JSON,
   last_running_time: DataTypes.NUMBER,
   last_execution_time: DataTypes.NUMBER,

back/loaders/db.ts

@@ -9,6 +9,7 @@ import { CrontabViewModel } from '../data/cronView';
 import { CrontabStatModel } from '../data/cronStats';
 import { RunningInstanceModel } from '../data/runningInstance';
 import { sequelize } from '../data';
+import { migrateSchema } from '../shared/schemaMigrations';
 
 export default async () => {
   try {
@@ -22,44 +23,11 @@ export default async () => {
     await CrontabStatModel.sync();
     await RunningInstanceModel.sync();
 
-    // 初始化新增字段
-    const migrations = [
-      {
-        table: 'CrontabViews',
-        column: 'filterRelation',
-        type: 'VARCHAR(255)',
-      },
-      { table: 'Subscriptions', column: 'proxy', type: 'VARCHAR(255)' },
-      { table: 'CrontabViews', column: 'type', type: 'NUMBER' },
-      { table: 'Subscriptions', column: 'autoAddCron', type: 'NUMBER' },
-      { table: 'Subscriptions', column: 'autoDelCron', type: 'NUMBER' },
-      { table: 'Crontabs', column: 'sub_id', type: 'NUMBER' },
-      { table: 'Crontabs', column: 'extra_schedules', type: 'JSON' },
-      { table: 'Crontabs', column: 'task_before', type: 'TEXT' },
-      { table: 'Crontabs', column: 'task_after', type: 'TEXT' },
-      { table: 'Crontabs', column: 'log_name', type: 'VARCHAR(255)' },
-      {
-        table: 'Crontabs',
-        column: 'allow_multiple_instances',
-        type: 'NUMBER',
-      },
-      { table: 'Crontabs', column: 'work_dir', type: 'VARCHAR(255)' },
-      { table: 'Envs', column: 'isPinned', type: 'NUMBER' },
-      { table: 'Envs', column: 'labels', type: 'JSON' },
-    ];
-
-    for (const migration of migrations) {
-      try {
-        await sequelize.query(
-          `alter table ${migration.table} add column ${migration.column} ${migration.type}`,
-        );
-      } catch (error) {
-        // Column already exists or other error, continue
-      }
-    }
+    await migrateSchema(sequelize);
 
     Logger.info('[boot] DB loaded');
   } catch (error) {
     Logger.error('[boot] DB load failed', error);
+    throw error;
   }
 };

back/loaders/initData.ts

@@ -17,6 +17,7 @@ import { createRandomString, fileExist, isDemoEnv, safeJSONParse } from '../conf
 import OpenService from '../services/open';
 import { shareStore } from '../shared/store';
 import Logger from './logger';
+import cronClient from '../schedule/client';
 import { AppModel } from '../data/open';
 import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
 import { setLang, systemLang } from '../shared/i18n';
@@ -236,7 +237,8 @@ export default async () => {
   } catch { }
 
   // 初始化保存一次ck和定时任务数据
-  await cronService.autosave_crontab();
+  cronClient.readiness.configure(() => cronService.autosave_crontab(true));
+  await cronClient.readiness.recover();
 
   await envService.set_envs();
 

back/protos/cron.proto

@@ -17,7 +17,10 @@ message ICron {
   string name = 5;
 }
 
-message AddCronRequest { repeated ICron crons = 1; }
+message AddCronRequest {
+  repeated ICron crons = 1;
+  bool replace = 2;
+}
 
 message AddCronResponse {}
 

back/protos/cron.ts

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-ts_proto. DO NOT EDIT.
 // versions:
 //   protoc-gen-ts_proto  v2.6.1
-//   protoc               v3.21.12
+//   protoc               v3.17.3
 // source: back/protos/cron.proto
 
 /* eslint-disable */
@@ -35,6 +35,7 @@ export interface ICron {
 
 export interface AddCronRequest {
   crons: ICron[];
+  replace: boolean;
 }
 
 export interface AddCronResponse {
@@ -232,7 +233,7 @@ export const ICron: MessageFns<ICron> = {
 };
 
 function createBaseAddCronRequest(): AddCronRequest {
-  return { crons: [] };
+  return { crons: [], replace: false };
 }
 
 export const AddCronRequest: MessageFns<AddCronRequest> = {
@@ -240,6 +241,9 @@ export const AddCronRequest: MessageFns<AddCronRequest> = {
     for (const v of message.crons) {
       ICron.encode(v!, writer.uint32(10).fork()).join();
     }
+    if (message.replace !== false) {
+      writer.uint32(16).bool(message.replace);
+    }
     return writer;
   },
 
@@ -258,6 +262,14 @@ export const AddCronRequest: MessageFns<AddCronRequest> = {
           message.crons.push(ICron.decode(reader, reader.uint32()));
           continue;
         }
+        case 2: {
+          if (tag !== 16) {
+            break;
+          }
+
+          message.replace = reader.bool();
+          continue;
+        }
       }
       if ((tag & 7) === 4 || tag === 0) {
         break;
@@ -268,7 +280,10 @@ export const AddCronRequest: MessageFns<AddCronRequest> = {
   },
 
   fromJSON(object: any): AddCronRequest {
-    return { crons: globalThis.Array.isArray(object?.crons) ? object.crons.map((e: any) => ICron.fromJSON(e)) : [] };
+    return {
+      crons: globalThis.Array.isArray(object?.crons) ? object.crons.map((e: any) => ICron.fromJSON(e)) : [],
+      replace: isSet(object.replace) ? globalThis.Boolean(object.replace) : false,
+    };
   },
 
   toJSON(message: AddCronRequest): unknown {
@@ -276,6 +291,9 @@ export const AddCronRequest: MessageFns<AddCronRequest> = {
     if (message.crons?.length) {
       obj.crons = message.crons.map((e) => ICron.toJSON(e));
     }
+    if (message.replace !== false) {
+      obj.replace = message.replace;
+    }
     return obj;
   },
 
@@ -285,6 +303,7 @@ export const AddCronRequest: MessageFns<AddCronRequest> = {
   fromPartial<I extends Exact<DeepPartial<AddCronRequest>, I>>(object: I): AddCronRequest {
     const message = createBaseAddCronRequest();
     message.crons = object.crons?.map((e) => ICron.fromPartial(e)) || [];
+    message.replace = object.replace ?? false;
     return message;
   },
 };

back/schedule/addCron.ts

@@ -69,6 +69,15 @@ const addCron = (
     return;
   }
 
+  // Recovery replaces the whole snapshot, including deletions and disabled jobs.
+  // Validation above must finish before touching the previous schedule.
+  if (call.request.replace) {
+    for (const jobs of scheduleStacks.values()) {
+      for (const job of jobs) job?.cancel();
+    }
+    scheduleStacks.clear();
+  }
+
   // ===== 第二遍：注册所有任务 =====
   for (const item of call.request.crons) {
     const { id, schedule, command, extra_schedules, name } = item;

back/schedule/client.ts

@@ -1,4 +1,4 @@
-import { credentials } from '@grpc/grpc-js';
+import { credentials, status, Metadata } from '@grpc/grpc-js';
 import {
   AddCronRequest,
   AddCronResponse,
@@ -9,7 +9,41 @@ import {
 import config from '../config';
 import { getGrpcCerts } from '../config/grpcCerts';
 
+import { HealthService } from '../protos/health';
+import { SchedulerReadiness } from '../shared/schedulerReadiness';
+
 class Client {
+  readonly readiness = new SchedulerReadiness(() => this.probe());
+
+  private async waitForReady(timeoutMs: number) {
+    try {
+      await new Promise<void>((resolve, reject) => {
+        this.client.waitForReady(Date.now() + timeoutMs, (err) =>
+          err ? reject(err) : resolve()
+        );
+      });
+    } catch (error) {
+      this.readiness.invalidate();
+      throw Object.assign(
+        error instanceof Error ? error : new Error(String(error)),
+        { status: 503 }
+      );
+    }
+  }
+
+  private async probe(): Promise<void> {
+    await this.waitForReady(1000);
+    await new Promise<void>((resolve, reject) => {
+      this.client.makeUnaryRequest(
+        HealthService.check.path,
+        HealthService.check.requestSerialize,
+        HealthService.check.responseDeserialize,
+        { service: 'scheduler' },
+        { deadline: Date.now() + 1000 },
+        (err, res) => err ? reject(err) : res?.status === 1 ? resolve() : reject(new Error('Scheduler unavailable')),
+      );
+    });
+  }
   private _client: CronClient | null = null;
 
   private get client(): CronClient {
@@ -28,22 +62,40 @@ class Client {
     return this._client;
   }
 
-  addCron(request: AddCronRequest['crons']): Promise<AddCronResponse> {
+  async addCron(
+    request: AddCronRequest['crons'],
+    replace = false
+  ): Promise<AddCronResponse> {
+    await this.waitForReady(2000);
     return new Promise((resolve, reject) => {
-      this.client.addCron({ crons: request }, (err, res) => {
-        if (err) {
-          reject(err);
+      this.client.addCron(
+        { crons: request, replace },
+        new Metadata(),
+        { deadline: Date.now() + 5000 },
+        (err, res) => {
+          if (err) {
+            if (err.code === status.UNAVAILABLE) {
+              this.readiness.invalidate();
+              Object.assign(err, { status: 503 });
+            }
+            return reject(err);
+          }
+          resolve(res);
         }
-        resolve(res);
-      });
+      );
     });
   }
 
-  delCron(request: DeleteCronRequest['ids']): Promise<DeleteCronResponse> {
+  async delCron(request: DeleteCronRequest['ids']): Promise<DeleteCronResponse> {
+    await this.waitForReady(2000);
     return new Promise((resolve, reject) => {
-      this.client.delCron({ ids: request }, (err, res) => {
+      this.client.delCron({ ids: request }, new Metadata(), { deadline: Date.now() + 5000 }, (err, res) => {
         if (err) {
-          reject(err);
+          if (err.code === status.UNAVAILABLE) {
+            this.readiness.invalidate();
+            Object.assign(err, { status: 503 });
+          }
+          return reject(err);
         }
         resolve(res);
       });

back/schedule/health.ts

@@ -52,6 +52,9 @@ const check = async (
   callback: sendUnaryData<HealthCheckResponse>,
 ) => {
   switch (call.request.service) {
+    // Local scheduler liveness only: never call HTTP from this probe.
+    case 'scheduler':
+      return callback(null, { status: 1 });
     case 'cron': {
       const healthUrl = `http://localhost:${config.port}${
         config.baseUrl || ''

back/services/cron.ts

@@ -1,3 +1,8 @@
+import { randomUUID } from 'crypto';
+import {
+  withSchedulerMutation,
+  schedulerRegistrationError,
+} from '../shared/schedulerMutationLock';
 import { Service, Inject } from 'typedi';
 import winston from 'winston';
 import config from '../config';
@@ -13,7 +18,6 @@ import {
   getFileContentByName,
   fileExist,
   killTask,
-  killAllTasks,
   getUniqPath,
   safeJSONParse,
   isDemoEnv,
@@ -31,8 +35,10 @@ import { writeFileWithLock } from '../shared/utils';
 import { t } from '../shared/i18n';
 import { ScheduleType } from '../interface/schedule';
 import { logStreamManager } from '../shared/logStreamManager';
+import { observeChildProcess, asError } from '../shared/childProcess';
 import { isEmpty } from 'lodash';
 import { LogReadOptions, readLogChunk } from '../shared/logReader';
+import { resolveFileAccess } from '../shared/fileAccess';
 
 @Service()
 export default class CronService {
@@ -97,42 +103,45 @@ export default class CronService {
   }
 
   public async create(payload: Crontab): Promise<Crontab> {
-    const tab = new Crontab(payload);
-    tab.saved = false;
-    tab.log_name = await this.getLogName(tab);
-    const doc = await this.insert(tab);
+    return withSchedulerMutation(async () => {
+      const tab = new Crontab(payload);
+      tab.saved = false;
+      tab.log_name = await this.getLogName(tab);
+      const doc = await this.insert(tab);
 
-    if (isDemoEnv()) {
-      return doc;
-    }
-
-    if (this.shouldUseCronClient(doc)) {
-      try {
-        await cronClient.addCron([
-          {
-            name: doc.name || '',
-            id: String(doc.id),
-            schedule: doc.schedule!,
-            command: this.makeCommand(doc),
-            extra_schedules: doc.extra_schedules || [],
-          },
-        ]);
-      } catch (error: any) {
-        // gRPC 注册失败时回滚 DB 记录，避免产生"僵尸任务"
-        // （DB 和 crontab.list 有记录但调度器永远不会执行）
-        await CrontabModel.destroy({ where: { id: doc.id } });
-        this.logger.error(
-          '[crontab] Failed to register cron job in scheduler, task creation rolled back:',
-          error?.message || error,
-        );
-        throw new Error(
-          `${t('调度器注册失败，任务创建已回滚')}: ${(error as any)?.details || error?.message}`,
-        );
+      if (isDemoEnv()) {
+        return doc;
       }
-    }
 
-    await this.setCrontab();
-    return doc;
+      if (this.shouldUseCronClient(doc)) {
+        try {
+          await cronClient.addCron([
+            {
+              name: doc.name || '',
+              id: String(doc.id),
+              schedule: doc.schedule!,
+              command: this.makeCommand(doc),
+              extra_schedules: doc.extra_schedules || [],
+            },
+          ]);
+        } catch (error: any) {
+          // gRPC 注册失败时回滚 DB 记录，避免产生"僵尸任务"
+          // （DB 和 crontab.list 有记录但调度器永远不会执行）
+          await CrontabModel.destroy({ where: { id: doc.id } });
+          this.logger.error(
+            '[crontab] Failed to register cron job in scheduler, task creation rolled back:',
+            error?.message || error,
+          );
+          throw schedulerRegistrationError(
+            `${t('调度器注册失败，任务创建已回滚')}: ${(error as any)?.details || error?.message}`,
+            error,
+          );
+        }
+      }
+
+      await this.setCrontab();
+      return doc;
+    });
   }
 
   public async insert(payload: Crontab): Promise<Crontab> {
@@ -140,69 +149,74 @@ export default class CronService {
   }
 
   public async update(payload: Partial<Crontab>): Promise<Crontab> {
-    const doc = await this.getDb({ id: payload.id });
-    const tab = new Crontab({ ...doc, ...payload });
-    tab.saved = false;
-    tab.log_name = await this.getLogName(tab);
-    const newDoc = await this.updateDb(tab);
+    return withSchedulerMutation(async () => {
+      const doc = await this.getDb({ id: payload.id });
+      const tab = new Crontab({ ...doc, ...payload });
+      tab.saved = false;
+      tab.log_name = await this.getLogName(tab);
+      const newDoc = await this.updateDb(tab);
 
-    if (doc.isDisabled === 1 || isDemoEnv()) {
-      return newDoc;
-    }
+      if (doc.isDisabled === 1 || isDemoEnv()) {
+        return newDoc;
+      }
 
-    try {
-      await cronClient.delCron([String(newDoc.id)]);
-    } catch (error: any) {
-      this.logger.warn(
-        '[crontab] Failed to unregister cron job in scheduler:',
-        error?.message || error,
-      );
-    }
-
-    if (this.shouldUseCronClient(newDoc)) {
       try {
-        await cronClient.addCron([
-          {
-            name: doc.name || '',
-            id: String(newDoc.id),
-            schedule: newDoc.schedule!,
-            command: this.makeCommand(newDoc),
-            extra_schedules: newDoc.extra_schedules || [],
-          },
-        ]);
+        await cronClient.delCron([String(newDoc.id)]);
       } catch (error: any) {
-        // gRPC 注册新任务失败 → 回滚 DB 到旧数据，并尝试恢复旧调度注册
-        await CrontabModel.update(doc, { where: { id: doc.id } });
-        if (this.shouldUseCronClient(doc)) {
-          try {
-            await cronClient.addCron([
-              {
-                name: doc.name || '',
-                id: String(doc.id),
-                schedule: doc.schedule!,
-                command: this.makeCommand(doc),
-                extra_schedules: doc.extra_schedules || [],
-              },
-            ]);
-          } catch (_recoveryError: any) {
-            this.logger.warn(
-              '[crontab] Failed to restore old cron job in scheduler after rollback:',
-              _recoveryError?.message || _recoveryError,
-            );
-          }
-        }
-        this.logger.error(
-          '[crontab] Failed to register updated cron job in scheduler, update rolled back:',
+        this.logger.warn(
+          '[crontab] Failed to unregister cron job in scheduler:',
           error?.message || error,
         );
-        throw new Error(
-          `${t('调度器注册失败，任务更新已回滚')}: ${(error as any)?.details || error?.message}`,
-        );
       }
-    }
 
-    await this.setCrontab();
-    return newDoc;
+      if (this.shouldUseCronClient(newDoc)) {
+        try {
+          await cronClient.addCron([
+            {
+              name: doc.name || '',
+              id: String(newDoc.id),
+              schedule: newDoc.schedule!,
+              command: this.makeCommand(newDoc),
+              extra_schedules: newDoc.extra_schedules || [],
+            },
+          ]);
+        } catch (error: any) {
+          // gRPC 注册新任务失败 → 回滚 DB 到旧数据，并尝试恢复旧调度注册
+          await CrontabModel.update(omit(doc, ['queued_token']), {
+            where: { id: doc.id },
+          });
+          if (this.shouldUseCronClient(doc)) {
+            try {
+              await cronClient.addCron([
+                {
+                  name: doc.name || '',
+                  id: String(doc.id),
+                  schedule: doc.schedule!,
+                  command: this.makeCommand(doc),
+                  extra_schedules: doc.extra_schedules || [],
+                },
+              ]);
+            } catch (_recoveryError: any) {
+              this.logger.warn(
+                '[crontab] Failed to restore old cron job in scheduler after rollback:',
+                _recoveryError?.message || _recoveryError,
+              );
+            }
+          }
+          this.logger.error(
+            '[crontab] Failed to register updated cron job in scheduler, update rolled back:',
+            error?.message || error,
+          );
+          throw schedulerRegistrationError(
+            `${t('调度器注册失败，任务更新已回滚')}: ${(error as any)?.details || error?.message}`,
+            error,
+          );
+        }
+      }
+
+      await this.setCrontab();
+      return newDoc;
+    });
   }
 
   public async updateDb(payload: Crontab): Promise<Crontab> {
@@ -290,16 +304,18 @@ export default class CronService {
   }
 
   public async remove(ids: number[]) {
-    await CrontabModel.destroy({ where: { id: ids } });
-    try {
-      await cronClient.delCron(ids.map(String));
-    } catch (error: any) {
-      this.logger.warn(
-        '[crontab] Failed to unregister cron job in scheduler:',
-        error?.message || error,
-      );
-    }
-    await this.setCrontab();
+    return withSchedulerMutation(async () => {
+      await CrontabModel.destroy({ where: { id: ids } });
+      try {
+        await cronClient.delCron(ids.map(String));
+      } catch (error: any) {
+        this.logger.warn(
+          '[crontab] Failed to unregister cron job in scheduler:',
+          error?.message || error,
+        );
+      }
+      await this.setCrontab();
+    });
   }
 
   public async pin(ids: number[]) {
@@ -570,46 +586,99 @@ export default class CronService {
   }
 
   public async run(ids: number[]) {
+    const queuedToken = randomUUID();
     await CrontabModel.update(
-      { status: CrontabStatus.queued },
+      { status: CrontabStatus.queued, queued_token: queuedToken },
       { where: { id: ids } },
     );
     ids.forEach((id) => {
-      this.runSingle(id);
+      this.runSingle(id, queuedToken);
     });
   }
 
   public async stop(ids: number[]) {
     const docs = await CrontabModel.findAll({ where: { id: ids } });
+    // Cancel the queued snapshot first, so a late spawn cannot claim it.
     for (const doc of docs) {
-      // Kill all running instances of this task
-      try {
-        if (doc.pid) {
-          await killTask(doc.pid);
-        }
-        const command = doc.command.replace(/\s+/g, ' ').trim();
-        await killAllTasks(command);
-        this.logger.info(
-          `[panel][停止所有运行中的任务实例] 任务ID: ${doc.id}, 命令: ${command}`,
+      if (doc.status === CrontabStatus.queued) {
+        const [cancelled] = await CrontabModel.update(
+          { status: CrontabStatus.idle, pid: null, queued_token: null } as any,
+          {
+            where: {
+              id: doc.id,
+              status: CrontabStatus.queued,
+              [Op.and]: [
+                where(colFn('log_path'), { [Op.eq]: doc.log_path ?? null }),
+                where(colFn('queued_token'), {
+                  [Op.eq]: doc.queued_token ?? null,
+                }),
+              ],
+            },
+          }
         );
+        // A concurrent claim may have won; capture its PID before signalling.
+        if (!cancelled) await doc.reload();
+      }
+    }
+    const stoppingInstances = await RunningInstanceModel.findAll({
+      attributes: ['id', 'pid', 'cron_id'],
+      where: { cron_id: ids, status: InstanceStatus.running },
+    });
+    const targets = new Set<number>(
+      [
+        ...stoppingInstances.map((instance) => instance.pid),
+        ...docs
+          .filter((doc) => doc.status === CrontabStatus.running)
+          .map((doc) => doc.pid),
+      ].filter((pid): pid is number => typeof pid === 'number' && pid > 0)
+    );
+    const stopped = new Set<number>();
+    for (const pid of targets) {
+      try {
+        await killTask(pid, true);
+        stopped.add(pid);
       } catch (error) {
         this.logger.error(
-          `[panel][停止任务失败] 任务ID: ${doc.id}, 错误: ${error}`,
+          '[panel][停止任务失败] PID: %s, 错误: %s',
+          pid,
+          asError(error).message
         );
       }
     }
-
-    // Mark all running instances as stopped
-    const finishedAt = dayjs().unix();
-    await RunningInstanceModel.update(
-      { status: InstanceStatus.stopped, finished_at: finishedAt },
-      { where: { cron_id: ids, status: InstanceStatus.running } },
-    );
-
-    await CrontabModel.update(
-      { status: CrontabStatus.idle, pid: undefined },
-      { where: { id: ids } },
-    );
+    const stoppedIds = stoppingInstances
+      .filter((instance) => instance.pid && stopped.has(instance.pid))
+      .map((instance) => instance.id!);
+    if (stoppedIds.length) {
+      await RunningInstanceModel.update(
+        { status: InstanceStatus.stopped, finished_at: dayjs().unix() },
+        { where: { id: stoppedIds } }
+      );
+    }
+    for (const doc of docs) {
+      if (
+        doc.status !== CrontabStatus.running ||
+        (doc.pid && !stopped.has(doc.pid))
+      )
+        continue;
+      const remaining = await RunningInstanceModel.count({
+        where: { cron_id: doc.id, status: InstanceStatus.running },
+      });
+      if (remaining) continue;
+      await CrontabModel.update(
+        { status: CrontabStatus.idle, pid: null, queued_token: null } as any,
+        {
+          where: {
+            id: doc.id,
+            status: CrontabStatus.running,
+            [Op.and]: [
+              where(colFn('queued_token'), { [Op.eq]: doc.queued_token ?? null }),
+              where(colFn('pid'), { [Op.eq]: doc.pid ?? null }),
+              where(colFn('log_path'), { [Op.eq]: doc.log_path ?? null }),
+            ],
+          },
+        }
+      );
+    }
   }
 
   public async stopInstance(instanceId: number) {
@@ -646,125 +715,191 @@ export default class CronService {
     return { code: 200, message: t('实例已停止') };
   }
 
-  private async runSingle(cronId: number): Promise<number | void> {
-    return taskLimit.manualRunWithCronLimit(() => {
-      return new Promise(async (resolve: any) => {
+  private async runSingle(
+    cronId: number,
+    expectedToken?: string,
+  ): Promise<number | void> {
+    return taskLimit.manualRunWithCronLimit(async () => {
+      let absolutePath: string | undefined;
+      let logPath: string | undefined;
+      let queuedLogPath: string | null | undefined;
+      let queuedToken: string | null = null;
+      let claimed = false;
+      try {
         const cron = await this.getDb({ id: cronId });
-        const params = {
-          name: cron.name,
-          command: cron.command,
-          schedule: cron.schedule,
-          extra_schedules: cron.extra_schedules,
-        };
-        if (cron.status !== CrontabStatus.queued) {
-          resolve(params);
-          return;
-        }
-
-        this.logger.info(
-          `[panel][开始执行任务] 参数: ${JSON.stringify(params)}`,
-        );
-
-        let { id, command, log_name } = cron;
-
+        if (
+          cron.status !== CrontabStatus.queued ||
+          (expectedToken !== undefined && cron.queued_token !== expectedToken)
+        ) return;
+        queuedToken = cron.queued_token ?? null;
+        queuedLogPath = cron.log_path ?? null;
+        const { id, command, log_name } = cron;
         const uniqPath =
           log_name === '/dev/null' || !log_name
             ? await getUniqPath(command, `${id}`)
             : log_name;
         const logTime = dayjs().format('YYYY-MM-DD-HH-mm-ss-SSS');
-        const logDirPath = path.resolve(config.logPath, `${uniqPath}`);
-        await fs.mkdir(logDirPath, { recursive: true });
-        const logPath = `${uniqPath}/${logTime}.log`;
-        const absolutePath = path.resolve(config.logPath, `${logPath}`);
+        logPath = `${uniqPath}/${logTime}.log`;
+        absolutePath = resolveFileAccess(config.logPath, [logPath]);
+        if (!absolutePath)
+          throw new Error('Log path is outside the log directory');
+        await fs.mkdir(path.dirname(absolutePath), { recursive: true });
+        const outputPath = absolutePath;
         const cp = spawn(
           `real_log_path=${logPath} no_delay=true ${this.makeCommand(
             cron,
-            true,
+            true
           )}`,
-          { shell: '/bin/bash' },
+          { shell: '/bin/bash' }
         );
-
-        await CrontabModel.update(
-          { status: CrontabStatus.running, pid: cp.pid, log_path: logPath },
-          { where: { id } },
-        );
-        cp.stdout.on('data', async (data) => {
-          await logStreamManager.write(absolutePath, data.toString());
+        // Install observers before the first await: very short children may already exit.
+        const { completed } = observeChildProcess(cp, {
+          onStart: async () => {
+            try {
+              const [count] = await CrontabModel.update(
+                {
+                  status: CrontabStatus.running,
+                  pid: cp.pid,
+                  log_path: logPath,
+                },
+                {
+                  where: {
+                    id,
+                    status: CrontabStatus.queued,
+                    [Op.and]: [
+                      where(colFn('queued_token'), { [Op.eq]: queuedToken }),
+                      where(colFn('log_path'), { [Op.eq]: queuedLogPath }),
+                    ],
+                  },
+                }
+              );
+              if (count !== 1)
+                throw new Error(
+                  'Task was stopped or superseded before startup'
+                );
+              claimed = true;
+            } catch (error) {
+              if (cp.pid) await killTask(cp.pid, true);
+              throw error;
+            }
+          },
+          onStdout: (message) => logStreamManager.write(outputPath, message),
+          onStderr: (message) => logStreamManager.write(outputPath, message),
         });
-        cp.stderr.on('data', async (data) => {
-          this.logger.info(
-            '[panel][执行任务失败] 命令: %s, 错误信息: %j',
-            command,
-            data.toString(),
-          );
-          await logStreamManager.write(absolutePath, data.toString());
-        });
-        cp.on('error', async (err) => {
+        const result = await completed;
+        if (result.error) {
           this.logger.error(
-            '[panel][创建任务失败] 命令: %s, 错误信息: %j',
-            command,
-            err,
+            '[panel][执行任务失败] 任务ID: %s, 错误: %s',
+            id,
+            result.error.message
           );
-          await logStreamManager.write(absolutePath, JSON.stringify(err));
-        });
-
-        cp.on('exit', async (code) => {
-          this.logger.info(
-            '[panel][执行任务结束] 参数: %s, 退出码: %j',
-            JSON.stringify(params),
-            code,
+        }
+        this.logger.info(
+          '[panel][执行任务结束] 任务ID: %s, 退出码: %j',
+          id,
+          result.code
+        );
+        return { ...cron, pid: cp.pid, ...result } as any;
+      } catch (error) {
+        this.logger.error(
+          '[panel][创建任务失败] 任务ID: %s, 错误: %s',
+          cronId,
+          asError(error).message
+        );
+      } finally {
+        try {
+          if (absolutePath) await logStreamManager.closeStream(absolutePath);
+        } catch (error) {
+          this.logger.error(
+            '[panel][关闭任务日志失败] %s',
+            asError(error).message
           );
-          await logStreamManager.closeStream(absolutePath);
-          resolve({ ...params, pid: cp.pid, code });
-        });
-      });
+        }
+        try {
+          // Do not overwrite a newer run's state or its script-reported exit code.
+          await CrontabModel.update(
+            { status: CrontabStatus.idle, pid: null, queued_token: null } as any,
+            {
+              where: {
+                id: cronId,
+                [Op.and]: where(colFn('queued_token'), { [Op.eq]: queuedToken }),
+                [Op.or]: [
+                  ...(queuedLogPath !== undefined && !claimed
+                    ? [
+                        {
+                          status: CrontabStatus.queued,
+                          [Op.and]: where(colFn('log_path'), {
+                            [Op.eq]: queuedLogPath,
+                          }),
+                        },
+                      ]
+                    : []),
+                  ...(claimed && logPath
+                    ? [{ log_path: logPath, status: CrontabStatus.running }]
+                    : []),
+                ],
+              },
+            }
+          );
+        } catch (error) {
+          this.logger.error(
+            '[panel][清理任务状态失败] %s',
+            asError(error).message
+          );
+        }
+      }
     });
   }
 
   public async disabled(ids: number[]) {
-    await CrontabModel.update({ isDisabled: 1 }, { where: { id: ids } });
-    try {
-      await cronClient.delCron(ids.map(String));
-    } catch (error: any) {
-      this.logger.warn(
-        '[crontab] Failed to unregister cron job in scheduler:',
-        error?.message || error,
-      );
-    }
-    await this.setCrontab();
+    return withSchedulerMutation(async () => {
+      await CrontabModel.update({ isDisabled: 1 }, { where: { id: ids } });
+      try {
+        await cronClient.delCron(ids.map(String));
+      } catch (error: any) {
+        this.logger.warn(
+          '[crontab] Failed to unregister cron job in scheduler:',
+          error?.message || error,
+        );
+      }
+      await this.setCrontab();
+    });
   }
 
   public async enabled(ids: number[]) {
-    await CrontabModel.update({ isDisabled: 0 }, { where: { id: ids } });
-    const docs = await CrontabModel.findAll({ where: { id: ids } });
-    const crons = docs
-      .filter((x) => this.shouldUseCronClient(x))
-      .map((doc) => ({
-        name: doc.name || '',
-        id: String(doc.id),
-        schedule: doc.schedule!,
-        command: this.makeCommand(doc),
-        extra_schedules: doc.extra_schedules || [],
-      }));
+    return withSchedulerMutation(async () => {
+      await CrontabModel.update({ isDisabled: 0 }, { where: { id: ids } });
+      const docs = await CrontabModel.findAll({ where: { id: ids } });
+      const crons = docs
+        .filter((x) => this.shouldUseCronClient(x))
+        .map((doc) => ({
+          name: doc.name || '',
+          id: String(doc.id),
+          schedule: doc.schedule!,
+          command: this.makeCommand(doc),
+          extra_schedules: doc.extra_schedules || [],
+        }));
 
-    if (isDemoEnv()) {
-      return;
-    }
+      if (isDemoEnv()) {
+        return;
+      }
 
-    try {
-      await cronClient.addCron(crons);
-    } catch (error: any) {
-      // gRPC 注册失败 → 回滚启用状态，避免 DB 显示已启用但调度器未注册
-      await CrontabModel.update({ isDisabled: 1 }, { where: { id: ids } });
-      this.logger.error(
-        '[crontab] Failed to register cron job in scheduler, enable rolled back:',
-        error?.message || error,
-      );
-      throw new Error(
-        `${t('调度器注册失败，任务启用已回滚')}: ${(error as any)?.details || error?.message}`,
-      );
-    }
-    await this.setCrontab();
+      try {
+        await cronClient.addCron(crons);
+      } catch (error: any) {
+        // gRPC 注册失败 → 回滚启用状态，避免 DB 显示已启用但调度器未注册
+        await CrontabModel.update({ isDisabled: 1 }, { where: { id: ids } });
+        this.logger.error(
+          '[crontab] Failed to register cron job in scheduler, enable rolled back:',
+          error?.message || error,
+        );
+        throw schedulerRegistrationError(
+          `${t('调度器注册失败，任务启用已回滚')}: ${(error as any)?.details || error?.message}`,
+          error,
+        );
+      }
+      await this.setCrontab();
+    });
   }
 
   public async log(
@@ -957,39 +1092,42 @@ export default class CronService {
     });
   }
 
-  public async autosave_crontab() {
-    const tabs = await this.crontabs();
-    const regularCrons = tabs.data
-      .filter(
-        (x) =>
-          x.isDisabled !== 1 &&
-          this.shouldUseCronClient(x),
-      )
-      .map((doc) => ({
-        name: doc.name || '',
-        id: String(doc.id),
-        schedule: doc.schedule!,
-        command: this.makeCommand(doc),
-        extra_schedules: doc.extra_schedules || [],
-      }));
+  public async autosave_crontab(requireScheduler = false) {
+    return withSchedulerMutation(async () => {
+      const tabs = await this.crontabs();
+      const regularCrons = tabs.data
+        .filter(
+          (x) =>
+            x.isDisabled !== 1 &&
+            this.shouldUseCronClient(x),
+        )
+        .map((doc) => ({
+          name: doc.name || '',
+          id: String(doc.id),
+          schedule: doc.schedule!,
+          command: this.makeCommand(doc),
+          extra_schedules: doc.extra_schedules || [],
+        }));
 
-    if (isDemoEnv()) {
-      await writeFileWithLock(config.crontabFile, '');
-      return;
-    }
+      if (isDemoEnv()) {
+        await writeFileWithLock(config.crontabFile, '');
+        return;
+      }
 
-    // 先同步 crontab.list 与系统 crontab，确保其始终反映数据库真实状态。
-    // gRPC 调度注册为尽力而为：失败时不阻断文件同步，调度器重启后会重新注册。
-    // 这避免了因调度器短暂不可用导致 crontab.list 与数据库脱节（订阅更新误判任务已存在）。
-    await this.setCrontab(tabs);
-    try {
-      await cronClient.addCron(regularCrons);
-    } catch (error: any) {
-      this.logger.warn(
-        '[crontab] Failed to register cron job in scheduler:',
-        error?.message || error,
-      );
-    }
+      // 先同步 crontab.list 与系统 crontab，确保其始终反映数据库真实状态。
+      // gRPC 调度注册为尽力而为：失败时不阻断文件同步，调度器重启后会重新注册。
+      // 这避免了因调度器短暂不可用导致 crontab.list 与数据库脱节（订阅更新误判任务已存在）。
+      await this.setCrontab(tabs);
+      try {
+        await cronClient.addCron(regularCrons, requireScheduler);
+      } catch (error: any) {
+        this.logger.warn(
+          '[crontab] Failed to register cron job in scheduler:',
+          error?.message || error,
+        );
+        if (requireScheduler) throw error;
+      }
+    });
   }
 
   public async bootTask() {
@@ -998,13 +1136,7 @@ export default class CronService {
       (x) => !x.isDisabled && this.isBootSchedule(x.schedule),
     );
     if (bootTasks.length > 0) {
-      await CrontabModel.update(
-        { status: CrontabStatus.queued },
-        { where: { id: bootTasks.map((t) => t.id!) } },
-      );
-      for (const task of bootTasks) {
-        this.runSingle(task.id!);
-      }
+      await this.run(bootTasks.map((task) => task.id!));
     }
   }
 }

back/services/health.ts

@@ -1,6 +1,6 @@
 import { Service } from 'typedi';
 import Logger from '../loaders/logger';
-import { GrpcServerService } from './grpc';
+import cronClient from '../schedule/client';
 import { HttpServerService } from './http';
 
 interface HealthStatus {
@@ -23,7 +23,6 @@ export class HealthService {
   private startTime = Date.now();
 
   constructor(
-    private grpcServerService: GrpcServerService,
     private httpServerService: HttpServerService,
   ) {}
 
@@ -56,8 +55,7 @@ export class HealthService {
     }
 
     try {
-      const grpcServer = this.grpcServerService.getServer();
-      if (!grpcServer) {
+      if (!(await cronClient.readiness.check())) {
         status.services.grpc = false;
         status.status = 'error';
       }

back/services/http.ts

@@ -39,7 +39,13 @@ export class HttpServerService {
 
   private async tryListen(expressApp: express.Application, port: number, host: string): Promise<Server> {
     return new Promise((resolve, reject) => {
-      const server = expressApp.listen(port, host, () => {
+      // There is one HTTP worker; accepting here avoids primary IPC handoff
+      // for every connection. Restore shared listening for custom clusters.
+      const server = expressApp.listen({
+        port,
+        host,
+        exclusive: process.env.QL_HTTP_SHARED_LISTEN !== 'true',
+      }, () => {
         resolve(server);
       });
 

back/services/schedule.ts

@@ -11,6 +11,7 @@ import {
 import dayjs from 'dayjs';
 import taskLimit from '../shared/pLimit';
 import { spawn } from 'cross-spawn';
+import { observeChildProcess, asError, ProcessResult } from '../shared/childProcess';
 
 export interface ScheduleTaskType {
   id?: number;
@@ -27,7 +28,7 @@ export interface TaskCallbacks {
     startTime: dayjs.Dayjs,
   ) => Promise<void>;
   onEnd?: (
-    cp: ChildProcessWithoutNullStreams,
+    cp: ChildProcessWithoutNullStreams | undefined,
     endTime: dayjs.Dayjs,
     diff: number,
   ) => Promise<void>;
@@ -63,73 +64,85 @@ export default class ScheduleService {
   ) {
     const { runOrigin, ...others } = params;
 
-    return taskLimit[this.taskLimitMap[runOrigin]](others, () => {
-      return new Promise(async (resolve, reject) => {
-        this.logger.info(
-          `[panel][开始执行任务] 参数: ${JSON.stringify({
-            ...others,
-            command,
-          })}`,
-        );
-
-        try {
-          const startTime = dayjs();
-          await callbacks.onBefore?.(startTime);
-
-          const cp = spawn(command, { shell: '/bin/bash' });
-
-          callbacks.onStart?.(cp, startTime);
-          completionTime === 'start' && resolve(cp.pid);
-
-          cp.stdout.on('data', async (data) => {
-            await callbacks.onLog?.(data.toString());
-          });
-
-          cp.stderr.on('data', async (data) => {
-            this.logger.info(
-              '[panel][执行任务失败] 命令: %s, 错误信息: %j',
-              command,
-              data.toString(),
-            );
-            await callbacks.onError?.(data.toString());
-          });
-
-          cp.on('error', async (err) => {
-            this.logger.error(
-              '[panel][创建任务失败] 命令: %s, 错误信息: %j',
-              command,
-              err,
-            );
-            await callbacks.onError?.(JSON.stringify(err));
-          });
-
-          cp.on('exit', async (code) => {
-            this.logger.info(
-              '[panel][执行任务结束] 参数: %s, 退出码: %j',
-              JSON.stringify({
-                ...others,
-                command,
-              }),
-              code,
-            );
-            const endTime = dayjs();
-            await callbacks.onEnd?.(
-              cp,
-              endTime,
-              endTime.diff(startTime, 'seconds'),
-            );
-            resolve({ ...others, pid: cp.pid, code });
-          });
-        } catch (error) {
-          this.logger.error(
-            '[panel][执行任务失败] 命令: %s, 错误信息: %j',
-            command,
-            error,
-          );
-          await callbacks.onError?.(JSON.stringify(error));
-        }
-      });
+    let resolveStart!: (pid: number | undefined) => void;
+    let rejectStart!: (error: Error) => void;
+    const startResult = new Promise<number | undefined>((resolve, reject) => {
+      resolveStart = resolve;
+      rejectStart = reject;
     });
+    // Most scheduled callers only observe completion (or intentionally detach).
+    void startResult.catch(() => {});
+    const completion = taskLimit[this.taskLimitMap[runOrigin]](
+      others,
+      async () => {
+        const startTime = dayjs();
+        let cp: ChildProcessWithoutNullStreams | undefined;
+        let result: ProcessResult = { code: null, signal: null };
+        try {
+          this.logger.info('[panel][开始执行任务] 任务ID: %s', others.id);
+          await callbacks.onBefore?.(startTime);
+          cp = spawn(command, { shell: '/bin/bash' });
+          const child = cp;
+          const observed = observeChildProcess(child, {
+            onStart: async () => {
+              await callbacks.onStart?.(child, startTime);
+            },
+            onStdout: callbacks.onLog,
+            onStderr: callbacks.onError,
+          });
+          observed.started.then(resolveStart, rejectStart);
+          result = await observed.completed;
+        } catch (error) {
+          result.error = asError(error);
+          rejectStart(result.error);
+        }
+
+        if (result.error) {
+          this.logger.error(
+            '[panel][执行任务失败] 任务ID: %s, 错误: %s',
+            others.id,
+            result.error.message,
+          );
+          try {
+            await callbacks.onError?.(result.error.message);
+          } catch (error) {
+            this.logger.error(
+              '[panel][任务错误回调失败] %s',
+              asError(error).message,
+            );
+          }
+        }
+        // Cleanup also runs after setup/spawn failure, and only after both pipes drain.
+        const endTime = dayjs();
+        try {
+          await callbacks.onEnd?.(
+            cp,
+            endTime,
+            endTime.diff(startTime, 'seconds'),
+          );
+        } catch (error) {
+          result.error ??= asError(error);
+          this.logger.error(
+            '[panel][任务结束回调失败] %s',
+            asError(error).message,
+          );
+        }
+        this.logger.info(
+          '[panel][执行任务结束] 任务ID: %s, 退出码: %j',
+          others.id,
+          result.code,
+        );
+        return { ...others, pid: cp?.pid, ...result };
+      },
+    ).catch((error) => {
+      // Queue/setup failures must not become unhandled rejections in detached callers.
+      rejectStart(asError(error));
+      this.logger.error('[panel][任务队列失败] %s', asError(error).message);
+      return { ...others, code: null, signal: null, error: asError(error) };
+    });
+
+    // Returning a PID must not release the execution slot while the process runs.
+    return completionTime === 'start' ? startResult : completion;
   }
 
   async createCronTask(

back/services/subscription.ts

@@ -159,48 +159,54 @@ export default class SubscriptionService {
         );
       },
       onEnd: async (cp, endTime, diff) => {
-        const sub = await this.getDb({ id: doc.id });
-        const absolutePath = await handleLogPath(sub.log_path as string);
-
-        // 执行 sub_after
-        let afterStr = '';
+        let absolutePath: string | undefined;
         try {
-          if (sub.sub_after) {
-            await logStreamManager.write(absolutePath, `\n\n## ${t('执行after命令...')}\n\n`);
-            afterStr = await promiseExec(sub.sub_after);
+          const sub = await this.getDb({ id: doc.id });
+          absolutePath = await handleLogPath(sub.log_path as string);
+
+          // 执行 sub_after
+          let afterStr = '';
+          try {
+            if (sub.sub_after) {
+              await logStreamManager.write(
+                absolutePath,
+                `\n\n## ${t('执行after命令...')}\n\n`,
+              );
+              afterStr = await promiseExec(sub.sub_after);
+            }
+          } catch (error: any) {
+            afterStr =
+              (error.stderr && error.stderr.toString()) || JSON.stringify(error);
+          }
+          if (afterStr) {
+            await logStreamManager.write(absolutePath, `${afterStr}\n`);
+          }
+
+          await logStreamManager.write(
+            absolutePath,
+            '\n' +
+              tf(
+                '## 执行结束... %s  耗时 %s 秒',
+                endTime.format('YYYY-MM-DD HH:mm:ss'),
+                String(diff),
+              ) +
+              LOG_END_SYMBOL,
+          );
+        } finally {
+          try {
+            if (absolutePath) await logStreamManager.closeStream(absolutePath);
+          } finally {
+            await SubscriptionModel.update(
+              { status: SubscriptionStatus.idle, pid: null } as any,
+              { where: { id: doc.id } },
+            );
+            this.sockService.sendMessage({
+              type: 'runSubscriptionEnd',
+              message: t('订阅执行完成'),
+              references: [doc.id as number],
+            });
           }
-        } catch (error: any) {
-          afterStr =
-            (error.stderr && error.stderr.toString()) || JSON.stringify(error);
         }
-        if (afterStr) {
-          await logStreamManager.write(absolutePath, `${afterStr}\n`);
-        }
-
-        await logStreamManager.write(
-          absolutePath,
-          '\n' +
-            tf(
-              '## 执行结束... %s  耗时 %s 秒',
-              endTime.format('YYYY-MM-DD HH:mm:ss'),
-              String(diff),
-            ) +
-            LOG_END_SYMBOL,
-        );
-
-        // Close the stream after task completion
-        await logStreamManager.closeStream(absolutePath);
-
-        await SubscriptionModel.update(
-          { status: SubscriptionStatus.idle, pid: undefined },
-          { where: { id: sub.id } },
-        );
-
-        this.sockService.sendMessage({
-          type: 'runSubscriptionEnd',
-          message: t('订阅执行完成'),
-          references: [doc.id as number],
-        });
       },
       onError: async (message: string) => {
         const sub = await this.getDb({ id: doc.id });

back/shared/childProcess.ts

@@ -0,0 +1,78 @@
+import { ChildProcessWithoutNullStreams } from 'child_process';
+import { Readable } from 'stream';
+
+export interface ProcessResult {
+  code: number | null;
+  signal: NodeJS.Signals | null;
+  error?: Error;
+}
+
+export function asError(error: unknown): Error {
+  return error instanceof Error ? error : new Error(String(error));
+}
+
+/** Attach immediately after spawn, before awaiting database or user callbacks. */
+export function observeChildProcess(
+  child: ChildProcessWithoutNullStreams,
+  callbacks: {
+    onStart?: () => Promise<void>;
+    onStdout?: (message: string) => Promise<void>;
+    onStderr?: (message: string) => Promise<void>;
+  } = {},
+) {
+  let failure: Error | undefined;
+  const recordError = (error: unknown) => {
+    failure ??= asError(error);
+  };
+  const spawned = new Promise<void>((resolve, reject) => {
+    child.once('spawn', resolve);
+    // Keep the listener through close: errors can occur after a successful spawn.
+    child.on('error', (error) => {
+      recordError(error);
+      reject(error);
+    });
+  });
+  const closed = new Promise<ProcessResult>((resolve) => {
+    child.once('close', (code, signal) => resolve({ code, signal }));
+  });
+  const started = spawned.then(async () => {
+    await callbacks.onStart?.();
+    return child.pid;
+  });
+  // The caller can ask only for completion, without an unhandled start rejection.
+  const ready = started.catch(recordError);
+
+  const consume = async (
+    stream: Readable,
+    callback?: (message: string) => Promise<void>,
+  ) => {
+    // StringDecoder in Readable preserves UTF-8 characters split across chunks.
+    stream.setEncoding('utf8');
+    let callbackFailed = false;
+    try {
+      for await (const chunk of stream) {
+        await ready;
+        if (!callbackFailed && callback) {
+          try {
+            await callback(String(chunk));
+          } catch (error) {
+            recordError(error);
+            callbackFailed = true;
+          }
+        }
+        // Even if a log sink fails, drain the pipe so the child can finish.
+      }
+    } catch (error) {
+      recordError(error);
+    }
+  };
+  const output = Promise.all([
+    consume(child.stdout, callbacks.onStdout),
+    consume(child.stderr, callbacks.onStderr),
+  ]);
+  const completed = Promise.all([closed, ready, output]).then(([result]) => ({
+    ...result,
+    error: failure,
+  }));
+  return { started, completed };
+}

back/shared/logStreamManager.ts

@@ -1,5 +1,8 @@
 import { createWriteStream, WriteStream } from 'fs';
 import { EventEmitter } from 'events';
+import path from 'path';
+import config from '../config';
+import { resolveFileAccess } from './fileAccess';
 
 /**
  * Manages write streams for log files to improve performance by avoiding repeated file opens
@@ -8,83 +11,90 @@ export class LogStreamManager extends EventEmitter {
   private streams: Map<string, WriteStream> = new Map();
   private pendingWrites: Map<string, Promise<void>> = new Map();
 
-  /**
-   * Write data to a log file using a managed stream
-   * @param filePath - Absolute path to the log file
-   * @param data - Data to write to the log file
-   */
-  async write(filePath: string, data: string): Promise<void> {
-    // Wait for any pending writes to this file to complete
-    const pending = this.pendingWrites.get(filePath);
-    if (pending) {
-      await pending;
-    }
+  private closingStreams = new Map<string, Promise<void>>();
+  private closedStreams = new WeakSet<WriteStream>();
+  private streamErrors = new Map<string, Error>();
 
-    // Create a new promise for this write operation
-    const writePromise = new Promise<void>((resolve, reject) => {
-      let stream = this.streams.get(filePath);
-
-      if (!stream) {
-        // Create a new write stream if one doesn't exist
-        stream = createWriteStream(filePath, { flags: 'a' });
-        this.streams.set(filePath, stream);
-
-        // Handle stream errors
-        stream.on('error', (error) => {
-          this.emit('error', { filePath, error });
-          // Remove the stream from the map on error
-          this.streams.delete(filePath);
-          reject(error);
-        });
-      }
-
-      // Write the data
-      const canContinue = stream.write(data, 'utf8', (error) => {
-        if (error) {
-          reject(error);
-        } else {
-          resolve();
-        }
-      });
-
-      // Handle backpressure
-      if (!canContinue) {
-        stream.once('drain', () => {
-          // Stream is ready for more data
-        });
-      }
-    });
-
-    this.pendingWrites.set(filePath, writePromise);
-
-    try {
-      await writePromise;
-    } finally {
-      this.pendingWrites.delete(filePath);
-    }
+  constructor(private readonly logRoot = config.logPath) {
+    super();
   }
 
-  /**
-   * Close the stream for a specific file path
-   * @param filePath - Absolute path to the log file
-   */
-  async closeStream(filePath: string): Promise<void> {
-    // Wait for any pending writes to complete
-    const pending = this.pendingWrites.get(filePath);
-    if (pending) {
-      await pending.catch(() => {
-        // Ignore errors on pending writes during close
-      });
+  /** Register each write synchronously, so concurrent callers cannot lose the tail. */
+  async write(filePath: string, data: string): Promise<void> {
+    if (this.closingStreams.has(filePath)) {
+      throw new Error(`Log stream is closing: ${filePath}`);
     }
+    const previous = this.pendingWrites.get(filePath) || Promise.resolve();
+    const pending = previous.then(
+      () =>
+        new Promise<void>((resolve, reject) => {
+          const failure = this.streamErrors.get(filePath);
+          if (failure) return reject(failure);
+          let stream = this.streams.get(filePath);
+          if (!stream) {
+            // Validate only when opening: subsequent chunks reuse the same descriptor.
+            const root = path.resolve(this.logRoot);
+            const target = path.resolve(filePath);
+            if (
+              !target.startsWith(root + path.sep) ||
+              !resolveFileAccess(root, [target])
+            ) {
+              return reject(new Error('Log path is outside the log directory'));
+            }
+            stream = createWriteStream(target, { flags: 'a' });
+            this.streams.set(filePath, stream);
+            const current = stream;
+            stream.once('close', () => this.closedStreams.add(current));
+            stream.on('error', (error) => {
+              this.streamErrors.set(filePath, error);
+              // EventEmitter's unobserved "error" event would crash the caller.
+              if (this.listenerCount('error') > 0)
+                this.emit('error', { filePath, error });
+            });
+          }
+          stream.write(data, 'utf8', (error) =>
+            error ? reject(error) : resolve(),
+          );
+        }),
+    );
+    this.pendingWrites.set(filePath, pending);
+    // Keep the tail until close, including failures; never reopen a failed log mid-run.
+    return pending;
+  }
 
-    const stream = this.streams.get(filePath);
-    if (stream) {
-      return new Promise<void>((resolve) => {
-        stream.end(() => {
-          this.streams.delete(filePath);
-          resolve();
-        });
-      });
+  async closeStream(filePath: string): Promise<void> {
+    const closing = this.closingStreams.get(filePath);
+    if (closing) return closing;
+    const pending = this.pendingWrites.get(filePath);
+    const result = (async () => {
+      let failure: unknown;
+      try {
+        await pending;
+      } catch (error) {
+        failure = error;
+      }
+      const stream = this.streams.get(filePath);
+      try {
+        if (stream && !this.closedStreams.has(stream)) {
+          await new Promise<void>((resolve) => {
+            stream.once('close', resolve);
+            if (failure || stream.destroyed) stream.destroy();
+            else stream.end();
+          });
+        }
+        failure ||= this.streamErrors.get(filePath);
+        if (failure) throw failure;
+      } finally {
+        this.streams.delete(filePath);
+        this.pendingWrites.delete(filePath);
+        this.streamErrors.delete(filePath);
+      }
+    })();
+    this.closingStreams.set(filePath, result);
+    try {
+      await result;
+    } finally {
+      this.closingStreams.delete(filePath);
     }
   }
 
@@ -92,7 +102,11 @@ export class LogStreamManager extends EventEmitter {
    * Close all open streams
    */
   async closeAll(): Promise<void> {
-    const closePromises = Array.from(this.streams.keys()).map((filePath) =>
+    const paths = new Set([
+      ...this.streams.keys(),
+      ...this.pendingWrites.keys(),
+    ]);
+    const closePromises = Array.from(paths).map((filePath) =>
       this.closeStream(filePath),
     );
     await Promise.all(closePromises);

back/shared/runCron.ts

@@ -4,15 +4,13 @@ import Logger from '../loaders/logger';
 import { ICron } from '../protos/cron';
 import { CrontabModel, CrontabStatus } from '../data/cron';
 import { killTask } from '../config/util';
-import {
-  RunningInstanceModel,
-  InstanceStatus,
-} from '../data/runningInstance';
+import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
 import dayjs from 'dayjs';
+import { observeChildProcess, asError } from './childProcess';
 
 export function runCron(cmd: string, cron: ICron): Promise<number | void> {
-  return taskLimit.runWithCronLimit(cron, () => {
-    return new Promise(async (resolve: any) => {
+  return taskLimit.runWithCronLimit(cron, async () => {
+    try {
       // Check if the cron is already running and stop it (only if multiple instances are not allowed)
       try {
         const existingCron = await CrontabModel.findOne({
@@ -38,7 +36,12 @@ export function runCron(cmd: string, cron: ICron): Promise<number | void> {
           const stoppedAt = dayjs().unix();
           await RunningInstanceModel.update(
             { status: InstanceStatus.stopped, finished_at: stoppedAt },
-            { where: { cron_id: Number(cron.id), status: InstanceStatus.running } },
+            {
+              where: {
+                cron_id: Number(cron.id),
+                status: InstanceStatus.running,
+              },
+            },
           );
           // Update the status to idle after killing
           await CrontabModel.update(
@@ -60,33 +63,37 @@ export function runCron(cmd: string, cron: ICron): Promise<number | void> {
       );
       const cp = spawn(cmd, { shell: '/bin/bash' });
 
-      cp.stderr.on('data', (data) => {
-        Logger.info(
-          '[schedule][执行任务失败] 命令: %s, 错误信息: %j',
-          cmd,
-          data.toString(),
-        );
+      const { completed } = observeChildProcess(cp, {
+        onStderr: async (message) => {
+          Logger.info(
+            '[schedule][任务标准错误] 命令: %s, 信息: %s',
+            cmd,
+            message,
+          );
+        },
       });
-      cp.on('error', (err) => {
+      const result = await completed;
+      if (result.error) {
         Logger.error(
-          '[schedule][创建任务失败] 命令: %s, 错误信息: %j',
+          '[schedule][执行任务失败] 命令: %s, 错误: %s',
           cmd,
-          err,
+          result.error.message,
         );
-      });
-
-      cp.on('exit', async (code) => {
-        taskLimit.removeQueuedCron(cron.id);
-        Logger.info(
-          '[schedule][执行任务结束] 参数: %s, 退出码: %j',
-          JSON.stringify({
-            ...cron,
-            command: cmd,
-          }),
-          code,
-        );
-        resolve({ ...cron, command: cmd, pid: cp.pid, code });
-      });
-    });
+      }
+      Logger.info(
+        '[schedule][执行任务结束] 任务ID: %s, 退出码: %j',
+        cron.id,
+        result.code,
+      );
+      return { ...cron, command: cmd, pid: cp.pid, ...result } as any;
+    } catch (error) {
+      Logger.error(
+        '[schedule][创建任务失败] 命令: %s, 错误: %s',
+        cmd,
+        asError(error).message,
+      );
+    } finally {
+      taskLimit.removeQueuedCron(cron.id);
+    }
   });
 }

back/shared/schedulerMutationLock.ts

@@ -0,0 +1,38 @@
+import lockfile from 'proper-lockfile';
+import config from '../config';
+
+// HTTP and gRPC both mutate cron definitions. Hold one shared lock from the
+// initial DB read/write through scheduler registration (including rollback).
+// Recovery takes the same lock before reading its replacement snapshot.
+export async function withSchedulerMutation<T>(
+  operation: () => Promise<T>,
+): Promise<T> {
+  let release: () => Promise<void>;
+  try {
+    release = await lockfile.lock(config.crontabFile, {
+      realpath: false,
+      lockfilePath: `${config.crontabFile}.scheduler.lock`,
+      stale: 30000,
+      update: 10000,
+      retries: { retries: 50, factor: 1, minTimeout: 100, maxTimeout: 100 },
+    });
+  } catch (cause) {
+    throw Object.assign(
+      new Error('Scheduler configuration is busy', { cause }),
+      {
+        status: 503,
+      },
+    );
+  }
+  try {
+    return await operation();
+  } finally {
+    await release();
+  }
+}
+
+export function schedulerRegistrationError(message: string, cause: any): Error {
+  return Object.assign(new Error(message, { cause }), {
+    status: cause?.status === 503 ? 503 : 500,
+  });
+}

back/shared/schedulerReadiness.ts

@@ -0,0 +1,77 @@
+// Recovery is single-flight and retried only while unavailable, never at idle.
+export class SchedulerReadiness {
+  private ready = false;
+  private generation = 0;
+  private restore?: () => Promise<void>;
+  private pending?: Promise<boolean>;
+  private retry?: NodeJS.Timeout;
+
+  constructor(private probe: () => Promise<void>, private retryMs = 1000) {}
+
+  configure(restore: () => Promise<void>) {
+    this.restore = restore;
+  }
+
+  invalidate() {
+    this.ready = false;
+    this.generation++;
+    void this.recover();
+  }
+
+  recover(): Promise<boolean> {
+    if (this.pending) return this.pending;
+    if (!this.restore) return Promise.resolve(false);
+    clearTimeout(this.retry);
+    const generation = this.generation;
+    this.ready = false;
+    this.pending = (async () => {
+      try {
+        await this.probe();
+        await this.restore!();
+        await this.probe();
+        this.ready = generation === this.generation;
+      } catch {
+        this.ready = false;
+      }
+      return this.ready;
+    })().finally(() => {
+      this.pending = undefined;
+      if (!this.ready) {
+        this.retry = setTimeout(() => void this.recover(), this.retryMs);
+        this.retry.unref();
+      }
+    });
+    return this.pending;
+  }
+
+  async check(): Promise<boolean> {
+    if (!this.ready) return false;
+    const generation = this.generation;
+    try {
+      await this.probe();
+      return this.ready && generation === this.generation;
+    } catch {
+      this.invalidate();
+      return false;
+    }
+  }
+
+  async ensureReady(timeoutMs = 2000): Promise<void> {
+    let timer: NodeJS.Timeout | undefined;
+    try {
+      const available = await Promise.race([
+        (async () => (await this.check()) || (await this.recover()))(),
+        new Promise<boolean>((resolve) => {
+          timer = setTimeout(() => resolve(false), timeoutMs);
+        }),
+      ]);
+      if (!available) {
+        throw Object.assign(new Error('Scheduler is recovering; try again later'), {
+          status: 503,
+        });
+      }
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+}

back/shared/schemaMigrations.ts

@@ -0,0 +1,70 @@
+import { QueryTypes, Sequelize } from 'sequelize';
+
+// Append new entries; IDs are persisted and must not be renumbered or reused.
+const columns = [
+  {
+    table: 'CrontabViews',
+    column: 'filterRelation',
+    type: 'VARCHAR(255)',
+  },
+  { table: 'Subscriptions', column: 'proxy', type: 'VARCHAR(255)' },
+  { table: 'CrontabViews', column: 'type', type: 'NUMBER' },
+  { table: 'Subscriptions', column: 'autoAddCron', type: 'NUMBER' },
+  { table: 'Subscriptions', column: 'autoDelCron', type: 'NUMBER' },
+  { table: 'Crontabs', column: 'sub_id', type: 'NUMBER' },
+  { table: 'Crontabs', column: 'extra_schedules', type: 'JSON' },
+  { table: 'Crontabs', column: 'task_before', type: 'TEXT' },
+  { table: 'Crontabs', column: 'task_after', type: 'TEXT' },
+  { table: 'Crontabs', column: 'log_name', type: 'VARCHAR(255)' },
+  {
+    table: 'Crontabs',
+    column: 'allow_multiple_instances',
+    type: 'NUMBER',
+  },
+  { table: 'Crontabs', column: 'work_dir', type: 'VARCHAR(255)' },
+  { table: 'Envs', column: 'isPinned', type: 'NUMBER' },
+  { table: 'Envs', column: 'labels', type: 'JSON' },
+  { table: 'Crontabs', column: 'queued_token', type: 'VARCHAR(255)' },
+];
+
+export async function migrateSchema(database: Sequelize): Promise<void> {
+  await database.transaction(async (transaction) => {
+    await database.query(
+      'CREATE TABLE IF NOT EXISTS "SchemaMigrations" ("id" TEXT PRIMARY KEY, "applied_at" TEXT NOT NULL)',
+      { transaction },
+    );
+    const applied = await database.query<{ id: string }>(
+      'SELECT "id" FROM "SchemaMigrations"',
+      { type: QueryTypes.SELECT, transaction },
+    );
+    const appliedIds = new Set(applied.map(({ id }) => id));
+    for (const { table, column, type } of columns) {
+      const id = `add-${table}-${column}`;
+      const fields = await database.query<{ name: string }>(
+        `PRAGMA table_info("${table}")`,
+        {
+          type: QueryTypes.SELECT,
+          transaction,
+        },
+      );
+      if (fields.length === 0)
+        throw new Error(`Migration table is missing: ${table}`);
+      if (!fields.some((field) => field.name === column)) {
+        // table/column/type come only from the static migration manifest above.
+        await database.query(
+          `ALTER TABLE "${table}" ADD COLUMN "${column}" ${type}`,
+          { transaction },
+        );
+      }
+      if (!appliedIds.has(id)) {
+        await database.query(
+          'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (:id, :appliedAt)',
+          {
+            replacements: { id, appliedAt: new Date().toISOString() },
+            transaction,
+          },
+        );
+      }
+    }
+  });
+}

docker/Dockerfile

@@ -20,7 +20,7 @@ RUN set -x \
   && npm_config_target_platform=linux \
     npm_config_target_arch="${NODE_ARCH}" \
     npm_config_target_libc=musl \
-    pnpm install --prod
+    pnpm install --prod --frozen-lockfile
 
 FROM python:3.11-alpine
 
@@ -73,15 +73,19 @@ RUN set -x \
   && ulimit -c 0
 
 ARG SOURCE_COMMIT
+LABEL org.opencontainers.image.revision=${SOURCE_COMMIT}
 RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} \
   && cd ${QL_DIR} \
+  && if [ -n "${SOURCE_COMMIT}" ]; then git fetch --depth=1 origin "${SOURCE_COMMIT}" && git reset --hard FETCH_HEAD; fi \
   && cp -f .env.example .env \
   && chmod 777 ${QL_DIR}/shell/*.sh \
-  && chmod 777 ${QL_DIR}/docker/*.sh \
-  && git clone --depth=1 -b ${QL_BRANCH} https://github.com/${QL_MAINTAINER}/qinglong-static.git /static \
-  && mkdir -p ${QL_DIR}/static \
-  && cp -rf /static/* ${QL_DIR}/static \
-  && rm -rf /static
+  && chmod 777 ${QL_DIR}/docker/*.sh
+
+# Downloaded by CI from the build-static job in this workflow run.
+COPY static/ /ql/static/
+COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/
+COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml
+RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml
 
 ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \
   PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \

docker/Dockerfile.310

@@ -20,7 +20,7 @@ RUN set -x \
   && npm_config_target_platform=linux \
     npm_config_target_arch="${NODE_ARCH}" \
     npm_config_target_libc=musl \
-    pnpm install --prod
+    pnpm install --prod --frozen-lockfile
 
 FROM python:3.10-alpine
 
@@ -73,15 +73,19 @@ RUN set -x \
   && ulimit -c 0
 
 ARG SOURCE_COMMIT
+LABEL org.opencontainers.image.revision=${SOURCE_COMMIT}
 RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} \
   && cd ${QL_DIR} \
+  && if [ -n "${SOURCE_COMMIT}" ]; then git fetch --depth=1 origin "${SOURCE_COMMIT}" && git reset --hard FETCH_HEAD; fi \
   && cp -f .env.example .env \
   && chmod 777 ${QL_DIR}/shell/*.sh \
-  && chmod 777 ${QL_DIR}/docker/*.sh \
-  && git clone --depth=1 -b ${QL_BRANCH} https://github.com/${QL_MAINTAINER}/qinglong-static.git /static \
-  && mkdir -p ${QL_DIR}/static \
-  && cp -rf /static/* ${QL_DIR}/static \
-  && rm -rf /static
+  && chmod 777 ${QL_DIR}/docker/*.sh
+
+# Downloaded by CI from the build-static job in this workflow run.
+COPY static/ /ql/static/
+COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/
+COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml
+RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml
 
 ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \
   PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \

docker/Dockerfile.debian

@@ -12,7 +12,7 @@ RUN set -x && \
   apt-get install --no-install-recommends -y libatomic1 && \
   npm i -g pnpm@8.3.1 && \
   cd /tmp/build && \
-  pnpm install --prod
+  pnpm install --prod --frozen-lockfile
 
 FROM python:3.11.14-slim-bookworm
 
@@ -82,15 +82,19 @@ RUN mkdir -p ${QL_DIR} && \
 
 USER qinglong
 ARG SOURCE_COMMIT
-RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} && \
-  cd ${QL_DIR} && \
-  cp -f .env.example .env && \
-  chmod 777 ${QL_DIR}/shell/*.sh && \
-  chmod 777 ${QL_DIR}/docker/*.sh && \
-  git clone --depth=1 -b ${QL_BRANCH} https://github.com/${QL_MAINTAINER}/qinglong-static.git /tmp/static && \
-  mkdir -p ${QL_DIR}/static && \
-  cp -rf /tmp/static/* ${QL_DIR}/static && \
-  rm -rf /tmp/static
+LABEL org.opencontainers.image.revision=${SOURCE_COMMIT}
+RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} \
+  && cd ${QL_DIR} \
+  && if [ -n "${SOURCE_COMMIT}" ]; then git fetch --depth=1 origin "${SOURCE_COMMIT}" && git reset --hard FETCH_HEAD; fi \
+  && cp -f .env.example .env \
+  && chmod 777 ${QL_DIR}/shell/*.sh \
+  && chmod 777 ${QL_DIR}/docker/*.sh
+
+# Downloaded by CI from the build-static job in this workflow run.
+COPY --chown=qinglong:qinglong static/ /ql/static/
+COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/
+COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml
+RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml
 
 ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \
   PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \

docker/Dockerfile.debian310

@@ -12,7 +12,7 @@ RUN set -x && \
   apt-get install --no-install-recommends -y libatomic1 && \
   npm i -g pnpm@8.3.1 && \
   cd /tmp/build && \
-  pnpm install --prod
+  pnpm install --prod --frozen-lockfile
 
 FROM python:3.10-slim-bookworm
 
@@ -82,15 +82,19 @@ RUN mkdir -p ${QL_DIR} && \
 USER qinglong
 
 ARG SOURCE_COMMIT
-RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} && \
-  cd ${QL_DIR} && \
-  cp -f .env.example .env && \
-  chmod 777 ${QL_DIR}/shell/*.sh && \
-  chmod 777 ${QL_DIR}/docker/*.sh && \
-  git clone --depth=1 -b ${QL_BRANCH} https://github.com/${QL_MAINTAINER}/qinglong-static.git /tmp/static && \
-  mkdir -p ${QL_DIR}/static && \
-  cp -rf /tmp/static/* ${QL_DIR}/static && \
-  rm -rf /tmp/static
+LABEL org.opencontainers.image.revision=${SOURCE_COMMIT}
+RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} \
+  && cd ${QL_DIR} \
+  && if [ -n "${SOURCE_COMMIT}" ]; then git fetch --depth=1 origin "${SOURCE_COMMIT}" && git reset --hard FETCH_HEAD; fi \
+  && cp -f .env.example .env \
+  && chmod 777 ${QL_DIR}/shell/*.sh \
+  && chmod 777 ${QL_DIR}/docker/*.sh
+
+# Downloaded by CI from the build-static job in this workflow run.
+COPY --chown=qinglong:qinglong static/ /ql/static/
+COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/
+COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml
+RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml
 
 ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \
   PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \

docker/build-manifest.cjs

@@ -0,0 +1,28 @@
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+
+// Include every artifact, including imported backend modules and frontend chunks.
+function collectBuildFiles(root = 'static') {
+  const files = {};
+  function visit(relative) {
+    for (const name of fs.readdirSync(path.join(root, relative)).sort()) {
+      const entry = relative ? `${relative}/${name}` : name;
+      if (entry === 'build-info.json') continue;
+      const file = path.join(root, entry);
+      const stat = fs.lstatSync(file);
+      if (stat.isSymbolicLink())
+        throw new Error(`Build output symlink: ${entry}`);
+      if (stat.isDirectory()) visit(entry);
+      else if (stat.isFile()) {
+        files[entry] = crypto
+          .createHash('sha256')
+          .update(fs.readFileSync(file))
+          .digest('hex');
+      } else throw new Error(`Unsupported build output: ${entry}`);
+    }
+  }
+  visit('');
+  return files;
+}
+module.exports = { collectBuildFiles };

docker/verify-build.cjs

@@ -0,0 +1,49 @@
+const fs = require('node:fs');
+const crypto = require('node:crypto');
+const { execFileSync } = require('node:child_process');
+const { collectBuildFiles } = require('./build-manifest.cjs');
+
+const manifest = JSON.parse(fs.readFileSync('static/build-info.json', 'utf8'));
+const sourceCommit = execFileSync('git', ['rev-parse', 'HEAD'], {
+  encoding: 'utf8',
+}).trim();
+const lockfileSha256 = crypto
+  .createHash('sha256')
+  .update(fs.readFileSync('pnpm-lock.yaml'))
+  .digest('hex');
+if (
+  manifest.dirty !== false ||
+  manifest.sourceCommit !== sourceCommit ||
+  manifest.lockfileSha256 !== lockfileSha256
+) {
+  throw new Error(
+    'Build artifacts do not match the checked-out source and lockfile. Rebuild from the same clean commit.',
+  );
+}
+
+if (process.argv[2]) {
+  const dependencyLock = crypto
+    .createHash('sha256')
+    .update(fs.readFileSync(process.argv[2]))
+    .digest('hex');
+  if (dependencyLock !== manifest.lockfileSha256)
+    throw new Error(
+      'Production dependencies were built from a different lockfile.',
+    );
+}
+
+const actual = collectBuildFiles();
+if (
+  manifest.version !== 1 ||
+  !manifest.files ||
+  JSON.stringify(Object.keys(actual).sort()) !==
+    JSON.stringify(Object.keys(manifest.files).sort())
+)
+  throw new Error(
+    'Build output file set mismatch. Rebuild the complete artifacts.',
+  );
+for (const [file, hash] of Object.entries(actual)) {
+  if (hash !== manifest.files[file])
+    throw new Error(`Build output checksum mismatch: ${file}`);
+}
+console.log(`Verified build artifacts for ${sourceCommit}`);

ecosystem.config.js

@@ -4,6 +4,9 @@ module.exports = {
   apps: [
     {
       name: 'qinglong',
+      // Keep process supervision; enable injected diagnostics only on demand
+      // in containers. Standalone installs retain PM2's monitoring default.
+      pmx: !isContainer || process.env.QL_PRIMARY_APM === 'true',
       max_restarts: 5,
       kill_timeout: 1000,
       wait_ready: true,

package.json

@@ -18,6 +18,9 @@
     "start:front": "max dev",
     "build:front": "max build",
     "build:back": "tsc -p back/tsconfig.json",
+    "test": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only --test test/back/*.test.cjs test/front/*.test.cjs",
+    "benchmark:execution": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only scripts/benchmark-execution.cjs",
+    "build:info": "node scripts/write-build-info.cjs",
     "panel": "npm run build:back && node static/build/app.js",
     "gen:proto": "protoc --experimental_allow_proto3_optional --plugin=./node_modules/.bin/protoc-gen-ts_proto ./back/protos/*.proto --ts_proto_out=./ --ts_proto_opt=outputServices=grpc-js,env=node,esModuleInterop=true,snakeToCamel=false",
     "prettier": "prettier --write '**/*.{js,jsx,tsx,ts,less,md,json}'",

scripts/benchmark-execution.cjs

@@ -0,0 +1,96 @@
+// Microbenchmark of the process observer, not a container or scheduler benchmark.
+const fs = require('node:fs');
+const os = require('node:os');
+const crypto = require('node:crypto');
+const { spawn, execFileSync } = require('node:child_process');
+const { performance } = require('node:perf_hooks');
+const { observeChildProcess } = require('../back/shared/childProcess');
+
+const args = process.argv.slice(2).filter((arg) => arg !== '--');
+const runsArg = args.find((arg) => arg.startsWith('--runs='));
+const runs = runsArg ? Number(runsArg.slice(7)) : 30;
+if (!Number.isInteger(runs) || runs < 3 || runs > 10000)
+  throw new Error('runs must be between 3 and 10000');
+const output = args.find((arg) => arg.startsWith('--output='))?.slice(9);
+const git = (...args) => execFileSync('git', args, { encoding: 'utf8' }).trim();
+const scenarios = [
+  { name: 'empty-node', command: '' },
+  {
+    name: 'log-1MiB',
+    command: 'process.stdout.write("x".repeat(1024 * 1024))',
+    bytes: 1024 * 1024,
+  },
+  { name: 'spawn-failure', shell: '/nonexistent-ql-benchmark-shell' },
+];
+
+(async () => {
+  const report = {
+    schema: 1,
+    observerSha256: crypto
+      .createHash('sha256')
+      .update(fs.readFileSync('back/shared/childProcess.ts'))
+      .digest('hex'),
+    benchmarkSha256: crypto
+      .createHash('sha256')
+      .update(fs.readFileSync(__filename))
+      .digest('hex'),
+    scope:
+      'local process observer only; parent CPU excludes child CPU; no panel/container load measured',
+    sourceCommit: git('rev-parse', 'HEAD'),
+    dirty: git('status', '--porcelain') !== '',
+    lockfileSha256: crypto
+      .createHash('sha256')
+      .update(fs.readFileSync('pnpm-lock.yaml'))
+      .digest('hex'),
+    node: process.version,
+    platform: process.platform,
+    arch: process.arch,
+    cpu: os.cpus()[0]?.model,
+    cores: os.cpus().length,
+    timestamp: new Date().toISOString(),
+    runs,
+    scenarios: [],
+  };
+  for (const scenario of scenarios) {
+    const samples = [];
+    const cpuStart = process.cpuUsage();
+    for (let index = 0; index < runs; index++) {
+      const start = performance.now();
+      let bytes = 0;
+      const child = scenario.shell
+        ? spawn('true', { shell: scenario.shell })
+        : spawn(process.execPath, ['-e', scenario.command]);
+      const result = await observeChildProcess(child, {
+        onStdout: async (chunk) => {
+          bytes += Buffer.byteLength(chunk);
+        },
+      }).completed;
+      if (scenario.shell ? !result.error : result.error || result.code !== 0)
+        throw new Error(`Unexpected result: ${scenario.name}`);
+      if (bytes !== (scenario.bytes || 0))
+        throw new Error(`Output loss: ${scenario.name}`);
+      samples.push({
+        wallMs: performance.now() - start,
+        bytes,
+        code: result.code,
+        error: result.error?.message,
+      });
+    }
+    const sorted = samples.map((sample) => sample.wallMs).sort((a, b) => a - b);
+    const percentile = (p) => sorted[Math.ceil(sorted.length * p) - 1];
+    report.scenarios.push({
+      name: scenario.name,
+      parentCpuMicros: process.cpuUsage(cpuStart),
+      p50Ms: percentile(0.5),
+      p95Ms: percentile(0.95),
+      ...(runs >= 100 ? { p99Ms: percentile(0.99) } : {}),
+      samples,
+    });
+  }
+  const json = JSON.stringify(report, null, 2) + '\n';
+  if (output) fs.writeFileSync(output, json);
+  else process.stdout.write(json);
+})().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

scripts/write-build-info.cjs

@@ -0,0 +1,27 @@
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const { execFileSync } = require('node:child_process');
+const { collectBuildFiles } = require('../docker/build-manifest.cjs');
+
+const root = process.cwd();
+for (const output of ['static/build/app.js', 'static/dist/index.html']) {
+  if (!fs.existsSync(path.join(root, output)))
+    throw new Error(`Missing build output: ${output}`);
+}
+const git = (...args) =>
+  execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim();
+const manifest = {
+  version: 1,
+  files: collectBuildFiles(),
+  sourceCommit: git('rev-parse', 'HEAD'),
+  dirty: git('status', '--porcelain', '--untracked-files=all') !== '',
+  lockfileSha256: crypto
+    .createHash('sha256')
+    .update(fs.readFileSync('pnpm-lock.yaml'))
+    .digest('hex'),
+};
+fs.writeFileSync(
+  'static/build-info.json',
+  JSON.stringify(manifest, null, 2) + '\n',
+);

shell/api.sh

@@ -10,16 +10,39 @@ create_token() {
 }
 
 get_token() {
-  if [[ -f $file_auth_token ]]; then
-    __ql_token__=$(cat $file_auth_token | jq -r .value)
-    local expiration=$(cat $file_auth_token | jq -r .expiration)
-    local currentTimeStamp=$(date +%s)
-    if [[ $currentTimeStamp -ge $expiration ]]; then
-      create_token
+  local token_data expiration token_value
+  local currentTimeStamp=${EPOCHSECONDS:-$(date +%s)}
+  if [[ -f "$file_auth_token" ]] && token_data=$(jq -er \
+    'select((.expiration | type) == "number" and (.value | type) == "string" and (.value | length) > 0 and (.value | test("[\r\n]") | not)) | .expiration, .value' \
+    "$file_auth_token" 2>/dev/null); then
+    expiration=${token_data%%$'\n'*}
+    token_value=${token_data#*$'\n'}
+    if [[ "$expiration" =~ ^[1-9][0-9]{0,10}$ && -n "$token_value" && "$token_value" != *$'\n'* && "$token_value" != *$'\r'* ]] && \
+      (( currentTimeStamp < expiration )); then
+      __ql_token__=$token_value
+      return 0
     fi
-  else
-    create_token
   fi
+  create_token
+}
+
+# Read the status response once; preserve multiline error messages and the
+# legacy code/message variables used by the shell API callers.
+ql_parse_status_response() {
+  # Both task status and statistics endpoints normally return this exact body.
+  # Match the whole document; all other JSON and malformed responses still
+  # use jq. Keep its missing-message result for existing callers.
+  if [[ "$1" == '{"code":200}' ]]; then
+    code=200
+    message=null
+    return 0
+  fi
+  local parsed
+  code=""
+  message=""
+  parsed=$(jq -r '.code, .message' <<< "$1") || return $?
+  code=${parsed%%$'\n'*}
+  message=${parsed#*$'\n'}
 }
 
 add_cron_api() {
@@ -142,7 +165,7 @@ update_cron() {
   local lastExecutingTime="${5:-0}"
   local runningTime="${6:-0}"
   local exitCode="${7:-}"
-  local currentTimeStamp=$(date +%s)
+  local currentTimeStamp=${EPOCHSECONDS:-$(date +%s)}
   local dataRaw="{\"ids\":[$ids],\"status\":\"$status\",\"pid\":\"$pid\",\"log_path\":\"$logPath\",\"last_execution_time\":$lastExecutingTime,\"last_running_time\":$runningTime"
   if [[ -n $exitCode ]]; then
     dataRaw="${dataRaw},\"exit_code\":$exitCode"
@@ -156,8 +179,7 @@ update_cron() {
       --data-raw "$dataRaw" \
       --compressed
   )
-  code=$(echo "$api" | jq -r .code)
-  message=$(echo "$api" | jq -r .message)
+  ql_parse_status_response "$api" || true
   if [[ $code != 200 ]]; then
     if [[ ! $message ]]; then
       message="$api"
@@ -240,8 +262,7 @@ record_cron_stat() {
     --data-raw "{\"ref_id\":$ref_id,\"code\":$exit_code,\"elapsed\":$elapsed}" \
     --compressed
   )
-  code=$(echo "$api" | jq -r .code)
-  message=$(echo "$api" | jq -r .message)
+  ql_parse_status_response "$api" || true
   if [[ $code != 200 ]]; then
     if [[ ! $message ]]; then
       message="$api"

shell/node_path_cache.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+
+# Cache discovery only, never the installed packages. Ordinary installs at the
+# same root are visible immediately. TTL bounds changes outside known inputs.
+ql_node_path_cache_key() {
+  local pnpm_bin node_bin name directory file
+  pnpm_bin=$(type -P pnpm) || return 1
+  node_bin=$(type -P node) || return 1
+  {
+    printf '%s\n' 'v1' "$EUID" "$PWD" "$PATH" "${HOME:-}" \
+      "${PNPM_HOME:-}" "${XDG_CONFIG_HOME:-}" "${XDG_DATA_HOME:-}" \
+      "$pnpm_bin" "$node_bin"
+    for name in ${!npm_config_@} ${!NPM_CONFIG_@}; do
+      printf '%s=%s\n' "$name" "${!name}"
+    done
+    # Follow executable symlinks, detecting upgrades at the same command path.
+    stat -Lc '%n:%i:%s:%Y:%Z' "$pnpm_bin" "$node_bin" 2>/dev/null || \
+      stat -Lf '%N:%i:%z:%m:%c' "$pnpm_bin" "$node_bin" 2>/dev/null
+    directory=$PWD
+    while :; do
+      for file in "$directory/.npmrc" "$directory/pnpm-workspace.yaml"; do
+        if [[ -f "$file" ]]; then
+          printf '%s\n' "$file"
+          cat -- "$file"
+          printf '\n'
+        fi
+      done
+      [[ "$directory" == / ]] && break
+      directory=${directory%/*}
+      [[ -n "$directory" ]] || directory=/
+    done
+    for file in "${HOME:-}/.npmrc" \
+      "${XDG_CONFIG_HOME:-${HOME:-}/.config}/pnpm/rc" \
+      "${npm_config_userconfig:-${NPM_CONFIG_USERCONFIG:-/dev/null}}" \
+      "${npm_config_globalconfig:-${NPM_CONFIG_GLOBALCONFIG:-/dev/null}}"; do
+      if [[ -f "$file" ]]; then
+        printf '%s\n' "$file"
+        cat -- "$file"
+        printf '\n'
+      fi
+    done
+  } | cksum
+}
+
+ql_read_node_path_cache() {
+  local cache="$1" key="$2" now
+  local stored_key="" stored_at="" stored_path=""
+  now=${EPOCHSECONDS:-$(date +%s)}
+  if [[ -f "$cache" && ! -L "$cache" && -O "$cache" ]]; then
+    if ! {
+      IFS= read -r stored_key && IFS= read -r stored_at && IFS= read -r stored_path
+    } < "$cache"; then
+      return 1
+    fi
+    if [[ "$stored_key" == "$key" && "$stored_at" =~ ^(0|[1-9][0-9]{0,10})$ && "$stored_path" == /* ]] && \
+      (( now >= stored_at && now - stored_at < 60 )); then
+      printf '%s\n' "$stored_path"
+      return 0
+    fi
+  fi
+  return 1
+}
+
+# Keep the lock descriptor in a subshell so sourcing this helper never changes
+# the caller's descriptors. A crashed refresher releases the kernel lock.
+ql_refresh_node_global_path() (
+  local cache="$1" key="$2" now result previous_umask
+  local lock="${cache}.lock"
+  previous_umask=$(umask)
+  umask 077
+  if type -P flock &>/dev/null && mkdir -p -- "$dir_tmp" 2>/dev/null; then
+    if [[ ! -L "$lock" && ( ! -e "$lock" || ( -f "$lock" && -O "$lock" ) ) ]] && \
+      { exec 9>> "$lock"; } 2>/dev/null; then
+      # Bound the wait; absent/unsupported flock or contention falls back to
+      # independent discovery. Never remove the lock file while waiters exist.
+      if flock -w 2 9 2>/dev/null; then
+        if ql_read_node_path_cache "$cache" "$key"; then
+          return 0
+        fi
+      fi
+    fi
+  fi
+
+  # Private lock creation must not change pnpm's inherited creation mask.
+  umask "$previous_umask"
+  now=${EPOCHSECONDS:-$(date +%s)}
+  result=$(pnpm root -g 9>&- 2>/dev/null) || return $?
+  # Never cache failed, empty, multiline or non-absolute answers.
+  if [[ "$result" == /* && "$result" != *$'\n'* && "$result" != *$'\r'* ]]; then
+    (
+      umask 077
+      mkdir -p -- "$dir_tmp" || exit 0
+      local temporary
+      temporary=$(mktemp "${cache}.XXXXXX") || exit 0
+      if printf '%s\n%s\n%s\n' "$key" "$now" "$result" > "$temporary"; then
+        mv -f -- "$temporary" "$cache" || rm -f -- "$temporary"
+      else
+        rm -f -- "$temporary"
+      fi
+    ) 2>/dev/null
+  fi
+  printf '%s\n' "$result"
+)
+
+ql_get_node_global_path() {
+  if [[ "${QL_NODE_PATH_CACHE:-1}" == 0 ]]; then
+    pnpm root -g 2>/dev/null
+    return $?
+  fi
+
+  local key cache
+  cache="${dir_tmp}/pnpm-root-${EUID}.cache"
+  key=$(ql_node_path_cache_key) || { pnpm root -g 2>/dev/null; return $?; }
+  if ql_read_node_path_cache "$cache" "$key"; then
+    return 0
+  fi
+  ql_refresh_node_global_path "$cache" "$key"
+}

shell/otask.sh

@@ -95,7 +95,13 @@ append_node_dependency_path() {
   # 用户依赖目录加入 NODE_PATH，替代 symlink 到 node_modules 的方式
   export NODE_PATH="${NODE_PATH:+${NODE_PATH}:}${dir_dep}"
 
-  local pnpm_global_path=$(pnpm root -g 2>/dev/null)
+  local pnpm_global_path
+  if [[ -f "$dir_shell/node_path_cache.sh" ]]; then
+    . "$dir_shell/node_path_cache.sh"
+    pnpm_global_path=$(ql_get_node_global_path) || true
+  else
+    pnpm_global_path=$(pnpm root -g 2>/dev/null) || true
+  fi
   if [[ -n "$pnpm_global_path" ]]; then
     export QL_NODE_GLOBAL_PATH="$pnpm_global_path"
     export NODE_PATH="${NODE_PATH:+${NODE_PATH}:}${pnpm_global_path}"

shell/share.sh

@@ -368,9 +368,16 @@ format_timestamp() {
 
 get_env_array() {
   exported_variables=()
-  while IFS= read -r line; do
-    exported_variables+=("$line")
-  done < <(grep '^export ' $file_env | awk '{print $2}' | cut -d= -f1)
+  # Preserve the legacy export-line/second-field rules without evaluating values.
+  local export_name_program='/^export / { name = $2; sub(/=.*/, "", name); print name }'
+  if [[ ${BASH_VERSINFO[0]} -ge 4 ]]; then
+    builtin mapfile -t exported_variables < <(awk "$export_name_program" "$file_env")
+  else
+    # macOS still ships Bash 3, which does not provide mapfile.
+    while IFS= read -r line; do
+      exported_variables+=("$line")
+    done < <(awk "$export_name_program" "$file_env")
+  fi
 }
 
 clear_env() {
@@ -411,9 +418,14 @@ run_task_after() {
 }
 
 handle_task_end() {
-  local etime=$(date "+$time_format")
-  local end_time=$(format_time "$time_format" "$etime")
-  local end_timestamp=$(format_timestamp "$time_format" "$etime")
+  local etime end_time end_timestamp
+  if [[ $is_macos -ne 1 && $time_format == '%Y-%m-%d %H:%M:%S' ]]; then
+    IFS='|' read -r end_time end_timestamp < <(date "+$time_format|%s")
+  else
+    etime=$(date "+$time_format")
+    end_time=$(format_time "$time_format" "$etime")
+    end_timestamp=$(format_timestamp "$time_format" "$etime")
+  fi
   local diff_time=$(($end_timestamp - $begin_timestamp))
   local exit_code="${_task_exit_code:-0}"
   [[ "$diff_time" == 0 ]] && diff_time=1

shell/task.sh

@@ -52,8 +52,14 @@ handle_log_path() {
     fi
   fi
 
-  time=$(date "+$mtime_format")
-  log_time=$(format_log_time "$mtime_format" "$time")
+  if [[ $is_macos -ne 1 && $mtime_format == '%Y-%m-%d %H:%M:%S.%3N' ]]; then
+    # Render both representations from one clock snapshot, without parsing it
+    # again in a second date process.
+    IFS='|' read -r time log_time < <(date "+$mtime_format|%Y-%m-%d-%H-%M-%S-%3N")
+  else
+    time=$(date "+$mtime_format")
+    log_time=$(format_log_time "$mtime_format" "$time")
+  fi
   if [[ -z $log_name ]]; then
     log_dir_tmp="${file_param##*/}"
     if [[ $file_param =~ "/" ]]; then
@@ -124,7 +130,11 @@ format_params() {
 }
 
 init_begin_time() {
-  begin_time=$(format_time "$time_format" "$time")
+  if [[ $is_macos -ne 1 && $mtime_format == '%Y-%m-%d %H:%M:%S.%3N' && $time_format == '%Y-%m-%d %H:%M:%S' ]]; then
+    begin_time=${time%.*}
+  else
+    begin_time=$(format_time "$time_format" "$time")
+  fi
   begin_timestamp=$(format_timestamp "$time_format" "$time")
 }
 

test/back/build-provenance.test.cjs

@@ -0,0 +1,150 @@
+const assert = require('node:assert/strict');
+const test = require('node:test');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { execFileSync, spawnSync } = require('node:child_process');
+
+test('build verification accepts matching source and rejects stale or dirty artifacts', (t) => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-build-source-'));
+  t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
+  const git = (...args) =>
+    execFileSync('git', args, { cwd: dir, stdio: 'pipe' });
+  git('init');
+  git('config', 'user.name', 'Build test');
+  git('config', 'user.email', 'build-test@example.invalid');
+  fs.writeFileSync(path.join(dir, 'pnpm-lock.yaml'), 'lockfileVersion: 6.0\n');
+  fs.writeFileSync(path.join(dir, '.gitignore'), 'static/\n');
+  git('add', '.');
+  git('commit', '-m', 'fixture');
+  fs.mkdirSync(path.join(dir, 'static/build'), { recursive: true });
+  fs.mkdirSync(path.join(dir, 'static/dist'), { recursive: true });
+  fs.writeFileSync(path.join(dir, 'static/build/app.js'), '');
+  fs.writeFileSync(path.join(dir, 'static/dist/index.html'), '');
+  const write = path.resolve('scripts/write-build-info.cjs');
+  const verify = path.resolve('docker/verify-build.cjs');
+  execFileSync(process.execPath, [write], { cwd: dir });
+  execFileSync(process.execPath, [verify], { cwd: dir });
+  const manifestPath = path.join(dir, 'static/build-info.json');
+  const manifest = JSON.parse(fs.readFileSync(manifestPath));
+  for (const invalid of [
+    { ...manifest, sourceCommit: '0'.repeat(40) },
+    { ...manifest, dirty: true },
+    { ...manifest, lockfileSha256: '0'.repeat(64) },
+  ]) {
+    fs.writeFileSync(manifestPath, JSON.stringify(invalid));
+    assert.notEqual(
+      spawnSync(process.execPath, [verify], { cwd: dir }).status,
+      0,
+    );
+  }
+});
+
+test('release images receive the same-run artifact and verify it before use', () => {
+  const yaml = require('js-yaml');
+  const workflow = yaml.load(
+    fs.readFileSync('.github/workflows/build-docker-image.yml', 'utf8'),
+  );
+  assert.equal(workflow.jobs['build-static'].needs, 'validate');
+  const upload = workflow.jobs['build-static'].steps.find(step => step.uses?.startsWith('actions/upload-artifact@'));
+  assert.equal(upload.with['include-hidden-files'], true);
+  for (const name of [
+    'build-alpine',
+    'build-debian',
+    'build-alpine310',
+    'build-debian310',
+  ]) {
+    const job = workflow.jobs[name];
+    assert.equal(job.needs, 'build-static');
+    const download = job.steps.find((step) =>
+      step.uses?.startsWith('actions/download-artifact@'),
+    );
+    assert.equal(download.with.name, 'qinglong-static-${{ github.sha }}');
+    assert.equal(download.with.path, 'static/');
+    const build = job.steps.find((step) =>
+      step.uses?.startsWith('docker/build-push-action@'),
+    );
+    assert.match(
+      build.with['build-args'],
+      /SOURCE_COMMIT=\$\{\{ github.sha \}\}/,
+    );
+    const dockerfile = fs.readFileSync(build.with.file, 'utf8');
+    assert.doesNotMatch(dockerfile, /git clone.*qinglong-static/);
+    assert.match(
+      dockerfile,
+      /git fetch --depth=1 origin "\$\{SOURCE_COMMIT\}"/,
+    );
+    assert.match(dockerfile, /node \/tmp\/verify-build.cjs/);
+  }
+});
+
+test('complete artifact manifests reject changed, missing, extra files and untracked build inputs', (t) => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-build-manifest-'));
+  t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
+  const git = (...args) =>
+    execFileSync('git', args, { cwd: dir, stdio: 'pipe' });
+  git('init');
+  git('config', 'user.name', 'Build test');
+  git('config', 'user.email', 'build-test@example.invalid');
+  fs.writeFileSync(path.join(dir, '.gitignore'), 'static/\n');
+  fs.writeFileSync(path.join(dir, 'pnpm-lock.yaml'), 'lockfileVersion: 6.0\n');
+  git('add', '.');
+  git('commit', '-m', 'fixture');
+  const artifacts = [
+    'build/app.js',
+    'build/services/http.js',
+    'dist/index.html',
+    'dist/chunks/main.js',
+    'dist/assets/main.css',
+    'dist/.well-known/config',
+  ];
+  for (const file of artifacts) {
+    fs.mkdirSync(path.dirname(path.join(dir, 'static', file)), {
+      recursive: true,
+    });
+    fs.writeFileSync(path.join(dir, 'static', file), file);
+  }
+  const write = path.resolve('scripts/write-build-info.cjs');
+  const verify = path.resolve('docker/verify-build.cjs');
+  const run = (script) =>
+    spawnSync(process.execPath, [script], { cwd: dir, encoding: 'utf8' });
+  assert.equal(run(write).status, 0);
+  const manifestFile = path.join(dir, 'static/build-info.json');
+  const original = fs.readFileSync(manifestFile, 'utf8');
+  assert.deepEqual(
+    Object.keys(JSON.parse(original).files).sort(),
+    artifacts.sort(),
+  );
+  assert.equal(run(write).status, 0);
+  assert.equal(fs.readFileSync(manifestFile, 'utf8'), original);
+  assert.equal(run(verify).status, 0);
+  for (const name of [
+    'build/services/http.js',
+    'dist/chunks/main.js',
+    'dist/assets/main.css',
+    'dist/.well-known/config',
+  ]) {
+    const file = path.join(dir, 'static', name);
+    fs.writeFileSync(file, 'stale');
+    assert.notEqual(run(verify).status, 0);
+    fs.unlinkSync(file);
+    assert.notEqual(run(verify).status, 0);
+    fs.writeFileSync(file, name);
+    assert.equal(run(verify).status, 0);
+  }
+  const extra = path.join(dir, 'static/build/stale.js');
+  fs.writeFileSync(extra, 'extra');
+  assert.notEqual(run(verify).status, 0);
+  fs.unlinkSync(extra);
+  const source = path.join(dir, 'custom.config.js');
+  fs.writeFileSync(source, 'untracked build input');
+  assert.equal(run(write).status, 0);
+  assert.equal(JSON.parse(fs.readFileSync(manifestFile)).dirty, true);
+  assert.notEqual(run(verify).status, 0);
+  fs.unlinkSync(source);
+  assert.equal(run(write).status, 0);
+  assert.equal(run(verify).status, 0);
+  fs.symlinkSync(path.join(dir, 'pnpm-lock.yaml'), extra);
+  assert.notEqual(run(write).status, 0);
+  assert.notEqual(run(verify).status, 0);
+});

test/back/env-name-parsing.test.cjs

@@ -0,0 +1,96 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const source = fs.readFileSync('shell/share.sh', 'utf8');
+function extract(name) {
+  const start = source.indexOf(name + '() {');
+  assert.ok(start >= 0);
+  return source.slice(start, source.indexOf('\n}', start) + 2);
+}
+function fixture(t, content) {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-env-names-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  const file = path.join(root, 'env file.sh');
+  fs.writeFileSync(file, content);
+  return {
+    root,
+    file,
+    run(script) {
+      const r = spawnSync(
+        '/bin/bash',
+        [
+          '-ec',
+          extract('get_env_array') +
+            '\n' +
+            extract('clear_env') +
+            '\n' +
+            script,
+        ],
+        {
+          encoding: 'utf8',
+          env: { ...process.env, file_env: file, FIXTURE_ROOT: root },
+        },
+      );
+      assert.equal(r.status, 0, r.stderr);
+      return r.stdout;
+    },
+  };
+}
+test('matches legacy pipeline for whitespace, malformed lines, CRLF and final unterminated line', (t) => {
+  const f = fixture(
+    t,
+    '# comment\nexport A=one\nexport B="two words"\n export IGNORED=1\nexport\tIGNORED2=1\nexport   C = spaced\nexport \nexport -x X=1\nexport D=a=b\nexport E=crlf\r\nexport F\r\nexport G=last',
+  );
+  assert.equal(
+    f.run('get_env_array; printf "%s\\0" "${exported_variables[@]}"'),
+    f.run(
+      'while IFS= read -r line; do printf "%s\\0" "$line"; done < <(grep "^export " "$file_env" | awk \'{print $2}\' | cut -d= -f1)',
+    ),
+  );
+});
+test('resets prior names for an empty file', (t) => {
+  const f = fixture(t, '');
+  assert.equal(
+    f.run(
+      'exported_variables=(OLD); get_env_array; echo "${#exported_variables[@]}"',
+    ),
+    '0\n',
+  );
+});
+test('extracting names never evaluates command substitutions or shell syntax in values', (t) => {
+  const f = fixture(
+    t,
+    'export A=$(touch "$FIXTURE_ROOT/unsafe")\nexport B=`touch "$FIXTURE_ROOT/unsafe2"`\n',
+  );
+  assert.equal(
+    f.run('get_env_array; printf "%s\\n" "${exported_variables[@]}"'),
+    'A\nB\n',
+  );
+  assert.equal(fs.existsSync(path.join(f.root, 'unsafe')), false);
+  assert.equal(fs.existsSync(path.join(f.root, 'unsafe2')), false);
+});
+test('clear_env removes listed exports and preserves unrelated values', (t) => {
+  const f = fixture(t, 'export USER_ONE=1\nexport USER_TWO=2\n');
+  f.run(
+    'USER_ONE=old; USER_TWO=old; UNRELATED=kept; get_env_array; clear_env; [[ ! ${USER_ONE+x} && ! ${USER_TWO+x} && "$UNRELATED" == kept ]]',
+  );
+});
+test('large configuration preserves order and duplicate names', (t) => {
+  const text = Array.from(
+    { length: 5000 },
+    (_, i) => 'export V' + (i % 1000) + '=value with spaces',
+  ).join('\n');
+  const f = fixture(t, text);
+  const lines = f
+    .run('get_env_array; printf "%s\\n" "${exported_variables[@]}"')
+    .trim()
+    .split('\n');
+  assert.equal(lines.length, 5000);
+  assert.deepEqual(
+    lines,
+    Array.from({ length: 5000 }, (_, i) => 'V' + (i % 1000)),
+  );
+});

test/back/execution-lifecycle.test.cjs

@@ -0,0 +1,236 @@
+const assert = require('node:assert/strict');
+const test = require('node:test');
+const { spawn } = require('node:child_process');
+const { EventEmitter } = require('node:events');
+const { PassThrough } = require('node:stream');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const load = require('../helpers/load-security-module.cjs');
+const { observeChildProcess } = require('../../back/shared/childProcess');
+const { LogStreamManager } = require('../../back/shared/logStreamManager');
+
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const logger = { info() {}, error() {} };
+
+test(
+  'spawn failure settles without exit, and the next scheduled run can proceed',
+  { timeout: 3000 },
+  async () => {
+    let releases = 0;
+    let active = 0;
+    const { runCron } = load(path.resolve('back/shared/runCron.ts'), {
+      'cross-spawn': {
+        spawn: () => spawn('true', { shell: '/nonexistent-ql-test-shell' }),
+      },
+      './pLimit': {
+        runWithCronLimit: async (_cron, fn) => {
+          active++;
+          try {
+            return await fn();
+          } finally {
+            active--;
+          }
+        },
+        removeQueuedCron: () => releases++,
+      },
+      '../loaders/logger': logger,
+      '../data/cron': {
+        CrontabModel: { findOne: async () => null },
+        CrontabStatus: {},
+      },
+      '../data/runningInstance': {
+        RunningInstanceModel: {},
+        InstanceStatus: {},
+      },
+      '../config/util': { killTask: async () => {} },
+    });
+    await runCron('true', { id: '1' });
+    await runCron('true', { id: '2' });
+    assert.equal(active, 0);
+    assert.equal(releases, 2);
+  },
+);
+
+test(
+  'completion waits for slow log consumers and preserves the final output',
+  { timeout: 5000 },
+  async () => {
+    const chunks = [];
+    const child = spawn(process.execPath, [
+      '-e',
+      'process.stdout.write("尾行\\n"); process.stderr.write("错误\\n")',
+    ]);
+    const observed = observeChildProcess(child, {
+      onStart: () => delay(25),
+      onStdout: async (message) => {
+        await delay(40);
+        chunks.push(message);
+      },
+      onStderr: async (message) => {
+        await delay(30);
+        chunks.push(message);
+      },
+    });
+    const result = await observed.completed;
+    assert.equal(result.code, 0);
+    assert.equal(result.error, undefined);
+    assert.match(chunks.join(''), /尾行/);
+    assert.match(chunks.join(''), /错误/);
+  },
+);
+
+test('exit is not completion, and UTF-8 split across writes remains intact', async () => {
+  const child = new EventEmitter();
+  child.stdout = new PassThrough();
+  child.stderr = new PassThrough();
+  child.pid = 123;
+  const chunks = [];
+  const observed = observeChildProcess(child, {
+    onStdout: async (data) => chunks.push(data),
+  });
+  child.emit('spawn');
+  let done = false;
+  observed.completed.then(() => {
+    done = true;
+  });
+  child.emit('exit', 0, null);
+  const data = Buffer.from('末尾中文');
+  child.stdout.write(data.subarray(0, 2));
+  await delay(10);
+  assert.equal(done, false);
+  child.stdout.end(data.subarray(2));
+  child.stderr.end();
+  child.emit('close', 0, null);
+  await observed.completed;
+  assert.equal(chunks.join(''), '末尾中文');
+});
+
+test(
+  'failed log sink drains large output instead of blocking the child',
+  { timeout: 5000 },
+  async () => {
+    const child = spawn(process.execPath, [
+      '-e',
+      'process.stdout.write("x".repeat(2 * 1024 * 1024))',
+    ]);
+    const { completed } = observeChildProcess(child, {
+      onStdout: async () => {
+        throw new Error('ENOSPC');
+      },
+    });
+    const result = await completed;
+    assert.equal(result.code, 0);
+    assert.equal(result.error.message, 'ENOSPC');
+  },
+);
+
+function scheduleFixture() {
+  let active = 0;
+  const limit = async (_params, fn) => {
+    active++;
+    try {
+      return await fn();
+    } finally {
+      active--;
+    }
+  };
+  const Schedule = load(path.resolve('back/services/schedule.ts'), {
+    '../shared/pLimit': {
+      runWithScriptLimit: limit,
+      runWithSystemLimit: limit,
+    },
+    'cross-spawn': {
+      spawn: () =>
+        spawn(process.execPath, [
+          '-e',
+          'setTimeout(() => process.stdout.write("done"), 150)',
+        ]),
+    },
+  }).default;
+  return { service: new Schedule(logger), active: () => active };
+}
+
+test(
+  'PID response keeps queue capacity occupied until cleanup completes',
+  { timeout: 4000 },
+  async () => {
+    const fixture = scheduleFixture();
+    let finish;
+    const finished = new Promise((resolve) => {
+      finish = resolve;
+    });
+    const pid = await fixture.service.runTask(
+      'ignored',
+      {
+        onEnd: async () => {
+          await delay(20);
+          finish();
+        },
+      },
+      { id: 'script', runOrigin: 'script' },
+      'start',
+    );
+    assert.ok(pid > 0);
+    assert.equal(fixture.active(), 1);
+    await finished;
+    await delay(0);
+    assert.equal(fixture.active(), 0);
+  },
+);
+
+test(
+  'before/error/end callback failures settle and release capacity',
+  { timeout: 3000 },
+  async () => {
+    const fixture = scheduleFixture();
+    let ended = 0;
+    const result = await fixture.service.runTask(
+      'ignored',
+      {
+        onBefore: async () => {
+          throw new Error('setup failed');
+        },
+        onError: async () => {
+          throw new Error('sink failed');
+        },
+        onEnd: async (child) => {
+          assert.equal(child, undefined);
+          ended++;
+          throw new Error('cleanup failed');
+        },
+      },
+      { id: 'system', runOrigin: 'system' },
+    );
+    assert.equal(result.error.message, 'setup failed');
+    assert.equal(ended, 1);
+    assert.equal(fixture.active(), 0);
+  },
+);
+
+test(
+  'concurrent writes and closes preserve all bytes, and failed files can be closed',
+  { timeout: 5000 },
+  async (t) => {
+    const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-lifecycle-'));
+    t.after(() => fs.rm(dir, { recursive: true, force: true }));
+    const manager = new LogStreamManager(dir);
+    const file = path.join(dir, 'out.log');
+    const lines = Array.from({ length: 500 }, (_, i) => `${i}:末尾\n`);
+    const writes = lines.map((line) => manager.write(file, line));
+    await Promise.all([
+      ...writes,
+      manager.closeStream(file),
+      manager.closeStream(file),
+    ]);
+    assert.equal(await fs.readFile(file, 'utf8'), lines.join(''));
+    assert.equal(manager.getOpenStreamCount(), 0);
+    const invalid = path.join(dir, 'missing', 'out.log');
+    await assert.rejects(manager.write(invalid, 'fail'), /ENOENT/);
+    await assert.rejects(manager.closeStream(invalid), /ENOENT/);
+    assert.equal(manager.getOpenStreamCount(), 0);
+    await manager.write(file, 'reopened');
+    await manager.closeAll();
+    assert.match(await fs.readFile(file, 'utf8'), /reopened$/);
+  },
+);

test/back/http-exclusive-listen.test.cjs

@@ -0,0 +1,31 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const express = require('express');
+const load = require('../helpers/load-security-module.cjs');
+const Http = load('back/services/http.ts', {
+  '../config': {bindHost: '127.0.0.1'},
+  '../loaders/logger': {debug(){},warn(){},error(){}},
+  './metrics': {metricsService: {record(){}}},
+  typedi: {Service:()=>x=>x},
+}).HttpServerService;
+test('HTTP listener preserves private binding and releases its port after shutdown',async(t)=>{
+ const app=express();app.get('/',(_q,r)=>r.send('ok'));
+ const first=new Http();const server=await first.initialize(app,0);
+ t.after(()=>first.shutdown());
+ assert.equal(server.address().address,'127.0.0.1');
+ const port=server.address().port;
+ const response=await fetch(`http://127.0.0.1:${port}/`,{headers:{connection:'close'}});assert.equal(await response.text(),'ok');
+ await assert.rejects(new Http().initialize(app,port),e=>e.code==='EADDRINUSE');
+ await first.shutdown();const replacement=new Http();t.after(()=>replacement.shutdown());
+ const next=await replacement.initialize(app,port);assert.equal(next.address().port,port);
+});
+test('custom cluster deployments can restore shared listening',async(t)=>{
+ const previous=process.env.QL_HTTP_SHARED_LISTEN;
+ t.after(()=>{if(previous===undefined)delete process.env.QL_HTTP_SHARED_LISTEN;else process.env.QL_HTTP_SHARED_LISTEN=previous;});
+ for(const value of [undefined,'true']){
+  if(value===undefined)delete process.env.QL_HTTP_SHARED_LISTEN;else process.env.QL_HTTP_SHARED_LISTEN=value;
+  let options;const service=new Http();const app=express();const listen=app.listen.bind(app);
+  app.listen=(opts,cb)=>{options=opts;return listen(opts,cb);};
+  await service.initialize(app,0);assert.equal(options.exclusive,value!=='true');await service.shutdown();
+ }
+});

test/back/log-path-security.test.cjs

@@ -0,0 +1,158 @@
+const assert = require('node:assert/strict');
+const test = require('node:test');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const load = require('../helpers/load-security-module.cjs');
+const { LogStreamManager } = require('../../back/shared/logStreamManager');
+
+async function fixture(t) {
+  const base = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-boundary-'));
+  t.after(() => fs.rm(base, { recursive: true, force: true }));
+  const root = path.join(base, 'log');
+  const outside = path.join(base, 'log-other');
+  await fs.mkdir(root);
+  await fs.mkdir(outside);
+  const victim = path.join(outside, 'victim.log');
+  await fs.writeFile(victim, 'unchanged');
+  await fs.symlink(outside, path.join(root, 'escape-dir'));
+  await fs.symlink(victim, path.join(root, 'escape-file'));
+  await fs.symlink(
+    path.join(outside, 'missing.log'),
+    path.join(root, 'dangling'),
+  );
+  const invalid = [
+    path.join(root, '..', 'log-other', 'new.log'),
+    path.join(root, 'escape-dir', 'new.log'),
+    path.join(root, 'escape-file'),
+    path.join(root, 'dangling'),
+    root,
+    path.join(root, 'bad\0name'),
+  ];
+  return { root, outside, victim, invalid };
+}
+
+test('log streams reject traversal, sibling prefixes and escaping symlinks before writing', async (t) => {
+  const { root, outside, victim, invalid } = await fixture(t);
+  const manager = new LogStreamManager(root);
+  for (const target of invalid) {
+    await assert.rejects(
+      manager.write(target, 'overwrite'),
+      /outside the log directory/,
+    );
+    await assert.rejects(
+      manager.closeStream(target),
+      /outside the log directory/,
+    );
+    assert.equal(manager.getOpenStreamCount(), 0);
+  }
+  assert.equal(await fs.readFile(victim, 'utf8'), 'unchanged');
+  assert.deepEqual(await fs.readdir(outside), ['victim.log']);
+  const folder = path.join(root, '中文 日志');
+  await fs.mkdir(folder);
+  const log = path.join(folder, 'task.log');
+  await Promise.all([
+    manager.write(log, '开始\n'),
+    manager.write(log, '结束\n'),
+  ]);
+  await manager.closeAll();
+  assert.equal(await fs.readFile(log, 'utf8'), '开始\n结束\n');
+});
+
+test('log initialization rejects unsafe paths before mkdir or file writes', async (t) => {
+  const { root, outside, victim, invalid } = await fixture(t);
+  const { handleLogPath } = load(path.resolve('back/config/util.ts'), {
+    './index': { logPath: root },
+    './share': {},
+    '../loaders/logger': {},
+    '../shared/utils': {
+      writeFileWithLock: (file, data) => fs.writeFile(file, data),
+    },
+    '../data/dependence': { DependenceTypes: {} },
+  });
+  for (const target of invalid) {
+    await assert.rejects(
+      handleLogPath(target, 'overwrite'),
+      /outside the log directory/,
+    );
+  }
+  await assert.rejects(
+    handleLogPath('../log-other/new/sub/task.log', 'overwrite'),
+    /outside the log directory/,
+  );
+  assert.equal(await fs.readFile(victim, 'utf8'), 'unchanged');
+  assert.deepEqual(await fs.readdir(outside), ['victim.log']);
+  const log = await handleLogPath('中文 日志/nested/task.log', 'initial');
+  assert.equal(await fs.readFile(log, 'utf8'), 'initial');
+  assert.equal(await handleLogPath(log, 'ignored'), log);
+  assert.equal(await fs.readFile(log, 'utf8'), 'initial');
+});
+
+test('manual execution rejects escaping log names before creating directories or spawning', async (t) => {
+  const { root, outside, victim } = await fixture(t);
+  let spawned = 0;
+  let releases = 0;
+  const errors = [];
+  const CronService = load(path.resolve('back/services/cron.ts'), {
+    '../config': { logPath: root },
+    '../data/cron': {
+      CrontabStatus: { queued: 3, idle: 1 },
+      CrontabModel: { update: async () => {} },
+    },
+    '../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} },
+    '../config/util': {},
+    '../config/const': {},
+    '../schedule/client': {},
+    '../shared/pLimit': {
+      manualRunWithCronLimit: async (fn) => {
+        try {
+          return await fn();
+        } finally {
+          releases++;
+        }
+      },
+    },
+    '../shared/utils': {},
+    '../shared/i18n': { t: (s) => s },
+    '../shared/logReader': {},
+    '../shared/logStreamManager': {
+      logStreamManager: { closeStream: async () => {} },
+    },
+    'cross-spawn': {
+      spawn: () => {
+        spawned++;
+        throw new Error('must not spawn');
+      },
+    },
+  }).default;
+  const service = new CronService({
+    info() {},
+    error: (...args) => errors.push(args),
+  });
+  for (const log_name of [
+    '../log-other/new',
+    outside,
+    'escape-dir/new',
+    'dangling',
+    'bad\0name',
+  ]) {
+    service.getDb = async () => ({
+      id: 1,
+      status: 3,
+      command: 'ignored',
+      log_path: '',
+      log_name,
+    });
+    await service.runSingle(1);
+  }
+  assert.equal(spawned, 0);
+  assert.equal(releases, 5);
+  assert.equal(
+    errors.filter((args) =>
+      args.includes('Log path is outside the log directory'),
+    ).length,
+    5,
+  );
+  assert.equal(await fs.readFile(victim, 'utf8'), 'unchanged');
+  assert.deepEqual(await fs.readdir(outside), ['victim.log']);
+});

test/back/manual-execution.test.cjs

@@ -0,0 +1,85 @@
+const assert = require('node:assert/strict');
+const test = require('node:test');
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const os = require('node:os');
+const { spawn } = require('node:child_process');
+const load = require('../helpers/load-security-module.cjs');
+const { LogStreamManager } = require('../../back/shared/logStreamManager');
+
+test(
+  'manual execution captures a short child before slow status storage and flushes before release',
+  { timeout: 5000 },
+  async (t) => {
+    const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-manual-'));
+    t.after(() => fs.rm(root, { recursive: true, force: true }));
+    const logs = new LogStreamManager(root);
+    const updates = [];
+    let active = 0;
+    let failSpawn = false;
+    const CronService = load(path.resolve('back/services/cron.ts'), {
+      '../config': { logPath: root },
+      '../data/cron': {
+        CrontabStatus: { queued: 3, running: 0, idle: 1 },
+        CrontabModel: {
+          update: async (data, options) => {
+            await new Promise((resolve) => setTimeout(resolve, 40));
+            updates.push({ data, options });
+            return [1];
+          },
+        },
+      },
+      '../data/runningInstance': {
+        RunningInstanceModel: {},
+        InstanceStatus: {},
+      },
+      '../config/util': { getUniqPath: async () => 'task' },
+      '../config/const': { TASK_PREFIX: 'task ', QL_PREFIX: 'ql ' },
+      '../schedule/client': {},
+      '../shared/pLimit': {
+        manualRunWithCronLimit: async (fn) => {
+          active++;
+          try {
+            return await fn();
+          } finally {
+            active--;
+          }
+        },
+      },
+      '../shared/utils': {},
+      '../shared/i18n': { t: (s) => s },
+      '../shared/logReader': {},
+      '../shared/logStreamManager': { logStreamManager: logs },
+      'cross-spawn': {
+        spawn: () =>
+          failSpawn
+            ? spawn('true', { shell: '/nonexistent-ql-manual-shell' })
+            : spawn(process.execPath, [
+                '-e',
+                'process.stdout.write("末尾\\n")',
+              ]),
+      },
+    }).default;
+    const service = new CronService({ info() {}, error() {} });
+    service.getDb = async () => ({
+      id: 1,
+      status: 3,
+      command: 'ignored',
+      log_path: '',
+    });
+    service.makeCommand = () => 'ignored';
+    await service.runSingle(1);
+    const [file] = await fs.readdir(path.join(root, 'task'));
+    assert.equal(
+      await fs.readFile(path.join(root, 'task', file), 'utf8'),
+      '末尾\n',
+    );
+    assert.equal(logs.getOpenStreamCount(), 0);
+    assert.equal(active, 0);
+    assert.equal(updates.at(-1).data.status, 1);
+    failSpawn = true;
+    await service.runSingle(1);
+    assert.equal(active, 0);
+    assert.equal(updates.at(-1).data.status, 1);
+  },
+);

test/back/manual-stop-claim.test.cjs

@@ -0,0 +1,215 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const { spawn } = require('node:child_process');
+const { Sequelize, DataTypes } = require('sequelize');
+const load = require('../helpers/load-security-module.cjs');
+const { killTask } = require('../../back/config/util');
+const { LogStreamManager } = require('../../back/shared/logStreamManager');
+
+for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
+  test(
+    `manual startup loses its conditional claim after ${conflict} and terminates the late child`,
+    { timeout: 10000 },
+    async (t) => {
+      const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-manual-claim-'));
+      const db = new Sequelize({
+        dialect: 'sqlite',
+        storage: ':memory:',
+        logging: false,
+      });
+      t.after(async () => {
+        await db.close();
+        await fs.rm(root, { recursive: true, force: true });
+      });
+      const crons = db.define('Cron', {
+        status: DataTypes.INTEGER,
+        pid: DataTypes.INTEGER,
+        log_path: DataTypes.STRING,
+        queued_token: DataTypes.STRING,
+        command: DataTypes.STRING,
+      });
+      const instances = db.define('Instance', {
+        cron_id: DataTypes.INTEGER,
+        status: DataTypes.INTEGER,
+        pid: DataTypes.INTEGER,
+      });
+      await db.sync();
+      await crons.create({
+        id: 1,
+        status: 3,
+        pid: null,
+        log_path: 'previous.log',
+        command: 'ignored',
+      });
+      let release,
+        entered,
+        child,
+        released = 0;
+      const pending = new Promise((resolve) => (entered = resolve));
+      const gate = new Promise((resolve) => (release = resolve));
+      const logs = new LogStreamManager(root);
+      t.after(() => logs.closeAll());
+      const CronService = load('back/services/cron.ts', {
+        '../config': { logPath: root },
+        '../data/cron': {
+          CrontabModel: crons,
+          CrontabStatus: { queued: 3, running: 0, idle: 1 },
+        },
+        '../data/runningInstance': {
+          RunningInstanceModel: instances,
+          InstanceStatus: { running: 0, stopped: 2 },
+        },
+        '../config/util': {
+          getUniqPath: async () => {
+            entered();
+            await gate;
+            return 'task';
+          },
+          killTask,
+        },
+        '../config/const': {},
+        '../schedule/client': {},
+        '../shared/pLimit': {
+          manualRunWithCronLimit: async (fn) => {
+            try {
+              return await fn();
+            } finally {
+              released++;
+            }
+          },
+        },
+        '../shared/utils': {},
+        '../shared/i18n': { t: (s) => s },
+        '../shared/logReader': {},
+        '../shared/logStreamManager': { logStreamManager: logs },
+        'cross-spawn': {
+          spawn: () => {
+            child = spawn(process.execPath, [
+              '-e',
+              'setInterval(() => {}, 1000)',
+            ]);
+            t.after(() => {
+              if (child.exitCode === null && child.signalCode === null)
+                child.kill('SIGKILL');
+            });
+            return child;
+          },
+        },
+      }).default;
+      const service = new CronService({ info() {}, error() {} });
+      service.getDb = async () =>
+        (await crons.findByPk(1)).get({ plain: true });
+      service.makeCommand = () => 'ignored';
+      const running = service.runSingle(1);
+      await pending;
+      if (conflict === 'stop-requeue') {
+        await service.stop([1]);
+        // Queue again through the real API; leave the new runner pending.
+        const pendingRuns = [];
+        service.runSingle = (id, token) => pendingRuns.push({ id, token });
+        await service.run([1]);
+        assert.equal(pendingRuns.length, 1);
+        assert.equal(
+          pendingRuns[0].token,
+          (await crons.findByPk(1)).queued_token,
+        );
+      } else if (conflict === 'stop') await service.stop([1]);
+      else await crons.update({ log_path: 'newer.log' }, { where: { id: 1 } });
+      release();
+      const result = await running;
+      assert.match(result.error.message, /stopped or superseded/);
+      assert.ok(child.signalCode || child.exitCode !== null);
+      assert.throws(() => process.kill(child.pid, 0), { code: 'ESRCH' });
+      const row = await crons.findByPk(1);
+      assert.equal(row.status, conflict === 'stop' ? 1 : 3);
+      assert.equal(
+        row.log_path,
+        conflict === 'newer-queue' ? 'newer.log' : 'previous.log',
+      );
+      assert.equal(row.pid, null);
+      assert.equal(released, 1);
+    },
+  );
+}
+
+test(
+  'verified termination waits for a SIGTERM-resistant target to exit',
+  { timeout: 5000 },
+  async (t) => {
+    const child = spawn(process.execPath, [
+      '-e',
+      'process.on("SIGTERM",()=>{}); process.stdout.write("ready"); setInterval(()=>{},1000)',
+    ]);
+    t.after(() => {
+      if (child.exitCode === null && child.signalCode === null)
+        child.kill('SIGKILL');
+    });
+    await new Promise((resolve) => child.stdout.once('data', resolve));
+    await killTask(child.pid, true);
+    assert.throws(() => process.kill(child.pid, 0), { code: 'ESRCH' });
+    assert.equal(child.signalCode, 'SIGKILL');
+  },
+);
+
+test('a runner waiting for a concurrency slot cannot adopt a newer queued generation', async (t) => {
+  const db = new Sequelize({
+    dialect: 'sqlite',
+    storage: ':memory:',
+    logging: false,
+  });
+  t.after(() => db.close());
+  const crons = db.define('Cron', {
+    status: DataTypes.INTEGER,
+    pid: DataTypes.INTEGER,
+    log_path: DataTypes.STRING,
+    queued_token: DataTypes.STRING,
+  });
+  await db.sync();
+  await crons.create({ id: 1, status: 1, log_path: 'same.log' });
+  const waiting = [];
+  const CronService = load('back/services/cron.ts', {
+    '../config': {},
+    '../data/cron': {
+      CrontabModel: crons,
+      CrontabStatus: { queued: 3, idle: 1, running: 0 },
+    },
+    '../data/runningInstance': {
+      RunningInstanceModel: { findAll: async () => [] },
+      InstanceStatus: { running: 0 },
+    },
+    '../config/util': {
+      getUniqPath: () => assert.fail('stale runner must not prepare a child'),
+    },
+    '../config/const': {},
+    '../schedule/client': {},
+    '../shared/pLimit': {
+      manualRunWithCronLimit: (fn) => {
+        waiting.push(fn);
+        return Promise.resolve();
+      },
+    },
+    '../shared/utils': {},
+    '../shared/i18n': {},
+    '../shared/logReader': {},
+    '../shared/logStreamManager': {},
+    'cross-spawn': { spawn: () => assert.fail('stale runner must not spawn') },
+  }).default;
+  const errors = [];
+  const service = new CronService({ error: (...args) => errors.push(args) });
+  await service.run([1]);
+  const first = (await crons.findByPk(1)).queued_token;
+  await service.stop([1]);
+  await service.run([1]);
+  const second = (await crons.findByPk(1)).queued_token;
+  assert.notEqual(first, second);
+  assert.equal(waiting.length, 2);
+  await waiting[0]();
+  const row = await crons.findByPk(1);
+  assert.equal(row.status, 3);
+  assert.equal(row.queued_token, second);
+  assert.equal(row.log_path, 'same.log');
+  assert.deepEqual(errors, []);
+});

test/back/node-path-cache.test.cjs

@@ -0,0 +1,173 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync, spawn } = require('node:child_process');
+const helper = path.resolve('shell/node_path_cache.sh');
+function fixture(t) {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-node-path-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  const bin = path.join(root, 'bin');
+  fs.mkdirSync(bin);
+  fs.symlinkSync(process.execPath, path.join(bin, 'node'));
+  const calls = path.join(root, 'calls');
+  fs.writeFileSync(
+    path.join(bin, 'pnpm'),
+    '#!/bin/bash\necho call >> "$CALLS"\n[[ "${FAIL:-0}" == 1 ]] && exit 1\nprintf "%s\\n" "${ANSWER:-/test/global/node_modules}"\n',
+    { mode: 0o755 },
+  );
+  const env = {
+    ...process.env,
+    HOME: root,
+    PATH: `${bin}:${process.env.PATH}`,
+    dir_tmp: path.join(root, 'cache'),
+    CALLS: calls,
+    HELPER: helper,
+  };
+  const command = '. "$HELPER"; ql_get_node_global_path';
+  const run = (extra = {}, code = command) =>
+    spawnSync('/bin/bash', ['-uc', code], {
+      cwd: root,
+      env: { ...env, ...extra },
+      encoding: 'utf8',
+    });
+  const count = () =>
+    fs.existsSync(calls)
+      ? fs.readFileSync(calls, 'utf8').trim().split('\n').length
+      : 0;
+  return { root, bin, env, command, run, count };
+}
+test('warm cache avoids pnpm, preserves spaces, and never evaluates cached text', (t) => {
+  const f = fixture(t);
+  const answer = '/test/path with spaces/$(touch injected)';
+  assert.equal(f.run({ ANSWER: answer }).stdout.trim(), answer);
+  assert.equal(f.run({ ANSWER: '/different' }).stdout.trim(), answer);
+  assert.equal(f.count(), 1);
+  assert.equal(fs.existsSync(path.join(f.root, 'injected')), false);
+});
+test('config, environment, cwd, and executable changes invalidate discovery', (t) => {
+  const f = fixture(t);
+  f.run();
+  fs.writeFileSync(path.join(f.root, '.npmrc'), 'global-dir=/other\n');
+  f.run();
+  assert.equal(f.count(), 2);
+  f.run({ npm_config_global_dir: '/third' });
+  assert.equal(f.count(), 3);
+  f.run({ PNPM_HOME: '/fourth' });
+  assert.equal(f.count(), 4);
+  fs.mkdirSync(path.join(f.root, 'child'));
+  f.run({}, 'cd child; ' + f.command);
+  assert.equal(f.count(), 5);
+  f.run();
+  const before = f.count();
+  fs.appendFileSync(path.join(f.bin, 'pnpm'), '# upgrade\n');
+  f.run();
+  assert.equal(f.count(), before + 1);
+});
+test('expired or malformed records refresh and failed lookups are not cached', (t) => {
+  const f = fixture(t);
+  f.run();
+  const file = path.join(
+    f.env.dir_tmp,
+    fs.readdirSync(f.env.dir_tmp).find((name) => name.endsWith('.cache')),
+  );
+  let lines = fs.readFileSync(file, 'utf8').split('\n');
+  lines[1] = '0';
+  fs.writeFileSync(file, lines.join('\n'));
+  f.run();
+  assert.equal(f.count(), 2);
+  fs.writeFileSync(file, lines[0] + '\n08\n/incorrect\n');
+  assert.equal(f.run().status, 0);
+  assert.equal(f.count(), 3);
+  fs.writeFileSync(file, lines[0]);
+  assert.equal(f.run().status, 0);
+  assert.equal(f.count(), 4);
+  fs.rmSync(file);
+  assert.equal(f.run({ FAIL: '1' }).status, 1);
+  assert.equal(fs.existsSync(file), false);
+  assert.equal(f.run().status, 0);
+  assert.equal(f.count(), 6);
+});
+test('disabled or unavailable cache falls back without changing discovery output', (t) => {
+  const f = fixture(t);
+  for (let i = 0; i < 2; i++)
+    assert.equal(
+      f.run({ QL_NODE_PATH_CACHE: '0' }).stdout.trim(),
+      '/test/global/node_modules',
+    );
+  assert.equal(f.count(), 2);
+  assert.equal(fs.existsSync(f.env.dir_tmp), false);
+  const blocked = path.join(f.root, 'file');
+  fs.writeFileSync(blocked, '');
+  assert.equal(
+    f.run({ dir_tmp: blocked }).stdout.trim(),
+    '/test/global/node_modules',
+  );
+  assert.equal(f.count(), 3);
+});
+test('concurrent refreshes publish complete records and do not overwrite symlink targets', async (t) => {
+  const f = fixture(t);
+  await Promise.all(
+    Array.from(
+      { length: 8 },
+      () =>
+        new Promise((resolve, reject) => {
+          const p = spawn('/bin/bash', ['-uc', f.command], {
+            cwd: f.root,
+            env: f.env,
+          });
+          let out = '';
+          p.stdout.on('data', (c) => (out += c));
+          p.once('error', reject);
+          p.once('close', (code) => {
+            try {
+              assert.equal(code, 0);
+              assert.equal(out.trim(), '/test/global/node_modules');
+              resolve();
+            } catch (e) {
+              reject(e);
+            }
+          });
+        }),
+    ),
+  );
+  const before = f.count();
+  f.run();
+  assert.equal(f.count(), before);
+  const files = fs
+    .readdirSync(f.env.dir_tmp)
+    .filter((file) => file.endsWith('.cache'));
+  assert.equal(files.length, 1);
+  const file = path.join(f.env.dir_tmp, files[0]);
+  fs.rmSync(file);
+  const victim = path.join(f.root, 'victim');
+  fs.writeFileSync(victim, 'unchanged');
+  fs.symlinkSync(victim, file);
+  assert.equal(f.run().status, 0);
+  assert.equal(fs.readFileSync(victim, 'utf8'), 'unchanged');
+  assert.equal(fs.lstatSync(file).isSymbolicLink(), false);
+});
+
+test('task entry tolerates discovery failure with errexit enabled', (t) => {
+  const f = fixture(t);
+  const source = fs.readFileSync(path.resolve('shell/otask.sh'), 'utf8');
+  const start = source.indexOf('append_node_dependency_path() {');
+  const end = source.indexOf('\nenter_script_workdir()', start);
+  const script =
+    source.slice(start, end) +
+    '\nappend_node_dependency_path; printf "continued:%s" "$NODE_PATH"';
+  for (const dir of [path.dirname(helper), f.root]) {
+    const r = f.run(
+      {
+        FAIL: '1',
+        dir_shell: dir,
+        dir_dep: '/legacy/deps',
+        NODE_PATH: '/existing',
+      },
+      'set -e; ' + script,
+    );
+    assert.equal(r.status, 0, r.stderr);
+    assert.equal(r.stdout, 'continued:/existing:/legacy/deps');
+  }
+});

test/back/node-path-lock.test.cjs

@@ -0,0 +1,178 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawn, spawnSync } = require('node:child_process');
+const helper = path.resolve('shell/node_path_cache.sh');
+const hasFlock = spawnSync('/bin/bash', ['-c', 'type -P flock']).status === 0;
+function fixture(t) {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-path-lock-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  const bin = path.join(root, 'bin');
+  fs.mkdirSync(bin);
+  fs.symlinkSync(process.execPath, path.join(bin, 'node'));
+  fs.writeFileSync(
+    path.join(bin, 'pnpm'),
+    '#!/bin/bash\nprintf "call\\n" >> "$CALLS"\n[[ -n "${UMASK_FILE:-}" ]] && umask > "$UMASK_FILE"\nsleep "${LOOKUP_DELAY:-0.2}"\n[[ "${FAIL:-0}" == 1 ]] && exit 17\nprintf "%s\\n" "${npm_config_global_dir:-/test/global/node_modules}"\n',
+    { mode: 0o755 },
+  );
+  const env = {
+    ...process.env,
+    HOME: root,
+    PATH: bin + ':' + process.env.PATH,
+    dir_tmp: path.join(root, 'cache'),
+    CALLS: path.join(root, 'calls'),
+    HELPER: helper,
+  };
+  const command = '. "$HELPER"; ql_get_node_global_path';
+  const run = (extra = {}, code = command) =>
+    spawnSync('/bin/bash', ['-euc', code], {
+      cwd: root,
+      env: { ...env, ...extra },
+      encoding: 'utf8',
+    });
+  const asyncRun = (extra = {}) =>
+    new Promise((resolve, reject) => {
+      const child = spawn('/bin/bash', ['-euc', command], {
+        cwd: root,
+        env: { ...env, ...extra },
+      });
+      let out = '',
+        err = '';
+      child.stdout.on('data', (c) => (out += c));
+      child.stderr.on('data', (c) => (err += c));
+      child.on('error', reject);
+      child.on('close', (status) =>
+        resolve({ status, stdout: out, stderr: err }),
+      );
+    });
+  const count = () =>
+    fs.existsSync(env.CALLS)
+      ? fs.readFileSync(env.CALLS, 'utf8').trim().split('\n').length
+      : 0;
+  const cache = path.join(env.dir_tmp, `pnpm-root-${process.getuid()}.cache`);
+  return {
+    root,
+    bin,
+    env,
+    command,
+    run,
+    asyncRun,
+    count,
+    cache,
+    lock: cache + '.lock',
+  };
+}
+function check(r, output = '/test/global/node_modules') {
+  assert.equal(r.status, 0, r.stderr);
+  assert.equal(r.stdout.trim(), output);
+}
+test(
+  'simultaneous cold and expired lookups share one successful refresh',
+  { skip: !hasFlock },
+  async (t) => {
+    const f = fixture(t);
+    for (let batch = 0; batch < 2; batch++) {
+      if (batch) {
+        const lines = fs.readFileSync(f.cache, 'utf8').split('\n');
+        lines[1] = '0';
+        fs.writeFileSync(f.cache, lines.join('\n'));
+      }
+      const rows = await Promise.all(
+        Array.from({ length: 8 }, () => f.asyncRun({ LOOKUP_DELAY: '0.5' })),
+      );
+      rows.forEach((r) => check(r));
+      assert.equal(f.count(), batch + 1);
+    }
+    assert.equal(fs.statSync(f.lock).mode & 0o777, 0o600);
+    check(f.run());
+    assert.equal(f.count(), 2);
+  },
+);
+test(
+  'waiters with different configuration do not reuse another key',
+  { skip: !hasFlock },
+  async (t) => {
+    const f = fixture(t);
+    const rows = await Promise.all(
+      ['/first', '/second'].map((v) =>
+        f.asyncRun({ npm_config_global_dir: v }),
+      ),
+    );
+    rows.forEach((r, i) => check(r, ['/first', '/second'][i]));
+    assert.equal(f.count(), 2);
+  },
+);
+test(
+  'lock timeout falls back while a holder is still alive',
+  { skip: !hasFlock },
+  async (t) => {
+    const f = fixture(t);
+    fs.mkdirSync(f.env.dir_tmp);
+    const holder = spawn(
+      '/bin/bash',
+      ['-c', 'exec 9>> "$LOCK"; flock 9; echo ready; sleep 30'],
+      { env: { ...f.env, LOCK: f.lock }, detached: true },
+    );
+    t.after(() => {
+      try {
+        process.kill(-holder.pid, 'SIGKILL');
+      } catch {}
+    });
+    await new Promise((resolve, reject) => {
+      holder.stdout.once('data', resolve);
+      holder.once('error', reject);
+    });
+    const start = Date.now();
+    check(f.run());
+    assert.ok(Date.now() - start >= 1800);
+    assert.equal(holder.exitCode, null);
+    assert.equal(f.count(), 1);
+  },
+);
+test('absent or unsupported flock and unsafe lock paths preserve discovery', (t) => {
+  const f = fixture(t);
+  check(
+    f.run(
+      {},
+      'type(){ if [[ "$*" == "-P flock" ]]; then return 1; fi; builtin type "$@"; }; ' +
+        f.command,
+    ),
+  );
+  fs.rmSync(f.cache);
+  fs.writeFileSync(path.join(f.bin, 'flock'), '#!/bin/bash\nexit 64\n', {
+    mode: 0o755,
+  });
+  check(f.run());
+  fs.rmSync(f.cache);
+  fs.rmSync(f.lock, { force: true });
+  const victim = path.join(f.root, 'victim');
+  fs.writeFileSync(victim, 'untouched');
+  fs.symlinkSync(victim, f.lock);
+  check(f.run());
+  assert.equal(fs.readFileSync(victim, 'utf8'), 'untouched');
+  assert.equal(f.count(), 3);
+});
+test('failed refresh releases lock and does not publish a success', (t) => {
+  const f = fixture(t);
+  assert.equal(f.run({ FAIL: '1' }).status, 17);
+  assert.equal(fs.existsSync(f.cache), false);
+  check(f.run());
+  assert.equal(f.count(), 2);
+});
+test('refresh leaves the caller file descriptor and umask unchanged', (t) => {
+  const f = fixture(t);
+  const sentinel = path.join(f.root, 'fd');
+  const mask = path.join(f.root, 'mask');
+  const r = f.run(
+    { SENTINEL: sentinel, UMASK_FILE: mask },
+    'umask 022; exec 9> "$SENTINEL"; ' +
+      f.command +
+      ' >/dev/null; printf preserved >&9; umask',
+  );
+  assert.equal(r.status, 0, r.stderr);
+  assert.equal(r.stdout.trim(), '0022');
+  assert.equal(fs.readFileSync(sentinel, 'utf8'), 'preserved');
+  assert.equal(fs.readFileSync(mask, 'utf8').trim(), '0022');
+});

test/back/primary-apm.test.cjs

@@ -0,0 +1,24 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const vm = require('node:vm');
+const fs = require('node:fs');
+const source = fs.readFileSync('ecosystem.config.js', 'utf8');
+const appSource = fs.readFileSync('back/app.ts', 'utf8');
+const ts = require('typescript');
+const compiled = ts.transpileModule(appSource.slice(0, appSource.indexOf('\nconst app = new Application();')) + '\nmodule.exports = Application;', {compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2022,esModuleInterop:true}}).outputText;
+function config(env) {const module={exports:{}};vm.runInNewContext(source,{module,process:{env}});return module.exports.apps[0];}
+test('container APM policy keeps worker isolation and can restore all inherited probes', () => {
+  for (const primary of [undefined, 'true']) for (const workers of [undefined, 'true']) {
+    const env={QL_CONTAINER:'true',QL_PRIMARY_APM:primary,QL_WORKER_APM:workers};
+    const conf=config(env);env.pm_id='0';env.pmx=String(conf.pmx);
+    const calls=[],module={exports:{}};
+    vm.runInNewContext(compiled,{module,exports:module.exports,process:{env},require(name){if(name==='cluster')return {fork(options){calls.push({...env,...options});return {id:1,process:{pid:123}}}};if(name==='express')return ()=>({use(){}});return {};}});
+    new module.exports().forkWorker('http');
+    assert.equal(conf.pmx,primary==='true');
+    assert.equal(calls[0].pmx,primary==='true'&&workers==='true'?'true':'false');
+    assert.equal(conf.max_restarts,5);assert.equal(conf.script,'static/build/app.js');
+  }
+});
+test('standalone PM2 monitoring stays enabled', () => {
+  for(const QL_PRIMARY_APM of [undefined,'true','false']) assert.equal(config({QL_PRIMARY_APM}).pmx,true);
+});

test/back/scheduler-mutation.test.cjs

@@ -0,0 +1,274 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const { spawn } = require('node:child_process');
+const { once } = require('node:events');
+const { setTimeout: delay } = require('node:timers/promises');
+const { Sequelize, DataTypes } = require('sequelize');
+const load = require('../helpers/load-security-module.cjs');
+
+function gate() {
+  let release;
+  const promise = new Promise((resolve) => {
+    release = resolve;
+  });
+  return { promise, release };
+}
+
+async function fixture(t) {
+  const root = await fs.mkdtemp(
+    path.join(os.tmpdir(), 'ql-scheduler-mutation-'),
+  );
+  const db = new Sequelize({
+    dialect: 'sqlite',
+    storage: ':memory:',
+    logging: false,
+  });
+  const crons = db.define('Cron', {
+    name: DataTypes.STRING,
+    command: DataTypes.STRING,
+    schedule: DataTypes.STRING,
+    isDisabled: DataTypes.INTEGER,
+    saved: DataTypes.BOOLEAN,
+    queued_token: DataTypes.STRING,
+  });
+  await db.sync();
+  t.after(async () => {
+    await db.close();
+    await fs.rm(root, { recursive: true, force: true });
+  });
+  const jobs = new Map();
+  const client = {
+    addCron: async (items, replace) => {
+      if (replace) jobs.clear();
+      items.forEach((item) => jobs.set(item.id, item.command));
+    },
+    delCron: async (ids) => ids.forEach((id) => jobs.delete(id)),
+  };
+  const CronService = load('back/services/cron.ts', {
+    '../config': {
+      crontabFile: path.join(root, 'crontab.list'),
+      logPath: root,
+    },
+    '../data/cron': {
+      CrontabModel: crons,
+      Crontab: class {
+        constructor(options) {
+          Object.assign(this, { isDisabled: 0 }, options);
+        }
+      },
+    },
+    '../data/runningInstance': {},
+    '../config/util': { isDemoEnv: () => false },
+    '../config/const': {},
+    '../schedule/client': client,
+    '../shared/pLimit': {},
+    '../shared/utils': {},
+    '../shared/i18n': { t: (s) => s },
+    '../shared/logReader': {},
+    '../shared/logStreamManager': {},
+  }).default;
+  const service = new CronService({ error() {}, warn() {}, info() {} });
+  service.getLogName = async () => 'task';
+  service.shouldUseCronClient = () => true;
+  service.makeCommand = (doc) => doc.command;
+  service.crontabs = async () => ({ data: await crons.findAll({ raw: true }) });
+  service.setCrontab = async () => {};
+  return { service, crons, jobs, client, root };
+}
+
+for (const operation of ['create', 'update', 'remove', 'disabled', 'enabled']) {
+  test(`recovery snapshot cannot overwrite concurrent ${operation}`, async (t) => {
+    const { service, crons, jobs } = await fixture(t);
+    await crons.create({
+      id: 1,
+      command: 'old',
+      schedule: '* * * * *',
+      isDisabled: operation === 'enabled' ? 1 : 0,
+    });
+    const entered = gate(),
+      release = gate();
+    let first = true;
+    service.setCrontab = async () => {
+      if (first) {
+        first = false;
+        entered.release();
+        await release.promise;
+      }
+    };
+    const recovery = service.autosave_crontab(true);
+    await entered.promise;
+    const before = await crons.findAll({ raw: true });
+    const mutation =
+      operation === 'create'
+        ? service.create({ command: 'new', schedule: '* * * * *' })
+        : operation === 'update'
+        ? service.update({ id: 1, command: 'new' })
+        : service[operation]([1]);
+    await delay(50);
+    assert.deepEqual(
+      await crons.findAll({ raw: true }),
+      before,
+      'DB writes must also wait for recovery',
+    );
+    release.release();
+    await Promise.all([recovery, mutation]);
+    const rows = await crons.findAll({ raw: true });
+    assert.deepEqual(
+      [...jobs.entries()].sort(),
+      rows
+        .filter((row) => !row.isDisabled)
+        .map((row) => [String(row.id), row.command])
+        .sort(),
+    );
+  });
+}
+
+test('recovery waits for an admitted mutation to finish registration before reading the DB', async (t) => {
+  const { service, client, jobs } = await fixture(t);
+  const entered = gate(),
+    release = gate();
+  const add = client.addCron;
+  client.addCron = async (items, replace) => {
+    if (!replace) {
+      entered.release();
+      await release.promise;
+    }
+    return add(items, replace);
+  };
+  const mutation = service.create({ command: 'new', schedule: '* * * * *' });
+  await entered.promise;
+  let reads = 0;
+  const read = service.crontabs;
+  service.crontabs = async () => {
+    reads++;
+    return read();
+  };
+  const recovery = service.autosave_crontab(true);
+  await delay(50);
+  assert.equal(reads, 0);
+  release.release();
+  await Promise.all([mutation, recovery]);
+  assert.deepEqual([...jobs.values()], ['new']);
+});
+
+for (const operation of ['create', 'update', 'enabled']) {
+  test(`${operation} preserves scheduler 503 after rollback and releases its lock`, async (t) => {
+    const { service, crons, client } = await fixture(t);
+    if (operation !== 'create')
+      await crons.create({
+        id: 1,
+        command: 'old',
+        schedule: '* * * * *',
+        isDisabled: operation === 'enabled' ? 1 : 0,
+      });
+    const cause = Object.assign(Error('unavailable'), { status: 503 });
+    client.addCron = async () => {
+      throw cause;
+    };
+    const mutation =
+      operation === 'create'
+        ? service.create({ command: 'new', schedule: '* * * * *' })
+        : operation === 'update'
+        ? service.update({ id: 1, command: 'new' })
+        : service.enabled([1]);
+    await assert.rejects(
+      mutation,
+      (err) =>
+        err.status === 503 && err.cause === cause && /回滚/.test(err.message),
+    );
+    if (operation === 'create') assert.equal(await crons.count(), 0);
+    else {
+      const row = await crons.findByPk(1);
+      assert.equal(row.command, 'old');
+      assert.equal(row.isDisabled, operation === 'enabled' ? 1 : 0);
+    }
+    client.addCron = async () => {};
+    await service.autosave_crontab(true);
+  });
+}
+
+test(
+  'scheduler lock excludes a second OS process and leaves no lock after completion',
+  { timeout: 10000 },
+  async (t) => {
+    const root = await fs.mkdtemp(
+      path.join(os.tmpdir(), 'ql-scheduler-process-'),
+    );
+    t.after(() => fs.rm(root, { recursive: true, force: true }));
+    const config = { crontabFile: path.join(root, 'crontab.list') };
+    const { withSchedulerMutation } = load(
+      'back/shared/schedulerMutationLock.ts',
+      { '../config': config },
+    );
+    const entered = gate(),
+      release = gate();
+    const owner = withSchedulerMutation(async () => {
+      entered.release();
+      await release.promise;
+    });
+    await entered.promise;
+    const child = spawn(
+      process.execPath,
+      [
+        '-e',
+        `
+    const load = require('./test/helpers/load-security-module.cjs');
+    const { withSchedulerMutation } = load('back/shared/schedulerMutationLock.ts', {'../config': JSON.parse(process.argv[1])});
+    process.send('attempting');
+    withSchedulerMutation(async () => process.send('entered')).then(() => process.disconnect()).catch(err => { console.error(err); process.exit(1); });
+  `,
+        JSON.stringify(config),
+      ],
+      { stdio: ['ignore', 'pipe', 'pipe', 'ipc'] },
+    );
+    t.after(() => {
+      if (child.exitCode === null) child.kill('SIGKILL');
+      release.release();
+    });
+    const closed = once(child, 'close');
+    assert.equal((await once(child, 'message'))[0], 'attempting');
+    let acquired = false;
+    const acquisition = once(child, 'message').then(([message]) => {
+      acquired = true;
+      assert.equal(message, 'entered');
+    });
+    await delay(150);
+    assert.equal(acquired, false);
+    release.release();
+    await owner;
+    await acquisition;
+    assert.equal((await closed)[0], 0);
+    await assert.rejects(fs.stat(`${config.crontabFile}.scheduler.lock`), {
+      code: 'ENOENT',
+    });
+  },
+);
+
+test('configuration rollback cannot restore an obsolete manual queue token', async (t) => {
+  const { service, crons, client } = await fixture(t);
+  await crons.create({
+    id: 1,
+    command: 'old',
+    schedule: '* * * * *',
+    isDisabled: 0,
+    queued_token: 'old-token',
+  });
+  let first = true;
+  client.addCron = async () => {
+    if (first) {
+      first = false;
+      await crons.update({ queued_token: 'new-token' }, { where: { id: 1 } });
+      throw Object.assign(Error('unavailable'), { status: 503 });
+    }
+  };
+  await assert.rejects(
+    service.update({ id: 1, command: 'new' }),
+    (err) => err.status === 503,
+  );
+  const row = await crons.findByPk(1);
+  assert.equal(row.command, 'old');
+  assert.equal(row.queued_token, 'new-token');
+});

test/back/scheduler-readiness.test.cjs

@@ -0,0 +1,85 @@
+const test=require('node:test');
+const assert=require('node:assert/strict');
+const {setTimeout:sleep}=require('node:timers/promises');
+const {SchedulerReadiness}=require('../../back/shared/schedulerReadiness');
+const load=require('../helpers/load-security-module.cjs');
+const fs=require('node:fs');
+const ts=require('typescript');
+
+test('readiness stays false through failed restoration and retries without idle polling',async()=>{
+ let probes=0,restores=0,fail=true;
+ const state=new SchedulerReadiness(async()=>{probes++;},10);
+ state.configure(async()=>{restores++;if(fail)throw Error('registration failed');});
+ assert.equal(await state.recover(),false);assert.equal(await state.check(),false);
+ fail=false;await sleep(40);assert.equal(await state.check(),true);assert.ok(restores>=2);
+ const before=probes;await sleep(40);assert.equal(probes,before);
+});
+test('single-flight recovery cannot mark an invalidated generation ready',async()=>{
+ let release,restores=0;
+ const state=new SchedulerReadiness(async()=>{},10);
+ state.configure(async()=>{if(++restores===1)await new Promise(r=>release=r);});
+ const a=state.recover();assert.equal(state.recover(),a);await sleep(1);
+ state.invalidate();release();assert.equal(await a,false);assert.equal(await state.check(),false);
+ await sleep(30);assert.equal(await state.check(),true);assert.equal(restores,2);
+});
+test('mutation waiting is bounded, a later recovery can succeed',async()=>{
+ let release;
+ const state=new SchedulerReadiness(async()=>{},10);state.configure(()=>new Promise(r=>release=r));
+ await assert.rejects(state.ensureReady(15),e=>e.status===503);
+ release();await sleep(1);assert.equal(await state.check(),true);
+});
+test('probe failure immediately invalidates a previously ready scheduler',async()=>{
+ let fail=false;
+ const state=new SchedulerReadiness(async()=>{if(fail)throw Error('unavailable');},10);
+ state.configure(async()=>{});assert.equal(await state.recover(),true);
+ fail=true;assert.equal(await state.check(),false);
+ fail=false;await sleep(40);assert.equal(await state.check(),true);
+});
+test('health uses actual readiness and returns HTTP 503 until recovery',async(t)=>{
+ let ready=false;const express=require('express');
+ const {HealthService}=load('back/services/health.ts',{
+  typedi:{Service:()=>x=>x},'../loaders/logger':{error(){}},'./http':{},
+  '../schedule/client':{readiness:{check:async()=>ready}},
+ });
+ const service=new HealthService({getServer:()=>({})});
+ const app=express();load('back/api/health.ts',{
+  typedi:{get:()=>service},'../services/health':{HealthService},'../loaders/logger':{error(){}},
+ }).default(app);
+ const server=app.listen(0,'127.0.0.1');await new Promise(r=>server.once('listening',r));
+ t.after(()=>new Promise(r=>{server.close(r);server.closeAllConnections();}));
+ for(const expected of [503,200,503]){
+  ready=expected===200;const r=await fetch(`http://127.0.0.1:${server.address().port}/health`);const b=await r.json();
+  assert.equal(r.status,expected);assert.equal(b.code,expected);assert.equal(b.data.services.grpc,ready);
+ }
+});
+test('recovery registration errors propagate while ordinary autosave retains file synchronization',async()=>{
+ const source=fs.readFileSync('back/services/cron.ts','utf8');const a=source.indexOf('  public async autosave_crontab('),z=source.indexOf('  public async bootTask',a);
+ const js=ts.transpileModule('class Fixture {\n'+source.slice(a,z)+'}\nmodule.exports=Fixture;', {compilerOptions:{target:ts.ScriptTarget.ES2020}}).outputText;
+ const module={exports:{}};let files=0;
+ new Function('module','isDemoEnv','cronClient','withSchedulerMutation',js)(module,()=>false,{addCron:async()=>{throw Error('registration unavailable');}},fn=>fn());
+ const fixture=new module.exports();fixture.crontabs=async()=>({data:[]});fixture.setCrontab=async()=>{files++;};fixture.logger={warn(){}};
+ await fixture.autosave_crontab();assert.equal(files,1);
+ await assert.rejects(fixture.autosave_crontab(true),/registration unavailable/);assert.equal(files,2);
+});
+test('scheduler probe uses the cron channel and failed writes are not replayed',async()=>{
+ const calls=[];const fake={
+  waitForReady(deadline,cb){calls.push(['wait',deadline]);cb();},
+  makeUnaryRequest(path,serialize,deserialize,request,options,cb){calls.push(['probe',path,request,options]);cb(null,{status:1});},
+  addCron(request,metadata,options,cb){calls.push(['add',request,options]);cb(Object.assign(Error('invalid'),{code:3}));},
+ };
+ const client=load('back/schedule/client.ts',{
+  '../protos/cron':{CronClient:class{constructor(){return fake;}}},
+  '../config':{grpcPort:5500},'../config/grpcCerts':{getGrpcCerts:()=>({caCert:'ca',clientKey:'key',clientCert:'cert'})},
+  '@grpc/grpc-js':{...require('@grpc/grpc-js'),credentials:{createSsl:()=>({})},status:{UNAVAILABLE:14},Metadata:class{}},
+ }).default;
+ client.readiness.configure(async()=>{});assert.equal(await client.readiness.recover(),true);
+ assert.ok(calls.filter(x=>x[0]==='probe').every(x=>x[1]==='/com.ql.health.Health/Check'&&x[2].service==='scheduler'&&x[3].deadline>Date.now()));
+ await assert.rejects(client.addCron([]),/invalid/);assert.equal(calls.filter(x=>x[0]==='add').length,1);
+});
+test('scheduler health probe never calls back into HTTP health',async()=>{
+ const {check}=load('back/schedule/health.ts',{
+  '../config':{},undici:{request:()=>{throw Error('recursive HTTP health call');}},
+ });
+ const result=await new Promise(resolve=>check({request:{service:'scheduler'}},(error,response)=>resolve({error,response})));
+ assert.equal(result.error,null);assert.deepEqual(result.response,{status:1});
+});

test/back/scheduler-reconciliation.test.cjs

@@ -0,0 +1,152 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const ts = require('typescript');
+const load = require('../helpers/load-security-module.cjs');
+const { SchedulerReadiness } = require('../../back/shared/schedulerReadiness');
+const { AddCronRequest } = require('../../back/protos/cron');
+
+function fixture(client) {
+  const source = fs.readFileSync('back/services/cron.ts', 'utf8');
+  const a = source.indexOf('  public async autosave_crontab(');
+  const z = source.indexOf('  public async bootTask', a);
+  const js = ts.transpileModule(
+    'class Fixture {\n' + source.slice(a, z) + '}\nmodule.exports=Fixture;',
+    { compilerOptions: { target: ts.ScriptTarget.ES2020 } },
+  ).outputText;
+  const module = { exports: {} };
+  new Function('module', 'isDemoEnv', 'cronClient', 'withSchedulerMutation', js)(
+    module,
+    () => false,
+    client,
+    (fn) => fn(),
+  );
+  const service = new module.exports();
+  service.setCrontab = async () => {};
+  service.logger = { warn() {} };
+  service.shouldUseCronClient = () => true;
+  service.makeCommand = () => 'true';
+  return service;
+}
+
+test('recovery reconciles a surviving scheduler after missed deletes and disables, including an empty DB', async () => {
+  const cancelled = [];
+  const stacks = new Map(
+    ['removed', 'disabled', 'kept'].map((id) => [
+      id,
+      [{ cancel: () => cancelled.push(id) }],
+    ]),
+  );
+  const { addCron } = load('back/schedule/addCron.ts', {
+    './data': { scheduleStacks: stacks },
+    'node-schedule': {
+      scheduleJob: (id) => ({ cancel: () => cancelled.push(id) }),
+    },
+    '../shared/runCron': {},
+    '../loaders/logger': { info() {}, warn() {} },
+    '../shared/i18n': { tf: (s) => s },
+  });
+  const client = {
+    addCron: async (crons, replace) => {
+      // Exercise the real protobuf field, not just a JavaScript-only flag.
+      const request = AddCronRequest.decode(
+        AddCronRequest.encode({ crons, replace }).finish(),
+      );
+      await new Promise((resolve, reject) =>
+        addCron({ request }, (err) => (err ? reject(err) : resolve())),
+      );
+    },
+  };
+  const service = fixture(client);
+  let rows = [
+    { id: 'kept', schedule: '* * * * *', isDisabled: 0 },
+    { id: 'disabled', schedule: '* * * * *', isDisabled: 1 },
+  ];
+  service.crontabs = async () => ({ data: rows });
+  const state = new SchedulerReadiness(async () => {});
+  state.configure(() => service.autosave_crontab(true));
+  assert.equal(await state.recover(), true);
+  assert.equal(await state.check(), true);
+  assert.deepEqual([...stacks.keys()], ['kept']);
+  assert.deepEqual(cancelled.sort(), ['disabled', 'kept', 'removed']);
+  rows = [];
+  state.invalidate();
+  assert.equal(await state.recover(), true);
+  assert.equal(stacks.size, 0);
+  assert.equal(await state.check(), true);
+});
+
+test('invalid replacement leaves the previous schedule intact', async () => {
+  const stacks = new Map([
+    [
+      'old',
+      [
+        {
+          cancel: () => {
+            throw Error('must not cancel');
+          },
+        },
+      ],
+    ],
+  ]);
+  const { addCron } = load('back/schedule/addCron.ts', {
+    './data': { scheduleStacks: stacks },
+    '../shared/runCron': {},
+    '../loaders/logger': {},
+    '../shared/i18n': { tf: (s) => s },
+  });
+  await assert.rejects(
+    new Promise((resolve, reject) =>
+      addCron(
+        {
+          request: {
+            replace: true,
+            crons: [{ id: 'bad', schedule: '?', extra_schedules: [] }],
+          },
+        },
+        (err) => (err ? reject(err) : resolve()),
+      ),
+    ),
+  );
+  assert.deepEqual([...stacks.keys()], ['old']);
+});
+
+test('pre-RPC channel failures invalidate readiness and return 503 without executing or replaying writes', async () => {
+  for (const method of ['addCron', 'delCron']) {
+    let invalidations = 0,
+      writes = 0;
+    const fake = {
+      waitForReady: (_deadline, cb) => cb(Error('channel unavailable')),
+      addCron: () => writes++,
+      delCron: () => writes++,
+    };
+    const client = load('back/schedule/client.ts', {
+      '../protos/cron': {
+        CronClient: class {
+          constructor() {
+            return fake;
+          }
+        },
+      },
+      '../config': { grpcPort: 5500 },
+      '../config/grpcCerts': {
+        getGrpcCerts: () => ({
+          caCert: 'ca',
+          clientKey: 'key',
+          clientCert: 'cert',
+        }),
+      },
+      '@grpc/grpc-js': {
+        ...require('@grpc/grpc-js'),
+        credentials: { createSsl: () => ({}) },
+      },
+    }).default;
+    client.readiness.invalidate = () => invalidations++;
+    await assert.rejects(
+      client[method]([]),
+      (err) => err.status === 503 && /channel unavailable/.test(err.message),
+    );
+    assert.equal(invalidations, 1);
+    assert.equal(writes, 0);
+  }
+});

test/back/schema-migrations.test.cjs

@@ -0,0 +1,127 @@
+const assert = require('node:assert/strict');
+const test = require('node:test');
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const os = require('node:os');
+const { Sequelize, QueryTypes } = require('sequelize');
+const { migrateSchema } = require('../../back/shared/schemaMigrations');
+const load = require('../helpers/load-security-module.cjs');
+
+async function createLegacy(storage = ':memory:', omitEnvs = false) {
+  const database = new Sequelize({
+    dialect: 'sqlite',
+    storage,
+    logging: false,
+  });
+  for (const table of [
+    'CrontabViews',
+    'Subscriptions',
+    'Crontabs',
+    ...(omitEnvs ? [] : ['Envs']),
+  ]) {
+    await database.query(
+      `CREATE TABLE "${table}" (id INTEGER PRIMARY KEY, name TEXT)`,
+    );
+  }
+  await database.query(
+    'INSERT INTO "Crontabs" (id, name) VALUES (1, \'keep-me\')',
+  );
+  return database;
+}
+
+test('legacy database upgrades without losing rows and repeated migration is idempotent', async (t) => {
+  const database = await createLegacy();
+  t.after(() => database.close());
+  await migrateSchema(database);
+  await migrateSchema(database);
+  const rows = await database.query('SELECT * FROM "Crontabs"', {
+    type: QueryTypes.SELECT,
+  });
+  assert.equal(rows[0].name, 'keep-me');
+  assert.equal(rows[0].queued_token, null);
+  assert.ok(Object.hasOwn(rows[0], 'allow_multiple_instances'));
+  const applied = await database.query('SELECT id FROM "SchemaMigrations"', {
+    type: QueryTypes.SELECT,
+  });
+  assert.equal(applied.length, 15);
+});
+
+test('migration failure rolls back added columns and can be retried after repair', async (t) => {
+  const database = await createLegacy(':memory:', true);
+  t.after(() => database.close());
+  await assert.rejects(migrateSchema(database));
+  const columns = await database.getQueryInterface().describeTable('Crontabs');
+  assert.equal(Object.hasOwn(columns, 'work_dir'), false);
+  await database.query(
+    'CREATE TABLE "Envs" (id INTEGER PRIMARY KEY, name TEXT)',
+  );
+  await migrateSchema(database);
+  assert.ok(
+    Object.hasOwn(
+      await database.getQueryInterface().describeTable('Crontabs'),
+      'work_dir',
+    ),
+  );
+});
+
+test('offline pre-upgrade backup restores the legacy schema and data', async (t) => {
+  const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-migration-'));
+  t.after(() => fs.rm(dir, { recursive: true, force: true }));
+  const file = path.join(dir, 'database.sqlite');
+  const backup = path.join(dir, 'before.sqlite');
+  let database = await createLegacy(file);
+  await database.close();
+  await fs.copyFile(file, backup);
+  database = new Sequelize({
+    dialect: 'sqlite',
+    storage: file,
+    logging: false,
+  });
+  await migrateSchema(database);
+  await database.close();
+  await fs.copyFile(backup, file);
+  database = new Sequelize({
+    dialect: 'sqlite',
+    storage: file,
+    logging: false,
+  });
+  try {
+    assert.equal(
+      Object.hasOwn(
+        await database.getQueryInterface().describeTable('Crontabs'),
+        'work_dir',
+      ),
+      false,
+    );
+    const rows = await database.query('SELECT name FROM "Crontabs"', {
+      type: QueryTypes.SELECT,
+    });
+    assert.equal(rows[0].name, 'keep-me');
+  } finally {
+    await database.close();
+  }
+});
+
+test('database loader rejects initialization failure rather than allowing workers to start', async () => {
+  const mocks = { './logger': { error() {}, info() {} } };
+  for (const [file, model] of [
+    ['env', 'EnvModel'],
+    ['cron', 'CrontabModel'],
+    ['dependence', 'DependenceModel'],
+    ['open', 'AppModel'],
+    ['system', 'SystemModel'],
+    ['subscription', 'SubscriptionModel'],
+    ['cronView', 'CrontabViewModel'],
+    ['cronStats', 'CrontabStatModel'],
+    ['runningInstance', 'RunningInstanceModel'],
+  ])
+    mocks[`../data/${file}`] = { [model]: { sync: async () => {} } };
+  mocks['../data'] = { sequelize: {} };
+  mocks['../shared/schemaMigrations'] = {
+    migrateSchema: async () => {
+      throw new Error('SQLITE_FULL');
+    },
+  };
+  const initialize = load(path.resolve('back/loaders/db.ts'), mocks).default;
+  await assert.rejects(initialize(), /SQLITE_FULL/);
+});

test/back/shell-api-parsing.test.cjs

@@ -0,0 +1,184 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const source = fs
+  .readFileSync(path.resolve('shell/api.sh'), 'utf8')
+  .replace(/\nget_token\s*$/, '\n');
+const realJq = spawnSync('which', ['jq'], { encoding: 'utf8' }).stdout.trim();
+function fixture(t) {
+  assert.ok(realJq, 'jq is required');
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-api-parse-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  const bin = path.join(root, 'bin');
+  fs.mkdirSync(bin);
+  fs.writeFileSync(
+    path.join(bin, 'jq'),
+    '#!/bin/bash\nprintf "call\\n" >> "$JQ_CALLS"\nexec "$REAL_JQ" "$@"\n',
+    { mode: 0o755 },
+  );
+  const token = path.join(root, 'token.json'),
+    response = path.join(root, 'response.json'),
+    calls = path.join(root, 'calls');
+  const run = (code) =>
+    spawnSync(
+      '/bin/bash',
+      [
+        '-euc',
+        source +
+          '\ncreate_token(){ __ql_token__=generated; generated=1; }; curl(){ cat "$RESPONSE_FILE"; };\n' +
+          code,
+      ],
+      {
+        encoding: 'utf8',
+        env: {
+          ...process.env,
+          PATH: bin + ':' + process.env.PATH,
+          REAL_JQ: realJq,
+          JQ_CALLS: calls,
+          file_auth_token: token,
+          RESPONSE_FILE: response,
+          ql_port: '5700',
+          __ql_token__: 'test-token',
+        },
+      },
+    );
+  return {
+    token,
+    response,
+    run,
+    count: () =>
+      fs.existsSync(calls)
+        ? fs.readFileSync(calls, 'utf8').trim().split('\n').length
+        : 0,
+  };
+}
+test('valid token cache is read with one jq and reused without generation', (t) => {
+  const f = fixture(t);
+  fs.writeFileSync(
+    f.token,
+    JSON.stringify({
+      value: 'header.payload.signature',
+      expiration: 4102444800,
+    }),
+  );
+  const r = f.run(
+    'get_token; printf "%s:%s" "$__ql_token__" "${generated:-0}"',
+  );
+  assert.equal(r.status, 0, r.stderr);
+  assert.equal(r.stdout, 'header.payload.signature:0');
+  assert.equal(f.count(), 1);
+});
+test('expired, corrupt, missing or header-unsafe token cache refreshes under errexit', (t) => {
+  const f = fixture(t);
+  for (const content of [
+    undefined,
+    'broken',
+    '{}',
+    JSON.stringify({ value: 'old', expiration: 1 }),
+    JSON.stringify({ value: 'old', expiration: '4102444800' }),
+    JSON.stringify({ value: '', expiration: 4102444800 }),
+    JSON.stringify({ value: 'bad\ninjected', expiration: 4102444800 }),
+    JSON.stringify({ value: 'bad\n', expiration: 4102444800 }),
+    JSON.stringify({ value: 'bad\rinjected', expiration: 4102444800 }),
+  ]) {
+    if (content === undefined) fs.rmSync(f.token, { force: true });
+    else fs.writeFileSync(f.token, content);
+    const r = f.run('get_token; printf "%s:%s" "$__ql_token__" "$generated"');
+    assert.equal(r.status, 0, r.stderr);
+    assert.equal(r.stdout, 'generated:1');
+  }
+});
+test('success with a message still parses once and stays silent', (t) => {
+  const f = fixture(t);
+  fs.writeFileSync(f.response, JSON.stringify({ code: 200, message: 'ok' }));
+  const r = f.run('update_cron 1 0 123 log 1');
+  assert.equal(r.status, 0, r.stderr);
+  assert.equal(r.stdout, '');
+  assert.equal(f.count(), 1);
+});
+test('status and statistics errors preserve multiline Unicode messages', (t) => {
+  const f = fixture(t);
+  fs.writeFileSync(
+    f.response,
+    JSON.stringify({ code: 500, message: '写入失败\n请重试 "原任务"' }),
+  );
+  for (const command of [
+    'update_cron 1 1 123 log 1 2 7',
+    'record_cron_stat 1 7 2',
+  ]) {
+    const r = f.run(command);
+    assert.equal(r.status, 0, r.stderr);
+    assert.equal(r.stdout, '写入失败\n请重试 "原任务"\n');
+  }
+  assert.equal(f.count(), 2);
+});
+test('non-JSON errors fall back to raw text and absent messages retain legacy null', (t) => {
+  const f = fixture(t);
+  fs.writeFileSync(f.response, 'upstream unavailable');
+  let r = f.run('update_cron 1 0 123 log 1');
+  assert.equal(r.status, 0);
+  assert.equal(r.stdout, 'upstream unavailable\n');
+  fs.writeFileSync(f.response, '{"code":500}');
+  r = f.run('record_cron_stat 1 7 2');
+  assert.equal(r.status, 0);
+  assert.equal(r.stdout, 'null\n');
+});
+test('statistics without a task id perform no JSON parsing', (t) => {
+  const f = fixture(t);
+  const r = f.run('record_cron_stat "" 0 1 || true');
+  assert.equal(r.status, 0, r.stderr);
+  assert.equal(f.count(), 0);
+});
+
+test('canonical success resets prior errors and matches jq missing-message semantics', (t) => {
+  const f = fixture(t);
+  fs.writeFileSync(f.response, '{"code":200}');
+  const r = f.run(
+    'code=500; message=old; ql_parse_status_response "$(cat "$RESPONSE_FILE")"; printf "%s|%s" "$code" "$message"; record_cron_stat 1 0 1',
+  );
+  assert.equal(r.status, 0, r.stderr);
+  assert.equal(r.stdout, '200|null');
+  assert.equal(f.count(), 0);
+});
+
+test('noncanonical JSON always uses jq, including whitespace, duplicate keys and embedded success text', (t) => {
+  const f = fixture(t);
+  const cases = [
+    [' {"code":200} ', '200|null'],
+    ['{"code": 200}', '200|null'],
+    ['{"code":200,"message":"完成\\n第二行"}', '200|完成\n第二行'],
+    ['{"code":200,"data":{"code":500}}', '200|null'],
+    ['{"code":"200"}', '200|null'],
+    ['{"code":200,"code":500,"message":"error"}', '500|error'],
+    ['{"code":500,"message":"{\\"code\\":200}"}', '500|{"code":200}'],
+    ['{"code":500}', '500|null'],
+  ];
+  for (const [body, expected] of cases) {
+    fs.writeFileSync(f.response, body);
+    const r = f.run(
+      'ql_parse_status_response "$(cat "$RESPONSE_FILE")"; printf "%s|%s" "$code" "$message"',
+    );
+    assert.equal(r.status, 0, r.stderr);
+    assert.equal(r.stdout, expected);
+  }
+  assert.equal(f.count(), cases.length);
+});
+
+test('success-shaped invalid documents retain the raw-response error fallback', (t) => {
+  const f = fixture(t);
+  const cases = [
+    '{"code":200}trailing',
+    '{"code":200',
+    '<html>{"code":200}</html>',
+  ];
+  for (const body of cases) {
+    fs.writeFileSync(f.response, body);
+    const r = f.run('update_cron 1 0 123 log 1');
+    assert.equal(r.status, 0, r.stderr);
+    assert.equal(r.stdout, body + '\n');
+  }
+  assert.equal(f.count(), cases.length);
+});

test/back/stop-race.test.cjs

@@ -0,0 +1,167 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const path = require('node:path');
+const { Sequelize, DataTypes } = require('sequelize');
+const load = require('../helpers/load-security-module.cjs');
+async function fixture(t, onKill = async () => {}) {
+  const db = new Sequelize({
+    dialect: 'sqlite',
+    storage: ':memory:',
+    logging: false,
+  });
+  t.after(() => db.close());
+  const crons = db.define('Cron', {
+    command: DataTypes.STRING,
+    status: DataTypes.INTEGER,
+    pid: DataTypes.INTEGER,
+    log_path: DataTypes.STRING,
+        queued_token: DataTypes.STRING,
+    last_execution_time: DataTypes.INTEGER,
+    last_running_time: DataTypes.INTEGER,
+  });
+  const instances = db.define('Instance', {
+    cron_id: DataTypes.INTEGER,
+    pid: DataTypes.INTEGER,
+    status: DataTypes.INTEGER,
+    exit_code: DataTypes.INTEGER,
+    finished_at: DataTypes.INTEGER,
+  });
+  await db.sync();
+  await crons.bulkCreate([
+    { id: 1, command: 'task one.js', status: 0, pid: 101, log_path: 'one.log' },
+    { id: 2, command: 'task two.js', status: 0, pid: 201, log_path: 'two.log' },
+  ]);
+  await instances.bulkCreate([
+    { id: 1, cron_id: 1, pid: 101, status: 0 },
+    { id: 2, cron_id: 1, pid: 102, status: 0 },
+    { id: 3, cron_id: 1, pid: 99, status: 1, exit_code: 0 },
+    { id: 4, cron_id: 1, pid: 98, status: 3, exit_code: 7 },
+    { id: 5, cron_id: 2, pid: 201, status: 0 },
+  ]);
+  let service;
+  const killed = [];
+  const CronService = load(path.resolve('back/services/cron.ts'), {
+    '../config': {},
+    '../data/cron': {
+      CrontabModel: crons,
+      CrontabStatus: { queued: 3, running: 0, idle: 1 },
+    },
+    '../data/runningInstance': {
+      RunningInstanceModel: instances,
+      InstanceStatus: { running: 0, finished: 1, stopped: 2, error: 3 },
+    },
+    '../config/util': {
+      killTask: async (pid, wait) => {
+        assert.equal(wait, true);
+        killed.push(pid);
+        await onKill({ pid, service, instances, crons });
+      },
+      killAllTasks: async () => {
+        throw Error('command-wide scans must not run');
+      },
+    },
+    '../config/const': {},
+    '../schedule/client': {},
+    '../shared/pLimit': {},
+    '../shared/utils': {},
+    '../shared/i18n': { t: (s) => s },
+    '../shared/logReader': {},
+    '../shared/logStreamManager': {},
+    '../shared/childProcess': require('../../back/shared/childProcess'),
+  }).default;
+  service = new CronService({ info() {}, error() {} });
+  service.getDb = ({ id }) => crons.findByPk(id);
+  return { service, instances, crons, killed };
+}
+async function report(service, pid, code) {
+  await service.status({
+    ids: [1],
+    status: 1,
+    pid,
+    log_path: 'one.log',
+    last_running_time: 1,
+    last_execution_time: 100,
+    exit_code: code,
+  });
+}
+test('stop wins when shell exit writes error or success during termination, preserving exit codes', async (t) => {
+  const f = await fixture(t, async ({ service }) => {
+    await report(service, 101, 143);
+    await report(service, 102, 0);
+  });
+  await f.service.stop([1]);
+  const rows = await f.instances.findAll({ order: [['id', 'ASC']], raw: true });
+  assert.deepEqual(
+    rows.map((r) => [r.id, r.status, r.exit_code]),
+    [
+      [1, 2, 143],
+      [2, 2, 0],
+      [3, 1, 0],
+      [4, 3, 7],
+      [5, 0, null],
+    ],
+  );
+});
+test('late shell completion cannot overwrite stopped instances', async (t) => {
+  const f = await fixture(t);
+  await f.service.stop([1]);
+  await report(f.service, 101, 143);
+  assert.equal((await f.instances.findByPk(1)).status, 2);
+  assert.equal((await f.instances.findByPk(2)).status, 2);
+});
+test('rows created after the stop snapshot are not relabelled', async (t) => {
+  const f = await fixture(t, async ({ instances }) => {
+    if (!(await instances.findByPk(6)))
+      await instances.create({ id: 6, cron_id: 1, pid: 103, status: 0 });
+  });
+  await f.service.stop([1]);
+  assert.equal((await f.instances.findByPk(6)).status, 0);
+  assert.equal((await f.instances.findByPk(1)).status, 2);
+});
+test('repeated stop does not rewrite historical rows or their finished timestamps', async (t) => {
+  const f = await fixture(t);
+  await f.service.stop([1]);
+  const before = await f.instances.findAll({
+    raw: true,
+    order: [['id', 'ASC']],
+  });
+  await f.service.stop([1]);
+  assert.deepEqual(
+    await f.instances.findAll({ raw: true, order: [['id', 'ASC']] }),
+    before,
+  );
+});
+test('batch stop captures running instances from every requested cron', async (t) => {
+  const f = await fixture(t);
+  await f.service.stop([1, 2]);
+  assert.equal((await f.instances.findByPk(5)).status, 2);
+  assert.equal((await f.instances.findByPk(3)).status, 1);
+});
+
+test('stop signals every snapshotted PID exactly once and preserves a later running instance', async (t) => {
+  const f = await fixture(t, async ({ instances, crons }) => {
+    if (!(await instances.findByPk(6))) {
+      await instances.create({ id: 6, cron_id: 1, pid: 103, status: 0 });
+      await crons.update(
+        { pid: 103, log_path: 'later.log', status: 0 },
+        { where: { id: 1 } },
+      );
+    }
+  });
+  await f.service.stop([1]);
+  assert.deepEqual(f.killed, [101, 102]);
+  assert.equal((await f.instances.findByPk(6)).status, 0);
+  assert.equal((await f.crons.findByPk(1)).pid, 103);
+  assert.equal((await f.crons.findByPk(1)).status, 0);
+});
+
+test('failed termination is not finalized as stopped', async (t) => {
+  const f = await fixture(t, async ({ pid }) => {
+    if (pid === 102) throw Error('still alive');
+  });
+  await f.service.stop([1]);
+  assert.deepEqual(f.killed, [101, 102]);
+  assert.equal((await f.instances.findByPk(1)).status, 2);
+  assert.equal((await f.instances.findByPk(2)).status, 0);
+  assert.equal((await f.crons.findByPk(1)).status, 0);
+});

test/back/subscription-cleanup.test.cjs

@@ -0,0 +1,53 @@
+const assert = require('node:assert/strict');
+const test = require('node:test');
+const path = require('node:path');
+const dayjs = require('dayjs');
+const load = require('../helpers/load-security-module.cjs');
+
+test('subscription becomes idle and closes its log even when completion logging fails', async () => {
+  const updates = [];
+  let closed = 0;
+  let notified = 0;
+  const Subscription = load(path.resolve('back/services/subscription.ts'), {
+    '../config': {},
+    '../data/subscription': {
+      SubscriptionModel: { update: async (values) => updates.push(values) },
+      SubscriptionStatus: { idle: 1 },
+    },
+    '../data/cron': {},
+    '../config/util': { handleLogPath: async () => '/tmp/unused-log' },
+    '../config/const': { LOG_END_SYMBOL: 'end' },
+    '../config/subscription': {},
+    '../shared/i18n': { t: (s) => s, tf: (s) => s },
+    '../shared/pLimit': {},
+    '../shared/logReader': {},
+    '../shared/logStreamManager': {
+      logStreamManager: {
+        write: async () => {
+          throw new Error('ENOSPC');
+        },
+        closeStream: async () => {
+          closed++;
+          throw new Error('ENOSPC');
+        },
+      },
+    },
+    './schedule': {},
+    './sock': {},
+    './sshKey': {},
+    './cron': {},
+  }).default;
+  const service = new Subscription(
+    {},
+    {},
+    { sendMessage: () => notified++ },
+    {},
+    {},
+  );
+  service.getDb = async () => ({ id: 1, log_path: 'log' });
+  const callbacks = service.taskCallbacks({ id: 1 });
+  await assert.rejects(callbacks.onEnd(undefined, dayjs(), 1), /ENOSPC/);
+  assert.equal(closed, 1);
+  assert.equal(updates.at(-1).status, 1);
+  assert.equal(notified, 1);
+});

test/back/task-time.test.cjs

@@ -0,0 +1,116 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+function extract(file, name) {
+  const text = fs.readFileSync(file, 'utf8');
+  const start = text.indexOf(name + '() {');
+  assert.ok(start >= 0);
+  return text.slice(start, text.indexOf('\n}', start) + 2);
+}
+const source =
+  ['handle_log_path', 'init_begin_time']
+    .map((n) => extract('shell/task.sh', n))
+    .join('\n') +
+  '\n' +
+  ['format_time', 'format_log_time', 'format_timestamp', 'handle_task_end']
+    .map((n) => extract('shell/share.sh', n))
+    .join('\n');
+function run(t, body) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-task-time-'));
+  t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
+  const result = spawnSync(
+    '/bin/bash',
+    [
+      '-ec',
+      source +
+        `
+is_macos=0
+mtime_format='%Y-%m-%d %H:%M:%S.%3N'
+time_format='%Y-%m-%d %H:%M:%S'
+ID=42; dir_log=/tmp; log_name=''; real_log_path=''; no_tee=''; real_time=''
+make_dir() { :; }
+t() { printf '%s\\n' "$*" >> "$TEST_DIR/messages"; }
+update_cron() { printf '%s\\n' "$*" >> "$TEST_DIR/status"; }
+record_cron_stat() { printf '%s\\n' "$*" >> "$TEST_DIR/stats"; }
+date() {
+ printf '%s\\n' "$*" >> "$TEST_DIR/dates"
+ case "$1" in
+  '+%Y-%m-%d %H:%M:%S.%3N|%Y-%m-%d-%H-%M-%S-%3N') printf '%s\\n' '2026-09-12 23:59:59.999|2026-09-12-23-59-59-999';;
+  '+%Y-%m-%d %H:%M:%S|%s') printf '%s\\n' '2026-09-13 00:00:00|101';;
+  '-d') [[ "$3" == '+%s' ]] && echo 100;;
+  *) echo raw-time;;
+ esac
+}
+` +
+        body,
+    ],
+    { encoding: 'utf8', env: { ...process.env, TEST_DIR: dir } },
+  );
+  assert.equal(result.status, 0, result.stderr + result.stdout);
+  return {
+    dir,
+    text: (name) =>
+      fs.existsSync(path.join(dir, name))
+        ? fs.readFileSync(path.join(dir, name), 'utf8')
+        : '',
+  };
+}
+test('Linux task timestamps share a snapshot and retain millisecond log names across midnight', (t) => {
+  const r = run(
+    t,
+    `handle_log_path 'folder/example.js'
+init_begin_time
+[[ "$log_path" == 'folder_example_42/2026-09-12-23-59-59-999.log' ]]
+[[ "$time" == '2026-09-12 23:59:59.999' && "$begin_time" == '2026-09-12 23:59:59' && "$begin_timestamp" == 100 ]]
+handle_task_end`,
+  );
+  assert.equal(r.text('dates').trim().split('\n').length, 3);
+  assert.match(r.text('stats'), /^42 0 1\n$/);
+  assert.match(r.text('messages'), /2026-09-13 00:00:00/);
+});
+test('explicit log paths and output mode remain intact', (t) => {
+  run(
+    t,
+    `real_log_path='custom/output.log'; real_time=true
+handle_log_path 'folder/example.js'
+[[ "$log_path" == 'custom/output.log' && "$cmd" == '' ]]`,
+  );
+});
+test('task failure keeps exit code and minimum one-second runtime', (t) => {
+  const r = run(
+    t,
+    `begin_timestamp=101; _task_exit_code=7; log_path=test.log
+handle_task_end`,
+  );
+  assert.equal(r.text('stats'), '42 7 1\n');
+  assert.match(r.text('messages'), /失败/);
+  assert.match(r.text('status'), /101 1 7/);
+});
+test('manual stop retains stopped message and exit status reporting', (t) => {
+  const r = run(
+    t,
+    `begin_timestamp=99; _task_exit_code=1; log_path=test.log; MANUAL=true
+handle_task_end`,
+  );
+  assert.equal(r.text('stats'), '42 1 2\n');
+  assert.match(r.text('messages'), /已停止/);
+});
+for (const mode of ['macos', 'custom'])
+  test(`${mode} keeps legacy time conversion helpers`, (t) => {
+    run(
+      t,
+      `
+${mode === 'macos' ? 'is_macos=1' : 'mtime_format=custom; time_format=custom'}
+format_log_time() { echo legacy-log; }
+format_time() { echo legacy-time; }
+format_timestamp() { echo 100; }
+handle_log_path example.js
+init_begin_time
+[[ "$log_time" == legacy-log && "$begin_time" == legacy-time && "$begin_timestamp" == 100 ]]
+handle_task_end
+`,
+    );
+  });

test/back/worker-apm.test.cjs

@@ -0,0 +1,84 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const vm = require('node:vm');
+const ts = require('typescript');
+const source = fs.readFileSync('back/app.ts', 'utf8');
+const entry = source.indexOf('\nconst app = new Application();');
+assert.ok(entry > 0, 'application entry point exists');
+const compiled = ts.transpileModule(
+  source.slice(0, entry) + '\nmodule.exports = Application;',
+  {
+    compilerOptions: {
+      module: ts.ModuleKind.CommonJS,
+      target: ts.ScriptTarget.ES2022,
+      esModuleInterop: true,
+    },
+  },
+).outputText;
+function create(env) {
+  const calls = [];
+  const cluster = {
+    fork(options) {
+      const worker = {
+        id: calls.length + 1,
+        process: { pid: 100 + calls.length },
+      };
+      calls.push({ options, inherited: { ...env, ...options }, worker });
+      return worker;
+    },
+  };
+  const module = { exports: {} };
+  const sandbox = {
+    module,
+    exports: module.exports,
+    process: { env },
+    require(name) {
+      if (name === 'cluster') return cluster;
+      if (name === 'express') return () => ({ use() {} });
+      return {};
+    },
+  };
+  vm.runInNewContext(compiled, sandbox);
+  return { app: new module.exports(), calls };
+}
+test('PM2-managed primary disables only inherited worker APM and keeps worker metadata', () => {
+  const env = { pm_id: '0', pmx: 'true', CUSTOM_VALUE: 'kept' };
+  const { app, calls } = create(env);
+  for (const role of ['grpc', 'http']) app.forkWorker(role);
+  assert.deepEqual(
+    calls.map((x) => ({ ...x.options })),
+    [
+      { SERVICE_TYPE: 'grpc', pmx: 'false' },
+      { SERVICE_TYPE: 'http', pmx: 'false' },
+    ],
+  );
+  assert.equal(env.pmx, 'true');
+  assert.equal(calls[0].inherited.CUSTOM_VALUE, 'kept');
+  for (const call of calls) {
+    const entry = app.workerMetadataMap.get(call.worker.id);
+    assert.equal(entry.pid, call.worker.process.pid);
+    assert.equal(entry.serviceType, call.options.SERVICE_TYPE);
+  }
+});
+test('worker APM opt-in restores inherited PM2 settings, including explicit disable', () => {
+  for (const pmx of ['true', 'false']) {
+    const { app, calls } = create({ pm_id: '0', pmx, QL_WORKER_APM: 'true' });
+    app.forkWorker('http');
+    assert.equal(Object.hasOwn(calls[0].options, 'pmx'), false);
+    assert.equal(calls[0].inherited.pmx, pmx);
+  }
+});
+test('standalone startup does not introduce a PM2-specific override', () => {
+  const { app, calls } = create({ CUSTOM_VALUE: 'kept' });
+  app.forkWorker('grpc');
+  assert.deepEqual({ ...calls[0].options }, { SERVICE_TYPE: 'grpc' });
+  assert.equal(calls[0].inherited.CUSTOM_VALUE, 'kept');
+});
+test('replacement workers receive the same monitoring policy', () => {
+  const { app, calls } = create({ pm_id: '0', pmx: 'true' });
+  app.forkWorker('grpc');
+  app.forkWorker('grpc');
+  assert.equal(calls[1].inherited.pmx, 'false');
+  assert.notEqual(calls[0].worker.id, calls[1].worker.id);
+});