批量运行任务添加并行数限制

back/config/index.ts

@@ -20,11 +20,16 @@ const dbPath = path.join(rootPath, 'db/');
 const manualLogPath = path.join(rootPath, 'manual_log/');
 const cronDbFile = path.join(rootPath, 'db/crontab.db');
 const cookieDbFile = path.join(rootPath, 'db/cookie.db');
+const configFound = dotenv.config({ path: confFile });
 
 if (envFound.error) {
   throw new Error("⚠️  Couldn't find .env file  ⚠️");
 }
 
+if (configFound.error) {
+  throw new Error("⚠️  Couldn't find config.sh file  ⚠️");
+}
+
 export default {
   port: parseInt(process.env.PORT as string, 10),
   cronPort: parseInt(process.env.CRON_PORT as string, 10),

back/data/cron.ts

@@ -9,6 +9,7 @@ export class Crontab {
   status?: CrontabStatus;
   isSystem?: 1 | 0;
   pid?: number;
+  isDisabled?: 1 | 0;
 
   constructor(options: Crontab) {
     this.name = options.name;
@@ -21,6 +22,7 @@ export class Crontab {
     this.timestamp = new Date().toString();
     this.isSystem = options.isSystem || 0;
     this.pid = options.pid;
+    this.isDisabled = options.isDisabled || 0;
   }
 }
 
@@ -28,4 +30,5 @@ export enum CrontabStatus {
   'running',
   'idle',
   'disabled',
+  'queued',
 }

back/loaders/initData.ts

@@ -74,6 +74,25 @@ export default async () => {
       }
     });
 
+  // patch 禁用状态字段改变
+  cronDb
+    .find({
+      status: CrontabStatus.disabled,
+    })
+    .exec((err, docs) => {
+      if (docs.length > 0) {
+        const ids = docs.map((x) => x._id);
+        cronDb.update(
+          { _id: { $in: ids } },
+          { $set: { status: CrontabStatus.idle, isDisabled: 1 } },
+          { multi: true },
+          (err) => {
+            cronService.autosave_crontab();
+          },
+        );
+      }
+    });
+
   // 初始化保存一次ck和定时任务数据
   await cronService.autosave_crontab();
   await cookieService.set_cookies();

back/schedule.ts

@@ -29,7 +29,8 @@ const run = async () => {
           if (
             _schedule &&
             _schedule.length > 5 &&
-            task.status !== CrontabStatus.disabled
+            task.status !== CrontabStatus.disabled &&
+            !task.isDisabled
           ) {
             schedule.scheduleJob(task.schedule, function () {
               let command = task.command as string;

back/services/cron.ts

@@ -7,11 +7,16 @@ import { exec, execSync, spawn } from 'child_process';
 import fs from 'fs';
 import cron_parser from 'cron-parser';
 import { getFileContentByName } from '../config/util';
+import PQueue from 'p-queue';
 
 @Service()
 export default class CronService {
   private cronDb = new DataStore({ filename: config.cronDbFile });
 
+  private queue = new PQueue({
+    concurrency: parseInt(process.env.MaxConcurrentNum) || 5,
+  });
+
   constructor(@Inject('logger') private logger: winston.Logger) {
     this.cronDb.loadDatabase((err) => {
       if (err) throw err;
@@ -124,97 +129,102 @@ export default class CronService {
   }
 
   public async run(ids: string[]) {
-    this.cronDb.find({ _id: { $in: ids } }).exec((err, docs: Crontab[]) => {
+    this.cronDb.update(
-      for (let i = 0; i < docs.length; i++) {
+      { _id: { $in: ids } },
-        const doc = docs[i];
+      { $set: { status: CrontabStatus.queued } },
-        this.runSingle(doc);
+      { multi: true },
-      }
+    );
-    });
+    for (let i = 0; i < ids.length; i++) {
+      const id = ids[i];
+      this.queue.add(() => this.runSingle(id));
+    }
   }
 
   public async stop(ids: string[]) {
     this.cronDb.find({ _id: { $in: ids } }).exec((err, docs: Crontab[]) => {
-      for (let i = 0; i < docs.length; i++) {
+      this.cronDb.update(
-        const doc = docs[i];
+        { _id: { $in: ids } },
-        if (doc.pid) {
+        { $set: { status: CrontabStatus.idle }, $unset: { pid: true } },
-          exec(`kill -9 ${doc.pid}`, (err, stdout, stderr) => {
+      );
-            this.cronDb.update(
+      const pids = docs
-              { _id: doc._id },
+        .map((x) => x.pid)
-              { $set: { status: CrontabStatus.idle }, $unset: { pid: true } },
+        .filter((x) => !!x)
-            );
+        .join('\n');
-          });
+      console.log(pids);
-        }
+      exec(`echo - e "${pids}" | xargs kill - 9`);
-      }
     });
   }
 
-  private async runSingle(cron: Crontab) {
+  private async runSingle(id: string): Promise<number> {
-    let { _id, command } = cron;
+    return new Promise(async (resolve) => {
+      const cron = await this.get(id);
+      if (cron.status !== CrontabStatus.queued) {
+        resolve(0);
+        return;
+      }
 
-    this.logger.silly('Running job');
+      let { _id, command } = cron;
-    this.logger.silly('ID: ' + _id);
-    this.logger.silly('Original command: ' + command);
 
-    let logFile = `${config.manualLogPath}${_id}.log`;
+      this.logger.silly('Running job');
-    fs.writeFileSync(logFile, `开始执行...\n\n${new Date().toString()}\n`);
+      this.logger.silly('ID: ' + _id);
+      this.logger.silly('Original command: ' + command);
 
-    let cmdStr = command;
+      let logFile = `${config.manualLogPath}${_id}.log`;
-    if (!cmdStr.includes('task ') && !cmdStr.includes('ql ')) {
+      fs.writeFileSync(logFile, `开始执行...\n${new Date().toString()}\n`);
-      cmdStr = `task ${cmdStr}`;
-    }
-    if (cmdStr.endsWith('.js')) {
-      cmdStr = `${cmdStr} now`;
-    }
-    const cmd = spawn(cmdStr, { shell: true });
 
-    this.cronDb.update(
+      let cmdStr = command;
-      { _id },
+      if (!cmdStr.includes('task ') && !cmdStr.includes('ql ')) {
-      { $set: { status: CrontabStatus.running, pid: cmd.pid } },
+        cmdStr = `task ${cmdStr}`;
-    );
+      }
+      if (cmdStr.endsWith('.js')) {
+        cmdStr = `${cmdStr} now`;
+      }
+      const cmd = spawn(cmdStr, { shell: true });
 
-    cmd.stdout.on('data', (data) => {
-      this.logger.info(`stdout: ${data}`);
-      fs.appendFileSync(logFile, data);
-    });
-
-    cmd.stderr.on('data', (data) => {
-      this.logger.info(`stderr: ${data}`);
-      fs.appendFileSync(logFile, data);
-    });
-
-    cmd.on('close', (code) => {
-      this.logger.info(`child process exited with code ${code}`);
       this.cronDb.update(
         { _id },
-        { $set: { status: CrontabStatus.idle }, $unset: { pid: true } },
+        { $set: { status: CrontabStatus.running, pid: cmd.pid } },
       );
-    });
 
-    cmd.on('error', (err) => {
+      cmd.stdout.on('data', (data) => {
-      this.logger.info(err);
+        this.logger.info(`stdout: ${data}`);
-      fs.appendFileSync(logFile, err.stack);
+        fs.appendFileSync(logFile, data);
-    });
+      });
 
-    cmd.on('exit', (code: number, signal: any) => {
+      cmd.stderr.on('data', (data) => {
-      this.logger.info(`cmd exit ${code}`);
+        this.logger.info(`stderr: ${data}`);
-      this.cronDb.update(
+        fs.appendFileSync(logFile, data);
-        { _id },
+      });
-        { $set: { status: CrontabStatus.idle }, $unset: { pid: true } },
-      );
-      fs.appendFileSync(logFile, `\n\n执行结束...`);
-    });
 
-    cmd.on('disconnect', () => {
+      cmd.on('close', (code) => {
-      this.logger.info(`cmd disconnect`);
+        this.logger.info(`child process exited with code ${code}`);
-      this.cronDb.update({ _id }, { $set: { status: CrontabStatus.idle } });
+        this.cronDb.update(
-      fs.appendFileSync(logFile, `\n\n连接断开...`);
+          { _id },
+          { $set: { status: CrontabStatus.idle }, $unset: { pid: true } },
+        );
+      });
+
+      cmd.on('error', (err) => {
+        this.logger.info(err);
+        fs.appendFileSync(logFile, err.stack);
+      });
+
+      cmd.on('exit', (code: number, signal: any) => {
+        this.logger.info(`cmd exit ${code}`);
+        this.cronDb.update(
+          { _id },
+          { $set: { status: CrontabStatus.idle }, $unset: { pid: true } },
+        );
+        fs.appendFileSync(logFile, `\n执行结束...`);
+        resolve(code);
+      });
     });
   }
 
   public async disabled(ids: string[]) {
     this.cronDb.update(
       { _id: { $in: ids } },
-      { $set: { status: CrontabStatus.disabled } },
+      { $set: { isDisabled: 1 } },
       { multi: true },
     );
     await this.set_crontab(true);
@@ -223,7 +233,7 @@ export default class CronService {
   public async enabled(ids: string[]) {
     this.cronDb.update(
       { _id: { $in: ids } },
-      { $set: { status: CrontabStatus.idle } },
+      { $set: { isDisabled: 0 } },
       { multi: true },
     );
     await this.set_crontab(true);
@@ -244,7 +254,7 @@ export default class CronService {
     var crontab_string = '';
     tabs.forEach((tab) => {
       const _schedule = tab.schedule && tab.schedule.split(' ');
-      if (tab.status === CrontabStatus.disabled || _schedule.length !== 5) {
+      if (tab.isDisabled === 1 || _schedule.length !== 5) {
         crontab_string += '# ';
         crontab_string += tab.schedule;
         crontab_string += ' ';

package.json

@@ -36,6 +36,7 @@
     "nedb": "^1.8.0",
     "node-fetch": "^2.6.1",
     "node-schedule": "^2.0.0",
+    "p-queue": "6.6.2",
     "reflect-metadata": "^0.1.13",
     "typedi": "^0.8.0",
     "winston": "^3.3.3"

sample/config.sample.sh

@@ -13,6 +13,9 @@ AutoAddCron="true"
 ## 设置定时任务执行的超时时间，默认1h，后缀"s"代表秒(默认值), "m"代表分, "h"代表小时, "d"代表天
 CommandTimeoutTime="1h"
 
+## 设置批量执行任务时的并发数，默认同时执行5个任务
+MaxConcurrentNum="5"
+
 ## 在运行 task 命令时，随机延迟启动任务的最大延迟时间
 ## 如果任务不是必须准点运行的任务，那么给它增加一个随机延迟，由你定义最大延迟时间，单位为秒，如 RandomDelay="300" ，表示任务将在 1-300 秒内随机延迟一个秒数，然后再运行
 ## 在crontab.list中，在每小时第0-2分、第30-31分、第59分这几个时间内启动的任务，均算作必须准点运行的任务，在启动这些任务时，即使你定义了RandomDelay，也将准点运行，不启用随机延迟

src/pages/crontab/index.tsx

@@ -24,6 +24,7 @@ import {
   StopOutlined,
   DeleteOutlined,
   PauseCircleOutlined,
+  SendOutlined,
 } from '@ant-design/icons';
 import config from '@/utils/config';
 import { PageContainer } from '@ant-design/pro-layout';
@@ -38,6 +39,7 @@ enum CrontabStatus {
   'running',
   'idle',
   'disabled',
+  'queued',
 }
 
 enum OperationName {
@@ -99,17 +101,29 @@ const Crontab = () => {
       align: 'center' as const,
       render: (text: string, record: any) => (
         <>
-          {record.status === CrontabStatus.idle && (
+          {!record.isDisabled && (
-            <Tag icon={<ClockCircleOutlined />} color="default">
+            <>
-              空闲中
+              {record.status === CrontabStatus.idle && (
-            </Tag>
+                <Tag icon={<ClockCircleOutlined />} color="default">
+                  空闲中
+                </Tag>
+              )}
+              {record.status === CrontabStatus.running && (
+                <Tag
+                  icon={<Loading3QuartersOutlined spin />}
+                  color="processing"
+                >
+                  运行中
+                </Tag>
+              )}
+              {record.status === CrontabStatus.queued && (
+                <Tag icon={<SendOutlined />} color="default">
+                  队列中
+                </Tag>
+              )}
+            </>
           )}
-          {record.status === CrontabStatus.running && (
+          {record.isDisabled === 1 && (
-            <Tag icon={<Loading3QuartersOutlined spin />} color="processing">
-              运行中
-            </Tag>
-          )}
-          {record.status === CrontabStatus.disabled && (
             <Tag icon={<CloseCircleOutlined />} color="error">
               已禁用
             </Tag>
@@ -123,7 +137,7 @@ const Crontab = () => {
       align: 'center' as const,
       render: (text: string, record: any, index: number) => (
         <Space size="middle">
-          {record.status !== CrontabStatus.running && (
+          {record.status === CrontabStatus.idle && (
             <Tooltip title="运行">
               <a
                 onClick={() => {
@@ -134,7 +148,7 @@ const Crontab = () => {
               </a>
             </Tooltip>
           )}
-          {record.status === CrontabStatus.running && (
+          {record.status !== CrontabStatus.idle && (
             <Tooltip title="停止">
               <a
                 onClick={() => {
@@ -303,12 +317,10 @@ const Crontab = () => {
 
   const enabledOrDisabledCron = (record: any, index: number) => {
     Modal.confirm({
-      title: `确认${
+      title: `确认${record.isDisabled === 1 ? '启用' : '禁用'}`,
-        record.status === CrontabStatus.disabled ? '启用' : '禁用'
-      }`,
       content: (
         <>
-          确认{record.status === CrontabStatus.disabled ? '启用' : '禁用'}
+          确认{record.isDisabled === 1 ? '启用' : '禁用'}
           定时任务{' '}
           <Text style={{ wordBreak: 'break-all' }} type="warning">
             {record.name}
@@ -320,7 +332,7 @@ const Crontab = () => {
         request
           .put(
             `${config.apiPrefix}crons/${
-              record.status === CrontabStatus.disabled ? 'enable' : 'disable'
+              record.isDisabled === 1 ? 'enable' : 'disable'
             }`,
             {
               data: [record._id],
@@ -328,14 +340,11 @@ const Crontab = () => {
           )
           .then((data: any) => {
             if (data.code === 200) {
-              const newStatus =
+              const newStatus = record.isDisabled === 1 ? 0 : 1;
-                record.status === CrontabStatus.disabled
-                  ? CrontabStatus.idle
-                  : CrontabStatus.disabled;
               const result = [...value];
               result.splice(index, 1, {
                 ...record,
-                status: newStatus,
+                isDisabled: newStatus,
               });
               setValue(result);
             } else {
@@ -366,14 +375,14 @@ const Crontab = () => {
           <Menu.Item
             key="enableordisable"
             icon={
-              record.status === CrontabStatus.disabled ? (
+              record.isDisabled === 1 ? (
                 <CheckCircleOutlined />
               ) : (
                 <StopOutlined />
               )
             }
           >
-            {record.status === CrontabStatus.disabled ? '启用' : '禁用'}
+            {record.isDisabled === 1 ? '启用' : '禁用'}
           </Menu.Item>
           {record.isSystem !== 1 && (
             <Menu.Item key="delete" icon={<DeleteOutlined />}>

src/pages/crontab/logModal.tsx

@@ -11,6 +11,7 @@ enum CrontabStatus {
   'running',
   'idle',
   'disabled',
+  'queued',
 }
 
 const CronLogModal = ({