重构任务并发执行逻辑

back/schedule/addCron.ts

@@ -2,7 +2,7 @@ import { ServerUnaryCall, sendUnaryData } from '@grpc/grpc-js';
 import { AddCronRequest, AddCronResponse } from '../protos/cron';
 import nodeSchedule from 'node-schedule';
 import { scheduleStacks } from './data';
-import { exec } from 'child_process';
+import { runCron } from '../shared/runCron';
 
 const addCron = (
   call: ServerUnaryCall<AddCronRequest, AddCronResponse>,
@@ -16,7 +16,7 @@ const addCron = (
     scheduleStacks.set(
       id,
       nodeSchedule.scheduleJob(id, schedule, async () => {
-        exec(`ID=${id} ${command}`);
+        runCron(`ID=${id} ${command}`)
       }),
     );
   }

back/schedule/health.ts

@@ -1,6 +1,5 @@
 import { ServerUnaryCall, sendUnaryData } from '@grpc/grpc-js';
 import { HealthCheckRequest, HealthCheckResponse } from '../protos/health';
-import { exec } from 'child_process';
 import config from '../config';
 import { promiseExec } from '../config/util';
 

back/services/cron.ts

@@ -16,6 +16,7 @@ import { Op, where, col as colFn, FindOptions } from 'sequelize';
 import path from 'path';
 import { TASK_PREFIX, QL_PREFIX } from '../config/const';
 import cronClient from '../schedule/client';
+import { runCronWithLimit } from '../shared/pLimit';
 
 @Service()
 export default class CronService {
@@ -365,10 +366,9 @@ export default class CronService {
       { status: CrontabStatus.queued },
       { where: { id: ids } },
     );
-    concurrentRun(
+    ids.forEach(id => {
-      ids.map((id) => async () => await this.runSingle(id)),
+      this.runSingle(id)
-      10,
+    })
-    );
   }
 
   public async stop(ids: number[]) {
@@ -390,6 +390,7 @@ export default class CronService {
   }
 
   private async runSingle(cronId: number): Promise<number> {
+    return runCronWithLimit(() => {
       return new Promise(async (resolve: any) => {
         const cron = await this.getDb({ id: cronId });
         if (cron.status !== CrontabStatus.queued) {
@@ -449,6 +450,7 @@ export default class CronService {
           resolve();
         });
       });
+    })
   }
 
   public async disabled(ids: number[]) {

back/services/dependence.ts

@@ -14,13 +14,14 @@ import SockService from './sock';
 import { FindOptions, Op } from 'sequelize';
 import { concurrentRun } from '../config/util';
 import dayjs from 'dayjs';
+import { runCronWithLimit } from 'back/shared/pLimit';
 
 @Service()
 export default class DependenceService {
   constructor(
     @Inject('logger') private logger: winston.Logger,
     private sockService: SockService,
-  ) {}
+  ) { }
 
   public async create(payloads: Dependence[]): Promise<Dependence[]> {
     const tabs = payloads.map((x) => {
@@ -104,20 +105,17 @@ export default class DependenceService {
     isInstall: boolean = true,
     force: boolean = false,
   ) {
-    concurrentRun(
+    docs.forEach(async (dep) => {
-      docs.map((dep) => async () => {
       const status = isInstall
         ? DependenceStatus.installing
         : DependenceStatus.removing;
       await DependenceModel.update({ status }, { where: { id: dep.id } });
-        return await this.installOrUninstallDependencies(
+      this.installOrUninstallDependencies(
         [dep],
         isInstall,
         force,
-        );
+      )
-      }),
+    })
-      1,
-    );
   }
 
   public async reInstall(ids: number[]): Promise<Dependence[]> {
@@ -157,6 +155,7 @@ export default class DependenceService {
     isInstall: boolean = true,
     force: boolean = false,
   ) {
+    return runCronWithLimit(() => {
       return new Promise(async (resolve) => {
         if (dependencies.length === 0) {
           resolve(null);
@@ -212,7 +211,6 @@ export default class DependenceService {
             references: depIds,
           });
           await this.updateLog(depIds, JSON.stringify(err));
-        resolve(null);
         });
 
         cp.on('close', async (code) => {
@@ -250,5 +248,6 @@ export default class DependenceService {
           resolve(null);
         });
       });
+    })
   }
 }

back/services/schedule.ts

@@ -9,6 +9,7 @@ import {
   Task,
 } from 'toad-scheduler';
 import dayjs from 'dayjs';
+import { runCronWithLimit } from '../shared/pLimit';
 
 interface ScheduleTaskType {
   id: number;
@@ -40,13 +41,14 @@ export default class ScheduleService {
 
   private maxBuffer = 200 * 1024 * 1024;
 
-  constructor(@Inject('logger') private logger: winston.Logger) {}
+  constructor(@Inject('logger') private logger: winston.Logger) { }
 
   async runTask(
     command: string,
     callbacks: TaskCallbacks = {},
     completionTime: 'start' | 'end' = 'end',
   ) {
+    return runCronWithLimit(() => {
       return new Promise(async (resolve, reject) => {
         try {
           const startTime = dayjs();
@@ -54,7 +56,6 @@ export default class ScheduleService {
 
           const cp = spawn(command, { shell: '/bin/bash' });
 
-        // TODO:
           callbacks.onStart?.(cp, startTime);
           completionTime === 'start' && resolve(cp.pid);
 
@@ -107,6 +108,7 @@ export default class ScheduleService {
           await callbacks.onError?.(JSON.stringify(error));
         }
       });
+    })
   }
 
   async createCronTask(
@@ -126,12 +128,12 @@ export default class ScheduleService {
     this.scheduleStacks.set(
       _id,
       nodeSchedule.scheduleJob(_id, schedule, async () => {
-        await this.runTask(command, callbacks);
+        this.runTask(command, callbacks);
       }),
     );
 
     if (runImmediately) {
-      await this.runTask(command, callbacks);
+      this.runTask(command, callbacks);
     }
   }
 

back/services/subscription.ts

@@ -8,9 +8,6 @@ import {
 } from '../data/subscription';
 import {
   ChildProcessWithoutNullStreams,
-  exec,
-  execSync,
-  spawn,
 } from 'child_process';
 import fs from 'fs';
 import {
@@ -20,6 +17,7 @@ import {
   createFile,
   killTask,
   handleLogPath,
+  promiseExec,
 } from '../config/util';
 import { promises, existsSync } from 'fs';
 import { FindOptions, Op } from 'sequelize';
@@ -39,7 +37,7 @@ export default class SubscriptionService {
     private scheduleService: ScheduleService,
     private sockService: SockService,
     private sshKeyService: SshKeyService,
-  ) {}
+  ) { }
 
   public async list(searchText?: string): Promise<Subscription[]> {
     let query = {};
@@ -110,18 +108,6 @@ export default class SubscriptionService {
     this.sshKeyService.setSshConfig(docs);
   }
 
-  private async promiseExec(command: string): Promise<string> {
-    return new Promise((resolve, reject) => {
-      exec(
-        command,
-        { maxBuffer: 200 * 1024 * 1024, encoding: 'utf8' },
-        (err, stdout, stderr) => {
-          resolve(stdout || stderr || JSON.stringify(err));
-        },
-      );
-    });
-  }
-
   private taskCallbacks(doc: Subscription): TaskCallbacks {
     return {
       onBefore: async (startTime) => {
@@ -144,7 +130,7 @@ export default class SubscriptionService {
         try {
           if (doc.sub_before) {
             fs.appendFileSync(absolutePath, `\n## 执行before命令...\n\n`);
-            beforeStr = await this.promiseExec(doc.sub_before);
+            beforeStr = await promiseExec(doc.sub_before);
           }
         } catch (error: any) {
           beforeStr =
@@ -171,7 +157,7 @@ export default class SubscriptionService {
         try {
           if (sub.sub_after) {
             fs.appendFileSync(absolutePath, `\n\n## 执行after命令...\n\n`);
-            afterStr = await this.promiseExec(sub.sub_after);
+            afterStr = await promiseExec(sub.sub_after);
           }
         } catch (error: any) {
           afterStr =
@@ -290,10 +276,9 @@ export default class SubscriptionService {
       { status: SubscriptionStatus.queued },
       { where: { id: ids } },
     );
-    concurrentRun(
+    ids.forEach(id => {
-      ids.map((id) => async () => await this.runSingle(id)),
+      this.runSingle(id)
-      10,
+    })
-    );
   }
 
   public async stop(ids: number[]) {
@@ -330,7 +315,7 @@ export default class SubscriptionService {
 
     const command = formatCommand(subscription);
 
-    await this.scheduleService.runTask(
+    this.scheduleService.runTask(
       command,
       this.taskCallbacks(subscription),
     );

back/services/user.ts

@@ -15,7 +15,6 @@ import { NotificationInfo } from '../data/notify';
 import NotificationService from './notify';
 import { Request } from 'express';
 import ScheduleService from './schedule';
-import { spawn } from 'child_process';
 import SockService from './sock';
 import dayjs from 'dayjs';
 

back/shared/pLimit.ts

@@ -0,0 +1,10 @@
+import pLimit from "p-limit";
+import os from 'os';
+
+const cronLimit = pLimit(os.cpus.length);
+
+export function runCronWithLimit<T>(fn: () => Promise<T>): Promise<T> {
+  return cronLimit(() => {
+    return fn();
+  });
+}

back/shared/runCron.ts

@@ -0,0 +1,37 @@
+import { spawn } from "child_process";
+import { runCronWithLimit } from "./pLimit";
+import Logger from '../loaders/logger';
+
+export function runCron(cmd: string): Promise<number> {
+  return runCronWithLimit(() => {
+    return new Promise(async (resolve: any) => {
+      Logger.silly('运行命令: ' + cmd);
+
+      const cp = spawn(cmd, { shell: '/bin/bash' });
+
+      cp.stderr.on('data', (data) => {
+        Logger.info(
+          '[执行任务失败] %s，时间：%s, 错误信息：%j',
+          cmd,
+          new Date().toLocaleString(),
+          data.toString(),
+        );
+      });
+      cp.on('error', (err) => {
+        Logger.error(
+          '[创建任务失败] %s，时间：%s, 错误信息：%j',
+          cmd,
+          new Date().toLocaleString(),
+          err,
+        );
+      });
+
+      cp.on('close', async (code) => {
+        Logger.info(
+          `[任务退出] ${cmd} 进程id: ${cp.pid} 退出，退出码 ${code}`,
+        );
+        resolve();
+      });
+    });
+  })
+}

package.json

@@ -81,6 +81,7 @@
     "nedb": "^1.8.0",
     "node-schedule": "^2.1.0",
     "nodemailer": "^6.7.2",
+    "p-limit": "^4.0.0",
     "protobufjs": "^7.2.3",
     "pstree.remy": "^1.1.8",
     "reflect-metadata": "^0.1.13",

pnpm-lock.yaml

@@ -79,6 +79,9 @@ dependencies:
   nodemailer:
     specifier: ^6.7.2
     version: 6.9.1
+  p-limit:
+    specifier: ^4.0.0
+    version: 4.0.0
   protobufjs:
     specifier: ^7.2.3
     version: 7.2.3
@@ -10844,6 +10847,13 @@ packages:
       yocto-queue: 0.1.0
     dev: true
 
+  /p-limit@4.0.0:
+    resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      yocto-queue: 1.0.0
+    dev: false
+
   /p-locate@4.1.0:
     resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==}
     engines: {node: '>=8'}
@@ -15471,6 +15481,11 @@ packages:
     engines: {node: '>=10'}
     dev: true
 
+  /yocto-queue@1.0.0:
+    resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==}
+    engines: {node: '>=12.20'}
+    dev: false
+
   /yorkie@2.0.0:
     resolution: {integrity: sha512-jcKpkthap6x63MB4TxwCyuIGkV0oYP/YRyuQU5UO0Yz/E/ZAu+653/uov+phdmO54n6BcvFRyyt0RRrWdN2mpw==}
     engines: {node: '>=4'}