修改并发逻辑，系统设置增加定时任务并发设置

back/api/system.ts

@@ -70,12 +70,12 @@ export default (app: Router) => {
   });
 
   route.get(
-    '/log/remove',
+    '/config',
     async (req: Request, res: Response, next: NextFunction) => {
       const logger: Logger = Container.get('logger');
       try {
         const systemService = Container.get(SystemService);
-        const data = await systemService.getLogRemoveFrequency();
+        const data = await systemService.getSystemConfig();
         res.send({ code: 200, data });
       } catch (e) {
         return next(e);
@@ -84,18 +84,19 @@ export default (app: Router) => {
   );
 
   route.put(
-    '/log/remove',
+    '/config',
     celebrate({
       body: Joi.object({
-        frequency: Joi.number().required(),
+        logRemoveFrequency: Joi.number().optional().allow(null),
+        cronConcurrency: Joi.number().optional().allow(null),
       }),
     }),
     async (req: Request, res: Response, next: NextFunction) => {
       const logger: Logger = Container.get('logger');
       try {
         const systemService = Container.get(SystemService);
-        const result = await systemService.updateLogRemoveFrequency(
-          req.body.frequency,
+        const result = await systemService.updateSystemConfig(
+          req.body,
         );
         res.send(result);
       } catch (e) {

back/data/auth.ts

@@ -1,10 +1,11 @@
 import { sequelize } from '.';
 import { DataTypes, Model, ModelDefined } from 'sequelize';
+import { NotificationInfo } from './notify';
 
 export class AuthInfo {
   ip?: string;
   type: AuthDataType;
-  info?: any;
+  info?: AuthModelInfo;
   id?: number;
 
   constructor(options: AuthInfo) {
@@ -25,9 +26,25 @@ export enum AuthDataType {
   'authToken' = 'authToken',
   'notification' = 'notification',
   'removeLogFrequency' = 'removeLogFrequency',
+  'systemConfig' = 'systemConfig',
 }
 
-interface AuthInstance extends Model<AuthInfo, AuthInfo>, AuthInfo {}
+export interface SystemConfigInfo {
+  logRemoveFrequency?: number;
+  cronConcurrency?: number;
+}
+
+export interface LoginLogInfo {
+  timestamp?: number;
+  address?: string;
+  ip?: string;
+  platform?: string;
+  status?: LoginStatus,
+}
+
+export type AuthModelInfo = SystemConfigInfo & Partial<NotificationInfo> & LoginLogInfo;
+
+interface AuthInstance extends Model<AuthInfo, AuthInfo>, AuthInfo { }
 export const AuthModel = sequelize.define<AuthInstance>('Auth', {
   ip: DataTypes.STRING,
   type: DataTypes.STRING,

back/data/cron.ts

@@ -44,9 +44,9 @@ export class Crontab {
 
 export enum CrontabStatus {
   'running',
+  'queued',
   'idle',
   'disabled',
-  'queued',
 }
 
 interface CronInstance extends Model<Crontab, Crontab>, Crontab {}

back/loaders/initData.ts

@@ -32,7 +32,7 @@ export default async () => {
   // 初始化更新所有任务状态为空闲
   await CrontabModel.update(
     { status: CrontabStatus.idle },
-    { where: { status: [CrontabStatus.running, CrontabStatus.queued] } },
+    { where: { status: { [Op.ne]: CrontabStatus.disabled } } },
   );
 
   // 初始化时安装所有处于安装中，安装成功，安装失败的依赖

back/loaders/initFile.ts

@@ -18,10 +18,13 @@ const confFile = path.join(configPath, 'config.sh');
 const authConfigFile = path.join(configPath, 'auth.json');
 const sampleConfigFile = path.join(samplePath, 'config.sample.sh');
 const sampleAuthFile = path.join(samplePath, 'auth.sample.json');
+const sampleTaskShellFile = path.join(samplePath, 'task.sample.sh');
 const sampleNotifyJsFile = path.join(samplePath, 'notify.js');
 const sampleNotifyPyFile = path.join(samplePath, 'notify.py');
 const scriptNotifyJsFile = path.join(scriptPath, 'sendNotify.js');
 const scriptNotifyPyFile = path.join(scriptPath, 'notify.py');
+const TaskBeforeFile = path.join(configPath, 'task_before.sh');
+const TaskAfterFile = path.join(configPath, 'task_after.sh');
 const homedir = os.homedir();
 const sshPath = path.resolve(homedir, '.ssh');
 const sshdPath = path.join(dataPath, 'ssh.d');
@@ -39,6 +42,8 @@ export default async () => {
   const tmpDirExist = await fileExist(tmpPath);
   const scriptNotifyJsFileExist = await fileExist(scriptNotifyJsFile);
   const scriptNotifyPyFileExist = await fileExist(scriptNotifyPyFile);
+  const TaskBeforeFileExist = await fileExist(TaskBeforeFile);
+  const TaskAfterFileExist = await fileExist(TaskAfterFile);
 
   if (!configDirExist) {
     fs.mkdirSync(configPath);
@@ -89,6 +94,14 @@ export default async () => {
     fs.writeFileSync(scriptNotifyPyFile, fs.readFileSync(sampleNotifyPyFile));
   }
 
+  if (!TaskBeforeFileExist) {
+    fs.writeFileSync(TaskBeforeFile, fs.readFileSync(sampleTaskShellFile));
+  }
+
+  if (!TaskAfterFileExist) {
+    fs.writeFileSync(TaskAfterFile, fs.readFileSync(sampleTaskShellFile));
+  }
+
   dotenv.config({ path: confFile });
 
   Logger.info('✌️ Init file down');

back/loaders/initTask.ts

@@ -29,7 +29,7 @@ export default async () => {
   });
 
   // 运行删除日志任务
-  const data = await systemService.getLogRemoveFrequency();
+  const data = await systemService.getSystemConfig();
   if (data && data.info && data.info.frequency) {
     const rmlogCron = {
       id: data.id,

back/services/cron.ts

@@ -7,21 +7,21 @@ import fs from 'fs';
 import cron_parser from 'cron-parser';
 import {
   getFileContentByName,
-  concurrentRun,
   fileExist,
   killTask,
 } from '../config/util';
 import { promises, existsSync } from 'fs';
-import { Op, where, col as colFn, FindOptions } from 'sequelize';
+import { Op, where, col as colFn, FindOptions, fn } from 'sequelize';
 import path from 'path';
 import { TASK_PREFIX, QL_PREFIX } from '../config/const';
 import cronClient from '../schedule/client';
-import { runWithCpuLimit } from '../shared/pLimit';
+import taskLimit from '../shared/pLimit';
 import { spawn } from 'cross-spawn';
+import { Fn } from 'sequelize/types/utils';
 
 @Service()
 export default class CronService {
-  constructor(@Inject('logger') private logger: winston.Logger) {}
+  constructor(@Inject('logger') private logger: winston.Logger) { }
 
   private isSixCron(cron: Crontab) {
     const { schedule } = cron;
@@ -281,7 +281,7 @@ export default class CronService {
     }
   }
 
-  private formatViewSort(order: string[][], viewQuery: any) {
+  private formatViewSort(order: (string | Fn)[][], viewQuery: any) {
     if (viewQuery.sorts && viewQuery.sorts.length > 0) {
       for (const { property, type } of viewQuery.sorts) {
         order.unshift([property, type]);
@@ -387,7 +387,7 @@ export default class CronService {
   }
 
   private async runSingle(cronId: number): Promise<number> {
-    return runWithCpuLimit(() => {
+    return taskLimit.runWithCpuLimit(() => {
       return new Promise(async (resolve: any) => {
         const cron = await this.getDb({ id: cronId });
         if (cron.status !== CrontabStatus.queued) {

back/services/dependence.ts

@@ -14,7 +14,7 @@ import SockService from './sock';
 import { FindOptions, Op } from 'sequelize';
 import { concurrentRun } from '../config/util';
 import dayjs from 'dayjs';
-import { runOneByOne, runWithCpuLimit } from '../shared/pLimit';
+import taskLimit from '../shared/pLimit';
 
 @Service()
 export default class DependenceService {
@@ -147,7 +147,7 @@ export default class DependenceService {
     isInstall: boolean = true,
     force: boolean = false,
   ) {
-    return runOneByOne(() => {
+    return taskLimit.runOneByOne(() => {
       return new Promise(async (resolve) => {
         const depIds = [dependency.id!];
         const status = isInstall

back/services/schedule.ts

@@ -9,7 +9,7 @@ import {
   Task,
 } from 'toad-scheduler';
 import dayjs from 'dayjs';
-import { runWithCpuLimit } from '../shared/pLimit';
+import taskLimit from '../shared/pLimit';
 import { spawn } from 'cross-spawn';
 
 interface ScheduleTaskType {
@@ -49,7 +49,7 @@ export default class ScheduleService {
     callbacks: TaskCallbacks = {},
     completionTime: 'start' | 'end' = 'end',
   ) {
-    return runWithCpuLimit(() => {
+    return taskLimit.runWithCpuLimit(() => {
       return new Promise(async (resolve, reject) => {
         try {
           const startTime = dayjs();

back/services/system.ts

@@ -2,7 +2,7 @@ import { Service, Inject } from 'typedi';
 import winston from 'winston';
 import config from '../config';
 import * as fs from 'fs';
-import { AuthDataType, AuthInfo, AuthModel, LoginStatus } from '../data/auth';
+import { AuthDataType, AuthInfo, AuthModel, AuthModelInfo } from '../data/auth';
 import { NotificationInfo } from '../data/notify';
 import NotificationService from './notify';
 import ScheduleService, { TaskCallbacks } from './schedule';
@@ -16,6 +16,7 @@ import {
   parseVersion,
 } from '../config/util';
 import { TASK_COMMAND } from '../config/const';
+import taskLimit from '../shared/pLimit'
 
 @Service()
 export default class SystemService {
@@ -28,8 +29,8 @@ export default class SystemService {
     private sockService: SockService,
   ) {}
 
-  public async getLogRemoveFrequency() {
-    const doc = await this.getDb({ type: AuthDataType.removeLogFrequency });
+  public async getSystemConfig() {
+    const doc = await this.getDb({ type: AuthDataType.systemConfig });
     return doc || {};
   }
 
@@ -62,25 +63,30 @@ export default class SystemService {
     }
   }
 
-  public async updateLogRemoveFrequency(frequency: number) {
-    const oDoc = await this.getLogRemoveFrequency();
+  public async updateSystemConfig(info: AuthModelInfo) {
+    const oDoc = await this.getSystemConfig();
     const result = await this.updateAuthDb({
       ...oDoc,
-      type: AuthDataType.removeLogFrequency,
-      info: { frequency },
+      type: AuthDataType.systemConfig,
+      info,
     });
-    const cron = {
-      id: result.id,
-      name: '删除日志',
-      command: `ql rmlog ${frequency}`,
-    };
-    await this.scheduleService.cancelIntervalTask(cron);
-    if (frequency > 0) {
-      this.scheduleService.createIntervalTask(cron, {
-        days: frequency,
-      });
+    if (info.logRemoveFrequency) {
+      const cron = {
+        id: result.id,
+        name: '删除日志',
+        command: `ql rmlog ${info.logRemoveFrequency}`,
+      };
+      await this.scheduleService.cancelIntervalTask(cron);
+      if (info.logRemoveFrequency > 0) {
+        this.scheduleService.createIntervalTask(cron, {
+          days: info.logRemoveFrequency,
+        });
+      }
     }
-    return { code: 200, data: { ...cron } };
+    if (info.cronConcurrency) {
+      await taskLimit.setCustomLimit(info.cronConcurrency);
+    }
+    return { code: 200, data: info };
   }
 
   public async checkUpdate() {

back/services/user.ts

@@ -10,7 +10,7 @@ import config from '../config';
 import * as fs from 'fs';
 import jwt from 'jsonwebtoken';
 import { authenticator } from '@otplib/preset-default';
-import { AuthDataType, AuthInfo, AuthModel, LoginStatus } from '../data/auth';
+import { AuthDataType, AuthInfo, AuthModel, AuthModelInfo, LoginStatus } from '../data/auth';
 import { NotificationInfo } from '../data/notify';
 import NotificationService from './notify';
 import { Request } from 'express';
@@ -27,7 +27,7 @@ export default class UserService {
     @Inject('logger') private logger: winston.Logger,
     private scheduleService: ScheduleService,
     private sockService: SockService,
-  ) {}
+  ) { }
 
   public async login(
     payloads: {
@@ -119,8 +119,7 @@ export default class UserService {
         });
         await this.notificationService.notify(
           '登录通知',
-          `你于${dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss')}在 ${address} ${
-            req.platform
+          `你于${dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss')}在 ${address} ${req.platform
           }端 登录成功，ip地址 ${ip}`,
         );
         await this.getLoginLog();
@@ -148,8 +147,7 @@ export default class UserService {
         });
         await this.notificationService.notify(
           '登录通知',
-          `你于${dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss')}在 ${address} ${
-            req.platform
+          `你于${dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss')}在 ${address} ${req.platform
           }端 登录失败，ip地址 ${ip}`,
         );
         await this.getLoginLog();
@@ -187,12 +185,12 @@ export default class UserService {
     });
   }
 
-  public async getLoginLog(): Promise<AuthInfo[]> {
+  public async getLoginLog(): Promise<Array<AuthModelInfo | undefined>> {
     const docs = await AuthModel.findAll({
       where: { type: AuthDataType.loginLog },
     });
     if (docs && docs.length > 0) {
-      const result = docs.sort((a, b) => b.info.timestamp - a.info.timestamp);
+      const result = docs.sort((a, b) => b.info!.timestamp! - a.info!.timestamp!);
       if (result.length > 100) {
         await AuthModel.destroy({
           where: { id: result[result.length - 1].id },

back/shared/pLimit.ts

@@ -1,17 +1,37 @@
 import pLimit from "p-limit";
 import os from 'os';
+import { AuthDataType, AuthModel } from "../data/auth";
 
-const cpuLimit = pLimit(os.cpus().length);
-const oneLimit = pLimit(1);
+class TaskLimit {
+  private oneLimit = pLimit(1);
+  private cpuLimit = pLimit(Math.max(os.cpus().length, 4));
 
-export function runWithCpuLimit<T>(fn: () => Promise<T>): Promise<T> {
-  return cpuLimit(() => {
-    return fn();
-  });
+  constructor() {
+    this.setCustomLimit();
+  }
+
+  public async setCustomLimit(limit?: number) {
+    if (limit) {
+      this.cpuLimit = pLimit(limit);
+      return;
+    }
+    const doc = await AuthModel.findOne({ where: { type: AuthDataType.systemConfig } });
+    if (doc?.info?.cronConcurrency) {
+      this.cpuLimit = pLimit(doc?.info?.cronConcurrency);
+    }
+  }
+
+  public runWithCpuLimit<T>(fn: () => Promise<T>): Promise<T> {
+    return this.cpuLimit(() => {
+      return fn();
+    });
+  }
+
+  public runOneByOne<T>(fn: () => Promise<T>): Promise<T> {
+    return this.oneLimit(() => {
+      return fn();
+    });
+  }
 }
 
-export function runOneByOne<T>(fn: () => Promise<T>): Promise<T> {
-  return oneLimit(() => {
-    return fn();
-  });
-}
+export default new TaskLimit();

back/shared/runCron.ts

@@ -1,9 +1,9 @@
 import { spawn } from 'cross-spawn';
-import { runWithCpuLimit } from "./pLimit";
+import taskLimit from "./pLimit";
 import Logger from '../loaders/logger';
 
 export function runCron(cmd: string): Promise<number> {
-  return runWithCpuLimit(() => {
+  return taskLimit.runWithCpuLimit(() => {
     return new Promise(async (resolve: any) => {
       Logger.silly('运行命令: ' + cmd);
 

sample/config.sample.sh

@@ -91,6 +91,10 @@ export TG_API_HOST=""
 export DD_BOT_TOKEN=""
 export DD_BOT_SECRET=""
 
+## 企业微信反向代理地址
+## (环境变量名 QYWX_ORIGIN)
+export QYWX_ORIGIN=""
+
 ## 5. 企业微信机器人
 ## 官方说明文档：https://work.weixin.qq.com/api/doc/90000/90136/91770
 ## 下方填写密钥，企业微信推送 webhook 后面的 key

src/pages/crontab/index.tsx

@@ -60,9 +60,9 @@ const { Search } = Input;
 
 export enum CrontabStatus {
   'running',
+  'queued',
   'idle',
   'disabled',
-  'queued',
 }
 
 const CrontabSort: any = { 0: 0, 5: 1, 3: 2, 1: 3, 4: 4 };

src/pages/error/index.less

@@ -1,6 +1,5 @@
 .error-wrapper {
   display: flex;
-  align-items: center;
   justify-content: center;
   height: 100vh;
 

src/pages/error/index.tsx

@@ -81,7 +81,7 @@ const Error = () => {
           </Typography.Paragraph>
         </div>
       ) : (
-        <PageLoading style={{ paddingTop: 0 }} tip="启动中，请稍后..." />
+        <PageLoading tip="启动中，请稍后..." />
       )}
     </div>
   );

src/pages/setting/other.tsx

@@ -19,7 +19,10 @@ const Other = ({
   reloadTheme,
 }: Pick<SharedContext, 'socketMessage' | 'reloadTheme' | 'systemInfo'>) => {
   const defaultTheme = localStorage.getItem('qinglong_dark_theme') || 'auto';
-  const [logRemoveFrequency, setLogRemoveFrequency] = useState<number | null>();
+  const [systemConfig, setSystemConfig] = useState<{
+    logRemoveFrequency?: number | null;
+    cronConcurrency?: number | null;
+  }>();
   const [form] = Form.useForm();
 
   const {
@@ -45,13 +48,12 @@ const Other = ({
     reloadTheme();
   };
 
-  const getLogRemoveFrequency = () => {
+  const getSystemConfig = () => {
     request
-      .get(`${config.apiPrefix}system/log/remove`)
+      .get(`${config.apiPrefix}system/config`)
       .then(({ code, data }) => {
         if (code === 200 && data.info) {
-          const { frequency } = data.info;
-          setLogRemoveFrequency(frequency);
+          setSystemConfig(data.info);
         }
       })
       .catch((error: any) => {
@@ -59,25 +61,23 @@ const Other = ({
       });
   };
 
-  const updateRemoveLogFrequency = () => {
-    setTimeout(() => {
-      request
-        .put(`${config.apiPrefix}system/log/remove`, {
-          data: { frequency: logRemoveFrequency },
-        })
-        .then(({ code, data }) => {
-          if (code === 200) {
-            message.success('更新成功');
-          }
-        })
-        .catch((error: any) => {
-          console.log(error);
-        });
-    });
+  const updateSystemConfig = () => {
+    request
+      .put(`${config.apiPrefix}system/config`, {
+        data: { ...systemConfig },
+      })
+      .then(({ code, data }) => {
+        if (code === 200) {
+          message.success('更新成功');
+        }
+      })
+      .catch((error: any) => {
+        console.log(error);
+      });
   };
 
   useEffect(() => {
-    getLogRemoveFrequency();
+    getSystemConfig();
   }, []);
 
   return (
@@ -100,12 +100,29 @@ const Other = ({
           <InputNumber
             addonBefore="每"
             addonAfter="天"
-            style={{ width: 150 }}
+            style={{ width: 142 }}
             min={0}
-            value={logRemoveFrequency}
-            onChange={(value) => setLogRemoveFrequency(value)}
+            value={systemConfig?.logRemoveFrequency}
+            onChange={(value) => {
+              setSystemConfig({ ...systemConfig, logRemoveFrequency: value });
+            }}
           />
-          <Button type="primary" onClick={updateRemoveLogFrequency}>
+          <Button type="primary" onClick={updateSystemConfig}>
+            确认
+          </Button>
+        </Input.Group>
+      </Form.Item>
+      <Form.Item label="定时任务并发数" name="frequency">
+        <Input.Group compact>
+          <InputNumber
+            style={{ width: 142 }}
+            min={1}
+            value={systemConfig?.cronConcurrency}
+            onChange={(value) => {
+              setSystemConfig({ ...systemConfig, cronConcurrency: value });
+            }}
+          />
+          <Button type="primary" onClick={updateSystemConfig}>
             确认
           </Button>
         </Input.Group>