From 702c3160ec41c8a990977cf7eb78735c901c948c Mon Sep 17 00:00:00 2001 From: whyour Date: Sat, 1 Jul 2023 15:26:20 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=B9=B6=E5=8F=91=E9=80=BB?= =?UTF-8?q?=E8=BE=91=EF=BC=8C=E7=B3=BB=E7=BB=9F=E8=AE=BE=E7=BD=AE=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E5=B9=B6=E5=8F=91?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- back/api/system.ts | 13 +++---- back/data/auth.ts | 21 ++++++++++-- back/data/cron.ts | 2 +- back/loaders/initData.ts | 2 +- back/loaders/initFile.ts | 13 +++++++ back/loaders/initTask.ts | 2 +- back/services/cron.ts | 12 +++---- back/services/dependence.ts | 4 +-- back/services/schedule.ts | 4 +-- back/services/system.ts | 42 +++++++++++++---------- back/services/user.ts | 14 ++++---- back/shared/pLimit.ts | 42 +++++++++++++++++------ back/shared/runCron.ts | 4 +-- sample/config.sample.sh | 4 +++ src/pages/crontab/index.tsx | 2 +- src/pages/error/index.less | 1 - src/pages/error/index.tsx | 2 +- src/pages/setting/other.tsx | 67 +++++++++++++++++++++++-------------- 18 files changed, 163 insertions(+), 88 deletions(-) diff --git a/back/api/system.ts b/back/api/system.ts index a324a3e1..1ef84b58 100644 --- a/back/api/system.ts +++ b/back/api/system.ts @@ -70,12 +70,12 @@ export default (app: Router) => { }); route.get( - '/log/remove', + '/config', async (req: Request, res: Response, next: NextFunction) => { const logger: Logger = Container.get('logger'); try { const systemService = Container.get(SystemService); - const data = await systemService.getLogRemoveFrequency(); + const data = await systemService.getSystemConfig(); res.send({ code: 200, data }); } catch (e) { return next(e); @@ -84,18 +84,19 @@ export default (app: Router) => { ); route.put( - '/log/remove', + '/config', celebrate({ body: Joi.object({ - frequency: Joi.number().required(), + logRemoveFrequency: Joi.number().optional().allow(null), + cronConcurrency: Joi.number().optional().allow(null), }), }), async (req: Request, res: Response, next: NextFunction) => { const logger: Logger = Container.get('logger'); try { const systemService = Container.get(SystemService); - const result = await systemService.updateLogRemoveFrequency( - req.body.frequency, + const result = await systemService.updateSystemConfig( + req.body, ); res.send(result); } catch (e) { diff --git a/back/data/auth.ts b/back/data/auth.ts index e3000f43..a24187a2 100644 --- a/back/data/auth.ts +++ b/back/data/auth.ts @@ -1,10 +1,11 @@ import { sequelize } from '.'; import { DataTypes, Model, ModelDefined } from 'sequelize'; +import { NotificationInfo } from './notify'; export class AuthInfo { ip?: string; type: AuthDataType; - info?: any; + info?: AuthModelInfo; id?: number; constructor(options: AuthInfo) { @@ -25,9 +26,25 @@ export enum AuthDataType { 'authToken' = 'authToken', 'notification' = 'notification', 'removeLogFrequency' = 'removeLogFrequency', + 'systemConfig' = 'systemConfig', } -interface AuthInstance extends Model, AuthInfo {} +export interface SystemConfigInfo { + logRemoveFrequency?: number; + cronConcurrency?: number; +} + +export interface LoginLogInfo { + timestamp?: number; + address?: string; + ip?: string; + platform?: string; + status?: LoginStatus, +} + +export type AuthModelInfo = SystemConfigInfo & Partial & LoginLogInfo; + +interface AuthInstance extends Model, AuthInfo { } export const AuthModel = sequelize.define('Auth', { ip: DataTypes.STRING, type: DataTypes.STRING, diff --git a/back/data/cron.ts b/back/data/cron.ts index 81198d89..3fc7fe4e 100644 --- a/back/data/cron.ts +++ b/back/data/cron.ts @@ -44,9 +44,9 @@ export class Crontab { export enum CrontabStatus { 'running', + 'queued', 'idle', 'disabled', - 'queued', } interface CronInstance extends Model, Crontab {} diff --git a/back/loaders/initData.ts b/back/loaders/initData.ts index a3e9541c..3b6dfc07 100644 --- a/back/loaders/initData.ts +++ b/back/loaders/initData.ts @@ -32,7 +32,7 @@ export default async () => { // 初始化更新所有任务状态为空闲 await CrontabModel.update( { status: CrontabStatus.idle }, - { where: { status: [CrontabStatus.running, CrontabStatus.queued] } }, + { where: { status: { [Op.ne]: CrontabStatus.disabled } } }, ); // 初始化时安装所有处于安装中，安装成功，安装失败的依赖 diff --git a/back/loaders/initFile.ts b/back/loaders/initFile.ts index 6937c202..e076f425 100644 --- a/back/loaders/initFile.ts +++ b/back/loaders/initFile.ts @@ -18,10 +18,13 @@ const confFile = path.join(configPath, 'config.sh'); const authConfigFile = path.join(configPath, 'auth.json'); const sampleConfigFile = path.join(samplePath, 'config.sample.sh'); const sampleAuthFile = path.join(samplePath, 'auth.sample.json'); +const sampleTaskShellFile = path.join(samplePath, 'task.sample.sh'); const sampleNotifyJsFile = path.join(samplePath, 'notify.js'); const sampleNotifyPyFile = path.join(samplePath, 'notify.py'); const scriptNotifyJsFile = path.join(scriptPath, 'sendNotify.js'); const scriptNotifyPyFile = path.join(scriptPath, 'notify.py'); +const TaskBeforeFile = path.join(configPath, 'task_before.sh'); +const TaskAfterFile = path.join(configPath, 'task_after.sh'); const homedir = os.homedir(); const sshPath = path.resolve(homedir, '.ssh'); const sshdPath = path.join(dataPath, 'ssh.d'); @@ -39,6 +42,8 @@ export default async () => { const tmpDirExist = await fileExist(tmpPath); const scriptNotifyJsFileExist = await fileExist(scriptNotifyJsFile); const scriptNotifyPyFileExist = await fileExist(scriptNotifyPyFile); + const TaskBeforeFileExist = await fileExist(TaskBeforeFile); + const TaskAfterFileExist = await fileExist(TaskAfterFile); if (!configDirExist) { fs.mkdirSync(configPath); @@ -89,6 +94,14 @@ export default async () => { fs.writeFileSync(scriptNotifyPyFile, fs.readFileSync(sampleNotifyPyFile)); } + if (!TaskBeforeFileExist) { + fs.writeFileSync(TaskBeforeFile, fs.readFileSync(sampleTaskShellFile)); + } + + if (!TaskAfterFileExist) { + fs.writeFileSync(TaskAfterFile, fs.readFileSync(sampleTaskShellFile)); + } + dotenv.config({ path: confFile }); Logger.info('✌️ Init file down'); diff --git a/back/loaders/initTask.ts b/back/loaders/initTask.ts index b219dbf9..06441242 100644 --- a/back/loaders/initTask.ts +++ b/back/loaders/initTask.ts @@ -29,7 +29,7 @@ export default async () => { }); // 运行删除日志任务 - const data = await systemService.getLogRemoveFrequency(); + const data = await systemService.getSystemConfig(); if (data && data.info && data.info.frequency) { const rmlogCron = { id: data.id, diff --git a/back/services/cron.ts b/back/services/cron.ts index 1e3559f5..47822adb 100644 --- a/back/services/cron.ts +++ b/back/services/cron.ts @@ -7,21 +7,21 @@ import fs from 'fs'; import cron_parser from 'cron-parser'; import { getFileContentByName, - concurrentRun, fileExist, killTask, } from '../config/util'; import { promises, existsSync } from 'fs'; -import { Op, where, col as colFn, FindOptions } from 'sequelize'; +import { Op, where, col as colFn, FindOptions, fn } from 'sequelize'; import path from 'path'; import { TASK_PREFIX, QL_PREFIX } from '../config/const'; import cronClient from '../schedule/client'; -import { runWithCpuLimit } from '../shared/pLimit'; +import taskLimit from '../shared/pLimit'; import { spawn } from 'cross-spawn'; +import { Fn } from 'sequelize/types/utils'; @Service() export default class CronService { - constructor(@Inject('logger') private logger: winston.Logger) {} + constructor(@Inject('logger') private logger: winston.Logger) { } private isSixCron(cron: Crontab) { const { schedule } = cron; @@ -281,7 +281,7 @@ export default class CronService { } } - private formatViewSort(order: string[][], viewQuery: any) { + private formatViewSort(order: (string | Fn)[][], viewQuery: any) { if (viewQuery.sorts && viewQuery.sorts.length > 0) { for (const { property, type } of viewQuery.sorts) { order.unshift([property, type]); @@ -387,7 +387,7 @@ export default class CronService { } private async runSingle(cronId: number): Promise { - return runWithCpuLimit(() => { + return taskLimit.runWithCpuLimit(() => { return new Promise(async (resolve: any) => { const cron = await this.getDb({ id: cronId }); if (cron.status !== CrontabStatus.queued) { diff --git a/back/services/dependence.ts b/back/services/dependence.ts index 5323882f..6b76f7b0 100644 --- a/back/services/dependence.ts +++ b/back/services/dependence.ts @@ -14,7 +14,7 @@ import SockService from './sock'; import { FindOptions, Op } from 'sequelize'; import { concurrentRun } from '../config/util'; import dayjs from 'dayjs'; -import { runOneByOne, runWithCpuLimit } from '../shared/pLimit'; +import taskLimit from '../shared/pLimit'; @Service() export default class DependenceService { @@ -147,7 +147,7 @@ export default class DependenceService { isInstall: boolean = true, force: boolean = false, ) { - return runOneByOne(() => { + return taskLimit.runOneByOne(() => { return new Promise(async (resolve) => { const depIds = [dependency.id!]; const status = isInstall diff --git a/back/services/schedule.ts b/back/services/schedule.ts index 4eac4fc1..a0d157a9 100644 --- a/back/services/schedule.ts +++ b/back/services/schedule.ts @@ -9,7 +9,7 @@ import { Task, } from 'toad-scheduler'; import dayjs from 'dayjs'; -import { runWithCpuLimit } from '../shared/pLimit'; +import taskLimit from '../shared/pLimit'; import { spawn } from 'cross-spawn'; interface ScheduleTaskType { @@ -49,7 +49,7 @@ export default class ScheduleService { callbacks: TaskCallbacks = {}, completionTime: 'start' | 'end' = 'end', ) { - return runWithCpuLimit(() => { + return taskLimit.runWithCpuLimit(() => { return new Promise(async (resolve, reject) => { try { const startTime = dayjs(); diff --git a/back/services/system.ts b/back/services/system.ts index 8bf00a5d..e754a51d 100644 --- a/back/services/system.ts +++ b/back/services/system.ts @@ -2,7 +2,7 @@ import { Service, Inject } from 'typedi'; import winston from 'winston'; import config from '../config'; import * as fs from 'fs'; -import { AuthDataType, AuthInfo, AuthModel, LoginStatus } from '../data/auth'; +import { AuthDataType, AuthInfo, AuthModel, AuthModelInfo } from '../data/auth'; import { NotificationInfo } from '../data/notify'; import NotificationService from './notify'; import ScheduleService, { TaskCallbacks } from './schedule'; @@ -16,6 +16,7 @@ import { parseVersion, } from '../config/util'; import { TASK_COMMAND } from '../config/const'; +import taskLimit from '../shared/pLimit' @Service() export default class SystemService { @@ -28,8 +29,8 @@ export default class SystemService { private sockService: SockService, ) {} - public async getLogRemoveFrequency() { - const doc = await this.getDb({ type: AuthDataType.removeLogFrequency }); + public async getSystemConfig() { + const doc = await this.getDb({ type: AuthDataType.systemConfig }); return doc || {}; } @@ -62,25 +63,30 @@ export default class SystemService { } } - public async updateLogRemoveFrequency(frequency: number) { - const oDoc = await this.getLogRemoveFrequency(); + public async updateSystemConfig(info: AuthModelInfo) { + const oDoc = await this.getSystemConfig(); const result = await this.updateAuthDb({ ...oDoc, - type: AuthDataType.removeLogFrequency, - info: { frequency }, + type: AuthDataType.systemConfig, + info, }); - const cron = { - id: result.id, - name: '删除日志', - command: `ql rmlog ${frequency}`, - }; - await this.scheduleService.cancelIntervalTask(cron); - if (frequency > 0) { - this.scheduleService.createIntervalTask(cron, { - days: frequency, - }); + if (info.logRemoveFrequency) { + const cron = { + id: result.id, + name: '删除日志', + command: `ql rmlog ${info.logRemoveFrequency}`, + }; + await this.scheduleService.cancelIntervalTask(cron); + if (info.logRemoveFrequency > 0) { + this.scheduleService.createIntervalTask(cron, { + days: info.logRemoveFrequency, + }); + } } - return { code: 200, data: { ...cron } }; + if (info.cronConcurrency) { + await taskLimit.setCustomLimit(info.cronConcurrency); + } + return { code: 200, data: info }; } public async checkUpdate() { diff --git a/back/services/user.ts b/back/services/user.ts index 844747cb..2a12b606 100644 --- a/back/services/user.ts +++ b/back/services/user.ts @@ -10,7 +10,7 @@ import config from '../config'; import * as fs from 'fs'; import jwt from 'jsonwebtoken'; import { authenticator } from '@otplib/preset-default'; -import { AuthDataType, AuthInfo, AuthModel, LoginStatus } from '../data/auth'; +import { AuthDataType, AuthInfo, AuthModel, AuthModelInfo, LoginStatus } from '../data/auth'; import { NotificationInfo } from '../data/notify'; import NotificationService from './notify'; import { Request } from 'express'; @@ -27,7 +27,7 @@ export default class UserService { @Inject('logger') private logger: winston.Logger, private scheduleService: ScheduleService, private sockService: SockService, - ) {} + ) { } public async login( payloads: { @@ -119,8 +119,7 @@ export default class UserService { }); await this.notificationService.notify( '登录通知', - `你于${dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss')}在 ${address} ${ - req.platform + `你于${dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss')}在 ${address} ${req.platform }端 登录成功，ip地址 ${ip}`, ); await this.getLoginLog(); @@ -148,8 +147,7 @@ export default class UserService { }); await this.notificationService.notify( '登录通知', - `你于${dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss')}在 ${address} ${ - req.platform + `你于${dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss')}在 ${address} ${req.platform }端 登录失败，ip地址 ${ip}`, ); await this.getLoginLog(); @@ -187,12 +185,12 @@ export default class UserService { }); } - public async getLoginLog(): Promise { + public async getLoginLog(): Promise> { const docs = await AuthModel.findAll({ where: { type: AuthDataType.loginLog }, }); if (docs && docs.length > 0) { - const result = docs.sort((a, b) => b.info.timestamp - a.info.timestamp); + const result = docs.sort((a, b) => b.info!.timestamp! - a.info!.timestamp!); if (result.length > 100) { await AuthModel.destroy({ where: { id: result[result.length - 1].id }, diff --git a/back/shared/pLimit.ts b/back/shared/pLimit.ts index a582c72a..07ee17ed 100644 --- a/back/shared/pLimit.ts +++ b/back/shared/pLimit.ts @@ -1,17 +1,37 @@ import pLimit from "p-limit"; import os from 'os'; +import { AuthDataType, AuthModel } from "../data/auth"; -const cpuLimit = pLimit(os.cpus().length); -const oneLimit = pLimit(1); +class TaskLimit { + private oneLimit = pLimit(1); + private cpuLimit = pLimit(Math.max(os.cpus().length, 4)); -export function runWithCpuLimit(fn: () => Promise): Promise { - return cpuLimit(() => { - return fn(); - }); + constructor() { + this.setCustomLimit(); + } + + public async setCustomLimit(limit?: number) { + if (limit) { + this.cpuLimit = pLimit(limit); + return; + } + const doc = await AuthModel.findOne({ where: { type: AuthDataType.systemConfig } }); + if (doc?.info?.cronConcurrency) { + this.cpuLimit = pLimit(doc?.info?.cronConcurrency); + } + } + + public runWithCpuLimit(fn: () => Promise): Promise { + return this.cpuLimit(() => { + return fn(); + }); + } + + public runOneByOne(fn: () => Promise): Promise { + return this.oneLimit(() => { + return fn(); + }); + } } -export function runOneByOne(fn: () => Promise): Promise { - return oneLimit(() => { - return fn(); - }); -} +export default new TaskLimit(); diff --git a/back/shared/runCron.ts b/back/shared/runCron.ts index cd11bbff..c9d2da9f 100644 --- a/back/shared/runCron.ts +++ b/back/shared/runCron.ts @@ -1,9 +1,9 @@ import { spawn } from 'cross-spawn'; -import { runWithCpuLimit } from "./pLimit"; +import taskLimit from "./pLimit"; import Logger from '../loaders/logger'; export function runCron(cmd: string): Promise { - return runWithCpuLimit(() => { + return taskLimit.runWithCpuLimit(() => { return new Promise(async (resolve: any) => { Logger.silly('运行命令: ' + cmd); diff --git a/sample/config.sample.sh b/sample/config.sample.sh index c5e4ab17..85782464 100644 --- a/sample/config.sample.sh +++ b/sample/config.sample.sh @@ -91,6 +91,10 @@ export TG_API_HOST="" export DD_BOT_TOKEN="" export DD_BOT_SECRET="" +## 企业微信反向代理地址 +## (环境变量名 QYWX_ORIGIN) +export QYWX_ORIGIN="" + ## 5. 企业微信机器人 ## 官方说明文档：https://work.weixin.qq.com/api/doc/90000/90136/91770 ## 下方填写密钥，企业微信推送 webhook 后面的 key diff --git a/src/pages/crontab/index.tsx b/src/pages/crontab/index.tsx index 95db490c..0192ce12 100644 --- a/src/pages/crontab/index.tsx +++ b/src/pages/crontab/index.tsx @@ -60,9 +60,9 @@ const { Search } = Input; export enum CrontabStatus { 'running', + 'queued', 'idle', 'disabled', - 'queued', } const CrontabSort: any = { 0: 0, 5: 1, 3: 2, 1: 3, 4: 4 }; diff --git a/src/pages/error/index.less b/src/pages/error/index.less index a1c5c974..cd3cddc1 100644 --- a/src/pages/error/index.less +++ b/src/pages/error/index.less @@ -1,6 +1,5 @@ .error-wrapper { display: flex; - align-items: center; justify-content: center; height: 100vh; diff --git a/src/pages/error/index.tsx b/src/pages/error/index.tsx index 6811ae47..c185e86b 100644 --- a/src/pages/error/index.tsx +++ b/src/pages/error/index.tsx @@ -81,7 +81,7 @@ const Error = () => { ) : ( - + )} ); diff --git a/src/pages/setting/other.tsx b/src/pages/setting/other.tsx index a216e24d..62b36e3e 100644 --- a/src/pages/setting/other.tsx +++ b/src/pages/setting/other.tsx @@ -19,7 +19,10 @@ const Other = ({ reloadTheme, }: Pick) => { const defaultTheme = localStorage.getItem('qinglong_dark_theme') || 'auto'; - const [logRemoveFrequency, setLogRemoveFrequency] = useState(); + const [systemConfig, setSystemConfig] = useState<{ + logRemoveFrequency?: number | null; + cronConcurrency?: number | null; + }>(); const [form] = Form.useForm(); const { @@ -45,13 +48,12 @@ const Other = ({ reloadTheme(); }; - const getLogRemoveFrequency = () => { + const getSystemConfig = () => { request - .get(`${config.apiPrefix}system/log/remove`) + .get(`${config.apiPrefix}system/config`) .then(({ code, data }) => { if (code === 200 && data.info) { - const { frequency } = data.info; - setLogRemoveFrequency(frequency); + setSystemConfig(data.info); } }) .catch((error: any) => { @@ -59,25 +61,23 @@ const Other = ({ }); }; - const updateRemoveLogFrequency = () => { - setTimeout(() => { - request - .put(`${config.apiPrefix}system/log/remove`, { - data: { frequency: logRemoveFrequency }, - }) - .then(({ code, data }) => { - if (code === 200) { - message.success('更新成功'); - } - }) - .catch((error: any) => { - console.log(error); - }); - }); + const updateSystemConfig = () => { + request + .put(`${config.apiPrefix}system/config`, { + data: { ...systemConfig }, + }) + .then(({ code, data }) => { + if (code === 200) { + message.success('更新成功'); + } + }) + .catch((error: any) => { + console.log(error); + }); }; useEffect(() => { - getLogRemoveFrequency(); + getSystemConfig(); }, []); return ( @@ -100,12 +100,29 @@ const Other = ({ setLogRemoveFrequency(value)} + value={systemConfig?.logRemoveFrequency} + onChange={(value) => { + setSystemConfig({ ...systemConfig, logRemoveFrequency: value }); + }} /> - + + + + + { + setSystemConfig({ ...systemConfig, cronConcurrency: value }); + }} + /> +