Compare commits

...

23 Commits

Author SHA1 Message Date
whyour ea4b474b83 更新版本 v2.17.10 2024-09-01 23:42:38 +08:00
whyour 53ef0fe296 修复订阅任务参数 2024-09-01 23:42:36 +08:00
whyour c573186e34 修复启动无法创建订阅任务 2024-09-01 22:47:29 +08:00
whyour c229eca315 修复依赖管理和脚本管理样式 2024-08-31 23:46:29 +08:00
whyour a41dd74048 修复 task_before/task_after 中换行后 crontab 不识别 2024-08-31 20:46:00 +08:00
whyour 459f465f3b 修复 shell 未定义变量错误 2024-08-31 20:23:37 +08:00
whyour b508e97dc1 修复依赖列表样式 2024-08-31 14:59:54 +08:00
whyour 7414a9d33d 执行任务增加结束日志 2024-08-31 14:43:44 +08:00
whyour a48d100b2d 修复表格样式 2024-08-30 00:30:31 +08:00
whyour a3044f9d29 修复 node 内置脚本运行错误 2024-08-28 00:28:25 +08:00
whyour 19cfc9e351 修复任务重复运行提示 2024-08-26 23:05:53 +08:00
whyour 6c61ac5106 修复 dockerfile 2024-08-26 00:56:55 +08:00
whyour 65f7483688 修复任务频繁运行通知 2024-08-25 16:28:32 +08:00
whyour 8b042d90f3 修复删除日志命令 2024-08-24 22:43:09 +08:00
whyour 4e5ad6d5f3 定时服务区分系统、订阅、脚本任务 2024-08-23 23:06:50 +08:00
whyour 8b8eae211b 增加任务重复运行提醒 2024-08-23 09:37:26 +08:00
whyour f4cb3eacf8 系统日志增加时间筛选和清空 2024-08-22 00:47:24 +08:00
whyour f6021c8157 增加内置 Python requests 模块 2024-08-19 23:06:16 +08:00
whyour 73601ca853 修复任务执行前命令字符转义 2024-08-19 22:27:09 +08:00
whyour 8218d4ba94 修复 Dockerfile 2024-08-18 17:26:33 +08:00
whyour d47f835531 更新 docker action 2024-08-18 17:03:49 +08:00
whyour 230a8f61b8 修复 smtp python 参数说明 2024-08-18 16:31:39 +08:00
whyour af5de8372c 修复 shell 变量初始化检查,更新 sentry 版本 2024-08-18 14:19:45 +08:00
47 changed files with 1441 additions and 397 deletions
+4 -4
View File
@@ -103,7 +103,7 @@ jobs:
cache: "pnpm"
- name: Setup timezone
uses: szenius/set-timezone@v1.2
uses: szenius/set-timezone@v2.0
with:
timezoneLinux: Asia/Shanghai
@@ -149,7 +149,7 @@ jobs:
- name: Build and push
id: docker_build
uses: docker/build-push-action@v5
uses: docker/build-push-action@v6
with:
build-args: |
MAINTAINER=${{ github.repository_owner }}
@@ -190,7 +190,7 @@ jobs:
cache: "pnpm"
- name: Setup timezone
uses: szenius/set-timezone@v1.2
uses: szenius/set-timezone@v2.0
with:
timezoneLinux: Asia/Shanghai
@@ -215,7 +215,7 @@ jobs:
- name: Build and push python3.10
id: docker_build_310
uses: docker/build-push-action@v5
uses: docker/build-push-action@v6
with:
build-args: |
MAINTAINER=${{ github.repository_owner }}
+1
View File
@@ -26,3 +26,4 @@
__pycache__
/shell/preload/env.*
/shell/preload/notify.*
/shell/preload/*-notify.json
+3 -2
View File
@@ -15,6 +15,7 @@ export default (app: Router) => {
const subscriptionService = Container.get(SubscriptionService);
const data = await subscriptionService.list(
req.query.searchValue as string,
req.query.ids as string,
);
return res.send({ code: 200, data });
} catch (e) {
@@ -212,8 +213,8 @@ export default (app: Router) => {
body: Joi.array().items(Joi.number().required()),
query: Joi.object({
force: Joi.boolean().optional(),
t: Joi.number()
})
t: Joi.number(),
}),
}),
async (req: Request, res: Response, next: NextFunction) => {
const logger: Logger = Container.get('logger');
+37 -8
View File
@@ -340,12 +340,41 @@ export default (app: Router) => {
},
);
route.get('/log', async (req: Request, res: Response, next: NextFunction) => {
try {
const systemService = Container.get(SystemService);
await systemService.getSystemLog(res);
} catch (e) {
return next(e);
}
});
route.get(
'/log',
celebrate({
query: {
startTime: Joi.string().allow('').optional(),
endTime: Joi.string().allow('').optional(),
t: Joi.string().optional(),
},
}),
async (req: Request, res: Response, next: NextFunction) => {
try {
const systemService = Container.get(SystemService);
await systemService.getSystemLog(
res,
req.query as {
startTime?: string;
endTime?: string;
},
);
} catch (e) {
return next(e);
}
},
);
route.delete(
'/log',
async (req: Request, res: Response, next: NextFunction) => {
try {
const systemService = Container.get(SystemService);
await systemService.deleteSystemLog();
res.send({ code: 200 });
} catch (e) {
return next(e);
}
},
);
};
+1 -2
View File
@@ -1,3 +1,4 @@
import './loaders/sentry'
import 'reflect-metadata'; // We need this in order to use @Decorators
import config from './config';
import express from 'express';
@@ -10,8 +11,6 @@ async function startServer() {
await require('./loaders/initFile').default();
await require('./loaders/sentry').default({ expressApp: app });
await require('./loaders/app').default({ expressApp: app });
const server = app
+2
View File
@@ -50,6 +50,7 @@ const extraFile = path.join(configPath, 'extra.sh');
const confBakDir = path.join(dataPath, 'config/bak/');
const sampleFile = path.join(samplePath, 'config.sample.sh');
const sqliteFile = path.join(samplePath, 'database.sqlite');
const systemNotifyFile = path.join(preloadPath, 'system-notify.json');
const authError = '错误的用户名密码,请重试';
const loginFaild = '请先登录!';
@@ -132,4 +133,5 @@ export default {
sqliteFile,
sshdPath,
systemLogPath,
systemNotifyFile,
};
+1 -1
View File
@@ -461,7 +461,7 @@ export async function parseVersion(path: string): Promise<IVersion> {
return load(await fs.readFile(path, 'utf8')) as IVersion;
}
export async function parseContentVersion(content: string): Promise<IVersion> {
export function parseContentVersion(content: string): IVersion {
return load(content) as IVersion;
}
+2 -2
View File
@@ -133,6 +133,8 @@ export default ({ app }: { app: Application }) => {
app.use(errors());
Sentry.setupExpressErrorHandler(app);
app.use(
(
err: Error & { status: number },
@@ -178,8 +180,6 @@ export default ({ app }: { app: Application }) => {
res: Response,
next: NextFunction,
) => {
Sentry.captureException(err);
res.status(err.status || 500);
res.json({
code: err.status || 500,
+7
View File
@@ -11,17 +11,24 @@ import { CrontabViewModel, CronViewType } from '../data/cronView';
import { initPosition } from '../data/env';
import { AuthDataType, SystemModel } from '../data/system';
import SystemService from '../services/system';
import UserService from '../services/user';
import { writeFile } from 'fs/promises';
export default async () => {
const cronService = Container.get(CronService);
const envService = Container.get(EnvService);
const dependenceService = Container.get(DependenceService);
const systemService = Container.get(SystemService);
const userService = Container.get(UserService);
// 初始化增加系统配置
await SystemModel.upsert({ type: AuthDataType.systemConfig });
await SystemModel.upsert({ type: AuthDataType.notification });
// 初始化通知配置
const notifyConfig = await userService.getNotificationMode();
await writeFile(config.systemNotifyFile, JSON.stringify(notifyConfig));
const installDependencies = () => {
// 初始化时安装所有处于安装中,安装成功,安装失败的依赖
DependenceModel.findAll({
+21 -9
View File
@@ -12,7 +12,10 @@ export default async () => {
const subscriptionService = Container.get(SubscriptionService);
// 生成内置token
let tokenCommand = `ts-node-transpile-only ${join(config.rootPath, 'back/token.ts')}`;
let tokenCommand = `ts-node-transpile-only ${join(
config.rootPath,
'back/token.ts',
)}`;
const tokenFile = join(config.rootPath, 'static/build/token.js');
if (await fileExist(tokenFile)) {
@@ -22,11 +25,16 @@ export default async () => {
id: NaN,
name: '生成token',
command: tokenCommand,
runOrigin: 'system',
} as ScheduleTaskType;
await scheduleService.cancelIntervalTask(cron);
scheduleService.createIntervalTask(cron, {
days: 28,
});
scheduleService.createIntervalTask(
cron,
{
days: 28,
},
true,
);
// 运行删除日志任务
const data = await systemService.getSystemConfig();
@@ -35,17 +43,21 @@ export default async () => {
id: data.id as number,
name: '删除日志',
command: `ql rmlog ${data.info.logRemoveFrequency}`,
runOrigin: 'system' as const,
};
await scheduleService.cancelIntervalTask(rmlogCron);
scheduleService.createIntervalTask(rmlogCron, {
days: data.info.logRemoveFrequency,
});
scheduleService.createIntervalTask(
rmlogCron,
{
days: data.info.logRemoveFrequency,
},
true,
);
}
// 运行所有订阅
await subscriptionService.setSshConfig();
const subs = await subscriptionService.list();
for (const sub of subs) {
subscriptionService.handleTask(sub, !sub.is_disabled, !sub.is_disabled);
subscriptionService.handleTask(sub.get({ plain: true }), !sub.is_disabled);
}
};
+20 -26
View File
@@ -1,32 +1,26 @@
import { Application } from 'express';
import * as Sentry from '@sentry/node';
import Logger from './logger';
import config from '../config';
import fs from 'fs';
import { parseVersion } from '../config/util';
import config from '../config';
import { parseContentVersion } from '../config/util';
export default async ({ expressApp }: { expressApp: Application }) => {
const { version } = await parseVersion(config.versionFile);
let version = '1.0.0';
try {
const content = fs.readFileSync(config.versionFile, 'utf-8');
({ version } = parseContentVersion(content));
} catch (error) {}
Sentry.init({
ignoreErrors: [
/SequelizeUniqueConstraintError/i,
/Validation error/i,
/UnauthorizedError/i,
/celebrate request validation failed/i,
],
dsn: 'https://8b5c84cfef3e22541bc84de0ed00497b@o1098464.ingest.sentry.io/6122819',
integrations: [
new Sentry.Integrations.Http({ tracing: true }),
new Sentry.Integrations.Express({ app: expressApp }),
],
tracesSampleRate: 0.8,
release: version,
});
Sentry.init({
ignoreErrors: [
/SequelizeUniqueConstraintError/i,
/Validation error/i,
/UnauthorizedError/i,
/celebrate request validation failed/i,
],
dsn: 'https://8b5c84cfef3e22541bc84de0ed00497b@o1098464.ingest.sentry.io/6122819',
tracesSampleRate: 0.5,
release: version,
});
expressApp.use(Sentry.Handlers.requestHandler());
expressApp.use(Sentry.Handlers.tracingHandler());
Logger.info('✌️ Sentry loaded');
console.log('✌️ Sentry loaded');
};
Logger.info('✌️ Sentry loaded');
console.log('✌️ Sentry loaded');
-3
View File
@@ -21,7 +21,6 @@ export default async ({ server }: { server: Server }) => {
if (data) {
const { token = '', tokens = {} } = safeJSONParse(data);
if (headerToken === token || tokens[platform] === headerToken) {
conn.write(JSON.stringify({ type: 'ping', message: 'hanhh' }));
sockService.addClient(conn);
conn.on('data', (message) => {
@@ -33,8 +32,6 @@ export default async ({ server }: { server: Server }) => {
});
return;
} else {
conn.write(JSON.stringify({ type: 'ping', message: 'whyour' }));
}
}
-1
View File
@@ -22,7 +22,6 @@ app.get('/api/health', (req, res) => {
app
.listen(config.publicPort, '0.0.0.0', async () => {
await require('./loaders/sentry').default({ expressApp: app });
await require('./loaders/db').default();
Logger.debug(`✌️ 公共服务启动成功!`);
+8 -8
View File
@@ -24,7 +24,7 @@ const addCron = (
);
if (extraSchedules?.length) {
extraSchedules.forEach(x => {
extraSchedules.forEach((x) => {
Logger.info(
'[schedule][创建定时任务], 任务ID: %s, 名称: %s, cron: %s, 执行命令: %s',
id,
@@ -32,21 +32,21 @@ const addCron = (
x.schedule,
command,
);
})
});
}
scheduleStacks.set(id, [
nodeSchedule.scheduleJob(id, schedule, async () => {
Logger.info(`[schedule][准备运行任务] 命令: ${command}`);
runCron(command, { name, schedule, extraSchedules });
runCron(command, item);
}),
...(extraSchedules?.length
? extraSchedules.map((x) =>
nodeSchedule.scheduleJob(id, x.schedule, async () => {
Logger.info(`[schedule][准备运行任务] 命令: ${command}`);
runCron(command, { name, schedule, extraSchedules });
}),
)
nodeSchedule.scheduleJob(id, x.schedule, async () => {
Logger.info(`[schedule][准备运行任务] 命令: ${command}`);
runCron(command, item);
}),
)
: []),
]);
}
+21 -4
View File
@@ -12,7 +12,7 @@ import {
getUniqPath,
safeJSONParse,
} from '../config/util';
import { Op, where, col as colFn, FindOptions, fn } from 'sequelize';
import { Op, where, col as colFn, FindOptions, fn, Order } from 'sequelize';
import path from 'path';
import { TASK_PREFIX, QL_PREFIX } from '../config/const';
import cronClient from '../schedule/client';
@@ -362,9 +362,9 @@ export default class CronService {
order.unshift([field, type]);
}
}
let condition: any = {
let condition: FindOptions<Crontab> = {
where: query,
order: order,
order: order as Order,
};
if (page && size) {
condition.offset = (page - 1) * size;
@@ -431,7 +431,7 @@ export default class CronService {
}
this.logger.info(
`[panel][开始执行任务] 参数 ${JSON.stringify(params)}`,
`[panel][开始执行任务] 参数: ${JSON.stringify(params)}`,
);
let { id, command, log_path } = cron;
@@ -459,13 +459,28 @@ export default class CronService {
await fs.appendFile(absolutePath, data.toString());
});
cp.stderr.on('data', async (data) => {
this.logger.info(
'[panel][执行任务失败] 命令: %s, 错误信息: %j',
command,
data.toString(),
);
await fs.appendFile(absolutePath, data.toString());
});
cp.on('error', async (err) => {
this.logger.error(
'[panel][创建任务失败] 命令: %s, 错误信息: %j',
command,
err,
);
await fs.appendFile(absolutePath, JSON.stringify(err));
});
cp.on('exit', async (code) => {
this.logger.info(
'[panel][执行任务结束] 参数: %s, 退出码: %j',
JSON.stringify(params),
code,
);
await CrontabModel.update(
{ status: CrontabStatus.idle, pid: undefined },
{ where: { id } },
@@ -549,11 +564,13 @@ export default class CronService {
if (tab.task_before) {
commandVariable += `task_before='${tab.task_before
.replace(/'/g, "'\\''")
.replace(/;? *\n/g, ';')
.trim()}' `;
}
if (tab.task_after) {
commandVariable += `task_after='${tab.task_after
.replace(/'/g, "'\\''")
.replace(/;? *\n/g, ';')
.trim()}' `;
}
+25 -2
View File
@@ -4,9 +4,11 @@ import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import nodemailer from 'nodemailer';
import { Inject, Service } from 'typedi';
import winston from 'winston';
import { parseBody, parseHeaders } from '../config/util';
import { parseBody, parseHeaders, safeJSONParse } from '../config/util';
import { NotificationInfo } from '../data/notify';
import UserService from './user';
import { readFile } from 'fs/promises';
import config from '../config';
@Service()
export default class NotificationService {
@@ -43,7 +45,28 @@ export default class NotificationService {
retry: 1,
};
constructor(@Inject('logger') private logger: winston.Logger) {}
constructor() {}
public async externalNotify(
title: string,
content: string,
): Promise<boolean | undefined> {
const { type, ...rest } = safeJSONParse(
await readFile(config.systemNotifyFile, 'utf-8'),
);
if (type) {
this.title = title;
this.content = content;
this.params = rest;
const notificationModeAction = this.modeMap.get(type);
try {
return await notificationModeAction?.call(this);
} catch (error: any) {
throw error;
}
}
return false;
}
public async notify(
title: string,
+43 -9
View File
@@ -17,6 +17,7 @@ export interface ScheduleTaskType {
command: string;
name?: string;
schedule?: string;
runOrigin: 'subscription' | 'system' | 'script';
}
export interface TaskCallbacks {
@@ -40,9 +41,13 @@ export default class ScheduleService {
private intervalSchedule = new ToadScheduler();
private maxBuffer = 200 * 1024 * 1024;
private taskLimitMap = {
system: 'runWithSystemLimit' as const,
script: 'runWithScriptLimit' as const,
subscription: 'runWithSubscriptionLimit' as const,
};
constructor(@Inject('logger') private logger: winston.Logger) { }
constructor(@Inject('logger') private logger: winston.Logger) {}
async runTask(
command: string,
@@ -51,12 +56,21 @@ export default class ScheduleService {
schedule?: string;
name?: string;
command?: string;
id: string;
runOrigin: 'subscription' | 'system' | 'script';
},
completionTime: 'start' | 'end' = 'end',
) {
return taskLimit.runWithCronLimit(() => {
const { runOrigin, ...others } = params;
return taskLimit[this.taskLimitMap[runOrigin]](others, () => {
return new Promise(async (resolve, reject) => {
this.logger.info(`[panel][开始执行任务] 参数 ${JSON.stringify({ ...params, command })}`);
this.logger.info(
`[panel][开始执行任务] 参数: ${JSON.stringify({
...others,
command,
})}`,
);
try {
const startTime = dayjs();
@@ -90,13 +104,21 @@ export default class ScheduleService {
});
cp.on('exit', async (code) => {
this.logger.info(
'[panel][执行任务结束] 参数: %s, 退出码: %j',
JSON.stringify({
...others,
command,
}),
code,
);
const endTime = dayjs();
await callbacks.onEnd?.(
cp,
endTime,
endTime.diff(startTime, 'seconds'),
);
resolve({ ...params, pid: cp.pid, code });
resolve({ ...others, pid: cp.pid, code });
});
} catch (error) {
this.logger.error(
@@ -111,7 +133,7 @@ export default class ScheduleService {
}
async createCronTask(
{ id = 0, command, name, schedule = '' }: ScheduleTaskType,
{ id = 0, command, name, schedule = '', runOrigin }: ScheduleTaskType,
callbacks?: TaskCallbacks,
runImmediately = false,
) {
@@ -131,6 +153,8 @@ export default class ScheduleService {
name,
schedule,
command,
id: _id,
runOrigin,
});
}),
);
@@ -140,6 +164,8 @@ export default class ScheduleService {
name,
schedule,
command,
id: _id,
runOrigin,
});
}
}
@@ -154,7 +180,7 @@ export default class ScheduleService {
}
async createIntervalTask(
{ id = 0, command, name = '' }: ScheduleTaskType,
{ id = 0, command, name = '', runOrigin }: ScheduleTaskType,
schedule: SimpleIntervalSchedule,
runImmediately = true,
callbacks?: TaskCallbacks,
@@ -172,11 +198,13 @@ export default class ScheduleService {
this.runTask(command, callbacks, {
name,
command,
id: _id,
runOrigin,
});
},
(err) => {
this.logger.error(
'[执行任务失败] 命令: %s, 错误信息: %j',
'[panel][执行任务失败] 命令: %s, 错误信息: %j',
command,
err,
);
@@ -195,13 +223,19 @@ export default class ScheduleService {
this.runTask(command, callbacks, {
name,
command,
id: _id,
runOrigin,
});
}
}
async cancelIntervalTask({ id = 0, name }: ScheduleTaskType) {
const _id = this.formatId(id);
this.logger.info('[取消interval任务], 任务ID: %s, 任务名: %s', _id, name);
this.logger.info(
'[panel][取消interval任务], 任务ID: %s, 任务名: %s',
_id,
name,
);
this.intervalSchedule.removeById(_id);
}
+3 -1
View File
@@ -7,6 +7,7 @@ import ScheduleService, { TaskCallbacks } from './schedule';
import config from '../config';
import { TASK_COMMAND } from '../config/const';
import { getFileContentByName, getPid, killTask, rmPath } from '../config/util';
import taskLimit from '../shared/pLimit';
@Service()
export default class ScriptService {
@@ -43,7 +44,7 @@ export default class ScriptService {
const pid = await this.scheduleService.runTask(
`real_time=true ${command}`,
this.taskCallbacks(filePath),
{ command },
{ command, id: relativePath.replace(/ /g, '-'), runOrigin: 'script' },
'start',
);
@@ -53,6 +54,7 @@ export default class ScriptService {
public async stopScript(filePath: string, pid: number) {
if (!pid) {
const relativePath = path.relative(config.scriptPath, filePath);
taskLimit.removeQueuedCron(relativePath.replace(/ /g, '-'));
pid = (await getPid(`${TASK_COMMAND} ${relativePath} now`)) as number;
}
try {
+17 -9
View File
@@ -3,6 +3,7 @@ import winston from 'winston';
import config from '../config';
import {
Subscription,
SubscriptionInstance,
SubscriptionModel,
SubscriptionStatus,
} from '../data/subscription';
@@ -29,6 +30,7 @@ import { LOG_END_SYMBOL } from '../config/const';
import { formatCommand, formatUrl } from '../config/subscription';
import { CrontabModel } from '../data/cron';
import CrontabService from './cron';
import taskLimit from '../shared/pLimit';
@Service()
export default class SubscriptionService {
@@ -40,8 +42,12 @@ export default class SubscriptionService {
private crontabService: CrontabService,
) {}
public async list(searchText?: string): Promise<Subscription[]> {
public async list(
searchText?: string,
ids?: string,
): Promise<SubscriptionInstance[]> {
let query = {};
const subIds = JSON.parse(ids || '[]');
if (searchText) {
const reg = {
[Op.or]: [
@@ -62,7 +68,7 @@ export default class SubscriptionService {
}
try {
const result = await SubscriptionModel.findAll({
where: query,
where: { ...query, ...(ids ? { id: subIds } : undefined) },
order: [
['is_disabled', 'ASC'],
['createdAt', 'DESC'],
@@ -87,7 +93,7 @@ export default class SubscriptionService {
this.scheduleService.cancelCronTask(doc as any);
needCreate &&
(await this.scheduleService.createCronTask(
doc as any,
{ ...doc, runOrigin: 'subscription' } as any,
this.taskCallbacks(doc),
runImmediately,
));
@@ -96,7 +102,7 @@ export default class SubscriptionService {
const { type, value } = doc.interval_schedule;
needCreate &&
(await this.scheduleService.createIntervalTask(
doc as any,
{ ...doc, runOrigin: 'subscription' } as any,
{ [type]: value } as SimpleIntervalSchedule,
runImmediately,
this.taskCallbacks(doc),
@@ -202,12 +208,12 @@ export default class SubscriptionService {
public async create(payload: Subscription): Promise<Subscription> {
const tab = new Subscription(payload);
const doc = await this.insert(tab);
await this.handleTask(doc);
await this.handleTask(doc.get({ plain: true }));
await this.setSshConfig();
return doc;
}
public async insert(payload: Subscription): Promise<Subscription> {
public async insert(payload: Subscription): Promise<SubscriptionInstance> {
return await SubscriptionModel.create(payload, { returning: true });
}
@@ -259,7 +265,7 @@ export default class SubscriptionService {
public async remove(ids: number[], query: { force?: boolean }) {
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
for (const doc of docs) {
await this.handleTask(doc, false);
await this.handleTask(doc.get({ plain: true }), false);
}
await SubscriptionModel.destroy({ where: { id: ids } });
await this.setSshConfig();
@@ -326,6 +332,8 @@ export default class SubscriptionService {
name: subscription.name,
schedule: subscription.schedule,
command,
id: String(subscription.id),
runOrigin: 'subscription',
});
}
@@ -334,7 +342,7 @@ export default class SubscriptionService {
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
await this.setSshConfig();
for (const doc of docs) {
await this.handleTask(doc, false);
await this.handleTask(doc.get({ plain: true }), false);
}
}
@@ -343,7 +351,7 @@ export default class SubscriptionService {
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
await this.setSshConfig();
for (const doc of docs) {
await this.handleTask(doc);
await this.handleTask(doc.get({ plain: true }));
}
}
+44 -7
View File
@@ -15,6 +15,7 @@ import {
parseVersion,
promiseExec,
readDirs,
rmPath,
} from '../config/util';
import {
DependenceModel,
@@ -34,6 +35,7 @@ import NotificationService from './notify';
import ScheduleService, { TaskCallbacks } from './schedule';
import SockService from './sock';
import os from 'os';
import dayjs from 'dayjs';
@Service()
export default class SystemService {
@@ -90,17 +92,22 @@ export default class SystemService {
info: { ...oDoc.info, ...info },
});
const cron = {
id: result.id || NaN,
id: result.id as number,
name: '删除日志',
command: `ql rmlog ${info.logRemoveFrequency}`,
runOrigin: 'system' as const,
};
if (oDoc.info?.logRemoveFrequency) {
await this.scheduleService.cancelIntervalTask(cron);
}
if (info.logRemoveFrequency && info.logRemoveFrequency > 0) {
this.scheduleService.createIntervalTask(cron, {
days: info.logRemoveFrequency,
});
this.scheduleService.createIntervalTask(
cron,
{
days: info.logRemoveFrequency,
},
true,
);
}
return { code: 200, data: info };
}
@@ -176,6 +183,8 @@ export default class SystemService {
},
{
command,
id: 'update-node-mirror',
runOrigin: 'system',
},
);
}
@@ -250,6 +259,8 @@ export default class SystemService {
},
{
command,
id: 'update-linux-mirror',
runOrigin: 'system',
},
);
}
@@ -266,7 +277,7 @@ export default class SystemService {
timeout: 30000,
},
);
lastVersionContent = await parseContentVersion(result.body);
lastVersionContent = parseContentVersion(result.body);
} catch (error) {}
if (!lastVersionContent) {
@@ -361,6 +372,8 @@ export default class SystemService {
}
this.scheduleService.runTask(`real_time=true ${command}`, callback, {
command,
id: command.replace(/ /g, '-'),
runOrigin: 'system',
});
}
@@ -409,9 +422,25 @@ export default class SystemService {
}
}
public async getSystemLog(res: Response) {
public async getSystemLog(
res: Response,
query: {
startTime?: string;
endTime?: string;
},
) {
const startTime = dayjs(query.startTime || undefined)
.startOf('d')
.valueOf();
const endTime = dayjs(query.endTime || undefined)
.endOf('d')
.valueOf();
const result = await readDirs(config.systemLogPath, config.systemLogPath);
const logs = result.reverse().filter((x) => x.title.endsWith('.log'));
const logs = result
.reverse()
.filter((x) => x.title.endsWith('.log'))
.filter((x) => x.mtime >= startTime && x.mtime <= endTime);
res.set({
'Content-Length': sum(logs.map((x) => x.size)),
});
@@ -433,4 +462,12 @@ export default class SystemService {
}
})(res, logs);
}
public async deleteSystemLog() {
const result = await readDirs(config.systemLogPath, config.systemLogPath);
const logs = result.reverse().filter((x) => x.title.endsWith('.log'));
for (const log of logs) {
await rmPath(path.join(config.systemLogPath, log.title));
}
}
}
+4 -1
View File
@@ -154,6 +154,7 @@ export default class UserService {
status: LoginStatus.success,
},
});
this.getLoginLog();
return {
code: 200,
data: { token, lastip, lastaddr, lastlogon, retries, platform },
@@ -182,6 +183,7 @@ export default class UserService {
status: LoginStatus.fail,
},
});
this.getLoginLog();
if (retries > 2) {
const waitTime = Math.round(Math.pow(3, retries + 1));
return {
@@ -215,8 +217,9 @@ export default class UserService {
(a, b) => b.info!.timestamp! - a.info!.timestamp!,
);
if (result.length > 100) {
const ids = result.slice(0, result.length - 100).map((x) => x.id!);
await SystemModel.destroy({
where: { id: result[result.length - 1].id },
where: { id: ids },
});
}
return result.map((x) => x.info);
+33
View File
@@ -0,0 +1,33 @@
import { Dependence } from '../data/dependence';
import { ICron } from '../protos/cron';
export type Override<
T,
K extends Partial<{ [P in keyof T]: any }> | string,
> = K extends string
? Omit<T, K> & { [P in keyof T]: T[P] | unknown }
: Omit<T, keyof K> & K;
export type TCron = Override<Partial<ICron>, { id: string }>;
export interface IDependencyFn<T> {
(): Promise<T>;
dependency?: Dependence;
}
export interface ICronFn<T> {
(): Promise<T>;
cron?: TCron;
}
export interface ISchedule {
schedule?: string;
name?: string;
command?: string;
id: string;
}
export interface IScheduleFn<T> {
(): Promise<T>;
schedule?: ISchedule;
}
+76 -5
View File
@@ -3,14 +3,20 @@ import os from 'os';
import { AuthDataType, SystemModel } from '../data/system';
import Logger from '../loaders/logger';
import { Dependence } from '../data/dependence';
import NotificationService from '../services/notify';
import {
ICronFn,
IDependencyFn,
ISchedule,
IScheduleFn,
TCron,
} from './interface';
interface IDependencyFn<T> {
(): Promise<T>;
dependency?: Dependence;
}
class TaskLimit {
private dependenyLimit = new PQueue({ concurrency: 1 });
private queuedDependencyIds = new Set<number>([]);
private queuedCrons = new Map<string, ICronFn<any>[]>();
private repeatCronNotifyMap = new Map<string, number>();
private updateLogLimit = new PQueue({ concurrency: 1 });
private cronLimit = new PQueue({
concurrency: Math.max(os.cpus().length, 4),
@@ -18,6 +24,15 @@ class TaskLimit {
private manualCronoLimit = new PQueue({
concurrency: Math.max(os.cpus().length, 4),
});
private subscriptionLimit = new PQueue({
concurrency: Math.max(os.cpus().length, 4),
});
private scriptLimit = new PQueue({
concurrency: Math.max(os.cpus().length, 4),
});
private systemLimit = new PQueue({
concurrency: Math.max(os.cpus().length, 4),
});
get cronLimitActiveCount() {
return this.cronLimit.pending;
@@ -31,6 +46,8 @@ class TaskLimit {
return [...this.queuedDependencyIds.values()][0];
}
private notificationService: NotificationService = new NotificationService();
constructor() {
this.setCustomLimit();
this.handleEvents();
@@ -71,6 +88,16 @@ class TaskLimit {
}
}
public removeQueuedCron(id: string) {
if (this.queuedCrons.has(id)) {
const runs = this.queuedCrons.get(id);
if (runs && runs.length > 0) {
runs.pop();
this.queuedCrons.set(id, runs);
}
}
}
public async setCustomLimit(limit?: number) {
if (limit) {
this.cronLimit.concurrency = limit;
@@ -88,9 +115,26 @@ class TaskLimit {
}
public async runWithCronLimit<T>(
fn: () => Promise<T>,
cron: TCron,
fn: ICronFn<T>,
options?: Partial<QueueAddOptions>,
): Promise<T | void> {
fn.cron = cron;
let runs = this.queuedCrons.get(cron.id);
const result = runs?.length ? [...runs, fn] : [fn];
const repeatTimes = this.repeatCronNotifyMap.get(cron.id) || 0;
if (result?.length > 5) {
if (repeatTimes < 3) {
this.repeatCronNotifyMap.set(cron.id, repeatTimes + 1);
this.notificationService.externalNotify(
'任务重复运行',
`任务:${cron.name},命令:${cron.command},定时:${cron.schedule},处于运行中的超过 5 个,请检查定时设置`,
);
}
Logger.warn(`[schedule][任务重复运行] 参数 ${JSON.stringify(cron)}`);
return;
}
this.queuedCrons.set(cron.id, result);
return this.cronLimit.add(fn, options);
}
@@ -101,6 +145,33 @@ class TaskLimit {
return this.manualCronoLimit.add(fn, options);
}
public async runWithSubscriptionLimit<T>(
schedule: TCron,
fn: IScheduleFn<T>,
options?: Partial<QueueAddOptions>,
): Promise<T | void> {
fn.schedule = schedule;
return this.subscriptionLimit.add(fn, options);
}
public async runWithSystemLimit<T>(
schedule: TCron,
fn: IScheduleFn<T>,
options?: Partial<QueueAddOptions>,
): Promise<T | void> {
fn.schedule = schedule;
return this.systemLimit.add(fn, options);
}
public async runWithScriptLimit<T>(
schedule: ISchedule,
fn: IScheduleFn<T>,
options?: Partial<QueueAddOptions>,
): Promise<T | void> {
fn.schedule = schedule;
return this.scriptLimit.add(fn, options);
}
public runDependeny<T>(
dependency: Dependence,
fn: IDependencyFn<T>,
+19 -4
View File
@@ -1,11 +1,17 @@
import { spawn } from 'cross-spawn';
import taskLimit from './pLimit';
import Logger from '../loaders/logger';
import { ICron } from '../protos/cron';
export function runCron(cmd: string, options?: { schedule: string; extraSchedules: Array<{ schedule: string }>; name: string }): Promise<number | void> {
return taskLimit.runWithCronLimit(() => {
export function runCron(cmd: string, cron: ICron): Promise<number | void> {
return taskLimit.runWithCronLimit(cron, () => {
return new Promise(async (resolve: any) => {
Logger.info(`[schedule][开始执行任务] 参数 ${JSON.stringify({ ...options, command: cmd })}`);
Logger.info(
`[schedule][开始执行任务] 参数 ${JSON.stringify({
...cron,
command: cmd,
})}`,
);
const cp = spawn(cmd, { shell: '/bin/bash' });
cp.stderr.on('data', (data) => {
@@ -24,7 +30,16 @@ export function runCron(cmd: string, options?: { schedule: string; extraSchedule
});
cp.on('exit', async (code) => {
resolve({ ...options, command: cmd, pid: cp.pid, code });
taskLimit.removeQueuedCron(cron.id);
Logger.info(
'[schedule][执行任务结束] 参数: %s, 退出码: %j',
JSON.stringify({
...cron,
command: cmd,
}),
code,
);
resolve({ ...cron, command: cmd, pid: cp.pid, code });
});
});
});
+4 -3
View File
@@ -1,4 +1,4 @@
FROM python:3.10-alpine3.18 as builder
FROM python:3.10-alpine3.18 AS builder
COPY package.json .npmrc pnpm-lock.yaml /tmp/build/
RUN set -x \
&& apk update \
@@ -15,7 +15,7 @@ ARG QL_URL=https://github.com/${QL_MAINTAINER}/qinglong.git
ARG QL_BRANCH=develop
ENV PNPM_HOME=/root/.local/share/pnpm \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.local/share/pnpm:/root/.local/share/pnpm/global/5/node_modules:$PNPM_HOME \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.local/share/pnpm:/root/.local/share/pnpm/global/5/node_modules \
NODE_PATH=/usr/local/bin:/usr/local/pnpm-global/5/node_modules:/usr/local/lib/node_modules:/root/.local/share/pnpm/global/5/node_modules \
LANG=C.UTF-8 \
SHELL=/bin/bash \
@@ -59,7 +59,8 @@ RUN set -x \
&& rm -rf /root/.pnpm-store \
&& rm -rf /root/.local/share/pnpm/store \
&& rm -rf /root/.cache \
&& ulimit -c 0
&& ulimit -c 0 \
&& pip3 install requests
ARG SOURCE_COMMIT
RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} \
+4 -3
View File
@@ -1,4 +1,4 @@
FROM python:3.11-alpine3.18 as builder
FROM python:3.11-alpine3.18 AS builder
COPY package.json .npmrc pnpm-lock.yaml /tmp/build/
RUN set -x \
&& apk update \
@@ -15,7 +15,7 @@ ARG QL_URL=https://github.com/${QL_MAINTAINER}/qinglong.git
ARG QL_BRANCH=develop
ENV PNPM_HOME=/root/.local/share/pnpm \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.local/share/pnpm:/root/.local/share/pnpm/global/5/node_modules:$PNPM_HOME \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.local/share/pnpm:/root/.local/share/pnpm/global/5/node_modules \
NODE_PATH=/usr/local/bin:/usr/local/pnpm-global/5/node_modules:/usr/local/lib/node_modules:/root/.local/share/pnpm/global/5/node_modules \
LANG=C.UTF-8 \
SHELL=/bin/bash \
@@ -59,7 +59,8 @@ RUN set -x \
&& rm -rf /root/.pnpm-store \
&& rm -rf /root/.local/share/pnpm/store \
&& rm -rf /root/.cache \
&& ulimit -c 0
&& ulimit -c 0 \
&& pip3 install requests
ARG SOURCE_COMMIT
RUN git clone --depth=1 -b ${QL_BRANCH} ${QL_URL} ${QL_DIR} \
+6 -4
View File
@@ -2,11 +2,11 @@
"private": true,
"scripts": {
"start": "concurrently -n w: npm:start:*",
"start:front": "max dev",
"start:back": "nodemon",
"start:update": "ts-node -P tsconfig.back.json ./back/update.ts",
"start:public": "ts-node -P tsconfig.back.json ./back/public.ts",
"start:rpc": "ts-node -P tsconfig.back.json ./back/schedule/index.ts",
"start:back": "nodemon",
"start:front": "max dev",
"build:front": "max build",
"build:back": "tsc -p tsconfig.back.json",
"panel": "npm run build:back && node static/build/app.js",
@@ -59,7 +59,7 @@
"dependencies": {
"@grpc/grpc-js": "^1.8.13",
"@otplib/preset-default": "^12.0.1",
"@sentry/node": "^7.12.1",
"@sentry/node": "^8.26.0",
"body-parser": "^1.19.2",
"celebrate": "^15.0.1",
"chokidar": "^3.5.3",
@@ -103,11 +103,13 @@
"ip2region": "2.3.0"
},
"devDependencies": {
"moment": "2.30.1",
"@ant-design/icons": "^4.7.0",
"@ant-design/pro-layout": "6.38.22",
"@monaco-editor/react": "4.2.1",
"@react-hook/resize-observer": "^1.2.6",
"@sentry/react": "^7.12.1",
"react-router-dom": "6.26.1",
"@sentry/react": "^8.26.0",
"@types/body-parser": "^1.19.2",
"@types/cors": "^2.8.12",
"@types/cross-spawn": "^6.0.2",
+710 -87
View File
File diff suppressed because it is too large Load Diff
+8
View File
@@ -175,8 +175,16 @@ export CHRONOCAT_QQ=""
export CHRONOCAT_TOKEN=""
## 16. SMTP
## JavaScript 参数
## 邮箱服务名称,比如126、163、Gmail、QQ等,支持列表 https://github.com/nodemailer/nodemailer/blob/master/lib/well-known/services.json
export SMTP_SERVICE=""
## Python 参数
## SMTP 发送邮件服务器,形如 smtp.exmail.qq.com:465
export SMTP_SERVER=""
## SMTP 发送邮件服务器是否使用 SSL,填写 true 或 false
export SMTP_SSL=""
## smtp_email 填写 SMTP 收发件邮箱,通知将会由自己发给自己
export SMTP_EMAIL=""
## smtp_password 填写 SMTP 登录密码,也可能为特殊口令,视具体邮件服务商说明而定
+1 -1
View File
@@ -817,7 +817,7 @@ function wePlusBotNotify(text, desp) {
const { WE_PLUS_BOT_TOKEN, WE_PLUS_BOT_RECEIVER, WE_PLUS_BOT_VERSION } =
push_config;
if (WE_PLUS_BOT_TOKEN) {
const template = 'txt';
let template = 'txt';
if (desp.length > 800) {
desp = desp.replace(/[\n\r]/g, '<br>');
template = 'html';
+9 -9
View File
@@ -6,12 +6,12 @@ create_token() {
if [[ -f $token_file ]]; then
token_command="node ${token_file}"
fi
token=$(eval "$token_command")
__ql_token__=$(eval "$token_command")
}
get_token() {
if [[ -f $file_auth_token ]]; then
token=$(cat $file_auth_token | jq -r .value)
__ql_token__=$(cat $file_auth_token | jq -r .value)
local expiration=$(cat $file_auth_token | jq -r .expiration)
local currentTimeStamp=$(date +%s)
if [[ $currentTimeStamp -ge $expiration ]]; then
@@ -43,7 +43,7 @@ add_cron_api() {
local api=$(
curl -s --noproxy "*" "http://0.0.0.0:5600/open/crons?t=$currentTimeStamp" \
-H "Accept: application/json" \
-H "Authorization: Bearer $token" \
-H "Authorization: Bearer ${__ql_token__}" \
-H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36" \
-H "Content-Type: application/json;charset=UTF-8" \
-H "Origin: http://0.0.0.0:5700" \
@@ -79,7 +79,7 @@ update_cron_api() {
curl -s --noproxy "*" "http://0.0.0.0:5600/open/crons?t=$currentTimeStamp" \
-X 'PUT' \
-H "Accept: application/json" \
-H "Authorization: Bearer $token" \
-H "Authorization: Bearer ${__ql_token__}" \
-H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36" \
-H "Content-Type: application/json;charset=UTF-8" \
-H "Origin: http://0.0.0.0:5700" \
@@ -111,7 +111,7 @@ update_cron_command_api() {
curl -s --noproxy "*" "http://0.0.0.0:5600/open/crons?t=$currentTimeStamp" \
-X 'PUT' \
-H "Accept: application/json" \
-H "Authorization: Bearer $token" \
-H "Authorization: Bearer ${__ql_token__}" \
-H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36" \
-H "Content-Type: application/json;charset=UTF-8" \
-H "Origin: http://0.0.0.0:5700" \
@@ -136,7 +136,7 @@ del_cron_api() {
curl -s --noproxy "*" "http://0.0.0.0:5600/open/crons?t=$currentTimeStamp" \
-X 'DELETE' \
-H "Accept: application/json" \
-H "Authorization: Bearer $token" \
-H "Authorization: Bearer ${__ql_token__}" \
-H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36" \
-H "Content-Type: application/json;charset=UTF-8" \
-H "Origin: http://0.0.0.0:5700" \
@@ -166,7 +166,7 @@ update_cron() {
curl -s --noproxy "*" "http://0.0.0.0:5600/open/crons/status?t=$currentTimeStamp" \
-X 'PUT' \
-H "Accept: application/json" \
-H "Authorization: Bearer $token" \
-H "Authorization: Bearer ${__ql_token__}" \
-H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36" \
-H "Content-Type: application/json;charset=UTF-8" \
-H "Origin: http://0.0.0.0:5700" \
@@ -190,7 +190,7 @@ notify_api() {
curl -s --noproxy "*" "http://0.0.0.0:5600/open/system/notify?t=$currentTimeStamp" \
-X 'PUT' \
-H "Accept: application/json" \
-H "Authorization: Bearer $token" \
-H "Authorization: Bearer ${__ql_token__}" \
-H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36" \
-H "Content-Type: application/json;charset=UTF-8" \
-H "Origin: http://0.0.0.0:5700" \
@@ -214,7 +214,7 @@ find_cron_api() {
local api=$(
curl -s --noproxy "*" "http://0.0.0.0:5600/open/crons/detail?$params&t=$currentTimeStamp" \
-H "Accept: application/json" \
-H "Authorization: Bearer $token" \
-H "Authorization: Bearer ${__ql_token__}" \
-H "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36" \
-H "Content-Type: application/json;charset=UTF-8" \
-H "Origin: http://0.0.0.0:5700" \
+12 -4
View File
@@ -210,15 +210,15 @@ run_else() {
check_file() {
isJsOrPythonFile="false"
if [[ $1 == *.js ]] || [[ $1 == *.py ]] || [[ $1 == *.pyc ]] || [[ $1 == *.ts ]]; then
if [[ $1 == *.js ]] || [[ $1 == *.mjs ]] || [[ $1 == *.py ]] || [[ $1 == *.pyc ]] || [[ $1 == *.ts ]]; then
isJsOrPythonFile="true"
fi
if [[ -f $file_env ]]; then
get_env_array
if [[ $isJsOrPythonFile == 'true' ]]; then
PREV_NODE_OPTIONS="${NODE_OPTIONS}"
PREV_PYTHONPATH="${PYTHONPATH}"
if [[ $1 == *.js ]] || [[ $1 == *.ts ]]; then
PREV_NODE_OPTIONS="${NODE_OPTIONS:=}"
PREV_PYTHONPATH="${PYTHONPATH:=}"
if [[ $1 == *.js ]] || [[ $1 == *.ts ]] || [[ $1 == *.mjs ]]; then
export NODE_OPTIONS="${NODE_OPTIONS} -r ${file_preload_js}"
else
export PYTHONPATH="${PYTHONPATH}:${dir_preload}:${dir_config}"
@@ -269,7 +269,15 @@ check_file "${task_shell_params[@]}"
if [[ $isJsOrPythonFile == 'false' ]]; then
run_task_before "${task_shell_params[@]}"
fi
set_u_on="false"
if set -o | grep -q 'nounset.*on'; then
set_u_on="true"
set +u
fi
main "${task_shell_params[@]}"
if [[ "$set_u_on" == 'true' ]]; then
set -u
fi
if [[ $isJsOrPythonFile == 'true' ]]; then
export NODE_OPTIONS="${PREV_NODE_OPTIONS}"
export PYTHONPATH="${PREV_PYTHONPATH}"
+12 -2
View File
@@ -37,8 +37,11 @@ function run() {
const fileName = process.argv[1].replace(`${dir_scripts}/`, '');
let command = `bash -c "source ${file_task_before} ${fileName}`;
if (task_before) {
const escapeTaskBefore = task_before.replace(/"/g, '\\"');
command = `${command} && echo -e '执行前置命令\n' && eval '${escapeTaskBefore}' && echo -e '\n执行前置命令结束\n'`;
const escapeTaskBefore = task_before
.replace(/"/g, '\\"')
.replace(/\$/g, '\\$');
command = `${command} && eval '${escapeTaskBefore}'`;
console.log('执行前置命令\n');
}
const res = execSync(
`${command} && echo -e '${splitStr}' && NODE_OPTIONS= node -p 'JSON.stringify(process.env)'"`,
@@ -52,6 +55,9 @@ function run() {
process.env[key] = newEnvObject[key];
}
console.log(output);
if (task_before) {
console.log('执行前置命令结束\n');
}
} catch (error) {
if (!error.message.includes('spawnSync /bin/sh E2BIG')) {
console.log(`run task before error: `, error);
@@ -68,6 +74,10 @@ function run() {
}
try {
if (!process.argv[1]) {
return;
}
run();
const { sendNotify } = require('./notify.js');
+6 -3
View File
@@ -44,8 +44,9 @@ def run():
task_before = os.getenv("task_before")
if task_before:
escape_task_before = task_before.replace('"', '\\"')
command += f" && echo -e '执行前置命令\n' && eval '{escape_task_before}' && echo -e '\n执行前置命令结束\n'"
escape_task_before = task_before.replace('"', '\\"').replace("$", "\\$")
command += f" && eval '{escape_task_before}'"
print("执行前置命令\n")
python_command = "PYTHONPATH= python3 -c 'import os, json; print(json.dumps(dict(os.environ)))'"
command += f" && echo -e '{split_str}' && {python_command}\""
@@ -59,6 +60,8 @@ def run():
os.environ[key] = value
print(output)
if task_before:
print("执行前置命令结束")
except subprocess.CalledProcessError as error:
print(f"run task before error: {error}")
@@ -82,7 +85,7 @@ def run():
try:
run()
from notify import send
class BaseApi:
+20 -18
View File
@@ -2,34 +2,37 @@
days=$1
## 删除运行脚本的旧日志
remove_js_log() {
local log_full_path_list=$(find $dir_log/ -name "*.log")
local log_full_path_list=$(find $dir_log -name "*.log")
local diff_time
for log in $log_full_path_list; do
local log_date=$(echo $log | awk -F "/" '{print $NF}' | cut -c1-10) #文件名比文件属性获得的日期要可靠
if [[ $(date +%s -d $log_date 2>/dev/null) ]]; then
local log_date=$(echo $log | awk -F "/" '{print $NF}' | cut -c1-10)
if ! [[ $log_date =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
if [[ $is_macos -eq 1 ]]; then
diff_time=$(($(date +%s) - $(date -j -f "%Y-%m-%d" "$log_date" +%s)))
log_date=$(stat -f %Sm -t "%Y-%m-%d" "$log")
else
diff_time=$(($(date +%s) - $(date +%s -d "$log_date")))
log_date=$(stat -c %y "$log" | cut -d ' ' -f 1)
fi
if [[ $diff_time -gt $((${days} * 86400)) ]]; then
local log_path=$(echo "$log" | sed "s,${dir_log}/,,g")
local result=$(find_cron_api "log_path=$log_path")
echo -e "查询文件 $log_path"
if [[ -z $result ]]; then
echo -e "删除中~"
rm -vf $log
else
echo -e "正在被 $result 使用,跳过~"
fi
fi
if [[ $is_macos -eq 1 ]]; then
diff_time=$(($(date +%s) - $(date -j -f "%Y-%m-%d" "$log_date" +%s)))
else
diff_time=$(($(date +%s) - $(date +%s -d "$log_date")))
fi
if [[ $diff_time -gt $((${days} * 86400)) ]]; then
local log_path=$(echo "$log" | sed "s,${dir_log}/,,g")
local result=$(find_cron_api "log_path=$log_path")
echo -e "查询文件 $log_path"
if [[ -z $result ]]; then
echo -e "删除中~"
rm -vf $log
else
echo -e "正在被 $result 使用,跳过~"
fi
fi
done
}
## 删除空文件夹
remove_empty_dir() {
cd $dir_log
for dir in $(ls); do
@@ -39,7 +42,6 @@ remove_empty_dir() {
done
}
## 运行
if [[ ${days} ]]; then
echo -e "查找旧日志文件中...\n"
remove_js_log
+6 -6
View File
@@ -5,7 +5,7 @@ export dir_root=$QL_DIR
export dir_tmp=$dir_root/.tmp
export dir_data=$dir_root/data
if [[ $QL_DATA_DIR ]]; then
if [[ ${QL_DATA_DIR:=} ]]; then
export dir_data="${QL_DATA_DIR%/}"
fi
@@ -84,7 +84,7 @@ import_config() {
command_timeout_time=${CommandTimeoutTime:-""}
file_extensions=${RepoFileExtensions:-"js py"}
proxy_url=${ProxyUrl:-""}
current_branch=${QL_BRANCH}
current_branch=${QL_BRANCH:-""}
if [[ -n "${DefaultCronRule}" ]]; then
default_cron="${DefaultCronRule}"
@@ -451,9 +451,9 @@ handle_task_start() {
run_task_before() {
. $file_task_before "$@"
if [[ $task_before ]]; then
if [[ ${task_before:=} ]]; then
echo -e "执行前置命令\n"
eval "$task_before" "$@"
eval "${task_before%;}" "$@"
echo -e "\n执行前置命令结束\n"
fi
}
@@ -461,9 +461,9 @@ run_task_before() {
run_task_after() {
. $file_task_after "$@"
if [[ $task_after ]]; then
if [[ ${task_after:=} ]]; then
echo -e "\n执行后置命令\n"
eval "$task_after" "$@"
eval "${task_after%;}" "$@"
echo -e "\n执行后置命令结束"
fi
}
+6 -8
View File
@@ -1,6 +1,5 @@
#!/usr/bin/env bash
## 导入通用变量与函数
dir_shell=$QL_DIR/shell
. $dir_shell/share.sh
. $dir_shell/api.sh
@@ -11,7 +10,6 @@ single_hanle() {
exit 1
}
## 选择python3还是node
define_program() {
local file_param=$1
if [[ $file_param == *.js ]] || [[ $file_param == *.mjs ]]; then
@@ -34,7 +32,7 @@ handle_log_path() {
file_param="task"
fi
if [[ -z $ID ]]; then
if [[ -z ${ID:=} ]]; then
ID=$(cat $list_crontab_user | grep -E "$cmd_task.* $file_param" | perl -pe "s|.*ID=(.*) $cmd_task.* $file_param\.*|\1|" | head -1 | awk -F " " '{print $1}')
fi
local suffix=""
@@ -62,17 +60,17 @@ handle_log_path() {
log_dir="${log_dir_tmp%.*}${suffix}"
log_path="$log_dir/$log_time.log"
if [[ $real_log_path ]]; then
if [[ ${real_log_path:=} ]]; then
log_path="$real_log_path"
fi
cmd="2>&1 | tee -a $dir_log/$log_path"
make_dir "$dir_log/$log_dir"
if [[ "$no_tee" == "true" ]]; then
if [[ "${no_tee:=}" == "true" ]]; then
cmd=">> $dir_log/$log_path 2>&1"
fi
if [[ "$real_time" == "true" ]]; then
if [[ "${real_time:=}" == "true" ]]; then
cmd=""
fi
}
@@ -124,8 +122,8 @@ while getopts ":lm:" opt; do
;;
esac
done
[[ $show_log ]] && shift $(($OPTIND - 1))
if [[ $max_time ]]; then
[[ ${show_log:=} ]] && shift $(($OPTIND - 1))
if [[ ${max_time:=} ]]; then
shift $(($OPTIND - 1))
command_timeout_time="$max_time"
fi
+5
View File
@@ -462,3 +462,8 @@ body[data-dark='true'] {
--antd-arrow-background-color: rgb(24, 26, 27);
}
}
.ant-tabs-content-holder {
flex: 1;
overflow-y: auto;
}
+15 -17
View File
@@ -267,22 +267,7 @@ const Crontab = () => {
{
title: intl.get('关联订阅'),
width: 185,
render: (text, record: any) =>
record.sub_id ? (
<Name
service={() =>
request.get(`${config.apiPrefix}subscriptions/${record.sub_id}`, {
onError: noop,
})
}
options={{
ready: record?.sub_id,
cacheKey: record.sub_id,
}}
/>
) : (
'-'
),
render: (text, record: any) => record?.subscription?.name || '-',
},
{
title: intl.get('操作'),
@@ -392,14 +377,27 @@ const Crontab = () => {
}
request
.get(url)
.then(({ code, data: _data }) => {
.then(async ({ code, data: _data }) => {
if (code === 200) {
const { data, total } = _data;
const subscriptions = await request.get(
`${config.apiPrefix}subscriptions?ids=${JSON.stringify([
...new Set(data.map((x) => x.sub_id).filter(Boolean)),
])}`,
{
onError: noop,
},
);
const subscriptionMap = Object.fromEntries(
subscriptions?.data?.map((x) => [x.id, x]),
);
setValue(
data.map((x) => {
return {
...x,
nextRunTime: getCrontabsNextDate(x.schedule, x.extra_schedules),
subscription: subscriptionMap?.[x.sub_id],
};
}),
);
+66 -48
View File
@@ -39,6 +39,7 @@ import dayjs from 'dayjs';
import WebSocketManager from '@/utils/websocket';
import { DependenceStatus, Status } from './type';
import IconFont from '@/components/iconfont';
import useResizeObserver from '@react-hook/resize-observer';
const { Text } = Typography;
const { Search } = Input;
@@ -240,7 +241,19 @@ const Dependence = () => {
const [isLogModalVisible, setIsLogModalVisible] = useState(false);
const [type, setType] = useState('nodejs');
const tableRef = useRef<HTMLDivElement>(null);
const tableScrollHeight = useTableScrollHeight(tableRef, 59);
const [height, setHeight] = useState<number>(0);
useResizeObserver(tableRef, (entry) => {
const _height =
entry.target?.parentElement?.parentElement?.parentElement?.offsetHeight;
let threshold = 113;
if (selectedRowIds.length) {
threshold += 53;
}
if (_height && height !== _height - threshold) {
setHeight(_height - threshold);
}
});
const getDependencies = (status?: number[]) => {
setLoading(true);
@@ -528,6 +541,56 @@ const Dependence = () => {
setType(activeKey);
};
const children = (
<div ref={tableRef}>
{selectedRowIds.length > 0 && (
<div style={{ marginBottom: 16 }}>
<Button
type="primary"
style={{ marginBottom: 5, marginLeft: 8 }}
onClick={() => handlereInstallDependencies()}
>
{intl.get('批量安装')}
</Button>
<Button
type="primary"
style={{ marginBottom: 5, marginLeft: 8 }}
onClick={() => delDependencies(false)}
>
{intl.get('批量删除')}
</Button>
<Button
type="primary"
style={{ marginBottom: 5, marginLeft: 8 }}
onClick={() => delDependencies(true)}
>
{intl.get('批量强制删除')}
</Button>
<span style={{ marginLeft: 8 }}>
{intl.get('已选择')}
<a>{selectedRowIds?.length}</a>
{intl.get('项')}
</span>
</div>
)}
<DndProvider backend={HTML5Backend}>
<Table
columns={columns}
rowSelection={rowSelection}
pagination={false}
dataSource={value}
rowKey="id"
size="middle"
scroll={{ x: 768, y: height }}
loading={loading}
onChange={(pagination, filters) => {
getDependencies(filters?.status as number[]);
}}
/>
</DndProvider>
</div>
);
return (
<PageContainer
className="ql-container-wrapper dependence-wrapper ql-container-wrapper-has-tab"
@@ -552,6 +615,7 @@ const Dependence = () => {
defaultActiveKey="nodejs"
size="small"
tabPosition="top"
destroyInactiveTabPane
onChange={onTabChange}
items={[
{
@@ -568,53 +632,7 @@ const Dependence = () => {
},
]}
/>
<div ref={tableRef}>
{selectedRowIds.length > 0 && (
<div style={{ marginBottom: 16 }}>
<Button
type="primary"
style={{ marginBottom: 5, marginLeft: 8 }}
onClick={() => handlereInstallDependencies()}
>
{intl.get('批量安装')}
</Button>
<Button
type="primary"
style={{ marginBottom: 5, marginLeft: 8 }}
onClick={() => delDependencies(false)}
>
{intl.get('批量删除')}
</Button>
<Button
type="primary"
style={{ marginBottom: 5, marginLeft: 8 }}
onClick={() => delDependencies(true)}
>
{intl.get('批量强制删除')}
</Button>
<span style={{ marginLeft: 8 }}>
{intl.get('已选择')}
<a>{selectedRowIds?.length}</a>
{intl.get('项')}
</span>
</div>
)}
<DndProvider backend={HTML5Backend}>
<Table
columns={columns}
rowSelection={rowSelection}
pagination={false}
dataSource={value}
rowKey="id"
size="middle"
scroll={{ x: 768, y: tableScrollHeight }}
loading={loading}
onChange={(pagination, filters) => {
getDependencies(filters?.status as number[]);
}}
/>
</DndProvider>
</div>
{children}
<DependenceModal
visible={isModalVisible}
handleCancel={handleCancel}
+2 -2
View File
@@ -34,7 +34,7 @@ const EditModal = ({
handleCancel: () => void;
}) => {
const [value, setValue] = useState('');
const [language, setLanguage] = useState<string>('javascript');
const [language, setLanguage] = useState<string>();
const [cNode, setCNode] = useState<any>();
const [selectedKey, setSelectedKey] = useState<string>();
const [saveModalVisible, setSaveModalVisible] = useState<boolean>(false);
@@ -242,7 +242,7 @@ const EditModal = ({
minimap: { enabled: false },
lineNumbersMinChars: 3,
glyphMargin: false,
accessibilitySupport: 'off'
accessibilitySupport: 'off',
}}
onMount={(editor) => {
editorRef.current = editor;
+1 -1
View File
@@ -377,7 +377,7 @@ const Script = () => {
useEffect(() => {
if (treeDom.current) {
setHeight(treeDom.current.clientHeight);
setHeight(treeDom.current.clientHeight - 6);
}
}, [treeDom.current, data]);
+10 -26
View File
@@ -124,15 +124,14 @@ const Setting = () => {
const [editedApp, setEditedApp] = useState<any>();
const [tabActiveKey, setTabActiveKey] = useState('security');
const [loginLogData, setLoginLogData] = useState<any[]>([]);
const [systemLogData, setSystemLogData] = useState<string>('');
const [notificationInfo, setNotificationInfo] = useState<any>();
const containergRef = useRef<HTMLDivElement>(null);
const [height, setHeight] = useState<number>(0);
useResizeObserver(containergRef, (entry) => {
const _height = entry.target.parentElement?.parentElement?.offsetHeight;
if (_height && height !== _height - 66) {
setHeight(_height - 66);
const _height = (entry.target as HTMLElement)?.offsetHeight;
if (_height && height !== _height - 101) {
setHeight(_height - 101);
}
});
@@ -253,19 +252,6 @@ const Setting = () => {
});
};
const getSystemLog = () => {
request
.get<Blob>(`${config.apiPrefix}system/log`, {
responseType: 'blob',
})
.then(async (res) => {
setSystemLogData(await res.text());
})
.catch((error: any) => {
console.log(error);
});
};
const tabChange = (activeKey: string) => {
setTabActiveKey(activeKey);
if (activeKey === 'app') {
@@ -274,8 +260,6 @@ const Setting = () => {
getLoginLog();
} else if (activeKey === 'notification') {
getNotification();
} else if (activeKey === 'syslog') {
getSystemLog();
}
};
@@ -315,12 +299,14 @@ const Setting = () => {
: []
}
>
<div ref={containergRef}>
<div ref={containergRef} style={{ height: '100%' }}>
<Tabs
style={{ height: '100%' }}
defaultActiveKey="security"
size="small"
tabPosition="top"
onChange={tabChange}
destroyInactiveTabPane
items={[
...(!isDemoEnv
? [
@@ -343,7 +329,7 @@ const Setting = () => {
dataSource={dataSource}
rowKey="id"
size="middle"
scroll={{ x: 1000 }}
scroll={{ x: 1000, y: height }}
loading={loading}
/>
),
@@ -356,14 +342,12 @@ const Setting = () => {
{
key: 'syslog',
label: intl.get('系统日志'),
children: (
<SystemLog data={systemLogData} height={height} theme={theme} />
),
children: <SystemLog height={height} theme={theme} />,
},
{
key: 'login',
label: intl.get('登录日志'),
children: <LoginLog data={loginLogData} />,
children: <LoginLog height={height} data={loginLogData} />,
},
{
key: 'dependence',
@@ -383,7 +367,7 @@ const Setting = () => {
children: <About systemInfo={systemInfo} />,
},
]}
></Tabs>
/>
</div>
<AppModal
visible={isModalVisible}
+8 -3
View File
@@ -67,7 +67,13 @@ const columns = [
},
];
const LoginLog = ({ data }: any) => {
const LoginLog = ({
data,
height,
}: {
data: Array<object>;
height: number;
}) => {
return (
<>
<Table
@@ -76,8 +82,7 @@ const LoginLog = ({ data }: any) => {
dataSource={data}
rowKey="id"
size="middle"
scroll={{ x: 1000 }}
sticky
scroll={{ x: 1000, y: height }}
/>
</>
);
+108 -31
View File
@@ -1,13 +1,39 @@
import React, { useRef } from 'react';
import React, { useRef, useState } from 'react';
import CodeMirror from '@uiw/react-codemirror';
import { Button } from 'antd';
import { Button, DatePicker, Empty, message, Spin } from 'antd';
import {
VerticalAlignBottomOutlined,
VerticalAlignTopOutlined,
} from '@ant-design/icons';
import { request } from '@/utils/http';
import config from '@/utils/config';
import { useRequest } from 'ahooks';
import moment from 'moment';
const SystemLog = ({ data, height, theme }: any) => {
const { RangePicker } = DatePicker;
const SystemLog = ({ height, theme }: any) => {
const editorRef = useRef<any>(null);
const panelVisiableRef = useRef<[string, string] | false>();
const [range, setRange] = useState<string[]>(['', '']);
const [systemLogData, setSystemLogData] = useState<string>('');
const { loading, refresh } = useRequest(
() => {
return request.get<Blob>(
`${config.apiPrefix}system/log?startTime=${range[0]}&endTime=${range[1]}`,
{
responseType: 'blob',
},
);
},
{
refreshDeps: [range],
async onSuccess(res) {
setSystemLogData(await res.text());
},
},
);
const scrollTo = (position: 'start' | 'end') => {
editorRef.current.scrollDOM.scrollTo({
@@ -15,42 +41,93 @@ const SystemLog = ({ data, height, theme }: any) => {
});
};
const deleteLog = () => {
request.delete(`${config.apiPrefix}system/log`).then((x) => {
message.success('删除成功');
refresh();
});
};
return (
<div style={{ position: 'relative' }}>
<CodeMirror
maxHeight={`${height}px`}
value={data}
onCreateEditor={(view) => {
editorRef.current = view;
}}
readOnly={true}
theme={theme.includes('dark') ? 'dark' : 'light'}
/>
<div
style={{
position: 'absolute',
bottom: 20,
right: 20,
display: 'flex',
flexDirection: 'column',
gap: 10,
}}
>
<Button
size='small'
icon={<VerticalAlignTopOutlined />}
onClick={() => {
scrollTo('start');
<div>
<RangePicker
style={{ marginBottom: 12, marginRight: 12 }}
disabledDate={(date) =>
date > moment() || date < moment().subtract(7, 'days')
}
defaultValue={[moment(), moment()]}
onOpenChange={(v) => {
panelVisiableRef.current = v ? ['', ''] : false;
}}
onCalendarChange={(_, dates, { range }) => {
if (
!panelVisiableRef.current ||
typeof panelVisiableRef.current === 'boolean'
) {
return;
}
if (range === 'start') {
panelVisiableRef.current[0] = dates[0];
}
if (range === 'end') {
panelVisiableRef.current[1] = dates[1];
}
if (panelVisiableRef.current[0] && panelVisiableRef.current[1]) {
setRange(dates);
}
}}
/>
<Button
size='small'
icon={<VerticalAlignBottomOutlined />}
onClick={() => {
scrollTo('end');
deleteLog();
}}
/>
>
</Button>
</div>
{systemLogData ? (
<>
<CodeMirror
maxHeight={`${height}px`}
value={systemLogData}
onCreateEditor={(view) => {
editorRef.current = view;
}}
readOnly={true}
theme={theme.includes('dark') ? 'dark' : 'light'}
/>
<div
style={{
position: 'absolute',
bottom: 20,
right: 20,
display: 'flex',
flexDirection: 'column',
gap: 10,
}}
>
<Button
size="small"
icon={<VerticalAlignTopOutlined />}
onClick={() => {
scrollTo('start');
}}
/>
<Button
size="small"
icon={<VerticalAlignBottomOutlined />}
onClick={() => {
scrollTo('end');
}}
/>
</div>
</>
) : loading ? (
<Spin />
) : (
<Empty image={Empty.PRESENTED_IMAGE_SIMPLE} />
)}
</div>
);
};
+20 -7
View File
@@ -1,27 +1,40 @@
import * as Sentry from '@sentry/react';
import { loader } from '@monaco-editor/react';
import config from './config';
import { useEffect } from 'react';
import {
createRoutesFromChildren,
matchRoutes,
useLocation,
useNavigationType,
} from 'react-router-dom';
export function init(version: string) {
// sentry监控 init
Sentry.init({
dsn: 'https://49b9ad1a6201bfe027db296ab7c6d672@o1098464.ingest.sentry.io/6122818',
integrations: [
new Sentry.BrowserTracing({
shouldCreateSpanForRequest(url) {
return !url.includes('/api/ws') && !url.includes('/api/static');
},
Sentry.reactRouterV6BrowserTracingIntegration({
useEffect,
useLocation,
useNavigationType,
createRoutesFromChildren,
matchRoutes,
}),
Sentry.replayIntegration(),
],
release: version,
tracesSampleRate: 0.1,
beforeBreadcrumb(breadcrumb, hint?) {
beforeBreadcrumb(breadcrumb) {
if (breadcrumb.data && breadcrumb.data.url) {
const url = breadcrumb.data.url.replace(/token=.*/, '');
breadcrumb.data.url = url;
}
return breadcrumb;
},
tracesSampleRate: 0.1,
tracePropagationTargets: [/^(?!\/api\/(ws|static)).*$/],
replaysSessionSampleRate: 0.1,
replaysOnErrorSampleRate: 0.1,
release: version,
});
// monaco 编辑器配置cdn和locale
+10 -6
View File
@@ -1,8 +1,12 @@
version: 2.17.9
changeLogLink: https://t.me/jiao_long/420
publishTime: 2024-07-23 23:00
version: 2.17.10
changeLogLink: https://t.me/jiao_long/421
publishTime: 2024-08-30 23:00
changeLog: |
1. Javascript 和 Python 增加内置API QLAPI.notify
2. 配置管理增加 task_before.js 和 task_before.py 文件,在执行任务前执行,避免环境变量过大报错
3. 修复执行任务 JavaScript 和 Python 任务前未执行 task_before.sh
1. 系统日志增加时间筛选和清空操作
2. 单个任务运行中实例限制最多 5 个,防止任务堆积过多,运行实例超过 5 个时,会发频繁通知
3. 修复 shell 任务无法正常结束
4. 修改 python 任务 smtp 参数说明,详见 config.sh
5. 修复任务执行前命令 (task_before) 执行报错
6. 修复删除日志命令可能失败
7. 其他样式修复