open-source/qinglong
1
0
Fork 0
mirror of https://github.com/whyour/qinglong.git synced 2026-07-01 04:40:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

Optimize log file writes using stream pooling (#2835)

Browse Source 
* Initial plan

* Implement LogStreamManager for optimized log writing

Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>

* Fix error handler in LogStreamManager to avoid race conditions

Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: whyour <22700758+whyour@users.noreply.github.com>
This commit is contained in:
Copilot
2025-11-16 21:11:10 +08:00
committed by GitHub
parent fbeb4f4a6c
commit 08ef509e27
4 changed files with 132 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
back/services/cron.ts
+6 -3
View File
@@ -24,6 +24,7 @@ import pickBy from 'lodash/pickBy';
import omit from 'lodash/omit';
import { writeFileWithLock } from '../shared/utils';
import { ScheduleType } from '../interface/schedule';
import { logStreamManager } from '../shared/logStreamManager';
@Service()
export default class CronService {
@@ -516,7 +517,7 @@ export default class CronService {
{ where: { id } },
);
cp.stdout.on('data', async (data) => {
await fs.appendFile(absolutePath, data.toString());
await logStreamManager.write(absolutePath, data.toString());
});
cp.stderr.on('data', async (data) => {
this.logger.info(
@@ -524,7 +525,7 @@ export default class CronService {
command,
data.toString(),
);
await fs.appendFile(absolutePath, data.toString());
await logStreamManager.write(absolutePath, data.toString());
});
cp.on('error', async (err) => {
this.logger.error(
@@ -532,7 +533,7 @@ export default class CronService {
command,
err,
);
await fs.appendFile(absolutePath, JSON.stringify(err));
await logStreamManager.write(absolutePath, JSON.stringify(err));
});
cp.on('exit', async (code) => {
@@ -541,6 +542,8 @@ export default class CronService {
JSON.stringify(params),
code,
);
// Close the stream after task completion
await logStreamManager.closeStream(absolutePath);
await CrontabModel.update(
{ status: CrontabStatus.idle, pid: undefined },
{ where: { id } },
back/services/subscription.ts
+11 -7
View File
@@ -31,6 +31,7 @@ import { formatCommand, formatUrl } from '../config/subscription';
import { CrontabModel } from '../data/cron';
import CrontabService from './cron';
import taskLimit from '../shared/pLimit';
import { logStreamManager } from '../shared/logStreamManager';
@Service()
export default class SubscriptionService {
@@ -136,7 +137,7 @@ export default class SubscriptionService {
let beforeStr = '';
try {
if (doc.sub_before) {
await fs.appendFile(absolutePath, `\n## 执行before命令...\n\n`);
await logStreamManager.write(absolutePath, `\n## 执行before命令...\n\n`);
beforeStr = await promiseExec(doc.sub_before);
}
} catch (error: any) {
@@ -144,7 +145,7 @@ export default class SubscriptionService {
(error.stderr && error.stderr.toString()) || JSON.stringify(error);
}
if (beforeStr) {
await fs.appendFile(absolutePath, `${beforeStr}\n`);
await logStreamManager.write(absolutePath, `${beforeStr}\n`);
}
},
onStart: async (cp: ChildProcessWithoutNullStreams, startTime) => {
@@ -163,7 +164,7 @@ export default class SubscriptionService {
let afterStr = '';
try {
if (sub.sub_after) {
await fs.appendFile(absolutePath, `\n\n## 执行after命令...\n\n`);
await logStreamManager.write(absolutePath, `\n\n## 执行after命令...\n\n`);
afterStr = await promiseExec(sub.sub_after);
}
} catch (error: any) {
@@ -171,16 +172,19 @@ export default class SubscriptionService {
(error.stderr && error.stderr.toString()) || JSON.stringify(error);
}
if (afterStr) {
await fs.appendFile(absolutePath, `${afterStr}\n`);
await logStreamManager.write(absolutePath, `${afterStr}\n`);
}
await fs.appendFile(
await logStreamManager.write(
absolutePath,
`\n## 执行结束... ${endTime.format(
'YYYY-MM-DD HH:mm:ss',
)} 耗时 ${diff} 秒${LOG_END_SYMBOL}`,
);
// Close the stream after task completion
await logStreamManager.closeStream(absolutePath);
await SubscriptionModel.update(
{ status: SubscriptionStatus.idle, pid: undefined },
{ where: { id: sub.id } },
@@ -195,12 +199,12 @@ export default class SubscriptionService {
onError: async (message: string) => {
const sub = await this.getDb({ id: doc.id });
const absolutePath = await handleLogPath(sub.log_path as string);
await fs.appendFile(absolutePath, `\n${message}`);
await logStreamManager.write(absolutePath, `\n${message}`);
},
onLog: async (message: string) => {
const sub = await this.getDb({ id: doc.id });
const absolutePath = await handleLogPath(sub.log_path as string);
await fs.appendFile(absolutePath, `\n${message}`);
await logStreamManager.write(absolutePath, `\n${message}`);
},
};
}