mirror of
https://github.com/whyour/qinglong.git
synced 2026-08-16 22:38:39 +08:00
perf: bound runtime memory and storage usage
This commit is contained in:
whyour
+24
-2
@@ -307,13 +307,35 @@ export default (app: Router) => {
|
||||
params: Joi.object({
|
||||
id: Joi.number().required(),
|
||||
}),
|
||||
query: Joi.object({
|
||||
offset: Joi.number().integer().min(0).optional(),
|
||||
limit: Joi.number()
|
||||
.integer()
|
||||
.min(1)
|
||||
.max(1024 * 1024)
|
||||
.optional(),
|
||||
tail: Joi.boolean().optional(),
|
||||
t: Joi.string().optional(),
|
||||
}).unknown(true),
|
||||
}),
|
||||
async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
|
||||
const logger: Logger = Container.get('logger');
|
||||
try {
|
||||
const cronService = Container.get(CronService);
|
||||
const result = await cronService.log(req.params.id);
|
||||
return res.send({ code: 200, data: result.content, logStatus: result.status });
|
||||
const result = await cronService.log(req.params.id, {
|
||||
offset: req.query.offset as unknown as number,
|
||||
limit: req.query.limit as unknown as number,
|
||||
tail: req.query.tail as unknown as boolean,
|
||||
});
|
||||
return res.send({
|
||||
code: 200,
|
||||
data: result.content,
|
||||
logStatus: result.status,
|
||||
offset: result.offset,
|
||||
nextOffset: result.nextOffset,
|
||||
total: result.total,
|
||||
truncated: result.truncated,
|
||||
});
|
||||
} catch (e) {
|
||||
return next(e);
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import update from './update';
|
||||
import dashboard from './dashboard';
|
||||
import health from './health';
|
||||
import clientIp from './clientIp';
|
||||
import retention from './retention';
|
||||
|
||||
export default () => {
|
||||
const app = Router();
|
||||
@@ -30,6 +31,7 @@ export default () => {
|
||||
dashboard(app);
|
||||
health(app);
|
||||
clientIp(app);
|
||||
retention(app);
|
||||
|
||||
return app;
|
||||
};
|
||||
|
||||
+25
-2
@@ -12,6 +12,7 @@ import {
|
||||
} from '../config/util';
|
||||
import LogService from '../services/log';
|
||||
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
|
||||
import { MAX_LOG_CHUNK_BYTES, readLogChunk } from '../shared/logReader';
|
||||
const route = Router();
|
||||
const blacklist = ['.tmp'];
|
||||
|
||||
@@ -34,6 +35,20 @@ export default (app: Router) => {
|
||||
|
||||
route.get(
|
||||
'/detail',
|
||||
celebrate({
|
||||
query: Joi.object({
|
||||
path: Joi.string().allow('').optional(),
|
||||
file: Joi.string().required(),
|
||||
offset: Joi.number().integer().min(0).optional(),
|
||||
limit: Joi.number()
|
||||
.integer()
|
||||
.min(1)
|
||||
.max(MAX_LOG_CHUNK_BYTES)
|
||||
.optional(),
|
||||
tail: Joi.boolean().optional(),
|
||||
t: Joi.string().optional(),
|
||||
}).unknown(true),
|
||||
}),
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const logService = Container.get(LogService);
|
||||
@@ -52,11 +67,19 @@ export default (app: Router) => {
|
||||
where: { log_path: logPath, status: InstanceStatus.running },
|
||||
});
|
||||
|
||||
const content = await getFileContentByName(finalPath);
|
||||
const chunk = await readLogChunk(finalPath, {
|
||||
offset: req.query.offset as unknown as number,
|
||||
limit: req.query.limit as unknown as number,
|
||||
tail: req.query.tail as unknown as boolean,
|
||||
});
|
||||
res.send({
|
||||
code: 200,
|
||||
data: removeAnsi(content),
|
||||
data: removeAnsi(chunk.content),
|
||||
logStatus: runningInstance ? 'running' : undefined,
|
||||
offset: chunk.offset,
|
||||
nextOffset: chunk.nextOffset,
|
||||
total: chunk.total,
|
||||
truncated: chunk.truncated,
|
||||
});
|
||||
} catch (e) {
|
||||
return next(e);
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { NextFunction, Request, Response, Router } from 'express';
|
||||
import { celebrate, Joi } from 'celebrate';
|
||||
import { Container } from 'typedi';
|
||||
import RetentionService from '../services/retention';
|
||||
import { MAX_RETENTION_DAYS } from '../shared/retention';
|
||||
|
||||
const route = Router();
|
||||
const policySchema = {
|
||||
runningInstanceRetentionDays: Joi.number()
|
||||
.integer()
|
||||
.min(0)
|
||||
.max(MAX_RETENTION_DAYS)
|
||||
.required(),
|
||||
cronStatRetentionDays: Joi.number()
|
||||
.integer()
|
||||
.min(0)
|
||||
.max(MAX_RETENTION_DAYS)
|
||||
.required(),
|
||||
};
|
||||
const cleanupSchema = {
|
||||
...policySchema,
|
||||
dependenceCacheTypes: Joi.array()
|
||||
.items(Joi.string().valid('node', 'python3'))
|
||||
.unique()
|
||||
.default([]),
|
||||
compactDatabase: Joi.boolean().default(false),
|
||||
};
|
||||
|
||||
export default (app: Router) => {
|
||||
app.use('/system/storage-retention', route);
|
||||
|
||||
route.put(
|
||||
'/config',
|
||||
celebrate({ body: Joi.object(policySchema) }),
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const service = Container.get(RetentionService);
|
||||
const data = await service.updatePolicy(req.body);
|
||||
res.send({ code: 200, data });
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
route.post(
|
||||
'/preview',
|
||||
celebrate({ body: Joi.object(cleanupSchema) }),
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const service = Container.get(RetentionService);
|
||||
const data = await service.preview(req.body);
|
||||
res.send({ code: 200, data });
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
route.post(
|
||||
'/cleanup',
|
||||
celebrate({
|
||||
body: Joi.object({
|
||||
...cleanupSchema,
|
||||
confirmation: Joi.string().valid('CLEAN').required(),
|
||||
}),
|
||||
}),
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const service = Container.get(RetentionService);
|
||||
const data = await service.cleanup(req.body);
|
||||
res.send({ code: 200, data });
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
};
|
||||
@@ -148,13 +148,27 @@ export default (app: Router) => {
|
||||
params: Joi.object({
|
||||
id: Joi.number().required(),
|
||||
}),
|
||||
query: Joi.object({
|
||||
offset: Joi.number().integer().min(0).optional(),
|
||||
limit: Joi.number()
|
||||
.integer()
|
||||
.min(1)
|
||||
.max(1024 * 1024)
|
||||
.optional(),
|
||||
tail: Joi.boolean().optional(),
|
||||
t: Joi.string().optional(),
|
||||
}).unknown(true),
|
||||
}),
|
||||
async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
|
||||
const logger: Logger = Container.get('logger');
|
||||
try {
|
||||
const subscriptionService = Container.get(SubscriptionService);
|
||||
const data = await subscriptionService.log(req.params.id);
|
||||
return res.send({ code: 200, data });
|
||||
const result = await subscriptionService.log(req.params.id, {
|
||||
offset: req.query.offset as unknown as number,
|
||||
limit: req.query.limit as unknown as number,
|
||||
tail: req.query.tail as unknown as boolean,
|
||||
});
|
||||
return res.send({ code: 200, data: result.content, ...result });
|
||||
} catch (e) {
|
||||
return next(e);
|
||||
}
|
||||
|
||||
@@ -350,6 +350,11 @@ export default (app: Router) => {
|
||||
query: {
|
||||
startTime: Joi.string().allow('').optional(),
|
||||
endTime: Joi.string().allow('').optional(),
|
||||
limit: Joi.number()
|
||||
.integer()
|
||||
.min(1)
|
||||
.max(1024 * 1024)
|
||||
.optional(),
|
||||
t: Joi.string().optional(),
|
||||
},
|
||||
}),
|
||||
@@ -361,6 +366,7 @@ export default (app: Router) => {
|
||||
req.query as {
|
||||
startTime?: string;
|
||||
endTime?: string;
|
||||
limit?: number;
|
||||
},
|
||||
);
|
||||
} catch (e) {
|
||||
|
||||
@@ -41,6 +41,8 @@ export interface SystemConfigInfo {
|
||||
linuxMirror?: string;
|
||||
timezone?: string;
|
||||
globalSshKey?: string;
|
||||
runningInstanceRetentionDays?: number;
|
||||
cronStatRetentionDays?: number;
|
||||
}
|
||||
|
||||
export interface LoginLogInfo {
|
||||
|
||||
+45
-8
@@ -32,6 +32,7 @@ import { t } from '../shared/i18n';
|
||||
import { ScheduleType } from '../interface/schedule';
|
||||
import { logStreamManager } from '../shared/logStreamManager';
|
||||
import { isEmpty } from 'lodash';
|
||||
import { LogReadOptions, readLogChunk } from '../shared/logReader';
|
||||
|
||||
@Service()
|
||||
export default class CronService {
|
||||
@@ -766,27 +767,63 @@ export default class CronService {
|
||||
await this.setCrontab();
|
||||
}
|
||||
|
||||
public async log(id: number): Promise<{ content: string; status: string }> {
|
||||
public async log(
|
||||
id: number,
|
||||
options: LogReadOptions = {},
|
||||
): Promise<{
|
||||
content: string;
|
||||
status: string;
|
||||
offset: number;
|
||||
nextOffset: number;
|
||||
total: number;
|
||||
truncated: boolean;
|
||||
}> {
|
||||
const doc = await this.getDb({ id });
|
||||
if (!doc) {
|
||||
return { content: '', status: 'empty' };
|
||||
return {
|
||||
content: '',
|
||||
status: 'empty',
|
||||
offset: 0,
|
||||
nextOffset: 0,
|
||||
total: 0,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
if (doc.log_name === '/dev/null') {
|
||||
return { content: t('日志设置为忽略'), status: 'ignored' };
|
||||
return {
|
||||
content: t('日志设置为忽略'),
|
||||
status: 'ignored',
|
||||
offset: 0,
|
||||
nextOffset: 0,
|
||||
total: 0,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
const absolutePath = path.resolve(config.logPath, `${doc.log_path}`);
|
||||
const logFileExist = doc.log_path && (await fileExist(absolutePath));
|
||||
if (logFileExist) {
|
||||
const content = await getFileContentByName(`${absolutePath}`);
|
||||
const chunk = await readLogChunk(`${absolutePath}`, options);
|
||||
const isRunning =
|
||||
typeof doc.status === 'number' &&
|
||||
[CrontabStatus.running, CrontabStatus.queued].includes(doc.status);
|
||||
return { content, status: isRunning ? 'running' : 'completed' };
|
||||
return {
|
||||
...chunk,
|
||||
status: isRunning ? 'running' : 'completed',
|
||||
};
|
||||
} else {
|
||||
return typeof doc.status === 'number' &&
|
||||
const status =
|
||||
typeof doc.status === 'number' &&
|
||||
[CrontabStatus.queued, CrontabStatus.running].includes(doc.status)
|
||||
? { content: t('运行中...'), status: 'running' }
|
||||
: { content: t('日志不存在...'), status: 'notFound' };
|
||||
? 'running'
|
||||
: 'notFound';
|
||||
return {
|
||||
content: status === 'running' ? t('运行中...') : t('日志不存在...'),
|
||||
status,
|
||||
offset: 0,
|
||||
nextOffset: 0,
|
||||
total: 0,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+58
-16
@@ -8,16 +8,19 @@ interface Metric {
|
||||
tags?: Record<string, string>;
|
||||
}
|
||||
|
||||
class MetricsService {
|
||||
private metrics: Metric[] = [];
|
||||
export const MAX_METRIC_SAMPLES = 1000;
|
||||
const METRIC_RETENTION_MS = 60 * 60 * 1000;
|
||||
|
||||
export class MetricsService {
|
||||
private metrics: Array<Metric | undefined> = new Array(MAX_METRIC_SAMPLES);
|
||||
private metricCount = 0;
|
||||
private nextMetricIndex = 0;
|
||||
private static instance: MetricsService;
|
||||
|
||||
private constructor() {
|
||||
// 定期清理旧数据
|
||||
setInterval(() => {
|
||||
const oneHourAgo = Date.now() - 3600000;
|
||||
this.metrics = this.metrics.filter(m => m.timestamp > oneHourAgo);
|
||||
}, 60000);
|
||||
const cleanupTimer = setInterval(() => this.removeExpiredMetrics(), 60000);
|
||||
cleanupTimer.unref();
|
||||
}
|
||||
|
||||
static getInstance(): MetricsService {
|
||||
@@ -28,12 +31,14 @@ class MetricsService {
|
||||
}
|
||||
|
||||
record(name: string, value: number, tags?: Record<string, string>) {
|
||||
this.metrics.push({
|
||||
this.metrics[this.nextMetricIndex] = {
|
||||
name,
|
||||
value,
|
||||
timestamp: Date.now(),
|
||||
tags,
|
||||
});
|
||||
};
|
||||
this.nextMetricIndex = (this.nextMetricIndex + 1) % MAX_METRIC_SAMPLES;
|
||||
this.metricCount = Math.min(this.metricCount + 1, MAX_METRIC_SAMPLES);
|
||||
}
|
||||
|
||||
measure(name: string, fn: () => void, tags?: Record<string, string>) {
|
||||
@@ -46,7 +51,11 @@ class MetricsService {
|
||||
}
|
||||
}
|
||||
|
||||
async measureAsync(name: string, fn: () => Promise<void>, tags?: Record<string, string>) {
|
||||
async measureAsync(
|
||||
name: string,
|
||||
fn: () => Promise<void>,
|
||||
tags?: Record<string, string>,
|
||||
) {
|
||||
const start = performance.now();
|
||||
try {
|
||||
await fn();
|
||||
@@ -57,28 +66,61 @@ class MetricsService {
|
||||
}
|
||||
|
||||
getMetrics(name?: string, tags?: Record<string, string>) {
|
||||
let filtered = this.metrics;
|
||||
this.removeExpiredMetrics();
|
||||
let filtered = this.getMetricSnapshot();
|
||||
|
||||
if (name) {
|
||||
filtered = filtered.filter(m => m.name === name);
|
||||
filtered = filtered.filter((m) => m.name === name);
|
||||
}
|
||||
|
||||
if (tags) {
|
||||
filtered = filtered.filter(m => {
|
||||
filtered = filtered.filter((m) => {
|
||||
if (!m.tags) return false;
|
||||
return Object.entries(tags).every(([key, value]) => m.tags![key] === value);
|
||||
return Object.entries(tags).every(
|
||||
([key, value]) => m.tags![key] === value,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
const values = filtered.map((metric) => metric.value);
|
||||
return {
|
||||
count: filtered.length,
|
||||
average: filtered.reduce((acc, curr) => acc + curr.value, 0) / filtered.length,
|
||||
min: Math.min(...filtered.map(m => m.value)),
|
||||
max: Math.max(...filtered.map(m => m.value)),
|
||||
average: values.length
|
||||
? values.reduce((acc, value) => acc + value, 0) / values.length
|
||||
: 0,
|
||||
min: values.length ? Math.min(...values) : 0,
|
||||
max: values.length ? Math.max(...values) : 0,
|
||||
metrics: filtered,
|
||||
};
|
||||
}
|
||||
|
||||
private getMetricSnapshot(): Metric[] {
|
||||
if (this.metricCount < MAX_METRIC_SAMPLES) {
|
||||
return this.metrics.slice(0, this.metricCount) as Metric[];
|
||||
}
|
||||
return [
|
||||
...this.metrics.slice(this.nextMetricIndex),
|
||||
...this.metrics.slice(0, this.nextMetricIndex),
|
||||
] as Metric[];
|
||||
}
|
||||
|
||||
private removeExpiredMetrics() {
|
||||
const oldestTimestamp = Date.now() - METRIC_RETENTION_MS;
|
||||
const retained = this.getMetricSnapshot().filter(
|
||||
(metric) => metric.timestamp > oldestTimestamp,
|
||||
);
|
||||
if (retained.length === this.metricCount) return;
|
||||
|
||||
this.metrics = new Array(MAX_METRIC_SAMPLES);
|
||||
this.metricCount = 0;
|
||||
this.nextMetricIndex = 0;
|
||||
for (const metric of retained) {
|
||||
this.metrics[this.nextMetricIndex] = metric;
|
||||
this.nextMetricIndex = (this.nextMetricIndex + 1) % MAX_METRIC_SAMPLES;
|
||||
this.metricCount++;
|
||||
}
|
||||
}
|
||||
|
||||
report() {
|
||||
const report = {
|
||||
timestamp: Date.now(),
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import dayjs from 'dayjs';
|
||||
import { Op } from 'sequelize';
|
||||
import { Service } from 'typedi';
|
||||
import config from '../config';
|
||||
import { CrontabStatModel } from '../data/cronStats';
|
||||
import { sequelize } from '../data';
|
||||
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
|
||||
import { AuthDataType, SystemModel } from '../data/system';
|
||||
import {
|
||||
DependenceCacheType,
|
||||
getDirectorySize,
|
||||
isDependenceCacheType,
|
||||
normalizeRetentionPolicy,
|
||||
RetentionPolicy,
|
||||
} from '../shared/retention';
|
||||
|
||||
export interface StorageCleanupRequest extends RetentionPolicy {
|
||||
dependenceCacheTypes?: DependenceCacheType[];
|
||||
compactDatabase?: boolean;
|
||||
}
|
||||
|
||||
function runningInstanceWhere(days: number) {
|
||||
const cutoff = dayjs().subtract(days, 'day').unix();
|
||||
return {
|
||||
status: { [Op.ne]: InstanceStatus.running },
|
||||
[Op.or]: [
|
||||
{ finished_at: { [Op.lt]: cutoff } },
|
||||
{
|
||||
finished_at: { [Op.is]: null },
|
||||
started_at: { [Op.lt]: cutoff },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function cronStatWhere(days: number) {
|
||||
return {
|
||||
date: { [Op.lt]: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
|
||||
};
|
||||
}
|
||||
|
||||
@Service()
|
||||
export default class RetentionService {
|
||||
public async updatePolicy(policy: Partial<RetentionPolicy>) {
|
||||
const normalized = normalizeRetentionPolicy(policy);
|
||||
const systemConfig = await SystemModel.findOne({
|
||||
where: { type: AuthDataType.systemConfig },
|
||||
});
|
||||
if (!systemConfig) {
|
||||
throw new Error('System config not found');
|
||||
}
|
||||
await SystemModel.update(
|
||||
{
|
||||
info: {
|
||||
...systemConfig.info,
|
||||
...normalized,
|
||||
},
|
||||
},
|
||||
{ where: { id: systemConfig.id } },
|
||||
);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
public async preview(request: StorageCleanupRequest) {
|
||||
const policy = normalizeRetentionPolicy(request);
|
||||
const dependenceCacheTypes = (request.dependenceCacheTypes || []).filter(
|
||||
isDependenceCacheType,
|
||||
);
|
||||
const [runningInstances, cronStats, dependenceCaches] = await Promise.all([
|
||||
policy.runningInstanceRetentionDays > 0
|
||||
? RunningInstanceModel.count({
|
||||
where: runningInstanceWhere(policy.runningInstanceRetentionDays),
|
||||
})
|
||||
: 0,
|
||||
policy.cronStatRetentionDays > 0
|
||||
? CrontabStatModel.count({
|
||||
where: cronStatWhere(policy.cronStatRetentionDays),
|
||||
})
|
||||
: 0,
|
||||
Promise.all(
|
||||
dependenceCacheTypes.map(async (type) => ({
|
||||
type,
|
||||
bytes: await getDirectorySize(
|
||||
path.join(config.dependenceCachePath, type),
|
||||
),
|
||||
})),
|
||||
),
|
||||
]);
|
||||
|
||||
return {
|
||||
policy,
|
||||
runningInstances,
|
||||
cronStats,
|
||||
dependenceCaches,
|
||||
dependenceCacheBytes: dependenceCaches.reduce(
|
||||
(total, cache) => total + cache.bytes,
|
||||
0,
|
||||
),
|
||||
compactDatabase: Boolean(request.compactDatabase),
|
||||
};
|
||||
}
|
||||
|
||||
public async cleanup(request: StorageCleanupRequest) {
|
||||
const preview = await this.preview(request);
|
||||
const deleted = await sequelize.transaction(async (transaction) => {
|
||||
const runningInstances =
|
||||
preview.policy.runningInstanceRetentionDays > 0
|
||||
? await RunningInstanceModel.destroy({
|
||||
where: runningInstanceWhere(
|
||||
preview.policy.runningInstanceRetentionDays,
|
||||
),
|
||||
transaction,
|
||||
})
|
||||
: 0;
|
||||
const cronStats =
|
||||
preview.policy.cronStatRetentionDays > 0
|
||||
? await CrontabStatModel.destroy({
|
||||
where: cronStatWhere(preview.policy.cronStatRetentionDays),
|
||||
transaction,
|
||||
})
|
||||
: 0;
|
||||
return { runningInstances, cronStats };
|
||||
});
|
||||
|
||||
const dependenceCaches = [];
|
||||
for (const cache of preview.dependenceCaches) {
|
||||
await fs.rm(path.join(config.dependenceCachePath, cache.type), {
|
||||
recursive: true,
|
||||
force: true,
|
||||
});
|
||||
dependenceCaches.push(cache);
|
||||
}
|
||||
|
||||
if (
|
||||
request.compactDatabase &&
|
||||
(deleted.runningInstances || deleted.cronStats)
|
||||
) {
|
||||
await sequelize.query('VACUUM');
|
||||
}
|
||||
|
||||
return {
|
||||
deleted: {
|
||||
...deleted,
|
||||
dependenceCaches,
|
||||
dependenceCacheBytes: preview.dependenceCacheBytes,
|
||||
},
|
||||
compactedDatabase: Boolean(
|
||||
request.compactDatabase &&
|
||||
(deleted.runningInstances || deleted.cronStats),
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,7 @@ import { CrontabModel } from '../data/cron';
|
||||
import CrontabService from './cron';
|
||||
import taskLimit from '../shared/pLimit';
|
||||
import { logStreamManager } from '../shared/logStreamManager';
|
||||
import { LogReadOptions, readLogChunk } from '../shared/logReader';
|
||||
|
||||
@Service()
|
||||
export default class SubscriptionService {
|
||||
@@ -366,14 +367,20 @@ export default class SubscriptionService {
|
||||
}
|
||||
}
|
||||
|
||||
public async log(id: number) {
|
||||
public async log(id: number, options: LogReadOptions = {}) {
|
||||
const doc = await this.getDb({ id });
|
||||
if (!doc || !doc.log_path) {
|
||||
return '';
|
||||
return {
|
||||
content: '',
|
||||
offset: 0,
|
||||
nextOffset: 0,
|
||||
total: 0,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
const absolutePath = await handleLogPath(doc.log_path as string);
|
||||
return await getFileContentByName(absolutePath);
|
||||
return await readLogChunk(absolutePath, options);
|
||||
}
|
||||
|
||||
public async logs(id: number) {
|
||||
|
||||
+29
-3
@@ -2,7 +2,6 @@ import { spawn } from 'cross-spawn';
|
||||
import { Response } from 'express';
|
||||
import fs from 'fs';
|
||||
import { Agent, request } from 'undici';
|
||||
import sum from 'lodash/sum';
|
||||
import path from 'path';
|
||||
import { Inject, Service } from 'typedi';
|
||||
import winston from 'winston';
|
||||
@@ -467,6 +466,7 @@ export default class SystemService {
|
||||
query: {
|
||||
startTime?: string;
|
||||
endTime?: string;
|
||||
limit?: number;
|
||||
},
|
||||
) {
|
||||
const startTime = dayjs(query.startTime || undefined)
|
||||
@@ -481,8 +481,30 @@ export default class SystemService {
|
||||
.filter((x) => x.title.endsWith('.log'))
|
||||
.filter((x) => x.createTime >= startTime && x.createTime <= endTime);
|
||||
|
||||
const limit = Math.min(
|
||||
Math.max(Number(query.limit) || 1024 * 1024, 1),
|
||||
1024 * 1024,
|
||||
);
|
||||
const total = logs.reduce((size, log) => size + (log.size || 0), 0);
|
||||
let remaining = limit;
|
||||
const selected: Array<
|
||||
(typeof logs)[number] & { start: number; length: number }
|
||||
> = [];
|
||||
for (let index = logs.length - 1; index >= 0 && remaining > 0; index--) {
|
||||
const log = logs[index];
|
||||
const size = log.size || 0;
|
||||
const length = Math.min(size, remaining);
|
||||
if (length > 0) {
|
||||
selected.unshift({ ...log, start: size - length, length });
|
||||
remaining -= length;
|
||||
}
|
||||
}
|
||||
|
||||
const contentLength = selected.reduce((size, log) => size + log.length, 0);
|
||||
res.set({
|
||||
'Content-Length': sum(logs.map((x) => x.size)),
|
||||
'Content-Length': contentLength,
|
||||
'X-QL-Log-Total': total,
|
||||
'X-QL-Log-Truncated': total > contentLength ? 'true' : 'false',
|
||||
});
|
||||
(function sendFiles(res, fileNames) {
|
||||
if (fileNames.length === 0) {
|
||||
@@ -494,13 +516,17 @@ export default class SystemService {
|
||||
if (currentLog) {
|
||||
const currentFileStream = fs.createReadStream(
|
||||
path.join(config.systemLogPath, currentLog.title),
|
||||
{
|
||||
start: currentLog.start,
|
||||
end: currentLog.start + currentLog.length - 1,
|
||||
},
|
||||
);
|
||||
currentFileStream.on('end', () => {
|
||||
sendFiles(res, fileNames);
|
||||
});
|
||||
currentFileStream.pipe(res, { end: false });
|
||||
}
|
||||
})(res, logs);
|
||||
})(res, selected);
|
||||
}
|
||||
|
||||
public async deleteSystemLog() {
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import fs from 'fs/promises';
|
||||
|
||||
export const DEFAULT_LOG_CHUNK_BYTES = 256 * 1024;
|
||||
export const MAX_LOG_CHUNK_BYTES = 1024 * 1024;
|
||||
|
||||
export interface LogReadOptions {
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
tail?: boolean;
|
||||
}
|
||||
|
||||
export interface LogChunk {
|
||||
content: string;
|
||||
offset: number;
|
||||
nextOffset: number;
|
||||
total: number;
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
function normalizeLimit(limit?: number) {
|
||||
if (!Number.isFinite(limit)) return DEFAULT_LOG_CHUNK_BYTES;
|
||||
return Math.min(Math.max(Math.trunc(limit!), 4), MAX_LOG_CHUNK_BYTES);
|
||||
}
|
||||
|
||||
function isUtf8ContinuationByte(byte: number) {
|
||||
return (byte & 0xc0) === 0x80;
|
||||
}
|
||||
|
||||
function completeUtf8End(buffer: Buffer, start: number, end: number) {
|
||||
if (end <= start) return start;
|
||||
|
||||
let sequenceStart = end - 1;
|
||||
while (
|
||||
sequenceStart > start &&
|
||||
isUtf8ContinuationByte(buffer[sequenceStart])
|
||||
) {
|
||||
sequenceStart--;
|
||||
}
|
||||
|
||||
const firstByte = buffer[sequenceStart];
|
||||
const expectedLength =
|
||||
firstByte < 0x80
|
||||
? 1
|
||||
: firstByte < 0xe0
|
||||
? 2
|
||||
: firstByte < 0xf0
|
||||
? 3
|
||||
: 4;
|
||||
return end - sequenceStart < expectedLength ? sequenceStart : end;
|
||||
}
|
||||
|
||||
export async function readLogChunk(
|
||||
filePath: string,
|
||||
options: LogReadOptions = {},
|
||||
): Promise<LogChunk> {
|
||||
let handle: fs.FileHandle | undefined;
|
||||
try {
|
||||
handle = await fs.open(filePath, 'r');
|
||||
const { size: total } = await handle.stat();
|
||||
const limit = normalizeLimit(options.limit);
|
||||
const requestedOffset = Number.isFinite(options.offset)
|
||||
? Math.trunc(options.offset!)
|
||||
: undefined;
|
||||
const requestedStart =
|
||||
options.tail || requestedOffset === undefined
|
||||
? Math.max(total - limit, 0)
|
||||
: Math.min(Math.max(requestedOffset, 0), total);
|
||||
const length = Math.min(limit + 3, total - requestedStart);
|
||||
const buffer = Buffer.allocUnsafe(length);
|
||||
const { bytesRead } = await handle.read(
|
||||
buffer,
|
||||
0,
|
||||
length,
|
||||
requestedStart,
|
||||
);
|
||||
let leadingBytes = 0;
|
||||
while (
|
||||
leadingBytes < bytesRead &&
|
||||
isUtf8ContinuationByte(buffer[leadingBytes])
|
||||
) {
|
||||
leadingBytes++;
|
||||
}
|
||||
const offset = requestedStart + leadingBytes;
|
||||
const candidateEnd = Math.min(leadingBytes + limit, bytesRead);
|
||||
const contentEnd = completeUtf8End(buffer, leadingBytes, candidateEnd);
|
||||
const nextOffset = requestedStart + contentEnd;
|
||||
|
||||
return {
|
||||
content: buffer.subarray(leadingBytes, contentEnd).toString('utf8'),
|
||||
offset,
|
||||
nextOffset,
|
||||
total,
|
||||
truncated: offset > 0 || nextOffset < total,
|
||||
};
|
||||
} catch (error: any) {
|
||||
if (error?.code === 'ENOENT') {
|
||||
return {
|
||||
content: '',
|
||||
offset: 0,
|
||||
nextOffset: 0,
|
||||
total: 0,
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
await handle?.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import fs from 'fs/promises';
|
||||
import { Dirent } from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
export const MAX_RETENTION_DAYS = 3650;
|
||||
export const DEPENDENCE_CACHE_TYPES = ['node', 'python3'] as const;
|
||||
|
||||
export type DependenceCacheType = (typeof DEPENDENCE_CACHE_TYPES)[number];
|
||||
|
||||
export interface RetentionPolicy {
|
||||
runningInstanceRetentionDays: number;
|
||||
cronStatRetentionDays: number;
|
||||
}
|
||||
|
||||
export function normalizeRetentionDays(value: unknown) {
|
||||
const days = Number(value);
|
||||
if (!Number.isFinite(days)) return 0;
|
||||
return Math.min(Math.max(Math.trunc(days), 0), MAX_RETENTION_DAYS);
|
||||
}
|
||||
|
||||
export function normalizeRetentionPolicy(
|
||||
policy: Partial<RetentionPolicy>,
|
||||
): RetentionPolicy {
|
||||
return {
|
||||
runningInstanceRetentionDays: normalizeRetentionDays(
|
||||
policy.runningInstanceRetentionDays,
|
||||
),
|
||||
cronStatRetentionDays: normalizeRetentionDays(policy.cronStatRetentionDays),
|
||||
};
|
||||
}
|
||||
|
||||
export function isDependenceCacheType(
|
||||
value: string,
|
||||
): value is DependenceCacheType {
|
||||
return DEPENDENCE_CACHE_TYPES.includes(value as DependenceCacheType);
|
||||
}
|
||||
|
||||
export async function getDirectorySize(rootPath: string): Promise<number> {
|
||||
const pending = [rootPath];
|
||||
let total = 0;
|
||||
|
||||
while (pending.length > 0) {
|
||||
const currentPath = pending.pop()!;
|
||||
let entries: Dirent[];
|
||||
try {
|
||||
entries = await fs.readdir(currentPath, { withFileTypes: true });
|
||||
} catch (error: any) {
|
||||
if (error?.code === 'ENOENT') continue;
|
||||
throw error;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (entry.isSymbolicLink()) continue;
|
||||
const entryPath = path.join(currentPath, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
pending.push(entryPath);
|
||||
} else if (entry.isFile()) {
|
||||
total += (await fs.stat(entryPath)).size;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
@@ -70,8 +70,10 @@ RUN set -x && \
|
||||
git config --global user.name "qinglong" && \
|
||||
git config --global http.postBuffer 524288000 && \
|
||||
npm install -g pnpm@8.3.1 pm2 ts-node typescript@5 && \
|
||||
npm cache clean --force && \
|
||||
rm -rf /root/.cache && \
|
||||
rm -rf /root/.npm && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
rm -rf /etc/apt/apt.conf.d/docker-clean && \
|
||||
ulimit -c 0
|
||||
|
||||
|
||||
@@ -69,8 +69,10 @@ RUN set -x && \
|
||||
git config --global user.name "qinglong" && \
|
||||
git config --global http.postBuffer 524288000 && \
|
||||
npm install -g pnpm@8.3.1 pm2 ts-node typescript@5 && \
|
||||
npm cache clean --force && \
|
||||
rm -rf /root/.cache && \
|
||||
rm -rf /root/.npm && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
rm -rf /etc/apt/apt.conf.d/docker-clean && \
|
||||
ulimit -c 0
|
||||
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
# 性能与未关闭 Issue 梳理（2026-08-16）
|
||||
|
||||
## 范围与结论
|
||||
|
||||
- 基线：`origin/develop` @ `31f78e4d`，运行镜像 `whyour/qinglong:debian`。
|
||||
- GitHub 当前有 86 个未关闭 issue；逐项读取了正文，并读取了其中 48 个有评论 issue 的全部评论。
|
||||
- 待机内存问题有效且可量化。主要问题不是 Node.js 语言本身，而是三个 Node 隔离进程、PM2，以及没有上限的逐请求指标保留共同叠加。
|
||||
- 稳定待机没有持续块设备写入；磁盘问题主要是活跃期的重复日志、每次任务生命周期的多次 SQLite 事务、无界历史表和日志读取/保留策略。
|
||||
- 建议先做可回归验证的小步优化，不建议以“改写为 Go”作为当前性能问题的前置条件。
|
||||
|
||||
## 实测基线
|
||||
|
||||
### 内存
|
||||
|
||||
| 项目 | 结果 |
|
||||
| --- | ---: |
|
||||
| `docker stats` 稳定待机 | 约 193 MiB |
|
||||
| cgroup `memory.current` | 约 234–244 MB |
|
||||
| HTTP worker PSS / RSS | 约 75 MiB / 107 MiB |
|
||||
| gRPC worker PSS / RSS | 约 57 MiB / 89 MiB |
|
||||
| cluster 主进程 PSS / RSS | 约 46 MiB / 77 MiB |
|
||||
| PM2 PSS / RSS | 约 24 MiB / 52 MiB |
|
||||
|
||||
对 `/api/health` 发出 50,000 次成功请求后，等待客户端退出及 5 秒回收：
|
||||
|
||||
- cgroup 从 `244,035,584` 增至 `348,925,952` 字节，净增约 105 MB；
|
||||
- HTTP worker RSS 从约 107 MB 增至约 208 MB；
|
||||
- 块设备写入量没有变化。
|
||||
|
||||
这与 `back/middlewares/monitoring.ts` 和 `back/services/metrics.ts` 一致：每个 HTTP 请求同时进入一个 1,000 条队列和一个按一小时保留、没有数量上限的原始指标数组。后者会随每小时请求数线性增长，并不是 GC 可以主动释放的短期对象。
|
||||
|
||||
### 磁盘写入与占用
|
||||
|
||||
- 稳定后连续 30 秒，cgroup `wbytes` 保持 `1,077,248` 不变；当前没有证据表明待机状态持续刷盘。
|
||||
- SQLite 使用 `journal_mode=delete`、`synchronous=2`（FULL）、4 KiB page。一次任务执行至少涉及运行实例创建、任务状态更新、运行实例完成、任务恢复空闲、统计更新等多次独立事务，活跃期会产生额外 journal 与同步写放大。
|
||||
- Winston 会写 `/ql/data/log`，同一日志又经 console 被 PM2 写入 `~/.pm2/logs`。主进程、HTTP worker、gRPC worker 都会创建 logger/file transport；PM2 日志没有代码内保留上限，只在 reload 时 `pm2 flush`。
|
||||
- 任务 stderr 同时进入任务日志和系统日志；错误输出较多时会再次放大写入。
|
||||
- 日志详情接口使用 `readFile(..., 'utf8')` 整文件加载，既造成瞬时内存峰值，也对应 #2256 的超长日志超时。
|
||||
|
||||
镜像与主要目录：
|
||||
|
||||
| 项目 | 大小 |
|
||||
| --- | ---: |
|
||||
| 镜像 | 599,130,109 bytes |
|
||||
| `/ql` | 215 MiB |
|
||||
| `/ql/node_modules` | 174 MiB |
|
||||
| `/ql/static` | 32 MiB |
|
||||
| `/usr/local/lib/node_modules` | 85 MiB |
|
||||
| `/usr/local/lib/python3.11` | 40 MiB |
|
||||
| `/var/lib/apt/lists` | 19 MiB |
|
||||
| 全新 `/ql/data` | 5.9 MiB |
|
||||
|
||||
## 建议修复批次
|
||||
|
||||
### P0：低风险、可量化收益
|
||||
|
||||
1. 将原始逐请求指标改为固定容量 ring buffer，或直接改成计数器、直方图和最近慢请求；给所有维度设置基数上限。增加 10 万请求内存回归测试，目标是回落后增量小于 10 MiB。
|
||||
2. 日志只保留一个持久化落点：生产环境选择“Winston 文件 + PM2 stdout 丢弃”或“console + 受控 PM2 rotation”，避免双写。统一由单进程持有系统日志文件 transport。
|
||||
3. 日志 API 增加 `tail/offset/limit` 与最大响应字节数，前端使用增量加载，不再整文件读入内存。
|
||||
4. 给 PM2 日志、`RunningInstances`、任务统计和依赖下载缓存增加明确的容量/时间保留策略；清理任务输出被删除的字节数和剩余大小。
|
||||
5. Dockerfile 删除 apt lists，审计全局 Node 模块与应用依赖的重复内容；构建产物中不要保留仅构建期需要的包。
|
||||
|
||||
当前修复分支落实情况：
|
||||
|
||||
- P0-1 已完成：逐请求原始指标改为 1,000 条固定容量环形缓冲区，并补充 50,000 次写入回归测试。
|
||||
- P0-2 已完成：容器内 PM2 输出直接接入 PID 1 标准输出/错误，停止写入 `~/.pm2/logs` 的重复持久化副本。
|
||||
- P0-3 已完成：日志接口默认 256 KiB、最大 1 MiB，支持 tail/offset 增量读取，并保证 UTF-8 字符不被分块破坏；浏览器视图最大保留 1 MiB。
|
||||
- P0-4 已完成：系统设置提供运行实例/任务统计保留天数，默认 0（禁用）；清理前展示记录数与依赖缓存字节数，清理接口要求明确确认。依赖缓存和 SQLite `VACUUM` 均为手动勾选，不在启动或升级时自动执行。
|
||||
- P0-5 已完成：Debian 镜像构建后清理 npm cache 和 apt lists。
|
||||
|
||||
### P1：轻量架构调整
|
||||
|
||||
1. 主进程只加载 cluster/bootstrap 必需模块，把 Express、监控、HTTP 中间件和业务容器全部延迟到 HTTP worker。目标是先回收主进程约 30–40 MiB private memory。
|
||||
2. Metrics 定时器只在实际使用指标的 worker 中启动，并对 path/tag 做归一化，防止高基数值撑大内存。
|
||||
3. 将任务“开始”和“结束”各自需要的状态、实例、统计写入合并到事务；避免一个生命周期五个以上独立提交。
|
||||
4. 在本地文件系统上基准比较 SQLite `WAL + synchronous=NORMAL + busy_timeout` 与现状；确认断电语义和网络文件系统兼容后再作为可选配置发布。
|
||||
5. 任务日志保留从全局天数扩展为“全局默认 + 单任务覆盖 + 总容量上限”，而不是只增加更多清理 cron。
|
||||
|
||||
### P2：需要设计评审
|
||||
|
||||
- 提供 Lite 模式：调度器与 HTTP 同进程运行，省去 gRPC worker；标准模式继续保持隔离。需要故障域、重启和任务注册一致性测试。
|
||||
- 若继续推进集群/多节点，先定义控制面、执行器、任务租约、心跳、幂等和日志归属；不要以“共享 SQLite/MySQL”替代分布式调度设计。
|
||||
|
||||
## 86 个未关闭 Issue 的逐项建议
|
||||
|
||||
状态含义：`立即处理` 是确认有效且影响当前主线；`轻量处理` 是小型代码/文档/回归测试；`架构议题` 需要 RFC 或独立 epic；`关闭/合并` 表示已修复、重复、支持问题或需要提交者用最新版本重现。
|
||||
|
||||
### 立即处理（10）
|
||||
|
||||
| Issue | 结论 | 建议 |
|
||||
| --- | --- | --- |
|
||||
| #2256 | 有效，整文件读取日志造成超时和内存峰值 | 日志分片、tail、最大响应大小 |
|
||||
| #2742 | 有效，与 #3057 合并 | 用内存基线和压力回归替代“运行几天观察” |
|
||||
| #2743 | 有效，与 #2742/#3057 合并 | 优先修复指标保留和进程基线，再复测受限容器 |
|
||||
| #2871 | 有效，2.21.0 仍有反馈 | 与 #2902 合并，修复源地址生成并加发行版矩阵测试 |
|
||||
| #2902 | 有效，重复 #2871 | 合并后关闭 |
|
||||
| #3017 | 有效安全 epic | 拆成凭据迁移、scope、SSRF、错误脱敏、更新完整性等独立任务 |
|
||||
| #3051 | 当前回归仍需验证 | 对 shell status payload 建端到端测试，重建最新镜像复测 |
|
||||
| #3055 | 用户报告有效，但依赖库单测 5/6 字段都可注册 | 增加真实 gRPC 注册与触发集成测试，排查发行镜像/存量任务迁移 |
|
||||
| #3057 | 有效，实测已复现 | 先修无界指标和多进程基线，不需要先改写语言 |
|
||||
| #3060 | 有效的容器权限/依赖安装回归 | 覆盖匿名卷、bind mount、rootless 三类安装 smoke test |
|
||||
|
||||
### 轻量处理或局部设计（36）
|
||||
|
||||
| Issue | 结论与最小处理 |
|
||||
| --- | --- |
|
||||
| #2015 | 当前仅支持整组 AND/OR；若要嵌套条件，定义轻量 filter AST，避免继续堆 UI 特判 |
|
||||
| #2236 | 与 #2421/#3014 合并为一个 cron 可视化 issue |
|
||||
| #2254 | 增加有限重试策略；必须有退避、最大次数和不可重试退出码 |
|
||||
| #2269 | 增加时区、5/6 字段、DST 的调度集成测试；最新版本无法复现则关闭 |
|
||||
| #2341 | 与 #2688 合并，把 boolean 实例模式升级为 `replace/skip/parallel/queue` 枚举 |
|
||||
| #2360 | 标签在列表中恢复可见与筛选入口，纯前端小改 |
|
||||
| #2418 | 配置方法已有评论解法；仅保留消息超长时的截断/分片问题 |
|
||||
| #2421 | 合并到 #3014 |
|
||||
| #2635 | 备注字段是小型 schema/UI 变更，可与名称/标签排序统一设计 |
|
||||
| #2645 | 增加依赖唯一键、恢复数据去重迁移和幂等安装测试 |
|
||||
| #2687 | 用最新版本和代理 URL 重现；补代理/超时错误提示 |
|
||||
| #2688 | 合并到 #2341；当前“单实例”会杀旧任务，不等同于“跳过新任务” |
|
||||
| #2701 | 需要最小仓库配置样本；JSON 控制字符应在输入边界给出字段级错误 |
|
||||
| #2715 | Server 酱 tags 是局部通知配置扩展 |
|
||||
| #2756 | 重测 Chrome、base path、缓存与 health 401；补登录跳转循环保护 |
|
||||
| #2757 | 复用订阅现有 `autoAddCron` 配置，明确新增任务默认启停状态 |
|
||||
| #2780 | `ql check` 应在系统包失败时停止并回滚/给出恢复指令，不能继续破坏可运行环境 |
|
||||
| #2782 | 通知免打扰时段是局部配置；明确延迟发送还是静默丢弃 |
|
||||
| #2797 | 保留；展开 AggregateError 内部原因并为每种通知后端返回可操作错误 |
|
||||
| #2801 | 私有仓库凭据与 SSH/HTTPS 模式做连通性诊断，信息不足时转支持问题 |
|
||||
| #2859 | 最新版请求重现；增加 `env.js` 镜像启动 smoke test和缓存头校验 |
|
||||
| #2863 | 区分官方 Docker 与 npm/Linux 安装支持等级；若继续支持后者则加 CI |
|
||||
| #2883 | 最新版复测临时编辑保存；失败时保留请求与后端路径错误 |
|
||||
| #2886 | 支持 requirements 文件需要锁定来源、工作目录和隔离策略，可先支持显式路径 |
|
||||
| #2896 | Synology 内核缺少随机字节能力，先做环境诊断与明确错误，不宜静默降级安全随机数 |
|
||||
| #2901 | 当前默认监听 `::` 并回退 `0.0.0.0`；增加 IPv6 Docker/host 网络 smoke test |
|
||||
| #2925 | 信息不足；收集完整日志与重启方式，最新镜像不可复现则关闭 |
|
||||
| #2927 | 使用成熟 ANSI 清理实现，覆盖复合 SGR，而不是只匹配 `\x1b[数字m` |
|
||||
| #2984 | 通知审计有效，但需先确定脱敏、保留期和容量上限，避免制造新磁盘问题 |
|
||||
| #3013 | 与性能修复合并：全局默认、单任务覆盖、总容量上限 |
|
||||
| #3014 | 合并 #2236/#2421，前端生成器必须始终展示最终 cron 文本 |
|
||||
| #3016 | 有效文档任务；从路由/validation 自动生成 OpenAPI，避免手工文档漂移 |
|
||||
| #3020 | 有效小改；配置编辑器启用 `wordWrap: 'on'` 并保留开关 |
|
||||
| #3027 | 需要明确旧值 `1` 的历史语义；用显式 `@once/@boot/disabled` 替代魔法数字 |
|
||||
| #3054 | 有效纯 UI 改进；整行点击打开历史日志，操作按钮阻止冒泡 |
|
||||
| #3058 | 有效安全需求；应用增加环境变量名称 allowlist，默认最小权限并审计拒绝事件 |
|
||||
|
||||
### 架构议题，单独立项（12）
|
||||
|
||||
| Issue | 处理建议 |
|
||||
| --- | --- |
|
||||
| #1656 | 捕获任意脚本网络请求需要代理或运行时注入，成本和隐私风险高；不作为普通日志小改 |
|
||||
| #1695 | 与 #2596 合并为“集群执行器”RFC |
|
||||
| #1821 | 多数据库不等于多节点；只有明确 HA/共享控制面设计后再评估 MySQL/PostgreSQL |
|
||||
| #2434 | PowerShell 运行时会增加镜像体积和维护矩阵，作为可选外部执行器评估 |
|
||||
| #2464 | Bun 属于额外运行时矩阵，不应为单一性能假设直接加入基础镜像 |
|
||||
| #2481 | 每任务 Python 虚拟环境需要依赖缓存、生命周期和磁盘配额设计 |
|
||||
| #2596 | 合并到 #1695 |
|
||||
| #2642 | 与 #2769/#2905 合并为多租户权限模型 RFC |
|
||||
| #2656 | .NET 8 作为可选运行时/派生镜像评估，不进入默认镜像 |
|
||||
| #2736 | 条件触发工作流已接近 n8n 类产品，应独立产品/RFC，不混入 cron 小功能 |
|
||||
| #2769 | 合并到多租户 RFC |
|
||||
| #2905 | 合并到多租户 RFC；PR #2818 作为设计输入 |
|
||||
|
||||
### 建议关闭、合并或转支持（28）
|
||||
|
||||
| Issue | 理由 |
|
||||
| --- | --- |
|
||||
| #795 | HTTP 明文本质上应通过 HTTPS 解决；前端自定义“加盐”不能阻止重放。补反代 HTTPS 文档后关闭 |
|
||||
| #796 | 可由数据库/API 导出；需求长期无上下文，关闭或改为 CSV 导出新需求 |
|
||||
| #854 | 客户端 IP 已由 #3059 对应功能覆盖；应用调用审计并入 #2984 |
|
||||
| #2042 | 当前 `NotReg` 已显式处理 NULL 与 notLike；最新版回归测试通过后关闭 |
|
||||
| #2217 | Home Assistant 插件仓库属于独立打包生态，转社区集成 |
|
||||
| #2219 | 当前已有运行实例历史与逐实例停止按钮，关闭为已实现 |
|
||||
| #2329 | 评论已给出 Open API/文件方案，转文档 FAQ 后关闭 |
|
||||
| #2340 | 老旧 IPQ/固件上的 V8 Bus error 属运行时/硬件兼容；用当前多架构镜像复测，否则关闭 |
|
||||
| #2350 | 环境变量值如何分隔主要由脚本协议定义；补文档后关闭 |
|
||||
| #2398 | 单一外站 DNS 解析问题且无环境信息，转网络支持 |
|
||||
| #2404 | 与 #3057 合并，保留一个性能 epic |
|
||||
| #2525 | 非官方旧容器/npm 安装路径；最新版本不可复现则关闭 |
|
||||
| #2567 | 2024 年无任何环境信息的容器 entrypoint 错误，要求当前镜像重现，否则关闭 |
|
||||
| #2705 | 当前已有全局 cron concurrency 与最多五个重复实例限制，补 UI/文档后关闭 |
|
||||
| #2772 | 与 #2340 合并为特定软路由 CPU/固件兼容问题 |
|
||||
| #2788 | maintainer 已说明黑名单语义；补订阅文档后关闭 |
|
||||
| #2792 | PM2 版本提示是镜像更新/支持问题，当前镜像复测后关闭 |
|
||||
| #2793 | 自动重启会掩盖 OOM；合并到性能问题，不单独实现定时自杀 |
|
||||
| #2798 | Node 在特定硬件启动即 fatal，合并硬件兼容问题 |
|
||||
| #2895 | Debian 中加载 musl 二进制通常是第三方依赖选错平台，转依赖支持 |
|
||||
| #2903 | 当前运行时代码未使用 chokidar 监听 node_modules；最新镜像复测后关闭 |
|
||||
| #2914 | maintainer 指定 2.20.1 已修复，当前版本复测后关闭 |
|
||||
| #2948 | 评论已有正确升级链和 `/ql/data` 挂载方案；补升级文档后关闭 |
|
||||
| #2964 | 本文已提供资源基线，并入 #3057 后关闭 |
|
||||
| #2967 | maintainer 已解释黑白名单语义；补文档后关闭 |
|
||||
| #3052 | `2ac4f07f` 已修复 dashboard open scope，关闭为已修复 |
|
||||
| #3053 | `2ac4f07f` 已加入 INI 预览/强制打开处理，关闭为已修复 |
|
||||
| #3059 | `31f78e4d` 已加入 IP 黑名单、客户端 IP 诊断和 trust proxy 配置，关闭为已修复 |
|
||||
|
||||
## 建议的 Issue 管理动作
|
||||
|
||||
1. 先关闭明确已实现的 #2219、#3052、#3053、#3059。
|
||||
2. 将 #2404、#2742、#2743 合并到 #3057，并把本文内存实测作为验收基线。
|
||||
3. 将 #2236、#2421 合并到 #3014；#2871 合并 #2902；#1695 合并 #2596；#2642/#2769/#2905 合并。
|
||||
4. 对“需要最新版复现”的旧 bug 统一贴模板，给出 14 天补充窗口；没有版本、架构、日志、最小复现则关闭为 stale/support。
|
||||
5. 轻量修复和架构 epic 使用不同 label/milestone，避免大设计长期占据回归队列。
|
||||
+5
-1
@@ -1,3 +1,5 @@
|
||||
const isContainer = process.env.QL_CONTAINER === 'true';
|
||||
|
||||
module.exports = {
|
||||
apps: [
|
||||
{
|
||||
@@ -7,7 +9,9 @@ module.exports = {
|
||||
wait_ready: true,
|
||||
listen_timeout: 5000,
|
||||
source_map_support: true,
|
||||
time: true,
|
||||
time: !isContainer,
|
||||
out_file: isContainer ? '/proc/1/fd/1' : undefined,
|
||||
error_file: isContainer ? '/proc/1/fd/2' : undefined,
|
||||
script: 'static/build/app.js',
|
||||
env: {
|
||||
http_proxy: '',
|
||||
|
||||
+6
-1
@@ -305,8 +305,13 @@ reload_pm2() {
|
||||
# Kill any existing node processes for qinglong
|
||||
pkill -f "node.*static/build/app.js" 2>/dev/null || true
|
||||
|
||||
# Start node directly in the background
|
||||
# Start node directly in the background. In containers, keep application
|
||||
# output on the container streams instead of duplicating Winston logs on disk.
|
||||
if [[ "$QL_CONTAINER" == "true" ]]; then
|
||||
nohup node static/build/app.js >/proc/1/fd/1 2>/proc/1/fd/2 &
|
||||
else
|
||||
nohup node static/build/app.js >$dir_log/qinglong.log 2>&1 &
|
||||
fi
|
||||
local node_pid=$!
|
||||
|
||||
t '已使用 Node.js 直接启动服务 (PID: %s)' "$node_pid"
|
||||
|
||||
@@ -366,6 +366,23 @@
|
||||
"添加标签成功": "Tags added successfully",
|
||||
"清空日志": "Clear Logs",
|
||||
"清除依赖缓存": "Clean dependency cache",
|
||||
"历史数据保留与手动清理": "History retention and manual cleanup",
|
||||
"保留天数为0时禁用对应清理，预览不会删除数据": "Set retention to 0 to disable that cleanup. Preview never deletes data.",
|
||||
"任务统计": "Task statistics",
|
||||
"清除 Node 依赖缓存": "Clean Node dependency cache",
|
||||
"清除 Python 依赖缓存": "Clean Python dependency cache",
|
||||
"清理后压缩数据库": "Compact database after cleanup",
|
||||
"保存设置": "Save settings",
|
||||
"预览清理": "Preview cleanup",
|
||||
"确认清理存储数据": "Confirm storage cleanup",
|
||||
"确认清理": "Clean now",
|
||||
"将删除历史运行实例": "Historical running instances to delete",
|
||||
"将删除任务统计": "Task statistics to delete",
|
||||
"将清除依赖缓存": "Dependency caches to clear",
|
||||
"清理依赖缓存后相关依赖需要重新安装": "Dependencies must be reinstalled after their cache is cleared.",
|
||||
"数据库压缩期间可能暂时阻塞请求": "Database compaction may briefly block requests.",
|
||||
"此操作不可恢复，请确认已完成必要备份": "This action cannot be undone. Confirm that required backups are complete.",
|
||||
"清理完成": "Cleanup complete",
|
||||
"清除成功": "Clean successful",
|
||||
"源文件": "Source File",
|
||||
"激活成功": "Activation successful",
|
||||
|
||||
@@ -366,6 +366,23 @@
|
||||
"添加标签成功": "添加标签成功",
|
||||
"清空日志": "清空日志",
|
||||
"清除依赖缓存": "清除依赖缓存",
|
||||
"历史数据保留与手动清理": "历史数据保留与手动清理",
|
||||
"保留天数为0时禁用对应清理，预览不会删除数据": "保留天数为0时禁用对应清理，预览不会删除数据",
|
||||
"任务统计": "任务统计",
|
||||
"清除 Node 依赖缓存": "清除 Node 依赖缓存",
|
||||
"清除 Python 依赖缓存": "清除 Python 依赖缓存",
|
||||
"清理后压缩数据库": "清理后压缩数据库",
|
||||
"保存设置": "保存设置",
|
||||
"预览清理": "预览清理",
|
||||
"确认清理存储数据": "确认清理存储数据",
|
||||
"确认清理": "确认清理",
|
||||
"将删除历史运行实例": "将删除历史运行实例",
|
||||
"将删除任务统计": "将删除任务统计",
|
||||
"将清除依赖缓存": "将清除依赖缓存",
|
||||
"清理依赖缓存后相关依赖需要重新安装": "清理依赖缓存后相关依赖需要重新安装",
|
||||
"数据库压缩期间可能暂时阻塞请求": "数据库压缩期间可能暂时阻塞请求",
|
||||
"此操作不可恢复，请确认已完成必要备份": "此操作不可恢复，请确认已完成必要备份",
|
||||
"清理完成": "清理完成",
|
||||
"清除成功": "清除成功",
|
||||
"源文件": "源文件",
|
||||
"激活成功": "激活成功",
|
||||
|
||||
@@ -16,11 +16,12 @@ import {
|
||||
CheckCircleOutlined,
|
||||
} from "@ant-design/icons";
|
||||
import { PageLoading } from "@ant-design/pro-layout";
|
||||
import { logEnded } from "@/utils";
|
||||
import { CrontabStatus } from "./type";
|
||||
import Ansi from "ansi-to-react";
|
||||
|
||||
const { Countdown } = Statistic;
|
||||
const LOG_CHUNK_BYTES = 256 * 1024;
|
||||
const MAX_LOG_VIEW_CHARS = 1024 * 1024;
|
||||
|
||||
const CronLogModal = ({
|
||||
cron,
|
||||
@@ -38,35 +39,49 @@ const CronLogModal = ({
|
||||
const [executing, setExecuting] = useState<any>(true);
|
||||
const [isPhone, setIsPhone] = useState(false);
|
||||
const scrollInfoRef = useRef({ value: 0, down: true });
|
||||
const logOffsetRef = useRef<number>();
|
||||
const valueRef = useRef(value);
|
||||
const pollTimerRef = useRef<ReturnType<typeof setTimeout>>();
|
||||
const uniqPath = logUrl ? logUrl : String(cron?.id);
|
||||
|
||||
const getCronLog = (isFirst?: boolean) => {
|
||||
if (isFirst) {
|
||||
setLoading(true);
|
||||
}
|
||||
const baseUrl = logUrl ? logUrl : `${config.apiPrefix}crons/${cron.id}/log`;
|
||||
const separator = baseUrl.includes("?") ? "&" : "?";
|
||||
const offset = isFirst ? undefined : logOffsetRef.current;
|
||||
const pagination = `${separator}limit=${LOG_CHUNK_BYTES}${
|
||||
isFirst ? "&tail=true" : offset !== undefined ? `&offset=${offset}` : ""
|
||||
}`;
|
||||
request
|
||||
.get(logUrl ? logUrl : `${config.apiPrefix}crons/${cron.id}/log`)
|
||||
.then(({ code, data, logStatus }) => {
|
||||
if (
|
||||
code === 200 &&
|
||||
localStorage.getItem("logCron") === uniqPath &&
|
||||
data !== value
|
||||
) {
|
||||
const log = (data as string) || intl.get("暂无日志");
|
||||
.get(`${baseUrl}${pagination}`)
|
||||
.then(({ code, data, logStatus, nextOffset }) => {
|
||||
if (code !== 200 || localStorage.getItem("logCron") !== uniqPath) {
|
||||
return;
|
||||
}
|
||||
|
||||
const hasNext = logStatus === "running";
|
||||
const chunk = (data as string) || "";
|
||||
let log = isFirst ? chunk : `${valueRef.current}${chunk}`;
|
||||
if (!log && !hasNext) {
|
||||
log = intl.get("暂无日志");
|
||||
}
|
||||
if (log.length > MAX_LOG_VIEW_CHARS) {
|
||||
log = log.slice(-MAX_LOG_VIEW_CHARS);
|
||||
}
|
||||
valueRef.current = log;
|
||||
setValue(log);
|
||||
const hasNext = logStatus === 'running';
|
||||
if (!hasNext && !logEnded(value) && value !== intl.get("启动中...")) {
|
||||
setTimeout(() => {
|
||||
autoScroll();
|
||||
});
|
||||
if (typeof nextOffset === "number") {
|
||||
logOffsetRef.current = nextOffset;
|
||||
}
|
||||
setExecuting(hasNext);
|
||||
if (hasNext) {
|
||||
setTimeout(() => {
|
||||
|
||||
if (chunk || !hasNext) {
|
||||
autoScroll();
|
||||
getCronLog();
|
||||
}, 2000);
|
||||
}
|
||||
if (hasNext) {
|
||||
pollTimerRef.current = setTimeout(() => getCronLog(), 2000);
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
@@ -89,6 +104,9 @@ const CronLogModal = ({
|
||||
};
|
||||
|
||||
const cancel = () => {
|
||||
if (pollTimerRef.current) {
|
||||
clearTimeout(pollTimerRef.current);
|
||||
}
|
||||
localStorage.removeItem("logCron");
|
||||
handleCancel();
|
||||
};
|
||||
@@ -117,12 +135,19 @@ const CronLogModal = ({
|
||||
|
||||
useEffect(() => {
|
||||
if (cron && cron.id) {
|
||||
logOffsetRef.current = undefined;
|
||||
getCronLog(true);
|
||||
}
|
||||
return () => {
|
||||
if (pollTimerRef.current) {
|
||||
clearTimeout(pollTimerRef.current);
|
||||
}
|
||||
};
|
||||
}, [cron]);
|
||||
|
||||
useEffect(() => {
|
||||
if (data) {
|
||||
valueRef.current = data;
|
||||
setValue(data);
|
||||
}
|
||||
}, [data]);
|
||||
|
||||
@@ -59,7 +59,7 @@ const Log = () => {
|
||||
.get(
|
||||
`${config.apiPrefix}logs/detail?file=${node.title}&path=${
|
||||
node.parent || ''
|
||||
}`,
|
||||
}&tail=true&limit=${1024 * 1024}`,
|
||||
)
|
||||
.then(({ code, data }) => {
|
||||
if (code === 200) {
|
||||
|
||||
@@ -51,7 +51,6 @@ const Setting = () => {
|
||||
reloadTheme,
|
||||
systemInfo,
|
||||
} = useOutletContext<SharedContext>();
|
||||
console.log('user', user);
|
||||
const columns = [
|
||||
{
|
||||
title: intl.get('名称'),
|
||||
|
||||
@@ -62,6 +62,8 @@ const Other = ({
|
||||
cronConcurrency?: number | null;
|
||||
timezone?: string | null;
|
||||
globalSshKey?: string | null;
|
||||
runningInstanceRetentionDays?: number | null;
|
||||
cronStatRetentionDays?: number | null;
|
||||
}>();
|
||||
const [form] = Form.useForm();
|
||||
const [exportLoading, setExportLoading] = useState(false);
|
||||
@@ -69,6 +71,11 @@ const Other = ({
|
||||
const showDownloadProgress = useProgress(intl.get('下载'));
|
||||
const [visible, setVisible] = useState(false);
|
||||
const [selectedModules, setSelectedModules] = useState<string[]>(['base']);
|
||||
const [cleanupLoading, setCleanupLoading] = useState(false);
|
||||
const [dependenceCacheTypes, setDependenceCacheTypes] = useState<string[]>(
|
||||
[],
|
||||
);
|
||||
const [compactDatabase, setCompactDatabase] = useState(false);
|
||||
|
||||
const {
|
||||
enable: enableDarkMode,
|
||||
@@ -136,6 +143,93 @@ const Other = ({
|
||||
});
|
||||
};
|
||||
|
||||
const retentionPayload = () => ({
|
||||
runningInstanceRetentionDays:
|
||||
systemConfig?.runningInstanceRetentionDays || 0,
|
||||
cronStatRetentionDays: systemConfig?.cronStatRetentionDays || 0,
|
||||
});
|
||||
|
||||
const saveRetentionPolicy = () => {
|
||||
request
|
||||
.put(
|
||||
`${config.apiPrefix}system/storage-retention/config`,
|
||||
retentionPayload(),
|
||||
)
|
||||
.then(({ code }) => {
|
||||
if (code === 200) {
|
||||
message.success(intl.get('更新成功'));
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const formatBytes = (bytes: number) => {
|
||||
if (!bytes) return '0 B';
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB'];
|
||||
const unit = Math.min(
|
||||
Math.floor(Math.log(bytes) / Math.log(1024)),
|
||||
units.length - 1,
|
||||
);
|
||||
return `${(bytes / 1024 ** unit).toFixed(unit ? 1 : 0)} ${units[unit]}`;
|
||||
};
|
||||
|
||||
const previewStorageCleanup = () => {
|
||||
setCleanupLoading(true);
|
||||
const payload = {
|
||||
...retentionPayload(),
|
||||
dependenceCacheTypes,
|
||||
compactDatabase,
|
||||
};
|
||||
request
|
||||
.post(`${config.apiPrefix}system/storage-retention/preview`, payload)
|
||||
.then(({ code, data }) => {
|
||||
if (code !== 200) return;
|
||||
Modal.confirm({
|
||||
width: 560,
|
||||
centered: true,
|
||||
title: intl.get('确认清理存储数据'),
|
||||
okText: intl.get('确认清理'),
|
||||
cancelText: intl.get('取消'),
|
||||
okButtonProps: { danger: true },
|
||||
content: (
|
||||
<div>
|
||||
<p>
|
||||
{intl.get('将删除历史运行实例')}: {data.runningInstances}
|
||||
</p>
|
||||
<p>
|
||||
{intl.get('将删除任务统计')}: {data.cronStats}
|
||||
</p>
|
||||
<p>
|
||||
{intl.get('将清除依赖缓存')}: {data.dependenceCaches.length} (
|
||||
{formatBytes(data.dependenceCacheBytes)})
|
||||
</p>
|
||||
{data.dependenceCaches.length > 0 && (
|
||||
<p>{intl.get('清理依赖缓存后相关依赖需要重新安装')}</p>
|
||||
)}
|
||||
{compactDatabase && (
|
||||
<p>{intl.get('数据库压缩期间可能暂时阻塞请求')}</p>
|
||||
)}
|
||||
<p>{intl.get('此操作不可恢复，请确认已完成必要备份')}</p>
|
||||
</div>
|
||||
),
|
||||
onOk: () => {
|
||||
setCleanupLoading(true);
|
||||
return request
|
||||
.post(`${config.apiPrefix}system/storage-retention/cleanup`, {
|
||||
...payload,
|
||||
confirmation: 'CLEAN',
|
||||
})
|
||||
.then(({ code }) => {
|
||||
if (code === 200) {
|
||||
message.success(intl.get('清理完成'));
|
||||
}
|
||||
})
|
||||
.finally(() => setCleanupLoading(false));
|
||||
},
|
||||
});
|
||||
})
|
||||
.finally(() => setCleanupLoading(false));
|
||||
};
|
||||
|
||||
const exportData = () => {
|
||||
setExportLoading(true);
|
||||
request
|
||||
@@ -302,6 +396,69 @@ const Other = ({
|
||||
</Button>
|
||||
</Input.Group>
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label={intl.get('历史数据保留与手动清理')}
|
||||
tooltip={intl.get('保留天数为0时禁用对应清理，预览不会删除数据')}
|
||||
>
|
||||
<div style={{ marginBottom: 8 }}>
|
||||
<InputNumber
|
||||
addonBefore={intl.get('运行实例')}
|
||||
addonAfter={intl.get('天')}
|
||||
min={0}
|
||||
max={3650}
|
||||
value={systemConfig?.runningInstanceRetentionDays || 0}
|
||||
onChange={(value) => {
|
||||
setSystemConfig({
|
||||
...systemConfig,
|
||||
runningInstanceRetentionDays: value,
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<div style={{ marginBottom: 8 }}>
|
||||
<InputNumber
|
||||
addonBefore={intl.get('任务统计')}
|
||||
addonAfter={intl.get('天')}
|
||||
min={0}
|
||||
max={3650}
|
||||
value={systemConfig?.cronStatRetentionDays || 0}
|
||||
onChange={(value) => {
|
||||
setSystemConfig({
|
||||
...systemConfig,
|
||||
cronStatRetentionDays: value,
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<div style={{ marginBottom: 8 }}>
|
||||
<Checkbox.Group
|
||||
value={dependenceCacheTypes}
|
||||
options={[
|
||||
{ label: intl.get('清除 Node 依赖缓存'), value: 'node' },
|
||||
{ label: intl.get('清除 Python 依赖缓存'), value: 'python3' },
|
||||
]}
|
||||
onChange={(value) => setDependenceCacheTypes(value as string[])}
|
||||
/>
|
||||
</div>
|
||||
<div style={{ marginBottom: 8 }}>
|
||||
<Checkbox
|
||||
checked={compactDatabase}
|
||||
onChange={(event) => setCompactDatabase(event.target.checked)}
|
||||
>
|
||||
{intl.get('清理后压缩数据库')}
|
||||
</Checkbox>
|
||||
</div>
|
||||
<Button onClick={saveRetentionPolicy} style={{ marginRight: 8 }}>
|
||||
{intl.get('保存设置')}
|
||||
</Button>
|
||||
<Button
|
||||
danger
|
||||
loading={cleanupLoading}
|
||||
onClick={previewStorageCleanup}
|
||||
>
|
||||
{intl.get('预览清理')}
|
||||
</Button>
|
||||
</Form.Item>
|
||||
<Form.Item label={intl.get('定时任务并发数')} name="frequency">
|
||||
<Input.Group compact>
|
||||
<InputNumber
|
||||
|
||||
@@ -29,7 +29,9 @@ const SystemLog = ({ height, theme }: any) => {
|
||||
const { loading, refresh } = useRequest(
|
||||
() => {
|
||||
return request.get<Blob>(
|
||||
`${config.apiPrefix}system/log?startTime=${range[0]}&endTime=${range[1]}`,
|
||||
`${config.apiPrefix}system/log?startTime=${range[0]}&endTime=${
|
||||
range[1]
|
||||
}&limit=${1024 * 1024}`,
|
||||
{
|
||||
responseType: 'blob',
|
||||
},
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const test = require('node:test');
|
||||
|
||||
const configPath = require.resolve('../../ecosystem.config');
|
||||
|
||||
function loadConfig(containerValue) {
|
||||
if (containerValue === undefined) {
|
||||
delete process.env.QL_CONTAINER;
|
||||
} else {
|
||||
process.env.QL_CONTAINER = containerValue;
|
||||
}
|
||||
delete require.cache[configPath];
|
||||
return require(configPath).apps[0];
|
||||
}
|
||||
|
||||
test('container logging goes to the container standard streams', () => {
|
||||
const app = loadConfig('true');
|
||||
assert.equal(app.out_file, '/proc/1/fd/1');
|
||||
assert.equal(app.error_file, '/proc/1/fd/2');
|
||||
assert.equal(app.time, false);
|
||||
});
|
||||
|
||||
test('non-container installs keep the PM2 logging defaults', () => {
|
||||
const app = loadConfig(undefined);
|
||||
assert.equal(app.out_file, undefined);
|
||||
assert.equal(app.error_file, undefined);
|
||||
assert.equal(app.time, true);
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const test = require('node:test');
|
||||
|
||||
const {
|
||||
DEFAULT_LOG_CHUNK_BYTES,
|
||||
MAX_LOG_CHUNK_BYTES,
|
||||
readLogChunk,
|
||||
} = require('../../back/shared/logReader');
|
||||
|
||||
test('log reader defaults to a bounded tail and supports incremental reads', async (t) => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-reader-'));
|
||||
const file = path.join(directory, 'task.log');
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
|
||||
const initial = 'a'.repeat(DEFAULT_LOG_CHUNK_BYTES * 2);
|
||||
await fs.writeFile(file, initial);
|
||||
|
||||
const tail = await readLogChunk(file);
|
||||
assert.equal(tail.content.length, DEFAULT_LOG_CHUNK_BYTES);
|
||||
assert.equal(tail.offset, DEFAULT_LOG_CHUNK_BYTES);
|
||||
assert.equal(tail.nextOffset, initial.length);
|
||||
assert.equal(tail.truncated, true);
|
||||
|
||||
await fs.appendFile(file, 'next');
|
||||
const incremental = await readLogChunk(file, { offset: tail.nextOffset });
|
||||
assert.equal(incremental.content, 'next');
|
||||
assert.equal(incremental.nextOffset, initial.length + 4);
|
||||
});
|
||||
|
||||
test('log reader enforces the maximum chunk size', async (t) => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-reader-'));
|
||||
const file = path.join(directory, 'large.log');
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
await fs.writeFile(file, 'x'.repeat(MAX_LOG_CHUNK_BYTES + 1024));
|
||||
|
||||
const chunk = await readLogChunk(file, { limit: Number.MAX_SAFE_INTEGER });
|
||||
assert.equal(Buffer.byteLength(chunk.content), MAX_LOG_CHUNK_BYTES);
|
||||
});
|
||||
|
||||
test('log reader preserves UTF-8 characters across byte boundaries', async (t) => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-reader-'));
|
||||
const file = path.join(directory, 'utf8.log');
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
await fs.writeFile(file, 'a'.repeat(5) + '中文日志');
|
||||
|
||||
const first = await readLogChunk(file, { offset: 0, limit: 7 });
|
||||
const second = await readLogChunk(file, {
|
||||
offset: first.nextOffset,
|
||||
limit: 7,
|
||||
});
|
||||
const third = await readLogChunk(file, {
|
||||
offset: second.nextOffset,
|
||||
limit: 7,
|
||||
});
|
||||
|
||||
assert.equal(first.content + second.content + third.content, 'aaaaa中文日志');
|
||||
assert.equal((first.content + second.content + third.content).includes('�'), false);
|
||||
});
|
||||
|
||||
test('missing logs return an empty chunk', async () => {
|
||||
const chunk = await readLogChunk('/path/that/does/not/exist.log');
|
||||
assert.deepEqual(chunk, {
|
||||
content: '',
|
||||
offset: 0,
|
||||
nextOffset: 0,
|
||||
total: 0,
|
||||
truncated: false,
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const test = require('node:test');
|
||||
|
||||
const {
|
||||
MAX_METRIC_SAMPLES,
|
||||
metricsService,
|
||||
} = require('../../back/services/metrics');
|
||||
|
||||
test('metrics keep only the newest bounded samples', () => {
|
||||
const sampleCount = MAX_METRIC_SAMPLES * 50;
|
||||
for (let index = 0; index < sampleCount; index++) {
|
||||
metricsService.record('http_request', index, {
|
||||
path: `/health/${index}`,
|
||||
});
|
||||
}
|
||||
|
||||
const result = metricsService.getMetrics('http_request');
|
||||
assert.equal(result.count, MAX_METRIC_SAMPLES);
|
||||
assert.equal(result.metrics[0].value, sampleCount - MAX_METRIC_SAMPLES);
|
||||
assert.equal(result.metrics.at(-1).value, sampleCount - 1);
|
||||
});
|
||||
|
||||
test('empty metric queries return finite aggregates', () => {
|
||||
const result = metricsService.getMetrics('missing_metric');
|
||||
assert.deepEqual(
|
||||
{
|
||||
count: result.count,
|
||||
average: result.average,
|
||||
min: result.min,
|
||||
max: result.max,
|
||||
},
|
||||
{ count: 0, average: 0, min: 0, max: 0 },
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const test = require('node:test');
|
||||
const { Op } = require('sequelize');
|
||||
|
||||
const {
|
||||
getDirectorySize,
|
||||
normalizeRetentionPolicy,
|
||||
} = require('../../back/shared/retention');
|
||||
|
||||
test('retention policy defaults to disabled and clamps invalid values', () => {
|
||||
assert.deepEqual(normalizeRetentionPolicy({}), {
|
||||
runningInstanceRetentionDays: 0,
|
||||
cronStatRetentionDays: 0,
|
||||
});
|
||||
assert.deepEqual(
|
||||
normalizeRetentionPolicy({
|
||||
runningInstanceRetentionDays: -10,
|
||||
cronStatRetentionDays: 99999,
|
||||
}),
|
||||
{
|
||||
runningInstanceRetentionDays: 0,
|
||||
cronStatRetentionDays: 3650,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('directory preview counts files recursively and ignores symlinks', async (t) => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-retention-'));
|
||||
const nested = path.join(directory, 'nested');
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
await fs.mkdir(nested);
|
||||
await fs.writeFile(path.join(directory, 'one'), Buffer.alloc(7));
|
||||
await fs.writeFile(path.join(nested, 'two'), Buffer.alloc(11));
|
||||
await fs.symlink(path.join(nested, 'two'), path.join(directory, 'link'));
|
||||
|
||||
assert.equal(await getDirectorySize(directory), 18);
|
||||
assert.equal(await getDirectorySize(path.join(directory, 'missing')), 0);
|
||||
});
|
||||
|
||||
test('cleanup previews first, protects running instances, and uses explicit options', async (t) => {
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-retention-'));
|
||||
const nodeCache = path.join(directory, 'node');
|
||||
await fs.mkdir(nodeCache);
|
||||
await fs.writeFile(path.join(nodeCache, 'cache'), Buffer.alloc(13));
|
||||
|
||||
const config = require('../../back/config').default;
|
||||
const originalCachePath = config.dependenceCachePath;
|
||||
const RunningInstanceModel = {};
|
||||
const CrontabStatModel = {};
|
||||
const InstanceStatus = { running: 0 };
|
||||
const sequelize = {};
|
||||
const stubModule = (modulePath, exports) => {
|
||||
require.cache[require.resolve(modulePath)] = {
|
||||
id: require.resolve(modulePath),
|
||||
filename: require.resolve(modulePath),
|
||||
loaded: true,
|
||||
exports,
|
||||
children: [],
|
||||
paths: [],
|
||||
};
|
||||
};
|
||||
stubModule('../../back/data', { sequelize });
|
||||
stubModule('../../back/data/cronStats', { CrontabStatModel });
|
||||
stubModule('../../back/data/runningInstance', {
|
||||
InstanceStatus,
|
||||
RunningInstanceModel,
|
||||
});
|
||||
stubModule('../../back/data/system', {
|
||||
AuthDataType: { systemConfig: 'systemConfig' },
|
||||
SystemModel: {},
|
||||
});
|
||||
delete require.cache[require.resolve('../../back/services/retention')];
|
||||
const RetentionService = require('../../back/services/retention').default;
|
||||
|
||||
t.after(async () => {
|
||||
config.dependenceCachePath = originalCachePath;
|
||||
await fs.rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
let instanceWhere;
|
||||
let vacuumed = false;
|
||||
let instanceCountCalls = 0;
|
||||
let statCountCalls = 0;
|
||||
config.dependenceCachePath = directory;
|
||||
RunningInstanceModel.count = async ({ where }) => {
|
||||
instanceCountCalls++;
|
||||
instanceWhere = where;
|
||||
return 2;
|
||||
};
|
||||
RunningInstanceModel.destroy = async ({ where }) => {
|
||||
instanceWhere = where;
|
||||
return 2;
|
||||
};
|
||||
CrontabStatModel.count = async () => {
|
||||
statCountCalls++;
|
||||
return 3;
|
||||
};
|
||||
CrontabStatModel.destroy = async () => 3;
|
||||
sequelize.transaction = async (callback) => callback({});
|
||||
sequelize.query = async (query) => {
|
||||
vacuumed = query === 'VACUUM';
|
||||
return [];
|
||||
};
|
||||
|
||||
const service = new RetentionService();
|
||||
const disabledPreview = await service.preview({
|
||||
runningInstanceRetentionDays: 0,
|
||||
cronStatRetentionDays: 0,
|
||||
});
|
||||
assert.equal(disabledPreview.runningInstances, 0);
|
||||
assert.equal(disabledPreview.cronStats, 0);
|
||||
assert.equal(instanceCountCalls, 0);
|
||||
assert.equal(statCountCalls, 0);
|
||||
|
||||
const result = await service.cleanup({
|
||||
runningInstanceRetentionDays: 30,
|
||||
cronStatRetentionDays: 90,
|
||||
dependenceCacheTypes: ['node'],
|
||||
compactDatabase: true,
|
||||
});
|
||||
|
||||
assert.equal(instanceWhere.status[Op.ne], InstanceStatus.running);
|
||||
assert.equal(result.deleted.runningInstances, 2);
|
||||
assert.equal(result.deleted.cronStats, 3);
|
||||
assert.equal(result.deleted.dependenceCacheBytes, 13);
|
||||
assert.equal(vacuumed, true);
|
||||
await assert.rejects(fs.stat(nodeCache), { code: 'ENOENT' });
|
||||
});
|
||||
Reference in New Issue
Block a user