perf: bound runtime memory and storage usage

back/api/cron.ts

@@ -307,13 +307,35 @@ export default (app: Router) => {
       params: Joi.object({
         id: Joi.number().required(),
       }),
+      query: Joi.object({
+        offset: Joi.number().integer().min(0).optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(1024 * 1024)
+          .optional(),
+        tail: Joi.boolean().optional(),
+        t: Joi.string().optional(),
+      }).unknown(true),
     }),
     async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
       const logger: Logger = Container.get('logger');
       try {
         const cronService = Container.get(CronService);
-        const result = await cronService.log(req.params.id);
+        const result = await cronService.log(req.params.id, {
-        return res.send({ code: 200, data: result.content, logStatus: result.status });
+          offset: req.query.offset as unknown as number,
+          limit: req.query.limit as unknown as number,
+          tail: req.query.tail as unknown as boolean,
+        });
+        return res.send({
+          code: 200,
+          data: result.content,
+          logStatus: result.status,
+          offset: result.offset,
+          nextOffset: result.nextOffset,
+          total: result.total,
+          truncated: result.truncated,
+        });
       } catch (e) {
         return next(e);
       }

back/api/index.ts

@@ -13,6 +13,7 @@ import update from './update';
 import dashboard from './dashboard';
 import health from './health';
 import clientIp from './clientIp';
+import retention from './retention';
 
 export default () => {
   const app = Router();
@@ -30,6 +31,7 @@ export default () => {
   dashboard(app);
   health(app);
   clientIp(app);
+  retention(app);
 
   return app;
 };

back/api/log.ts

@@ -12,6 +12,7 @@ import {
 } from '../config/util';
 import LogService from '../services/log';
 import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
+import { MAX_LOG_CHUNK_BYTES, readLogChunk } from '../shared/logReader';
 const route = Router();
 const blacklist = ['.tmp'];
 
@@ -34,6 +35,20 @@ export default (app: Router) => {
 
   route.get(
     '/detail',
+    celebrate({
+      query: Joi.object({
+        path: Joi.string().allow('').optional(),
+        file: Joi.string().required(),
+        offset: Joi.number().integer().min(0).optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(MAX_LOG_CHUNK_BYTES)
+          .optional(),
+        tail: Joi.boolean().optional(),
+        t: Joi.string().optional(),
+      }).unknown(true),
+    }),
     async (req: Request, res: Response, next: NextFunction) => {
       try {
         const logService = Container.get(LogService);
@@ -52,11 +67,19 @@ export default (app: Router) => {
           where: { log_path: logPath, status: InstanceStatus.running },
         });
 
-        const content = await getFileContentByName(finalPath);
+        const chunk = await readLogChunk(finalPath, {
+          offset: req.query.offset as unknown as number,
+          limit: req.query.limit as unknown as number,
+          tail: req.query.tail as unknown as boolean,
+        });
         res.send({
           code: 200,
-          data: removeAnsi(content),
+          data: removeAnsi(chunk.content),
           logStatus: runningInstance ? 'running' : undefined,
+          offset: chunk.offset,
+          nextOffset: chunk.nextOffset,
+          total: chunk.total,
+          truncated: chunk.truncated,
         });
       } catch (e) {
         return next(e);

back/api/retention.ts

@@ -0,0 +1,78 @@
+import { NextFunction, Request, Response, Router } from 'express';
+import { celebrate, Joi } from 'celebrate';
+import { Container } from 'typedi';
+import RetentionService from '../services/retention';
+import { MAX_RETENTION_DAYS } from '../shared/retention';
+
+const route = Router();
+const policySchema = {
+  runningInstanceRetentionDays: Joi.number()
+    .integer()
+    .min(0)
+    .max(MAX_RETENTION_DAYS)
+    .required(),
+  cronStatRetentionDays: Joi.number()
+    .integer()
+    .min(0)
+    .max(MAX_RETENTION_DAYS)
+    .required(),
+};
+const cleanupSchema = {
+  ...policySchema,
+  dependenceCacheTypes: Joi.array()
+    .items(Joi.string().valid('node', 'python3'))
+    .unique()
+    .default([]),
+  compactDatabase: Joi.boolean().default(false),
+};
+
+export default (app: Router) => {
+  app.use('/system/storage-retention', route);
+
+  route.put(
+    '/config',
+    celebrate({ body: Joi.object(policySchema) }),
+    async (req: Request, res: Response, next: NextFunction) => {
+      try {
+        const service = Container.get(RetentionService);
+        const data = await service.updatePolicy(req.body);
+        res.send({ code: 200, data });
+      } catch (error) {
+        next(error);
+      }
+    },
+  );
+
+  route.post(
+    '/preview',
+    celebrate({ body: Joi.object(cleanupSchema) }),
+    async (req: Request, res: Response, next: NextFunction) => {
+      try {
+        const service = Container.get(RetentionService);
+        const data = await service.preview(req.body);
+        res.send({ code: 200, data });
+      } catch (error) {
+        next(error);
+      }
+    },
+  );
+
+  route.post(
+    '/cleanup',
+    celebrate({
+      body: Joi.object({
+        ...cleanupSchema,
+        confirmation: Joi.string().valid('CLEAN').required(),
+      }),
+    }),
+    async (req: Request, res: Response, next: NextFunction) => {
+      try {
+        const service = Container.get(RetentionService);
+        const data = await service.cleanup(req.body);
+        res.send({ code: 200, data });
+      } catch (error) {
+        next(error);
+      }
+    },
+  );
+};

back/api/subscription.ts

@@ -148,13 +148,27 @@ export default (app: Router) => {
       params: Joi.object({
         id: Joi.number().required(),
       }),
+      query: Joi.object({
+        offset: Joi.number().integer().min(0).optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(1024 * 1024)
+          .optional(),
+        tail: Joi.boolean().optional(),
+        t: Joi.string().optional(),
+      }).unknown(true),
     }),
     async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
       const logger: Logger = Container.get('logger');
       try {
         const subscriptionService = Container.get(SubscriptionService);
-        const data = await subscriptionService.log(req.params.id);
+        const result = await subscriptionService.log(req.params.id, {
-        return res.send({ code: 200, data });
+          offset: req.query.offset as unknown as number,
+          limit: req.query.limit as unknown as number,
+          tail: req.query.tail as unknown as boolean,
+        });
+        return res.send({ code: 200, data: result.content, ...result });
       } catch (e) {
         return next(e);
       }

back/api/system.ts

@@ -350,6 +350,11 @@ export default (app: Router) => {
       query: {
         startTime: Joi.string().allow('').optional(),
         endTime: Joi.string().allow('').optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(1024 * 1024)
+          .optional(),
         t: Joi.string().optional(),
       },
     }),
@@ -361,6 +366,7 @@ export default (app: Router) => {
           req.query as {
             startTime?: string;
             endTime?: string;
+            limit?: number;
           },
         );
       } catch (e) {

back/data/system.ts

@@ -41,6 +41,8 @@ export interface SystemConfigInfo {
   linuxMirror?: string;
   timezone?: string;
   globalSshKey?: string;
+  runningInstanceRetentionDays?: number;
+  cronStatRetentionDays?: number;
 }
 
 export interface LoginLogInfo {

back/services/cron.ts

@@ -32,6 +32,7 @@ import { t } from '../shared/i18n';
 import { ScheduleType } from '../interface/schedule';
 import { logStreamManager } from '../shared/logStreamManager';
 import { isEmpty } from 'lodash';
+import { LogReadOptions, readLogChunk } from '../shared/logReader';
 
 @Service()
 export default class CronService {
@@ -766,27 +767,63 @@ export default class CronService {
     await this.setCrontab();
   }
 
-  public async log(id: number): Promise<{ content: string; status: string }> {
+  public async log(
+    id: number,
+    options: LogReadOptions = {},
+  ): Promise<{
+    content: string;
+    status: string;
+    offset: number;
+    nextOffset: number;
+    total: number;
+    truncated: boolean;
+  }> {
     const doc = await this.getDb({ id });
     if (!doc) {
-      return { content: '', status: 'empty' };
+      return {
+        content: '',
+        status: 'empty',
+        offset: 0,
+        nextOffset: 0,
+        total: 0,
+        truncated: false,
+      };
     }
     if (doc.log_name === '/dev/null') {
-      return { content: t('日志设置为忽略'), status: 'ignored' };
+      return {
+        content: t('日志设置为忽略'),
+        status: 'ignored',
+        offset: 0,
+        nextOffset: 0,
+        total: 0,
+        truncated: false,
+      };
     }
     const absolutePath = path.resolve(config.logPath, `${doc.log_path}`);
     const logFileExist = doc.log_path && (await fileExist(absolutePath));
     if (logFileExist) {
-      const content = await getFileContentByName(`${absolutePath}`);
+      const chunk = await readLogChunk(`${absolutePath}`, options);
       const isRunning =
         typeof doc.status === 'number' &&
         [CrontabStatus.running, CrontabStatus.queued].includes(doc.status);
-      return { content, status: isRunning ? 'running' : 'completed' };
+      return {
+        ...chunk,
+        status: isRunning ? 'running' : 'completed',
+      };
     } else {
-      return typeof doc.status === 'number' &&
+      const status =
+        typeof doc.status === 'number' &&
         [CrontabStatus.queued, CrontabStatus.running].includes(doc.status)
-        ? { content: t('运行中...'), status: 'running' }
+          ? 'running'
-        : { content: t('日志不存在...'), status: 'notFound' };
+          : 'notFound';
+      return {
+        content: status === 'running' ? t('运行中...') : t('日志不存在...'),
+        status,
+        offset: 0,
+        nextOffset: 0,
+        total: 0,
+        truncated: false,
+      };
     }
   }
 

back/services/metrics.ts

@@ -8,16 +8,19 @@ interface Metric {
   tags?: Record<string, string>;
 }
 
-class MetricsService {
+export const MAX_METRIC_SAMPLES = 1000;
-  private metrics: Metric[] = [];
+const METRIC_RETENTION_MS = 60 * 60 * 1000;
+
+export class MetricsService {
+  private metrics: Array<Metric | undefined> = new Array(MAX_METRIC_SAMPLES);
+  private metricCount = 0;
+  private nextMetricIndex = 0;
   private static instance: MetricsService;
 
   private constructor() {
     // 定期清理旧数据
-    setInterval(() => {
+    const cleanupTimer = setInterval(() => this.removeExpiredMetrics(), 60000);
-      const oneHourAgo = Date.now() - 3600000;
+    cleanupTimer.unref();
-      this.metrics = this.metrics.filter(m => m.timestamp > oneHourAgo);
-    }, 60000);
   }
 
   static getInstance(): MetricsService {
@@ -28,12 +31,14 @@ class MetricsService {
   }
 
   record(name: string, value: number, tags?: Record<string, string>) {
-    this.metrics.push({
+    this.metrics[this.nextMetricIndex] = {
       name,
       value,
       timestamp: Date.now(),
       tags,
-    });
+    };
+    this.nextMetricIndex = (this.nextMetricIndex + 1) % MAX_METRIC_SAMPLES;
+    this.metricCount = Math.min(this.metricCount + 1, MAX_METRIC_SAMPLES);
   }
 
   measure(name: string, fn: () => void, tags?: Record<string, string>) {
@@ -46,7 +51,11 @@ class MetricsService {
     }
   }
 
-  async measureAsync(name: string, fn: () => Promise<void>, tags?: Record<string, string>) {
+  async measureAsync(
+    name: string,
+    fn: () => Promise<void>,
+    tags?: Record<string, string>,
+  ) {
     const start = performance.now();
     try {
       await fn();
@@ -57,28 +66,61 @@ class MetricsService {
   }
 
   getMetrics(name?: string, tags?: Record<string, string>) {
-    let filtered = this.metrics;
+    this.removeExpiredMetrics();
-    
+    let filtered = this.getMetricSnapshot();
+
     if (name) {
-      filtered = filtered.filter(m => m.name === name);
+      filtered = filtered.filter((m) => m.name === name);
     }
-    
+
     if (tags) {
-      filtered = filtered.filter(m => {
+      filtered = filtered.filter((m) => {
         if (!m.tags) return false;
-        return Object.entries(tags).every(([key, value]) => m.tags![key] === value);
+        return Object.entries(tags).every(
+          ([key, value]) => m.tags![key] === value,
+        );
       });
     }
 
+    const values = filtered.map((metric) => metric.value);
     return {
       count: filtered.length,
-      average: filtered.reduce((acc, curr) => acc + curr.value, 0) / filtered.length,
+      average: values.length
-      min: Math.min(...filtered.map(m => m.value)),
+        ? values.reduce((acc, value) => acc + value, 0) / values.length
-      max: Math.max(...filtered.map(m => m.value)),
+        : 0,
+      min: values.length ? Math.min(...values) : 0,
+      max: values.length ? Math.max(...values) : 0,
       metrics: filtered,
     };
   }
 
+  private getMetricSnapshot(): Metric[] {
+    if (this.metricCount < MAX_METRIC_SAMPLES) {
+      return this.metrics.slice(0, this.metricCount) as Metric[];
+    }
+    return [
+      ...this.metrics.slice(this.nextMetricIndex),
+      ...this.metrics.slice(0, this.nextMetricIndex),
+    ] as Metric[];
+  }
+
+  private removeExpiredMetrics() {
+    const oldestTimestamp = Date.now() - METRIC_RETENTION_MS;
+    const retained = this.getMetricSnapshot().filter(
+      (metric) => metric.timestamp > oldestTimestamp,
+    );
+    if (retained.length === this.metricCount) return;
+
+    this.metrics = new Array(MAX_METRIC_SAMPLES);
+    this.metricCount = 0;
+    this.nextMetricIndex = 0;
+    for (const metric of retained) {
+      this.metrics[this.nextMetricIndex] = metric;
+      this.nextMetricIndex = (this.nextMetricIndex + 1) % MAX_METRIC_SAMPLES;
+      this.metricCount++;
+    }
+  }
+
   report() {
     const report = {
       timestamp: Date.now(),
@@ -89,4 +131,4 @@ class MetricsService {
   }
 }
 
-export const metricsService = MetricsService.getInstance(); 
+export const metricsService = MetricsService.getInstance();

back/services/retention.ts

@@ -0,0 +1,155 @@
+import fs from 'fs/promises';
+import path from 'path';
+import dayjs from 'dayjs';
+import { Op } from 'sequelize';
+import { Service } from 'typedi';
+import config from '../config';
+import { CrontabStatModel } from '../data/cronStats';
+import { sequelize } from '../data';
+import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
+import { AuthDataType, SystemModel } from '../data/system';
+import {
+  DependenceCacheType,
+  getDirectorySize,
+  isDependenceCacheType,
+  normalizeRetentionPolicy,
+  RetentionPolicy,
+} from '../shared/retention';
+
+export interface StorageCleanupRequest extends RetentionPolicy {
+  dependenceCacheTypes?: DependenceCacheType[];
+  compactDatabase?: boolean;
+}
+
+function runningInstanceWhere(days: number) {
+  const cutoff = dayjs().subtract(days, 'day').unix();
+  return {
+    status: { [Op.ne]: InstanceStatus.running },
+    [Op.or]: [
+      { finished_at: { [Op.lt]: cutoff } },
+      {
+        finished_at: { [Op.is]: null },
+        started_at: { [Op.lt]: cutoff },
+      },
+    ],
+  };
+}
+
+function cronStatWhere(days: number) {
+  return {
+    date: { [Op.lt]: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
+  };
+}
+
+@Service()
+export default class RetentionService {
+  public async updatePolicy(policy: Partial<RetentionPolicy>) {
+    const normalized = normalizeRetentionPolicy(policy);
+    const systemConfig = await SystemModel.findOne({
+      where: { type: AuthDataType.systemConfig },
+    });
+    if (!systemConfig) {
+      throw new Error('System config not found');
+    }
+    await SystemModel.update(
+      {
+        info: {
+          ...systemConfig.info,
+          ...normalized,
+        },
+      },
+      { where: { id: systemConfig.id } },
+    );
+    return normalized;
+  }
+
+  public async preview(request: StorageCleanupRequest) {
+    const policy = normalizeRetentionPolicy(request);
+    const dependenceCacheTypes = (request.dependenceCacheTypes || []).filter(
+      isDependenceCacheType,
+    );
+    const [runningInstances, cronStats, dependenceCaches] = await Promise.all([
+      policy.runningInstanceRetentionDays > 0
+        ? RunningInstanceModel.count({
+            where: runningInstanceWhere(policy.runningInstanceRetentionDays),
+          })
+        : 0,
+      policy.cronStatRetentionDays > 0
+        ? CrontabStatModel.count({
+            where: cronStatWhere(policy.cronStatRetentionDays),
+          })
+        : 0,
+      Promise.all(
+        dependenceCacheTypes.map(async (type) => ({
+          type,
+          bytes: await getDirectorySize(
+            path.join(config.dependenceCachePath, type),
+          ),
+        })),
+      ),
+    ]);
+
+    return {
+      policy,
+      runningInstances,
+      cronStats,
+      dependenceCaches,
+      dependenceCacheBytes: dependenceCaches.reduce(
+        (total, cache) => total + cache.bytes,
+        0,
+      ),
+      compactDatabase: Boolean(request.compactDatabase),
+    };
+  }
+
+  public async cleanup(request: StorageCleanupRequest) {
+    const preview = await this.preview(request);
+    const deleted = await sequelize.transaction(async (transaction) => {
+      const runningInstances =
+        preview.policy.runningInstanceRetentionDays > 0
+          ? await RunningInstanceModel.destroy({
+              where: runningInstanceWhere(
+                preview.policy.runningInstanceRetentionDays,
+              ),
+              transaction,
+            })
+          : 0;
+      const cronStats =
+        preview.policy.cronStatRetentionDays > 0
+          ? await CrontabStatModel.destroy({
+              where: cronStatWhere(preview.policy.cronStatRetentionDays),
+              transaction,
+            })
+          : 0;
+      return { runningInstances, cronStats };
+    });
+
+    const dependenceCaches = [];
+    for (const cache of preview.dependenceCaches) {
+      await fs.rm(path.join(config.dependenceCachePath, cache.type), {
+        recursive: true,
+        force: true,
+      });
+      dependenceCaches.push(cache);
+    }
+
+    if (
+      request.compactDatabase &&
+      (deleted.runningInstances || deleted.cronStats)
+    ) {
+      await sequelize.query('VACUUM');
+    }
+
+    return {
+      deleted: {
+        ...deleted,
+        dependenceCaches,
+        dependenceCacheBytes: preview.dependenceCacheBytes,
+      },
+      compactedDatabase: Boolean(
+        request.compactDatabase &&
+          (deleted.runningInstances || deleted.cronStats),
+      ),
+    };
+  }
+}

back/services/subscription.ts

@@ -33,6 +33,7 @@ import { CrontabModel } from '../data/cron';
 import CrontabService from './cron';
 import taskLimit from '../shared/pLimit';
 import { logStreamManager } from '../shared/logStreamManager';
+import { LogReadOptions, readLogChunk } from '../shared/logReader';
 
 @Service()
 export default class SubscriptionService {
@@ -366,14 +367,20 @@ export default class SubscriptionService {
     }
   }
 
-  public async log(id: number) {
+  public async log(id: number, options: LogReadOptions = {}) {
     const doc = await this.getDb({ id });
     if (!doc || !doc.log_path) {
-      return '';
+      return {
+        content: '',
+        offset: 0,
+        nextOffset: 0,
+        total: 0,
+        truncated: false,
+      };
     }
 
     const absolutePath = await handleLogPath(doc.log_path as string);
-    return await getFileContentByName(absolutePath);
+    return await readLogChunk(absolutePath, options);
   }
 
   public async logs(id: number) {

back/services/system.ts

@@ -2,7 +2,6 @@ import { spawn } from 'cross-spawn';
 import { Response } from 'express';
 import fs from 'fs';
 import { Agent, request } from 'undici';
-import sum from 'lodash/sum';
 import path from 'path';
 import { Inject, Service } from 'typedi';
 import winston from 'winston';
@@ -467,6 +466,7 @@ export default class SystemService {
     query: {
       startTime?: string;
       endTime?: string;
+      limit?: number;
     },
   ) {
     const startTime = dayjs(query.startTime || undefined)
@@ -481,8 +481,30 @@ export default class SystemService {
       .filter((x) => x.title.endsWith('.log'))
       .filter((x) => x.createTime >= startTime && x.createTime <= endTime);
 
+    const limit = Math.min(
+      Math.max(Number(query.limit) || 1024 * 1024, 1),
+      1024 * 1024,
+    );
+    const total = logs.reduce((size, log) => size + (log.size || 0), 0);
+    let remaining = limit;
+    const selected: Array<
+      (typeof logs)[number] & { start: number; length: number }
+    > = [];
+    for (let index = logs.length - 1; index >= 0 && remaining > 0; index--) {
+      const log = logs[index];
+      const size = log.size || 0;
+      const length = Math.min(size, remaining);
+      if (length > 0) {
+        selected.unshift({ ...log, start: size - length, length });
+        remaining -= length;
+      }
+    }
+
+    const contentLength = selected.reduce((size, log) => size + log.length, 0);
     res.set({
-      'Content-Length': sum(logs.map((x) => x.size)),
+      'Content-Length': contentLength,
+      'X-QL-Log-Total': total,
+      'X-QL-Log-Truncated': total > contentLength ? 'true' : 'false',
     });
     (function sendFiles(res, fileNames) {
       if (fileNames.length === 0) {
@@ -494,13 +516,17 @@ export default class SystemService {
       if (currentLog) {
         const currentFileStream = fs.createReadStream(
           path.join(config.systemLogPath, currentLog.title),
+          {
+            start: currentLog.start,
+            end: currentLog.start + currentLog.length - 1,
+          },
         );
         currentFileStream.on('end', () => {
           sendFiles(res, fileNames);
         });
         currentFileStream.pipe(res, { end: false });
       }
-    })(res, logs);
+    })(res, selected);
   }
 
   public async deleteSystemLog() {

back/shared/logReader.ts

@@ -0,0 +1,109 @@
+import fs from 'fs/promises';
+
+export const DEFAULT_LOG_CHUNK_BYTES = 256 * 1024;
+export const MAX_LOG_CHUNK_BYTES = 1024 * 1024;
+
+export interface LogReadOptions {
+  offset?: number;
+  limit?: number;
+  tail?: boolean;
+}
+
+export interface LogChunk {
+  content: string;
+  offset: number;
+  nextOffset: number;
+  total: number;
+  truncated: boolean;
+}
+
+function normalizeLimit(limit?: number) {
+  if (!Number.isFinite(limit)) return DEFAULT_LOG_CHUNK_BYTES;
+  return Math.min(Math.max(Math.trunc(limit!), 4), MAX_LOG_CHUNK_BYTES);
+}
+
+function isUtf8ContinuationByte(byte: number) {
+  return (byte & 0xc0) === 0x80;
+}
+
+function completeUtf8End(buffer: Buffer, start: number, end: number) {
+  if (end <= start) return start;
+
+  let sequenceStart = end - 1;
+  while (
+    sequenceStart > start &&
+    isUtf8ContinuationByte(buffer[sequenceStart])
+  ) {
+    sequenceStart--;
+  }
+
+  const firstByte = buffer[sequenceStart];
+  const expectedLength =
+    firstByte < 0x80
+      ? 1
+      : firstByte < 0xe0
+        ? 2
+        : firstByte < 0xf0
+          ? 3
+          : 4;
+  return end - sequenceStart < expectedLength ? sequenceStart : end;
+}
+
+export async function readLogChunk(
+  filePath: string,
+  options: LogReadOptions = {},
+): Promise<LogChunk> {
+  let handle: fs.FileHandle | undefined;
+  try {
+    handle = await fs.open(filePath, 'r');
+    const { size: total } = await handle.stat();
+    const limit = normalizeLimit(options.limit);
+    const requestedOffset = Number.isFinite(options.offset)
+      ? Math.trunc(options.offset!)
+      : undefined;
+    const requestedStart =
+      options.tail || requestedOffset === undefined
+        ? Math.max(total - limit, 0)
+        : Math.min(Math.max(requestedOffset, 0), total);
+    const length = Math.min(limit + 3, total - requestedStart);
+    const buffer = Buffer.allocUnsafe(length);
+    const { bytesRead } = await handle.read(
+      buffer,
+      0,
+      length,
+      requestedStart,
+    );
+    let leadingBytes = 0;
+    while (
+      leadingBytes < bytesRead &&
+      isUtf8ContinuationByte(buffer[leadingBytes])
+    ) {
+      leadingBytes++;
+    }
+    const offset = requestedStart + leadingBytes;
+    const candidateEnd = Math.min(leadingBytes + limit, bytesRead);
+    const contentEnd = completeUtf8End(buffer, leadingBytes, candidateEnd);
+    const nextOffset = requestedStart + contentEnd;
+
+    return {
+      content: buffer.subarray(leadingBytes, contentEnd).toString('utf8'),
+      offset,
+      nextOffset,
+      total,
+      truncated: offset > 0 || nextOffset < total,
+    };
+  } catch (error: any) {
+    if (error?.code === 'ENOENT') {
+      return {
+        content: '',
+        offset: 0,
+        nextOffset: 0,
+        total: 0,
+        truncated: false,
+      };
+    }
+    throw error;
+  } finally {
+    await handle?.close();
+  }
+}

back/shared/retention.ts

@@ -0,0 +1,64 @@
+import fs from 'fs/promises';
+import { Dirent } from 'fs';
+import path from 'path';
+
+export const MAX_RETENTION_DAYS = 3650;
+export const DEPENDENCE_CACHE_TYPES = ['node', 'python3'] as const;
+
+export type DependenceCacheType = (typeof DEPENDENCE_CACHE_TYPES)[number];
+
+export interface RetentionPolicy {
+  runningInstanceRetentionDays: number;
+  cronStatRetentionDays: number;
+}
+
+export function normalizeRetentionDays(value: unknown) {
+  const days = Number(value);
+  if (!Number.isFinite(days)) return 0;
+  return Math.min(Math.max(Math.trunc(days), 0), MAX_RETENTION_DAYS);
+}
+
+export function normalizeRetentionPolicy(
+  policy: Partial<RetentionPolicy>,
+): RetentionPolicy {
+  return {
+    runningInstanceRetentionDays: normalizeRetentionDays(
+      policy.runningInstanceRetentionDays,
+    ),
+    cronStatRetentionDays: normalizeRetentionDays(policy.cronStatRetentionDays),
+  };
+}
+
+export function isDependenceCacheType(
+  value: string,
+): value is DependenceCacheType {
+  return DEPENDENCE_CACHE_TYPES.includes(value as DependenceCacheType);
+}
+
+export async function getDirectorySize(rootPath: string): Promise<number> {
+  const pending = [rootPath];
+  let total = 0;
+
+  while (pending.length > 0) {
+    const currentPath = pending.pop()!;
+    let entries: Dirent[];
+    try {
+      entries = await fs.readdir(currentPath, { withFileTypes: true });
+    } catch (error: any) {
+      if (error?.code === 'ENOENT') continue;
+      throw error;
+    }
+
+    for (const entry of entries) {
+      if (entry.isSymbolicLink()) continue;
+      const entryPath = path.join(currentPath, entry.name);
+      if (entry.isDirectory()) {
+        pending.push(entryPath);
+      } else if (entry.isFile()) {
+        total += (await fs.stat(entryPath)).size;
+      }
+    }
+  }
+
+  return total;
+}

docker/Dockerfile.debian

@@ -70,8 +70,10 @@ RUN set -x && \
   git config --global user.name "qinglong" && \
   git config --global http.postBuffer 524288000 && \
   npm install -g pnpm@8.3.1 pm2 ts-node typescript@5 && \
+  npm cache clean --force && \
   rm -rf /root/.cache && \
   rm -rf /root/.npm && \
+  rm -rf /var/lib/apt/lists/* && \
   rm -rf /etc/apt/apt.conf.d/docker-clean && \
   ulimit -c 0
 

docker/Dockerfile.debian310

@@ -69,8 +69,10 @@ RUN set -x && \
   git config --global user.name "qinglong" && \
   git config --global http.postBuffer 524288000 && \
   npm install -g pnpm@8.3.1 pm2 ts-node typescript@5 && \
+  npm cache clean --force && \
   rm -rf /root/.cache && \
   rm -rf /root/.npm && \
+  rm -rf /var/lib/apt/lists/* && \
   rm -rf /etc/apt/apt.conf.d/docker-clean && \
   ulimit -c 0
 

docs/PERFORMANCE_AND_OPEN_ISSUE_TRIAGE_2026-08-16.md

@@ -0,0 +1,200 @@
+# 性能与未关闭 Issue 梳理（2026-08-16）
+
+## 范围与结论
+
+- 基线：`origin/develop` @ `31f78e4d`，运行镜像 `whyour/qinglong:debian`。
+- GitHub 当前有 86 个未关闭 issue；逐项读取了正文，并读取了其中 48 个有评论 issue 的全部评论。
+- 待机内存问题有效且可量化。主要问题不是 Node.js 语言本身，而是三个 Node 隔离进程、PM2，以及没有上限的逐请求指标保留共同叠加。
+- 稳定待机没有持续块设备写入；磁盘问题主要是活跃期的重复日志、每次任务生命周期的多次 SQLite 事务、无界历史表和日志读取/保留策略。
+- 建议先做可回归验证的小步优化，不建议以“改写为 Go”作为当前性能问题的前置条件。
+
+## 实测基线
+
+### 内存
+
+| 项目 | 结果 |
+| --- | ---: |
+| `docker stats` 稳定待机 | 约 193 MiB |
+| cgroup `memory.current` | 约 234–244 MB |
+| HTTP worker PSS / RSS | 约 75 MiB / 107 MiB |
+| gRPC worker PSS / RSS | 约 57 MiB / 89 MiB |
+| cluster 主进程 PSS / RSS | 约 46 MiB / 77 MiB |
+| PM2 PSS / RSS | 约 24 MiB / 52 MiB |
+
+对 `/api/health` 发出 50,000 次成功请求后，等待客户端退出及 5 秒回收：
+
+- cgroup 从 `244,035,584` 增至 `348,925,952` 字节，净增约 105 MB；
+- HTTP worker RSS 从约 107 MB 增至约 208 MB；
+- 块设备写入量没有变化。
+
+这与 `back/middlewares/monitoring.ts` 和 `back/services/metrics.ts` 一致：每个 HTTP 请求同时进入一个 1,000 条队列和一个按一小时保留、没有数量上限的原始指标数组。后者会随每小时请求数线性增长，并不是 GC 可以主动释放的短期对象。
+
+### 磁盘写入与占用
+
+- 稳定后连续 30 秒，cgroup `wbytes` 保持 `1,077,248` 不变；当前没有证据表明待机状态持续刷盘。
+- SQLite 使用 `journal_mode=delete`、`synchronous=2`（FULL）、4 KiB page。一次任务执行至少涉及运行实例创建、任务状态更新、运行实例完成、任务恢复空闲、统计更新等多次独立事务，活跃期会产生额外 journal 与同步写放大。
+- Winston 会写 `/ql/data/log`，同一日志又经 console 被 PM2 写入 `~/.pm2/logs`。主进程、HTTP worker、gRPC worker 都会创建 logger/file transport；PM2 日志没有代码内保留上限，只在 reload 时 `pm2 flush`。
+- 任务 stderr 同时进入任务日志和系统日志；错误输出较多时会再次放大写入。
+- 日志详情接口使用 `readFile(..., 'utf8')` 整文件加载，既造成瞬时内存峰值，也对应 #2256 的超长日志超时。
+
+镜像与主要目录：
+
+| 项目 | 大小 |
+| --- | ---: |
+| 镜像 | 599,130,109 bytes |
+| `/ql` | 215 MiB |
+| `/ql/node_modules` | 174 MiB |
+| `/ql/static` | 32 MiB |
+| `/usr/local/lib/node_modules` | 85 MiB |
+| `/usr/local/lib/python3.11` | 40 MiB |
+| `/var/lib/apt/lists` | 19 MiB |
+| 全新 `/ql/data` | 5.9 MiB |
+
+## 建议修复批次
+
+### P0：低风险、可量化收益
+
+1. 将原始逐请求指标改为固定容量 ring buffer，或直接改成计数器、直方图和最近慢请求；给所有维度设置基数上限。增加 10 万请求内存回归测试，目标是回落后增量小于 10 MiB。
+2. 日志只保留一个持久化落点：生产环境选择“Winston 文件 + PM2 stdout 丢弃”或“console + 受控 PM2 rotation”，避免双写。统一由单进程持有系统日志文件 transport。
+3. 日志 API 增加 `tail/offset/limit` 与最大响应字节数，前端使用增量加载，不再整文件读入内存。
+4. 给 PM2 日志、`RunningInstances`、任务统计和依赖下载缓存增加明确的容量/时间保留策略；清理任务输出被删除的字节数和剩余大小。
+5. Dockerfile 删除 apt lists，审计全局 Node 模块与应用依赖的重复内容；构建产物中不要保留仅构建期需要的包。
+
+当前修复分支落实情况：
+
+- P0-1 已完成：逐请求原始指标改为 1,000 条固定容量环形缓冲区，并补充 50,000 次写入回归测试。
+- P0-2 已完成：容器内 PM2 输出直接接入 PID 1 标准输出/错误，停止写入 `~/.pm2/logs` 的重复持久化副本。
+- P0-3 已完成：日志接口默认 256 KiB、最大 1 MiB，支持 tail/offset 增量读取，并保证 UTF-8 字符不被分块破坏；浏览器视图最大保留 1 MiB。
+- P0-4 已完成：系统设置提供运行实例/任务统计保留天数，默认 0（禁用）；清理前展示记录数与依赖缓存字节数，清理接口要求明确确认。依赖缓存和 SQLite `VACUUM` 均为手动勾选，不在启动或升级时自动执行。
+- P0-5 已完成：Debian 镜像构建后清理 npm cache 和 apt lists。
+
+### P1：轻量架构调整
+
+1. 主进程只加载 cluster/bootstrap 必需模块，把 Express、监控、HTTP 中间件和业务容器全部延迟到 HTTP worker。目标是先回收主进程约 30–40 MiB private memory。
+2. Metrics 定时器只在实际使用指标的 worker 中启动，并对 path/tag 做归一化，防止高基数值撑大内存。
+3. 将任务“开始”和“结束”各自需要的状态、实例、统计写入合并到事务；避免一个生命周期五个以上独立提交。
+4. 在本地文件系统上基准比较 SQLite `WAL + synchronous=NORMAL + busy_timeout` 与现状；确认断电语义和网络文件系统兼容后再作为可选配置发布。
+5. 任务日志保留从全局天数扩展为“全局默认 + 单任务覆盖 + 总容量上限”，而不是只增加更多清理 cron。
+
+### P2：需要设计评审
+
+- 提供 Lite 模式：调度器与 HTTP 同进程运行，省去 gRPC worker；标准模式继续保持隔离。需要故障域、重启和任务注册一致性测试。
+- 若继续推进集群/多节点，先定义控制面、执行器、任务租约、心跳、幂等和日志归属；不要以“共享 SQLite/MySQL”替代分布式调度设计。
+
+## 86 个未关闭 Issue 的逐项建议
+
+状态含义：`立即处理` 是确认有效且影响当前主线；`轻量处理` 是小型代码/文档/回归测试；`架构议题` 需要 RFC 或独立 epic；`关闭/合并` 表示已修复、重复、支持问题或需要提交者用最新版本重现。
+
+### 立即处理（10）
+
+| Issue | 结论 | 建议 |
+| --- | --- | --- |
+| #2256 | 有效，整文件读取日志造成超时和内存峰值 | 日志分片、tail、最大响应大小 |
+| #2742 | 有效，与 #3057 合并 | 用内存基线和压力回归替代“运行几天观察” |
+| #2743 | 有效，与 #2742/#3057 合并 | 优先修复指标保留和进程基线，再复测受限容器 |
+| #2871 | 有效，2.21.0 仍有反馈 | 与 #2902 合并，修复源地址生成并加发行版矩阵测试 |
+| #2902 | 有效，重复 #2871 | 合并后关闭 |
+| #3017 | 有效安全 epic | 拆成凭据迁移、scope、SSRF、错误脱敏、更新完整性等独立任务 |
+| #3051 | 当前回归仍需验证 | 对 shell status payload 建端到端测试，重建最新镜像复测 |
+| #3055 | 用户报告有效，但依赖库单测 5/6 字段都可注册 | 增加真实 gRPC 注册与触发集成测试，排查发行镜像/存量任务迁移 |
+| #3057 | 有效，实测已复现 | 先修无界指标和多进程基线，不需要先改写语言 |
+| #3060 | 有效的容器权限/依赖安装回归 | 覆盖匿名卷、bind mount、rootless 三类安装 smoke test |
+
+### 轻量处理或局部设计（36）
+
+| Issue | 结论与最小处理 |
+| --- | --- |
+| #2015 | 当前仅支持整组 AND/OR；若要嵌套条件，定义轻量 filter AST，避免继续堆 UI 特判 |
+| #2236 | 与 #2421/#3014 合并为一个 cron 可视化 issue |
+| #2254 | 增加有限重试策略；必须有退避、最大次数和不可重试退出码 |
+| #2269 | 增加时区、5/6 字段、DST 的调度集成测试；最新版本无法复现则关闭 |
+| #2341 | 与 #2688 合并，把 boolean 实例模式升级为 `replace/skip/parallel/queue` 枚举 |
+| #2360 | 标签在列表中恢复可见与筛选入口，纯前端小改 |
+| #2418 | 配置方法已有评论解法；仅保留消息超长时的截断/分片问题 |
+| #2421 | 合并到 #3014 |
+| #2635 | 备注字段是小型 schema/UI 变更，可与名称/标签排序统一设计 |
+| #2645 | 增加依赖唯一键、恢复数据去重迁移和幂等安装测试 |
+| #2687 | 用最新版本和代理 URL 重现；补代理/超时错误提示 |
+| #2688 | 合并到 #2341；当前“单实例”会杀旧任务，不等同于“跳过新任务” |
+| #2701 | 需要最小仓库配置样本；JSON 控制字符应在输入边界给出字段级错误 |
+| #2715 | Server 酱 tags 是局部通知配置扩展 |
+| #2756 | 重测 Chrome、base path、缓存与 health 401；补登录跳转循环保护 |
+| #2757 | 复用订阅现有 `autoAddCron` 配置，明确新增任务默认启停状态 |
+| #2780 | `ql check` 应在系统包失败时停止并回滚/给出恢复指令，不能继续破坏可运行环境 |
+| #2782 | 通知免打扰时段是局部配置；明确延迟发送还是静默丢弃 |
+| #2797 | 保留；展开 AggregateError 内部原因并为每种通知后端返回可操作错误 |
+| #2801 | 私有仓库凭据与 SSH/HTTPS 模式做连通性诊断，信息不足时转支持问题 |
+| #2859 | 最新版请求重现；增加 `env.js` 镜像启动 smoke test和缓存头校验 |
+| #2863 | 区分官方 Docker 与 npm/Linux 安装支持等级；若继续支持后者则加 CI |
+| #2883 | 最新版复测临时编辑保存；失败时保留请求与后端路径错误 |
+| #2886 | 支持 requirements 文件需要锁定来源、工作目录和隔离策略，可先支持显式路径 |
+| #2896 | Synology 内核缺少随机字节能力，先做环境诊断与明确错误，不宜静默降级安全随机数 |
+| #2901 | 当前默认监听 `::` 并回退 `0.0.0.0`；增加 IPv6 Docker/host 网络 smoke test |
+| #2925 | 信息不足；收集完整日志与重启方式，最新镜像不可复现则关闭 |
+| #2927 | 使用成熟 ANSI 清理实现，覆盖复合 SGR，而不是只匹配 `\x1b[数字m` |
+| #2984 | 通知审计有效，但需先确定脱敏、保留期和容量上限，避免制造新磁盘问题 |
+| #3013 | 与性能修复合并：全局默认、单任务覆盖、总容量上限 |
+| #3014 | 合并 #2236/#2421，前端生成器必须始终展示最终 cron 文本 |
+| #3016 | 有效文档任务；从路由/validation 自动生成 OpenAPI，避免手工文档漂移 |
+| #3020 | 有效小改；配置编辑器启用 `wordWrap: 'on'` 并保留开关 |
+| #3027 | 需要明确旧值 `1` 的历史语义；用显式 `@once/@boot/disabled` 替代魔法数字 |
+| #3054 | 有效纯 UI 改进；整行点击打开历史日志，操作按钮阻止冒泡 |
+| #3058 | 有效安全需求；应用增加环境变量名称 allowlist，默认最小权限并审计拒绝事件 |
+
+### 架构议题，单独立项（12）
+
+| Issue | 处理建议 |
+| --- | --- |
+| #1656 | 捕获任意脚本网络请求需要代理或运行时注入，成本和隐私风险高；不作为普通日志小改 |
+| #1695 | 与 #2596 合并为“集群执行器”RFC |
+| #1821 | 多数据库不等于多节点；只有明确 HA/共享控制面设计后再评估 MySQL/PostgreSQL |
+| #2434 | PowerShell 运行时会增加镜像体积和维护矩阵，作为可选外部执行器评估 |
+| #2464 | Bun 属于额外运行时矩阵，不应为单一性能假设直接加入基础镜像 |
+| #2481 | 每任务 Python 虚拟环境需要依赖缓存、生命周期和磁盘配额设计 |
+| #2596 | 合并到 #1695 |
+| #2642 | 与 #2769/#2905 合并为多租户权限模型 RFC |
+| #2656 | .NET 8 作为可选运行时/派生镜像评估，不进入默认镜像 |
+| #2736 | 条件触发工作流已接近 n8n 类产品，应独立产品/RFC，不混入 cron 小功能 |
+| #2769 | 合并到多租户 RFC |
+| #2905 | 合并到多租户 RFC；PR #2818 作为设计输入 |
+
+### 建议关闭、合并或转支持（28）
+
+| Issue | 理由 |
+| --- | --- |
+| #795 | HTTP 明文本质上应通过 HTTPS 解决；前端自定义“加盐”不能阻止重放。补反代 HTTPS 文档后关闭 |
+| #796 | 可由数据库/API 导出；需求长期无上下文，关闭或改为 CSV 导出新需求 |
+| #854 | 客户端 IP 已由 #3059 对应功能覆盖；应用调用审计并入 #2984 |
+| #2042 | 当前 `NotReg` 已显式处理 NULL 与 notLike；最新版回归测试通过后关闭 |
+| #2217 | Home Assistant 插件仓库属于独立打包生态，转社区集成 |
+| #2219 | 当前已有运行实例历史与逐实例停止按钮，关闭为已实现 |
+| #2329 | 评论已给出 Open API/文件方案，转文档 FAQ 后关闭 |
+| #2340 | 老旧 IPQ/固件上的 V8 Bus error 属运行时/硬件兼容；用当前多架构镜像复测，否则关闭 |
+| #2350 | 环境变量值如何分隔主要由脚本协议定义；补文档后关闭 |
+| #2398 | 单一外站 DNS 解析问题且无环境信息，转网络支持 |
+| #2404 | 与 #3057 合并，保留一个性能 epic |
+| #2525 | 非官方旧容器/npm 安装路径；最新版本不可复现则关闭 |
+| #2567 | 2024 年无任何环境信息的容器 entrypoint 错误，要求当前镜像重现，否则关闭 |
+| #2705 | 当前已有全局 cron concurrency 与最多五个重复实例限制，补 UI/文档后关闭 |
+| #2772 | 与 #2340 合并为特定软路由 CPU/固件兼容问题 |
+| #2788 | maintainer 已说明黑名单语义；补订阅文档后关闭 |
+| #2792 | PM2 版本提示是镜像更新/支持问题，当前镜像复测后关闭 |
+| #2793 | 自动重启会掩盖 OOM；合并到性能问题，不单独实现定时自杀 |
+| #2798 | Node 在特定硬件启动即 fatal，合并硬件兼容问题 |
+| #2895 | Debian 中加载 musl 二进制通常是第三方依赖选错平台，转依赖支持 |
+| #2903 | 当前运行时代码未使用 chokidar 监听 node_modules；最新镜像复测后关闭 |
+| #2914 | maintainer 指定 2.20.1 已修复，当前版本复测后关闭 |
+| #2948 | 评论已有正确升级链和 `/ql/data` 挂载方案；补升级文档后关闭 |
+| #2964 | 本文已提供资源基线，并入 #3057 后关闭 |
+| #2967 | maintainer 已解释黑白名单语义；补文档后关闭 |
+| #3052 | `2ac4f07f` 已修复 dashboard open scope，关闭为已修复 |
+| #3053 | `2ac4f07f` 已加入 INI 预览/强制打开处理，关闭为已修复 |
+| #3059 | `31f78e4d` 已加入 IP 黑名单、客户端 IP 诊断和 trust proxy 配置，关闭为已修复 |
+
+## 建议的 Issue 管理动作
+
+1. 先关闭明确已实现的 #2219、#3052、#3053、#3059。
+2. 将 #2404、#2742、#2743 合并到 #3057，并把本文内存实测作为验收基线。
+3. 将 #2236、#2421 合并到 #3014；#2871 合并 #2902；#1695 合并 #2596；#2642/#2769/#2905 合并。
+4. 对“需要最新版复现”的旧 bug 统一贴模板，给出 14 天补充窗口；没有版本、架构、日志、最小复现则关闭为 stale/support。
+5. 轻量修复和架构 epic 使用不同 label/milestone，避免大设计长期占据回归队列。

ecosystem.config.js

@@ -1,3 +1,5 @@
+const isContainer = process.env.QL_CONTAINER === 'true';
+
 module.exports = {
   apps: [
     {
@@ -7,7 +9,9 @@ module.exports = {
       wait_ready: true,
       listen_timeout: 5000,
       source_map_support: true,
-      time: true,
+      time: !isContainer,
+      out_file: isContainer ? '/proc/1/fd/1' : undefined,
+      error_file: isContainer ? '/proc/1/fd/2' : undefined,
       script: 'static/build/app.js',
       env: {
         http_proxy: '',

shell/share.sh

@@ -305,8 +305,13 @@ reload_pm2() {
     # Kill any existing node processes for qinglong
     pkill -f "node.*static/build/app.js" 2>/dev/null || true
 
-    # Start node directly in the background
+    # Start node directly in the background. In containers, keep application
-    nohup node static/build/app.js > $dir_log/qinglong.log 2>&1 &
+    # output on the container streams instead of duplicating Winston logs on disk.
+    if [[ "$QL_CONTAINER" == "true" ]]; then
+      nohup node static/build/app.js >/proc/1/fd/1 2>/proc/1/fd/2 &
+    else
+      nohup node static/build/app.js >$dir_log/qinglong.log 2>&1 &
+    fi
     local node_pid=$!
 
     t '已使用 Node.js 直接启动服务 (PID: %s)' "$node_pid"

src/locales/en-US.json

@@ -366,6 +366,23 @@
   "添加标签成功": "Tags added successfully",
   "清空日志": "Clear Logs",
   "清除依赖缓存": "Clean dependency cache",
+  "历史数据保留与手动清理": "History retention and manual cleanup",
+  "保留天数为0时禁用对应清理，预览不会删除数据": "Set retention to 0 to disable that cleanup. Preview never deletes data.",
+  "任务统计": "Task statistics",
+  "清除 Node 依赖缓存": "Clean Node dependency cache",
+  "清除 Python 依赖缓存": "Clean Python dependency cache",
+  "清理后压缩数据库": "Compact database after cleanup",
+  "保存设置": "Save settings",
+  "预览清理": "Preview cleanup",
+  "确认清理存储数据": "Confirm storage cleanup",
+  "确认清理": "Clean now",
+  "将删除历史运行实例": "Historical running instances to delete",
+  "将删除任务统计": "Task statistics to delete",
+  "将清除依赖缓存": "Dependency caches to clear",
+  "清理依赖缓存后相关依赖需要重新安装": "Dependencies must be reinstalled after their cache is cleared.",
+  "数据库压缩期间可能暂时阻塞请求": "Database compaction may briefly block requests.",
+  "此操作不可恢复，请确认已完成必要备份": "This action cannot be undone. Confirm that required backups are complete.",
+  "清理完成": "Cleanup complete",
   "清除成功": "Clean successful",
   "源文件": "Source File",
   "激活成功": "Activation successful",

src/locales/zh-CN.json

@@ -366,6 +366,23 @@
   "添加标签成功": "添加标签成功",
   "清空日志": "清空日志",
   "清除依赖缓存": "清除依赖缓存",
+  "历史数据保留与手动清理": "历史数据保留与手动清理",
+  "保留天数为0时禁用对应清理，预览不会删除数据": "保留天数为0时禁用对应清理，预览不会删除数据",
+  "任务统计": "任务统计",
+  "清除 Node 依赖缓存": "清除 Node 依赖缓存",
+  "清除 Python 依赖缓存": "清除 Python 依赖缓存",
+  "清理后压缩数据库": "清理后压缩数据库",
+  "保存设置": "保存设置",
+  "预览清理": "预览清理",
+  "确认清理存储数据": "确认清理存储数据",
+  "确认清理": "确认清理",
+  "将删除历史运行实例": "将删除历史运行实例",
+  "将删除任务统计": "将删除任务统计",
+  "将清除依赖缓存": "将清除依赖缓存",
+  "清理依赖缓存后相关依赖需要重新安装": "清理依赖缓存后相关依赖需要重新安装",
+  "数据库压缩期间可能暂时阻塞请求": "数据库压缩期间可能暂时阻塞请求",
+  "此操作不可恢复，请确认已完成必要备份": "此操作不可恢复，请确认已完成必要备份",
+  "清理完成": "清理完成",
   "清除成功": "清除成功",
   "源文件": "源文件",
   "激活成功": "激活成功",

src/pages/crontab/logModal.tsx

@@ -16,11 +16,12 @@ import {
   CheckCircleOutlined,
 } from "@ant-design/icons";
 import { PageLoading } from "@ant-design/pro-layout";
-import { logEnded } from "@/utils";
 import { CrontabStatus } from "./type";
 import Ansi from "ansi-to-react";
 
 const { Countdown } = Statistic;
+const LOG_CHUNK_BYTES = 256 * 1024;
+const MAX_LOG_VIEW_CHARS = 1024 * 1024;
 
 const CronLogModal = ({
   cron,
@@ -38,35 +39,49 @@ const CronLogModal = ({
   const [executing, setExecuting] = useState<any>(true);
   const [isPhone, setIsPhone] = useState(false);
   const scrollInfoRef = useRef({ value: 0, down: true });
+  const logOffsetRef = useRef<number>();
+  const valueRef = useRef(value);
+  const pollTimerRef = useRef<ReturnType<typeof setTimeout>>();
   const uniqPath = logUrl ? logUrl : String(cron?.id);
 
   const getCronLog = (isFirst?: boolean) => {
     if (isFirst) {
       setLoading(true);
     }
+    const baseUrl = logUrl ? logUrl : `${config.apiPrefix}crons/${cron.id}/log`;
+    const separator = baseUrl.includes("?") ? "&" : "?";
+    const offset = isFirst ? undefined : logOffsetRef.current;
+    const pagination = `${separator}limit=${LOG_CHUNK_BYTES}${
+      isFirst ? "&tail=true" : offset !== undefined ? `&offset=${offset}` : ""
+    }`;
     request
-      .get(logUrl ? logUrl : `${config.apiPrefix}crons/${cron.id}/log`)
+      .get(`${baseUrl}${pagination}`)
-      .then(({ code, data, logStatus }) => {
+      .then(({ code, data, logStatus, nextOffset }) => {
-        if (
+        if (code !== 200 || localStorage.getItem("logCron") !== uniqPath) {
-          code === 200 &&
+          return;
-          localStorage.getItem("logCron") === uniqPath &&
+        }
-          data !== value
+
-        ) {
+        const hasNext = logStatus === "running";
-          const log = (data as string) || intl.get("暂无日志");
+        const chunk = (data as string) || "";
-          setValue(log);
+        let log = isFirst ? chunk : `${valueRef.current}${chunk}`;
-          const hasNext = logStatus === 'running';
+        if (!log && !hasNext) {
-          if (!hasNext && !logEnded(value) && value !== intl.get("启动中...")) {
+          log = intl.get("暂无日志");
-            setTimeout(() => {
+        }
-              autoScroll();
+        if (log.length > MAX_LOG_VIEW_CHARS) {
-            });
+          log = log.slice(-MAX_LOG_VIEW_CHARS);
-          }
+        }
-          setExecuting(hasNext);
+        valueRef.current = log;
-          if (hasNext) {
+        setValue(log);
-            setTimeout(() => {
+        if (typeof nextOffset === "number") {
-              autoScroll();
+          logOffsetRef.current = nextOffset;
-              getCronLog();
+        }
-            }, 2000);
+        setExecuting(hasNext);
-          }
+
+        if (chunk || !hasNext) {
+          autoScroll();
+        }
+        if (hasNext) {
+          pollTimerRef.current = setTimeout(() => getCronLog(), 2000);
         }
       })
       .finally(() => {
@@ -89,6 +104,9 @@ const CronLogModal = ({
   };
 
   const cancel = () => {
+    if (pollTimerRef.current) {
+      clearTimeout(pollTimerRef.current);
+    }
     localStorage.removeItem("logCron");
     handleCancel();
   };
@@ -117,12 +135,19 @@ const CronLogModal = ({
 
   useEffect(() => {
     if (cron && cron.id) {
+      logOffsetRef.current = undefined;
       getCronLog(true);
     }
+    return () => {
+      if (pollTimerRef.current) {
+        clearTimeout(pollTimerRef.current);
+      }
+    };
   }, [cron]);
 
   useEffect(() => {
     if (data) {
+      valueRef.current = data;
       setValue(data);
     }
   }, [data]);

src/pages/log/index.tsx

@@ -59,7 +59,7 @@ const Log = () => {
       .get(
         `${config.apiPrefix}logs/detail?file=${node.title}&path=${
           node.parent || ''
-        }`,
+        }&tail=true&limit=${1024 * 1024}`,
       )
       .then(({ code, data }) => {
         if (code === 200) {

src/pages/setting/index.tsx

@@ -51,7 +51,6 @@ const Setting = () => {
     reloadTheme,
     systemInfo,
   } = useOutletContext<SharedContext>();
-  console.log('user', user);
   const columns = [
     {
       title: intl.get('名称'),

src/pages/setting/other.tsx

@@ -62,6 +62,8 @@ const Other = ({
     cronConcurrency?: number | null;
     timezone?: string | null;
     globalSshKey?: string | null;
+    runningInstanceRetentionDays?: number | null;
+    cronStatRetentionDays?: number | null;
   }>();
   const [form] = Form.useForm();
   const [exportLoading, setExportLoading] = useState(false);
@@ -69,6 +71,11 @@ const Other = ({
   const showDownloadProgress = useProgress(intl.get('下载'));
   const [visible, setVisible] = useState(false);
   const [selectedModules, setSelectedModules] = useState<string[]>(['base']);
+  const [cleanupLoading, setCleanupLoading] = useState(false);
+  const [dependenceCacheTypes, setDependenceCacheTypes] = useState<string[]>(
+    [],
+  );
+  const [compactDatabase, setCompactDatabase] = useState(false);
 
   const {
     enable: enableDarkMode,
@@ -136,6 +143,93 @@ const Other = ({
       });
   };
 
+  const retentionPayload = () => ({
+    runningInstanceRetentionDays:
+      systemConfig?.runningInstanceRetentionDays || 0,
+    cronStatRetentionDays: systemConfig?.cronStatRetentionDays || 0,
+  });
+
+  const saveRetentionPolicy = () => {
+    request
+      .put(
+        `${config.apiPrefix}system/storage-retention/config`,
+        retentionPayload(),
+      )
+      .then(({ code }) => {
+        if (code === 200) {
+          message.success(intl.get('更新成功'));
+        }
+      });
+  };
+
+  const formatBytes = (bytes: number) => {
+    if (!bytes) return '0 B';
+    const units = ['B', 'KiB', 'MiB', 'GiB'];
+    const unit = Math.min(
+      Math.floor(Math.log(bytes) / Math.log(1024)),
+      units.length - 1,
+    );
+    return `${(bytes / 1024 ** unit).toFixed(unit ? 1 : 0)} ${units[unit]}`;
+  };
+
+  const previewStorageCleanup = () => {
+    setCleanupLoading(true);
+    const payload = {
+      ...retentionPayload(),
+      dependenceCacheTypes,
+      compactDatabase,
+    };
+    request
+      .post(`${config.apiPrefix}system/storage-retention/preview`, payload)
+      .then(({ code, data }) => {
+        if (code !== 200) return;
+        Modal.confirm({
+          width: 560,
+          centered: true,
+          title: intl.get('确认清理存储数据'),
+          okText: intl.get('确认清理'),
+          cancelText: intl.get('取消'),
+          okButtonProps: { danger: true },
+          content: (
+            <div>
+              <p>
+                {intl.get('将删除历史运行实例')}: {data.runningInstances}
+              </p>
+              <p>
+                {intl.get('将删除任务统计')}: {data.cronStats}
+              </p>
+              <p>
+                {intl.get('将清除依赖缓存')}: {data.dependenceCaches.length} (
+                {formatBytes(data.dependenceCacheBytes)})
+              </p>
+              {data.dependenceCaches.length > 0 && (
+                <p>{intl.get('清理依赖缓存后相关依赖需要重新安装')}</p>
+              )}
+              {compactDatabase && (
+                <p>{intl.get('数据库压缩期间可能暂时阻塞请求')}</p>
+              )}
+              <p>{intl.get('此操作不可恢复，请确认已完成必要备份')}</p>
+            </div>
+          ),
+          onOk: () => {
+            setCleanupLoading(true);
+            return request
+              .post(`${config.apiPrefix}system/storage-retention/cleanup`, {
+                ...payload,
+                confirmation: 'CLEAN',
+              })
+              .then(({ code }) => {
+                if (code === 200) {
+                  message.success(intl.get('清理完成'));
+                }
+              })
+              .finally(() => setCleanupLoading(false));
+          },
+        });
+      })
+      .finally(() => setCleanupLoading(false));
+  };
+
   const exportData = () => {
     setExportLoading(true);
     request
@@ -302,6 +396,69 @@ const Other = ({
             </Button>
           </Input.Group>
         </Form.Item>
+        <Form.Item
+          label={intl.get('历史数据保留与手动清理')}
+          tooltip={intl.get('保留天数为0时禁用对应清理，预览不会删除数据')}
+        >
+          <div style={{ marginBottom: 8 }}>
+            <InputNumber
+              addonBefore={intl.get('运行实例')}
+              addonAfter={intl.get('天')}
+              min={0}
+              max={3650}
+              value={systemConfig?.runningInstanceRetentionDays || 0}
+              onChange={(value) => {
+                setSystemConfig({
+                  ...systemConfig,
+                  runningInstanceRetentionDays: value,
+                });
+              }}
+            />
+          </div>
+          <div style={{ marginBottom: 8 }}>
+            <InputNumber
+              addonBefore={intl.get('任务统计')}
+              addonAfter={intl.get('天')}
+              min={0}
+              max={3650}
+              value={systemConfig?.cronStatRetentionDays || 0}
+              onChange={(value) => {
+                setSystemConfig({
+                  ...systemConfig,
+                  cronStatRetentionDays: value,
+                });
+              }}
+            />
+          </div>
+          <div style={{ marginBottom: 8 }}>
+            <Checkbox.Group
+              value={dependenceCacheTypes}
+              options={[
+                { label: intl.get('清除 Node 依赖缓存'), value: 'node' },
+                { label: intl.get('清除 Python 依赖缓存'), value: 'python3' },
+              ]}
+              onChange={(value) => setDependenceCacheTypes(value as string[])}
+            />
+          </div>
+          <div style={{ marginBottom: 8 }}>
+            <Checkbox
+              checked={compactDatabase}
+              onChange={(event) => setCompactDatabase(event.target.checked)}
+            >
+              {intl.get('清理后压缩数据库')}
+            </Checkbox>
+          </div>
+          <Button onClick={saveRetentionPolicy} style={{ marginRight: 8 }}>
+            {intl.get('保存设置')}
+          </Button>
+          <Button
+            danger
+            loading={cleanupLoading}
+            onClick={previewStorageCleanup}
+          >
+            {intl.get('预览清理')}
+          </Button>
+        </Form.Item>
         <Form.Item label={intl.get('定时任务并发数')} name="frequency">
           <Input.Group compact>
             <InputNumber

src/pages/setting/systemLog.tsx

@@ -29,7 +29,9 @@ const SystemLog = ({ height, theme }: any) => {
   const { loading, refresh } = useRequest(
     () => {
       return request.get<Blob>(
-        `${config.apiPrefix}system/log?startTime=${range[0]}&endTime=${range[1]}`,
+        `${config.apiPrefix}system/log?startTime=${range[0]}&endTime=${
+          range[1]
+        }&limit=${1024 * 1024}`,
         {
           responseType: 'blob',
         },

test/back/ecosystem.test.cjs

@@ -0,0 +1,28 @@
+const assert = require('node:assert/strict');
+const test = require('node:test');
+
+const configPath = require.resolve('../../ecosystem.config');
+
+function loadConfig(containerValue) {
+  if (containerValue === undefined) {
+    delete process.env.QL_CONTAINER;
+  } else {
+    process.env.QL_CONTAINER = containerValue;
+  }
+  delete require.cache[configPath];
+  return require(configPath).apps[0];
+}
+
+test('container logging goes to the container standard streams', () => {
+  const app = loadConfig('true');
+  assert.equal(app.out_file, '/proc/1/fd/1');
+  assert.equal(app.error_file, '/proc/1/fd/2');
+  assert.equal(app.time, false);
+});
+
+test('non-container installs keep the PM2 logging defaults', () => {
+  const app = loadConfig(undefined);
+  assert.equal(app.out_file, undefined);
+  assert.equal(app.error_file, undefined);
+  assert.equal(app.time, true);
+});

test/back/logReader.test.cjs

@@ -0,0 +1,72 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const test = require('node:test');
+
+const {
+  DEFAULT_LOG_CHUNK_BYTES,
+  MAX_LOG_CHUNK_BYTES,
+  readLogChunk,
+} = require('../../back/shared/logReader');
+
+test('log reader defaults to a bounded tail and supports incremental reads', async (t) => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-reader-'));
+  const file = path.join(directory, 'task.log');
+  t.after(() => fs.rm(directory, { recursive: true, force: true }));
+
+  const initial = 'a'.repeat(DEFAULT_LOG_CHUNK_BYTES * 2);
+  await fs.writeFile(file, initial);
+
+  const tail = await readLogChunk(file);
+  assert.equal(tail.content.length, DEFAULT_LOG_CHUNK_BYTES);
+  assert.equal(tail.offset, DEFAULT_LOG_CHUNK_BYTES);
+  assert.equal(tail.nextOffset, initial.length);
+  assert.equal(tail.truncated, true);
+
+  await fs.appendFile(file, 'next');
+  const incremental = await readLogChunk(file, { offset: tail.nextOffset });
+  assert.equal(incremental.content, 'next');
+  assert.equal(incremental.nextOffset, initial.length + 4);
+});
+
+test('log reader enforces the maximum chunk size', async (t) => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-reader-'));
+  const file = path.join(directory, 'large.log');
+  t.after(() => fs.rm(directory, { recursive: true, force: true }));
+  await fs.writeFile(file, 'x'.repeat(MAX_LOG_CHUNK_BYTES + 1024));
+
+  const chunk = await readLogChunk(file, { limit: Number.MAX_SAFE_INTEGER });
+  assert.equal(Buffer.byteLength(chunk.content), MAX_LOG_CHUNK_BYTES);
+});
+
+test('log reader preserves UTF-8 characters across byte boundaries', async (t) => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-reader-'));
+  const file = path.join(directory, 'utf8.log');
+  t.after(() => fs.rm(directory, { recursive: true, force: true }));
+  await fs.writeFile(file, 'a'.repeat(5) + '中文日志');
+
+  const first = await readLogChunk(file, { offset: 0, limit: 7 });
+  const second = await readLogChunk(file, {
+    offset: first.nextOffset,
+    limit: 7,
+  });
+  const third = await readLogChunk(file, {
+    offset: second.nextOffset,
+    limit: 7,
+  });
+
+  assert.equal(first.content + second.content + third.content, 'aaaaa中文日志');
+  assert.equal((first.content + second.content + third.content).includes('�'), false);
+});
+
+test('missing logs return an empty chunk', async () => {
+  const chunk = await readLogChunk('/path/that/does/not/exist.log');
+  assert.deepEqual(chunk, {
+    content: '',
+    offset: 0,
+    nextOffset: 0,
+    total: 0,
+    truncated: false,
+  });
+});

test/back/metrics.test.cjs

@@ -0,0 +1,34 @@
+const assert = require('node:assert/strict');
+const test = require('node:test');
+
+const {
+  MAX_METRIC_SAMPLES,
+  metricsService,
+} = require('../../back/services/metrics');
+
+test('metrics keep only the newest bounded samples', () => {
+  const sampleCount = MAX_METRIC_SAMPLES * 50;
+  for (let index = 0; index < sampleCount; index++) {
+    metricsService.record('http_request', index, {
+      path: `/health/${index}`,
+    });
+  }
+
+  const result = metricsService.getMetrics('http_request');
+  assert.equal(result.count, MAX_METRIC_SAMPLES);
+  assert.equal(result.metrics[0].value, sampleCount - MAX_METRIC_SAMPLES);
+  assert.equal(result.metrics.at(-1).value, sampleCount - 1);
+});
+
+test('empty metric queries return finite aggregates', () => {
+  const result = metricsService.getMetrics('missing_metric');
+  assert.deepEqual(
+    {
+      count: result.count,
+      average: result.average,
+      min: result.min,
+      max: result.max,
+    },
+    { count: 0, average: 0, min: 0, max: 0 },
+  );
+});

test/back/retention.test.cjs

@@ -0,0 +1,131 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs/promises');
+const os = require('node:os');
+const path = require('node:path');
+const test = require('node:test');
+const { Op } = require('sequelize');
+
+const {
+  getDirectorySize,
+  normalizeRetentionPolicy,
+} = require('../../back/shared/retention');
+
+test('retention policy defaults to disabled and clamps invalid values', () => {
+  assert.deepEqual(normalizeRetentionPolicy({}), {
+    runningInstanceRetentionDays: 0,
+    cronStatRetentionDays: 0,
+  });
+  assert.deepEqual(
+    normalizeRetentionPolicy({
+      runningInstanceRetentionDays: -10,
+      cronStatRetentionDays: 99999,
+    }),
+    {
+      runningInstanceRetentionDays: 0,
+      cronStatRetentionDays: 3650,
+    },
+  );
+});
+
+test('directory preview counts files recursively and ignores symlinks', async (t) => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-retention-'));
+  const nested = path.join(directory, 'nested');
+  t.after(() => fs.rm(directory, { recursive: true, force: true }));
+  await fs.mkdir(nested);
+  await fs.writeFile(path.join(directory, 'one'), Buffer.alloc(7));
+  await fs.writeFile(path.join(nested, 'two'), Buffer.alloc(11));
+  await fs.symlink(path.join(nested, 'two'), path.join(directory, 'link'));
+
+  assert.equal(await getDirectorySize(directory), 18);
+  assert.equal(await getDirectorySize(path.join(directory, 'missing')), 0);
+});
+
+test('cleanup previews first, protects running instances, and uses explicit options', async (t) => {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-retention-'));
+  const nodeCache = path.join(directory, 'node');
+  await fs.mkdir(nodeCache);
+  await fs.writeFile(path.join(nodeCache, 'cache'), Buffer.alloc(13));
+
+  const config = require('../../back/config').default;
+  const originalCachePath = config.dependenceCachePath;
+  const RunningInstanceModel = {};
+  const CrontabStatModel = {};
+  const InstanceStatus = { running: 0 };
+  const sequelize = {};
+  const stubModule = (modulePath, exports) => {
+    require.cache[require.resolve(modulePath)] = {
+      id: require.resolve(modulePath),
+      filename: require.resolve(modulePath),
+      loaded: true,
+      exports,
+      children: [],
+      paths: [],
+    };
+  };
+  stubModule('../../back/data', { sequelize });
+  stubModule('../../back/data/cronStats', { CrontabStatModel });
+  stubModule('../../back/data/runningInstance', {
+    InstanceStatus,
+    RunningInstanceModel,
+  });
+  stubModule('../../back/data/system', {
+    AuthDataType: { systemConfig: 'systemConfig' },
+    SystemModel: {},
+  });
+  delete require.cache[require.resolve('../../back/services/retention')];
+  const RetentionService = require('../../back/services/retention').default;
+
+  t.after(async () => {
+    config.dependenceCachePath = originalCachePath;
+    await fs.rm(directory, { recursive: true, force: true });
+  });
+
+  let instanceWhere;
+  let vacuumed = false;
+  let instanceCountCalls = 0;
+  let statCountCalls = 0;
+  config.dependenceCachePath = directory;
+  RunningInstanceModel.count = async ({ where }) => {
+    instanceCountCalls++;
+    instanceWhere = where;
+    return 2;
+  };
+  RunningInstanceModel.destroy = async ({ where }) => {
+    instanceWhere = where;
+    return 2;
+  };
+  CrontabStatModel.count = async () => {
+    statCountCalls++;
+    return 3;
+  };
+  CrontabStatModel.destroy = async () => 3;
+  sequelize.transaction = async (callback) => callback({});
+  sequelize.query = async (query) => {
+    vacuumed = query === 'VACUUM';
+    return [];
+  };
+
+  const service = new RetentionService();
+  const disabledPreview = await service.preview({
+    runningInstanceRetentionDays: 0,
+    cronStatRetentionDays: 0,
+  });
+  assert.equal(disabledPreview.runningInstances, 0);
+  assert.equal(disabledPreview.cronStats, 0);
+  assert.equal(instanceCountCalls, 0);
+  assert.equal(statCountCalls, 0);
+
+  const result = await service.cleanup({
+    runningInstanceRetentionDays: 30,
+    cronStatRetentionDays: 90,
+    dependenceCacheTypes: ['node'],
+    compactDatabase: true,
+  });
+
+  assert.equal(instanceWhere.status[Op.ne], InstanceStatus.running);
+  assert.equal(result.deleted.runningInstances, 2);
+  assert.equal(result.deleted.cronStats, 3);
+  assert.equal(result.deleted.dependenceCacheBytes, 13);
+  assert.equal(vacuumed, true);
+  await assert.rejects(fs.stat(nodeCache), { code: 'ENOENT' });
+});