perf: bound runtime memory and storage usage

back/api/cron.ts

@@ -307,13 +307,35 @@ export default (app: Router) => {
       params: Joi.object({
         id: Joi.number().required(),
       }),
+      query: Joi.object({
+        offset: Joi.number().integer().min(0).optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(1024 * 1024)
+          .optional(),
+        tail: Joi.boolean().optional(),
+        t: Joi.string().optional(),
+      }).unknown(true),
     }),
     async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
       const logger: Logger = Container.get('logger');
       try {
         const cronService = Container.get(CronService);
-        const result = await cronService.log(req.params.id);
-        return res.send({ code: 200, data: result.content, logStatus: result.status });
+        const result = await cronService.log(req.params.id, {
+          offset: req.query.offset as unknown as number,
+          limit: req.query.limit as unknown as number,
+          tail: req.query.tail as unknown as boolean,
+        });
+        return res.send({
+          code: 200,
+          data: result.content,
+          logStatus: result.status,
+          offset: result.offset,
+          nextOffset: result.nextOffset,
+          total: result.total,
+          truncated: result.truncated,
+        });
       } catch (e) {
         return next(e);
       }

back/api/index.ts

@@ -13,6 +13,7 @@ import update from './update';
 import dashboard from './dashboard';
 import health from './health';
 import clientIp from './clientIp';
+import retention from './retention';
 
 export default () => {
   const app = Router();
@@ -30,6 +31,7 @@ export default () => {
   dashboard(app);
   health(app);
   clientIp(app);
+  retention(app);
 
   return app;
 };

back/api/log.ts

@@ -12,6 +12,7 @@ import {
 } from '../config/util';
 import LogService from '../services/log';
 import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
+import { MAX_LOG_CHUNK_BYTES, readLogChunk } from '../shared/logReader';
 const route = Router();
 const blacklist = ['.tmp'];
 
@@ -34,6 +35,20 @@ export default (app: Router) => {
 
   route.get(
     '/detail',
+    celebrate({
+      query: Joi.object({
+        path: Joi.string().allow('').optional(),
+        file: Joi.string().required(),
+        offset: Joi.number().integer().min(0).optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(MAX_LOG_CHUNK_BYTES)
+          .optional(),
+        tail: Joi.boolean().optional(),
+        t: Joi.string().optional(),
+      }).unknown(true),
+    }),
     async (req: Request, res: Response, next: NextFunction) => {
       try {
         const logService = Container.get(LogService);
@@ -52,11 +67,19 @@ export default (app: Router) => {
           where: { log_path: logPath, status: InstanceStatus.running },
         });
 
-        const content = await getFileContentByName(finalPath);
+        const chunk = await readLogChunk(finalPath, {
+          offset: req.query.offset as unknown as number,
+          limit: req.query.limit as unknown as number,
+          tail: req.query.tail as unknown as boolean,
+        });
         res.send({
           code: 200,
-          data: removeAnsi(content),
+          data: removeAnsi(chunk.content),
           logStatus: runningInstance ? 'running' : undefined,
+          offset: chunk.offset,
+          nextOffset: chunk.nextOffset,
+          total: chunk.total,
+          truncated: chunk.truncated,
         });
       } catch (e) {
         return next(e);

back/api/retention.ts

@@ -0,0 +1,78 @@
+import { NextFunction, Request, Response, Router } from 'express';
+import { celebrate, Joi } from 'celebrate';
+import { Container } from 'typedi';
+import RetentionService from '../services/retention';
+import { MAX_RETENTION_DAYS } from '../shared/retention';
+
+const route = Router();
+const policySchema = {
+  runningInstanceRetentionDays: Joi.number()
+    .integer()
+    .min(0)
+    .max(MAX_RETENTION_DAYS)
+    .required(),
+  cronStatRetentionDays: Joi.number()
+    .integer()
+    .min(0)
+    .max(MAX_RETENTION_DAYS)
+    .required(),
+};
+const cleanupSchema = {
+  ...policySchema,
+  dependenceCacheTypes: Joi.array()
+    .items(Joi.string().valid('node', 'python3'))
+    .unique()
+    .default([]),
+  compactDatabase: Joi.boolean().default(false),
+};
+
+export default (app: Router) => {
+  app.use('/system/storage-retention', route);
+
+  route.put(
+    '/config',
+    celebrate({ body: Joi.object(policySchema) }),
+    async (req: Request, res: Response, next: NextFunction) => {
+      try {
+        const service = Container.get(RetentionService);
+        const data = await service.updatePolicy(req.body);
+        res.send({ code: 200, data });
+      } catch (error) {
+        next(error);
+      }
+    },
+  );
+
+  route.post(
+    '/preview',
+    celebrate({ body: Joi.object(cleanupSchema) }),
+    async (req: Request, res: Response, next: NextFunction) => {
+      try {
+        const service = Container.get(RetentionService);
+        const data = await service.preview(req.body);
+        res.send({ code: 200, data });
+      } catch (error) {
+        next(error);
+      }
+    },
+  );
+
+  route.post(
+    '/cleanup',
+    celebrate({
+      body: Joi.object({
+        ...cleanupSchema,
+        confirmation: Joi.string().valid('CLEAN').required(),
+      }),
+    }),
+    async (req: Request, res: Response, next: NextFunction) => {
+      try {
+        const service = Container.get(RetentionService);
+        const data = await service.cleanup(req.body);
+        res.send({ code: 200, data });
+      } catch (error) {
+        next(error);
+      }
+    },
+  );
+};

back/api/subscription.ts

@@ -148,13 +148,27 @@ export default (app: Router) => {
       params: Joi.object({
         id: Joi.number().required(),
       }),
+      query: Joi.object({
+        offset: Joi.number().integer().min(0).optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(1024 * 1024)
+          .optional(),
+        tail: Joi.boolean().optional(),
+        t: Joi.string().optional(),
+      }).unknown(true),
     }),
     async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
       const logger: Logger = Container.get('logger');
       try {
         const subscriptionService = Container.get(SubscriptionService);
-        const data = await subscriptionService.log(req.params.id);
-        return res.send({ code: 200, data });
+        const result = await subscriptionService.log(req.params.id, {
+          offset: req.query.offset as unknown as number,
+          limit: req.query.limit as unknown as number,
+          tail: req.query.tail as unknown as boolean,
+        });
+        return res.send({ code: 200, data: result.content, ...result });
       } catch (e) {
         return next(e);
       }

back/api/system.ts

@@ -350,6 +350,11 @@ export default (app: Router) => {
       query: {
         startTime: Joi.string().allow('').optional(),
         endTime: Joi.string().allow('').optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(1024 * 1024)
+          .optional(),
         t: Joi.string().optional(),
       },
     }),
@@ -361,6 +366,7 @@ export default (app: Router) => {
           req.query as {
             startTime?: string;
             endTime?: string;
+            limit?: number;
           },
         );
       } catch (e) {

back/data/system.ts

@@ -41,6 +41,8 @@ export interface SystemConfigInfo {
   linuxMirror?: string;
   timezone?: string;
   globalSshKey?: string;
+  runningInstanceRetentionDays?: number;
+  cronStatRetentionDays?: number;
 }
 
 export interface LoginLogInfo {

back/services/cron.ts

@@ -32,6 +32,7 @@ import { t } from '../shared/i18n';
 import { ScheduleType } from '../interface/schedule';
 import { logStreamManager } from '../shared/logStreamManager';
 import { isEmpty } from 'lodash';
+import { LogReadOptions, readLogChunk } from '../shared/logReader';
 
 @Service()
 export default class CronService {
@@ -766,27 +767,63 @@ export default class CronService {
     await this.setCrontab();
   }
 
-  public async log(id: number): Promise<{ content: string; status: string }> {
+  public async log(
+    id: number,
+    options: LogReadOptions = {},
+  ): Promise<{
+    content: string;
+    status: string;
+    offset: number;
+    nextOffset: number;
+    total: number;
+    truncated: boolean;
+  }> {
     const doc = await this.getDb({ id });
     if (!doc) {
-      return { content: '', status: 'empty' };
+      return {
+        content: '',
+        status: 'empty',
+        offset: 0,
+        nextOffset: 0,
+        total: 0,
+        truncated: false,
+      };
     }
     if (doc.log_name === '/dev/null') {
-      return { content: t('日志设置为忽略'), status: 'ignored' };
+      return {
+        content: t('日志设置为忽略'),
+        status: 'ignored',
+        offset: 0,
+        nextOffset: 0,
+        total: 0,
+        truncated: false,
+      };
     }
     const absolutePath = path.resolve(config.logPath, `${doc.log_path}`);
     const logFileExist = doc.log_path && (await fileExist(absolutePath));
     if (logFileExist) {
-      const content = await getFileContentByName(`${absolutePath}`);
+      const chunk = await readLogChunk(`${absolutePath}`, options);
       const isRunning =
         typeof doc.status === 'number' &&
         [CrontabStatus.running, CrontabStatus.queued].includes(doc.status);
-      return { content, status: isRunning ? 'running' : 'completed' };
+      return {
+        ...chunk,
+        status: isRunning ? 'running' : 'completed',
+      };
     } else {
-      return typeof doc.status === 'number' &&
+      const status =
+        typeof doc.status === 'number' &&
         [CrontabStatus.queued, CrontabStatus.running].includes(doc.status)
-        ? { content: t('运行中...'), status: 'running' }
-        : { content: t('日志不存在...'), status: 'notFound' };
+          ? 'running'
+          : 'notFound';
+      return {
+        content: status === 'running' ? t('运行中...') : t('日志不存在...'),
+        status,
+        offset: 0,
+        nextOffset: 0,
+        total: 0,
+        truncated: false,
+      };
     }
   }
 

back/services/metrics.ts

@@ -8,16 +8,19 @@ interface Metric {
   tags?: Record<string, string>;
 }
 
-class MetricsService {
-  private metrics: Metric[] = [];
+export const MAX_METRIC_SAMPLES = 1000;
+const METRIC_RETENTION_MS = 60 * 60 * 1000;
+
+export class MetricsService {
+  private metrics: Array<Metric | undefined> = new Array(MAX_METRIC_SAMPLES);
+  private metricCount = 0;
+  private nextMetricIndex = 0;
   private static instance: MetricsService;
 
   private constructor() {
     // 定期清理旧数据
-    setInterval(() => {
-      const oneHourAgo = Date.now() - 3600000;
-      this.metrics = this.metrics.filter(m => m.timestamp > oneHourAgo);
-    }, 60000);
+    const cleanupTimer = setInterval(() => this.removeExpiredMetrics(), 60000);
+    cleanupTimer.unref();
   }
 
   static getInstance(): MetricsService {
@@ -28,12 +31,14 @@ class MetricsService {
   }
 
   record(name: string, value: number, tags?: Record<string, string>) {
-    this.metrics.push({
+    this.metrics[this.nextMetricIndex] = {
       name,
       value,
       timestamp: Date.now(),
       tags,
-    });
+    };
+    this.nextMetricIndex = (this.nextMetricIndex + 1) % MAX_METRIC_SAMPLES;
+    this.metricCount = Math.min(this.metricCount + 1, MAX_METRIC_SAMPLES);
   }
 
   measure(name: string, fn: () => void, tags?: Record<string, string>) {
@@ -46,7 +51,11 @@ class MetricsService {
     }
   }
 
-  async measureAsync(name: string, fn: () => Promise<void>, tags?: Record<string, string>) {
+  async measureAsync(
+    name: string,
+    fn: () => Promise<void>,
+    tags?: Record<string, string>,
+  ) {
     const start = performance.now();
     try {
       await fn();
@@ -57,28 +66,61 @@ class MetricsService {
   }
 
   getMetrics(name?: string, tags?: Record<string, string>) {
-    let filtered = this.metrics;
-    
+    this.removeExpiredMetrics();
+    let filtered = this.getMetricSnapshot();
+
     if (name) {
-      filtered = filtered.filter(m => m.name === name);
+      filtered = filtered.filter((m) => m.name === name);
     }
-    
+
     if (tags) {
-      filtered = filtered.filter(m => {
+      filtered = filtered.filter((m) => {
         if (!m.tags) return false;
-        return Object.entries(tags).every(([key, value]) => m.tags![key] === value);
+        return Object.entries(tags).every(
+          ([key, value]) => m.tags![key] === value,
+        );
       });
     }
 
+    const values = filtered.map((metric) => metric.value);
     return {
       count: filtered.length,
-      average: filtered.reduce((acc, curr) => acc + curr.value, 0) / filtered.length,
-      min: Math.min(...filtered.map(m => m.value)),
-      max: Math.max(...filtered.map(m => m.value)),
+      average: values.length
+        ? values.reduce((acc, value) => acc + value, 0) / values.length
+        : 0,
+      min: values.length ? Math.min(...values) : 0,
+      max: values.length ? Math.max(...values) : 0,
       metrics: filtered,
     };
   }
 
+  private getMetricSnapshot(): Metric[] {
+    if (this.metricCount < MAX_METRIC_SAMPLES) {
+      return this.metrics.slice(0, this.metricCount) as Metric[];
+    }
+    return [
+      ...this.metrics.slice(this.nextMetricIndex),
+      ...this.metrics.slice(0, this.nextMetricIndex),
+    ] as Metric[];
+  }
+
+  private removeExpiredMetrics() {
+    const oldestTimestamp = Date.now() - METRIC_RETENTION_MS;
+    const retained = this.getMetricSnapshot().filter(
+      (metric) => metric.timestamp > oldestTimestamp,
+    );
+    if (retained.length === this.metricCount) return;
+
+    this.metrics = new Array(MAX_METRIC_SAMPLES);
+    this.metricCount = 0;
+    this.nextMetricIndex = 0;
+    for (const metric of retained) {
+      this.metrics[this.nextMetricIndex] = metric;
+      this.nextMetricIndex = (this.nextMetricIndex + 1) % MAX_METRIC_SAMPLES;
+      this.metricCount++;
+    }
+  }
+
   report() {
     const report = {
       timestamp: Date.now(),
@@ -89,4 +131,4 @@ class MetricsService {
   }
 }
 
-export const metricsService = MetricsService.getInstance(); 
+export const metricsService = MetricsService.getInstance();

back/services/retention.ts

@@ -0,0 +1,155 @@
+import fs from 'fs/promises';
+import path from 'path';
+import dayjs from 'dayjs';
+import { Op } from 'sequelize';
+import { Service } from 'typedi';
+import config from '../config';
+import { CrontabStatModel } from '../data/cronStats';
+import { sequelize } from '../data';
+import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
+import { AuthDataType, SystemModel } from '../data/system';
+import {
+  DependenceCacheType,
+  getDirectorySize,
+  isDependenceCacheType,
+  normalizeRetentionPolicy,
+  RetentionPolicy,
+} from '../shared/retention';
+
+export interface StorageCleanupRequest extends RetentionPolicy {
+  dependenceCacheTypes?: DependenceCacheType[];
+  compactDatabase?: boolean;
+}
+
+function runningInstanceWhere(days: number) {
+  const cutoff = dayjs().subtract(days, 'day').unix();
+  return {
+    status: { [Op.ne]: InstanceStatus.running },
+    [Op.or]: [
+      { finished_at: { [Op.lt]: cutoff } },
+      {
+        finished_at: { [Op.is]: null },
+        started_at: { [Op.lt]: cutoff },
+      },
+    ],
+  };
+}
+
+function cronStatWhere(days: number) {
+  return {
+    date: { [Op.lt]: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
+  };
+}
+
+@Service()
+export default class RetentionService {
+  public async updatePolicy(policy: Partial<RetentionPolicy>) {
+    const normalized = normalizeRetentionPolicy(policy);
+    const systemConfig = await SystemModel.findOne({
+      where: { type: AuthDataType.systemConfig },
+    });
+    if (!systemConfig) {
+      throw new Error('System config not found');
+    }
+    await SystemModel.update(
+      {
+        info: {
+          ...systemConfig.info,
+          ...normalized,
+        },
+      },
+      { where: { id: systemConfig.id } },
+    );
+    return normalized;
+  }
+
+  public async preview(request: StorageCleanupRequest) {
+    const policy = normalizeRetentionPolicy(request);
+    const dependenceCacheTypes = (request.dependenceCacheTypes || []).filter(
+      isDependenceCacheType,
+    );
+    const [runningInstances, cronStats, dependenceCaches] = await Promise.all([
+      policy.runningInstanceRetentionDays > 0
+        ? RunningInstanceModel.count({
+            where: runningInstanceWhere(policy.runningInstanceRetentionDays),
+          })
+        : 0,
+      policy.cronStatRetentionDays > 0
+        ? CrontabStatModel.count({
+            where: cronStatWhere(policy.cronStatRetentionDays),
+          })
+        : 0,
+      Promise.all(
+        dependenceCacheTypes.map(async (type) => ({
+          type,
+          bytes: await getDirectorySize(
+            path.join(config.dependenceCachePath, type),
+          ),
+        })),
+      ),
+    ]);
+
+    return {
+      policy,
+      runningInstances,
+      cronStats,
+      dependenceCaches,
+      dependenceCacheBytes: dependenceCaches.reduce(
+        (total, cache) => total + cache.bytes,
+        0,
+      ),
+      compactDatabase: Boolean(request.compactDatabase),
+    };
+  }
+
+  public async cleanup(request: StorageCleanupRequest) {
+    const preview = await this.preview(request);
+    const deleted = await sequelize.transaction(async (transaction) => {
+      const runningInstances =
+        preview.policy.runningInstanceRetentionDays > 0
+          ? await RunningInstanceModel.destroy({
+              where: runningInstanceWhere(
+                preview.policy.runningInstanceRetentionDays,
+              ),
+              transaction,
+            })
+          : 0;
+      const cronStats =
+        preview.policy.cronStatRetentionDays > 0
+          ? await CrontabStatModel.destroy({
+              where: cronStatWhere(preview.policy.cronStatRetentionDays),
+              transaction,
+            })
+          : 0;
+      return { runningInstances, cronStats };
+    });
+
+    const dependenceCaches = [];
+    for (const cache of preview.dependenceCaches) {
+      await fs.rm(path.join(config.dependenceCachePath, cache.type), {
+        recursive: true,
+        force: true,
+      });
+      dependenceCaches.push(cache);
+    }
+
+    if (
+      request.compactDatabase &&
+      (deleted.runningInstances || deleted.cronStats)
+    ) {
+      await sequelize.query('VACUUM');
+    }
+
+    return {
+      deleted: {
+        ...deleted,
+        dependenceCaches,
+        dependenceCacheBytes: preview.dependenceCacheBytes,
+      },
+      compactedDatabase: Boolean(
+        request.compactDatabase &&
+          (deleted.runningInstances || deleted.cronStats),
+      ),
+    };
+  }
+}

back/services/subscription.ts

@@ -33,6 +33,7 @@ import { CrontabModel } from '../data/cron';
 import CrontabService from './cron';
 import taskLimit from '../shared/pLimit';
 import { logStreamManager } from '../shared/logStreamManager';
+import { LogReadOptions, readLogChunk } from '../shared/logReader';
 
 @Service()
 export default class SubscriptionService {
@@ -366,14 +367,20 @@ export default class SubscriptionService {
     }
   }
 
-  public async log(id: number) {
+  public async log(id: number, options: LogReadOptions = {}) {
     const doc = await this.getDb({ id });
     if (!doc || !doc.log_path) {
-      return '';
+      return {
+        content: '',
+        offset: 0,
+        nextOffset: 0,
+        total: 0,
+        truncated: false,
+      };
     }
 
     const absolutePath = await handleLogPath(doc.log_path as string);
-    return await getFileContentByName(absolutePath);
+    return await readLogChunk(absolutePath, options);
   }
 
   public async logs(id: number) {

back/services/system.ts

@@ -2,7 +2,6 @@ import { spawn } from 'cross-spawn';
 import { Response } from 'express';
 import fs from 'fs';
 import { Agent, request } from 'undici';
-import sum from 'lodash/sum';
 import path from 'path';
 import { Inject, Service } from 'typedi';
 import winston from 'winston';
@@ -467,6 +466,7 @@ export default class SystemService {
     query: {
       startTime?: string;
       endTime?: string;
+      limit?: number;
     },
   ) {
     const startTime = dayjs(query.startTime || undefined)
@@ -481,8 +481,30 @@ export default class SystemService {
       .filter((x) => x.title.endsWith('.log'))
       .filter((x) => x.createTime >= startTime && x.createTime <= endTime);
 
+    const limit = Math.min(
+      Math.max(Number(query.limit) || 1024 * 1024, 1),
+      1024 * 1024,
+    );
+    const total = logs.reduce((size, log) => size + (log.size || 0), 0);
+    let remaining = limit;
+    const selected: Array<
+      (typeof logs)[number] & { start: number; length: number }
+    > = [];
+    for (let index = logs.length - 1; index >= 0 && remaining > 0; index--) {
+      const log = logs[index];
+      const size = log.size || 0;
+      const length = Math.min(size, remaining);
+      if (length > 0) {
+        selected.unshift({ ...log, start: size - length, length });
+        remaining -= length;
+      }
+    }
+
+    const contentLength = selected.reduce((size, log) => size + log.length, 0);
     res.set({
-      'Content-Length': sum(logs.map((x) => x.size)),
+      'Content-Length': contentLength,
+      'X-QL-Log-Total': total,
+      'X-QL-Log-Truncated': total > contentLength ? 'true' : 'false',
     });
     (function sendFiles(res, fileNames) {
       if (fileNames.length === 0) {
@@ -494,13 +516,17 @@ export default class SystemService {
       if (currentLog) {
         const currentFileStream = fs.createReadStream(
           path.join(config.systemLogPath, currentLog.title),
+          {
+            start: currentLog.start,
+            end: currentLog.start + currentLog.length - 1,
+          },
         );
         currentFileStream.on('end', () => {
           sendFiles(res, fileNames);
         });
         currentFileStream.pipe(res, { end: false });
       }
-    })(res, logs);
+    })(res, selected);
   }
 
   public async deleteSystemLog() {

back/shared/logReader.ts

@@ -0,0 +1,109 @@
+import fs from 'fs/promises';
+
+export const DEFAULT_LOG_CHUNK_BYTES = 256 * 1024;
+export const MAX_LOG_CHUNK_BYTES = 1024 * 1024;
+
+export interface LogReadOptions {
+  offset?: number;
+  limit?: number;
+  tail?: boolean;
+}
+
+export interface LogChunk {
+  content: string;
+  offset: number;
+  nextOffset: number;
+  total: number;
+  truncated: boolean;
+}
+
+function normalizeLimit(limit?: number) {
+  if (!Number.isFinite(limit)) return DEFAULT_LOG_CHUNK_BYTES;
+  return Math.min(Math.max(Math.trunc(limit!), 4), MAX_LOG_CHUNK_BYTES);
+}
+
+function isUtf8ContinuationByte(byte: number) {
+  return (byte & 0xc0) === 0x80;
+}
+
+function completeUtf8End(buffer: Buffer, start: number, end: number) {
+  if (end <= start) return start;
+
+  let sequenceStart = end - 1;
+  while (
+    sequenceStart > start &&
+    isUtf8ContinuationByte(buffer[sequenceStart])
+  ) {
+    sequenceStart--;
+  }
+
+  const firstByte = buffer[sequenceStart];
+  const expectedLength =
+    firstByte < 0x80
+      ? 1
+      : firstByte < 0xe0
+        ? 2
+        : firstByte < 0xf0
+          ? 3
+          : 4;
+  return end - sequenceStart < expectedLength ? sequenceStart : end;
+}
+
+export async function readLogChunk(
+  filePath: string,
+  options: LogReadOptions = {},
+): Promise<LogChunk> {
+  let handle: fs.FileHandle | undefined;
+  try {
+    handle = await fs.open(filePath, 'r');
+    const { size: total } = await handle.stat();
+    const limit = normalizeLimit(options.limit);
+    const requestedOffset = Number.isFinite(options.offset)
+      ? Math.trunc(options.offset!)
+      : undefined;
+    const requestedStart =
+      options.tail || requestedOffset === undefined
+        ? Math.max(total - limit, 0)
+        : Math.min(Math.max(requestedOffset, 0), total);
+    const length = Math.min(limit + 3, total - requestedStart);
+    const buffer = Buffer.allocUnsafe(length);
+    const { bytesRead } = await handle.read(
+      buffer,
+      0,
+      length,
+      requestedStart,
+    );
+    let leadingBytes = 0;
+    while (
+      leadingBytes < bytesRead &&
+      isUtf8ContinuationByte(buffer[leadingBytes])
+    ) {
+      leadingBytes++;
+    }
+    const offset = requestedStart + leadingBytes;
+    const candidateEnd = Math.min(leadingBytes + limit, bytesRead);
+    const contentEnd = completeUtf8End(buffer, leadingBytes, candidateEnd);
+    const nextOffset = requestedStart + contentEnd;
+
+    return {
+      content: buffer.subarray(leadingBytes, contentEnd).toString('utf8'),
+      offset,
+      nextOffset,
+      total,
+      truncated: offset > 0 || nextOffset < total,
+    };
+  } catch (error: any) {
+    if (error?.code === 'ENOENT') {
+      return {
+        content: '',
+        offset: 0,
+        nextOffset: 0,
+        total: 0,
+        truncated: false,
+      };
+    }
+    throw error;
+  } finally {
+    await handle?.close();
+  }
+}

back/shared/retention.ts

@@ -0,0 +1,64 @@
+import fs from 'fs/promises';
+import { Dirent } from 'fs';
+import path from 'path';
+
+export const MAX_RETENTION_DAYS = 3650;
+export const DEPENDENCE_CACHE_TYPES = ['node', 'python3'] as const;
+
+export type DependenceCacheType = (typeof DEPENDENCE_CACHE_TYPES)[number];
+
+export interface RetentionPolicy {
+  runningInstanceRetentionDays: number;
+  cronStatRetentionDays: number;
+}
+
+export function normalizeRetentionDays(value: unknown) {
+  const days = Number(value);
+  if (!Number.isFinite(days)) return 0;
+  return Math.min(Math.max(Math.trunc(days), 0), MAX_RETENTION_DAYS);
+}
+
+export function normalizeRetentionPolicy(
+  policy: Partial<RetentionPolicy>,
+): RetentionPolicy {
+  return {
+    runningInstanceRetentionDays: normalizeRetentionDays(
+      policy.runningInstanceRetentionDays,
+    ),
+    cronStatRetentionDays: normalizeRetentionDays(policy.cronStatRetentionDays),
+  };
+}
+
+export function isDependenceCacheType(
+  value: string,
+): value is DependenceCacheType {
+  return DEPENDENCE_CACHE_TYPES.includes(value as DependenceCacheType);
+}
+
+export async function getDirectorySize(rootPath: string): Promise<number> {
+  const pending = [rootPath];
+  let total = 0;
+
+  while (pending.length > 0) {
+    const currentPath = pending.pop()!;
+    let entries: Dirent[];
+    try {
+      entries = await fs.readdir(currentPath, { withFileTypes: true });
+    } catch (error: any) {
+      if (error?.code === 'ENOENT') continue;
+      throw error;
+    }
+
+    for (const entry of entries) {
+      if (entry.isSymbolicLink()) continue;
+      const entryPath = path.join(currentPath, entry.name);
+      if (entry.isDirectory()) {
+        pending.push(entryPath);
+      } else if (entry.isFile()) {
+        total += (await fs.stat(entryPath)).size;
+      }
+    }
+  }
+
+  return total;
+}