perf: bound runtime memory and storage usage

back/api/cron.ts

@@ -307,13 +307,35 @@ export default (app: Router) => {
       params: Joi.object({
         id: Joi.number().required(),
       }),
+      query: Joi.object({
+        offset: Joi.number().integer().min(0).optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(1024 * 1024)
+          .optional(),
+        tail: Joi.boolean().optional(),
+        t: Joi.string().optional(),
+      }).unknown(true),
     }),
     async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
       const logger: Logger = Container.get('logger');
       try {
         const cronService = Container.get(CronService);
-        const result = await cronService.log(req.params.id);
-        return res.send({ code: 200, data: result.content, logStatus: result.status });
+        const result = await cronService.log(req.params.id, {
+          offset: req.query.offset as unknown as number,
+          limit: req.query.limit as unknown as number,
+          tail: req.query.tail as unknown as boolean,
+        });
+        return res.send({
+          code: 200,
+          data: result.content,
+          logStatus: result.status,
+          offset: result.offset,
+          nextOffset: result.nextOffset,
+          total: result.total,
+          truncated: result.truncated,
+        });
       } catch (e) {
         return next(e);
       }

back/api/index.ts

@@ -13,6 +13,7 @@ import update from './update';
 import dashboard from './dashboard';
 import health from './health';
 import clientIp from './clientIp';
+import retention from './retention';
 
 export default () => {
   const app = Router();
@@ -30,6 +31,7 @@ export default () => {
   dashboard(app);
   health(app);
   clientIp(app);
+  retention(app);
 
   return app;
 };

back/api/log.ts

@@ -12,6 +12,7 @@ import {
 } from '../config/util';
 import LogService from '../services/log';
 import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
+import { MAX_LOG_CHUNK_BYTES, readLogChunk } from '../shared/logReader';
 const route = Router();
 const blacklist = ['.tmp'];
 
@@ -34,6 +35,20 @@ export default (app: Router) => {
 
   route.get(
     '/detail',
+    celebrate({
+      query: Joi.object({
+        path: Joi.string().allow('').optional(),
+        file: Joi.string().required(),
+        offset: Joi.number().integer().min(0).optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(MAX_LOG_CHUNK_BYTES)
+          .optional(),
+        tail: Joi.boolean().optional(),
+        t: Joi.string().optional(),
+      }).unknown(true),
+    }),
     async (req: Request, res: Response, next: NextFunction) => {
       try {
         const logService = Container.get(LogService);
@@ -52,11 +67,19 @@ export default (app: Router) => {
           where: { log_path: logPath, status: InstanceStatus.running },
         });
 
-        const content = await getFileContentByName(finalPath);
+        const chunk = await readLogChunk(finalPath, {
+          offset: req.query.offset as unknown as number,
+          limit: req.query.limit as unknown as number,
+          tail: req.query.tail as unknown as boolean,
+        });
         res.send({
           code: 200,
-          data: removeAnsi(content),
+          data: removeAnsi(chunk.content),
           logStatus: runningInstance ? 'running' : undefined,
+          offset: chunk.offset,
+          nextOffset: chunk.nextOffset,
+          total: chunk.total,
+          truncated: chunk.truncated,
         });
       } catch (e) {
         return next(e);

back/api/retention.ts

@@ -0,0 +1,78 @@
+import { NextFunction, Request, Response, Router } from 'express';
+import { celebrate, Joi } from 'celebrate';
+import { Container } from 'typedi';
+import RetentionService from '../services/retention';
+import { MAX_RETENTION_DAYS } from '../shared/retention';
+
+const route = Router();
+const policySchema = {
+  runningInstanceRetentionDays: Joi.number()
+    .integer()
+    .min(0)
+    .max(MAX_RETENTION_DAYS)
+    .required(),
+  cronStatRetentionDays: Joi.number()
+    .integer()
+    .min(0)
+    .max(MAX_RETENTION_DAYS)
+    .required(),
+};
+const cleanupSchema = {
+  ...policySchema,
+  dependenceCacheTypes: Joi.array()
+    .items(Joi.string().valid('node', 'python3'))
+    .unique()
+    .default([]),
+  compactDatabase: Joi.boolean().default(false),
+};
+
+export default (app: Router) => {
+  app.use('/system/storage-retention', route);
+
+  route.put(
+    '/config',
+    celebrate({ body: Joi.object(policySchema) }),
+    async (req: Request, res: Response, next: NextFunction) => {
+      try {
+        const service = Container.get(RetentionService);
+        const data = await service.updatePolicy(req.body);
+        res.send({ code: 200, data });
+      } catch (error) {
+        next(error);
+      }
+    },
+  );
+
+  route.post(
+    '/preview',
+    celebrate({ body: Joi.object(cleanupSchema) }),
+    async (req: Request, res: Response, next: NextFunction) => {
+      try {
+        const service = Container.get(RetentionService);
+        const data = await service.preview(req.body);
+        res.send({ code: 200, data });
+      } catch (error) {
+        next(error);
+      }
+    },
+  );
+
+  route.post(
+    '/cleanup',
+    celebrate({
+      body: Joi.object({
+        ...cleanupSchema,
+        confirmation: Joi.string().valid('CLEAN').required(),
+      }),
+    }),
+    async (req: Request, res: Response, next: NextFunction) => {
+      try {
+        const service = Container.get(RetentionService);
+        const data = await service.cleanup(req.body);
+        res.send({ code: 200, data });
+      } catch (error) {
+        next(error);
+      }
+    },
+  );
+};

back/api/subscription.ts

@@ -148,13 +148,27 @@ export default (app: Router) => {
       params: Joi.object({
         id: Joi.number().required(),
       }),
+      query: Joi.object({
+        offset: Joi.number().integer().min(0).optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(1024 * 1024)
+          .optional(),
+        tail: Joi.boolean().optional(),
+        t: Joi.string().optional(),
+      }).unknown(true),
     }),
     async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
       const logger: Logger = Container.get('logger');
       try {
         const subscriptionService = Container.get(SubscriptionService);
-        const data = await subscriptionService.log(req.params.id);
-        return res.send({ code: 200, data });
+        const result = await subscriptionService.log(req.params.id, {
+          offset: req.query.offset as unknown as number,
+          limit: req.query.limit as unknown as number,
+          tail: req.query.tail as unknown as boolean,
+        });
+        return res.send({ code: 200, data: result.content, ...result });
       } catch (e) {
         return next(e);
       }

back/api/system.ts

@@ -350,6 +350,11 @@ export default (app: Router) => {
       query: {
         startTime: Joi.string().allow('').optional(),
         endTime: Joi.string().allow('').optional(),
+        limit: Joi.number()
+          .integer()
+          .min(1)
+          .max(1024 * 1024)
+          .optional(),
         t: Joi.string().optional(),
       },
     }),
@@ -361,6 +366,7 @@ export default (app: Router) => {
           req.query as {
             startTime?: string;
             endTime?: string;
+            limit?: number;
           },
         );
       } catch (e) {