mirror of
https://github.com/whyour/qinglong.git
synced 2026-11-04 00:42:01 +08:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8976447b1e | ||
|
|
15d54866a8 |
@@ -550,10 +550,14 @@ jobs:
|
||||
cache-from: type=registry,ref=whyour/qinglong:cache-debian-python3.10
|
||||
cache-to: type=registry,ref=whyour/qinglong:cache-debian-python3.10,mode=max
|
||||
|
||||
# One unmodified amd64 package for migration acceptance only. This does not
|
||||
# Unmodified amd64 packages for glibc and musl migration acceptance. This does not
|
||||
# publish bundles or advance development/formal channel indexes.
|
||||
build-npm-migration-bundle:
|
||||
needs: build-debian
|
||||
needs: [build-debian, build-alpine]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
libc: [glibc, musl]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
@@ -561,7 +565,7 @@ jobs:
|
||||
- uses: docker/setup-buildx-action@v4
|
||||
- name: Export the official platform package from this run's image digest
|
||||
env:
|
||||
IMAGE_DIGEST: ${{ needs.build-debian.outputs.image-digest }}
|
||||
IMAGE_DIGEST: ${{ matrix.libc == 'musl' && needs.build-alpine.outputs.image-digest || needs.build-debian.outputs.image-digest }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$IMAGE_DIGEST" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
@@ -569,20 +573,21 @@ jobs:
|
||||
docker buildx build --platform linux/amd64 --build-arg "BASE_IMAGE=$image" --file docker/Dockerfile.upgrade-bundle --output "type=local,dest=$RUNNER_TEMP/npm-migration-bundle" .
|
||||
- uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: npm-migration-bundle-${{ github.sha }}
|
||||
name: npm-migration-bundle-${{ matrix.libc }}-${{ github.sha }}
|
||||
path: ${{ runner.temp }}/npm-migration-bundle/*
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
|
||||
verify-npm-package:
|
||||
needs: [build-static, build-npm-migration-bundle]
|
||||
needs: [build-static, build-npm-migration-bundle, build-alpine]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
node: ['20.18.1', '24']
|
||||
container: node:${{ matrix.node }}-bookworm
|
||||
node: ['20.18.1', '22.17.0', '24', 'alpine']
|
||||
container:
|
||||
image: ${{ matrix.node == 'alpine' && format('ghcr.io/{0}@{1}', github.repository, needs.build-alpine.outputs.image-digest) || format('node:{0}-bookworm', matrix.node) }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/download-artifact@v6
|
||||
@@ -591,22 +596,30 @@ jobs:
|
||||
path: ${{ runner.temp }}/npm-panel/
|
||||
- uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: npm-migration-bundle-${{ github.sha }}
|
||||
name: npm-migration-bundle-${{ matrix.node == 'alpine' && 'musl' || 'glibc' }}-${{ github.sha }}
|
||||
path: ${{ runner.temp }}/npm-migration-bundle/
|
||||
- name: Install isolated service test tools
|
||||
run: |
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends python3 openssl curl jq nginx procps
|
||||
if [ -f /etc/alpine-release ]; then
|
||||
apk add --no-cache python3 openssl curl jq nginx procps make g++
|
||||
else
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends python3 openssl curl jq nginx procps
|
||||
fi
|
||||
# The published legacy launcher resets PATH to standard directories.
|
||||
# /usr/local is confined to this disposable job container; PM2_HOME
|
||||
# remains private to each verification run.
|
||||
npm install --global --prefix /usr/local --no-audit --no-fund pm2@6.0.14
|
||||
- name: Install and upgrade the exact npm archive
|
||||
if: matrix.node != '20.18.1'
|
||||
env:
|
||||
QL_PANEL_PACKAGE_TEST_ISOLATED: '1'
|
||||
QL_PANEL_TEST_PM2: /usr/local/bin/pm2
|
||||
QL_PANEL_TEST_BUNDLE_DIR: ${{ runner.temp }}/npm-migration-bundle
|
||||
run: node scripts/verify-panel-package.cjs verify "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results"
|
||||
- name: Reject an unsupported runtime while preserving the old installation
|
||||
if: matrix.node == '20.18.1'
|
||||
run: node scripts/verify-panel-runtime-floor.cjs "$RUNNER_TEMP/npm-panel" "$RUNNER_TEMP/npm-panel-results"
|
||||
- uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: qinglong-npm-verification-${{ matrix.node }}-${{ github.sha }}
|
||||
@@ -721,7 +734,7 @@ jobs:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '24'
|
||||
- uses: pnpm/action-setup@v6
|
||||
with:
|
||||
version: '8.3.1'
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
version: '8.3.1'
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '22'
|
||||
cache: pnpm
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: pnpm build:front && pnpm build:back && pnpm build:info
|
||||
@@ -120,7 +120,7 @@ jobs:
|
||||
run: |
|
||||
docker run --rm --network none --entrypoint node \
|
||||
qinglong-readiness:${{ matrix.image }} \
|
||||
-e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.4.0",sequelize:"6.37.8"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); require("sqlite3"); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")'
|
||||
-e 'const assert=require("node:assert/strict"); const {createRequire}=require("node:module"); for(const [name,want] of Object.entries({"@grpc/grpc-js":"1.14.5",compression:"1.8.2",multer:"2.4.0"}))assert.equal(require(name+"/package.json").version,want); const expressRequire=createRequire(require.resolve("express")); assert.equal(expressRequire("proxy-addr/package.json").version,"2.0.8"); const sockRequire=createRequire(require.resolve("sockjs")); assert.equal(sockRequire("websocket-driver/package.json").version,"0.7.5"); assert.equal(typeof require("@mikro-orm/sql").NodeSqliteDialect,"function"); const db=new (require("node:sqlite").DatabaseSync)(":memory:"); assert.equal(db.prepare("SELECT 42 AS value").get().value,42); db.close(); require("nodemailer"); require("undici"); console.log("Minimal production patches and runtime imports verified")'
|
||||
- name: Export native Debian image for isolated dev validation
|
||||
if: matrix.image == 'debian'
|
||||
run: |
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
version: '8.3.1'
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '20'
|
||||
node-version: '24'
|
||||
cache: pnpm
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: cp .env.example .env
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# Node.js 原生部署与 Python venv
|
||||
|
||||
面板要求 Node.js >=22.17,推荐 Node.js 24,并要求该 Node 构建提供 `node:sqlite`。应用升级保留现有 Node;旧 Node 20 环境需先升级运行时。远程管理 CLI 的最低版本要求独立于面板。
|
||||
|
||||
设置 `QL_DIR` 为安装目录,`QL_DATA_DIR` 为以 `/data` 结尾的绝对持久化数据路径。
|
||||
如需隔离 Python,请在启动进程的环境中显式设置 `PYTHON_VENV_DIR`(绝对路径或相对 `QL_DIR` 的路径),例如 `export PYTHON_VENV_DIR=/srv/qinglong/.venv`。未设置时保留现有 Python prefix 行为。
|
||||
|
||||
|
||||
+19
-13
@@ -5,10 +5,7 @@ import CronService from '../services/cron';
|
||||
import CronViewService from '../services/cronView';
|
||||
import { celebrate, Joi } from 'celebrate';
|
||||
import { commonCronSchema } from '../validation/schedule';
|
||||
import {
|
||||
RunningInstanceModel,
|
||||
InstanceStatus,
|
||||
} from '../data/runningInstance';
|
||||
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
|
||||
import { t } from '../shared/i18n';
|
||||
import cronClient from '../schedule/client';
|
||||
|
||||
@@ -19,8 +16,17 @@ export default (app: Router) => {
|
||||
|
||||
route.use(async (req, res, next) => {
|
||||
// Keep stop/status callbacks available even when the scheduler is down.
|
||||
if (['POST', 'PUT', 'DELETE'].includes(req.method) &&
|
||||
['/', '/run', '/enable', '/disable', '/views/enable', '/views/disable'].includes(req.path)) {
|
||||
if (
|
||||
['POST', 'PUT', 'DELETE'].includes(req.method) &&
|
||||
[
|
||||
'/',
|
||||
'/run',
|
||||
'/enable',
|
||||
'/disable',
|
||||
'/views/enable',
|
||||
'/views/disable',
|
||||
].includes(req.path)
|
||||
) {
|
||||
try {
|
||||
await cronClient.readiness.ensureReady();
|
||||
} catch (error) {
|
||||
@@ -497,12 +503,8 @@ export default (app: Router) => {
|
||||
}),
|
||||
async (req: Request<{ id: number }>, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const instances = await RunningInstanceModel.findAll({
|
||||
where: {
|
||||
cron_id: req.params.id,
|
||||
},
|
||||
order: [['started_at', 'DESC']],
|
||||
raw: true,
|
||||
const instances = await RunningInstanceModel.find({
|
||||
cron_id: req.params.id,
|
||||
});
|
||||
return res.send({ code: 200, data: instances });
|
||||
} catch (e) {
|
||||
@@ -519,7 +521,11 @@ export default (app: Router) => {
|
||||
instanceId: Joi.number().required(),
|
||||
}),
|
||||
}),
|
||||
async (req: Request<{ id: number; instanceId: number }>, res: Response, next: NextFunction) => {
|
||||
async (
|
||||
req: Request<{ id: number; instanceId: number }>,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) => {
|
||||
try {
|
||||
const cronService = Container.get(CronService);
|
||||
const data = await cronService.stopInstance(req.params.instanceId);
|
||||
|
||||
+93
-157
@@ -1,38 +1,32 @@
|
||||
import { statistics } from '../data/statistics';
|
||||
import { Query, QuerySpec, orderBy } from '../data/queries';
|
||||
import { Router, Request, Response, NextFunction } from 'express';
|
||||
import { Container } from 'typedi';
|
||||
import { fn, col, where, Op } from 'sequelize';
|
||||
import { CrontabModel } from '../data/cron';
|
||||
import { CrontabStatModel } from '../data/cronStats';
|
||||
import {
|
||||
RunningInstanceModel,
|
||||
InstanceStatus,
|
||||
} from '../data/runningInstance';
|
||||
import { RunningInstanceModel, InstanceStatus } from '../data/runningInstance';
|
||||
import dayjs from 'dayjs';
|
||||
import os from 'os';
|
||||
import { isEmpty } from 'lodash';
|
||||
import { t, tf } from '../shared/i18n';
|
||||
|
||||
const route = Router();
|
||||
|
||||
export default (app: Router) => {
|
||||
app.use('/dashboard', route);
|
||||
|
||||
route.post('/record', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const { ref_id, code, elapsed } = req.body;
|
||||
if (!ref_id) return res.send({ code: 400, message: 'ref_id required' });
|
||||
|
||||
const today = dayjs().format('YYYY-MM-DD');
|
||||
const isSuccess = code === 0 ? 1 : 0;
|
||||
const isFail = code !== 0 ? 1 : 0;
|
||||
const elapsedMs = (Number(elapsed) || 0) * 1000;
|
||||
|
||||
const existing = await CrontabStatModel.findOne({
|
||||
where: { ref_id: Number(ref_id), date: today },
|
||||
});
|
||||
|
||||
const existing = await CrontabStatModel.findOne(
|
||||
{ ref_id: Number(ref_id), date: today },
|
||||
{},
|
||||
);
|
||||
if (existing) {
|
||||
await CrontabStatModel.update(
|
||||
await CrontabStatModel.nativeUpdate(
|
||||
{ id: existing.id },
|
||||
{
|
||||
run_count: (existing.run_count || 0) + 1,
|
||||
success_count: (existing.success_count || 0) + isSuccess,
|
||||
@@ -40,10 +34,9 @@ export default (app: Router) => {
|
||||
total_time: (existing.total_time || 0) + elapsedMs,
|
||||
max_time: Math.max(existing.max_time || 0, elapsedMs),
|
||||
},
|
||||
{ where: { id: existing.id } },
|
||||
);
|
||||
} else {
|
||||
await CrontabStatModel.create({
|
||||
await CrontabStatModel.insert({
|
||||
ref_id: Number(ref_id),
|
||||
date: today,
|
||||
run_count: 1,
|
||||
@@ -53,41 +46,27 @@ export default (app: Router) => {
|
||||
max_time: elapsedMs,
|
||||
});
|
||||
}
|
||||
|
||||
res.send({ code: 200 });
|
||||
} catch (e) {
|
||||
res.send({ code: 500 });
|
||||
}
|
||||
});
|
||||
|
||||
route.get(
|
||||
'/overview',
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const today = dayjs().format('YYYY-MM-DD');
|
||||
|
||||
const [total, enabled, disabled, stats] = await Promise.all([
|
||||
CrontabModel.count(),
|
||||
CrontabModel.count({ where: { isDisabled: 0 } }),
|
||||
CrontabModel.count({ where: { isDisabled: 1 } }),
|
||||
CrontabStatModel.findOne({
|
||||
attributes: [
|
||||
[fn('SUM', col('run_count')), 'total_runs'],
|
||||
[fn('SUM', col('success_count')), 'total_success'],
|
||||
[fn('SUM', col('fail_count')), 'total_fail'],
|
||||
[fn('SUM', col('total_time')), 'total_time'],
|
||||
],
|
||||
where: { date: today },
|
||||
raw: true,
|
||||
}),
|
||||
CrontabModel.count({}),
|
||||
CrontabModel.count({ isDisabled: 0 }),
|
||||
CrontabModel.count({ isDisabled: 1 }),
|
||||
statistics.totals(today),
|
||||
]);
|
||||
|
||||
const row = stats as any;
|
||||
const totalRuns = Number(row?.total_runs) || 0;
|
||||
const totalSuccess = Number(row?.total_success) || 0;
|
||||
const totalFail = Number(row?.total_fail) || 0;
|
||||
const totalTime = Number(row?.total_time) || 0;
|
||||
|
||||
res.send({
|
||||
code: 200,
|
||||
data: {
|
||||
@@ -97,7 +76,10 @@ export default (app: Router) => {
|
||||
todayRuns: totalRuns,
|
||||
todaySuccess: totalSuccess,
|
||||
todayFail: totalFail,
|
||||
successRate: totalRuns > 0 ? ((totalSuccess / totalRuns) * 100).toFixed(1) : '0',
|
||||
successRate:
|
||||
totalRuns > 0
|
||||
? ((totalSuccess / totalRuns) * 100).toFixed(1)
|
||||
: '0',
|
||||
avgTime: totalRuns > 0 ? Math.round(totalTime / totalRuns) : 0,
|
||||
},
|
||||
});
|
||||
@@ -106,25 +88,25 @@ export default (app: Router) => {
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
for (const isSuccess of [false, true]) {
|
||||
route.get(
|
||||
isSuccess ? '/successes' : '/failures',
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const countColumn = isSuccess ? 'success_count' : 'fail_count';
|
||||
const rows = (await CrontabStatModel.findAll({
|
||||
attributes: ['ref_id', [fn('SUM', col(countColumn)), 'result_count']],
|
||||
where: { date: dayjs().format('YYYY-MM-DD'), [countColumn]: { [Op.gt]: 0 } },
|
||||
group: ['ref_id'],
|
||||
order: [[fn('SUM', col(countColumn)), 'DESC'], ['ref_id', 'ASC']],
|
||||
raw: true,
|
||||
})) as any[];
|
||||
const crons = rows.length > 0 ? await CrontabModel.findAll({
|
||||
attributes: ['id', 'name', 'command'],
|
||||
where: { id: { [Op.in]: rows.map((row) => Number(row.ref_id)) } },
|
||||
raw: true,
|
||||
}) : [];
|
||||
const rows = (await statistics.results(
|
||||
dayjs().format('YYYY-MM-DD'),
|
||||
countColumn,
|
||||
)) as any[];
|
||||
const crons =
|
||||
rows.length > 0
|
||||
? await CrontabModel.find(
|
||||
{ id: { ['$in']: rows.map((row) => Number(row.ref_id)) } },
|
||||
{
|
||||
fields: ['id', 'name', 'command'],
|
||||
},
|
||||
)
|
||||
: [];
|
||||
const cronMap = new Map(crons.map((cron) => [cron.id, cron]));
|
||||
res.send({
|
||||
code: 200,
|
||||
@@ -135,7 +117,9 @@ export default (app: Router) => {
|
||||
id,
|
||||
name: cron?.name || cron?.command || tf('任务#%s', id),
|
||||
command: cron?.command || '',
|
||||
[isSuccess ? 'successCount' : 'failCount']: Number(row.result_count),
|
||||
[isSuccess ? 'successCount' : 'failCount']: Number(
|
||||
row.result_count,
|
||||
),
|
||||
deleted: !cron,
|
||||
};
|
||||
}),
|
||||
@@ -146,7 +130,6 @@ export default (app: Router) => {
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
route.get(
|
||||
'/trend',
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
@@ -156,22 +139,7 @@ export default (app: Router) => {
|
||||
for (let i = days - 1; i >= 0; i--) {
|
||||
dates.push(dayjs().subtract(i, 'day').format('YYYY-MM-DD'));
|
||||
}
|
||||
|
||||
const rows = (await CrontabStatModel.findAll({
|
||||
attributes: [
|
||||
'date',
|
||||
[fn('SUM', col('run_count')), 'total_runs'],
|
||||
[fn('SUM', col('success_count')), 'total_success'],
|
||||
[fn('SUM', col('fail_count')), 'total_fail'],
|
||||
],
|
||||
where: {
|
||||
date: { [Op.in]: dates },
|
||||
},
|
||||
group: ['date'],
|
||||
order: [['date', 'ASC']],
|
||||
raw: true,
|
||||
})) as any[];
|
||||
|
||||
const rows = (await statistics.trend(dates)) as any[];
|
||||
const dataMap: Record<string, any> = {};
|
||||
rows.forEach((r: any) => {
|
||||
dataMap[r.date] = {
|
||||
@@ -180,89 +148,52 @@ export default (app: Router) => {
|
||||
fail: Number(r.total_fail) || 0,
|
||||
};
|
||||
});
|
||||
|
||||
const data = dates.map((d) => ({
|
||||
date: dayjs(d).format('MM-DD'),
|
||||
...(dataMap[d] || { total: 0, success: 0, fail: 0 }),
|
||||
}));
|
||||
|
||||
res.send({ code: 200, data });
|
||||
} catch (e) {
|
||||
next(e);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
route.get(
|
||||
'/top-time',
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const today = dayjs().format('YYYY-MM-DD');
|
||||
|
||||
const rows = (await CrontabStatModel.findAll({
|
||||
attributes: [
|
||||
'ref_id',
|
||||
[fn('SUM', col('total_time')), 'total_time'],
|
||||
[fn('SUM', col('run_count')), 'run_count'],
|
||||
[fn('MAX', col('max_time')), 'max_time'],
|
||||
],
|
||||
where: { date: today, run_count: { [Op.gt]: 0 } },
|
||||
group: ['ref_id'],
|
||||
order: [[fn('SUM', col('total_time')), 'DESC']],
|
||||
limit: 5,
|
||||
raw: true,
|
||||
})) as any[];
|
||||
|
||||
const rows = (await statistics.ranking(today, 'total_time')) as any[];
|
||||
const ids = rows.map((r) => Number(r.ref_id));
|
||||
const crons = await CrontabModel.findAll({
|
||||
where: { id: { [Op.in]: ids } },
|
||||
raw: true,
|
||||
});
|
||||
const crons = await CrontabModel.find({ id: { ['$in']: ids } }, {});
|
||||
const nameMap: Record<number, string> = {};
|
||||
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; });
|
||||
|
||||
crons.forEach((c: any) => {
|
||||
nameMap[c.id] = c.name || c.command;
|
||||
});
|
||||
const data = rows.map((r: any, i) => ({
|
||||
rank: i + 1,
|
||||
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
|
||||
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
|
||||
maxTime: Number(r.max_time),
|
||||
}));
|
||||
|
||||
res.send({ code: 200, data });
|
||||
} catch (e) {
|
||||
next(e);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
route.get(
|
||||
'/top-count',
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const today = dayjs().format('YYYY-MM-DD');
|
||||
|
||||
const rows = (await CrontabStatModel.findAll({
|
||||
attributes: [
|
||||
'ref_id',
|
||||
[fn('SUM', col('run_count')), 'run_count'],
|
||||
[fn('SUM', col('total_time')), 'total_time'],
|
||||
[fn('SUM', col('success_count')), 'success_count'],
|
||||
],
|
||||
where: { date: today, run_count: { [Op.gt]: 0 } },
|
||||
group: ['ref_id'],
|
||||
order: [[fn('SUM', col('run_count')), 'DESC']],
|
||||
limit: 5,
|
||||
raw: true,
|
||||
})) as any[];
|
||||
|
||||
const rows = (await statistics.ranking(today, 'run_count')) as any[];
|
||||
const ids = rows.map((r) => Number(r.ref_id));
|
||||
const crons = await CrontabModel.findAll({
|
||||
where: { id: { [Op.in]: ids } },
|
||||
raw: true,
|
||||
});
|
||||
const crons = await CrontabModel.find({ id: { ['$in']: ids } }, {});
|
||||
const nameMap: Record<number, string> = {};
|
||||
crons.forEach((c: any) => { nameMap[c.id] = c.name || c.command; });
|
||||
|
||||
crons.forEach((c: any) => {
|
||||
nameMap[c.id] = c.name || c.command;
|
||||
});
|
||||
const data = rows.map((r: any, i) => ({
|
||||
rank: i + 1,
|
||||
name: nameMap[Number(r.ref_id)] || tf('任务#%s', r.ref_id),
|
||||
@@ -270,48 +201,42 @@ export default (app: Router) => {
|
||||
avgTime: Math.round(Number(r.total_time) / Number(r.run_count)),
|
||||
successRate:
|
||||
Number(r.run_count) > 0
|
||||
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(1)
|
||||
? ((Number(r.success_count) / Number(r.run_count)) * 100).toFixed(
|
||||
1,
|
||||
)
|
||||
: '0',
|
||||
}));
|
||||
|
||||
res.send({ code: 200, data });
|
||||
} catch (e) {
|
||||
next(e);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
route.get(
|
||||
'/runtime',
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const runningInstances = await RunningInstanceModel.findAll({
|
||||
where: {
|
||||
const runningInstances = await RunningInstanceModel.find(
|
||||
{
|
||||
status: InstanceStatus.running,
|
||||
},
|
||||
raw: true,
|
||||
});
|
||||
|
||||
const queuedCrons = await CrontabModel.findAll({
|
||||
where: {
|
||||
{},
|
||||
);
|
||||
const queuedCrons = await CrontabModel.find(
|
||||
{
|
||||
status: 3, // queued
|
||||
},
|
||||
raw: true,
|
||||
});
|
||||
|
||||
{},
|
||||
);
|
||||
// Fetch cron names for running instances
|
||||
const cronIds = [
|
||||
...new Set(runningInstances.map((i: any) => i.cron_id)),
|
||||
];
|
||||
const crons =
|
||||
cronIds.length > 0
|
||||
? await CrontabModel.findAll({
|
||||
where: { id: cronIds },
|
||||
raw: true,
|
||||
})
|
||||
? await CrontabModel.find({ id: cronIds }, {})
|
||||
: [];
|
||||
const cronMap = new Map(crons.map((c: any) => [c.id, c]));
|
||||
|
||||
const now = dayjs().unix();
|
||||
const running = runningInstances.map((inst: any) => {
|
||||
const cron = cronMap.get(inst.cron_id);
|
||||
@@ -324,19 +249,18 @@ export default (app: Router) => {
|
||||
logPath: inst.log_path,
|
||||
};
|
||||
});
|
||||
|
||||
const dayAgo = dayjs().subtract(24, 'hour').unix();
|
||||
const idleTasks = await CrontabModel.findAll({
|
||||
where: {
|
||||
const idleTasks = await CrontabModel.find(
|
||||
{
|
||||
isDisabled: 0,
|
||||
status: 1,
|
||||
last_execution_time: { [Op.lt]: dayAgo },
|
||||
last_execution_time: { ['$lt']: dayAgo },
|
||||
},
|
||||
order: [['last_execution_time', 'ASC']],
|
||||
limit: 5,
|
||||
raw: true,
|
||||
});
|
||||
|
||||
{
|
||||
orderBy: orderBy([['last_execution_time', 'ASC']]),
|
||||
limit: 5,
|
||||
},
|
||||
);
|
||||
res.send({
|
||||
code: 200,
|
||||
data: {
|
||||
@@ -357,33 +281,45 @@ export default (app: Router) => {
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
route.get(
|
||||
'/labels',
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const today = dayjs().format('YYYY-MM-DD');
|
||||
const [crons, stats] = (await Promise.all([
|
||||
CrontabModel.findAll({ where: { isDisabled: 0 }, raw: true }),
|
||||
CrontabStatModel.findAll({ where: { date: today }, raw: true }),
|
||||
]));
|
||||
|
||||
const [crons, stats] = await Promise.all([
|
||||
CrontabModel.find({ isDisabled: 0 }, {}),
|
||||
CrontabStatModel.find({ date: today }, {}),
|
||||
]);
|
||||
const statMap: Record<number, any> = {};
|
||||
stats.forEach((s: any) => { statMap[s.ref_id] = s; });
|
||||
|
||||
const labelMap: Record<string, { count: number; runs: number; success: number; totalTime: number }> = {};
|
||||
stats.forEach((s: any) => {
|
||||
statMap[s.ref_id] = s;
|
||||
});
|
||||
const labelMap: Record<
|
||||
string,
|
||||
{
|
||||
count: number;
|
||||
runs: number;
|
||||
success: number;
|
||||
totalTime: number;
|
||||
}
|
||||
> = {};
|
||||
crons.forEach((c) => {
|
||||
let rawLabels = c.labels;
|
||||
if (typeof rawLabels === 'string') rawLabels = JSON.parse(rawLabels);
|
||||
const labels: string[] = Array.isArray(rawLabels)
|
||||
? [...new Set((rawLabels as string[]).filter((l: string) => !isEmpty(l)))]
|
||||
? [
|
||||
...new Set(
|
||||
(rawLabels as string[]).filter((l: string) => !isEmpty(l)),
|
||||
),
|
||||
]
|
||||
: [];
|
||||
if (labels.length === 0) {
|
||||
labels.push(t('未分类'));
|
||||
}
|
||||
const st = statMap[c.id!];
|
||||
labels.forEach((label: string) => {
|
||||
if (!labelMap[label]) labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 };
|
||||
if (!labelMap[label])
|
||||
labelMap[label] = { count: 0, runs: 0, success: 0, totalTime: 0 };
|
||||
labelMap[label].count += 1;
|
||||
if (st) {
|
||||
labelMap[label].runs += Number(st.run_count) || 0;
|
||||
@@ -392,24 +328,22 @@ export default (app: Router) => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const data = Object.entries(labelMap)
|
||||
.map(([label, v]) => ({
|
||||
label,
|
||||
count: v.count,
|
||||
todayRuns: v.runs,
|
||||
successRate: v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
|
||||
successRate:
|
||||
v.runs > 0 ? ((v.success / v.runs) * 100).toFixed(1) : '0',
|
||||
avgTime: v.runs > 0 ? Math.round(v.totalTime / v.runs) : 0,
|
||||
}))
|
||||
.sort((a, b) => b.todayRuns - a.todayRuns);
|
||||
|
||||
res.send({ code: 200, data });
|
||||
} catch (e) {
|
||||
next(e);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
route.get(
|
||||
'/system',
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
@@ -422,7 +356,9 @@ export default (app: Router) => {
|
||||
uptime: Math.floor(process.uptime()),
|
||||
memTotal: os.totalmem(),
|
||||
memFree: os.freemem(),
|
||||
memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(1),
|
||||
memUsagePercent: ((1 - os.freemem() / os.totalmem()) * 100).toFixed(
|
||||
1,
|
||||
),
|
||||
heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024),
|
||||
heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024),
|
||||
loadAvg: os.loadavg().map((v) => Number(v.toFixed(2))),
|
||||
|
||||
@@ -1,19 +1,16 @@
|
||||
import { errStack } from '../shared/errors';
|
||||
|
||||
// Also terminate if the primary disappears without forwarding a signal.
|
||||
const onParentDisconnect = () => process.exit(1);
|
||||
process.once('disconnect', onParentDisconnect);
|
||||
|
||||
async function initializeDatabase() {
|
||||
const { sequelize } = await import('../data');
|
||||
const { database } = await import('../data');
|
||||
try {
|
||||
const { default: loadDatabase } = await import('../loaders/db');
|
||||
await loadDatabase();
|
||||
} finally {
|
||||
await sequelize.close();
|
||||
await database.close();
|
||||
}
|
||||
}
|
||||
|
||||
initializeDatabase()
|
||||
.catch((error) => {
|
||||
console.error(`[boot] Database initialization failed:\n${errStack(error)}`);
|
||||
|
||||
@@ -3,11 +3,12 @@
|
||||
import 'reflect-metadata';
|
||||
import dns from 'dns';
|
||||
import http from 'http';
|
||||
|
||||
// The first scheduler connection can outlive its short RPC deadline. Let the
|
||||
// normal single-flight recovery finish within the supervised startup budget.
|
||||
async function waitForUpgradeScheduler(
|
||||
readiness: { recover(): Promise<boolean> },
|
||||
readiness: {
|
||||
recover(): Promise<boolean>;
|
||||
},
|
||||
timeoutMs: number,
|
||||
retryMs = 250,
|
||||
) {
|
||||
@@ -22,13 +23,12 @@ async function waitForUpgradeScheduler(
|
||||
} while (Date.now() < deadline);
|
||||
throw new Error('Upgrade scheduler readiness failed');
|
||||
}
|
||||
|
||||
async function probeUpgrade() {
|
||||
if (!process.send || process.env.QL_UPGRADE_PROBE !== '1')
|
||||
throw new Error('Upgrade probe requires its isolated supervisor');
|
||||
dns.setDefaultResultOrder('ipv4first');
|
||||
const { default: config } = await import('../config');
|
||||
const { sequelize } = await import('../data');
|
||||
const { database } = await import('../data');
|
||||
const { default: loadDatabase } = await import('../loaders/db');
|
||||
const { Container } = await import('typedi');
|
||||
const { GrpcServerService } = await import('../services/grpc');
|
||||
@@ -43,17 +43,17 @@ async function probeUpgrade() {
|
||||
try {
|
||||
await loadDatabase();
|
||||
// Exercise an actual on-disk transaction as well as schema initialization.
|
||||
await sequelize.transaction(async (transaction) => {
|
||||
await sequelize.query('CREATE TABLE upgrade_probe (value INTEGER)', {
|
||||
transaction,
|
||||
});
|
||||
await sequelize.query('INSERT INTO upgrade_probe VALUES (42)', {
|
||||
transaction,
|
||||
});
|
||||
const [rows] = await sequelize.query('SELECT value FROM upgrade_probe', {
|
||||
transaction,
|
||||
});
|
||||
if ((rows[0] as { value: number })?.value !== 42)
|
||||
await database.transaction(async () => {
|
||||
await database.query('CREATE TABLE upgrade_probe (value INTEGER)');
|
||||
await database.query('INSERT INTO upgrade_probe VALUES (42)');
|
||||
const rows = await database.query('SELECT value FROM upgrade_probe');
|
||||
if (
|
||||
(
|
||||
rows[0] as {
|
||||
value: number;
|
||||
}
|
||||
)?.value !== 42
|
||||
)
|
||||
throw new Error('Upgrade database readback failed');
|
||||
});
|
||||
inject();
|
||||
@@ -64,8 +64,11 @@ async function probeUpgrade() {
|
||||
cronClient.readiness,
|
||||
process.arch === 'arm' &&
|
||||
Number(
|
||||
(process.config.variables as { arm_version?: string | number })
|
||||
.arm_version,
|
||||
(
|
||||
process.config.variables as {
|
||||
arm_version?: string | number;
|
||||
}
|
||||
).arm_version,
|
||||
) === 6
|
||||
? 60000
|
||||
: 10000,
|
||||
@@ -119,10 +122,9 @@ async function probeUpgrade() {
|
||||
} finally {
|
||||
await server.shutdown();
|
||||
grpc.getServer().forceShutdown();
|
||||
await sequelize.close();
|
||||
await database.close();
|
||||
}
|
||||
}
|
||||
|
||||
process.once('disconnect', () => process.exit(1));
|
||||
probeUpgrade()
|
||||
.then(() => process.exit(0))
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import { database } from '.';
|
||||
import { DatabaseClient } from './databaseClient';
|
||||
|
||||
// Keep the existing Keyv serialization and namespace. Only its SQLite transport
|
||||
// changes; old cached authentication records remain readable in either version.
|
||||
export class CacheStore extends EventEmitter {
|
||||
readonly opts = { dialect: 'sqlite' };
|
||||
namespace?: string;
|
||||
constructor(private client: () => DatabaseClient = () => database) {
|
||||
super();
|
||||
}
|
||||
get(key: string): Promise<string | undefined> {
|
||||
return this.client().request('cache', 'get', key);
|
||||
}
|
||||
async set(key: string, value: string) {
|
||||
return this.client().request('cache', 'set', key, value);
|
||||
}
|
||||
async delete(key: string): Promise<boolean> {
|
||||
return this.client().request('cache', 'delete', key);
|
||||
}
|
||||
async clear(): Promise<void> {
|
||||
throw new Error('Clearing all authentication namespaces is not supported');
|
||||
}
|
||||
async disconnect(): Promise<void> {
|
||||
/* The shared database client owns the worker. */
|
||||
}
|
||||
}
|
||||
+3
-36
@@ -1,5 +1,4 @@
|
||||
import { sequelize } from '.';
|
||||
import { DataTypes, Model, ModelDefined } from 'sequelize';
|
||||
import { Repository, RecordRow } from './repository';
|
||||
|
||||
export class Crontab {
|
||||
name?: string;
|
||||
@@ -62,37 +61,5 @@ export enum CrontabStatus {
|
||||
'queued' = 3,
|
||||
}
|
||||
|
||||
export interface CronInstance extends Model<Crontab, Crontab>, Crontab {}
|
||||
export const CrontabModel = sequelize.define<CronInstance>('Crontab', {
|
||||
name: {
|
||||
unique: 'compositeIndex',
|
||||
type: DataTypes.STRING,
|
||||
},
|
||||
command: {
|
||||
unique: 'compositeIndex',
|
||||
type: DataTypes.STRING,
|
||||
},
|
||||
schedule: {
|
||||
unique: 'compositeIndex',
|
||||
type: DataTypes.STRING,
|
||||
},
|
||||
timestamp: DataTypes.STRING,
|
||||
saved: DataTypes.BOOLEAN,
|
||||
status: DataTypes.NUMBER,
|
||||
isSystem: DataTypes.NUMBER,
|
||||
pid: DataTypes.NUMBER,
|
||||
isDisabled: DataTypes.NUMBER,
|
||||
isPinned: DataTypes.NUMBER,
|
||||
log_path: DataTypes.STRING,
|
||||
queued_token: DataTypes.STRING,
|
||||
labels: DataTypes.JSON,
|
||||
last_running_time: DataTypes.NUMBER,
|
||||
last_execution_time: DataTypes.NUMBER,
|
||||
sub_id: { type: DataTypes.NUMBER, allowNull: true },
|
||||
extra_schedules: DataTypes.JSON,
|
||||
task_before: DataTypes.STRING,
|
||||
task_after: DataTypes.STRING,
|
||||
log_name: DataTypes.STRING,
|
||||
allow_multiple_instances: DataTypes.NUMBER,
|
||||
work_dir: DataTypes.STRING,
|
||||
});
|
||||
export type CronInstance = RecordRow<Crontab>;
|
||||
export const CrontabModel = new Repository<Crontab>('Crontab');
|
||||
|
||||
+3
-48
@@ -1,5 +1,4 @@
|
||||
import { DataTypes, Model } from 'sequelize';
|
||||
import { sequelize } from '.';
|
||||
import { Repository, RecordRow } from './repository';
|
||||
|
||||
export class CrontabStat {
|
||||
id?: number;
|
||||
@@ -23,49 +22,5 @@ export class CrontabStat {
|
||||
}
|
||||
}
|
||||
|
||||
export interface CrontabStatInstance extends Model<CrontabStat, CrontabStat>, CrontabStat {}
|
||||
|
||||
export const CrontabStatModel = sequelize.define<CrontabStatInstance>(
|
||||
'CrontabStat',
|
||||
{
|
||||
id: {
|
||||
type: DataTypes.INTEGER,
|
||||
primaryKey: true,
|
||||
autoIncrement: true,
|
||||
},
|
||||
ref_id: {
|
||||
type: DataTypes.NUMBER,
|
||||
allowNull: false,
|
||||
},
|
||||
date: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: false,
|
||||
},
|
||||
run_count: {
|
||||
type: DataTypes.NUMBER,
|
||||
defaultValue: 0,
|
||||
},
|
||||
success_count: {
|
||||
type: DataTypes.NUMBER,
|
||||
defaultValue: 0,
|
||||
},
|
||||
fail_count: {
|
||||
type: DataTypes.NUMBER,
|
||||
defaultValue: 0,
|
||||
},
|
||||
total_time: {
|
||||
type: DataTypes.NUMBER,
|
||||
defaultValue: 0,
|
||||
},
|
||||
max_time: {
|
||||
type: DataTypes.NUMBER,
|
||||
defaultValue: 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
indexes: [
|
||||
{ unique: true, fields: ['ref_id', 'date'] },
|
||||
{ fields: ['date'] },
|
||||
],
|
||||
},
|
||||
);
|
||||
export type CrontabStatInstance = RecordRow<CrontabStat>;
|
||||
export const CrontabStatModel = new Repository<CrontabStat>('CrontabStat');
|
||||
|
||||
+3
-23
@@ -1,5 +1,4 @@
|
||||
import { sequelize } from '.';
|
||||
import { DataTypes, Model } from 'sequelize';
|
||||
import { Repository, RecordRow } from './repository';
|
||||
|
||||
export enum CronViewType {
|
||||
'系统' = 1,
|
||||
@@ -39,24 +38,5 @@ export class CrontabView {
|
||||
}
|
||||
}
|
||||
|
||||
export interface CronViewInstance
|
||||
extends Model<CrontabView, CrontabView>,
|
||||
CrontabView {}
|
||||
export const CrontabViewModel = sequelize.define<CronViewInstance>(
|
||||
'CrontabView',
|
||||
{
|
||||
name: {
|
||||
unique: 'name',
|
||||
type: DataTypes.STRING,
|
||||
},
|
||||
position: DataTypes.NUMBER,
|
||||
isDisabled: DataTypes.NUMBER,
|
||||
filters: DataTypes.JSON,
|
||||
sorts: DataTypes.JSON,
|
||||
filterRelation: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
},
|
||||
type: DataTypes.NUMBER,
|
||||
},
|
||||
);
|
||||
export type CronViewInstance = RecordRow<CrontabView>;
|
||||
export const CrontabViewModel = new Repository<CrontabView>('CrontabView');
|
||||
|
||||
@@ -0,0 +1,325 @@
|
||||
import { AsyncLocalStorage } from 'async_hooks';
|
||||
import { ChildProcess, fork } from 'child_process';
|
||||
import { join } from 'path';
|
||||
import { setTimeout, clearTimeout } from 'timers';
|
||||
import { DatabaseError } from './databaseErrors';
|
||||
|
||||
type Scope = { active: boolean; child: boolean; token: number };
|
||||
type Reply = {
|
||||
id: number;
|
||||
value?: any;
|
||||
error?: { code: string; message: string };
|
||||
};
|
||||
type Limits = { timeoutMs?: number; queueLimit?: number };
|
||||
|
||||
// One worker per backend process; SQLite itself coordinates independent processes.
|
||||
// A transaction owns this client's queue until its callback and commit finish.
|
||||
export class DatabaseClient {
|
||||
private worker?: ChildProcess;
|
||||
private stopped: Promise<void> = Promise.resolve();
|
||||
private failed?: Error;
|
||||
private closed = false;
|
||||
private nextId = 0;
|
||||
private queued = 0;
|
||||
private tail: Promise<void> = Promise.resolve();
|
||||
private pending = new Map<
|
||||
number,
|
||||
{
|
||||
resolve(value: any): void;
|
||||
reject(error: Error): void;
|
||||
timer: NodeJS.Timeout;
|
||||
}
|
||||
>();
|
||||
private scope = new AsyncLocalStorage<Scope>();
|
||||
|
||||
private readonly timeoutMs: number;
|
||||
private readonly queueLimit: number;
|
||||
|
||||
constructor(
|
||||
public readonly storage: string,
|
||||
private cacheStorage?: string,
|
||||
limits: Limits = {},
|
||||
) {
|
||||
this.timeoutMs =
|
||||
limits.timeoutMs ?? (process.arch === 'arm' ? 300000 : 60000);
|
||||
this.queueLimit = limits.queueLimit ?? 256;
|
||||
if (
|
||||
!Number.isSafeInteger(this.timeoutMs) ||
|
||||
this.timeoutMs < 1 ||
|
||||
!Number.isSafeInteger(this.queueLimit) ||
|
||||
this.queueLimit < 1
|
||||
)
|
||||
throw new Error('Invalid database worker limits');
|
||||
}
|
||||
|
||||
private abort(error: Error) {
|
||||
this.failed ||= error;
|
||||
for (const request of this.pending.values()) {
|
||||
clearTimeout(request.timer);
|
||||
request.reject(this.failed);
|
||||
}
|
||||
this.pending.clear();
|
||||
this.unref();
|
||||
this.worker?.kill('SIGKILL');
|
||||
}
|
||||
|
||||
private unref() {
|
||||
this.worker?.unref();
|
||||
this.worker?.channel?.unref();
|
||||
}
|
||||
|
||||
private start() {
|
||||
if (this.worker) return this.worker;
|
||||
const [major, minor] = process.versions.node.split('.').map(Number);
|
||||
if (major < 22 || (major === 22 && minor < 17))
|
||||
throw new DatabaseError(
|
||||
'Qinglong requires Node >=22.17; keep the previous installation until Node is upgraded',
|
||||
'UNSUPPORTED_NODE',
|
||||
);
|
||||
// ts-node's CLI places its executable in execArgv, rather than using -r.
|
||||
// Source workers register their own compiler; compiled workers need none.
|
||||
const preloads = __filename.endsWith('.ts')
|
||||
? ['--require', require.resolve('ts-node/register/transpile-only')]
|
||||
: [];
|
||||
const armv6 =
|
||||
process.arch === 'arm' &&
|
||||
Number(
|
||||
(process.config.variables as { arm_version?: string | number })
|
||||
.arm_version,
|
||||
) === 6;
|
||||
const worker = fork(require.resolve('./databaseProcess'), [], {
|
||||
serialization: 'advanced',
|
||||
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
||||
execArgv: [
|
||||
...preloads,
|
||||
...(armv6 ? ['--jitless'] : []),
|
||||
'--max-old-space-size=128',
|
||||
],
|
||||
env: {
|
||||
...process.env,
|
||||
NODE_OPTIONS: '',
|
||||
...(__filename.endsWith('.ts')
|
||||
? { TS_NODE_PROJECT: join(__dirname, 'tsconfig.worker.json') }
|
||||
: {}),
|
||||
},
|
||||
});
|
||||
this.stopped = new Promise((resolve) =>
|
||||
worker.once('close', () => resolve()),
|
||||
);
|
||||
const onOwnerExit = () => worker.kill('SIGKILL');
|
||||
process.once('exit', onOwnerExit);
|
||||
worker.once('close', () => process.removeListener('exit', onOwnerExit));
|
||||
this.worker = worker;
|
||||
worker.on('message', (reply: Reply & { type?: string }) => {
|
||||
if (reply.type === 'fatal' && reply.error) {
|
||||
this.abort(new DatabaseError(reply.error.message, reply.error.code));
|
||||
return;
|
||||
}
|
||||
const request = this.pending.get(reply.id);
|
||||
if (!request) return;
|
||||
this.pending.delete(reply.id);
|
||||
clearTimeout(request.timer);
|
||||
if (!this.pending.size) this.unref();
|
||||
if (reply.error)
|
||||
request.reject(
|
||||
new DatabaseError(reply.error.message, reply.error.code),
|
||||
);
|
||||
else request.resolve(reply.value);
|
||||
});
|
||||
const fail = () =>
|
||||
this.abort(
|
||||
new DatabaseError(
|
||||
'Database worker stopped; restart the service before retrying a write',
|
||||
'DATABASE_WORKER_EXIT',
|
||||
),
|
||||
);
|
||||
worker.on('error', fail);
|
||||
worker.on('exit', () => {
|
||||
if (!this.closed) fail();
|
||||
});
|
||||
worker.send({
|
||||
storage: this.storage,
|
||||
cacheStorage: this.cacheStorage,
|
||||
ownerPid: process.pid,
|
||||
});
|
||||
this.unref();
|
||||
return worker;
|
||||
}
|
||||
|
||||
private send(
|
||||
operation: string,
|
||||
args: any[] = [],
|
||||
token?: number,
|
||||
): Promise<any> {
|
||||
if (this.failed) return Promise.reject(this.failed);
|
||||
if (this.closed)
|
||||
return Promise.reject(
|
||||
new DatabaseError('Database is closed', 'DATABASE_CLOSED'),
|
||||
);
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
const worker = this.start();
|
||||
const id = ++this.nextId;
|
||||
const timer = setTimeout(
|
||||
() =>
|
||||
this.abort(
|
||||
new DatabaseError(
|
||||
'Database worker timed out; writes are never automatically replayed',
|
||||
'DATABASE_WORKER_TIMEOUT',
|
||||
),
|
||||
),
|
||||
this.timeoutMs,
|
||||
);
|
||||
timer.unref();
|
||||
this.pending.set(id, { resolve, reject, timer });
|
||||
worker.ref();
|
||||
worker.channel?.ref();
|
||||
try {
|
||||
worker.send({ id, operation, args, token }, (error) => {
|
||||
if (error)
|
||||
this.abort(
|
||||
new DatabaseError(
|
||||
'Database IPC disconnected',
|
||||
'DATABASE_WORKER_EXIT',
|
||||
),
|
||||
);
|
||||
});
|
||||
} catch (error) {
|
||||
this.pending.delete(id);
|
||||
clearTimeout(timer);
|
||||
if (!this.pending.size) this.unref();
|
||||
reject(error);
|
||||
}
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private exclusive<T>(
|
||||
operation: () => Promise<T>,
|
||||
shutdown = false,
|
||||
): Promise<T> {
|
||||
if (!shutdown && this.queued >= this.queueLimit)
|
||||
return Promise.reject(
|
||||
new DatabaseError('Database request queue is full', 'DATABASE_BUSY'),
|
||||
);
|
||||
this.queued++;
|
||||
const previous = this.tail;
|
||||
let release!: () => void;
|
||||
this.tail = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
return previous.then(operation).finally(() => {
|
||||
this.queued--;
|
||||
release();
|
||||
});
|
||||
}
|
||||
|
||||
request<T = any>(operation: string, ...args: any[]): Promise<T> {
|
||||
const scope = this.scope.getStore();
|
||||
if (scope) {
|
||||
if (!scope.active || scope.child)
|
||||
return Promise.reject(
|
||||
new DatabaseError(
|
||||
'Transaction scope is no longer available',
|
||||
'TRANSACTION_SCOPE',
|
||||
),
|
||||
);
|
||||
if (this.pending.size >= this.queueLimit)
|
||||
return Promise.reject(
|
||||
new DatabaseError('Database request queue is full', 'DATABASE_BUSY'),
|
||||
);
|
||||
return this.send(operation, args, scope.token);
|
||||
}
|
||||
return this.exclusive(() => this.send(operation, args));
|
||||
}
|
||||
|
||||
async transaction<T>(operation: () => Promise<T>): Promise<T> {
|
||||
const parent = this.scope.getStore();
|
||||
if (parent && (!parent.active || parent.child))
|
||||
throw new DatabaseError(
|
||||
'Concurrent or expired nested transaction',
|
||||
'TRANSACTION_SCOPE',
|
||||
);
|
||||
const run = async () => {
|
||||
const token = ++this.nextId;
|
||||
if (parent) parent.child = true;
|
||||
const scope: Scope = { active: true, child: false, token };
|
||||
let began = false;
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
await this.send('begin', [], token);
|
||||
began = true;
|
||||
const deadline = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(
|
||||
() =>
|
||||
reject(
|
||||
new DatabaseError(
|
||||
'Database transaction timed out',
|
||||
'DATABASE_TRANSACTION_TIMEOUT',
|
||||
),
|
||||
),
|
||||
this.timeoutMs,
|
||||
);
|
||||
});
|
||||
const value = await Promise.race([
|
||||
this.scope.run(scope, operation),
|
||||
deadline,
|
||||
]);
|
||||
scope.active = false;
|
||||
await this.send('commit', [], token);
|
||||
return value;
|
||||
} catch (error) {
|
||||
scope.active = false;
|
||||
if (began) {
|
||||
try {
|
||||
await this.send('rollback', [], token);
|
||||
} catch {
|
||||
this.abort(
|
||||
new DatabaseError(
|
||||
'Transaction rollback failed; restart the service',
|
||||
'DATABASE_ROLLBACK_FAILED',
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
if (parent) parent.child = false;
|
||||
}
|
||||
};
|
||||
return parent ? run() : this.exclusive(run);
|
||||
}
|
||||
|
||||
query<T = Record<string, unknown>>(
|
||||
sql: string,
|
||||
params: unknown[] = [],
|
||||
): Promise<T[]> {
|
||||
return this.request('query', sql, params);
|
||||
}
|
||||
|
||||
get healthy() {
|
||||
return !this.failed && !this.closed;
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.scope.getStore())
|
||||
throw new DatabaseError(
|
||||
'Cannot close within a transaction',
|
||||
'TRANSACTION_SCOPE',
|
||||
);
|
||||
await this.exclusive(async () => {
|
||||
if (this.closed) return;
|
||||
try {
|
||||
if (this.worker && !this.failed) await this.send('close');
|
||||
} finally {
|
||||
this.closed = true;
|
||||
this.worker?.ref();
|
||||
this.worker?.kill('SIGKILL');
|
||||
await this.stopped;
|
||||
}
|
||||
}, true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
// Error details (SQL, bindings, credentials) stay inside the database worker.
|
||||
export class DatabaseError extends Error {
|
||||
constructor(message: string, public readonly code: string) {
|
||||
super(message);
|
||||
this.name = 'DatabaseError';
|
||||
}
|
||||
}
|
||||
|
||||
export function databaseStartupFailure(value: unknown) {
|
||||
const error = (value && typeof value === 'object' ? value : {}) as Error & { code?: string };
|
||||
const missing = /Cannot find (?:module|package) ['"]([^'"]+)['"]/.exec(
|
||||
error.message || '',
|
||||
)?.[1];
|
||||
const importer = /Require stack:\r?\n-\s+([^\r\n]+)/.exec(
|
||||
error.message || '',
|
||||
)?.[1];
|
||||
const dependency =
|
||||
/MODULE_NOT_FOUND/.test(error.code || '') &&
|
||||
Boolean(
|
||||
missing &&
|
||||
((!missing.startsWith('.') &&
|
||||
!missing.startsWith('/') &&
|
||||
!missing.startsWith('node:')) ||
|
||||
/node_modules[/\\]/.test(missing) ||
|
||||
/node_modules[/\\]/.test(importer || '')),
|
||||
);
|
||||
return dependency
|
||||
? {
|
||||
code: 'MODULE_NOT_FOUND',
|
||||
message: "Cannot find module '@mikro-orm/sql'",
|
||||
}
|
||||
: {
|
||||
code: 'DATABASE_WORKER_EXIT',
|
||||
message: 'Database process failed to start',
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { databaseStartupFailure } from './databaseErrors';
|
||||
import { Worker } from 'worker_threads';
|
||||
|
||||
// IPC disconnect cannot run while DatabaseSync is executing SQL. Start this
|
||||
// independent, allocation-light monitor before opening SQLite so a SIGKILLed
|
||||
// backend cannot leave an orphan holding the old service's database write lock.
|
||||
async function monitorOwner(ownerPid: number) {
|
||||
if (!Number.isSafeInteger(ownerPid) || ownerPid < 1)
|
||||
throw new Error('Invalid database owner');
|
||||
const monitor = new Worker(
|
||||
`
|
||||
const {parentPort,workerData}=require('node:worker_threads');
|
||||
process.on('uncaughtException',()=>process.kill(process.pid,'SIGKILL'));
|
||||
const check=()=>{
|
||||
if(process.ppid!==workerData.ownerPid)process.kill(process.pid,'SIGKILL');
|
||||
};
|
||||
check();
|
||||
setInterval(check,500);
|
||||
parentPort.postMessage('ready');
|
||||
`,
|
||||
{
|
||||
eval: true,
|
||||
workerData: { ownerPid },
|
||||
// V8's process-wide --jitless remains active on ARMv6. Do not load the
|
||||
// source worker's ts-node compiler into this tiny, plain-JavaScript monitor.
|
||||
execArgv: [],
|
||||
resourceLimits: {
|
||||
maxOldGenerationSizeMb: 8,
|
||||
maxYoungGenerationSizeMb: 2,
|
||||
stackSizeMb: 1,
|
||||
},
|
||||
},
|
||||
);
|
||||
monitor.once('exit', () => process.exit(1));
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
monitor.once('message', () => resolve());
|
||||
monitor.once('error', reject);
|
||||
});
|
||||
monitor.unref();
|
||||
}
|
||||
|
||||
// The bootstrap catches dependency import errors before the ORM module loads.
|
||||
// The advanced IPC serializer preserves Date, Buffer and undefined values.
|
||||
process.once('disconnect', () => process.exit(1));
|
||||
process.once('message', async (settings) => {
|
||||
const pending: unknown[] = [];
|
||||
const buffer = (request: unknown) => pending.push(request);
|
||||
// Keep a listener throughout asynchronous monitor startup. Node may flush
|
||||
// several buffered IPC messages while delivering the initial settings.
|
||||
process.on('message', buffer);
|
||||
try {
|
||||
await monitorOwner((settings as { ownerPid: number }).ownerPid);
|
||||
require('./databaseWorker').startDatabaseWorker(settings);
|
||||
process.removeListener('message', buffer);
|
||||
for (const request of pending) process.emit('message', request, undefined);
|
||||
} catch (error) {
|
||||
process.send?.({ type: 'fatal', error: databaseStartupFailure(error) });
|
||||
} finally {
|
||||
process.removeListener('message', buffer);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,280 @@
|
||||
import { mkdirSync } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
import { EntitySchema, Type } from '@mikro-orm/core';
|
||||
import { MikroORM, SqliteDriver, NodeSqliteDialect, raw } from '@mikro-orm/sql';
|
||||
import { entityDefinitions } from './schema';
|
||||
|
||||
// Sequelize DATE readers expect TEXT. Numeric epoch dates would break rollback.
|
||||
class LegacyDateType extends Type<Date, string> {
|
||||
convertToDatabaseValue(value: Date) {
|
||||
return value == null
|
||||
? value
|
||||
: value.toISOString().replace('T', ' ').replace('Z', ' +00:00');
|
||||
}
|
||||
convertToJSValue(value: string | Date) {
|
||||
return value == null
|
||||
? value
|
||||
: value instanceof Date
|
||||
? value
|
||||
: new Date(value);
|
||||
}
|
||||
getColumnType() {
|
||||
return 'DATETIME';
|
||||
}
|
||||
compareAsType() {
|
||||
return 'string' as const;
|
||||
}
|
||||
get runtimeType() {
|
||||
return 'Date';
|
||||
}
|
||||
}
|
||||
|
||||
const entities = Object.values(entityDefinitions).map(
|
||||
(definition) =>
|
||||
new EntitySchema({
|
||||
...definition,
|
||||
properties: Object.fromEntries(
|
||||
Object.entries(definition.properties).map(([name, field]) => [
|
||||
name,
|
||||
{
|
||||
...field,
|
||||
type:
|
||||
field.type === 'legacyDate' ? new LegacyDateType() : field.type,
|
||||
},
|
||||
]),
|
||||
),
|
||||
} as any),
|
||||
);
|
||||
let orm: Awaited<ReturnType<typeof MikroORM.init<SqliteDriver>>>;
|
||||
let cache: any;
|
||||
const transactions: number[] = [];
|
||||
let ready: Promise<void>;
|
||||
let workerData: { storage: string; cacheStorage?: string; ownerPid: number };
|
||||
|
||||
async function initialize() {
|
||||
if (workerData.storage !== ':memory:')
|
||||
mkdirSync(dirname(workerData.storage), { recursive: true });
|
||||
orm = await MikroORM.init<SqliteDriver>({
|
||||
driver: SqliteDriver,
|
||||
driverOptions: new NodeSqliteDialect(workerData.storage),
|
||||
dbName: workerData.storage,
|
||||
entities,
|
||||
debug: false,
|
||||
allowGlobalContext: false,
|
||||
discovery: { warnWhenNoEntities: false },
|
||||
});
|
||||
await orm.em.getConnection().execute('PRAGMA busy_timeout=10000');
|
||||
}
|
||||
|
||||
function filter(name: keyof typeof entityDefinitions, value: any): any {
|
||||
if (value === undefined) throw new Error('Undefined database predicate');
|
||||
if (value == null || typeof value !== 'object' || value instanceof Date)
|
||||
return value;
|
||||
if (Array.isArray(value)) return value.map((part) => filter(name, part));
|
||||
const properties = entityDefinitions[name].properties;
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, item]: [string, any]) => {
|
||||
if (item === undefined) throw new Error('Undefined database predicate');
|
||||
// LIKE searches JSON's serialized text, matching the pre-migration behavior.
|
||||
if (
|
||||
key in properties &&
|
||||
(properties as any)[key].type === 'json' &&
|
||||
item &&
|
||||
typeof item === 'object' &&
|
||||
'$like' in item
|
||||
)
|
||||
return [raw(`cast("${key}" as text)`), item];
|
||||
return [key, filter(name, item)];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function values(
|
||||
name: keyof typeof entityDefinitions,
|
||||
payload: Record<string, any>,
|
||||
insert = false,
|
||||
) {
|
||||
const properties = entityDefinitions[name].properties;
|
||||
const data = Object.fromEntries(
|
||||
Object.entries(payload).filter(
|
||||
([key, value]) => key in properties && value !== undefined,
|
||||
),
|
||||
);
|
||||
for (const [key, value] of Object.entries(data)) {
|
||||
if ((properties as any)[key].type === 'json' && value !== null)
|
||||
data[key] = JSON.parse(JSON.stringify(value));
|
||||
}
|
||||
const now = new Date();
|
||||
if (insert && !data.createdAt) data.createdAt = now;
|
||||
data.updatedAt = now;
|
||||
return data;
|
||||
}
|
||||
|
||||
function plain(name: keyof typeof entityDefinitions, row: any) {
|
||||
if (!row) return null;
|
||||
return Object.fromEntries(
|
||||
Object.keys(entityDefinitions[name].properties)
|
||||
.filter((key) => row[key] !== undefined)
|
||||
.map((key) => [key, row[key]]),
|
||||
);
|
||||
}
|
||||
|
||||
async function execute(
|
||||
operation: string,
|
||||
args: any[],
|
||||
token?: number,
|
||||
): Promise<any> {
|
||||
await ready;
|
||||
// Reject queued work after owner death; the monitor interrupts running SQL.
|
||||
if (process.ppid !== workerData.ownerPid) process.exit(1);
|
||||
const connection = orm.em.getConnection();
|
||||
if (operation === 'begin') {
|
||||
await connection.execute(
|
||||
transactions.length ? `SAVEPOINT "ql_${token}"` : 'BEGIN IMMEDIATE',
|
||||
);
|
||||
transactions.push(token!);
|
||||
return;
|
||||
}
|
||||
if (token !== transactions[transactions.length - 1])
|
||||
throw new Error('Database transaction ownership mismatch');
|
||||
if (operation === 'commit' || operation === 'rollback') {
|
||||
if (!transactions.length) throw new Error('No active database transaction');
|
||||
const nested = transactions.length > 1;
|
||||
if (operation === 'rollback' && nested)
|
||||
await connection.execute(`ROLLBACK TO SAVEPOINT "ql_${token}"`);
|
||||
await connection.execute(
|
||||
nested ? `RELEASE SAVEPOINT "ql_${token}"` : operation.toUpperCase(),
|
||||
);
|
||||
transactions.pop();
|
||||
return;
|
||||
}
|
||||
if (operation === 'close') {
|
||||
if (transactions.length)
|
||||
throw new Error('Cannot close an active transaction');
|
||||
cache?.close();
|
||||
await orm.close();
|
||||
return;
|
||||
}
|
||||
if (operation === 'query') return connection.execute(args[0], args[1]);
|
||||
if (operation === 'cache') {
|
||||
if (!workerData.cacheStorage)
|
||||
throw new Error('No cache database configured');
|
||||
if (!cache) {
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
mkdirSync(dirname(workerData.cacheStorage), { recursive: true });
|
||||
cache = new DatabaseSync(workerData.cacheStorage);
|
||||
cache.exec(
|
||||
'PRAGMA busy_timeout=10000;CREATE TABLE IF NOT EXISTS keyv(key VARCHAR(255) PRIMARY KEY,value TEXT)',
|
||||
);
|
||||
}
|
||||
const [action, key, value] = args;
|
||||
switch (action) {
|
||||
case 'get':
|
||||
return cache.prepare('SELECT value FROM keyv WHERE key=?').get(key)
|
||||
?.value;
|
||||
case 'set':
|
||||
cache
|
||||
.prepare(
|
||||
'INSERT INTO keyv(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',
|
||||
)
|
||||
.run(key, value);
|
||||
return true;
|
||||
case 'delete':
|
||||
return (
|
||||
cache.prepare('DELETE FROM keyv WHERE key=?').run(key).changes > 0
|
||||
);
|
||||
default:
|
||||
throw new Error('Unsupported cache operation');
|
||||
}
|
||||
}
|
||||
const [name, condition, options = {}] = args as [
|
||||
keyof typeof entityDefinitions,
|
||||
any,
|
||||
any,
|
||||
];
|
||||
if (!(name in entityDefinitions)) throw new Error('Unknown entity');
|
||||
const em = orm.em.fork();
|
||||
const entityName = entities.find((entity) => entity.meta.className === name)!;
|
||||
if (operation !== 'insert' && condition == null)
|
||||
throw new Error('Missing database predicate');
|
||||
const query = operation === 'insert' ? {} : filter(name, condition);
|
||||
const readOptions = {
|
||||
...options,
|
||||
orderBy: options.orderBy?.map((order: Record<string, string>) =>
|
||||
Object.fromEntries(
|
||||
Object.entries(order).map(([field, direction]) => [
|
||||
name === 'Env' && field === 'isPinned'
|
||||
? raw('COALESCE("isPinned",0)')
|
||||
: field,
|
||||
direction,
|
||||
]),
|
||||
),
|
||||
),
|
||||
};
|
||||
switch (operation) {
|
||||
case 'find':
|
||||
return (await em.find(entityName, query, readOptions)).map((row) =>
|
||||
plain(name, row),
|
||||
);
|
||||
case 'findOne':
|
||||
return plain(name, await em.findOne(entityName, query, readOptions));
|
||||
case 'count':
|
||||
return em.count(entityName, query);
|
||||
case 'insert': {
|
||||
const id = await em.insert(entityName, values(name, condition, true));
|
||||
return plain(name, await em.findOneOrFail(entityName, { id }));
|
||||
}
|
||||
case 'nativeUpdate':
|
||||
return em.nativeUpdate(entityName, query, values(name, options));
|
||||
case 'nativeDelete':
|
||||
return em.nativeDelete(entityName, query);
|
||||
default:
|
||||
throw new Error('Unsupported database operation');
|
||||
}
|
||||
}
|
||||
|
||||
export function startDatabaseWorker(settings: typeof workerData) {
|
||||
workerData = settings;
|
||||
ready = initialize();
|
||||
void ready.catch(() => {});
|
||||
let queue = Promise.resolve();
|
||||
process.on('message', ({ id, operation, args, token }) => {
|
||||
// Continue after a failed statement so the owner can still roll back.
|
||||
queue = queue.then(async () => {
|
||||
try {
|
||||
process.send?.({
|
||||
id,
|
||||
value: await execute(operation, args, token),
|
||||
});
|
||||
} catch (error: any) {
|
||||
const code =
|
||||
error.name === 'UniqueConstraintViolationException'
|
||||
? 'UNIQUE_CONSTRAINT'
|
||||
: error.name === 'TableNotFoundException'
|
||||
? 'DATABASE_SCHEMA_MISSING'
|
||||
: (
|
||||
{
|
||||
5: 'SQLITE_BUSY',
|
||||
6: 'SQLITE_LOCKED',
|
||||
8: 'SQLITE_READONLY',
|
||||
10: 'SQLITE_IOERR',
|
||||
11: 'SQLITE_CORRUPT',
|
||||
13: 'SQLITE_FULL',
|
||||
14: 'SQLITE_CANTOPEN',
|
||||
26: 'SQLITE_NOTADB',
|
||||
} as Record<number, string>
|
||||
)[Number(error.errcode) & 255] || 'DATABASE_QUERY_FAILED';
|
||||
process.send?.({
|
||||
id,
|
||||
error: {
|
||||
code,
|
||||
message:
|
||||
code === 'UNIQUE_CONSTRAINT'
|
||||
? 'Resource already exists'
|
||||
: `Database operation failed (${code})`,
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
+3
-16
@@ -1,5 +1,4 @@
|
||||
import { sequelize } from '.';
|
||||
import { DataTypes, Model, ModelDefined } from 'sequelize';
|
||||
import { Repository, RecordRow } from './repository';
|
||||
|
||||
export class Dependence {
|
||||
timestamp?: string;
|
||||
@@ -47,17 +46,5 @@ export enum versionDependenceCommandTypes {
|
||||
'=',
|
||||
}
|
||||
|
||||
export interface DependenceInstance
|
||||
extends Model<Dependence, Dependence>,
|
||||
Dependence {}
|
||||
export const DependenceModel = sequelize.define<DependenceInstance>(
|
||||
'Dependence',
|
||||
{
|
||||
name: DataTypes.STRING,
|
||||
type: DataTypes.NUMBER,
|
||||
timestamp: DataTypes.STRING,
|
||||
status: DataTypes.NUMBER,
|
||||
log: DataTypes.JSON,
|
||||
remark: DataTypes.STRING,
|
||||
},
|
||||
);
|
||||
export type DependenceInstance = RecordRow<Dependence>;
|
||||
export const DependenceModel = new Repository<Dependence>('Dependence');
|
||||
|
||||
+3
-13
@@ -1,5 +1,4 @@
|
||||
import { DataTypes, Model } from 'sequelize';
|
||||
import { sequelize } from '.';
|
||||
import { Repository, RecordRow } from './repository';
|
||||
|
||||
export class Env {
|
||||
value?: string;
|
||||
@@ -38,14 +37,5 @@ export const initPosition = 4500000000000000;
|
||||
export const stepPosition = 10000000000;
|
||||
export const minPosition = 100;
|
||||
|
||||
export interface EnvInstance extends Model<Env, Env>, Env {}
|
||||
export const EnvModel = sequelize.define<EnvInstance>('Env', {
|
||||
value: { type: DataTypes.STRING, unique: 'compositeIndex' },
|
||||
timestamp: DataTypes.STRING,
|
||||
status: DataTypes.NUMBER,
|
||||
position: DataTypes.NUMBER,
|
||||
name: { type: DataTypes.STRING, unique: 'compositeIndex' },
|
||||
remarks: DataTypes.STRING,
|
||||
isPinned: DataTypes.NUMBER,
|
||||
labels: DataTypes.JSON,
|
||||
});
|
||||
export type EnvInstance = RecordRow<Env>;
|
||||
export const EnvModel = new Repository<Env>('Env');
|
||||
|
||||
+5
-18
@@ -1,23 +1,10 @@
|
||||
import { Sequelize, Transaction } from 'sequelize';
|
||||
import config from '../config/index';
|
||||
import { join } from 'path';
|
||||
import { DatabaseClient } from './databaseClient';
|
||||
|
||||
export const sequelize = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: join(config.dbPath, 'database.sqlite'),
|
||||
logging: false,
|
||||
retry: {
|
||||
max: 10,
|
||||
match: ['SQLITE_BUSY: database is locked'],
|
||||
},
|
||||
pool: {
|
||||
max: 5,
|
||||
min: 2,
|
||||
idle: 30000,
|
||||
acquire: 30000,
|
||||
evict: 10000,
|
||||
},
|
||||
transactionType: Transaction.TYPES.IMMEDIATE,
|
||||
});
|
||||
export const database = new DatabaseClient(
|
||||
join(config.dbPath, 'database.sqlite'),
|
||||
join(config.dbPath, 'keyv.sqlite'),
|
||||
);
|
||||
|
||||
export type ResponseType<T> = { code: number; data?: T; message?: string };
|
||||
|
||||
+11
-11
@@ -1,5 +1,4 @@
|
||||
import { sequelize } from '.';
|
||||
import { DataTypes, Model, ModelDefined } from 'sequelize';
|
||||
import { Repository, RecordRow } from './repository';
|
||||
|
||||
export class App {
|
||||
name: string;
|
||||
@@ -24,13 +23,14 @@ export interface AppToken {
|
||||
expiration: number;
|
||||
}
|
||||
|
||||
export type AppScope = 'envs' | 'crons' | 'configs' | 'scripts' | 'logs' | 'system' | 'dashboard';
|
||||
export type AppScope =
|
||||
| 'envs'
|
||||
| 'crons'
|
||||
| 'configs'
|
||||
| 'scripts'
|
||||
| 'logs'
|
||||
| 'system'
|
||||
| 'dashboard';
|
||||
|
||||
export interface AppInstance extends Model<App, App>, App {}
|
||||
export const AppModel = sequelize.define<AppInstance>('App', {
|
||||
name: { type: DataTypes.STRING, unique: 'name' },
|
||||
scopes: DataTypes.JSON,
|
||||
client_id: DataTypes.STRING,
|
||||
client_secret: DataTypes.STRING,
|
||||
tokens: DataTypes.JSON,
|
||||
});
|
||||
export type AppInstance = RecordRow<App>;
|
||||
export const AppModel = new Repository<App>('App');
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
export type Query = Record<string, any>;
|
||||
export type QuerySpec = {
|
||||
where: Query;
|
||||
orderBy?: Record<string, string>[];
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
};
|
||||
|
||||
export function orderBy(order: string[][]): Record<string, string>[] {
|
||||
return order.map(([field, direction]) => {
|
||||
if (!/^(asc|desc)$/i.test(direction))
|
||||
throw new Error('Invalid sort direction');
|
||||
return { [field]: direction.toLowerCase() };
|
||||
});
|
||||
}
|
||||
|
||||
export function textSearch(fields: string[], text: string): Query {
|
||||
return {
|
||||
$or: fields.flatMap((field) =>
|
||||
[...new Set([text, encodeURI(text)])].map((value) => ({
|
||||
[field]: { $like: `%${value}%` },
|
||||
})),
|
||||
),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { database } from '.';
|
||||
import { DatabaseClient } from './databaseClient';
|
||||
|
||||
export type RecordRow<T> = T & { id: number; createdAt: Date; updatedAt: Date };
|
||||
export type Query = Record<string, any>;
|
||||
export type ReadOptions = {
|
||||
fields?: string[];
|
||||
orderBy?: Record<string, string>[];
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
};
|
||||
|
||||
// A small IPC boundary with MikroORM's native query/operation names. No ORM
|
||||
// entities or request identity maps escape the single database worker.
|
||||
export class Repository<T> {
|
||||
constructor(
|
||||
private name: string,
|
||||
private client: () => DatabaseClient = () => database,
|
||||
) {}
|
||||
find(where: Query = {}, options: ReadOptions = {}): Promise<RecordRow<T>[]> {
|
||||
return this.client().request('find', this.name, where, options);
|
||||
}
|
||||
findOne(
|
||||
where: Query,
|
||||
options: ReadOptions = {},
|
||||
): Promise<RecordRow<T> | null> {
|
||||
return this.client().request('findOne', this.name, where, options);
|
||||
}
|
||||
count(where: Query = {}): Promise<number> {
|
||||
return this.client().request('count', this.name, where);
|
||||
}
|
||||
insert(data: Partial<T>): Promise<RecordRow<T>> {
|
||||
return this.client().request('insert', this.name, data);
|
||||
}
|
||||
nativeUpdate(where: Query, data: Partial<T>): Promise<number> {
|
||||
return this.client().request('nativeUpdate', this.name, where, data);
|
||||
}
|
||||
nativeDelete(where: Query): Promise<number> {
|
||||
return this.client().request('nativeDelete', this.name, where);
|
||||
}
|
||||
async insertMany(data: Partial<T>[]): Promise<RecordRow<T>[]> {
|
||||
return this.client().transaction(async () => {
|
||||
const rows = [];
|
||||
for (const item of data) rows.push(await this.insert(item));
|
||||
return rows;
|
||||
});
|
||||
}
|
||||
async getOrCreate(where: Query): Promise<RecordRow<T>> {
|
||||
return this.client().transaction(
|
||||
async () =>
|
||||
(await this.findOne(where)) || this.insert(where as Partial<T>),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
import { sequelize } from '.';
|
||||
import { DataTypes, Model } from 'sequelize';
|
||||
import { Repository, RecordRow } from './repository';
|
||||
|
||||
export enum InstanceStatus {
|
||||
'running' = 0,
|
||||
@@ -41,41 +40,7 @@ export class RunningInstance {
|
||||
}
|
||||
}
|
||||
|
||||
export interface RunningInstanceModel
|
||||
extends Model<RunningInstanceAttributes, RunningInstanceAttributes>,
|
||||
RunningInstanceAttributes {}
|
||||
|
||||
export const RunningInstanceModel = sequelize.define<RunningInstanceModel>(
|
||||
export type RunningInstanceModel = RecordRow<RunningInstanceAttributes>;
|
||||
export const RunningInstanceModel = new Repository<RunningInstanceAttributes>(
|
||||
'RunningInstance',
|
||||
{
|
||||
cron_id: {
|
||||
type: DataTypes.NUMBER,
|
||||
allowNull: false,
|
||||
},
|
||||
pid: {
|
||||
type: DataTypes.NUMBER,
|
||||
allowNull: true,
|
||||
},
|
||||
log_path: {
|
||||
type: DataTypes.STRING,
|
||||
allowNull: true,
|
||||
},
|
||||
started_at: {
|
||||
type: DataTypes.NUMBER,
|
||||
allowNull: false,
|
||||
},
|
||||
finished_at: {
|
||||
type: DataTypes.NUMBER,
|
||||
allowNull: true,
|
||||
},
|
||||
status: {
|
||||
type: DataTypes.NUMBER,
|
||||
allowNull: false,
|
||||
defaultValue: InstanceStatus.running,
|
||||
},
|
||||
exit_code: {
|
||||
type: DataTypes.NUMBER,
|
||||
allowNull: true,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
// Persisted SQLite names and types are part of the rollback contract.
|
||||
type Field = {
|
||||
type: string;
|
||||
fieldName: string;
|
||||
nullable?: boolean;
|
||||
primary?: boolean;
|
||||
autoincrement?: boolean;
|
||||
};
|
||||
function entity<Name extends string>(
|
||||
name: Name,
|
||||
tableName: string,
|
||||
groups: Record<string, string[]>,
|
||||
required: string[] = [],
|
||||
) {
|
||||
const properties: Record<string, Field> = {
|
||||
id: { type: 'number', fieldName: 'id', primary: true, autoincrement: true },
|
||||
};
|
||||
for (const [type, fields] of Object.entries(groups))
|
||||
for (const field of fields)
|
||||
properties[field] = {
|
||||
type,
|
||||
fieldName: field,
|
||||
nullable: !required.includes(field),
|
||||
};
|
||||
for (const field of ['createdAt', 'updatedAt'])
|
||||
properties[field] = {
|
||||
type: 'legacyDate',
|
||||
fieldName: field,
|
||||
nullable: false,
|
||||
};
|
||||
return { name, tableName, properties };
|
||||
}
|
||||
export const entityDefinitions = {
|
||||
App: entity(
|
||||
'App',
|
||||
'Apps',
|
||||
{
|
||||
string: ['name', 'client_id', 'client_secret'],
|
||||
json: ['scopes', 'tokens'],
|
||||
},
|
||||
[],
|
||||
),
|
||||
Auth: entity('Auth', 'Auths', { string: ['ip', 'type'], json: ['info'] }, []),
|
||||
CrontabStat: entity(
|
||||
'CrontabStat',
|
||||
'CrontabStats',
|
||||
{
|
||||
number: [
|
||||
'ref_id',
|
||||
'run_count',
|
||||
'success_count',
|
||||
'fail_count',
|
||||
'total_time',
|
||||
'max_time',
|
||||
],
|
||||
string: ['date'],
|
||||
},
|
||||
['ref_id', 'date'],
|
||||
),
|
||||
CrontabView: entity(
|
||||
'CrontabView',
|
||||
'CrontabViews',
|
||||
{
|
||||
string: ['name', 'filterRelation'],
|
||||
number: ['position', 'isDisabled', 'type'],
|
||||
json: ['filters', 'sorts'],
|
||||
},
|
||||
[],
|
||||
),
|
||||
Crontab: entity(
|
||||
'Crontab',
|
||||
'Crontabs',
|
||||
{
|
||||
string: [
|
||||
'name',
|
||||
'command',
|
||||
'schedule',
|
||||
'timestamp',
|
||||
'log_path',
|
||||
'queued_token',
|
||||
'task_before',
|
||||
'task_after',
|
||||
'log_name',
|
||||
'work_dir',
|
||||
],
|
||||
boolean: ['saved'],
|
||||
number: [
|
||||
'status',
|
||||
'isSystem',
|
||||
'pid',
|
||||
'isDisabled',
|
||||
'isPinned',
|
||||
'last_running_time',
|
||||
'last_execution_time',
|
||||
'sub_id',
|
||||
'allow_multiple_instances',
|
||||
],
|
||||
json: ['labels', 'extra_schedules'],
|
||||
},
|
||||
[],
|
||||
),
|
||||
Dependence: entity(
|
||||
'Dependence',
|
||||
'Dependences',
|
||||
{
|
||||
string: ['name', 'timestamp', 'remark'],
|
||||
number: ['type', 'status'],
|
||||
json: ['log'],
|
||||
},
|
||||
[],
|
||||
),
|
||||
Env: entity(
|
||||
'Env',
|
||||
'Envs',
|
||||
{
|
||||
string: ['value', 'timestamp', 'name', 'remarks'],
|
||||
number: ['status', 'position', 'isPinned'],
|
||||
json: ['labels'],
|
||||
},
|
||||
[],
|
||||
),
|
||||
RunningInstance: entity(
|
||||
'RunningInstance',
|
||||
'RunningInstances',
|
||||
{
|
||||
number: [
|
||||
'cron_id',
|
||||
'pid',
|
||||
'started_at',
|
||||
'finished_at',
|
||||
'status',
|
||||
'exit_code',
|
||||
],
|
||||
string: ['log_path'],
|
||||
},
|
||||
['cron_id', 'started_at', 'status'],
|
||||
),
|
||||
Subscription: entity(
|
||||
'Subscription',
|
||||
'Subscriptions',
|
||||
{
|
||||
string: [
|
||||
'name',
|
||||
'url',
|
||||
'schedule',
|
||||
'type',
|
||||
'whitelist',
|
||||
'blacklist',
|
||||
'dependences',
|
||||
'extensions',
|
||||
'sub_before',
|
||||
'sub_after',
|
||||
'branch',
|
||||
'pull_type',
|
||||
'log_path',
|
||||
'schedule_type',
|
||||
'alias',
|
||||
'proxy',
|
||||
],
|
||||
json: ['interval_schedule', 'pull_option'],
|
||||
number: ['status', 'pid', 'is_disabled', 'autoAddCron', 'autoDelCron'],
|
||||
},
|
||||
[],
|
||||
),
|
||||
};
|
||||
export const initialSchema = [
|
||||
'CREATE TABLE IF NOT EXISTS `Apps` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `scopes` JSON, `client_id` VARCHAR(255), `client_secret` VARCHAR(255), `tokens` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`))',
|
||||
'CREATE TABLE IF NOT EXISTS `Auths` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `ip` VARCHAR(255), `type` VARCHAR(255), `info` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
|
||||
'CREATE TABLE IF NOT EXISTS `CrontabStats` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `ref_id` NUMBER NOT NULL, `date` VARCHAR(255) NOT NULL, `run_count` NUMBER DEFAULT 0, `success_count` NUMBER DEFAULT 0, `fail_count` NUMBER DEFAULT 0, `total_time` NUMBER DEFAULT 0, `max_time` NUMBER DEFAULT 0, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
|
||||
'CREATE TABLE IF NOT EXISTS `CrontabViews` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `position` NUMBER, `isDisabled` NUMBER, `filters` JSON, `sorts` JSON, `filterRelation` VARCHAR(255), `type` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`))',
|
||||
'CREATE TABLE IF NOT EXISTS `Crontabs` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `command` VARCHAR(255), `schedule` VARCHAR(255), `timestamp` VARCHAR(255), `saved` TINYINT(1), `status` NUMBER, `isSystem` NUMBER, `pid` NUMBER, `isDisabled` NUMBER, `isPinned` NUMBER, `log_path` VARCHAR(255), `queued_token` VARCHAR(255), `labels` JSON, `last_running_time` NUMBER, `last_execution_time` NUMBER, `sub_id` NUMBER, `extra_schedules` JSON, `task_before` VARCHAR(255), `task_after` VARCHAR(255), `log_name` VARCHAR(255), `allow_multiple_instances` NUMBER, `work_dir` VARCHAR(255), `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`, `command`, `schedule`))',
|
||||
'CREATE TABLE IF NOT EXISTS `Dependences` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `type` NUMBER, `timestamp` VARCHAR(255), `status` NUMBER, `log` JSON, `remark` VARCHAR(255), `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
|
||||
'CREATE TABLE IF NOT EXISTS `Envs` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `value` VARCHAR(255), `timestamp` VARCHAR(255), `status` NUMBER, `position` NUMBER, `name` VARCHAR(255), `remarks` VARCHAR(255), `isPinned` NUMBER, `labels` JSON, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`value`, `name`))',
|
||||
'CREATE TABLE IF NOT EXISTS `RunningInstances` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `cron_id` NUMBER NOT NULL, `pid` NUMBER, `log_path` VARCHAR(255), `started_at` NUMBER NOT NULL, `finished_at` NUMBER, `status` NUMBER NOT NULL DEFAULT 0, `exit_code` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL)',
|
||||
'CREATE TABLE IF NOT EXISTS `Subscriptions` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `name` VARCHAR(255), `url` VARCHAR(255), `schedule` VARCHAR(255), `interval_schedule` JSON, `type` VARCHAR(255), `whitelist` VARCHAR(255), `blacklist` VARCHAR(255), `status` NUMBER, `dependences` VARCHAR(255), `extensions` VARCHAR(255), `sub_before` VARCHAR(255), `sub_after` VARCHAR(255), `branch` VARCHAR(255), `pull_type` VARCHAR(255), `pull_option` JSON, `pid` NUMBER, `is_disabled` NUMBER, `log_path` VARCHAR(255), `schedule_type` VARCHAR(255), `alias` VARCHAR(255), `proxy` VARCHAR(255), `autoAddCron` NUMBER, `autoDelCron` NUMBER, `createdAt` DATETIME NOT NULL, `updatedAt` DATETIME NOT NULL, UNIQUE (`name`, `url`, `schedule`, `interval_schedule`), UNIQUE (`alias`))',
|
||||
'CREATE INDEX IF NOT EXISTS `crontab_stats_date` ON `CrontabStats` (`date`)',
|
||||
'CREATE UNIQUE INDEX IF NOT EXISTS `crontab_stats_ref_id_date` ON `CrontabStats` (`ref_id`, `date`)',
|
||||
];
|
||||
@@ -0,0 +1,45 @@
|
||||
import { database } from '.';
|
||||
|
||||
// Aggregate SQL stays explicit and bound; public input is never used as an SQL
|
||||
// identifier. These queries preserve the dashboard response names.
|
||||
export const statistics = {
|
||||
async totals(date: string) {
|
||||
const rows = await database.query(
|
||||
`SELECT SUM(run_count) AS total_runs,
|
||||
SUM(success_count) AS total_success, SUM(fail_count) AS total_fail,
|
||||
SUM(total_time) AS total_time FROM CrontabStats WHERE date=?`,
|
||||
[date],
|
||||
);
|
||||
return rows[0];
|
||||
},
|
||||
results(date: string, column: 'success_count' | 'fail_count') {
|
||||
if (!['success_count', 'fail_count'].includes(column))
|
||||
throw new Error('Invalid statistic');
|
||||
return database.query(
|
||||
`SELECT ref_id,SUM(${column}) AS result_count FROM CrontabStats
|
||||
WHERE date=? AND ${column}>0 GROUP BY ref_id ORDER BY result_count DESC,ref_id ASC`,
|
||||
[date],
|
||||
);
|
||||
},
|
||||
trend(dates: string[]) {
|
||||
if (!dates.length) return Promise.resolve([]);
|
||||
return database.query(
|
||||
`SELECT date,SUM(run_count) AS total_runs,
|
||||
SUM(success_count) AS total_success,SUM(fail_count) AS total_fail FROM CrontabStats
|
||||
WHERE date IN (${dates
|
||||
.map(() => '?')
|
||||
.join(',')}) GROUP BY date ORDER BY date ASC`,
|
||||
dates,
|
||||
);
|
||||
},
|
||||
ranking(date: string, metric: 'total_time' | 'run_count') {
|
||||
if (!['total_time', 'run_count'].includes(metric))
|
||||
throw new Error('Invalid statistic');
|
||||
return database.query(
|
||||
`SELECT ref_id,SUM(total_time) AS total_time,SUM(run_count) AS run_count,
|
||||
SUM(success_count) AS success_count,MAX(max_time) AS max_time FROM CrontabStats
|
||||
WHERE date=? AND run_count>0 GROUP BY ref_id ORDER BY SUM(${metric}) DESC LIMIT 5`,
|
||||
[date],
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -1,5 +1,4 @@
|
||||
import { sequelize } from '.';
|
||||
import { DataTypes, Model, ModelDefined } from 'sequelize';
|
||||
import { Repository, RecordRow } from './repository';
|
||||
import { SimpleIntervalSchedule } from 'toad-scheduler';
|
||||
|
||||
type SimpleIntervalScheduleUnit = keyof SimpleIntervalSchedule;
|
||||
@@ -70,46 +69,5 @@ export enum SubscriptionStatus {
|
||||
'queued',
|
||||
}
|
||||
|
||||
export interface SubscriptionInstance
|
||||
extends Model<Subscription, Subscription>,
|
||||
Subscription {}
|
||||
export const SubscriptionModel = sequelize.define<SubscriptionInstance>(
|
||||
'Subscription',
|
||||
{
|
||||
name: {
|
||||
unique: 'compositeIndex',
|
||||
type: DataTypes.STRING,
|
||||
},
|
||||
url: {
|
||||
unique: 'compositeIndex',
|
||||
type: DataTypes.STRING,
|
||||
},
|
||||
schedule: {
|
||||
unique: 'compositeIndex',
|
||||
type: DataTypes.STRING,
|
||||
},
|
||||
interval_schedule: {
|
||||
unique: 'compositeIndex',
|
||||
type: DataTypes.JSON,
|
||||
},
|
||||
type: DataTypes.STRING,
|
||||
whitelist: DataTypes.STRING,
|
||||
blacklist: DataTypes.STRING,
|
||||
status: DataTypes.NUMBER,
|
||||
dependences: DataTypes.STRING,
|
||||
extensions: DataTypes.STRING,
|
||||
sub_before: DataTypes.STRING,
|
||||
sub_after: DataTypes.STRING,
|
||||
branch: DataTypes.STRING,
|
||||
pull_type: DataTypes.STRING,
|
||||
pull_option: DataTypes.JSON,
|
||||
pid: DataTypes.NUMBER,
|
||||
is_disabled: DataTypes.NUMBER,
|
||||
log_path: DataTypes.STRING,
|
||||
schedule_type: DataTypes.STRING,
|
||||
alias: { type: DataTypes.STRING, unique: 'alias' },
|
||||
proxy: { type: DataTypes.STRING, allowNull: true },
|
||||
autoAddCron: { type: DataTypes.NUMBER, allowNull: true },
|
||||
autoDelCron: { type: DataTypes.NUMBER, allowNull: true },
|
||||
},
|
||||
);
|
||||
export type SubscriptionInstance = RecordRow<Subscription>;
|
||||
export const SubscriptionModel = new Repository<Subscription>('Subscription');
|
||||
|
||||
+3
-13
@@ -1,5 +1,4 @@
|
||||
import { sequelize } from '.';
|
||||
import { DataTypes, Model, ModelDefined } from 'sequelize';
|
||||
import { Repository, RecordRow } from './repository';
|
||||
import { NotificationInfo } from './notify';
|
||||
|
||||
export class SystemInfo {
|
||||
@@ -106,14 +105,5 @@ export type SystemModelInfo = SystemConfigInfo &
|
||||
Partial<NotifyLogInfo> &
|
||||
Partial<AuthInfo>;
|
||||
|
||||
export interface SystemInstance
|
||||
extends Model<SystemInfo, SystemInfo>,
|
||||
SystemInfo {}
|
||||
export const SystemModel = sequelize.define<SystemInstance>('Auth', {
|
||||
ip: DataTypes.STRING,
|
||||
type: DataTypes.STRING,
|
||||
info: {
|
||||
type: DataTypes.JSON,
|
||||
allowNull: true,
|
||||
},
|
||||
});
|
||||
export type SystemInstance = RecordRow<SystemInfo>;
|
||||
export const SystemModel = new Repository<SystemInfo>('Auth');
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"extends": "../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"module": "node20",
|
||||
"moduleResolution": "node16",
|
||||
"rootDir": "..",
|
||||
"outDir": "../../static/build"
|
||||
},
|
||||
"include": ["./databaseWorker.ts", "./schema.ts"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
+6
-22
@@ -1,30 +1,14 @@
|
||||
import Logger from './logger';
|
||||
import { EnvModel } from '../data/env';
|
||||
import { CrontabModel } from '../data/cron';
|
||||
import { DependenceModel } from '../data/dependence';
|
||||
import { AppModel } from '../data/open';
|
||||
import { SystemModel } from '../data/system';
|
||||
import { SubscriptionModel } from '../data/subscription';
|
||||
import { CrontabViewModel } from '../data/cronView';
|
||||
import { CrontabStatModel } from '../data/cronStats';
|
||||
import { RunningInstanceModel } from '../data/runningInstance';
|
||||
import { sequelize } from '../data';
|
||||
import { database } from '../data';
|
||||
import { initialSchema } from '../data/schema';
|
||||
import { migrateSchema } from '../shared/schemaMigrations';
|
||||
|
||||
export default async () => {
|
||||
try {
|
||||
await CrontabModel.sync();
|
||||
await DependenceModel.sync();
|
||||
await AppModel.sync();
|
||||
await SystemModel.sync();
|
||||
await EnvModel.sync();
|
||||
await SubscriptionModel.sync();
|
||||
await CrontabViewModel.sync();
|
||||
await CrontabStatModel.sync();
|
||||
await RunningInstanceModel.sync();
|
||||
|
||||
await migrateSchema(sequelize);
|
||||
|
||||
await database.transaction(async () => {
|
||||
for (const sql of initialSchema) await database.query(sql);
|
||||
await migrateSchema(database);
|
||||
});
|
||||
Logger.info('[boot] DB loaded');
|
||||
} catch (error) {
|
||||
Logger.error('[boot] DB load failed', error);
|
||||
|
||||
+85
-99
@@ -16,7 +16,7 @@ import { t } from '../shared/i18n';
|
||||
import { AppScope } from '../data/open';
|
||||
import uniqueConstraintError from '../middlewares/uniqueConstraintError';
|
||||
import protectedPathCase from '../middlewares/protectedPathCase';
|
||||
|
||||
import { withAuthAvailability } from '../middlewares/authAvailability';
|
||||
function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
|
||||
const setting = value?.trim();
|
||||
if (!setting) {
|
||||
@@ -30,24 +30,20 @@ function resolveTrustProxy(value = process.env.QL_TRUST_PROXY) {
|
||||
}
|
||||
return setting;
|
||||
}
|
||||
|
||||
export default ({ app }: { app: Application }) => {
|
||||
// Security: Enable strict routing to prevent case-insensitive path bypass
|
||||
app.set('case sensitive routing', true);
|
||||
app.set('strict routing', true);
|
||||
app.set('trust proxy', resolveTrustProxy());
|
||||
app.use(cors());
|
||||
|
||||
// Security: Reject case variations under protected API namespaces before
|
||||
// authentication checks can interpret the request differently from routing.
|
||||
app.use(protectedPathCase);
|
||||
|
||||
// Rewrite URLs to strip baseUrl prefix if configured
|
||||
// This allows the rest of the app to work without baseUrl awareness
|
||||
if (config.baseUrl) {
|
||||
app.use(rewrite(`${config.baseUrl}/*`, '/$1'));
|
||||
}
|
||||
|
||||
app.get(`${config.api.prefix}/env.js`, serveEnv);
|
||||
app.use(
|
||||
`${config.api.prefix}/static`,
|
||||
@@ -58,7 +54,6 @@ export default ({ app }: { app: Application }) => {
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const credentialPaths = ['/api', '/open'].flatMap((prefix) =>
|
||||
['/user/login', '/user/init', '/user/two-factor/login'].map(
|
||||
(route) => `${prefix}${route}`,
|
||||
@@ -69,13 +64,10 @@ export default ({ app }: { app: Application }) => {
|
||||
bodyParser.json({ limit: '16kb' }),
|
||||
bodyParser.urlencoded({ limit: '16kb', extended: false }),
|
||||
);
|
||||
|
||||
app.use(bodyParser.json({ limit: '50mb' }));
|
||||
app.use(bodyParser.urlencoded({ limit: '50mb', extended: true }));
|
||||
|
||||
const frontendPath = path.join(config.rootPath, 'static/dist');
|
||||
app.use(express.static(frontendPath));
|
||||
|
||||
app.use(
|
||||
expressjwt({
|
||||
secret: config.jwt.secret,
|
||||
@@ -84,7 +76,6 @@ export default ({ app }: { app: Application }) => {
|
||||
path: [...config.apiWhiteList, /^(\/(?!api\/).*)$/i],
|
||||
}),
|
||||
);
|
||||
|
||||
app.use((req: Request, res, next) => {
|
||||
if (!req.headers) {
|
||||
req.platform = 'desktop';
|
||||
@@ -94,91 +85,84 @@ export default ({ app }: { app: Application }) => {
|
||||
}
|
||||
return next();
|
||||
});
|
||||
|
||||
app.use(async (req: Request, res, next) => {
|
||||
const pathLower = req.path.toLowerCase();
|
||||
if (!['/open/', '/api/'].some((x) => pathLower.startsWith(x))) {
|
||||
return next();
|
||||
}
|
||||
|
||||
const headerToken = getToken(req);
|
||||
if (pathLower.startsWith('/open/')) {
|
||||
const apps = await shareStore.getApps();
|
||||
const doc = apps?.filter((x) =>
|
||||
x.tokens?.find((y) => y.value === headerToken),
|
||||
)?.[0];
|
||||
if (doc && doc.tokens && doc.tokens.length > 0) {
|
||||
const currentToken = doc.tokens.find((x) => x.value === headerToken);
|
||||
const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
|
||||
const key = keyMatch && keyMatch[1];
|
||||
|
||||
if (!doc.scopes.includes(key as AppScope)) {
|
||||
const err = new UnauthorizedError('credentials_bad_scheme', {
|
||||
message: t('暂无权限'),
|
||||
});
|
||||
return next(err);
|
||||
}
|
||||
|
||||
if (
|
||||
!currentToken ||
|
||||
currentToken.expiration < Math.round(Date.now() / 1000)
|
||||
) {
|
||||
const err = new UnauthorizedError('invalid_token', {
|
||||
message: t('Token 已失效'),
|
||||
});
|
||||
return next(err);
|
||||
}
|
||||
|
||||
app.use(
|
||||
withAuthAvailability(async (req: Request, res, next) => {
|
||||
const pathLower = req.path.toLowerCase();
|
||||
if (!['/open/', '/api/'].some((x) => pathLower.startsWith(x))) {
|
||||
return next();
|
||||
}
|
||||
}
|
||||
|
||||
const originPath = `${req.baseUrl}${pathLower === '/' ? '' : pathLower}`;
|
||||
if (
|
||||
!headerToken &&
|
||||
originPath &&
|
||||
config.apiWhiteList.includes(originPath)
|
||||
) {
|
||||
return next();
|
||||
}
|
||||
|
||||
const authInfo = await shareStore.getAuthInfo();
|
||||
if (isValidToken(authInfo, headerToken, req.platform, config.jwt.secret)) {
|
||||
return next();
|
||||
}
|
||||
|
||||
const errorCode = headerToken ? 'invalid_token' : 'credentials_required';
|
||||
const errorMessage = headerToken ? t('Token 已失效') : t('请先登录');
|
||||
const err = new UnauthorizedError(errorCode, { message: errorMessage });
|
||||
next(err);
|
||||
});
|
||||
|
||||
app.use(async (req, res, next) => {
|
||||
const pathLower = req.path.toLowerCase();
|
||||
if (
|
||||
![
|
||||
'/api/user/init',
|
||||
'/api/user/notification/init',
|
||||
'/open/user/init',
|
||||
'/open/user/notification/init',
|
||||
].includes(pathLower)
|
||||
) {
|
||||
return next();
|
||||
}
|
||||
const authInfo = (await shareStore.getAuthInfo()) || ({} as AuthInfo);
|
||||
|
||||
let isInitialized = !isDefaultAuthInfo(authInfo);
|
||||
|
||||
if (isInitialized) {
|
||||
return res.send({ code: 450, message: t('未知错误') });
|
||||
} else {
|
||||
return next();
|
||||
}
|
||||
});
|
||||
|
||||
const headerToken = getToken(req);
|
||||
if (pathLower.startsWith('/open/')) {
|
||||
const apps = await shareStore.getApps();
|
||||
const doc = apps?.filter((x) =>
|
||||
x.tokens?.find((y) => y.value === headerToken),
|
||||
)?.[0];
|
||||
if (doc && doc.tokens && doc.tokens.length > 0) {
|
||||
const currentToken = doc.tokens.find((x) => x.value === headerToken);
|
||||
const keyMatch = pathLower.match(/\/open\/([a-z]+)\/*/);
|
||||
const key = keyMatch && keyMatch[1];
|
||||
if (!doc.scopes.includes(key as AppScope)) {
|
||||
const err = new UnauthorizedError('credentials_bad_scheme', {
|
||||
message: t('暂无权限'),
|
||||
});
|
||||
return next(err);
|
||||
}
|
||||
if (
|
||||
!currentToken ||
|
||||
currentToken.expiration < Math.round(Date.now() / 1000)
|
||||
) {
|
||||
const err = new UnauthorizedError('invalid_token', {
|
||||
message: t('Token 已失效'),
|
||||
});
|
||||
return next(err);
|
||||
}
|
||||
return next();
|
||||
}
|
||||
}
|
||||
const originPath = `${req.baseUrl}${pathLower === '/' ? '' : pathLower}`;
|
||||
if (
|
||||
!headerToken &&
|
||||
originPath &&
|
||||
config.apiWhiteList.includes(originPath)
|
||||
) {
|
||||
return next();
|
||||
}
|
||||
const authInfo = await shareStore.getAuthInfo();
|
||||
if (
|
||||
isValidToken(authInfo, headerToken, req.platform, config.jwt.secret)
|
||||
) {
|
||||
return next();
|
||||
}
|
||||
const errorCode = headerToken ? 'invalid_token' : 'credentials_required';
|
||||
const errorMessage = headerToken ? t('Token 已失效') : t('请先登录');
|
||||
const err = new UnauthorizedError(errorCode, { message: errorMessage });
|
||||
next(err);
|
||||
}),
|
||||
);
|
||||
app.use(
|
||||
withAuthAvailability(async (req, res, next) => {
|
||||
const pathLower = req.path.toLowerCase();
|
||||
if (
|
||||
![
|
||||
'/api/user/init',
|
||||
'/api/user/notification/init',
|
||||
'/open/user/init',
|
||||
'/open/user/notification/init',
|
||||
].includes(pathLower)
|
||||
) {
|
||||
return next();
|
||||
}
|
||||
const authInfo = (await shareStore.getAuthInfo()) || ({} as AuthInfo);
|
||||
let isInitialized = !isDefaultAuthInfo(authInfo);
|
||||
if (isInitialized) {
|
||||
return res.send({ code: 450, message: t('未知错误') });
|
||||
} else {
|
||||
return next();
|
||||
}
|
||||
}),
|
||||
);
|
||||
app.use(rewrite('/open/*', '/api/$1'));
|
||||
app.use(config.api.prefix, routes());
|
||||
|
||||
app.get('*', (req, res, next) => {
|
||||
// Don't serve index.html for API routes
|
||||
if (
|
||||
@@ -196,13 +180,13 @@ export default ({ app }: { app: Application }) => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
app.use(validationErrors);
|
||||
app.use(uniqueConstraintError);
|
||||
|
||||
app.use(
|
||||
(
|
||||
err: Error & { status: number },
|
||||
err: Error & {
|
||||
status: number;
|
||||
},
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
@@ -216,15 +200,16 @@ export default ({ app }: { app: Application }) => {
|
||||
return next(err);
|
||||
},
|
||||
);
|
||||
|
||||
app.use(
|
||||
(
|
||||
err: Error & { errors: any[] },
|
||||
err: Error & {
|
||||
errors: any[];
|
||||
},
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
) => {
|
||||
if (err.name.includes('Sequelize')) {
|
||||
if (err.name === 'DatabaseError') {
|
||||
return res
|
||||
.status(500)
|
||||
.send({
|
||||
@@ -237,10 +222,11 @@ export default ({ app }: { app: Application }) => {
|
||||
return next(err);
|
||||
},
|
||||
);
|
||||
|
||||
app.use(
|
||||
(
|
||||
err: Error & { status: number },
|
||||
err: Error & {
|
||||
status: number;
|
||||
},
|
||||
req: Request,
|
||||
res: Response,
|
||||
next: NextFunction,
|
||||
|
||||
+65
-82
@@ -1,3 +1,5 @@
|
||||
import { textSearch } from '../data/queries';
|
||||
import { Query, QuerySpec, orderBy } from '../data/queries';
|
||||
import DependenceService from '../services/dependence';
|
||||
import { exec } from 'child_process';
|
||||
import { Container } from 'typedi';
|
||||
@@ -5,7 +7,6 @@ import { Crontab, CrontabModel, CrontabStatus } from '../data/cron';
|
||||
import CronService from '../services/cron';
|
||||
import EnvService from '../services/env';
|
||||
import { DependenceModel, DependenceStatus } from '../data/dependence';
|
||||
import { Op } from 'sequelize';
|
||||
import config from '../config';
|
||||
import { CrontabViewModel, CronViewType } from '../data/cronView';
|
||||
import { initPosition } from '../data/env';
|
||||
@@ -20,7 +21,6 @@ import Logger from './logger';
|
||||
import cronClient from '../schedule/client';
|
||||
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
|
||||
import { setLang, systemLang } from '../shared/i18n';
|
||||
|
||||
export default async () => {
|
||||
const cronService = Container.get(CronService);
|
||||
const envService = Container.get(EnvService);
|
||||
@@ -28,17 +28,14 @@ export default async () => {
|
||||
const systemService = Container.get(SystemService);
|
||||
const userService = Container.get(UserService);
|
||||
const openService = Container.get(OpenService);
|
||||
|
||||
// 初始化增加系统配置
|
||||
await openService.initializeSystemApp();
|
||||
const [systemConfig] = await SystemModel.findOrCreate({
|
||||
where: { type: AuthDataType.systemConfig },
|
||||
const systemConfig = await SystemModel.getOrCreate({
|
||||
type: AuthDataType.systemConfig,
|
||||
});
|
||||
await SystemModel.findOrCreate({
|
||||
where: { type: AuthDataType.notification },
|
||||
});
|
||||
const [authConfig] = await SystemModel.findOrCreate({
|
||||
where: { type: AuthDataType.authConfig },
|
||||
await SystemModel.getOrCreate({ type: AuthDataType.notification });
|
||||
const authConfig = await SystemModel.getOrCreate({
|
||||
type: AuthDataType.authConfig,
|
||||
});
|
||||
if (!authConfig?.info || isDemoEnv()) {
|
||||
let authInfo = {
|
||||
@@ -54,36 +51,34 @@ export default async () => {
|
||||
} catch (error) {
|
||||
Logger.warn('Failed to read auth config file, using default credentials');
|
||||
}
|
||||
await SystemModel.upsert({
|
||||
id: authConfig?.id,
|
||||
info: authInfo,
|
||||
type: AuthDataType.authConfig,
|
||||
});
|
||||
await SystemModel.nativeUpdate(
|
||||
{ id: authConfig.id },
|
||||
{
|
||||
info: authInfo,
|
||||
type: AuthDataType.authConfig,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const installDependencies = async () => {
|
||||
const docs = await DependenceModel.findAll({
|
||||
where: {},
|
||||
order: [
|
||||
['type', 'DESC'],
|
||||
['createdAt', 'DESC'],
|
||||
],
|
||||
raw: true,
|
||||
});
|
||||
|
||||
await DependenceModel.update(
|
||||
const docs = await DependenceModel.find(
|
||||
{},
|
||||
{
|
||||
orderBy: orderBy([
|
||||
['type', 'DESC'],
|
||||
['createdAt', 'DESC'],
|
||||
]),
|
||||
},
|
||||
);
|
||||
await DependenceModel.nativeUpdate(
|
||||
{ id: docs.map((x) => x.id!) },
|
||||
{ status: DependenceStatus.queued, log: [] },
|
||||
{ where: { id: docs.map((x) => x.id!) } },
|
||||
);
|
||||
|
||||
setTimeout(async () => {
|
||||
await dependenceService.installDependenceOneByOne(docs);
|
||||
|
||||
const bootAfterLoader = await import('./bootAfter');
|
||||
bootAfterLoader.default();
|
||||
}, 5000);
|
||||
};
|
||||
|
||||
// 初始化更新 linux/python/nodejs 镜像源配置
|
||||
if (systemConfig.info?.pythonMirror) {
|
||||
systemService.updatePythonMirror({
|
||||
@@ -106,43 +101,40 @@ export default async () => {
|
||||
nodeMirror: systemConfig.info?.nodeMirror,
|
||||
});
|
||||
}
|
||||
|
||||
// 初始化新增默认全部任务视图
|
||||
CrontabViewModel.findAll({
|
||||
where: { type: CronViewType.系统, name: '全部任务' },
|
||||
raw: true,
|
||||
}).then((docs) => {
|
||||
if (docs.length === 0) {
|
||||
CrontabViewModel.create({
|
||||
name: '全部任务',
|
||||
type: CronViewType.系统,
|
||||
position: initPosition / 2,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
CrontabViewModel.find({ type: CronViewType.系统, name: '全部任务' }, {}).then(
|
||||
(docs) => {
|
||||
if (docs.length === 0) {
|
||||
CrontabViewModel.insert({
|
||||
name: '全部任务',
|
||||
type: CronViewType.系统,
|
||||
position: initPosition / 2,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
// 初始化更新所有任务状态为空闲
|
||||
await CrontabModel.update({ status: CrontabStatus.idle }, { where: {} });
|
||||
|
||||
await CrontabModel.nativeUpdate({}, { status: CrontabStatus.idle });
|
||||
// 清空所有运行中的实例记录(服务重启后进程已不存在)
|
||||
await RunningInstanceModel.update(
|
||||
await RunningInstanceModel.nativeUpdate(
|
||||
{ status: InstanceStatus.running },
|
||||
{
|
||||
status: InstanceStatus.stopped,
|
||||
finished_at: Math.floor(Date.now() / 1000),
|
||||
// The previous process's exit code cannot be recovered after a restart.
|
||||
},
|
||||
{ where: { status: InstanceStatus.running } },
|
||||
);
|
||||
|
||||
// 初始化时执行一次所有的 ql repo 任务
|
||||
CrontabModel.findAll({
|
||||
where: {
|
||||
isDisabled: { [Op.ne]: 1 },
|
||||
command: {
|
||||
[Op.or]: [{ [Op.like]: `%ql repo%` }, { [Op.like]: `%ql raw%` }],
|
||||
},
|
||||
CrontabModel.find(
|
||||
{
|
||||
isDisabled: { ['$ne']: 1 },
|
||||
$or: [
|
||||
...textSearch(['command'], 'ql repo').$or,
|
||||
...textSearch(['command'], 'ql raw').$or,
|
||||
],
|
||||
},
|
||||
}).then((docs) => {
|
||||
{},
|
||||
).then((docs) => {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
const doc = docs[i];
|
||||
if (doc) {
|
||||
@@ -150,87 +142,78 @@ export default async () => {
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 更新2.11.3以前的脚本路径
|
||||
CrontabModel.findAll({
|
||||
where: {
|
||||
command: {
|
||||
[Op.or]: [
|
||||
{ [Op.like]: `%\/${config.rootPath}\/scripts\/%` },
|
||||
{ [Op.like]: `%\/${config.rootPath}\/config\/%` },
|
||||
{ [Op.like]: `%\/${config.rootPath}\/log\/%` },
|
||||
{ [Op.like]: `%\/${config.rootPath}\/db\/%` },
|
||||
],
|
||||
},
|
||||
CrontabModel.find(
|
||||
{
|
||||
$or: ['scripts', 'config', 'log', 'db'].flatMap(
|
||||
(folder) =>
|
||||
textSearch(['command'], `/${config.rootPath}/${folder}/`).$or,
|
||||
),
|
||||
},
|
||||
}).then(async (docs) => {
|
||||
{},
|
||||
).then(async (docs) => {
|
||||
for (let i = 0; i < docs.length; i++) {
|
||||
const doc = docs[i];
|
||||
if (doc) {
|
||||
if (doc.command.includes(`${config.rootPath}/scripts/`)) {
|
||||
await CrontabModel.update(
|
||||
await CrontabModel.nativeUpdate(
|
||||
{ id: doc.id },
|
||||
{ command: doc.command.replace(`${config.rootPath}/scripts/`, '') },
|
||||
{ where: { id: doc.id } },
|
||||
);
|
||||
}
|
||||
if (doc.command.includes(`${config.rootPath}/log/`)) {
|
||||
await CrontabModel.update(
|
||||
await CrontabModel.nativeUpdate(
|
||||
{ id: doc.id },
|
||||
{
|
||||
command: `${config.dataPath}/log/${doc.command.replace(
|
||||
`${config.rootPath}/log/`,
|
||||
'',
|
||||
)}`,
|
||||
},
|
||||
{ where: { id: doc.id } },
|
||||
);
|
||||
}
|
||||
if (doc.command.includes(`${config.rootPath}/config/`)) {
|
||||
await CrontabModel.update(
|
||||
await CrontabModel.nativeUpdate(
|
||||
{ id: doc.id },
|
||||
{
|
||||
command: `${config.dataPath}/config/${doc.command.replace(
|
||||
`${config.rootPath}/config/`,
|
||||
'',
|
||||
)}`,
|
||||
},
|
||||
{ where: { id: doc.id } },
|
||||
);
|
||||
}
|
||||
if (doc.command.includes(`${config.rootPath}/db/`)) {
|
||||
await CrontabModel.update(
|
||||
await CrontabModel.nativeUpdate(
|
||||
{ id: doc.id },
|
||||
{
|
||||
command: `${config.dataPath}/db/${doc.command.replace(
|
||||
`${config.rootPath}/db/`,
|
||||
'',
|
||||
)}`,
|
||||
},
|
||||
{ where: { id: doc.id } },
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 初始化语言(必须在 autosave_crontab 之前)
|
||||
const lang = systemConfig.info?.lang || systemLang();
|
||||
setLang(lang);
|
||||
|
||||
// 确保 lang_env.sh 存在
|
||||
try {
|
||||
const langEnvExist = await fileExist(config.langEnvFile);
|
||||
if (!langEnvExist) {
|
||||
await writeFile(config.langEnvFile, `export QL_LANG='${lang}'\n`);
|
||||
}
|
||||
} catch { }
|
||||
|
||||
} catch {}
|
||||
// 初始化保存一次ck和定时任务数据
|
||||
cronClient.readiness.configure(async () => {
|
||||
await cronService.autosave_crontab(true);
|
||||
await systemService.restoreCronConcurrency();
|
||||
});
|
||||
await cronClient.readiness.recover();
|
||||
|
||||
await envService.set_envs();
|
||||
|
||||
const authInfo = await userService.getAuthInfo();
|
||||
await shareStore.updateAuthInfo(authInfo);
|
||||
await openService.refreshApps();
|
||||
|
||||
@@ -9,20 +9,17 @@ import { join } from 'path';
|
||||
import { t } from '../shared/i18n';
|
||||
import Logger from './logger';
|
||||
import { assertSubscriptionAlias } from '../shared/subscriptionPath';
|
||||
|
||||
export default async () => {
|
||||
const systemService = Container.get(SystemService);
|
||||
const scheduleService = Container.get(ScheduleService);
|
||||
const subscriptionService = Container.get(SubscriptionService);
|
||||
const sshKeyService = Container.get(SshKeyService);
|
||||
|
||||
// 生成内置token
|
||||
let tokenCommand = `ts-node-transpile-only ${join(
|
||||
config.rootPath,
|
||||
'back/token.ts',
|
||||
)}`;
|
||||
const tokenFile = join(config.rootPath, 'static/build/token.js');
|
||||
|
||||
if (await fileExist(tokenFile)) {
|
||||
tokenCommand = `node ${tokenFile}`;
|
||||
}
|
||||
@@ -41,7 +38,6 @@ export default async () => {
|
||||
},
|
||||
true,
|
||||
);
|
||||
|
||||
// 运行删除日志任务
|
||||
const data = await systemService.getSystemConfig();
|
||||
if (data && data.info) {
|
||||
@@ -61,20 +57,17 @@ export default async () => {
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
systemService.updateTimezone(data.info);
|
||||
|
||||
// Apply global SSH key if configured
|
||||
if (data.info.globalSshKey) {
|
||||
await sshKeyService.addGlobalSSHKey(data.info.globalSshKey, 'global');
|
||||
}
|
||||
}
|
||||
|
||||
await subscriptionService.setSshConfig();
|
||||
const subs = await subscriptionService.list();
|
||||
for (const sub of subs) {
|
||||
try {
|
||||
const doc = sub.get({ plain: true });
|
||||
const doc = sub;
|
||||
assertSubscriptionAlias(doc.alias);
|
||||
await subscriptionService.handleTask(doc, !sub.is_disabled);
|
||||
} catch (error) {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { RequestHandler } from 'express';
|
||||
import { t } from '../shared/i18n';
|
||||
|
||||
// Express 4 does not forward rejected async middleware promises itself.
|
||||
export function withAuthAvailability(handler: RequestHandler): RequestHandler {
|
||||
return (req, res, next) => {
|
||||
void Promise.resolve()
|
||||
.then(() => handler(req, res, next))
|
||||
.catch(() => {
|
||||
res.setHeader('Retry-After', '1');
|
||||
next(
|
||||
Object.assign(new Error(t('认证服务暂时不可用,请稍后重试')), {
|
||||
status: 503,
|
||||
}),
|
||||
);
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -1,14 +1,13 @@
|
||||
import { ErrorRequestHandler } from 'express';
|
||||
import { UniqueConstraintError } from 'sequelize';
|
||||
import { DatabaseError } from '../data/databaseErrors';
|
||||
import { t } from '../shared/i18n';
|
||||
|
||||
// Handle the database constraint itself so concurrent writes are covered too.
|
||||
const uniqueConstraintError: ErrorRequestHandler = (err, req, res, next) => {
|
||||
if (!(err instanceof UniqueConstraintError)) return next(err);
|
||||
if (!(err instanceof DatabaseError) || err.code !== 'UNIQUE_CONSTRAINT')
|
||||
return next(err);
|
||||
// Constraint details can contain environment values or application credentials.
|
||||
res
|
||||
.status(409)
|
||||
.json({ code: 409, message: t('资源已存在,请检查重复的名称或值') });
|
||||
};
|
||||
|
||||
export default uniqueConstraintError;
|
||||
|
||||
+1
-39
@@ -39,10 +39,7 @@ import {
|
||||
RunCronsRequest,
|
||||
} from '../protos/api';
|
||||
import { NotificationInfo } from '../data/notify';
|
||||
import { Model } from 'sequelize';
|
||||
|
||||
Container.set('logger', LoggerInstance);
|
||||
|
||||
export const getEnvs = async (
|
||||
call: ServerUnaryCall<GetEnvsRequest, EnvsResponse>,
|
||||
callback: sendUnaryData<EnvsResponse>,
|
||||
@@ -62,7 +59,6 @@ export const getEnvs = async (
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export const createEnv = async (
|
||||
call: ServerUnaryCall<CreateEnvRequest, EnvsResponse>,
|
||||
callback: sendUnaryData<EnvsResponse>,
|
||||
@@ -75,7 +71,6 @@ export const createEnv = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const updateEnv = async (
|
||||
call: ServerUnaryCall<UpdateEnvRequest, EnvResponse>,
|
||||
callback: sendUnaryData<EnvResponse>,
|
||||
@@ -88,7 +83,6 @@ export const updateEnv = async (
|
||||
message: 'id parameter is required',
|
||||
});
|
||||
}
|
||||
|
||||
const envService = Container.get(EnvService);
|
||||
const data = await envService.update(
|
||||
pick(call.request.env, ['id', 'name', 'value', 'remarks']) as EnvItem,
|
||||
@@ -98,7 +92,6 @@ export const updateEnv = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const deleteEnvs = async (
|
||||
call: ServerUnaryCall<DeleteEnvsRequest, Response>,
|
||||
callback: sendUnaryData<Response>,
|
||||
@@ -110,7 +103,6 @@ export const deleteEnvs = async (
|
||||
message: 'ids parameter is required',
|
||||
});
|
||||
}
|
||||
|
||||
const envService = Container.get(EnvService);
|
||||
await envService.remove(call.request.ids);
|
||||
callback(null, { code: 200 });
|
||||
@@ -118,7 +110,6 @@ export const deleteEnvs = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const moveEnv = async (
|
||||
call: ServerUnaryCall<MoveEnvRequest, EnvResponse>,
|
||||
callback: sendUnaryData<EnvResponse>,
|
||||
@@ -131,7 +122,6 @@ export const moveEnv = async (
|
||||
message: 'id parameter is required',
|
||||
});
|
||||
}
|
||||
|
||||
const envService = Container.get(EnvService);
|
||||
const data = await envService.move(call.request.id, {
|
||||
fromIndex: call.request.fromIndex,
|
||||
@@ -142,7 +132,6 @@ export const moveEnv = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const disableEnvs = async (
|
||||
call: ServerUnaryCall<DisableEnvsRequest, Response>,
|
||||
callback: sendUnaryData<Response>,
|
||||
@@ -154,7 +143,6 @@ export const disableEnvs = async (
|
||||
message: 'ids parameter is required',
|
||||
});
|
||||
}
|
||||
|
||||
const envService = Container.get(EnvService);
|
||||
await envService.disabled(call.request.ids);
|
||||
callback(null, { code: 200 });
|
||||
@@ -162,7 +150,6 @@ export const disableEnvs = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const enableEnvs = async (
|
||||
call: ServerUnaryCall<EnableEnvsRequest, Response>,
|
||||
callback: sendUnaryData<Response>,
|
||||
@@ -174,7 +161,6 @@ export const enableEnvs = async (
|
||||
message: 'ids parameter is required',
|
||||
});
|
||||
}
|
||||
|
||||
const envService = Container.get(EnvService);
|
||||
await envService.enabled(call.request.ids);
|
||||
callback(null, { code: 200 });
|
||||
@@ -182,7 +168,6 @@ export const enableEnvs = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const updateEnvNames = async (
|
||||
call: ServerUnaryCall<UpdateEnvNamesRequest, Response>,
|
||||
callback: sendUnaryData<Response>,
|
||||
@@ -194,7 +179,6 @@ export const updateEnvNames = async (
|
||||
message: 'ids parameter is required',
|
||||
});
|
||||
}
|
||||
|
||||
const envService = Container.get(EnvService);
|
||||
await envService.updateNames({
|
||||
ids: call.request.ids,
|
||||
@@ -205,7 +189,6 @@ export const updateEnvNames = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const getEnvById = async (
|
||||
call: ServerUnaryCall<GetEnvByIdRequest, EnvResponse>,
|
||||
callback: sendUnaryData<EnvResponse>,
|
||||
@@ -218,7 +201,6 @@ export const getEnvById = async (
|
||||
message: 'id parameter is required',
|
||||
});
|
||||
}
|
||||
|
||||
const envService = Container.get(EnvService);
|
||||
const data = await envService.getDb({ id: call.request.id });
|
||||
callback(null, {
|
||||
@@ -229,7 +211,6 @@ export const getEnvById = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const systemNotify = async (
|
||||
call: ServerUnaryCall<SystemNotifyRequest, Response>,
|
||||
callback: sendUnaryData<Response>,
|
||||
@@ -247,7 +228,6 @@ export const systemNotify = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const recordNotifyLog = async (
|
||||
call: ServerUnaryCall<RecordNotifyLogRequest, Response>,
|
||||
callback: sendUnaryData<Response>,
|
||||
@@ -267,12 +247,9 @@ export const recordNotifyLog = async (
|
||||
callback(error);
|
||||
}
|
||||
};
|
||||
|
||||
const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
|
||||
if (!data) return undefined;
|
||||
// create() returns a Sequelize instance; spreading it omits attribute getters.
|
||||
const cron =
|
||||
data instanceof Model ? (data.get({ plain: true }) as CronItem) : data;
|
||||
const cron = data;
|
||||
return {
|
||||
...cron,
|
||||
labels: cron.labels ?? [],
|
||||
@@ -283,7 +260,6 @@ const normalizeCronData = (data: CronItem | null): CronItem | undefined => {
|
||||
task_after: cron.task_after ?? undefined,
|
||||
};
|
||||
};
|
||||
|
||||
export const getCronDetail = async (
|
||||
call: ServerUnaryCall<CronDetailRequest, CronDetailResponse>,
|
||||
callback: sendUnaryData<CronDetailResponse>,
|
||||
@@ -305,7 +281,6 @@ export const getCronDetail = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const createCron = async (
|
||||
call: ServerUnaryCall<CreateCronRequest, CronResponse>,
|
||||
callback: sendUnaryData<CronResponse>,
|
||||
@@ -318,7 +293,6 @@ export const createCron = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const updateCron = async (
|
||||
call: ServerUnaryCall<UpdateCronRequest, CronResponse>,
|
||||
callback: sendUnaryData<CronResponse>,
|
||||
@@ -326,7 +300,6 @@ export const updateCron = async (
|
||||
try {
|
||||
const cronService = Container.get(CronService);
|
||||
const { id, ...fields } = call.request;
|
||||
|
||||
const updateRequest = {
|
||||
id,
|
||||
...Object.entries(fields).reduce((acc: any, [key, value]) => {
|
||||
@@ -336,14 +309,12 @@ export const updateCron = async (
|
||||
return acc;
|
||||
}, {}),
|
||||
} as UpdateCronRequest;
|
||||
|
||||
const data = (await cronService.update(updateRequest)) as CronItem;
|
||||
callback(null, { code: 200, data: normalizeCronData(data) });
|
||||
} catch (e: any) {
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const deleteCrons = async (
|
||||
call: ServerUnaryCall<DeleteCronsRequest, Response>,
|
||||
callback: sendUnaryData<Response>,
|
||||
@@ -356,7 +327,6 @@ export const deleteCrons = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const getCrons = async (
|
||||
call: ServerUnaryCall<GetCronsRequest, CronsResponse>,
|
||||
callback: sendUnaryData<CronsResponse>,
|
||||
@@ -384,7 +354,6 @@ export const getCrons = async (
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export const getCronById = async (
|
||||
call: ServerUnaryCall<GetCronByIdRequest, CronResponse>,
|
||||
callback: sendUnaryData<CronResponse>,
|
||||
@@ -397,7 +366,6 @@ export const getCronById = async (
|
||||
message: 'id parameter is required',
|
||||
});
|
||||
}
|
||||
|
||||
const cronService = Container.get(CronService);
|
||||
const data = (await cronService.getDb({ id: call.request.id })) as CronItem;
|
||||
callback(null, { code: 200, data: normalizeCronData(data) });
|
||||
@@ -409,7 +377,6 @@ export const getCronById = async (
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export const enableCrons = async (
|
||||
call: ServerUnaryCall<EnableCronsRequest, Response>,
|
||||
callback: sendUnaryData<Response>,
|
||||
@@ -421,7 +388,6 @@ export const enableCrons = async (
|
||||
message: 'ids parameter is required',
|
||||
});
|
||||
}
|
||||
|
||||
const cronService = Container.get(CronService);
|
||||
await cronService.enabled(call.request.ids);
|
||||
callback(null, { code: 200 });
|
||||
@@ -429,7 +395,6 @@ export const enableCrons = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const disableCrons = async (
|
||||
call: ServerUnaryCall<DisableCronsRequest, Response>,
|
||||
callback: sendUnaryData<Response>,
|
||||
@@ -441,7 +406,6 @@ export const disableCrons = async (
|
||||
message: 'ids parameter is required',
|
||||
});
|
||||
}
|
||||
|
||||
const cronService = Container.get(CronService);
|
||||
await cronService.disabled(call.request.ids);
|
||||
callback(null, { code: 200 });
|
||||
@@ -449,7 +413,6 @@ export const disableCrons = async (
|
||||
callback(e);
|
||||
}
|
||||
};
|
||||
|
||||
export const runCrons = async (
|
||||
call: ServerUnaryCall<RunCronsRequest, Response>,
|
||||
callback: sendUnaryData<Response>,
|
||||
@@ -461,7 +424,6 @@ export const runCrons = async (
|
||||
message: 'ids parameter is required',
|
||||
});
|
||||
}
|
||||
|
||||
const cronService = Container.get(CronService);
|
||||
await cronService.run(call.request.ids);
|
||||
callback(null, { code: 200 });
|
||||
|
||||
@@ -4,6 +4,7 @@ import path from 'path';
|
||||
import { request } from 'undici';
|
||||
import { HealthCheckRequest, HealthCheckResponse } from '../protos/health';
|
||||
import config from '../config';
|
||||
import { database } from '../data';
|
||||
|
||||
function formatError(error: unknown): string {
|
||||
if (!(error instanceof Error)) {
|
||||
@@ -54,7 +55,7 @@ const check = async (
|
||||
switch (call.request.service) {
|
||||
// Local scheduler liveness only: never call HTTP from this probe.
|
||||
case 'scheduler':
|
||||
return callback(null, { status: 1 });
|
||||
return callback(null, { status: database.healthy ? 1 : 2 });
|
||||
case 'cron': {
|
||||
const healthUrl = `http://localhost:${config.port}${
|
||||
config.baseUrl || ''
|
||||
|
||||
+233
-283
File diff suppressed because it is too large
Load Diff
+19
-31
@@ -1,3 +1,4 @@
|
||||
import { Query, QuerySpec, orderBy } from '../data/queries';
|
||||
import { Service, Inject } from 'typedi';
|
||||
import winston from 'winston';
|
||||
import { CrontabView, CrontabViewModel } from '../data/cronView';
|
||||
@@ -7,12 +8,12 @@ import {
|
||||
minPosition,
|
||||
stepPosition,
|
||||
} from '../data/env';
|
||||
import { FindOptions } from 'sequelize';
|
||||
|
||||
@Service()
|
||||
export default class CronViewService {
|
||||
constructor(@Inject('logger') private logger: winston.Logger) {}
|
||||
|
||||
constructor(
|
||||
@Inject('logger')
|
||||
private logger: winston.Logger,
|
||||
) {}
|
||||
public async create(payload: CrontabView): Promise<CrontabView> {
|
||||
let position = initPosition;
|
||||
const views = await this.list();
|
||||
@@ -22,61 +23,51 @@ export default class CronViewService {
|
||||
position = position / 2;
|
||||
const tab = new CrontabView({ ...payload, position });
|
||||
const doc = await this.insert(tab);
|
||||
|
||||
await this.checkPosition(tab.position!);
|
||||
return doc;
|
||||
}
|
||||
|
||||
public async insert(payload: CrontabView): Promise<CrontabView> {
|
||||
return await CrontabViewModel.create(payload, { returning: true });
|
||||
return await CrontabViewModel.insert(payload);
|
||||
}
|
||||
|
||||
public async update(payload: CrontabView): Promise<CrontabView> {
|
||||
const doc = await this.getDb({ id: payload.id });
|
||||
const tab = new CrontabView({ ...doc, ...payload });
|
||||
const newDoc = await this.updateDb(tab);
|
||||
return newDoc;
|
||||
}
|
||||
|
||||
public async updateDb(payload: CrontabView): Promise<CrontabView> {
|
||||
await CrontabViewModel.update(payload, { where: { id: payload.id } });
|
||||
await CrontabViewModel.nativeUpdate({ id: payload.id }, payload);
|
||||
return await this.getDb({ id: payload.id });
|
||||
}
|
||||
|
||||
public async remove(ids: number[]) {
|
||||
await CrontabViewModel.destroy({ where: { id: ids } });
|
||||
await CrontabViewModel.nativeDelete({ id: ids });
|
||||
}
|
||||
|
||||
public async list(): Promise<CrontabView[]> {
|
||||
try {
|
||||
const result = await CrontabViewModel.findAll({
|
||||
where: {},
|
||||
order: [['position', 'DESC']],
|
||||
});
|
||||
const result = await CrontabViewModel.find(
|
||||
{},
|
||||
{
|
||||
orderBy: orderBy([['position', 'DESC']]),
|
||||
},
|
||||
);
|
||||
return result;
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public async getDb(
|
||||
query: FindOptions<CrontabView>['where'],
|
||||
): Promise<CrontabView> {
|
||||
const doc: any = await CrontabViewModel.findOne({ where: { ...query } });
|
||||
public async getDb(query: QuerySpec['where']): Promise<CrontabView> {
|
||||
const doc: any = await CrontabViewModel.findOne({ ...query }, {});
|
||||
if (!doc) {
|
||||
throw new Error(`CronView ${JSON.stringify(query)} not found`);
|
||||
}
|
||||
return doc.get({ plain: true });
|
||||
return doc;
|
||||
}
|
||||
|
||||
public async disabled(ids: number[]) {
|
||||
await CrontabViewModel.update({ isDisabled: 1 }, { where: { id: ids } });
|
||||
await CrontabViewModel.nativeUpdate({ id: ids }, { isDisabled: 1 });
|
||||
}
|
||||
|
||||
public async enabled(ids: number[]) {
|
||||
await CrontabViewModel.update({ isDisabled: 0 }, { where: { id: ids } });
|
||||
await CrontabViewModel.nativeUpdate({ id: ids }, { isDisabled: 0 });
|
||||
}
|
||||
|
||||
private async checkPosition(position: number) {
|
||||
const precisionPosition = parseFloat(position.toPrecision(16));
|
||||
if (precisionPosition < minPosition || precisionPosition > maxPosition) {
|
||||
@@ -88,11 +79,9 @@ export default class CronViewService {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private getPrecisionPosition(position: number): number {
|
||||
return parseFloat(position.toPrecision(16));
|
||||
}
|
||||
|
||||
public async move({
|
||||
id,
|
||||
fromIndex,
|
||||
@@ -118,7 +107,6 @@ export default class CronViewService {
|
||||
id,
|
||||
position: this.getPrecisionPosition(targetPosition),
|
||||
});
|
||||
|
||||
await this.checkPosition(targetPosition);
|
||||
return newDoc;
|
||||
}
|
||||
|
||||
+48
-93
@@ -1,3 +1,5 @@
|
||||
import { textSearch } from '../data/queries';
|
||||
import { Query, QuerySpec, orderBy } from '../data/queries';
|
||||
import { Service, Inject } from 'typedi';
|
||||
import winston from 'winston';
|
||||
import config from '../config';
|
||||
@@ -10,7 +12,6 @@ import {
|
||||
} from '../data/dependence';
|
||||
import { spawn } from 'cross-spawn';
|
||||
import SockService from './sock';
|
||||
import { FindOptions, Op } from 'sequelize';
|
||||
import {
|
||||
fileExist,
|
||||
getPid,
|
||||
@@ -27,14 +28,13 @@ import taskLimit from '../shared/pLimit';
|
||||
import { detectOS } from '../config/util';
|
||||
import { LINUX_DEPENDENCE_COMMAND } from '../config/const';
|
||||
import { t, tf } from '../shared/i18n';
|
||||
|
||||
@Service()
|
||||
export default class DependenceService {
|
||||
constructor(
|
||||
@Inject('logger') private logger: winston.Logger,
|
||||
@Inject('logger')
|
||||
private logger: winston.Logger,
|
||||
private sockService: SockService,
|
||||
) {}
|
||||
|
||||
public async create(payloads: Dependence[]): Promise<Dependence[]> {
|
||||
const tabs = payloads.map((x) => {
|
||||
getInstallCommand(x.type, x.name);
|
||||
@@ -45,14 +45,14 @@ export default class DependenceService {
|
||||
this.installDependenceOneByOne(docs);
|
||||
return docs;
|
||||
}
|
||||
|
||||
public async insert(payloads: Dependence[]): Promise<Dependence[]> {
|
||||
const docs = await DependenceModel.bulkCreate(payloads);
|
||||
const docs = await DependenceModel.insertMany(payloads);
|
||||
return docs;
|
||||
}
|
||||
|
||||
public async update(
|
||||
payload: Dependence & { id: string },
|
||||
payload: Dependence & {
|
||||
id: string;
|
||||
},
|
||||
): Promise<Dependence> {
|
||||
const { id, ...other } = payload;
|
||||
const doc = await this.getDb({ id });
|
||||
@@ -66,14 +66,12 @@ export default class DependenceService {
|
||||
this.installDependenceOneByOne([newDoc]);
|
||||
return newDoc;
|
||||
}
|
||||
|
||||
private async updateDb(payload: Dependence): Promise<Dependence> {
|
||||
await DependenceModel.update(payload, { where: { id: payload.id } });
|
||||
await DependenceModel.nativeUpdate({ id: payload.id }, payload);
|
||||
return await this.getDb({ id: payload.id });
|
||||
}
|
||||
|
||||
public async remove(ids: number[], force = false): Promise<Dependence[]> {
|
||||
const docs = await DependenceModel.findAll({ where: { id: ids } });
|
||||
const docs = await DependenceModel.find({ id: ids }, {});
|
||||
for (const doc of docs) {
|
||||
taskLimit.removeQueuedDependency(doc);
|
||||
}
|
||||
@@ -88,22 +86,18 @@ export default class DependenceService {
|
||||
);
|
||||
const unInstalledDeps = docs.filter((x) => !installedDeps.includes(x));
|
||||
await this.removeDb(unInstalledDeps.map((x) => x.id!));
|
||||
|
||||
if (installedDeps.length) {
|
||||
await DependenceModel.update(
|
||||
await DependenceModel.nativeUpdate(
|
||||
{ id: installedDeps.map((x) => x.id!) },
|
||||
{ status: DependenceStatus.queued, log: [] },
|
||||
{ where: { id: installedDeps.map((x) => x.id!) } },
|
||||
);
|
||||
|
||||
this.installDependenceOneByOne(installedDeps, false, force);
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
|
||||
public async removeDb(ids: number[]) {
|
||||
await DependenceModel.destroy({ where: { id: ids } });
|
||||
await DependenceModel.nativeDelete({ id: ids });
|
||||
}
|
||||
|
||||
public async dependencies(
|
||||
{
|
||||
searchValue,
|
||||
@@ -130,13 +124,7 @@ export default class DependenceService {
|
||||
condition.status = status.split(',').map(Number);
|
||||
}
|
||||
if (searchValue) {
|
||||
const encodeText = encodeURI(searchValue);
|
||||
condition.name = {
|
||||
[Op.or]: [
|
||||
{ [Op.like]: `%${searchValue}%` },
|
||||
{ [Op.like]: `%${encodeText}%` },
|
||||
],
|
||||
};
|
||||
Object.assign(condition, textSearch(['name'], searchValue));
|
||||
}
|
||||
try {
|
||||
return await this.find(condition, sort);
|
||||
@@ -144,7 +132,6 @@ export default class DependenceService {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async refreshInstalledStatuses(type?: DependenceTypes) {
|
||||
const cacheDependenceTypes = [
|
||||
DependenceTypes.nodejs,
|
||||
@@ -153,13 +140,13 @@ export default class DependenceService {
|
||||
if (type !== undefined && !cacheDependenceTypes.includes(type)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const docs = await DependenceModel.findAll({
|
||||
where: {
|
||||
const docs = await DependenceModel.find(
|
||||
{
|
||||
status: DependenceStatus.installed,
|
||||
type: type === undefined ? { [Op.in]: cacheDependenceTypes } : type,
|
||||
type: type === undefined ? { ['$in']: cacheDependenceTypes } : type,
|
||||
},
|
||||
});
|
||||
{},
|
||||
);
|
||||
const checks = await concurrentRun(
|
||||
docs
|
||||
.filter(
|
||||
@@ -175,15 +162,13 @@ export default class DependenceService {
|
||||
const missingIds = (checks || []).filter(
|
||||
(id): id is number => id !== undefined,
|
||||
);
|
||||
|
||||
if (missingIds.length) {
|
||||
await DependenceModel.update(
|
||||
await DependenceModel.nativeUpdate(
|
||||
{ id: missingIds },
|
||||
{ status: DependenceStatus.installFailed },
|
||||
{ where: { id: missingIds } },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async isDependenceInstalled(dependency: Dependence) {
|
||||
let depName = dependency.name.trim();
|
||||
const depVersionStr = versionDependenceCommandTypes[dependency.type];
|
||||
@@ -201,7 +186,6 @@ export default class DependenceService {
|
||||
depVersion = parsedVersion;
|
||||
}
|
||||
}
|
||||
|
||||
const depInfo = (
|
||||
await promiseExecSuccess(getGetCommand(dependency.type, depName))
|
||||
)
|
||||
@@ -211,12 +195,10 @@ export default class DependenceService {
|
||||
(dependency.type === DependenceTypes.nodejs &&
|
||||
depInfo.split(' ')?.[0] === depName) ||
|
||||
dependency.type === DependenceTypes.python3;
|
||||
|
||||
return Boolean(
|
||||
depInfo && nameMatches && (!depVersion || depInfo.includes(depVersion)),
|
||||
);
|
||||
}
|
||||
|
||||
public installDependenceOneByOne(
|
||||
docs: Dependence[],
|
||||
isInstall: boolean = true,
|
||||
@@ -225,32 +207,27 @@ export default class DependenceService {
|
||||
docs.forEach((dep) => {
|
||||
this.installOrUninstallDependency(dep, isInstall, force);
|
||||
});
|
||||
|
||||
return taskLimit.waitDependencyQueueDone();
|
||||
}
|
||||
|
||||
public async reInstall(ids: number[]): Promise<Dependence[]> {
|
||||
await DependenceModel.update(
|
||||
await DependenceModel.nativeUpdate(
|
||||
{ id: ids },
|
||||
{ status: DependenceStatus.queued, log: [] },
|
||||
{ where: { id: ids } },
|
||||
);
|
||||
|
||||
const docs = await DependenceModel.findAll({ where: { id: ids } });
|
||||
const docs = await DependenceModel.find({ id: ids }, {});
|
||||
for (const doc of docs) {
|
||||
taskLimit.removeQueuedDependency(doc);
|
||||
}
|
||||
this.installDependenceOneByOne(docs, true, true);
|
||||
return docs;
|
||||
}
|
||||
|
||||
public async cancel(ids: number[]) {
|
||||
const docs = await DependenceModel.findAll({ where: { id: ids } });
|
||||
const docs = await DependenceModel.find({ id: ids }, {});
|
||||
for (const doc of docs) {
|
||||
taskLimit.removeQueuedDependency(doc);
|
||||
let depInstallCommand = getInstallCommand(doc.type, doc.name);
|
||||
let depUnInstallCommand = getUninstallCommand(doc.type, doc.name);
|
||||
const isLinuxDependence = doc.type === DependenceTypes.linux;
|
||||
|
||||
if (isLinuxDependence) {
|
||||
const osType = await detectOS();
|
||||
if (!osType) {
|
||||
@@ -268,44 +245,37 @@ export default class DependenceService {
|
||||
pid && (await killTask(pid));
|
||||
}
|
||||
}
|
||||
await DependenceModel.update(
|
||||
await DependenceModel.nativeUpdate(
|
||||
{ id: ids },
|
||||
{ status: DependenceStatus.cancelled },
|
||||
{ where: { id: ids } },
|
||||
);
|
||||
}
|
||||
|
||||
private async find(query: any, sort: any = []): Promise<Dependence[]> {
|
||||
const docs = await DependenceModel.findAll({
|
||||
where: { ...query },
|
||||
order: [...sort, ['createdAt', 'DESC']],
|
||||
});
|
||||
const docs = await DependenceModel.find(
|
||||
{ ...query },
|
||||
{
|
||||
orderBy: orderBy([...sort, ['createdAt', 'DESC']]),
|
||||
},
|
||||
);
|
||||
return docs;
|
||||
}
|
||||
|
||||
public async getDb(
|
||||
query: FindOptions<Dependence>['where'],
|
||||
): Promise<Dependence> {
|
||||
const doc: any = await DependenceModel.findOne({ where: { ...query } });
|
||||
public async getDb(query: QuerySpec['where']): Promise<Dependence> {
|
||||
const doc: any = await DependenceModel.findOne({ ...query }, {});
|
||||
if (!doc) {
|
||||
throw new Error(`Dependency ${JSON.stringify(query)} not found`);
|
||||
}
|
||||
return doc.get({ plain: true });
|
||||
return doc;
|
||||
}
|
||||
|
||||
private async updateLog(ids: number[], log: string): Promise<void> {
|
||||
taskLimit.updateDepLog(async () => {
|
||||
const docs = await DependenceModel.findAll({ where: { id: ids } });
|
||||
const docs = await DependenceModel.find({ id: ids }, {});
|
||||
for (const doc of docs) {
|
||||
const newLog = doc?.log ? [...doc.log, log] : [log];
|
||||
await DependenceModel.update(
|
||||
{ log: newLog },
|
||||
{ where: { id: doc.id } },
|
||||
);
|
||||
await DependenceModel.nativeUpdate({ id: doc.id }, { log: newLog });
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
public installOrUninstallDependency(
|
||||
dependency: Dependence,
|
||||
isInstall: boolean = true,
|
||||
@@ -322,14 +292,14 @@ export default class DependenceService {
|
||||
if (!isInstall && force) {
|
||||
return this.removeDb([dependency.id!]).then(() => null);
|
||||
}
|
||||
return DependenceModel.update(
|
||||
return DependenceModel.nativeUpdate(
|
||||
{ id: dependency.id },
|
||||
{
|
||||
status: isInstall
|
||||
? DependenceStatus.installFailed
|
||||
: DependenceStatus.removeFailed,
|
||||
log: [String(error)],
|
||||
},
|
||||
{ where: { id: dependency.id } },
|
||||
).then(() => null);
|
||||
}
|
||||
return new Promise(async (resolve) => {
|
||||
@@ -350,9 +320,9 @@ export default class DependenceService {
|
||||
taskLimit.removeQueuedDependency(dependency);
|
||||
if (isLinuxDependence) {
|
||||
if (!osType) {
|
||||
await DependenceModel.update(
|
||||
await DependenceModel.nativeUpdate(
|
||||
{ id: depIds },
|
||||
{ status: DependenceStatus.installFailed },
|
||||
{ where: { id: depIds } },
|
||||
);
|
||||
const startTime = dayjs();
|
||||
const message = tf(
|
||||
@@ -375,12 +345,10 @@ export default class DependenceService {
|
||||
}
|
||||
linuxCommand = LINUX_DEPENDENCE_COMMAND[osType];
|
||||
}
|
||||
|
||||
const status = isInstall
|
||||
? DependenceStatus.installing
|
||||
: DependenceStatus.removing;
|
||||
await DependenceModel.update({ status }, { where: { id: depIds } });
|
||||
|
||||
await DependenceModel.nativeUpdate({ id: depIds }, { status });
|
||||
let command = validatedCommand;
|
||||
if (isLinuxDependence) {
|
||||
command = isInstall
|
||||
@@ -388,7 +356,6 @@ export default class DependenceService {
|
||||
: `${linuxCommand.uninstall} ${depName.trim()}`;
|
||||
}
|
||||
const startTime = dayjs();
|
||||
|
||||
const message = tf(
|
||||
'开始%s依赖 %s,开始时间 %s\n\n',
|
||||
actionText,
|
||||
@@ -402,7 +369,6 @@ export default class DependenceService {
|
||||
status,
|
||||
});
|
||||
this.updateLog(depIds, message);
|
||||
|
||||
// 判断是否已经安装过依赖
|
||||
// pip must resolve optional dependencies even if the base distribution
|
||||
// is already present; its version alone cannot prove extras are ready.
|
||||
@@ -416,7 +382,6 @@ export default class DependenceService {
|
||||
if (isLinuxDependence) {
|
||||
getCommand = `${linuxCommand.info} ${depName}`;
|
||||
}
|
||||
|
||||
let depVersion = '';
|
||||
if (!isPythonDependence && depName.includes(depVersionStr)) {
|
||||
const symbolRegx = new RegExp(
|
||||
@@ -431,7 +396,6 @@ export default class DependenceService {
|
||||
const depInfo = (await promiseExecSuccess(getCommand))
|
||||
.replace(/\s{2,}/, ' ')
|
||||
.replace(/\s+$/, '');
|
||||
|
||||
if (
|
||||
depInfo &&
|
||||
((isNodeDependence && depInfo.split(' ')?.[0] === depName) ||
|
||||
@@ -456,9 +420,9 @@ export default class DependenceService {
|
||||
status: DependenceStatus.installed,
|
||||
});
|
||||
this.updateLog(depIds, _message);
|
||||
await DependenceModel.update(
|
||||
await DependenceModel.nativeUpdate(
|
||||
{ id: depIds },
|
||||
{ status: DependenceStatus.installed },
|
||||
{ where: { id: depIds } },
|
||||
);
|
||||
return resolve(null);
|
||||
}
|
||||
@@ -472,7 +436,6 @@ export default class DependenceService {
|
||||
const cp = spawn(`${proxyStr} ${command}`, {
|
||||
shell: '/bin/bash',
|
||||
});
|
||||
|
||||
cp.stdout.on('data', async (data) => {
|
||||
this.sockService.sendMessage({
|
||||
type: socketMessageType,
|
||||
@@ -482,7 +445,6 @@ export default class DependenceService {
|
||||
});
|
||||
this.updateLog(depIds, data.toString());
|
||||
});
|
||||
|
||||
cp.stderr.on('data', async (data) => {
|
||||
this.sockService.sendMessage({
|
||||
type: socketMessageType,
|
||||
@@ -492,7 +454,6 @@ export default class DependenceService {
|
||||
});
|
||||
this.updateLog(depIds, data.toString());
|
||||
});
|
||||
|
||||
cp.on('error', async (err) => {
|
||||
this.sockService.sendMessage({
|
||||
type: socketMessageType,
|
||||
@@ -502,12 +463,10 @@ export default class DependenceService {
|
||||
});
|
||||
this.updateLog(depIds, JSON.stringify(err));
|
||||
});
|
||||
|
||||
cp.on('exit', async (code) => {
|
||||
const endTime = dayjs();
|
||||
const isSucceed = code === 0;
|
||||
const resultText = isSucceed ? t('成功') : t('失败');
|
||||
|
||||
const message =
|
||||
'\n' +
|
||||
tf(
|
||||
@@ -531,24 +490,20 @@ export default class DependenceService {
|
||||
status: exitStatus,
|
||||
});
|
||||
this.updateLog(depIds, message);
|
||||
|
||||
const docs = await DependenceModel.findAll({ where: { id: depIds } });
|
||||
const docs = await DependenceModel.find({ id: depIds }, {});
|
||||
const _docIds = docs
|
||||
.filter((x) => x.status !== DependenceStatus.cancelled)
|
||||
.map((x) => x.id!);
|
||||
|
||||
if (_docIds.length > 0) {
|
||||
await DependenceModel.update(
|
||||
await DependenceModel.nativeUpdate(
|
||||
{ id: _docIds },
|
||||
{ status: exitStatus },
|
||||
{ where: { id: _docIds } },
|
||||
);
|
||||
}
|
||||
|
||||
// 如果删除依赖成功或者强制删除
|
||||
if ((isSucceed || force) && !isInstall) {
|
||||
this.removeDb(depIds);
|
||||
}
|
||||
|
||||
resolve(null);
|
||||
});
|
||||
});
|
||||
|
||||
+41
-85
@@ -1,5 +1,6 @@
|
||||
import { textSearch } from '../data/queries';
|
||||
import { Query, QuerySpec, orderBy } from '../data/queries';
|
||||
import groupBy from 'lodash/groupBy';
|
||||
import { FindOptions, Op } from 'sequelize';
|
||||
import { Inject, Service } from 'typedi';
|
||||
import winston from 'winston';
|
||||
import config from '../config';
|
||||
@@ -13,12 +14,13 @@ import {
|
||||
stepPosition,
|
||||
} from '../data/env';
|
||||
import { writeFileWithLock } from '../shared/utils';
|
||||
import { sequelize } from '../data';
|
||||
|
||||
import { database } from '../data';
|
||||
@Service()
|
||||
export default class EnvService {
|
||||
constructor(@Inject('logger') private logger: winston.Logger) { }
|
||||
|
||||
constructor(
|
||||
@Inject('logger')
|
||||
private logger: winston.Logger,
|
||||
) {}
|
||||
public async create(payloads: Env[]): Promise<Env[]> {
|
||||
const envs = await this.envs();
|
||||
let position = initPosition;
|
||||
@@ -39,16 +41,14 @@ export default class EnvService {
|
||||
await this.checkPosition(tabs[tabs.length - 1].position!);
|
||||
return docs;
|
||||
}
|
||||
|
||||
public async insert(payloads: Env[]): Promise<Env[]> {
|
||||
const result: Env[] = [];
|
||||
for (const env of payloads) {
|
||||
const doc = await EnvModel.create(env, { returning: true });
|
||||
const doc = await EnvModel.insert(env);
|
||||
result.push(doc);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public async update(payload: Env): Promise<Env> {
|
||||
const doc = await this.getDb({ id: payload.id });
|
||||
const tab = new Env({ ...doc, ...payload });
|
||||
@@ -56,17 +56,14 @@ export default class EnvService {
|
||||
await this.set_envs();
|
||||
return newDoc;
|
||||
}
|
||||
|
||||
private async updateDb(payload: Env): Promise<Env> {
|
||||
await EnvModel.update({ ...payload }, { where: { id: payload.id } });
|
||||
await EnvModel.nativeUpdate({ id: payload.id }, { ...payload });
|
||||
return await this.getDb({ id: payload.id });
|
||||
}
|
||||
|
||||
public async remove(ids: number[]) {
|
||||
await EnvModel.destroy({ where: { id: ids } });
|
||||
await EnvModel.nativeDelete({ id: ids });
|
||||
await this.set_envs();
|
||||
}
|
||||
|
||||
public async move(
|
||||
id: number,
|
||||
{
|
||||
@@ -89,16 +86,13 @@ export default class EnvService {
|
||||
? (envs[toIndex].position! + envs[toIndex - 1].position!) / 2
|
||||
: (envs[toIndex].position! + envs[toIndex + 1].position!) / 2;
|
||||
}
|
||||
|
||||
const newDoc = await this.update({
|
||||
id,
|
||||
position: this.getPrecisionPosition(targetPosition),
|
||||
});
|
||||
|
||||
await this.checkPosition(targetPosition, envs[toIndex].position!);
|
||||
return newDoc;
|
||||
}
|
||||
|
||||
private async checkPosition(position: number, edge: number = 0) {
|
||||
const precisionPosition = this.getPrecisionPosition(position);
|
||||
if (
|
||||
@@ -114,40 +108,20 @@ export default class EnvService {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private getPrecisionPosition(position: number): number {
|
||||
return Math.trunc(parseFloat(position.toPrecision(16)));
|
||||
}
|
||||
|
||||
public async envs(searchText: string = '', query: any = {}): Promise<Env[]> {
|
||||
let condition = { ...query };
|
||||
if (searchText) {
|
||||
const encodeText = encodeURI(searchText);
|
||||
const reg = {
|
||||
[Op.or]: [
|
||||
{ [Op.like]: `%${searchText}%` },
|
||||
{ [Op.like]: `%${encodeText}%` },
|
||||
],
|
||||
};
|
||||
|
||||
condition = {
|
||||
...condition,
|
||||
[Op.or]: [
|
||||
{
|
||||
name: reg,
|
||||
},
|
||||
{
|
||||
value: reg,
|
||||
},
|
||||
{
|
||||
remarks: reg,
|
||||
},
|
||||
],
|
||||
...textSearch(['name', 'value', 'remarks'], searchText),
|
||||
};
|
||||
}
|
||||
try {
|
||||
const result = await this.find(condition, [
|
||||
[sequelize.literal('COALESCE(`isPinned`, 0)'), 'DESC'],
|
||||
['isPinned', 'DESC'],
|
||||
['position', 'DESC'],
|
||||
['createdAt', 'ASC'],
|
||||
]);
|
||||
@@ -156,90 +130,73 @@ export default class EnvService {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async find(query: any, sort: any = []): Promise<Env[]> {
|
||||
const docs = await EnvModel.findAll({
|
||||
where: { ...query },
|
||||
order: [...sort],
|
||||
});
|
||||
return docs.map((x) => x.get({ plain: true }));
|
||||
const docs = await EnvModel.find(
|
||||
{ ...query },
|
||||
{
|
||||
orderBy: orderBy([...sort]),
|
||||
},
|
||||
);
|
||||
return docs.map((x) => x);
|
||||
}
|
||||
|
||||
public async getDb(query: FindOptions<Env>['where']): Promise<Env> {
|
||||
const doc: any = await EnvModel.findOne({ where: { ...query } });
|
||||
public async getDb(query: QuerySpec['where']): Promise<Env> {
|
||||
const doc: any = await EnvModel.findOne({ ...query }, {});
|
||||
if (!doc) {
|
||||
throw new Error(`Env ${JSON.stringify(query)} not found`);
|
||||
}
|
||||
return doc.get({ plain: true });
|
||||
return doc;
|
||||
}
|
||||
|
||||
public async disabled(ids: number[]) {
|
||||
await EnvModel.update(
|
||||
{ status: EnvStatus.disabled },
|
||||
{ where: { id: ids } },
|
||||
);
|
||||
await EnvModel.nativeUpdate({ id: ids }, { status: EnvStatus.disabled });
|
||||
await this.set_envs();
|
||||
}
|
||||
|
||||
public async enabled(ids: number[]) {
|
||||
await EnvModel.update({ status: EnvStatus.normal }, { where: { id: ids } });
|
||||
await EnvModel.nativeUpdate({ id: ids }, { status: EnvStatus.normal });
|
||||
await this.set_envs();
|
||||
}
|
||||
|
||||
public async updateNames({ ids, name }: { ids: number[]; name: string }) {
|
||||
await EnvModel.update({ name }, { where: { id: ids } });
|
||||
await EnvModel.nativeUpdate({ id: ids }, { name });
|
||||
await this.set_envs();
|
||||
}
|
||||
|
||||
public async pin(ids: number[]) {
|
||||
await EnvModel.update({ isPinned: 1 }, { where: { id: ids } });
|
||||
await EnvModel.nativeUpdate({ id: ids }, { isPinned: 1 });
|
||||
}
|
||||
|
||||
public async unPin(ids: number[]) {
|
||||
await EnvModel.update({ isPinned: 0 }, { where: { id: ids } });
|
||||
await EnvModel.nativeUpdate({ id: ids }, { isPinned: 0 });
|
||||
}
|
||||
|
||||
public async addLabels(ids: number[], labels: string[]) {
|
||||
await sequelize.transaction(async (transaction) => {
|
||||
const docs = await EnvModel.findAll({
|
||||
where: { id: ids },
|
||||
transaction,
|
||||
});
|
||||
await database.transaction(async () => {
|
||||
const docs = await EnvModel.find({ id: ids }, {});
|
||||
for (const doc of docs) {
|
||||
const env = doc.get({ plain: true });
|
||||
await EnvModel.update(
|
||||
const env = doc;
|
||||
await EnvModel.nativeUpdate(
|
||||
{ id: env.id },
|
||||
{ labels: Array.from(new Set([...(env.labels || []), ...labels])) },
|
||||
{ where: { id: env.id }, transaction },
|
||||
);
|
||||
}
|
||||
});
|
||||
return await this.find({ id: ids });
|
||||
}
|
||||
|
||||
public async removeLabels(ids: number[], labels: string[]) {
|
||||
await sequelize.transaction(async (transaction) => {
|
||||
const docs = await EnvModel.findAll({
|
||||
where: { id: ids },
|
||||
transaction,
|
||||
});
|
||||
await database.transaction(async () => {
|
||||
const docs = await EnvModel.find({ id: ids }, {});
|
||||
for (const doc of docs) {
|
||||
const env = doc.get({ plain: true });
|
||||
await EnvModel.update(
|
||||
const env = doc;
|
||||
await EnvModel.nativeUpdate(
|
||||
{ id: env.id },
|
||||
{
|
||||
labels: (env.labels || []).filter(
|
||||
(label: string) => !labels.includes(label),
|
||||
),
|
||||
},
|
||||
{ where: { id: env.id }, transaction },
|
||||
);
|
||||
}
|
||||
});
|
||||
return await this.find({ id: ids });
|
||||
}
|
||||
|
||||
public async set_envs() {
|
||||
const envs = await this.envs('', {
|
||||
name: { [Op.not]: null },
|
||||
name: { ['$ne']: null },
|
||||
status: EnvStatus.normal,
|
||||
});
|
||||
const groups = groupBy(envs, 'name');
|
||||
@@ -249,7 +206,6 @@ export default class EnvService {
|
||||
for (const key in groups) {
|
||||
if (Object.prototype.hasOwnProperty.call(groups, key)) {
|
||||
const group = groups[key];
|
||||
|
||||
// 忽略不符合bash要求的环境变量名称
|
||||
if (/^[a-zA-Z_][0-9a-zA-Z_]*$/.test(key)) {
|
||||
let value = group
|
||||
@@ -262,9 +218,9 @@ export default class EnvService {
|
||||
.map((x) => x.value)
|
||||
.join('&')
|
||||
.replace(/\\/g, '\\\\')}`;
|
||||
js_env_string += `process.env[${JSON.stringify(key)}]=${JSON.stringify(
|
||||
group.map((x) => x.value).join('&'),
|
||||
)};\n`;
|
||||
js_env_string += `process.env[${JSON.stringify(
|
||||
key,
|
||||
)}]=${JSON.stringify(group.map((x) => x.value).join('&'))};\n`;
|
||||
py_env_string += `os.environ['${key}']='''${_env_value.replace(
|
||||
/\'/g,
|
||||
"\\'",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { database } from '../data';
|
||||
import { Service } from 'typedi';
|
||||
import Logger from '../loaders/logger';
|
||||
import cronClient from '../schedule/client';
|
||||
@@ -10,6 +11,7 @@ interface HealthStatus {
|
||||
services: {
|
||||
http: boolean;
|
||||
grpc: boolean;
|
||||
database: boolean;
|
||||
};
|
||||
metrics: {
|
||||
uptime: number;
|
||||
@@ -33,6 +35,7 @@ export class HealthService {
|
||||
services: {
|
||||
http: true,
|
||||
grpc: true,
|
||||
database: database.healthy,
|
||||
},
|
||||
metrics: {
|
||||
uptime: Math.floor((Date.now() - this.startTime) / 1000),
|
||||
@@ -66,6 +69,7 @@ export class HealthService {
|
||||
Logger.error('gRPC server check failed:', err);
|
||||
}
|
||||
|
||||
if (!status.services.database) status.status = 'error';
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
+21
-49
@@ -1,9 +1,9 @@
|
||||
import { textSearch } from '../data/queries';
|
||||
import { Service, Inject } from 'typedi';
|
||||
import winston from 'winston';
|
||||
import { createRandomString } from '../config/util';
|
||||
import { App, AppModel } from '../data/open';
|
||||
import { v4 as uuidV4 } from 'uuid';
|
||||
import sequelize, { Op } from 'sequelize';
|
||||
import { shareStore } from '../shared/store';
|
||||
import { t } from '../shared/i18n';
|
||||
import { lock } from 'proper-lockfile';
|
||||
@@ -11,18 +11,19 @@ import path from 'path';
|
||||
import os from 'os';
|
||||
import { writeFile, rename, rm, stat, chmod } from 'fs/promises';
|
||||
import config from '../config';
|
||||
|
||||
@Service()
|
||||
export default class OpenService {
|
||||
constructor(@Inject('logger') private logger: winston.Logger) {}
|
||||
|
||||
constructor(
|
||||
@Inject('logger')
|
||||
private logger: winston.Logger,
|
||||
) {}
|
||||
// Every database mutation and full authentication-cache publication shares
|
||||
// this cross-process lock. Lock a separate marker, not token.json itself.
|
||||
private async withAppsLock<T>(operation: () => Promise<T>): Promise<T> {
|
||||
const target = path.join(config.configPath, '.apps-state');
|
||||
const release = await lock(target, {
|
||||
realpath: false,
|
||||
stale: 10_000,
|
||||
stale: 10000,
|
||||
retries: { retries: 10, factor: 2, minTimeout: 100, maxTimeout: 3000 },
|
||||
});
|
||||
try {
|
||||
@@ -31,11 +32,10 @@ export default class OpenService {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
public async initializeSystemApp(): Promise<void> {
|
||||
await this.withAppsLock(async () => {
|
||||
const doc = await AppModel.findOne({ where: { name: 'system' } });
|
||||
const app = doc?.get({ plain: true });
|
||||
const doc = await AppModel.findOne({ name: 'system' }, {});
|
||||
const app = doc;
|
||||
if (!app) {
|
||||
await this.insert({
|
||||
name: 'system',
|
||||
@@ -44,26 +44,23 @@ export default class OpenService {
|
||||
client_secret: createRandomString(24, 24),
|
||||
});
|
||||
} else if (!app.scopes.includes('dashboard')) {
|
||||
await AppModel.update(
|
||||
await AppModel.nativeUpdate(
|
||||
{ id: app.id },
|
||||
{ scopes: [...app.scopes, 'dashboard'] },
|
||||
{ where: { id: app.id } },
|
||||
);
|
||||
}
|
||||
await shareStore.updateApps(await this.find({}));
|
||||
});
|
||||
}
|
||||
|
||||
public async refreshApps(): Promise<void> {
|
||||
await this.withAppsLock(async () => {
|
||||
await shareStore.updateApps(await this.find({}));
|
||||
});
|
||||
}
|
||||
|
||||
public async findApps(): Promise<App[] | null> {
|
||||
const docs = await this.find({});
|
||||
return docs;
|
||||
}
|
||||
|
||||
public async create(payload: App): Promise<App> {
|
||||
return this.withAppsLock(async () => {
|
||||
const tab = { ...payload };
|
||||
@@ -75,12 +72,10 @@ export default class OpenService {
|
||||
return { ...doc, tokens: [] };
|
||||
});
|
||||
}
|
||||
|
||||
private async insert(payload: App): Promise<App> {
|
||||
const doc = await AppModel.create(payload, { returning: true });
|
||||
return doc.get({ plain: true });
|
||||
const doc = await AppModel.insert(payload);
|
||||
return doc;
|
||||
}
|
||||
|
||||
public async update(payload: App): Promise<App> {
|
||||
const newDoc = await this.updateDb({
|
||||
name: payload.name,
|
||||
@@ -89,32 +84,28 @@ export default class OpenService {
|
||||
} as App);
|
||||
return { ...newDoc, tokens: [] };
|
||||
}
|
||||
|
||||
private async updateDb(payload: Partial<App>): Promise<App> {
|
||||
return this.withAppsLock(async () => {
|
||||
await AppModel.update(payload, { where: { id: payload.id } });
|
||||
await AppModel.nativeUpdate({ id: payload.id }, payload);
|
||||
const apps = await this.find({});
|
||||
await shareStore.updateApps(apps);
|
||||
return apps?.find((x) => x.id === payload.id) as App;
|
||||
});
|
||||
}
|
||||
|
||||
public async getDb(query: Record<string, any>): Promise<App> {
|
||||
const doc = await AppModel.findOne({ where: query });
|
||||
const doc = await AppModel.findOne(query, {});
|
||||
if (!doc) {
|
||||
throw new Error(`App ${JSON.stringify(query)} not found`);
|
||||
}
|
||||
return doc.get({ plain: true });
|
||||
return doc;
|
||||
}
|
||||
|
||||
public async remove(ids: number[]) {
|
||||
return this.withAppsLock(async () => {
|
||||
await AppModel.destroy({ where: { id: ids } });
|
||||
await AppModel.nativeDelete({ id: ids });
|
||||
const apps = await this.find({});
|
||||
await shareStore.updateApps(apps);
|
||||
});
|
||||
}
|
||||
|
||||
public async resetSecret(id: number): Promise<App> {
|
||||
const tab: Partial<App> = {
|
||||
client_secret: createRandomString(24, 24),
|
||||
@@ -130,7 +121,6 @@ export default class OpenService {
|
||||
const newDoc = await this.updateDb(tab);
|
||||
return newDoc;
|
||||
}
|
||||
|
||||
public async list(
|
||||
searchText: string = '',
|
||||
sort: any = {},
|
||||
@@ -138,18 +128,7 @@ export default class OpenService {
|
||||
): Promise<App[]> {
|
||||
let condition = { ...query };
|
||||
if (searchText) {
|
||||
const encodeText = encodeURI(searchText);
|
||||
const reg = {
|
||||
[Op.or]: [
|
||||
{ [Op.like]: `%${searchText}%` },
|
||||
{ [Op.like]: `%${encodeText}%` },
|
||||
],
|
||||
};
|
||||
|
||||
condition = {
|
||||
...condition,
|
||||
name: reg,
|
||||
};
|
||||
condition = { ...condition, ...textSearch(['name'], searchText) };
|
||||
}
|
||||
try {
|
||||
const result = await this.find(condition);
|
||||
@@ -160,19 +139,16 @@ export default class OpenService {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async find(query: Record<string, any>, sort?: any): Promise<App[]> {
|
||||
const docs = await AppModel.findAll({ where: { ...query } });
|
||||
return docs.map((x) => x.get({ plain: true }));
|
||||
const docs = await AppModel.find({ ...query }, {});
|
||||
return docs.map((x) => x);
|
||||
}
|
||||
|
||||
public async authToken(credentials: {
|
||||
client_id: string;
|
||||
client_secret: string;
|
||||
}): Promise<any> {
|
||||
return this.withAppsLock(() => this.issueToken(credentials));
|
||||
}
|
||||
|
||||
// Caller holds withAppsLock through database update and cache publication.
|
||||
private async issueToken({
|
||||
client_id,
|
||||
@@ -183,7 +159,7 @@ export default class OpenService {
|
||||
}): Promise<any> {
|
||||
let token = uuidV4();
|
||||
const expiration = Math.round(Date.now() / 1000) + 2592000; // 2592000 30天
|
||||
const doc = await AppModel.findOne({ where: { client_id, client_secret } });
|
||||
const doc = await AppModel.findOne({ client_id, client_secret }, {});
|
||||
if (doc) {
|
||||
const timestamp = Math.round(Date.now() / 1000);
|
||||
const invalidTokens = (doc.tokens || []).filter(
|
||||
@@ -199,10 +175,7 @@ export default class OpenService {
|
||||
} else {
|
||||
tokens = [...invalidTokens, { value: token, expiration }];
|
||||
}
|
||||
await AppModel.update(
|
||||
{ tokens },
|
||||
{ where: { client_id, client_secret } },
|
||||
);
|
||||
await AppModel.nativeUpdate({ client_id, client_secret }, { tokens });
|
||||
const apps = await this.find({});
|
||||
await shareStore.updateApps(apps);
|
||||
return {
|
||||
@@ -217,7 +190,6 @@ export default class OpenService {
|
||||
return { code: 400, message: t('client_id 或 client_seret 有误') };
|
||||
}
|
||||
}
|
||||
|
||||
public async generateSystemToken(renew = false): Promise<{
|
||||
value: string;
|
||||
expiration: number;
|
||||
|
||||
+25
-41
@@ -1,11 +1,10 @@
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import dayjs from 'dayjs';
|
||||
import { Op } from 'sequelize';
|
||||
import { Service } from 'typedi';
|
||||
import config from '../config';
|
||||
import { CrontabStatModel } from '../data/cronStats';
|
||||
import { sequelize } from '../data';
|
||||
import { database } from '../data';
|
||||
import { InstanceStatus, RunningInstanceModel } from '../data/runningInstance';
|
||||
import { AuthDataType, SystemModel } from '../data/system';
|
||||
import {
|
||||
@@ -15,54 +14,50 @@ import {
|
||||
normalizeRetentionPolicy,
|
||||
RetentionPolicy,
|
||||
} from '../shared/retention';
|
||||
|
||||
export interface StorageCleanupRequest extends RetentionPolicy {
|
||||
dependenceCacheTypes?: DependenceCacheType[];
|
||||
compactDatabase?: boolean;
|
||||
}
|
||||
|
||||
function runningInstanceWhere(days: number) {
|
||||
const cutoff = dayjs().subtract(days, 'day').unix();
|
||||
return {
|
||||
status: { [Op.ne]: InstanceStatus.running },
|
||||
[Op.or]: [
|
||||
{ finished_at: { [Op.lt]: cutoff } },
|
||||
status: { ['$ne']: InstanceStatus.running },
|
||||
['$or']: [
|
||||
{ finished_at: { ['$lt']: cutoff } },
|
||||
{
|
||||
finished_at: { [Op.is]: null },
|
||||
started_at: { [Op.lt]: cutoff },
|
||||
finished_at: { ['$eq']: null },
|
||||
started_at: { ['$lt']: cutoff },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function cronStatWhere(days: number) {
|
||||
return {
|
||||
date: { [Op.lt]: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
|
||||
date: { ['$lt']: dayjs().subtract(days, 'day').format('YYYY-MM-DD') },
|
||||
};
|
||||
}
|
||||
|
||||
@Service()
|
||||
export default class RetentionService {
|
||||
public async updatePolicy(policy: Partial<RetentionPolicy>) {
|
||||
const normalized = normalizeRetentionPolicy(policy);
|
||||
const systemConfig = await SystemModel.findOne({
|
||||
where: { type: AuthDataType.systemConfig },
|
||||
});
|
||||
const systemConfig = await SystemModel.findOne(
|
||||
{ type: AuthDataType.systemConfig },
|
||||
{},
|
||||
);
|
||||
if (!systemConfig) {
|
||||
throw new Error('System config not found');
|
||||
}
|
||||
await SystemModel.update(
|
||||
await SystemModel.nativeUpdate(
|
||||
{ id: systemConfig.id },
|
||||
{
|
||||
info: {
|
||||
...systemConfig.info,
|
||||
...normalized,
|
||||
},
|
||||
},
|
||||
{ where: { id: systemConfig.id } },
|
||||
);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
public async preview(request: StorageCleanupRequest) {
|
||||
const policy = normalizeRetentionPolicy(request);
|
||||
const dependenceCacheTypes = (request.dependenceCacheTypes || []).filter(
|
||||
@@ -70,14 +65,12 @@ export default class RetentionService {
|
||||
);
|
||||
const [runningInstances, cronStats, dependenceCaches] = await Promise.all([
|
||||
policy.runningInstanceRetentionDays > 0
|
||||
? RunningInstanceModel.count({
|
||||
where: runningInstanceWhere(policy.runningInstanceRetentionDays),
|
||||
})
|
||||
? RunningInstanceModel.count(
|
||||
runningInstanceWhere(policy.runningInstanceRetentionDays),
|
||||
)
|
||||
: 0,
|
||||
policy.cronStatRetentionDays > 0
|
||||
? CrontabStatModel.count({
|
||||
where: cronStatWhere(policy.cronStatRetentionDays),
|
||||
})
|
||||
? CrontabStatModel.count(cronStatWhere(policy.cronStatRetentionDays))
|
||||
: 0,
|
||||
Promise.all(
|
||||
dependenceCacheTypes.map(async (type) => ({
|
||||
@@ -88,7 +81,6 @@ export default class RetentionService {
|
||||
})),
|
||||
),
|
||||
]);
|
||||
|
||||
return {
|
||||
policy,
|
||||
runningInstances,
|
||||
@@ -101,29 +93,23 @@ export default class RetentionService {
|
||||
compactDatabase: Boolean(request.compactDatabase),
|
||||
};
|
||||
}
|
||||
|
||||
public async cleanup(request: StorageCleanupRequest) {
|
||||
const preview = await this.preview(request);
|
||||
const deleted = await sequelize.transaction(async (transaction) => {
|
||||
const deleted = await database.transaction(async () => {
|
||||
const runningInstances =
|
||||
preview.policy.runningInstanceRetentionDays > 0
|
||||
? await RunningInstanceModel.destroy({
|
||||
where: runningInstanceWhere(
|
||||
preview.policy.runningInstanceRetentionDays,
|
||||
),
|
||||
transaction,
|
||||
})
|
||||
? await RunningInstanceModel.nativeDelete(
|
||||
runningInstanceWhere(preview.policy.runningInstanceRetentionDays),
|
||||
)
|
||||
: 0;
|
||||
const cronStats =
|
||||
preview.policy.cronStatRetentionDays > 0
|
||||
? await CrontabStatModel.destroy({
|
||||
where: cronStatWhere(preview.policy.cronStatRetentionDays),
|
||||
transaction,
|
||||
})
|
||||
? await CrontabStatModel.nativeDelete(
|
||||
cronStatWhere(preview.policy.cronStatRetentionDays),
|
||||
)
|
||||
: 0;
|
||||
return { runningInstances, cronStats };
|
||||
});
|
||||
|
||||
const dependenceCaches = [];
|
||||
for (const cache of preview.dependenceCaches) {
|
||||
await fs.rm(path.join(config.dependenceCachePath, cache.type), {
|
||||
@@ -132,14 +118,12 @@ export default class RetentionService {
|
||||
});
|
||||
dependenceCaches.push(cache);
|
||||
}
|
||||
|
||||
if (
|
||||
request.compactDatabase &&
|
||||
(deleted.runningInstances || deleted.cronStats)
|
||||
) {
|
||||
await sequelize.query('VACUUM');
|
||||
await database.query('VACUUM');
|
||||
}
|
||||
|
||||
return {
|
||||
deleted: {
|
||||
...deleted,
|
||||
|
||||
+70
-101
@@ -1,3 +1,5 @@
|
||||
import { textSearch } from '../data/queries';
|
||||
import { Query, QuerySpec, orderBy } from '../data/queries';
|
||||
import { Service, Inject } from 'typedi';
|
||||
import winston from 'winston';
|
||||
import config from '../config';
|
||||
@@ -19,7 +21,6 @@ import {
|
||||
rmPath,
|
||||
} from '../config/util';
|
||||
import fs from 'fs/promises';
|
||||
import { FindOptions, Op } from 'sequelize';
|
||||
import path from 'path';
|
||||
import ScheduleService, { TaskCallbacks } from './schedule';
|
||||
import { SimpleIntervalSchedule } from 'toad-scheduler';
|
||||
@@ -42,17 +43,16 @@ import CrontabService from './cron';
|
||||
import taskLimit from '../shared/pLimit';
|
||||
import { logStreamManager } from '../shared/logStreamManager';
|
||||
import { LogReadOptions, readLogChunk } from '../shared/logReader';
|
||||
|
||||
@Service()
|
||||
export default class SubscriptionService {
|
||||
constructor(
|
||||
@Inject('logger') private logger: winston.Logger,
|
||||
@Inject('logger')
|
||||
private logger: winston.Logger,
|
||||
private scheduleService: ScheduleService,
|
||||
private sockService: SockService,
|
||||
private sshKeyService: SshKeyService,
|
||||
private crontabService: CrontabService,
|
||||
) {}
|
||||
|
||||
public async list(
|
||||
searchText?: string,
|
||||
ids?: string,
|
||||
@@ -60,37 +60,23 @@ export default class SubscriptionService {
|
||||
let query = {};
|
||||
const subIds = JSON.parse(ids || '[]');
|
||||
if (searchText) {
|
||||
const reg = {
|
||||
[Op.or]: [
|
||||
{ [Op.like]: `%${searchText}%` },
|
||||
{ [Op.like]: `%${encodeURI(searchText)}%` },
|
||||
],
|
||||
};
|
||||
query = {
|
||||
[Op.or]: [
|
||||
{
|
||||
name: reg,
|
||||
},
|
||||
{
|
||||
url: reg,
|
||||
},
|
||||
],
|
||||
};
|
||||
query = textSearch(['name', 'url'], searchText);
|
||||
}
|
||||
try {
|
||||
const result = await SubscriptionModel.findAll({
|
||||
where: { ...query, ...(ids ? { id: subIds } : undefined) },
|
||||
order: [
|
||||
['is_disabled', 'ASC'],
|
||||
['createdAt', 'DESC'],
|
||||
],
|
||||
});
|
||||
const result = await SubscriptionModel.find(
|
||||
{ ...query, ...(ids ? { id: subIds } : undefined) },
|
||||
{
|
||||
orderBy: orderBy([
|
||||
['is_disabled', 'ASC'],
|
||||
['createdAt', 'DESC'],
|
||||
]),
|
||||
},
|
||||
);
|
||||
return result;
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public async handleTask(
|
||||
doc: Subscription,
|
||||
needCreate = true,
|
||||
@@ -101,7 +87,6 @@ export default class SubscriptionService {
|
||||
const { url } = formatUrl(doc);
|
||||
doc.command = formatCommand(doc, url as string);
|
||||
}
|
||||
|
||||
if (doc.schedule_type === 'crontab') {
|
||||
this.scheduleService.cancelCronTask(doc as any);
|
||||
needCreate &&
|
||||
@@ -122,34 +107,34 @@ export default class SubscriptionService {
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
public async setSshConfig() {
|
||||
const docs = await SubscriptionModel.findAll();
|
||||
const docs = await SubscriptionModel.find({}, {});
|
||||
await this.sshKeyService.setSshConfig(docs);
|
||||
}
|
||||
|
||||
private taskCallbacks(doc: Subscription): TaskCallbacks {
|
||||
return {
|
||||
onBefore: async (startTime) => {
|
||||
const logTime = startTime.format('YYYY-MM-DD-HH-mm-ss');
|
||||
const logPath = `${doc.alias}/${logTime}.log`;
|
||||
await SubscriptionModel.update(
|
||||
await SubscriptionModel.nativeUpdate(
|
||||
{ id: doc.id },
|
||||
{
|
||||
status: SubscriptionStatus.running,
|
||||
log_path: logPath,
|
||||
},
|
||||
{ where: { id: doc.id } },
|
||||
);
|
||||
const absolutePath = await handleLogPath(
|
||||
logPath as string,
|
||||
tf('## 开始执行... %s\n', startTime.format('YYYY-MM-DD HH:mm:ss')),
|
||||
);
|
||||
|
||||
// 执行sub_before
|
||||
let beforeStr = '';
|
||||
try {
|
||||
if (doc.sub_before) {
|
||||
await logStreamManager.write(absolutePath, `\n## ${t('执行before命令...')}\n\n`);
|
||||
await logStreamManager.write(
|
||||
absolutePath,
|
||||
`\n## ${t('执行before命令...')}\n\n`,
|
||||
);
|
||||
beforeStr = await promiseExec(doc.sub_before);
|
||||
}
|
||||
} catch (error: any) {
|
||||
@@ -161,11 +146,11 @@ export default class SubscriptionService {
|
||||
}
|
||||
},
|
||||
onStart: async (cp: ChildProcessWithoutNullStreams, startTime) => {
|
||||
await SubscriptionModel.update(
|
||||
await SubscriptionModel.nativeUpdate(
|
||||
{ id: doc.id },
|
||||
{
|
||||
pid: cp.pid,
|
||||
},
|
||||
{ where: { id: doc.id } },
|
||||
);
|
||||
},
|
||||
onEnd: async (cp, endTime, diff) => {
|
||||
@@ -173,7 +158,6 @@ export default class SubscriptionService {
|
||||
try {
|
||||
const sub = await this.getDb({ id: doc.id });
|
||||
absolutePath = await handleLogPath(sub.log_path as string);
|
||||
|
||||
// 执行 sub_after
|
||||
let afterStr = '';
|
||||
try {
|
||||
@@ -186,12 +170,12 @@ export default class SubscriptionService {
|
||||
}
|
||||
} catch (error: any) {
|
||||
afterStr =
|
||||
(error.stderr && error.stderr.toString()) || JSON.stringify(error);
|
||||
(error.stderr && error.stderr.toString()) ||
|
||||
JSON.stringify(error);
|
||||
}
|
||||
if (afterStr) {
|
||||
await logStreamManager.write(absolutePath, `${afterStr}\n`);
|
||||
}
|
||||
|
||||
await logStreamManager.write(
|
||||
absolutePath,
|
||||
'\n' +
|
||||
@@ -206,10 +190,10 @@ export default class SubscriptionService {
|
||||
try {
|
||||
if (absolutePath) await logStreamManager.closeStream(absolutePath);
|
||||
} finally {
|
||||
await SubscriptionModel.update(
|
||||
{ status: SubscriptionStatus.idle, pid: null } as any,
|
||||
{ where: { id: doc.id } },
|
||||
);
|
||||
await SubscriptionModel.nativeUpdate({ id: doc.id }, {
|
||||
status: SubscriptionStatus.idle,
|
||||
pid: null,
|
||||
} as any);
|
||||
this.sockService.sendMessage({
|
||||
type: 'runSubscriptionEnd',
|
||||
message: t('订阅执行完成'),
|
||||
@@ -230,22 +214,19 @@ export default class SubscriptionService {
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
public async create(payload: Subscription): Promise<Subscription> {
|
||||
assertSubscriptionAlias(payload.alias);
|
||||
return withSubscriptionMutation(async () => {
|
||||
const tab = new Subscription(payload);
|
||||
const doc = await this.insert(tab);
|
||||
await this.handleTask(doc.get({ plain: true }));
|
||||
await this.handleTask(doc);
|
||||
await this.setSshConfig();
|
||||
return doc;
|
||||
});
|
||||
}
|
||||
|
||||
public async insert(payload: Subscription): Promise<SubscriptionInstance> {
|
||||
return await SubscriptionModel.create(payload, { returning: true });
|
||||
return await SubscriptionModel.insert(payload);
|
||||
}
|
||||
|
||||
public async update(payload: Subscription): Promise<Subscription> {
|
||||
assertSubscriptionAlias(payload.alias);
|
||||
return withSubscriptionMutation(async () => {
|
||||
@@ -258,7 +239,6 @@ export default class SubscriptionService {
|
||||
return newDoc;
|
||||
});
|
||||
}
|
||||
|
||||
private async removeSshConfigForSubscription(doc: Subscription) {
|
||||
if (doc.type !== 'private-repo' || doc.pull_type !== 'ssh-key') return;
|
||||
// Invalid historical rows must remain deletable without deriving unsafe paths.
|
||||
@@ -269,12 +249,10 @@ export default class SubscriptionService {
|
||||
}
|
||||
await this.sshKeyService.removeSSHKey(doc.alias, '');
|
||||
}
|
||||
|
||||
public async updateDb(payload: Subscription): Promise<Subscription> {
|
||||
await SubscriptionModel.update(payload, { where: { id: payload.id } });
|
||||
await SubscriptionModel.nativeUpdate({ id: payload.id }, payload);
|
||||
return await this.getDb({ id: payload.id });
|
||||
}
|
||||
|
||||
public async status({
|
||||
ids,
|
||||
status,
|
||||
@@ -300,16 +278,16 @@ export default class SubscriptionService {
|
||||
if (last_running_time > 0) {
|
||||
options.last_running_time = last_running_time;
|
||||
}
|
||||
|
||||
return await SubscriptionModel.update(
|
||||
{ ...options },
|
||||
{ where: { id: ids } },
|
||||
);
|
||||
return await SubscriptionModel.nativeUpdate({ id: ids }, { ...options });
|
||||
}
|
||||
|
||||
public async remove(ids: number[], query: { force?: boolean }) {
|
||||
public async remove(
|
||||
ids: number[],
|
||||
query: {
|
||||
force?: boolean;
|
||||
},
|
||||
) {
|
||||
return withSubscriptionMutation(async () => {
|
||||
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
|
||||
const docs = await SubscriptionModel.find({ id: ids }, {});
|
||||
const storagePaths: string[] = [];
|
||||
const retainedPaths: string[] = [];
|
||||
const collectPaths = (
|
||||
@@ -321,7 +299,9 @@ export default class SubscriptionService {
|
||||
// A URL accepted by one engine can produce an invalid name in another.
|
||||
// Validate each candidate independently, retaining the safe candidate.
|
||||
if (!Object.values(names).every(isSubscriptionStorageName)) continue;
|
||||
const resolve = deleting ? resolveSubscriptionStoragePath : path.resolve;
|
||||
const resolve = deleting
|
||||
? resolveSubscriptionStoragePath
|
||||
: path.resolve;
|
||||
const scriptPath = resolve(config.scriptPath, names.script);
|
||||
if (scriptPath) targets.push(scriptPath);
|
||||
if (names.repo) {
|
||||
@@ -329,7 +309,10 @@ export default class SubscriptionService {
|
||||
if (repoPath) targets.push(repoPath);
|
||||
}
|
||||
if (names.raw) {
|
||||
const rawPath = resolve(path.join(config.dataPath, 'raw'), names.raw);
|
||||
const rawPath = resolve(
|
||||
path.join(config.dataPath, 'raw'),
|
||||
names.raw,
|
||||
);
|
||||
if (rawPath) targets.push(rawPath);
|
||||
}
|
||||
}
|
||||
@@ -342,70 +325,65 @@ export default class SubscriptionService {
|
||||
resolveSubscriptionStoragePath(config.repoPath, doc.alias);
|
||||
collectPaths(doc, storagePaths, true);
|
||||
}
|
||||
const remaining = await SubscriptionModel.findAll({ where: {} });
|
||||
const remaining = await SubscriptionModel.find({}, {});
|
||||
for (const doc of remaining) {
|
||||
if (!ids.includes(doc.id!)) collectPaths(doc, retainedPaths);
|
||||
}
|
||||
}
|
||||
for (const doc of docs) {
|
||||
await this.handleTask(doc.get({ plain: true }), false);
|
||||
await this.handleTask(doc, false);
|
||||
}
|
||||
await SubscriptionModel.destroy({ where: { id: ids } });
|
||||
await SubscriptionModel.nativeDelete({ id: ids });
|
||||
for (const doc of docs) {
|
||||
await this.removeSshConfigForSubscription(doc);
|
||||
}
|
||||
await this.setSshConfig();
|
||||
|
||||
if (query?.force === true) {
|
||||
const crons = await CrontabModel.findAll({ where: { sub_id: ids } });
|
||||
const crons = await CrontabModel.find({ sub_id: ids }, {});
|
||||
if (crons?.length) {
|
||||
await this.crontabService.remove(crons.map((x) => x.id!));
|
||||
}
|
||||
for (const storagePath of new Set(storagePaths)) {
|
||||
// rm is recursive: an ancestor must also be retained if a surviving
|
||||
// subscription uses a nested branch, and vice versa.
|
||||
const shared = retainedPaths.some((retained) =>
|
||||
retained === storagePath ||
|
||||
retained.startsWith(`${storagePath}${path.sep}`) ||
|
||||
storagePath.startsWith(`${retained}${path.sep}`),
|
||||
const shared = retainedPaths.some(
|
||||
(retained) =>
|
||||
retained === storagePath ||
|
||||
retained.startsWith(`${storagePath}${path.sep}`) ||
|
||||
storagePath.startsWith(`${retained}${path.sep}`),
|
||||
);
|
||||
if (!shared) await rmPath(storagePath);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public async getDb(
|
||||
query: FindOptions<Subscription>['where'],
|
||||
): Promise<Subscription> {
|
||||
const doc = await SubscriptionModel.findOne({ where: { ...query } });
|
||||
public async getDb(query: QuerySpec['where']): Promise<Subscription> {
|
||||
const doc = await SubscriptionModel.findOne({ ...query }, {});
|
||||
if (!doc) {
|
||||
throw new Error(`Subscription ${JSON.stringify(query)} not found`);
|
||||
}
|
||||
return doc.get({ plain: true });
|
||||
return doc;
|
||||
}
|
||||
|
||||
public async run(ids: number[]) {
|
||||
await SubscriptionModel.update(
|
||||
await SubscriptionModel.nativeUpdate(
|
||||
{ id: ids },
|
||||
{ status: SubscriptionStatus.queued },
|
||||
{ where: { id: ids } },
|
||||
);
|
||||
ids.forEach((id) => {
|
||||
this.runSingle(id);
|
||||
});
|
||||
}
|
||||
|
||||
public async stop(ids: number[]) {
|
||||
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
|
||||
const docs = await SubscriptionModel.find({ id: ids }, {});
|
||||
let failure: unknown;
|
||||
for (const doc of docs) {
|
||||
try {
|
||||
if (doc.pid) {
|
||||
await killTask(doc.pid, true);
|
||||
}
|
||||
await SubscriptionModel.update(
|
||||
await SubscriptionModel.nativeUpdate(
|
||||
{ id: doc.id, pid: doc.pid ?? null },
|
||||
{ status: SubscriptionStatus.idle, pid: null } as any,
|
||||
{ where: { id: doc.id, pid: doc.pid ?? null } },
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
@@ -414,15 +392,12 @@ export default class SubscriptionService {
|
||||
}
|
||||
if (failure) throw failure;
|
||||
}
|
||||
|
||||
private async runSingle(subscriptionId: number) {
|
||||
const subscription = await this.getDb({ id: subscriptionId });
|
||||
if (subscription.status !== SubscriptionStatus.queued) {
|
||||
return;
|
||||
}
|
||||
|
||||
const command = formatCommand(subscription);
|
||||
|
||||
this.scheduleService.runTask(command, this.taskCallbacks(subscription), {
|
||||
name: subscription.name,
|
||||
schedule: subscription.schedule,
|
||||
@@ -431,25 +406,22 @@ export default class SubscriptionService {
|
||||
runOrigin: 'subscription',
|
||||
});
|
||||
}
|
||||
|
||||
public async disabled(ids: number[]) {
|
||||
await SubscriptionModel.update({ is_disabled: 1 }, { where: { id: ids } });
|
||||
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
|
||||
await SubscriptionModel.nativeUpdate({ id: ids }, { is_disabled: 1 });
|
||||
const docs = await SubscriptionModel.find({ id: ids }, {});
|
||||
await this.setSshConfig();
|
||||
for (const doc of docs) {
|
||||
await this.handleTask(doc.get({ plain: true }), false);
|
||||
await this.handleTask(doc, false);
|
||||
}
|
||||
}
|
||||
|
||||
public async enabled(ids: number[]) {
|
||||
await SubscriptionModel.update({ is_disabled: 0 }, { where: { id: ids } });
|
||||
const docs = await SubscriptionModel.findAll({ where: { id: ids } });
|
||||
await SubscriptionModel.nativeUpdate({ id: ids }, { is_disabled: 0 });
|
||||
const docs = await SubscriptionModel.find({ id: ids }, {});
|
||||
await this.setSshConfig();
|
||||
for (const doc of docs) {
|
||||
await this.handleTask(doc.get({ plain: true }));
|
||||
await this.handleTask(doc);
|
||||
}
|
||||
}
|
||||
|
||||
public async log(id: number, options: LogReadOptions = {}) {
|
||||
const doc = await this.getDb({ id });
|
||||
if (!doc || !doc.log_path) {
|
||||
@@ -461,17 +433,14 @@ export default class SubscriptionService {
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
const absolutePath = await handleLogPath(doc.log_path as string);
|
||||
return await readLogChunk(absolutePath, options);
|
||||
}
|
||||
|
||||
public async logs(id: number) {
|
||||
const doc = await this.getDb({ id });
|
||||
if (!doc) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (doc.log_path) {
|
||||
const relativeDir = path.dirname(`${doc.log_path}`);
|
||||
const dir = path.dirname(resolveLogPath(config.logPath, doc.log_path));
|
||||
|
||||
+33
-67
@@ -1,3 +1,4 @@
|
||||
import { Query, QuerySpec, orderBy } from '../data/queries';
|
||||
import { persistNotificationAudit } from '../shared/notificationAudit';
|
||||
import { spawn } from 'cross-spawn';
|
||||
import { execFile } from 'child_process';
|
||||
@@ -46,18 +47,16 @@ import os from 'os';
|
||||
import dayjs from 'dayjs';
|
||||
import { t, setLang } from '../shared/i18n';
|
||||
import { updateLinuxMirrorFile } from '../config/util';
|
||||
|
||||
@Service()
|
||||
export default class SystemService {
|
||||
@Inject((type) => NotificationService)
|
||||
private notificationService!: NotificationService;
|
||||
|
||||
constructor(
|
||||
@Inject('logger') private logger: winston.Logger,
|
||||
@Inject('logger')
|
||||
private logger: winston.Logger,
|
||||
private scheduleService: ScheduleService,
|
||||
private sockService: SockService,
|
||||
) {}
|
||||
|
||||
public async getSystemConfig() {
|
||||
const doc = await this.getDb({ type: AuthDataType.systemConfig });
|
||||
return {
|
||||
@@ -65,22 +64,19 @@ export default class SystemService {
|
||||
info: { ...doc.info, timezone: doc.info?.timezone || 'Asia/Shanghai' },
|
||||
};
|
||||
}
|
||||
|
||||
private async updateAuthDb(payload: SystemInfo): Promise<SystemInfo> {
|
||||
const { id, ...others } = payload;
|
||||
await SystemModel.update(others, { where: { id } });
|
||||
await SystemModel.nativeUpdate({ id }, others);
|
||||
const doc = await this.getDb({ id });
|
||||
return doc;
|
||||
}
|
||||
|
||||
public async getDb(query: any): Promise<SystemInfo> {
|
||||
const doc = await SystemModel.findOne({ where: query });
|
||||
const doc = await SystemModel.findOne(query, {});
|
||||
if (!doc) {
|
||||
throw new Error(`System ${JSON.stringify(query)} not found`);
|
||||
}
|
||||
return doc.get({ plain: true });
|
||||
return doc;
|
||||
}
|
||||
|
||||
public async updateNotificationMode(notificationInfo: NotificationInfo) {
|
||||
const code = Math.random().toString().slice(-6);
|
||||
const isSuccess = await this.notificationService.testNotify(
|
||||
@@ -98,7 +94,6 @@ export default class SystemService {
|
||||
return { code: 400, message: t('通知发送失败,请检查参数') };
|
||||
}
|
||||
}
|
||||
|
||||
public async updateLogRemoveFrequency(info: SystemModelInfo) {
|
||||
const oDoc = await this.getSystemConfig();
|
||||
const result = await this.updateAuthDb({
|
||||
@@ -125,7 +120,6 @@ export default class SystemService {
|
||||
}
|
||||
return { code: 200, data: info };
|
||||
}
|
||||
|
||||
public async updateCronConcurrency(info: SystemModelInfo) {
|
||||
const limit = info.cronConcurrency ?? 0;
|
||||
if (!Number.isInteger(limit) || limit < 0 || limit > 2147483647) {
|
||||
@@ -142,7 +136,6 @@ export default class SystemService {
|
||||
return { code: 200, data: info };
|
||||
});
|
||||
}
|
||||
|
||||
public async restoreCronConcurrency() {
|
||||
await withSchedulerMutation(async () => {
|
||||
const doc = await this.getSystemConfig();
|
||||
@@ -151,7 +144,6 @@ export default class SystemService {
|
||||
await cronClient.setConcurrency(limit);
|
||||
});
|
||||
}
|
||||
|
||||
public async updateDependenceProxy(info: SystemModelInfo) {
|
||||
const oDoc = await this.getSystemConfig();
|
||||
await this.updateAuthDb({
|
||||
@@ -170,7 +162,6 @@ export default class SystemService {
|
||||
}
|
||||
return { code: 200, data: info };
|
||||
}
|
||||
|
||||
public async updateNodeMirror(info: SystemModelInfo, res?: Response) {
|
||||
const oDoc = await this.getSystemConfig();
|
||||
await this.updateAuthDb({
|
||||
@@ -182,12 +173,13 @@ export default class SystemService {
|
||||
cmd = `pnpm config set registry ${shellQuote(info.nodeMirror)}`;
|
||||
}
|
||||
let command = `cd && ${cmd}`;
|
||||
const docs = await DependenceModel.findAll({
|
||||
where: {
|
||||
const docs = await DependenceModel.find(
|
||||
{
|
||||
type: DependenceTypes.nodejs,
|
||||
status: DependenceStatus.installed,
|
||||
},
|
||||
});
|
||||
{},
|
||||
);
|
||||
if (docs.length > 0) {
|
||||
command += ` && pnpm i -g`;
|
||||
}
|
||||
@@ -219,7 +211,6 @@ export default class SystemService {
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
public async updatePythonMirror(info: SystemModelInfo) {
|
||||
const oDoc = await this.getSystemConfig();
|
||||
await this.updateAuthDb({
|
||||
@@ -233,7 +224,6 @@ export default class SystemService {
|
||||
await promiseExec(cmd);
|
||||
return { code: 200, data: info };
|
||||
}
|
||||
|
||||
public async updateLinuxMirror(
|
||||
info: SystemModelInfo,
|
||||
res?: Response,
|
||||
@@ -280,13 +270,11 @@ export default class SystemService {
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
public async checkUpdate() {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 15000);
|
||||
try {
|
||||
const currentVersionContent = await parseVersion(config.versionFile);
|
||||
|
||||
const { body, statusCode } = await request(
|
||||
`${config.lastVersionFile}?t=${Date.now()}`,
|
||||
{ signal: controller.signal },
|
||||
@@ -301,7 +289,6 @@ export default class SystemService {
|
||||
) {
|
||||
throw new Error('Invalid version response');
|
||||
}
|
||||
|
||||
return {
|
||||
code: 200,
|
||||
data: {
|
||||
@@ -323,7 +310,6 @@ export default class SystemService {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
private checkHasNewVersion(curVersion: string, lastVersion: string) {
|
||||
const curArr = curVersion.split('.').map((x) => parseInt(x, 10));
|
||||
const lastArr = lastVersion.split('.').map((x) => parseInt(x, 10));
|
||||
@@ -342,25 +328,21 @@ export default class SystemService {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public async updateSystem() {
|
||||
return this.startUpgradeJob('download');
|
||||
}
|
||||
|
||||
private upgradeJobs() {
|
||||
return require(path.join(
|
||||
config.rootPath,
|
||||
'cli/src/internal/maintenance/panelUpgrade.cjs',
|
||||
));
|
||||
}
|
||||
|
||||
public async getUpgradeStatus() {
|
||||
return {
|
||||
code: 200,
|
||||
data: await this.upgradeJobs().getStatus(config.tmpPath),
|
||||
};
|
||||
}
|
||||
|
||||
private async startUpgradeJob(action: 'download' | 'apply') {
|
||||
try {
|
||||
const data = await this.upgradeJobs().startJob(
|
||||
@@ -383,7 +365,6 @@ export default class SystemService {
|
||||
return { code: 400, message: t('升级任务启动失败,请检查日志后重试') };
|
||||
}
|
||||
}
|
||||
|
||||
public async reloadSystem(target?: 'system' | 'data') {
|
||||
if (target && !['system', 'data'].includes(target)) {
|
||||
return { code: 400, message: t('参数错误') };
|
||||
@@ -401,7 +382,6 @@ export default class SystemService {
|
||||
});
|
||||
return { code: 200 };
|
||||
}
|
||||
|
||||
public async notify({
|
||||
title,
|
||||
content,
|
||||
@@ -418,35 +398,38 @@ export default class SystemService {
|
||||
if (notificationInfo && typeString) {
|
||||
notificationInfo.type = typeString;
|
||||
}
|
||||
|
||||
const isSuccess = await this.notificationService.notify(
|
||||
title,
|
||||
content,
|
||||
notificationInfo,
|
||||
);
|
||||
|
||||
if (isSuccess) {
|
||||
return { code: 200, message: t('通知发送成功') };
|
||||
} else {
|
||||
return { code: 400, message: t('通知发送失败,请检查系统设置/通知配置') };
|
||||
}
|
||||
}
|
||||
|
||||
public async recordNotifyLog(info: NotifyLogInfo): Promise<void> {
|
||||
await persistNotificationAudit(info);
|
||||
}
|
||||
|
||||
public async getNotifyLog(): Promise<Array<NotifyLogInfo>> {
|
||||
const docs = await SystemModel.findAll({
|
||||
where: { type: AuthDataType.notifyLog },
|
||||
order: [['id', 'DESC']],
|
||||
limit: 200,
|
||||
});
|
||||
const docs = await SystemModel.find(
|
||||
{ type: AuthDataType.notifyLog },
|
||||
{
|
||||
orderBy: orderBy([['id', 'DESC']]),
|
||||
limit: 200,
|
||||
},
|
||||
);
|
||||
return docs.map((x) => ({ ...x.info, id: x.id }));
|
||||
}
|
||||
|
||||
public async run(
|
||||
{ command, logPath }: { command: string; logPath?: string },
|
||||
{
|
||||
command,
|
||||
logPath,
|
||||
}: {
|
||||
command: string;
|
||||
logPath?: string;
|
||||
},
|
||||
callback: TaskCallbacks,
|
||||
) {
|
||||
if (!command.startsWith(TASK_COMMAND)) {
|
||||
@@ -463,17 +446,14 @@ export default class SystemService {
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
public async stop({ command, pid }: { command: string; pid: number }) {
|
||||
if (!pid && !command) {
|
||||
return { code: 400, message: t('参数错误') };
|
||||
}
|
||||
|
||||
if (pid) {
|
||||
await killTask(pid);
|
||||
return { code: 200 };
|
||||
}
|
||||
|
||||
if (!command.startsWith(TASK_COMMAND)) {
|
||||
command = `${TASK_COMMAND} ${command}`;
|
||||
}
|
||||
@@ -485,7 +465,6 @@ export default class SystemService {
|
||||
return { code: 400, message: t('任务未找到') };
|
||||
}
|
||||
}
|
||||
|
||||
public async exportData(res: Response, type?: string[]) {
|
||||
try {
|
||||
let dataDirs = ['db', 'upload'];
|
||||
@@ -527,7 +506,6 @@ export default class SystemService {
|
||||
return res.send({ code: 400, message: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
public async importData() {
|
||||
try {
|
||||
await promiseExec(`rm -rf ${path.join(config.tmpPath, 'data')}`);
|
||||
@@ -539,7 +517,6 @@ export default class SystemService {
|
||||
return { code: 400, message: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
public async getSystemLog(
|
||||
res: Response,
|
||||
query: {
|
||||
@@ -559,7 +536,6 @@ export default class SystemService {
|
||||
.reverse()
|
||||
.filter((x) => x.title.endsWith('.log'))
|
||||
.filter((x) => x.createTime >= startTime && x.createTime <= endTime);
|
||||
|
||||
const limit = Math.min(
|
||||
Math.max(Number(query.limit) || 1024 * 1024, 1),
|
||||
1024 * 1024,
|
||||
@@ -567,7 +543,10 @@ export default class SystemService {
|
||||
const total = logs.reduce((size, log) => size + (log.size || 0), 0);
|
||||
let remaining = limit;
|
||||
const selected: Array<
|
||||
(typeof logs)[number] & { start: number; length: number }
|
||||
(typeof logs)[number] & {
|
||||
start: number;
|
||||
length: number;
|
||||
}
|
||||
> = [];
|
||||
for (let index = logs.length - 1; index >= 0 && remaining > 0; index--) {
|
||||
const log = logs[index];
|
||||
@@ -578,7 +557,6 @@ export default class SystemService {
|
||||
remaining -= length;
|
||||
}
|
||||
}
|
||||
|
||||
const contentLength = selected.reduce((size, log) => size + log.length, 0);
|
||||
res.set({
|
||||
'Content-Length': contentLength,
|
||||
@@ -590,7 +568,6 @@ export default class SystemService {
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
const currentLog = fileNames.shift();
|
||||
if (currentLog) {
|
||||
const currentFileStream = fs.createReadStream(
|
||||
@@ -607,7 +584,6 @@ export default class SystemService {
|
||||
}
|
||||
})(res, selected);
|
||||
}
|
||||
|
||||
public async deleteSystemLog() {
|
||||
const result = await readDirs(config.systemLogPath, config.systemLogPath);
|
||||
const logs = result.reverse().filter((x) => x.title.endsWith('.log'));
|
||||
@@ -625,7 +601,6 @@ export default class SystemService {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async updateTimezone(info: SystemModelInfo) {
|
||||
if (!info.timezone) {
|
||||
info.timezone = 'Asia/Shanghai';
|
||||
@@ -642,7 +617,6 @@ export default class SystemService {
|
||||
return { code: 400, message: t('设置时区失败') };
|
||||
}
|
||||
}
|
||||
|
||||
public async updateLanguage(info: SystemModelInfo) {
|
||||
const oDoc = await this.getSystemConfig();
|
||||
const lang = info.lang || 'zh';
|
||||
@@ -662,7 +636,6 @@ export default class SystemService {
|
||||
setLang(lang);
|
||||
return { code: 200, data: { lang } };
|
||||
}
|
||||
|
||||
public async updatePanelTitle(info: SystemModelInfo) {
|
||||
const oDoc = await this.getSystemConfig();
|
||||
const panelTitle = info.panelTitle?.trim() || '';
|
||||
@@ -672,43 +645,36 @@ export default class SystemService {
|
||||
});
|
||||
return { code: 200, data: { panelTitle } };
|
||||
}
|
||||
|
||||
public async updateGlobalSshKey(info: SystemModelInfo) {
|
||||
const oDoc = await this.getSystemConfig();
|
||||
const result = await this.updateAuthDb({
|
||||
...oDoc,
|
||||
info: { ...oDoc.info, ...info },
|
||||
});
|
||||
|
||||
// Apply the global SSH key
|
||||
const SshKeyService = require('./sshKey').default;
|
||||
const Container = require('typedi').Container;
|
||||
const sshKeyService = Container.get(SshKeyService);
|
||||
|
||||
if (info.globalSshKey) {
|
||||
await sshKeyService.addGlobalSSHKey(info.globalSshKey, 'global');
|
||||
} else {
|
||||
await sshKeyService.removeGlobalSSHKey('global');
|
||||
}
|
||||
|
||||
return { code: 200, data: result };
|
||||
}
|
||||
|
||||
public async cleanDependence(type: 'node' | 'python3') {
|
||||
if (!type || !['node', 'python3'].includes(type)) {
|
||||
return { code: 400, message: t('参数错误') };
|
||||
}
|
||||
const finalPath = path.join(config.dependenceCachePath, type);
|
||||
await fs.promises.rm(finalPath, { recursive: true, force: true });
|
||||
await DependenceModel.update(
|
||||
{ status: DependenceStatus.installFailed },
|
||||
await DependenceModel.nativeUpdate(
|
||||
{
|
||||
where: {
|
||||
type:
|
||||
type === 'node' ? DependenceTypes.nodejs : DependenceTypes.python3,
|
||||
status: DependenceStatus.installed,
|
||||
},
|
||||
type:
|
||||
type === 'node' ? DependenceTypes.nodejs : DependenceTypes.python3,
|
||||
status: DependenceStatus.installed,
|
||||
},
|
||||
{ status: DependenceStatus.installFailed },
|
||||
);
|
||||
return { code: 200 };
|
||||
}
|
||||
|
||||
+20
-71
@@ -34,27 +34,27 @@ import {
|
||||
} from '../shared/password';
|
||||
import { serializeAuthMutation } from '../shared/authMutation';
|
||||
import { LoginThrottle } from '../shared/loginThrottle';
|
||||
|
||||
@Service()
|
||||
export default class UserService {
|
||||
private loginThrottle = new LoginThrottle();
|
||||
@Inject((type) => NotificationService)
|
||||
private notificationService!: NotificationService;
|
||||
|
||||
constructor(
|
||||
@Inject('logger') private logger: winston.Logger,
|
||||
@Inject('logger')
|
||||
private logger: winston.Logger,
|
||||
private scheduleService: ScheduleService,
|
||||
private sockService: SockService,
|
||||
) {}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async login(
|
||||
payloads: { username: string; password: string },
|
||||
payloads: {
|
||||
username: string;
|
||||
password: string;
|
||||
},
|
||||
req: Request,
|
||||
): Promise<any> {
|
||||
return this.authenticate(payloads, req);
|
||||
}
|
||||
|
||||
private async authenticate(
|
||||
payloads: {
|
||||
username: string;
|
||||
@@ -89,7 +89,6 @@ export default class UserService {
|
||||
platform,
|
||||
blockedIps = [],
|
||||
} = content;
|
||||
|
||||
if (
|
||||
ip &&
|
||||
blockedIps.some((blockedIp) => normalizeClientIp(blockedIp) === ip)
|
||||
@@ -107,7 +106,6 @@ export default class UserService {
|
||||
this.getLoginLog();
|
||||
return { code: 403, message: t('该 IP 已被列入黑名单') };
|
||||
}
|
||||
|
||||
const waitTime = this.loginThrottle.retryAfter(ip, timestamp);
|
||||
if (waitTime > 0) {
|
||||
return {
|
||||
@@ -116,10 +114,8 @@ export default class UserService {
|
||||
data: waitTime,
|
||||
};
|
||||
}
|
||||
|
||||
const passwordMatches =
|
||||
username === cUsername && (await verifyPassword(password, cPassword));
|
||||
|
||||
if (passwordMatches && twoFactorActivated && needTwoFactor) {
|
||||
await this.updateAuthInfo(content, {
|
||||
isTwoFactorChecking: true,
|
||||
@@ -130,7 +126,6 @@ export default class UserService {
|
||||
message: '',
|
||||
};
|
||||
}
|
||||
|
||||
if (passwordMatches) {
|
||||
this.loginThrottle.reset(ip);
|
||||
const data = createRandomString(50, 100);
|
||||
@@ -139,7 +134,6 @@ export default class UserService {
|
||||
expiresIn: config.jwt.expiresIn || expiration,
|
||||
algorithm: 'HS384',
|
||||
});
|
||||
|
||||
const tokenInfo: TokenInfo = {
|
||||
value: token,
|
||||
timestamp,
|
||||
@@ -147,13 +141,11 @@ export default class UserService {
|
||||
address,
|
||||
platform: req.platform,
|
||||
};
|
||||
|
||||
const updatedTokens = this.addTokenToList(
|
||||
tokens,
|
||||
req.platform,
|
||||
tokenInfo,
|
||||
);
|
||||
|
||||
await this.updateAuthInfo(content, {
|
||||
password: isPasswordHash(cPassword)
|
||||
? cPassword
|
||||
@@ -252,16 +244,13 @@ export default class UserService {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async logout(platform: string, tokenValue: string): Promise<any> {
|
||||
if (!platform || !tokenValue) {
|
||||
this.logger.warn('Invalid logout parameters - empty platform or token');
|
||||
return;
|
||||
}
|
||||
|
||||
const authInfo = await this.getAuthInfo();
|
||||
|
||||
// Verify the token exists before attempting to remove it
|
||||
const tokenExists = this.findTokenInList(
|
||||
authInfo.tokens,
|
||||
@@ -275,45 +264,36 @@ export default class UserService {
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const updatedTokens = this.removeTokenFromList(
|
||||
authInfo.tokens,
|
||||
platform,
|
||||
tokenValue,
|
||||
);
|
||||
|
||||
await this.updateAuthInfo(authInfo, {
|
||||
token: authInfo.token === tokenValue ? '' : authInfo.token,
|
||||
tokens: updatedTokens,
|
||||
});
|
||||
}
|
||||
|
||||
public async getLoginLog(): Promise<Array<SystemModelInfo | undefined>> {
|
||||
const docs = await SystemModel.findAll({
|
||||
where: { type: AuthDataType.loginLog },
|
||||
});
|
||||
const docs = await SystemModel.find({ type: AuthDataType.loginLog }, {});
|
||||
if (docs && docs.length > 0) {
|
||||
const result = docs.sort(
|
||||
(a, b) => b.info!.timestamp! - a.info!.timestamp!,
|
||||
);
|
||||
if (result.length > 100) {
|
||||
const ids = result.slice(100).map((x) => x.id!);
|
||||
await SystemModel.destroy({
|
||||
where: { id: ids },
|
||||
});
|
||||
await SystemModel.nativeDelete({ id: ids });
|
||||
}
|
||||
return result.map((x) => x.info);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
public async getIpBlacklist(): Promise<string[]> {
|
||||
const authInfo = await this.getAuthInfo();
|
||||
return uniq((authInfo.blockedIps || []).map(normalizeClientIp)).filter(
|
||||
Boolean,
|
||||
);
|
||||
}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async blockIp(ip: string): Promise<string[]> {
|
||||
const authInfo = await this.getAuthInfo();
|
||||
@@ -324,7 +304,6 @@ export default class UserService {
|
||||
await this.updateAuthInfo(authInfo, { blockedIps });
|
||||
return blockedIps;
|
||||
}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async unblockIp(ip: string): Promise<string[]> {
|
||||
const authInfo = await this.getAuthInfo();
|
||||
@@ -335,12 +314,10 @@ export default class UserService {
|
||||
await this.updateAuthInfo(authInfo, { blockedIps });
|
||||
return blockedIps;
|
||||
}
|
||||
|
||||
private async insertDb(payload: SystemInfo): Promise<SystemInfo> {
|
||||
const doc = await SystemModel.create({ ...payload }, { returning: true });
|
||||
const doc = await SystemModel.insert({ ...payload });
|
||||
return doc;
|
||||
}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async initializeUser({
|
||||
username,
|
||||
@@ -366,7 +343,6 @@ export default class UserService {
|
||||
});
|
||||
return { code: 200, message: t('更新成功') };
|
||||
}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async updateUsernameAndPassword({
|
||||
username,
|
||||
@@ -389,14 +365,12 @@ export default class UserService {
|
||||
});
|
||||
return { code: 200, message: t('更新成功') };
|
||||
}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async updateAvatar(avatar: string) {
|
||||
const authInfo = await this.getAuthInfo();
|
||||
await this.updateAuthInfo(authInfo, { avatar });
|
||||
return { code: 200, data: avatar, message: t('更新成功') };
|
||||
}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async initTwoFactor() {
|
||||
const secret = authenticator.generateSecret();
|
||||
@@ -408,7 +382,6 @@ export default class UserService {
|
||||
await this.updateAuthInfo(authInfo, { twoFactorSecret: secret });
|
||||
return { secret, url: otpauth };
|
||||
}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async activeTwoFactor(code: string) {
|
||||
const authInfo = await this.getAuthInfo();
|
||||
@@ -427,14 +400,17 @@ export default class UserService {
|
||||
}
|
||||
return isValid;
|
||||
}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async twoFactorLogin(
|
||||
{
|
||||
username,
|
||||
password,
|
||||
code,
|
||||
}: { username: string; password: string; code: string },
|
||||
}: {
|
||||
username: string;
|
||||
password: string;
|
||||
code: string;
|
||||
},
|
||||
req: any,
|
||||
) {
|
||||
const authInfo = await this.getAuthInfo();
|
||||
@@ -482,7 +458,6 @@ export default class UserService {
|
||||
return { code: 430, message: t('验证失败') };
|
||||
}
|
||||
}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async deactivateTwoFactor() {
|
||||
const authInfo = await this.getAuthInfo();
|
||||
@@ -496,7 +471,6 @@ export default class UserService {
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
public async getAuthInfo() {
|
||||
const authInfo = await shareStore.getAuthInfo();
|
||||
if (authInfo) {
|
||||
@@ -505,7 +479,6 @@ export default class UserService {
|
||||
const doc = await this.getDb({ type: AuthDataType.authConfig });
|
||||
return (doc.info || {}) as AuthInfo;
|
||||
}
|
||||
|
||||
private async updateAuthInfo(authInfo: AuthInfo, info: Partial<AuthInfo>) {
|
||||
const result = { ...authInfo, ...info };
|
||||
await shareStore.updateAuthInfo(result);
|
||||
@@ -517,34 +490,27 @@ export default class UserService {
|
||||
this.sockService.getClients().forEach((conn) => conn.close('401'));
|
||||
}
|
||||
}
|
||||
|
||||
public async getNotificationMode(): Promise<NotificationInfo> {
|
||||
const doc = await this.getDb({ type: AuthDataType.notification });
|
||||
return (doc.info || {}) as NotificationInfo;
|
||||
}
|
||||
|
||||
private async updateAuthDb(payload: SystemInfo): Promise<any> {
|
||||
let doc = await SystemModel.findOne({ where: { type: payload.type } });
|
||||
let doc = await SystemModel.findOne({ type: payload.type }, {});
|
||||
if (doc) {
|
||||
const updateResult = await SystemModel.update(payload, {
|
||||
where: { id: doc.id },
|
||||
returning: true,
|
||||
});
|
||||
doc = updateResult[1][0];
|
||||
await SystemModel.nativeUpdate({ id: doc.id }, payload);
|
||||
doc = await SystemModel.findOne({ id: doc.id });
|
||||
} else {
|
||||
doc = await SystemModel.create(payload, { returning: true });
|
||||
doc = await SystemModel.insert(payload);
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
public async getDb(query: any): Promise<SystemInfo> {
|
||||
const doc = await SystemModel.findOne({ where: { ...query } });
|
||||
const doc = await SystemModel.findOne({ ...query }, {});
|
||||
if (!doc) {
|
||||
throw new Error(`${JSON.stringify(query)} not found`);
|
||||
}
|
||||
return doc.get({ plain: true });
|
||||
return doc;
|
||||
}
|
||||
|
||||
public async updateNotificationMode(notificationInfo: NotificationInfo) {
|
||||
const code = Math.random().toString().slice(-6);
|
||||
const isSuccess = await this.notificationService.testNotify(
|
||||
@@ -562,12 +528,10 @@ export default class UserService {
|
||||
return { code: 400, message: t('通知发送失败,请检查参数') };
|
||||
}
|
||||
}
|
||||
|
||||
private normalizeTokens(
|
||||
tokens: Record<string, string | TokenInfo[]>,
|
||||
): Record<string, TokenInfo[]> {
|
||||
const normalized: Record<string, TokenInfo[]> = {};
|
||||
|
||||
for (const [platform, value] of Object.entries(tokens)) {
|
||||
if (typeof value === 'string') {
|
||||
// Legacy format: convert string token to TokenInfo array
|
||||
@@ -589,10 +553,8 @@ export default class UserService {
|
||||
normalized[platform] = value || [];
|
||||
}
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private addTokenToList(
|
||||
tokens: Record<string, string | TokenInfo[]>,
|
||||
platform: string,
|
||||
@@ -606,16 +568,12 @@ export default class UserService {
|
||||
);
|
||||
maxTokensPerPlatform = config.maxTokensPerPlatform;
|
||||
}
|
||||
|
||||
const normalized = this.normalizeTokens(tokens);
|
||||
|
||||
if (!normalized[platform]) {
|
||||
normalized[platform] = [];
|
||||
}
|
||||
|
||||
// Add new token
|
||||
normalized[platform].unshift(tokenInfo);
|
||||
|
||||
// Limit the number of active tokens per platform
|
||||
if (normalized[platform].length > maxTokensPerPlatform) {
|
||||
normalized[platform] = normalized[platform].slice(
|
||||
@@ -623,40 +581,32 @@ export default class UserService {
|
||||
maxTokensPerPlatform,
|
||||
);
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private removeTokenFromList(
|
||||
tokens: Record<string, string | TokenInfo[]>,
|
||||
platform: string,
|
||||
tokenValue: string,
|
||||
): Record<string, TokenInfo[]> {
|
||||
const normalized = this.normalizeTokens(tokens);
|
||||
|
||||
if (normalized[platform]) {
|
||||
normalized[platform] = normalized[platform].filter(
|
||||
(t) => t.value !== tokenValue,
|
||||
);
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private findTokenInList(
|
||||
tokens: Record<string, string | TokenInfo[]>,
|
||||
platform: string,
|
||||
tokenValue: string,
|
||||
): TokenInfo | undefined {
|
||||
const normalized = this.normalizeTokens(tokens);
|
||||
|
||||
if (normalized[platform]) {
|
||||
return normalized[platform].find((t) => t.value === tokenValue);
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@serializeAuthMutation
|
||||
public async resetAuthInfo(info: Partial<AuthInfo>) {
|
||||
const { retries, twoFactorActivated, password, username } = info;
|
||||
@@ -673,7 +623,6 @@ export default class UserService {
|
||||
},
|
||||
(x) => !isNil(x),
|
||||
);
|
||||
|
||||
if (password !== undefined) {
|
||||
payload.password = await hashPassword(password);
|
||||
}
|
||||
|
||||
@@ -135,8 +135,10 @@ export function parseCronSchedule(schedule: unknown): CronSchedule {
|
||||
};
|
||||
Object.freeze(patterns);
|
||||
Object.freeze(parsed);
|
||||
if (scheduleCache.size >= 512)
|
||||
scheduleCache.delete(scheduleCache.keys().next().value);
|
||||
if (scheduleCache.size >= 512) {
|
||||
const oldest = scheduleCache.keys().next().value;
|
||||
if (oldest !== undefined) scheduleCache.delete(oldest);
|
||||
}
|
||||
scheduleCache.set(cacheKey, parsed);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
const messages: Record<string, Record<string, string>> = {
|
||||
zh: {},
|
||||
en: {
|
||||
'认证服务暂时不可用,请稍后重试':
|
||||
'Authentication service is temporarily unavailable; try again shortly.',
|
||||
'升级任务启动失败,请检查日志后重试':
|
||||
'Failed to start the update. Check the logs and try again.',
|
||||
'[schedule][停止已运行任务] 任务ID: %s, PID: %s':
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
import { Op } from 'sequelize';
|
||||
import { Query, QuerySpec, orderBy } from '../data/queries';
|
||||
import {
|
||||
SystemModel,
|
||||
AuthDataType,
|
||||
NotifyStatus,
|
||||
NotifyLogInfo,
|
||||
} from '../data/system';
|
||||
|
||||
// Both backend sends and script reports use the same bounded, best-effort sink.
|
||||
export async function persistNotificationAudit(
|
||||
info: NotifyLogInfo,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await SystemModel.create({
|
||||
await SystemModel.insert({
|
||||
type: AuthDataType.notifyLog,
|
||||
info: {
|
||||
timestamp: Date.now(),
|
||||
@@ -27,19 +26,19 @@ export async function persistNotificationAudit(
|
||||
taskId: /^\d+$/.test(info.taskId || '') ? info.taskId : undefined,
|
||||
},
|
||||
});
|
||||
const oldest = await SystemModel.findAll({
|
||||
where: { type: AuthDataType.notifyLog },
|
||||
attributes: ['id'],
|
||||
order: [['id', 'DESC']],
|
||||
offset: 200,
|
||||
limit: 1,
|
||||
});
|
||||
const oldest = await SystemModel.find(
|
||||
{ type: AuthDataType.notifyLog },
|
||||
{
|
||||
fields: ['id'],
|
||||
orderBy: orderBy([['id', 'DESC']]),
|
||||
offset: 200,
|
||||
limit: 1,
|
||||
},
|
||||
);
|
||||
if (oldest[0]?.id) {
|
||||
await SystemModel.destroy({
|
||||
where: {
|
||||
type: AuthDataType.notifyLog,
|
||||
id: { [Op.lte]: oldest[0].id },
|
||||
},
|
||||
await SystemModel.nativeDelete({
|
||||
type: AuthDataType.notifyLog,
|
||||
id: { ['$lte']: oldest[0].id },
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
+14
-7
@@ -137,13 +137,20 @@ class TaskLimit {
|
||||
this.manualCronoLimit.concurrency = limit;
|
||||
return;
|
||||
}
|
||||
await SystemModel.sync();
|
||||
const doc = await SystemModel.findOne({
|
||||
where: { type: AuthDataType.systemConfig },
|
||||
});
|
||||
if (doc?.info?.cronConcurrency) {
|
||||
this.cronLimit.concurrency = doc.info.cronConcurrency;
|
||||
this.manualCronoLimit.concurrency = doc.info.cronConcurrency;
|
||||
|
||||
try {
|
||||
const doc = await SystemModel.findOne({
|
||||
type: AuthDataType.systemConfig,
|
||||
});
|
||||
if (doc?.info?.cronConcurrency) {
|
||||
this.cronLimit.concurrency = doc.info.cronConcurrency;
|
||||
this.manualCronoLimit.concurrency = doc.info.cronConcurrency;
|
||||
}
|
||||
} catch (error) {
|
||||
// Imports can precede the isolated initializer on a fresh installation.
|
||||
// Use defaults until that initializer creates Auths; never create schema here.
|
||||
if ((error as { code?: string }).code !== 'DATABASE_SCHEMA_MISSING')
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ export function runCron(
|
||||
// Check if the cron is already running and stop it (only if multiple instances are not allowed)
|
||||
try {
|
||||
const existingCron = await CrontabModel.findOne({
|
||||
where: { id: Number(cron.id) },
|
||||
id: Number(cron.id),
|
||||
});
|
||||
|
||||
// A mutation may commit while this queued run is reading the database.
|
||||
@@ -45,19 +45,17 @@ export function runCron(
|
||||
await killTask(existingCron.pid, true);
|
||||
// Mark old running instances as stopped
|
||||
const stoppedAt = dayjs().unix();
|
||||
await RunningInstanceModel.update(
|
||||
{ status: InstanceStatus.stopped, finished_at: stoppedAt },
|
||||
await RunningInstanceModel.nativeUpdate(
|
||||
{
|
||||
where: {
|
||||
cron_id: Number(cron.id),
|
||||
status: InstanceStatus.running,
|
||||
},
|
||||
cron_id: Number(cron.id),
|
||||
status: InstanceStatus.running,
|
||||
},
|
||||
{ status: InstanceStatus.stopped, finished_at: stoppedAt },
|
||||
);
|
||||
// Update the status to idle after killing
|
||||
await CrontabModel.update(
|
||||
await CrontabModel.nativeUpdate(
|
||||
{ id: Number(cron.id) },
|
||||
{ status: CrontabStatus.idle, pid: undefined },
|
||||
{ where: { id: Number(cron.id) } },
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { QueryTypes, Sequelize } from 'sequelize';
|
||||
import { DatabaseClient } from '../data/databaseClient';
|
||||
|
||||
// Append new entries; IDs are persisted and must not be renumbered or reused.
|
||||
const columns = [
|
||||
@@ -27,25 +27,19 @@ const columns = [
|
||||
{ table: 'Crontabs', column: 'queued_token', type: 'VARCHAR(255)' },
|
||||
];
|
||||
|
||||
export async function migrateSchema(database: Sequelize): Promise<void> {
|
||||
await database.transaction(async (transaction) => {
|
||||
export async function migrateSchema(database: DatabaseClient): Promise<void> {
|
||||
await database.transaction(async () => {
|
||||
await database.query(
|
||||
'CREATE TABLE IF NOT EXISTS "SchemaMigrations" ("id" TEXT PRIMARY KEY, "applied_at" TEXT NOT NULL)',
|
||||
{ transaction },
|
||||
);
|
||||
const applied = await database.query<{ id: string }>(
|
||||
'SELECT "id" FROM "SchemaMigrations"',
|
||||
{ type: QueryTypes.SELECT, transaction },
|
||||
);
|
||||
const appliedIds = new Set(applied.map(({ id }) => id));
|
||||
for (const { table, column, type } of columns) {
|
||||
const id = `add-${table}-${column}`;
|
||||
const fields = await database.query<{ name: string }>(
|
||||
`PRAGMA table_info("${table}")`,
|
||||
{
|
||||
type: QueryTypes.SELECT,
|
||||
transaction,
|
||||
},
|
||||
);
|
||||
if (fields.length === 0)
|
||||
throw new Error(`Migration table is missing: ${table}`);
|
||||
@@ -53,16 +47,12 @@ export async function migrateSchema(database: Sequelize): Promise<void> {
|
||||
// table/column/type come only from the static migration manifest above.
|
||||
await database.query(
|
||||
`ALTER TABLE "${table}" ADD COLUMN "${column}" ${type}`,
|
||||
{ transaction },
|
||||
);
|
||||
}
|
||||
if (!appliedIds.has(id)) {
|
||||
await database.query(
|
||||
'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (:id, :appliedAt)',
|
||||
{
|
||||
replacements: { id, appliedAt: new Date().toISOString() },
|
||||
transaction,
|
||||
},
|
||||
'INSERT INTO "SchemaMigrations" ("id", "applied_at") VALUES (?, ?)',
|
||||
[id, new Date().toISOString()],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import { AuthInfo } from '../data/system';
|
||||
import { App } from '../data/open';
|
||||
import Keyv from 'keyv';
|
||||
import KeyvSqlite from '@keyv/sqlite';
|
||||
import config from '../config';
|
||||
import path from 'path';
|
||||
import { CacheStore } from '../data/cacheStore';
|
||||
|
||||
export enum EKeyv {
|
||||
'apps' = 'apps',
|
||||
@@ -17,8 +15,12 @@ export interface IKeyvStore {
|
||||
lang: string;
|
||||
}
|
||||
|
||||
const keyvSqlite = new KeyvSqlite(path.join(config.dbPath, 'keyv.sqlite'));
|
||||
export const keyvStore = new Keyv<IKeyvStore>({ store: keyvSqlite });
|
||||
const keyvSqlite = new CacheStore();
|
||||
// A stopped database is unavailable, not a cache miss or an expired session.
|
||||
export const keyvStore = new Keyv<IKeyvStore>({
|
||||
store: keyvSqlite,
|
||||
throwOnErrors: true,
|
||||
});
|
||||
|
||||
export const shareStore = {
|
||||
getAuthInfo() {
|
||||
|
||||
+12
-31
@@ -2,21 +2,15 @@ import express, { Application, Request } from 'express';
|
||||
import { AuthDataType, SystemModel } from '../data/system';
|
||||
import { normalizeClientIp } from './clientIp';
|
||||
import { t } from './i18n';
|
||||
|
||||
const DEFAULT_TRUST_PROXY = 'loopback';
|
||||
|
||||
type TrustProxyValue = boolean | number | string;
|
||||
|
||||
let activeApp: Application | undefined;
|
||||
|
||||
function getEnvironmentSetting(): string {
|
||||
return process.env.QL_TRUST_PROXY?.trim() || '';
|
||||
}
|
||||
|
||||
function normalizeSetting(value?: string): string {
|
||||
return value?.trim() || DEFAULT_TRUST_PROXY;
|
||||
}
|
||||
|
||||
export function resolveTrustProxy(value?: string): TrustProxyValue {
|
||||
const setting = normalizeSetting(value);
|
||||
if (setting === 'true' || setting === 'false') {
|
||||
@@ -27,7 +21,6 @@ export function resolveTrustProxy(value?: string): TrustProxyValue {
|
||||
}
|
||||
return setting;
|
||||
}
|
||||
|
||||
function validateTrustProxy(value: string): string {
|
||||
const setting = normalizeSetting(value);
|
||||
if (setting.length > 500 || /[\r\n]/.test(setting)) {
|
||||
@@ -36,25 +29,22 @@ function validateTrustProxy(value: string): string {
|
||||
if (/^\d+$/.test(setting) && Number(setting) > 20) {
|
||||
throw new Error(t('代理层数不能超过 20'));
|
||||
}
|
||||
|
||||
const probe = express();
|
||||
probe.set('trust proxy', resolveTrustProxy(setting));
|
||||
return setting;
|
||||
}
|
||||
|
||||
async function getStoredSetting(): Promise<string> {
|
||||
const doc = await SystemModel.findOne({
|
||||
where: { type: AuthDataType.systemConfig },
|
||||
});
|
||||
const info = (doc?.get('info') || {}) as Record<string, unknown>;
|
||||
const doc = await SystemModel.findOne(
|
||||
{ type: AuthDataType.systemConfig },
|
||||
{},
|
||||
);
|
||||
const info = (doc?.info || {}) as Record<string, unknown>;
|
||||
return typeof info.trustProxy === 'string' ? info.trustProxy : '';
|
||||
}
|
||||
|
||||
export async function getTrustProxyConfig() {
|
||||
const environmentSetting = getEnvironmentSetting();
|
||||
const storedSetting = await getStoredSetting();
|
||||
const trustProxy = normalizeSetting(environmentSetting || storedSetting);
|
||||
|
||||
return {
|
||||
trustProxy,
|
||||
source: environmentSetting
|
||||
@@ -65,36 +55,31 @@ export async function getTrustProxyConfig() {
|
||||
editable: !environmentSetting,
|
||||
};
|
||||
}
|
||||
|
||||
export async function initializeTrustProxy(app: Application) {
|
||||
activeApp = app;
|
||||
const { trustProxy } = await getTrustProxyConfig();
|
||||
app.set('trust proxy', resolveTrustProxy(trustProxy));
|
||||
}
|
||||
|
||||
export async function updateTrustProxy(value: string) {
|
||||
if (getEnvironmentSetting()) {
|
||||
throw new Error(t('环境变量 QL_TRUST_PROXY 已生效,系统设置不可覆盖'));
|
||||
}
|
||||
|
||||
const trustProxy = validateTrustProxy(value);
|
||||
const doc = await SystemModel.findOne({
|
||||
where: { type: AuthDataType.systemConfig },
|
||||
});
|
||||
const doc = await SystemModel.findOne(
|
||||
{ type: AuthDataType.systemConfig },
|
||||
{},
|
||||
);
|
||||
if (!doc) {
|
||||
throw new Error(t('系统配置不存在'));
|
||||
}
|
||||
|
||||
const plain = doc.get({ plain: true });
|
||||
await SystemModel.update(
|
||||
const plain = doc;
|
||||
await SystemModel.nativeUpdate(
|
||||
{ id: plain.id },
|
||||
{ info: { ...(plain.info || {}), trustProxy } as any },
|
||||
{ where: { id: plain.id } },
|
||||
);
|
||||
activeApp?.set('trust proxy', resolveTrustProxy(trustProxy));
|
||||
|
||||
return getTrustProxyConfig();
|
||||
}
|
||||
|
||||
function parseForwardedFor(value: string | string[] | undefined): string[] {
|
||||
const values = Array.isArray(value) ? value : value ? [value] : [];
|
||||
return values
|
||||
@@ -102,7 +87,6 @@ function parseForwardedFor(value: string | string[] | undefined): string[] {
|
||||
.map((item) => normalizeClientIp(item))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
export async function diagnoseClientIp(req: Request) {
|
||||
const remoteAddress = normalizeClientIp(req.socket.remoteAddress);
|
||||
const forwardedFor = parseForwardedFor(req.headers['x-forwarded-for']);
|
||||
@@ -112,7 +96,6 @@ export async function diagnoseClientIp(req: Request) {
|
||||
const trust = req.app.get('trust proxy fn') as
|
||||
| ((ip: string, hop: number) => boolean)
|
||||
| undefined;
|
||||
|
||||
let selectedIndex = Math.max(hopsFromApp.length - 1, 0);
|
||||
for (let index = 0; index < hopsFromApp.length - 1; index += 1) {
|
||||
if (!trust?.(hopsFromApp[index], index)) {
|
||||
@@ -120,7 +103,6 @@ export async function diagnoseClientIp(req: Request) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const hops = hopsFromApp.map((ip, index) => ({
|
||||
ip,
|
||||
hop: index,
|
||||
@@ -131,7 +113,6 @@ export async function diagnoseClientIp(req: Request) {
|
||||
? 'client'
|
||||
: 'not_checked',
|
||||
}));
|
||||
|
||||
return {
|
||||
...(await getTrustProxyConfig()),
|
||||
remoteAddress,
|
||||
|
||||
+1
-1
@@ -23,5 +23,5 @@
|
||||
"esModuleInterop": true
|
||||
},
|
||||
"include": ["./**/*"],
|
||||
"exclude": ["node_modules"]
|
||||
"exclude": ["node_modules", "./data/databaseWorker.ts"]
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// One installed verifier handles release downloads and local offline archives.
|
||||
const fs = require('node:fs/promises');
|
||||
const { constants, createReadStream } = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const crypto = require('node:crypto');
|
||||
const os = require('node:os');
|
||||
@@ -176,23 +177,39 @@ async function verifyPayload(selected, manifest, target) {
|
||||
});
|
||||
const names = Object.keys(actual).sort(),
|
||||
expected = Object.keys(manifest.files).sort();
|
||||
if (
|
||||
JSON.stringify(names) !== JSON.stringify(expected) ||
|
||||
names.some(
|
||||
(name) =>
|
||||
JSON.stringify(actual[name]) !== JSON.stringify(manifest.files[name]),
|
||||
)
|
||||
)
|
||||
const differences = [...new Set([...expected, ...names])].filter(
|
||||
(name) =>
|
||||
!Object.hasOwn(actual, name) ||
|
||||
!Object.hasOwn(manifest.files, name) ||
|
||||
JSON.stringify(actual[name]) !== JSON.stringify(manifest.files[name]),
|
||||
);
|
||||
if (differences.length)
|
||||
throw new Error(
|
||||
'Platform bundle files or dependency checksums do not match',
|
||||
'Platform bundle files or dependency checksums do not match: ' +
|
||||
JSON.stringify({
|
||||
count: differences.length,
|
||||
files: differences.slice(0, 5).map((name) => ({
|
||||
name,
|
||||
expected: manifest.files[name] || null,
|
||||
actual: actual[name] || null,
|
||||
})),
|
||||
}),
|
||||
);
|
||||
if (
|
||||
sha256(await fs.readFile(path.join(selected.source, 'pnpm-lock.yaml'))) !==
|
||||
manifest.lockfileSha256
|
||||
)
|
||||
throw new Error('Platform bundle lockfile mismatch');
|
||||
const pkg = JSON.parse(
|
||||
await fs.readFile(path.join(selected.source, 'package.json'), 'utf8'),
|
||||
);
|
||||
await fs.access(
|
||||
path.join(selected.source, 'node_modules/sqlite3/package.json'),
|
||||
path.join(
|
||||
selected.source,
|
||||
pkg.dependencies?.['@mikro-orm/sql']
|
||||
? 'node_modules/@mikro-orm/sql/package.json'
|
||||
: 'node_modules/sqlite3/package.json',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -202,34 +219,58 @@ async function smokeDependencies(source, signal) {
|
||||
const path = require('node:path'),fs = require('node:fs');
|
||||
const root = fs.realpathSync(process.cwd());
|
||||
const r = require('node:module').createRequire(path.join(root,'package.json'));
|
||||
for (const name of ['sqlite3','sequelize','cron-parser','@grpc/grpc-js','undici']) {
|
||||
const pkg=r('./package.json'), modern=Boolean(pkg.dependencies?.['@mikro-orm/sql']);
|
||||
if (modern) {
|
||||
const [major,minor]=process.versions.node.split('.').map(Number);
|
||||
if(major<22 || (major===22 && minor<17)) throw new Error('Bundle requires Node >=22.17; the existing Node is retained');
|
||||
}
|
||||
for (const name of [...(modern?['@mikro-orm/core','@mikro-orm/sql']:['sqlite3','sequelize']),'cron-parser','@grpc/grpc-js','undici']) {
|
||||
if (!fs.realpathSync(r.resolve(name)).startsWith(root + path.sep)) throw new Error('Dependency outside staged payload: '+name);
|
||||
r(name);
|
||||
}
|
||||
if (modern) {
|
||||
const {DatabaseSync}=require('node:sqlite'), db=new DatabaseSync(':memory:');
|
||||
try {
|
||||
db.exec('CREATE TABLE verify_bundle(value INTEGER);INSERT INTO verify_bundle VALUES(42)');
|
||||
if(db.prepare('SELECT value FROM verify_bundle').get().value!==42) throw new Error('SQLite readback failed');
|
||||
} finally {db.close();}
|
||||
} else {
|
||||
const sqlite=r('sqlite3'), db=new sqlite.Database(':memory:');
|
||||
db.serialize(()=>{db.run('CREATE TABLE verify_bundle(value INTEGER)');db.run('INSERT INTO verify_bundle VALUES(42)');
|
||||
db.get('SELECT value FROM verify_bundle',(err,row)=>{if(err||row.value!==42){console.error(err||'SQLite readback failed');process.exitCode=1;}db.close();});});
|
||||
}
|
||||
`;
|
||||
// Match the server's normal runtime. Node 20.18's built-in HTTP parser needs
|
||||
// WebAssembly, which --jitless disables; only archive workers need that flag.
|
||||
await execute(process.execPath, [...serverRuntimeArgs(process), '-e', probe], {
|
||||
cwd: source,
|
||||
env: { ...process.env, NODE_PATH: '', NODE_OPTIONS: '' },
|
||||
// ARM1176 devices need the same startup allowance as the backend probe.
|
||||
// Keep this bounded and cancellable without forcing an unnecessary repair.
|
||||
timeout: nodeRuntimeArgs().length ? 300000 : 30000,
|
||||
signal,
|
||||
});
|
||||
await execute(
|
||||
process.execPath,
|
||||
[...serverRuntimeArgs(process), '-e', probe],
|
||||
{
|
||||
cwd: source,
|
||||
env: { ...process.env, NODE_PATH: '', NODE_OPTIONS: '' },
|
||||
// ARM1176 devices need the same startup allowance as the backend probe.
|
||||
// Keep this bounded and cancellable without forcing an unnecessary repair.
|
||||
timeout: nodeRuntimeArgs().length ? 300000 : 30000,
|
||||
signal,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function extractArchive(archive, destination) {
|
||||
const entries = new Map(),
|
||||
links = new Set();
|
||||
directories = new Map([['qinglong', 0o755]]);
|
||||
const header = (entry) => ({
|
||||
type: entry.type,
|
||||
size: entry.size,
|
||||
mode: entry.mode,
|
||||
linkpath: entry.linkpath,
|
||||
});
|
||||
let unpacked = 0,
|
||||
invalid;
|
||||
await tar.t({
|
||||
file: archive,
|
||||
strict: true,
|
||||
maxReadSize: 256 * 1024,
|
||||
onReadEntry(entry) {
|
||||
try {
|
||||
const name = entry.path.replace(/\/$/, '');
|
||||
@@ -237,16 +278,19 @@ async function extractArchive(archive, destination) {
|
||||
!name ||
|
||||
name.includes('\\') ||
|
||||
/[\x00-\x1f\x7f]/.test(name) ||
|
||||
name.split('/').length > 1024 ||
|
||||
name.split('/').some((p) => !p || p === '.' || p === '..') ||
|
||||
!(name === 'qinglong' || name.startsWith('qinglong/')) ||
|
||||
entries.has(name) ||
|
||||
!['File', 'OldFile', 'Directory', 'SymbolicLink'].includes(entry.type)
|
||||
)
|
||||
throw new Error(`Unsafe bundle archive entry: ${name}`);
|
||||
if (!Number.isSafeInteger(entry.size) || entry.size < 0)
|
||||
throw new Error('Invalid bundle archive entry size');
|
||||
unpacked += entry.size;
|
||||
if (unpacked > 2 * 1024 ** 3 || entries.size > 200000)
|
||||
throw new Error('Bundle archive exceeds extraction limits');
|
||||
entries.set(name, entry.type);
|
||||
entries.set(name, header(entry));
|
||||
if (entry.type === 'SymbolicLink') {
|
||||
if (
|
||||
path.posix.isAbsolute(entry.linkpath) ||
|
||||
@@ -257,7 +301,6 @@ async function extractArchive(archive, destination) {
|
||||
)
|
||||
)
|
||||
throw new Error('Unsafe bundle symlink');
|
||||
links.add(name);
|
||||
}
|
||||
} catch (error) {
|
||||
invalid ||= error;
|
||||
@@ -265,22 +308,101 @@ async function extractArchive(archive, destination) {
|
||||
},
|
||||
});
|
||||
if (invalid) throw invalid;
|
||||
for (const name of entries.keys()) {
|
||||
for (const [name, entry] of entries) {
|
||||
if (entry.type === 'Directory')
|
||||
directories.set(name, ((entry.mode ?? 0o755) & 0o777) | 0o700);
|
||||
let parent = path.posix.dirname(name);
|
||||
while (parent !== '.') {
|
||||
if (links.has(parent))
|
||||
throw new Error('Archive writes through a symlink');
|
||||
if (entries.has(parent) && entries.get(parent).type !== 'Directory')
|
||||
throw new Error('Archive writes through a symlink or non-directory');
|
||||
if (!directories.has(parent)) directories.set(parent, 0o755);
|
||||
parent = path.posix.dirname(parent);
|
||||
}
|
||||
}
|
||||
await tar.x({
|
||||
file: archive,
|
||||
// The extraction root must be new and private. Create every parent before
|
||||
// links, and keep directories private/searchable until all writes finish.
|
||||
const directoryNames = [...directories.keys()].sort(
|
||||
(a, b) => a.split('/').length - b.split('/').length,
|
||||
);
|
||||
for (const name of directoryNames) {
|
||||
const filename = path.join(destination, name);
|
||||
await fs.mkdir(filename, { mode: 0o700 });
|
||||
await fs.chmod(filename, 0o700);
|
||||
}
|
||||
let writes = Promise.resolve(),
|
||||
failure;
|
||||
const active = new Set(),
|
||||
seen = new Set();
|
||||
const parser = tar.t({
|
||||
cwd: destination,
|
||||
strict: true,
|
||||
preservePaths: false,
|
||||
noChown: true,
|
||||
umask: 0,
|
||||
noResume: true,
|
||||
onReadEntry(entry) {
|
||||
active.add(entry);
|
||||
// A parser failure can destroy queued entries before their turn starts.
|
||||
entry.on('error', () => {});
|
||||
writes = writes
|
||||
.then(async () => {
|
||||
if (failure) return entry.resume();
|
||||
const name = entry.path.replace(/\/$/, '');
|
||||
if (
|
||||
seen.has(name) ||
|
||||
JSON.stringify(entries.get(name)) !== JSON.stringify(header(entry))
|
||||
)
|
||||
throw new Error('Archive changed during extraction');
|
||||
seen.add(name);
|
||||
const filename = path.join(destination, name);
|
||||
if (entry.type === 'Directory') return entry.resume();
|
||||
if (entry.type === 'SymbolicLink') {
|
||||
await fs.symlink(entry.linkpath, filename);
|
||||
return entry.resume();
|
||||
}
|
||||
const mode = (entry.mode ?? 0o644) & 0o777;
|
||||
const handle = await fs.open(
|
||||
filename,
|
||||
constants.O_WRONLY |
|
||||
constants.O_CREAT |
|
||||
constants.O_EXCL |
|
||||
constants.O_NOFOLLOW,
|
||||
0o600,
|
||||
);
|
||||
try {
|
||||
// tar's fs-minipass writer discards successful partial write/writev
|
||||
// results. Node's writeFile loops until each chunk is fully written.
|
||||
// One entry/chunk at a time bounds memory and open FDs on ARMv6.
|
||||
for await (const chunk of entry) await handle.writeFile(chunk);
|
||||
await handle.chmod(mode);
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
failure ||= error;
|
||||
entry.resume();
|
||||
})
|
||||
.finally(() => active.delete(entry));
|
||||
},
|
||||
});
|
||||
const input = createReadStream(archive, { highWaterMark: 256 * 1024 });
|
||||
try {
|
||||
await new Promise((resolve, reject) => {
|
||||
input.once('error', reject);
|
||||
parser.once('error', reject);
|
||||
parser.once('end', resolve);
|
||||
input.pipe(parser);
|
||||
});
|
||||
} catch (error) {
|
||||
failure ||= error;
|
||||
for (const entry of active) entry.destroy(error);
|
||||
} finally {
|
||||
input.destroy();
|
||||
}
|
||||
await writes;
|
||||
if (failure) throw failure;
|
||||
if (seen.size !== entries.size)
|
||||
throw new Error('Archive changed during extraction');
|
||||
for (const name of directoryNames.reverse())
|
||||
await fs.chmod(path.join(destination, name), directories.get(name));
|
||||
}
|
||||
|
||||
async function stageBundle({
|
||||
|
||||
@@ -299,7 +299,8 @@ async function repairDependencies(source, { target, signal, env }) {
|
||||
'--child-concurrency=1',
|
||||
'--network-concurrency=2',
|
||||
]);
|
||||
await run(['rebuild', 'sqlite3', '@whyour/sqlite3']);
|
||||
if (pkg.dependencies?.sqlite3)
|
||||
await run(['rebuild', 'sqlite3', '@whyour/sqlite3']);
|
||||
if (
|
||||
!(await fs.readFile(path.join(source, 'pnpm-lock.yaml'))).equals(
|
||||
await fs.readFile(path.join(workspace, 'pnpm-lock.yaml')),
|
||||
|
||||
+18
-2
@@ -1,6 +1,6 @@
|
||||
# Run Node package installation natively on the builder. Node/npm can spin at
|
||||
# 100% CPU when Alpine s390x is emulated through QEMU.
|
||||
FROM --platform=$BUILDPLATFORM node:18-alpine3.18 AS builder
|
||||
FROM --platform=$BUILDPLATFORM node:24-alpine3.24 AS builder
|
||||
|
||||
ARG TARGETARCH
|
||||
ENV NPM_CONFIG_PREFIX=/opt/node-global
|
||||
@@ -20,10 +20,18 @@ RUN set -x \
|
||||
&& npm_config_target_platform=linux \
|
||||
npm_config_target_arch="${NODE_ARCH}" \
|
||||
npm_config_target_libc=musl \
|
||||
pnpm install --prod --frozen-lockfile
|
||||
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
|
||||
|
||||
# The amd64 APK runtime has crashed in native URL parsing on dev. Keep its
|
||||
# URL implementation bundled with the pinned upstream musl Node binary.
|
||||
FROM --platform=linux/amd64 node:24.18.1-alpine3.24@sha256:f70403e87646dc51b45295f4b8b70cdad0b63d2297c4c9899119b03f7af7a6b3 AS node-runtime-amd64
|
||||
|
||||
FROM python:3.11-alpine
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
COPY --from=node-runtime-amd64 /usr/local/bin/node /tmp/node-runtime-amd64
|
||||
|
||||
ARG QL_MAINTAINER="whyour"
|
||||
LABEL maintainer="${QL_MAINTAINER}"
|
||||
ARG QL_URL=https://github.com/${QL_MAINTAINER}/qinglong.git
|
||||
@@ -62,6 +70,10 @@ RUN set -x \
|
||||
netcat-openbsd \
|
||||
unzip \
|
||||
npm \
|
||||
&& if [ "$TARGETARCH" = amd64 ]; then \
|
||||
mv /tmp/node-runtime-amd64 /usr/local/bin/node; \
|
||||
ln -sf /usr/local/bin/node /usr/bin/node; \
|
||||
else rm /tmp/node-runtime-amd64; fi \
|
||||
&& rm -rf /var/cache/apk/* \
|
||||
&& apk update \
|
||||
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
|
||||
@@ -86,6 +98,10 @@ COPY static/ /ql/static/
|
||||
COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/
|
||||
COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml
|
||||
RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml
|
||||
RUN if [ "$TARGETARCH" = arm ] && [ "${TARGETVARIANT:-}" = v6 ]; then \
|
||||
QEMU_CPU=arm1176 node --jitless ${QL_DIR}/shell/node-runtime-probe.cjs; \
|
||||
else node ${QL_DIR}/shell/node-runtime-probe.cjs; fi
|
||||
|
||||
|
||||
ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \
|
||||
PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \
|
||||
|
||||
+18
-2
@@ -1,6 +1,6 @@
|
||||
# Run Node package installation natively on the builder. Node/npm can spin at
|
||||
# 100% CPU when Alpine s390x is emulated through QEMU.
|
||||
FROM --platform=$BUILDPLATFORM node:18-alpine3.18 AS builder
|
||||
FROM --platform=$BUILDPLATFORM node:24-alpine3.24 AS builder
|
||||
|
||||
ARG TARGETARCH
|
||||
ENV NPM_CONFIG_PREFIX=/opt/node-global
|
||||
@@ -20,10 +20,18 @@ RUN set -x \
|
||||
&& npm_config_target_platform=linux \
|
||||
npm_config_target_arch="${NODE_ARCH}" \
|
||||
npm_config_target_libc=musl \
|
||||
pnpm install --prod --frozen-lockfile
|
||||
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
|
||||
|
||||
# The amd64 APK runtime has crashed in native URL parsing on dev. Keep its
|
||||
# URL implementation bundled with the pinned upstream musl Node binary.
|
||||
FROM --platform=linux/amd64 node:24.18.1-alpine3.24@sha256:f70403e87646dc51b45295f4b8b70cdad0b63d2297c4c9899119b03f7af7a6b3 AS node-runtime-amd64
|
||||
|
||||
FROM python:3.10-alpine
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
COPY --from=node-runtime-amd64 /usr/local/bin/node /tmp/node-runtime-amd64
|
||||
|
||||
ARG QL_MAINTAINER="whyour"
|
||||
LABEL maintainer="${QL_MAINTAINER}"
|
||||
ARG QL_URL=https://github.com/${QL_MAINTAINER}/qinglong.git
|
||||
@@ -62,6 +70,10 @@ RUN set -x \
|
||||
netcat-openbsd \
|
||||
unzip \
|
||||
npm \
|
||||
&& if [ "$TARGETARCH" = amd64 ]; then \
|
||||
mv /tmp/node-runtime-amd64 /usr/local/bin/node; \
|
||||
ln -sf /usr/local/bin/node /usr/bin/node; \
|
||||
else rm /tmp/node-runtime-amd64; fi \
|
||||
&& rm -rf /var/cache/apk/* \
|
||||
&& apk update \
|
||||
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
|
||||
@@ -86,6 +98,10 @@ COPY static/ /ql/static/
|
||||
COPY docker/verify-build.cjs docker/build-manifest.cjs /tmp/
|
||||
COPY --from=builder /tmp/build/pnpm-lock.yaml /tmp/dependency-lock.yaml
|
||||
RUN cd ${QL_DIR} && node /tmp/verify-build.cjs /tmp/dependency-lock.yaml
|
||||
RUN if [ "$TARGETARCH" = arm ] && [ "${TARGETVARIANT:-}" = v6 ]; then \
|
||||
QEMU_CPU=arm1176 node --jitless ${QL_DIR}/shell/node-runtime-probe.cjs; \
|
||||
else node ${QL_DIR}/shell/node-runtime-probe.cjs; fi
|
||||
|
||||
|
||||
ENV PNPM_HOME=${QL_DIR}/data/dep_cache/node \
|
||||
PYTHON_HOME=${QL_DIR}/data/dep_cache/python3 \
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# Node 20 Bookworm is the latest official Node image variant that covers the
|
||||
# full Debian build matrix, including arm/v7, ppc64le, and s390x.
|
||||
FROM node:20-bookworm-slim AS nodebuilder
|
||||
FROM python:3.11.14-slim-bookworm AS nodebuilder
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
COPY docker/install-node-runtime.sh /tmp/install-node-runtime.sh
|
||||
RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates curl xz-utils libatomic1 libstdc++6 && \
|
||||
sh /tmp/install-node-runtime.sh /usr/local
|
||||
|
||||
FROM python:3.11.14-slim-bookworm AS builder
|
||||
COPY package.json .npmrc pnpm-lock.yaml /tmp/build/
|
||||
@@ -13,7 +16,7 @@ RUN set -x && \
|
||||
npm i -g pnpm@8.3.1 && \
|
||||
test "$(pnpm --version)" = "8.3.1" && \
|
||||
cd /tmp/build && \
|
||||
pnpm install --prod --frozen-lockfile
|
||||
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
|
||||
|
||||
FROM python:3.11.14-slim-bookworm
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# Node 20 Bookworm is the latest official Node image variant that covers the
|
||||
# full Debian build matrix, including arm/v7, ppc64le, and s390x.
|
||||
FROM node:20-bookworm-slim AS nodebuilder
|
||||
FROM python:3.10.19-slim-bookworm@sha256:23f63358922e79a794f71be8f3723c84e5ccca9638af3f74456dc73d6184499e AS nodebuilder
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
COPY docker/install-node-runtime.sh /tmp/install-node-runtime.sh
|
||||
RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates curl xz-utils libatomic1 libstdc++6 && \
|
||||
sh /tmp/install-node-runtime.sh /usr/local
|
||||
|
||||
# Keep the full release matrix, including s390x, on the same immutable base.
|
||||
# The floating 3.10-slim-bookworm tag no longer includes s390x.
|
||||
@@ -14,7 +17,7 @@ RUN set -x && \
|
||||
apt-get install --no-install-recommends -y libatomic1 && \
|
||||
npm i -g pnpm@8.3.1 && \
|
||||
cd /tmp/build && \
|
||||
pnpm install --prod --frozen-lockfile
|
||||
QL_BUILD_ONLY=1 pnpm install --prod --frozen-lockfile
|
||||
|
||||
FROM python:3.10.19-slim-bookworm@sha256:23f63358922e79a794f71be8f3723c84e5ccca9638af3f74456dc73d6184499e
|
||||
|
||||
|
||||
@@ -6,18 +6,7 @@ ENV QL_BUNDLE_BASE_IMAGE=${BASE_IMAGE}
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
USER root
|
||||
# The generic "arm" sqlite release is ARMv7. ARMv6 must compile its own addon.
|
||||
RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
|
||||
apk add --no-cache --virtual .ql-bundle-build build-base binutils python3 nodejs-dev && \
|
||||
cd "$(readlink -f /ql/node_modules/sqlite3)" && \
|
||||
CFLAGS='-march=armv6 -mfpu=vfp -mfloat-abi=hard' \
|
||||
CXXFLAGS='-march=armv6 -mfpu=vfp -mfloat-abi=hard' \
|
||||
npm_config_nodedir=/usr npm_config_build_from_source=true QEMU_CPU=arm1176 \
|
||||
node -e 'require("node:child_process").execFileSync(process.execPath,[require.resolve("@mapbox/node-pre-gyp/bin/node-pre-gyp"),"rebuild","--napi_build_version=6"],{stdio:"inherit"})' && \
|
||||
readelf -A lib/binding/napi-v6-linux-arm/node_sqlite3.node | grep -E 'Tag_CPU_arch: v6($|K)' && \
|
||||
rm -rf build && \
|
||||
apk del .ql-bundle-build; \
|
||||
fi
|
||||
# SQLite comes from the retained Node runtime; no architecture-specific addon.
|
||||
RUN ulimit -c 0; if [ "$TARGETARCH/$TARGETVARIANT" = arm/v6 ]; then \
|
||||
export QEMU_CPU=arm1176; \
|
||||
node --jitless /ql/scripts/build-upgrade-bundle.cjs /ql /out && \
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# Official binaries, pinned to their release checksums. Node 24 no longer
|
||||
# publishes ARMv7 binaries, so that existing Debian target uses Node 22 LTS.
|
||||
node_version=24.18.1
|
||||
case "${TARGETARCH:?}/${TARGETVARIANT:-}" in
|
||||
amd64/*) node_arch=x64; node_sha=d6c664df3f3f61458e8c277585571328522d705166723a7c7823a9253a4d15a0 ;;
|
||||
arm64/*) node_arch=arm64; node_sha=7201e3a09dc825bac57867c81913e2b8f0ef87d04cb9082af4cda82f6ff3d88c ;;
|
||||
ppc64le/*) node_arch=ppc64le; node_sha=c47812c13862be7f6e71194c434c4d78a99096e72efb9c31a2627437e056d669 ;;
|
||||
s390x/*) node_arch=s390x; node_sha=acf57bd726f35afc2ba8475194e5a8bd532c00e1a3b2436295b2fbdde199d0e9 ;;
|
||||
arm/v7) node_version=22.23.2; node_arch=armv7l; node_sha=ef8f26a3de19acd8c23548e6c3cfc2052610b0e67abb5fd64dbd92c8b1c1245b ;;
|
||||
*) echo "Unsupported Debian Node architecture: ${TARGETARCH}/${TARGETVARIANT:-}" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
if [ "${1:-}" = --print ]; then
|
||||
printf '%s %s %s\n' "$node_version" "$node_arch" "$node_sha"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
node_prefix=${1:-/usr/local}
|
||||
node_work_dir=$(mktemp -d)
|
||||
trap 'rm -rf "$node_work_dir"' EXIT
|
||||
node_archive="node-v${node_version}-linux-${node_arch}.tar.xz"
|
||||
curl --fail --location --retry 3 --connect-timeout 20 --max-time 300 \
|
||||
"https://nodejs.org/dist/v${node_version}/${node_archive}" -o "$node_work_dir/$node_archive"
|
||||
printf '%s %s\n' "$node_sha" "$node_work_dir/$node_archive" | sha256sum -c -
|
||||
mkdir -p "$node_prefix"
|
||||
tar -xJf "$node_work_dir/$node_archive" -C "$node_prefix" --strip-components=1 --no-same-owner
|
||||
"$node_prefix/bin/node" -e 'const {DatabaseSync}=require("node:sqlite"); const db=new DatabaseSync(":memory:"); try {db.exec("CREATE TABLE node_probe(value INTEGER);INSERT INTO node_probe VALUES(42)");if(db.prepare("SELECT value FROM node_probe").get().value!==42)throw Error("Node SQLite probe failed");}finally{db.close();}'
|
||||
@@ -1,12 +1,12 @@
|
||||
# 平台完整升级包
|
||||
|
||||
CI 从同次构建的不可变镜像摘要导出完整 tar.gz,包含源码中的运行脚本和模板、前后端编译产物、锁文件和已经安装的生产 `node_modules`。不包含 Node 可执行文件、用户数据、`.env`、Git 历史或 npm 缓存。保留现有 Node 和用户脚本依赖。
|
||||
CI 从同次构建的不可变镜像摘要导出完整 tar.gz,包含源码中的运行脚本和模板、前后端编译产物、锁文件和已经安装的生产 `node_modules`。不包含 Node 可执行文件、用户数据、`.env`、Git 历史或 npm 缓存。保留现有 Node 和用户脚本依赖。面板最低需要 Node 22.17,推荐 Node 24;Node 20 安装先升级运行时,应用更新包不会替换 Node。
|
||||
|
||||
## 在线升级
|
||||
|
||||
`package.json` 中 `qinglongUpgradeFormat: 2` 表示使用完整包。旧版升级器仍可通过原归档路径完成首次迁移;完整包版本不会在平台包缺失、损坏或不兼容时退回设备端 npm 安装。仅当完整包已通过校验、依赖加载或后端预启动明确报告依赖错误时,在线升级会按同一冻结锁文件修复一次生产依赖,然后再次预启动;第二次失败即停止,保留正在运行的旧服务。
|
||||
`package.json` 中 `qinglongUpgradeFormat: 2` 表示使用完整包。旧源码归档升级器可在运行时满足最低版本后迁移;已安装的旧格式 2 校验器仍要求 sqlite3 原生模块,首次切换到 node:sqlite 包需使用同批次新版独立安装器执行完整安装和重启,或先更新镜像。旧校验器会拒绝新依赖布局并保留旧服务;完整包版本不会在平台包缺失、损坏或不兼容时退回设备端 npm 安装。仅当完整包已通过校验、依赖加载或后端预启动明确报告依赖错误时,在线升级会按同一冻结锁文件修复一次生产依赖,然后再次预启动;第二次失败即停止,保留正在运行的旧服务。
|
||||
|
||||
面板下载任务与本地离线入口共用 `platformBundle.cjs`:平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → SQLite/Sequelize/cron-parser/gRPC/undici 加载及 SQLite 内存库读写 → 隔离后端预启动 → 发布就绪指针。预启动使用现有 Node、临时数据目录和 loopback 随机端口,运行真实数据库建表/迁移和事务读写、gRPC 服务,以及与正常启动共用的 HTTP 中间件、路由和 WebSocket 初始化,再通过 `/api/health` 确认双服务与目标版本、提交和后端摘要;不运行任务、订阅、通知、用户启动钩子和系统依赖安装。临时库不是用户数据库副本,因此实际应用后的健康检查和回退仍不可省略。点击重启时,接口只检查就绪记录并提交后台任务;完整文件与依赖复验由独立工作进程执行,复验通过后才停止旧服务,避免小设备的预检耗时超过 HTTP 请求期限。离线升级同步面板任务状态,并与在线任务互斥;中断准备时保留之前的可用就绪包。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。
|
||||
面板下载任务与本地离线入口共用 `platformBundle.cjs`:平台匹配 → 归档安全检查 → 全文件/链接/权限和锁文件校验 → MikroORM/cron-parser/gRPC/undici 加载及 node:sqlite 内存库读写(旧包继续检查 SQLite/Sequelize) → 隔离后端预启动 → 发布就绪指针。预启动使用现有 Node、临时数据目录和 loopback 随机端口,运行真实数据库建表/迁移和事务读写、gRPC 服务,以及与正常启动共用的 HTTP 中间件、路由和 WebSocket 初始化,再通过 `/api/health` 确认双服务与目标版本、提交和后端摘要;不运行任务、订阅、通知、用户启动钩子和系统依赖安装。临时库不是用户数据库副本,因此实际应用后的健康检查和回退仍不可省略。点击重启时,接口只检查就绪记录并提交后台任务;完整文件与依赖复验由独立工作进程执行,复验通过后才停止旧服务,避免小设备的预检耗时超过 HTTP 请求期限。离线升级同步面板任务状态,并与在线任务互斥;中断准备时保留之前的可用就绪包。原有服务停止、文件备份、启动健康检查及失败恢复仍由共享升级引擎处理。
|
||||
|
||||
正式升级包跟随 master 同一次版本镜像构建;develop 的普通提交也自动构建开发包。两者都直接使用该次 Alpine/Debian 镜像的不可变 digest。Git tag 工作流和 `static_sync_only` 不生成完整包。需要重建当前开发包时,可手动运行同一工作流:
|
||||
|
||||
@@ -32,7 +32,7 @@ gh workflow run build-docker-image.yml --ref develop
|
||||
node install-bundle.cjs --root /ql --file /absolute/path/qinglong-<commit>-linux-arm64-musl.tar.gz
|
||||
```
|
||||
|
||||
只准备,稍后手动重启:
|
||||
已迁移的安装只准备,稍后手动重启(首次从旧格式 2 迁移应直接执行上一条新版独立安装器命令,不能调用旧版 `ql reload system`):
|
||||
|
||||
```sh
|
||||
node install-bundle.cjs --root /ql --file /absolute/path/package.tar.gz --download-only
|
||||
@@ -45,7 +45,7 @@ ql reload system
|
||||
node install-bundle.cjs --root /ql --file /absolute/path/package.tar.gz --download-only --repair-deps
|
||||
```
|
||||
|
||||
修复需要已有 pnpm 8.3.1 及必要编译工具。依赖在私有目录按 `--prod --frozen-lockfile` 重装,仅重建 SQLite;ARMv6 强制源码编译和 ARMv6 硬浮点指令,禁止下载 generic ARM 的 ARMv7 二进制。修复仅执行一次;不安装或替换 Node、不改锁文件、不改已校验源码和静态产物。应用前以原包校验清单和独立记录的修复依赖摘要复验。
|
||||
修复需要已有 pnpm 8.3.1。新包在私有目录按 `--prod --frozen-lockfile` 重装生产 JavaScript 依赖,不编译 SQLite。只有旧包显式依赖 sqlite3 时才需要编译工具并重建 SQLite;旧 ARMv6 包强制源码编译和 ARMv6 硬浮点指令,禁止下载 generic ARM 的 ARMv7 二进制。修复仅执行一次;不安装或替换 Node、不改锁文件、不改已校验源码和静态产物。应用前以原包校验清单和独立记录的修复依赖摘要复验。
|
||||
|
||||
修复继承调用环境中的标准 HTTP/HTTPS/ALL/NO_PROXY(含小写形式),其余 Node 预加载和编译参数仍隔离。修复安装或 SQLite 重建失败时,清理私有目录,保留当前服务、当前依赖及上一次就绪包;不会标记本次更新就绪。预启动或修复主进程退出时立即终止其进程组,防止持有输出管道的子进程继续写入已清理目录。
|
||||
|
||||
@@ -74,9 +74,9 @@ npm 会排除根目录的 `pnpm-lock.yaml`。主包的原生 `npm pack/publish`
|
||||
|
||||
`package.json#files` 是 npm 包内容的唯一项目规则,不再维护根 `.npmignore`:npm 在根包使用 `files` 时会停用该目录的 `.npmignore` 和 `.gitignore` 规则。打包测试在没有 `.npmignore` 的独立源码目录中注入私有配置、密钥占位文件、缓存及生成的预加载配置,执行真实 `npm pack` 和安装,检查运行文件完整、私有文件及其内容不进入归档。此规则只用于 npm 包,完整平台 tar.gz 仍由平台包构建器处理。
|
||||
|
||||
干净源码完成 `pnpm build:front`、`pnpm build:back`、`pnpm build:info` 后,运行 `node scripts/verify-panel-package.cjs pack /tmp/qinglong-npm` 生成并检查真实 npm 归档。CI 在一次性 Linux 容器中,以 Node 20.18.1 和 Node 24 安装同一个归档,使用真实生产依赖和已安装的 `qinglong`、`ql` 入口,验证启动、损坏下载保留旧服务、在线准备与重启、离线准备与 Bash 重启及账号、配置、环境变量、任务和脚本保留。测试还安装完整性摘要固定的已发布 `@whyour/qinglong@2.23.0`,在旧服务中建立真实数据,再用新 `.tgz` 执行 npm 覆盖安装,验证账号、配置、环境变量、任务和脚本保留。之后升级到同次 Debian 镜像不可变 digest 通过正式 `Dockerfile.upgrade-bundle` 导出的原始平台包,确认运行目录变为 pnpm 布局。该单平台验收包仅上传短期 CI artifact,不发布 Release,也不推进开发或正式更新索引。
|
||||
干净源码完成 `pnpm build:front`、`pnpm build:back`、`pnpm build:info` 后,运行 `node scripts/verify-panel-package.cjs pack /tmp/qinglong-npm` 生成并检查真实 npm 归档。CI 在一次性 Linux 容器中,以 Node 22.17.0 和 Node 24 安装同一个归档,使用真实生产依赖和已安装的 `qinglong`、`ql` 入口,验证启动、损坏下载保留旧服务、在线准备与重启、离线准备与 Bash 重启及账号、配置、环境变量、任务和脚本保留。测试还安装完整性摘要固定的已发布 `@whyour/qinglong@2.23.0`,在旧服务中建立真实数据,再用新 `.tgz` 执行 npm 覆盖安装,验证账号、配置、环境变量、任务和脚本保留。之后升级到同次 Debian 镜像不可变 digest 通过正式 `Dockerfile.upgrade-bundle` 导出的原始平台包,确认运行目录变为 pnpm 布局。该单平台验收包仅上传短期 CI artifact,不发布 Release,也不推进开发或正式更新索引。
|
||||
|
||||
Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端源码构建使用 `.nvmrc` 中的 Node 22,CI 从该文件读取版本,Vercel 前后端项目均配置为 Node 22.x。`package.json` 不声明 `engines.node`,避免 Vercel 用开放的最低运行版本范围覆盖项目的构建版本;`pnpm build:front` 直接执行构建,不下载或切换 Node。面板运行时仍支持 Node 20.18.1 及以上,平台包的清单与安装器独立校验最低 Node 要求,npm 实装验收继续覆盖 Node 20/24。
|
||||
Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端源码构建使用 `.nvmrc` 中的 Node 22,CI 从该文件读取版本,Vercel 前后端项目均配置为 Node 22.x。`package.json` 不声明 `engines.node`,避免 Vercel 用开放的最低运行版本范围覆盖项目的构建版本;`pnpm build:front` 直接执行构建,不下载或切换 Node。面板运行时最低为 Node 22.17,平台包清单、npm preinstall 和数据库进程均检查最低 Node 要求。CI 的 Node 20.18.1 项使用实际新 npm 归档验证拒绝安装,以及本地/全局旧包、`.env`、外置数据和旧服务保留;它不启动新后端。构建专用环境可跳过安装阶段的运行时探测,后端启动不跳过版本检查。
|
||||
|
||||
服务集成测试只能在专用容器中运行:设置 `QL_PANEL_PACKAGE_TEST_ISOLATED=1`、指向隔离 PM2 的 `QL_PANEL_TEST_PM2` 和指向正式构建脚本输出目录的 `QL_PANEL_TEST_BUNDLE_DIR`,然后执行 `node scripts/verify-panel-package.cjs verify /tmp/qinglong-npm /tmp/qinglong-npm-results`。容器需已有 Bash、curl、OpenSSL、Python、jq、nginx 和 procps。旧正式包会重置 PATH,因此验收容器将 PM2 6.0.14 安装到 `/usr/local/bin/pm2`,并以此设置 `QL_PANEL_TEST_PM2`;隔离通过专属容器和独立 `PM2_HOME` 保证。测试创建独立的 npm prefix、HOME、PM2_HOME 和数据目录,结束后清理。
|
||||
|
||||
@@ -84,13 +84,21 @@ Umi 4.4.4 的构建依赖仍使用 Node 24 已移除的 `http_parser`。前端
|
||||
|
||||
master 发布任务等待上述测试通过,直接发布已验证的 `.tgz`,不重新构建或打包。develop 只生成测试归档与验收记录,不发布 npm。
|
||||
|
||||
## 数据库兼容与回滚
|
||||
|
||||
生产环境使用 MikroORM 7 的官方 NodeSqliteDialect。Sequelize 和 sqlite3 只作为开发测试依赖,用于验证旧库读写,不进入生产依赖。九张业务表及既有迁移记录保留原名和结构,不运行 ORM 自动改表;日期保持旧版 UTC 文本格式,JSON、ID、排序精度及 Keyv 缓存格式保持兼容,旧版可读新版写入的数据。
|
||||
|
||||
每个后端进程延迟启动一个有界数据库子进程,避免同步 node:sqlite 阻塞 HTTP 和调度事件循环。事务独占请求队列,嵌套事务使用 SAVEPOINT;事务异常回滚。数据库进程退出或超时后拒绝后续请求、健康检查失败,不自动重放写入;恢复由服务重启完成。后端正常退出会终止其数据库子进程,预启动监督器还会终止整个临时进程组。数据库进程在打开 SQLite 前启动一个仅监视父进程的轻量线程;即使主 SQL 线程阻塞,父进程被强杀后也会终止整个数据库进程并释放事务锁,避免阻止旧服务重启。
|
||||
|
||||
## 平台兼容
|
||||
|
||||
- 最低 Node 为 20.18.1,Node-API 至少 6;安装器只检查,不替换 Node。
|
||||
- 最低 Node 为 22.17,Node-API 至少 6;安装器只检查,不替换 Node。Debian amd64、arm64、ppc64le、s390x 使用 Node 24;官方 Node 24 不提供 Linux ARMv7 二进制,Debian ARMv7 使用 Node 22 LTS。
|
||||
- 以 Node 实际运行架构、ARM 指令集和 libc 类型/版本选择包;不依据 Docker 架构别名猜测。
|
||||
- glibc、musl 分包,并记录构建系统 libc 下限。系统库仍以目标设备加载测试作为最后检查。
|
||||
- ARMv6 与 ARMv7 使用不同包名。ARMv6 在目标镜像中从源码编译 SQLite,禁止复用 generic `arm` 的 ARMv7 预编译件;构建需通过加载和内存库读写,失败则不发布渠道。
|
||||
- ARMv6 的归档构建、在线更新工作进程及独立离线安装器使用 `--jitless`,避开 ARM1176 上 V8 的优化编译崩溃;保留现有 Node 二进制,Node 24 及以上的 ARMv6 面板进程通过共享 `nodeRuntime` 使用 `--no-opt --no-sparkplug --no-maglev`,其他架构和较低 Node 版本保持原启动参数。ARMv6 的 gRPC 工作进程首次启动及重建最多等待 2 分钟;升级后的整体健康检查最多等待 5 分钟,覆盖慢设备按顺序启动数据库、调度与 HTTP 工作进程的耗时;其他平台仍为 60 秒,超时或版本校验失败仍执行回退。依赖加载与后端预启动使用现有 Node 的正常模式,使 Node 20.18.1 的内置 HTTP/WebAssembly 组件保持可用。预启动只有一个后端进程,按顺序初始化数据库、gRPC、HTTP,不启动 PM2/cluster;ARMv6 V8 堆上限 128 MiB、期限 5 分钟,其他平台为 192 MiB/60 秒。堆上限不是 RSS 总内存上限,系统仍需为现有服务和 OpenSSL 留出内存。依赖修复的包管理器及编译使用单并发、低网络并发和有界时长;ARMv6 包管理器采用 `--jitless`、256 MiB 堆上限,并优先复用版本匹配的现有 Node 头文件与实际运行时编译配置;超时、中断会终止整个进程组并清理临时目录。
|
||||
- ARMv6 与 ARMv7 使用不同包名。当前生产依赖没有 SQLite 原生 addon,使用设备 Node 自带的 node:sqlite。ARMv6 Node 本身仍须兼容 ARM1176 和硬浮点;构建需通过内置 SQLite 的加载和内存库读写,失败则不发布渠道。
|
||||
- ARMv6 的归档构建、在线更新工作进程及独立离线安装器使用 `--jitless`,避开 ARM1176 上 V8 的优化编译崩溃;保留现有 Node 二进制,Node 24 及以上的 ARMv6 面板进程通过共享 `nodeRuntime` 使用 `--no-opt --no-sparkplug --no-maglev`,其他架构和较低 Node 版本保持原启动参数。ARMv6 的 gRPC 工作进程首次启动及重建最多等待 2 分钟;升级后的整体健康检查最多等待 5 分钟,覆盖慢设备按顺序启动数据库、调度与 HTTP 工作进程的耗时;其他平台仍为 60 秒,超时或版本校验失败仍执行回退。依赖加载与后端预启动保留 HTTP/WebAssembly 所需能力。预启动使用一个后端和一个数据库子进程,按顺序初始化数据库、gRPC、HTTP,不启动 PM2/cluster;后端的 ARMv6 V8 堆上限 128 MiB、期限 5 分钟,其他平台为 192 MiB/60 秒。数据库执行主线程的 V8 堆上限 128 MiB,ARMv6 使用 `--jitless`;监视线程不加载 ORM、HTTP 或 ts-node,并保留进程的 V8 模式。主线程和监视线程的 V8 限制不是 RSS 总内存上限,系统仍需为现有服务和 OpenSSL 留出内存。依赖修复的包管理器及编译使用单并发、低网络并发和有界时长;ARMv6 包管理器采用 `--jitless`、256 MiB 堆上限,并优先复用版本匹配的现有 Node 头文件与实际运行时编译配置;超时、中断会终止整个进程组并清理临时目录。
|
||||
- tar 中允许 pnpm 的包内符号链接;拒绝绝对路径、越界路径、包外链接、硬链接、设备条目和经符号链接写入的条目。重启前再次校验依赖,暂存失败保留上一次可用的就绪包。
|
||||
|
||||
归档先完整预检路径、类型、大小与链接,再向新建的私有目录逐个流式写入文件。文件写入使用 Node 的完整写入接口,处理短写后才关闭文件;目录权限在写入结束后恢复,避免受调用进程的 umask 影响。读取、解析或写入失败时,等候待写文件关闭后再清理,不发布就绪状态。校验失败会列出最多五个文件的预期和实际权限、哈希或链接信息,不输出文件内容。
|
||||
|
||||
自托管镜像可设置 `QL_UPGRADE_INDEX_URL` 指向兼容索引;要求 HTTPS,测试时仅放行 loopback HTTP。索引和所有平台归档应部署到同一可访问源,避免依赖额外的 raw 域名。
|
||||
|
||||
+14
-11
@@ -49,6 +49,7 @@
|
||||
"version.yaml"
|
||||
],
|
||||
"scripts": {
|
||||
"preinstall": "node -e \"if (process.env.QL_BUILD_ONLY !== '1' && process.env.VERCEL !== '1') require('./shell/runtime-guard.cjs').assertNodeRuntime()\"",
|
||||
"build:cli": "node cli/scripts/build.cjs",
|
||||
"check:cli": "npm --prefix cli run check",
|
||||
"test:cli": "npm run build:cli && node cli/scripts/test.cjs",
|
||||
@@ -57,7 +58,7 @@
|
||||
"start:back": "nodemon ./back/app.ts",
|
||||
"start:front": "max dev",
|
||||
"build:front": "max build",
|
||||
"build:back": "tsc -p back/tsconfig.json",
|
||||
"build:back": "tsc -p back/tsconfig.json && tsc -p back/data/tsconfig.worker.json",
|
||||
"test": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only --test test/back/*.test.cjs test/front/*.test.cjs",
|
||||
"benchmark:execution": "TS_NODE_PROJECT=back/tsconfig.json node -r ts-node/register/transpile-only scripts/benchmark-execution.cjs",
|
||||
"build:info": "node scripts/write-build-info.cjs",
|
||||
@@ -121,11 +122,11 @@
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@ant-design/plots": "^2.6.8",
|
||||
"@bufbuild/protobuf": "^2.10.0",
|
||||
"@grpc/grpc-js": "1.14.5",
|
||||
"@grpc/proto-loader": "^0.8.0",
|
||||
"@keyv/sqlite": "^4.0.1",
|
||||
"@mikro-orm/core": "7.2.5",
|
||||
"@mikro-orm/sql": "7.2.5",
|
||||
"@otplib/preset-default": "^12.0.1",
|
||||
"body-parser": "^1.20.3",
|
||||
"celebrate": "^15.0.3",
|
||||
@@ -147,7 +148,7 @@
|
||||
"ip2region": "2.3.0",
|
||||
"js-yaml": "^4.1.0",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"keyv": "^5.2.3",
|
||||
"keyv": "5.6.0",
|
||||
"lodash": "^4.17.21",
|
||||
"multer": "2.4.0",
|
||||
"node-cron": "4.6.0",
|
||||
@@ -157,19 +158,18 @@
|
||||
"ps-tree": "^1.2.0",
|
||||
"reflect-metadata": "^0.2.2",
|
||||
"request-ip": "3.3.0",
|
||||
"sequelize": "6.37.8",
|
||||
"sockjs": "^0.3.24",
|
||||
"sqlite3": "npm:@whyour/sqlite3@1.1.2",
|
||||
"tar": "7.5.20",
|
||||
"toad-scheduler": "^3.0.1",
|
||||
"typedi": "^0.10.0",
|
||||
"undici": "^7.9.0",
|
||||
"uuid": "^11.0.3",
|
||||
"winston": "^3.17.0",
|
||||
"winston-daily-rotate-file": "^5.0.0",
|
||||
"tar": "7.5.20"
|
||||
"winston-daily-rotate-file": "^5.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@ant-design/icons": "^5.0.1",
|
||||
"@ant-design/plots": "^2.6.8",
|
||||
"@ant-design/pro-layout": "6.38.22",
|
||||
"@codemirror/state": "6.5.4",
|
||||
"@codemirror/view": "6.39.16",
|
||||
@@ -187,7 +187,7 @@
|
||||
"@types/jsonwebtoken": "^8.5.8",
|
||||
"@types/lodash": "^4.14.185",
|
||||
"@types/multer": "^1.4.7",
|
||||
"@types/node": "^17.0.21",
|
||||
"@types/node": "^22.20.5",
|
||||
"@types/nodemailer": "^6.4.4",
|
||||
"@types/proper-lockfile": "^4.1.4",
|
||||
"@types/ps-tree": "^1.1.6",
|
||||
@@ -233,13 +233,16 @@
|
||||
"react-intl-universal": "^2.12.0",
|
||||
"react-router-dom": "6.26.1",
|
||||
"react-split-pane": "^0.1.92",
|
||||
"sequelize": "6.37.8",
|
||||
"sockjs-client": "^1.6.0",
|
||||
"sqlite3": "npm:@whyour/sqlite3@1.1.2",
|
||||
"ts-node": "^10.9.2",
|
||||
"ts-proto": "^2.6.1",
|
||||
"tslib": "^2.4.0",
|
||||
"typescript": "5.2.2",
|
||||
"typescript": "5.9.3",
|
||||
"vh-check": "^2.0.5",
|
||||
"virtualizedtableforantd4": "1.3.0"
|
||||
},
|
||||
"qinglongUpgradeFormat": 2
|
||||
"qinglongUpgradeFormat": 2,
|
||||
"qinglongRuntimeMinimum": "22.17.0"
|
||||
}
|
||||
|
||||
Generated
+539
-646
File diff suppressed because it is too large
Load Diff
@@ -1,10 +1,15 @@
|
||||
const fs = require('node:fs/promises');
|
||||
const { createWriteStream } = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { pipeline } = require('node:stream/promises');
|
||||
const tar = require('tar');
|
||||
const bundle = require('../cli/src/internal/maintenance/platformBundle.cjs');
|
||||
const {
|
||||
verifyUpgrade,
|
||||
} = require('../cli/src/internal/maintenance/upgradeArtifacts.cjs');
|
||||
const {
|
||||
prestartBackend,
|
||||
} = require('../cli/src/internal/maintenance/upgradePrestart.cjs');
|
||||
|
||||
async function normalizeModes(directory) {
|
||||
for (const entry of await fs.readdir(directory, { withFileTypes: true })) {
|
||||
@@ -19,6 +24,17 @@ async function normalizeModes(directory) {
|
||||
}
|
||||
}
|
||||
|
||||
async function writeBundleArchive(workspace, filename) {
|
||||
// Core streams complete partial writes; tar's file option uses fs-minipass,
|
||||
// whose writer can silently discard a successful short write/writev result.
|
||||
await pipeline(
|
||||
tar.c({ gzip: true, portable: true, noMtime: true, cwd: workspace }, [
|
||||
'qinglong',
|
||||
]),
|
||||
createWriteStream(filename, { highWaterMark: 256 * 1024 }),
|
||||
);
|
||||
}
|
||||
|
||||
async function buildBundle(root, output) {
|
||||
root = path.resolve(root);
|
||||
output = path.resolve(output);
|
||||
@@ -76,7 +92,7 @@ async function buildBundle(root, output) {
|
||||
endianness: target.endianness,
|
||||
armFloatAbi: target.armFloatAbi,
|
||||
},
|
||||
nodeMinimum: '20.18.1',
|
||||
nodeMinimum: '22.17.0',
|
||||
nodeApi: 6,
|
||||
files: await bundle.inventory({ source, static: staticRoot }),
|
||||
};
|
||||
@@ -88,16 +104,7 @@ async function buildBundle(root, output) {
|
||||
const filename = `qinglong-${build.sourceCommit}-${bundle.targetKey(
|
||||
target,
|
||||
)}.tar.gz`;
|
||||
await tar.c(
|
||||
{
|
||||
gzip: true,
|
||||
portable: true,
|
||||
noMtime: true,
|
||||
cwd: workspace,
|
||||
file: path.join(output, filename),
|
||||
},
|
||||
['qinglong'],
|
||||
);
|
||||
await writeBundleArchive(workspace, path.join(output, filename));
|
||||
// Verify the bytes users receive, not just the pre-archive build directory.
|
||||
const roundtrip = path.join(workspace, 'roundtrip');
|
||||
await fs.mkdir(roundtrip);
|
||||
@@ -111,6 +118,8 @@ async function buildBundle(root, output) {
|
||||
await verifyUpgrade(extracted.source, extracted.static);
|
||||
await bundle.smokeDependencies(extracted.source);
|
||||
console.log('Archived production dependencies and SQLite verified');
|
||||
await prestartBackend(extracted, { target });
|
||||
console.log('Archived backend database, gRPC and HTTP startup verified');
|
||||
const bytes = await fs.readFile(path.join(output, filename));
|
||||
const record = {
|
||||
format: 2,
|
||||
@@ -132,7 +141,7 @@ async function buildBundle(root, output) {
|
||||
await fs.rm(workspace, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
module.exports = { buildBundle, normalizeModes };
|
||||
module.exports = { buildBundle, normalizeModes, writeBundleArchive };
|
||||
if (require.main === module)
|
||||
buildBundle(process.argv[2] || '/ql', process.argv[3] || '/out').catch(
|
||||
(e) => {
|
||||
|
||||
+525
-112
@@ -16,61 +16,134 @@ const execute = promisify(execFile);
|
||||
const repository = path.resolve(__dirname, '..');
|
||||
const hash = (bytes) => createHash('sha256').update(bytes).digest('hex');
|
||||
const read = async (file) => JSON.parse(await fs.readFile(file, 'utf8'));
|
||||
const run = (program, args, options = {}) => execute(program, args, {
|
||||
encoding: 'utf8', maxBuffer: 8 * 1024 * 1024, timeout: 120000,
|
||||
...options,
|
||||
});
|
||||
const run = (program, args, options = {}) =>
|
||||
execute(program, args, {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
timeout: 120000,
|
||||
...options,
|
||||
});
|
||||
const requiredFiles = [
|
||||
'.env.example', 'back/protos/api.proto', 'ecosystem.config.js', 'version.yaml',
|
||||
'shell/upgrade.cjs', 'shell/install-bundle.cjs', 'shell/install-package.cjs',
|
||||
'shell/upgrade-lock.yaml', 'shell/start.sh', 'shell/start-simplify.sh',
|
||||
'shell/task.sh', 'shell/preload/client.js', 'shell/preload/client.py',
|
||||
'shell/preload/sitecustomize.js', 'shell/preload/sitecustomize.py',
|
||||
'shell/preload/esm-loader.mjs', 'shell/preload/notificationAudit.js',
|
||||
'sample/config.sample.sh', 'sample/auth.sample.json',
|
||||
'static/build-info.json', 'static/build/app.js', 'static/build/token.js',
|
||||
'.env.example',
|
||||
'back/protos/api.proto',
|
||||
'ecosystem.config.js',
|
||||
'version.yaml',
|
||||
'shell/upgrade.cjs',
|
||||
'shell/install-bundle.cjs',
|
||||
'shell/install-package.cjs',
|
||||
'shell/upgrade-lock.yaml',
|
||||
'shell/runtime-guard.cjs',
|
||||
'shell/node-runtime-probe.cjs',
|
||||
'shell/start.sh',
|
||||
'shell/start-simplify.sh',
|
||||
'shell/task.sh',
|
||||
'shell/preload/client.js',
|
||||
'shell/preload/client.py',
|
||||
'shell/preload/sitecustomize.js',
|
||||
'shell/preload/sitecustomize.py',
|
||||
'shell/preload/esm-loader.mjs',
|
||||
'shell/preload/notificationAudit.js',
|
||||
'sample/config.sample.sh',
|
||||
'sample/auth.sample.json',
|
||||
'static/build-info.json',
|
||||
'static/build/app.js',
|
||||
'static/build/token.js',
|
||||
'static/build/bootstrap/upgradeProbe.js',
|
||||
...[
|
||||
'databaseClient',
|
||||
'databaseErrors',
|
||||
'databaseProcess',
|
||||
'databaseWorker',
|
||||
'repository',
|
||||
'schema',
|
||||
'queries',
|
||||
'cacheStore',
|
||||
'statistics',
|
||||
].map((name) => `static/build/data/${name}.js`),
|
||||
'static/dist/index.html',
|
||||
...['upgradeArtifacts', 'platformBundle', 'upgradePrestart', 'panelUpgrade', 'nodeRuntime']
|
||||
.map((name) => `cli/src/internal/maintenance/${name}.cjs`),
|
||||
...[
|
||||
'upgradeArtifacts',
|
||||
'platformBundle',
|
||||
'upgradePrestart',
|
||||
'panelUpgrade',
|
||||
'nodeRuntime',
|
||||
].map((name) => `cli/src/internal/maintenance/${name}.cjs`),
|
||||
];
|
||||
|
||||
function checkPackedFiles(files) {
|
||||
const names = new Set(files.map((file) => file.path));
|
||||
for (const name of requiredFiles) assert.ok(names.has(name), `Missing npm runtime: ${name}`);
|
||||
for (const name of requiredFiles)
|
||||
assert.ok(names.has(name), `Missing npm runtime: ${name}`);
|
||||
for (const name of names) {
|
||||
assert.ok(/^(?:package\.json$|LICENSE$|README[^/]*\.md$|\.env\.example$|ecosystem\.config\.js$|version\.yaml$|back\/protos\/api\.proto$|cli\/(?:README[^/]*\.md$|src\/internal\/maintenance\/[^/]+\.cjs$)|sample\/|shell\/|static\/)/.test(name), `Unexpected npm file: ${name}`);
|
||||
assert.ok(!/^shell\/(?:pub\.sh$|preload\/(?:env\.|notify\.|__ql_notify__\.|lang_env\.sh|.*-notify\.json))/.test(name), `Generated or release-only runtime file: ${name}`);
|
||||
assert.ok(!/(?:^|\/)__pycache__\/|\.pyc$|(?:^|\/)\.DS_Store$/.test(name), name);
|
||||
assert.ok(
|
||||
/^(?:package\.json$|LICENSE$|README[^/]*\.md$|\.env\.example$|ecosystem\.config\.js$|version\.yaml$|back\/protos\/api\.proto$|cli\/(?:README[^/]*\.md$|src\/internal\/maintenance\/[^/]+\.cjs$)|sample\/|shell\/|static\/)/.test(
|
||||
name,
|
||||
),
|
||||
`Unexpected npm file: ${name}`,
|
||||
);
|
||||
assert.ok(
|
||||
!/^shell\/(?:pub\.sh$|preload\/(?:env\.|notify\.|__ql_notify__\.|lang_env\.sh|.*-notify\.json))/.test(
|
||||
name,
|
||||
),
|
||||
`Generated or release-only runtime file: ${name}`,
|
||||
);
|
||||
assert.ok(
|
||||
!/(?:^|\/)__pycache__\/|\.pyc$|(?:^|\/)\.DS_Store$/.test(name),
|
||||
name,
|
||||
);
|
||||
assert.notEqual(name, 'sample/tool.ts');
|
||||
}
|
||||
}
|
||||
|
||||
async function packPanel(output) {
|
||||
const git = (...args) => run('git', args, { cwd: repository });
|
||||
assert.equal((await git('status', '--porcelain', '--untracked-files=all')).stdout.trim(), '', 'Pack only from a clean source checkout');
|
||||
await run(process.execPath, [path.join(repository, 'docker/verify-build.cjs')], { cwd: repository });
|
||||
assert.equal(
|
||||
(await git('status', '--porcelain', '--untracked-files=all')).stdout.trim(),
|
||||
'',
|
||||
'Pack only from a clean source checkout',
|
||||
);
|
||||
await run(
|
||||
process.execPath,
|
||||
[path.join(repository, 'docker/verify-build.cjs')],
|
||||
{ cwd: repository },
|
||||
);
|
||||
await fs.mkdir(output, { recursive: true });
|
||||
const cache = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-pack-cache-'));
|
||||
try {
|
||||
const result = await run('npm', ['pack', '--json', '--cache', cache, '--pack-destination', output], { cwd: repository });
|
||||
const result = await run(
|
||||
'npm',
|
||||
['pack', '--json', '--cache', cache, '--pack-destination', output],
|
||||
{ cwd: repository },
|
||||
);
|
||||
const [manifest] = JSON.parse(result.stdout);
|
||||
checkPackedFiles(manifest.files);
|
||||
assert.equal(manifest.name, '@whyour/qinglong');
|
||||
const build = await read(path.join(repository, 'static/build-info.json'));
|
||||
const names = new Set(manifest.files.map((file) => file.path));
|
||||
for (const name of Object.keys(build.files)) assert.ok(names.has(`static/${name}`), `npm filtered a verified build artifact: ${name}`);
|
||||
for (const name of Object.keys(build.files))
|
||||
assert.ok(
|
||||
names.has(`static/${name}`),
|
||||
`npm filtered a verified build artifact: ${name}`,
|
||||
);
|
||||
const receipt = {
|
||||
filename: manifest.filename, sha256: hash(await fs.readFile(path.join(output, manifest.filename))),
|
||||
sourceCommit: build.sourceCommit, size: manifest.size, fileCount: manifest.files.length,
|
||||
filename: manifest.filename,
|
||||
sha256: hash(await fs.readFile(path.join(output, manifest.filename))),
|
||||
sourceCommit: build.sourceCommit,
|
||||
size: manifest.size,
|
||||
fileCount: manifest.files.length,
|
||||
};
|
||||
await fs.writeFile(path.join(output, 'package-receipt.json'), JSON.stringify(receipt, null, 2));
|
||||
await fs.writeFile(
|
||||
path.join(output, 'package-receipt.json'),
|
||||
JSON.stringify(receipt, null, 2),
|
||||
);
|
||||
console.log(JSON.stringify(receipt));
|
||||
return receipt;
|
||||
} finally {
|
||||
await fs.rm(cache, { recursive: true, force: true });
|
||||
// Also recover an interrupted npm pack whose postpack hook did not execute.
|
||||
await fs.rm(path.join(repository, 'shell/upgrade-lock.yaml'), { force: true });
|
||||
await fs.rm(path.join(repository, 'shell/upgrade-lock.yaml'), {
|
||||
force: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,8 +157,13 @@ async function unusedPort() {
|
||||
|
||||
// Consume the unmodified output of Dockerfile.upgrade-bundle / buildBundle.
|
||||
async function platformFixture(root, directory, expectedCommit) {
|
||||
assert.ok(directory, 'Set QL_PANEL_TEST_BUNDLE_DIR to the official platform bundle output');
|
||||
const names = (await fs.readdir(directory)).filter((name) => name.endsWith('.tar.gz'));
|
||||
assert.ok(
|
||||
directory,
|
||||
'Set QL_PANEL_TEST_BUNDLE_DIR to the official platform bundle output',
|
||||
);
|
||||
const names = (await fs.readdir(directory)).filter((name) =>
|
||||
name.endsWith('.tar.gz'),
|
||||
);
|
||||
assert.equal(names.length, 1, 'Expected one official platform package');
|
||||
const archive = path.resolve(directory, names[0]);
|
||||
const record = await read(archive + '.json');
|
||||
@@ -105,64 +183,140 @@ async function platformFixture(root, directory, expectedCommit) {
|
||||
async function legacyArchive(temp, env) {
|
||||
const directory = path.join(temp, 'legacy');
|
||||
await fs.mkdir(directory);
|
||||
const result = await run('npm', ['pack', '@whyour/qinglong@2.23.0', '--ignore-scripts', '--json', '--registry=https://registry.npmjs.org', '--pack-destination', directory, '--cache', path.join(temp, 'cache')], { cwd: temp, env, timeout: 300000 });
|
||||
const result = await run(
|
||||
'npm',
|
||||
[
|
||||
'pack',
|
||||
'@whyour/qinglong@2.23.0',
|
||||
'--ignore-scripts',
|
||||
'--json',
|
||||
'--registry=https://registry.npmjs.org',
|
||||
'--pack-destination',
|
||||
directory,
|
||||
'--cache',
|
||||
path.join(temp, 'cache'),
|
||||
],
|
||||
{ cwd: temp, env, timeout: 300000 },
|
||||
);
|
||||
const [record] = JSON.parse(result.stdout);
|
||||
assert.equal(record.name, '@whyour/qinglong');
|
||||
assert.equal(record.version, '2.23.0');
|
||||
assert.equal(record.filename, 'whyour-qinglong-2.23.0.tgz');
|
||||
const archive = path.join(directory, record.filename);
|
||||
const integrity = 'sha512-' + createHash('sha512').update(await fs.readFile(archive)).digest('base64');
|
||||
assert.equal(integrity, 'sha512-x6B9s1/SRKkHe7jRQtb3LzMluJ3O3PAAssrr/5jaFMMl7DZvABSEiqkkZMu0r1BNtiCF72uj8OqXN9pCnF2WVw==', 'Published legacy package integrity changed');
|
||||
const integrity =
|
||||
'sha512-' +
|
||||
createHash('sha512')
|
||||
.update(await fs.readFile(archive))
|
||||
.digest('base64');
|
||||
assert.equal(
|
||||
integrity,
|
||||
'sha512-x6B9s1/SRKkHe7jRQtb3LzMluJ3O3PAAssrr/5jaFMMl7DZvABSEiqkkZMu0r1BNtiCF72uj8OqXN9pCnF2WVw==',
|
||||
'Published legacy package integrity changed',
|
||||
);
|
||||
return { archive, integrity, version: record.version };
|
||||
}
|
||||
|
||||
async function verifyPanel(input, output) {
|
||||
assert.equal(process.platform, 'linux', 'Service tests require Linux');
|
||||
assert.equal(process.env.QL_PANEL_PACKAGE_TEST_ISOLATED, '1', 'Run only inside a disposable Linux container: QL_PANEL_PACKAGE_TEST_ISOLATED=1');
|
||||
assert.equal(
|
||||
process.env.QL_PANEL_PACKAGE_TEST_ISOLATED,
|
||||
'1',
|
||||
'Run only inside a disposable Linux container: QL_PANEL_PACKAGE_TEST_ISOLATED=1',
|
||||
);
|
||||
const stat = await fs.stat(input);
|
||||
if (stat.isDirectory()) {
|
||||
const names = (await fs.readdir(input)).filter((name) => name.endsWith('.tgz'));
|
||||
const names = (await fs.readdir(input)).filter((name) =>
|
||||
name.endsWith('.tgz'),
|
||||
);
|
||||
assert.equal(names.length, 1, 'Expected one exact npm archive');
|
||||
input = path.join(input, names[0]);
|
||||
}
|
||||
const expected = await read(path.join(path.dirname(input), 'package-receipt.json'));
|
||||
const expected = await read(
|
||||
path.join(path.dirname(input), 'package-receipt.json'),
|
||||
);
|
||||
assert.equal(path.basename(input), expected.filename);
|
||||
assert.equal(hash(await fs.readFile(input)), expected.sha256, 'npm archive differs from its build receipt');
|
||||
assert.equal(
|
||||
hash(await fs.readFile(input)),
|
||||
expected.sha256,
|
||||
'npm archive differs from its build receipt',
|
||||
);
|
||||
assert.equal((await fs.stat(input)).size, expected.size);
|
||||
assert.match(expected.sourceCommit, /^[a-f0-9]{40}$/);
|
||||
if (process.env.GITHUB_SHA) assert.equal(expected.sourceCommit, process.env.GITHUB_SHA);
|
||||
if (process.env.GITHUB_SHA)
|
||||
assert.equal(expected.sourceCommit, process.env.GITHUB_SHA);
|
||||
const nodeSha = hash(await fs.readFile(process.execPath));
|
||||
const pm2 = process.env.QL_PANEL_TEST_PM2;
|
||||
assert.ok(pm2 && path.isAbsolute(pm2), 'Set QL_PANEL_TEST_PM2 to an isolated PM2 executable');
|
||||
const temp = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-install-')));
|
||||
const home = path.join(temp, 'home'), prefix = path.join(temp, 'prefix'), data = path.join(temp, 'data');
|
||||
assert.ok(
|
||||
pm2 && path.isAbsolute(pm2),
|
||||
'Set QL_PANEL_TEST_PM2 to an isolated PM2 executable',
|
||||
);
|
||||
const temp = await fs.realpath(
|
||||
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-install-')),
|
||||
);
|
||||
const home = path.join(temp, 'home'),
|
||||
prefix = path.join(temp, 'prefix'),
|
||||
data = path.join(temp, 'data');
|
||||
await fs.mkdir(home);
|
||||
await fs.mkdir(data);
|
||||
const port = await unusedPort();
|
||||
let grpcPort = await unusedPort();
|
||||
while (grpcPort === port) grpcPort = await unusedPort();
|
||||
const env = { ...process.env, HOME: home, PM2_HOME: path.join(temp, '.pm2'),
|
||||
QL_DATA_DIR: data, QL_BRANCH: 'develop', NODE_ENV: 'production', QL_CONTAINER: 'false',
|
||||
BACK_PORT: String(port), GRPC_PORT: String(grpcPort), QlPort: String(port), QlGrpcPort: String(grpcPort),
|
||||
BIND_HOST: '127.0.0.1', BIND_HOST_GRPC: '127.0.0.1',
|
||||
NODE_PATH: '', NODE_OPTIONS: '',
|
||||
PATH: [path.join(prefix, 'bin'), path.dirname(pm2), process.env.PATH].join(path.delimiter),
|
||||
const env = {
|
||||
...process.env,
|
||||
HOME: home,
|
||||
PM2_HOME: path.join(temp, '.pm2'),
|
||||
QL_DATA_DIR: data,
|
||||
QL_BRANCH: 'develop',
|
||||
NODE_ENV: 'production',
|
||||
QL_CONTAINER: 'false',
|
||||
BACK_PORT: String(port),
|
||||
GRPC_PORT: String(grpcPort),
|
||||
QlPort: String(port),
|
||||
QlGrpcPort: String(grpcPort),
|
||||
BIND_HOST: '127.0.0.1',
|
||||
BIND_HOST_GRPC: '127.0.0.1',
|
||||
NODE_PATH: '',
|
||||
NODE_OPTIONS: '',
|
||||
PATH: [path.join(prefix, 'bin'), path.dirname(pm2), process.env.PATH].join(
|
||||
path.delimiter,
|
||||
),
|
||||
};
|
||||
for (const key of ['QL_CLI_ROOT', 'QL_DIR', 'QL_UPGRADE_FILE', 'QL_UPGRADE_REPAIR_DEPS', 'QL_UPGRADE_LAUNCHER_PID', 'QL_UPGRADE_INDEX_URL']) delete env[key];
|
||||
for (const key of [
|
||||
'QL_CLI_ROOT',
|
||||
'QL_DIR',
|
||||
'QL_UPGRADE_FILE',
|
||||
'QL_UPGRADE_REPAIR_DEPS',
|
||||
'QL_UPGRADE_LAUNCHER_PID',
|
||||
'QL_UPGRADE_INDEX_URL',
|
||||
])
|
||||
delete env[key];
|
||||
let root, fixtureServer, token;
|
||||
const api = async (route, method = 'GET', body, anonymous = false) => {
|
||||
const response = await fetch(`http://127.0.0.1:${port}/api/${route}`, {
|
||||
method, headers: { 'Content-Type': 'application/json', ...(!anonymous && token ? { Authorization: `Bearer ${token}` } : {}) },
|
||||
body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(5000),
|
||||
method,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...(!anonymous && token ? { Authorization: `Bearer ${token}` } : {}),
|
||||
},
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
const result = await response.json();
|
||||
assert.equal(result.code, 200, `${method} ${route}: ${result.message}`);
|
||||
try {
|
||||
assert.equal(result.code, 200, `${method} ${route}: ${result.message}`);
|
||||
} catch (error) {
|
||||
error.status = response.status;
|
||||
throw error;
|
||||
}
|
||||
return result.data;
|
||||
};
|
||||
const healthy = async () => {
|
||||
const deadline = Date.now() + 120000;
|
||||
while (Date.now() < deadline) {
|
||||
try { const value = await api('health', 'GET', undefined, true); if (value.status === 'ok') return value; } catch {}
|
||||
try {
|
||||
const value = await api('health', 'GET', undefined, true);
|
||||
if (value.status === 'ok') return value;
|
||||
} catch {}
|
||||
await delay(300);
|
||||
}
|
||||
throw new Error('Installed npm backend did not become healthy');
|
||||
@@ -174,26 +328,81 @@ async function verifyPanel(input, output) {
|
||||
const value = await api('system/update-status');
|
||||
assert.equal(value.id, id);
|
||||
if (value.status === status) return value;
|
||||
if (value.status === 'failed' && status !== 'failed') throw new Error(value.error);
|
||||
if (value.status === 'failed' && status !== 'failed')
|
||||
throw new Error(value.error);
|
||||
} catch (error) {
|
||||
if (!/fetch failed|ECONNRESET|ECONNREFUSED|aborted|timeout/i.test(error.message)) throw error;
|
||||
if (
|
||||
error.status !== 503 &&
|
||||
!/fetch failed|ECONNRESET|ECONNREFUSED|aborted|timeout/i.test(
|
||||
error.message,
|
||||
)
|
||||
)
|
||||
throw error;
|
||||
}
|
||||
await delay(250);
|
||||
}
|
||||
throw new Error(`Installed npm update never reached ${status}`);
|
||||
};
|
||||
try {
|
||||
const install = (archive) => run('npm', ['install', '--global', '--prefix', prefix, '--force', '--omit=dev', '--no-audit', '--no-fund', '--cache', path.join(temp, 'cache'), archive], { cwd: temp, env, timeout: 600000 });
|
||||
const start = () => run(path.join(prefix, 'bin/qinglong'), ['reload'], { cwd: temp, env, timeout: 180000 });
|
||||
const stop = () => run(pm2, ['delete', 'ecosystem.config.js'], { cwd: root, env });
|
||||
console.log(`Installing actual npm archive with production lifecycle scripts on ${process.version}`);
|
||||
const install = (archive) =>
|
||||
run(
|
||||
'npm',
|
||||
[
|
||||
'install',
|
||||
'--global',
|
||||
'--prefix',
|
||||
prefix,
|
||||
'--force',
|
||||
'--omit=dev',
|
||||
'--no-audit',
|
||||
'--no-fund',
|
||||
'--cache',
|
||||
path.join(temp, 'cache'),
|
||||
archive,
|
||||
],
|
||||
{ cwd: temp, env, timeout: 600000 },
|
||||
);
|
||||
const start = () =>
|
||||
run(path.join(prefix, 'bin/qinglong'), ['reload'], {
|
||||
cwd: temp,
|
||||
env,
|
||||
timeout: 180000,
|
||||
});
|
||||
const stop = () =>
|
||||
run(pm2, ['delete', 'ecosystem.config.js'], { cwd: root, env });
|
||||
console.log(
|
||||
`Installing actual npm archive with production lifecycle scripts on ${process.version}`,
|
||||
);
|
||||
await install(input);
|
||||
const globalRoot = (await run('npm', ['root', '--global', '--prefix', prefix], { cwd: temp, env })).stdout.trim();
|
||||
const globalRoot = (
|
||||
await run('npm', ['root', '--global', '--prefix', prefix], {
|
||||
cwd: temp,
|
||||
env,
|
||||
})
|
||||
).stdout.trim();
|
||||
root = path.join(globalRoot, '@whyour/qinglong');
|
||||
env.QL_DIR = root;
|
||||
const runtimeProbes = [];
|
||||
const checkRuntime = async () => {
|
||||
const result = await run(
|
||||
process.execPath,
|
||||
[path.join(root, 'shell/node-runtime-probe.cjs')],
|
||||
{ cwd: temp, env },
|
||||
);
|
||||
runtimeProbes.push(JSON.parse(result.stdout.trim()));
|
||||
};
|
||||
await checkRuntime();
|
||||
// Use a subprocess: cached candidate modules must not mask missing files
|
||||
// after old-package installation and the real npm overwrite below.
|
||||
await run(process.execPath, ['-e', `const root=process.argv[1],p=require('node:path');const m=require(root+'/package.json');for(const target of Object.values(m.bin))require('node:fs').accessSync(p.join(root,target));for(const name of ['platformBundle','upgradePrestart','panelUpgrade'])require(root+'/cli/src/internal/maintenance/'+name+'.cjs');require(root+'/shell/upgrade.cjs').verifyUpgrade(root,p.join(root,'static')).catch(e=>{console.error(e);process.exitCode=1});`, root], { env, cwd: temp });
|
||||
await run(
|
||||
process.execPath,
|
||||
[
|
||||
'-e',
|
||||
`const root=process.argv[1],p=require('node:path');const m=require(root+'/package.json');for(const target of Object.values(m.bin))require('node:fs').accessSync(p.join(root,target));for(const name of ['platformBundle','upgradePrestart','panelUpgrade'])require(root+'/cli/src/internal/maintenance/'+name+'.cjs');require(root+'/shell/upgrade.cjs').verifyUpgrade(root,p.join(root,'static')).catch(e=>{console.error(e);process.exitCode=1});`,
|
||||
root,
|
||||
],
|
||||
{ env, cwd: temp },
|
||||
);
|
||||
env.QL_DATA_DIR = path.join(temp, 'fresh/data');
|
||||
await fs.mkdir(env.QL_DATA_DIR, { recursive: true });
|
||||
console.log('Starting the freshly installed npm package');
|
||||
@@ -201,40 +410,75 @@ async function verifyPanel(input, output) {
|
||||
const fresh = await healthy();
|
||||
assert.equal(fresh.build.sourceCommit, expected.sourceCommit);
|
||||
assert.equal(fresh.services.grpc, true);
|
||||
assert.match(await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()), /<html/i);
|
||||
assert.match(
|
||||
await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()),
|
||||
/<html/i,
|
||||
);
|
||||
await stop();
|
||||
await fs.rm(path.join(temp, 'fresh'), { recursive: true, force: true });
|
||||
env.QL_DATA_DIR = data;
|
||||
console.log('Installing the integrity-pinned published legacy npm package');
|
||||
const legacy = await legacyArchive(temp, env);
|
||||
await install(legacy.archive);
|
||||
assert.equal((await read(path.join(root, 'package.json'))).version, legacy.version);
|
||||
await assert.rejects(fs.access(path.join(root, 'cli/src/internal/maintenance/platformBundle.cjs')), { code: 'ENOENT' });
|
||||
const legacyBackendSha = hash(await fs.readFile(path.join(root, 'static/build/app.js')));
|
||||
assert.equal(
|
||||
(await read(path.join(root, 'package.json'))).version,
|
||||
legacy.version,
|
||||
);
|
||||
await assert.rejects(
|
||||
fs.access(
|
||||
path.join(root, 'cli/src/internal/maintenance/platformBundle.cjs'),
|
||||
),
|
||||
{ code: 'ENOENT' },
|
||||
);
|
||||
const legacyBackendSha = hash(
|
||||
await fs.readFile(path.join(root, 'static/build/app.js')),
|
||||
);
|
||||
await start();
|
||||
await healthy();
|
||||
const credentials = { username: 'npm_package_verification', password: `Test_${randomUUID()}!` };
|
||||
const credentials = {
|
||||
username: 'npm_package_verification',
|
||||
password: `Test_${randomUUID()}!`,
|
||||
};
|
||||
await api('user/init', 'PUT', credentials, true);
|
||||
token = (await api('user/login', 'POST', credentials, true)).token;
|
||||
const nonce = randomUUID();
|
||||
await api('envs', 'POST', [{ name: 'NPM_PACKAGE_PRESERVE', value: nonce }]);
|
||||
const script = path.join(data, 'scripts/npm-package-check.js');
|
||||
await fs.writeFile(script, `console.log(${JSON.stringify(nonce)});\n`);
|
||||
const cron = await api('crons', 'POST', { name: 'npm package retained task', command: `node ${script}`, schedule: '0 0 1 1 *' });
|
||||
const cron = await api('crons', 'POST', {
|
||||
name: 'npm package retained task',
|
||||
command: `node ${script}`,
|
||||
schedule: '0 0 1 1 *',
|
||||
});
|
||||
const userConfig = path.join(data, 'config/config.sh');
|
||||
await fs.appendFile(userConfig, `\n# retained npm package setting ${nonce}\n`);
|
||||
await fs.appendFile(
|
||||
userConfig,
|
||||
`\n# retained npm package setting ${nonce}\n`,
|
||||
);
|
||||
const userConfigSha = hash(await fs.readFile(userConfig));
|
||||
const preservedData = async () => {
|
||||
assert.equal((await api('user')).username, credentials.username);
|
||||
assert.ok((await api('envs')).some((x) => x.name === 'NPM_PACKAGE_PRESERVE' && x.value === nonce));
|
||||
assert.ok(
|
||||
(await api('envs')).some(
|
||||
(x) => x.name === 'NPM_PACKAGE_PRESERVE' && x.value === nonce,
|
||||
),
|
||||
);
|
||||
assert.equal((await api(`crons/${cron.id}`)).command, cron.command);
|
||||
assert.equal(hash(await fs.readFile(userConfig)), userConfigSha);
|
||||
assert.equal(await fs.readFile(script, 'utf8'), `console.log(${JSON.stringify(nonce)});\n`);
|
||||
assert.equal(
|
||||
await fs.readFile(script, 'utf8'),
|
||||
`console.log(${JSON.stringify(nonce)});\n`,
|
||||
);
|
||||
};
|
||||
await preservedData();
|
||||
console.log('Stopping legacy service and verifying npm automatically preserves its .env during overwrite');
|
||||
console.log(
|
||||
'Stopping legacy service and verifying npm automatically preserves its .env during overwrite',
|
||||
);
|
||||
await stop();
|
||||
await fs.appendFile(path.join(root, '.env'), `\n# retained custom npm environment ${nonce}\n`);
|
||||
await fs.appendFile(
|
||||
path.join(root, '.env'),
|
||||
`\n# retained custom npm environment ${nonce}\n`,
|
||||
);
|
||||
const savedEnv = await fs.readFile(path.join(root, '.env'));
|
||||
await install(input);
|
||||
assert.deepEqual(await fs.readFile(path.join(root, '.env')), savedEnv);
|
||||
@@ -242,25 +486,62 @@ async function verifyPanel(input, output) {
|
||||
const configSha = hash(savedEnv);
|
||||
const local = createRequire(path.join(root, 'package.json'));
|
||||
const artifacts = local('./shell/upgrade.cjs');
|
||||
const build = await artifacts.verifyUpgrade(root, path.join(root, 'static'));
|
||||
const build = await artifacts.verifyUpgrade(
|
||||
root,
|
||||
path.join(root, 'static'),
|
||||
);
|
||||
assert.equal(build.sourceCommit, expected.sourceCommit);
|
||||
assert.notEqual(hash(await fs.readFile(path.join(root, 'static/build/app.js'))), legacyBackendSha);
|
||||
assert.equal(hash(await fs.readFile(path.join(root, 'pnpm-lock.yaml'))), build.lockfileSha256);
|
||||
const fixture = await platformFixture(root, process.env.QL_PANEL_TEST_BUNDLE_DIR, expected.sourceCommit);
|
||||
let badChecksum = true, requests = 0;
|
||||
assert.notEqual(
|
||||
hash(await fs.readFile(path.join(root, 'static/build/app.js'))),
|
||||
legacyBackendSha,
|
||||
);
|
||||
assert.equal(
|
||||
hash(await fs.readFile(path.join(root, 'pnpm-lock.yaml'))),
|
||||
build.lockfileSha256,
|
||||
);
|
||||
const fixture = await platformFixture(
|
||||
root,
|
||||
process.env.QL_PANEL_TEST_BUNDLE_DIR,
|
||||
expected.sourceCommit,
|
||||
);
|
||||
let badChecksum = true,
|
||||
requests = 0;
|
||||
fixtureServer = http.createServer((request, response) => {
|
||||
requests++;
|
||||
if (request.url === '/latest.json') {
|
||||
response.setHeader('Content-Type', 'application/json');
|
||||
response.end(JSON.stringify({ format: 2, sourceCommit: fixture.sourceCommit, packages: { [fixture.target]: { size: fixture.size, sha256: badChecksum ? '0'.repeat(64) : fixture.sha256, url: `http://127.0.0.1:${fixtureServer.address().port}/bundle.tar.gz` } } }));
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
format: 2,
|
||||
sourceCommit: fixture.sourceCommit,
|
||||
packages: {
|
||||
[fixture.target]: {
|
||||
size: fixture.size,
|
||||
sha256: badChecksum ? '0'.repeat(64) : fixture.sha256,
|
||||
url: `http://127.0.0.1:${
|
||||
fixtureServer.address().port
|
||||
}/bundle.tar.gz`,
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
} else if (request.url === '/bundle.tar.gz') {
|
||||
response.setHeader('Content-Length', fixture.size);
|
||||
sync.createReadStream(fixture.archive).pipe(response);
|
||||
} else { response.statusCode = 404; response.end(); }
|
||||
} else {
|
||||
response.statusCode = 404;
|
||||
response.end();
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => fixtureServer.listen(0, '127.0.0.1', resolve));
|
||||
env.QL_UPGRADE_INDEX_URL = `http://127.0.0.1:${fixtureServer.address().port}/latest.json`;
|
||||
console.log('Starting the overwritten npm package against legacy user data');
|
||||
await new Promise((resolve) =>
|
||||
fixtureServer.listen(0, '127.0.0.1', resolve),
|
||||
);
|
||||
env.QL_UPGRADE_INDEX_URL = `http://127.0.0.1:${
|
||||
fixtureServer.address().port
|
||||
}/latest.json`;
|
||||
console.log(
|
||||
'Starting the overwritten npm package against legacy user data',
|
||||
);
|
||||
await start();
|
||||
const original = await healthy();
|
||||
assert.equal(original.build.sourceCommit, build.sourceCommit);
|
||||
@@ -277,32 +558,82 @@ async function verifyPanel(input, output) {
|
||||
assert.equal((await healthy()).build.instanceId, original.build.instanceId);
|
||||
await preserved();
|
||||
badChecksum = false;
|
||||
console.log('Testing installed panel download, prestart and explicit application');
|
||||
console.log(
|
||||
'Testing installed panel download, prestart and explicit application',
|
||||
);
|
||||
const accepted = await api('system/update', 'PUT');
|
||||
const ready = await job(accepted.id, 'ready');
|
||||
assert.equal((await healthy()).build.instanceId, original.build.instanceId);
|
||||
const pointer = await read(path.join(root, '.tmp/upgrade-ready-develop.json'));
|
||||
const receipt = await read(path.join(root, '.tmp', pointer.directory, 'prestart.json'));
|
||||
assert.ok(receipt.completedAt <= ready.updatedAt, 'Readiness was recorded before backend prestart');
|
||||
const pointer = await read(
|
||||
path.join(root, '.tmp/upgrade-ready-develop.json'),
|
||||
);
|
||||
const receipt = await read(
|
||||
path.join(root, '.tmp', pointer.directory, 'prestart.json'),
|
||||
);
|
||||
assert.ok(
|
||||
receipt.completedAt <= ready.updatedAt,
|
||||
'Readiness was recorded before backend prestart',
|
||||
);
|
||||
const applying = await api('update/system', 'PUT');
|
||||
await job(applying.id, 'completed');
|
||||
const afterOnline = await healthy();
|
||||
await checkRuntime();
|
||||
assert.notEqual(afterOnline.build.instanceId, original.build.instanceId);
|
||||
assert.equal(afterOnline.build.sourceCommit, fixture.sourceCommit);
|
||||
await fs.access(path.join(root, 'node_modules/.pnpm'));
|
||||
assert.match(await fs.realpath(path.join(root, 'node_modules/sqlite3')), /[/\\]node_modules[/\\]\.pnpm[/\\]/);
|
||||
assert.match(
|
||||
await fs.realpath(path.join(root, 'node_modules/@mikro-orm/sql')),
|
||||
/[/\\]node_modules[/\\]\.pnpm[/\\]/,
|
||||
);
|
||||
await assert.rejects(fs.access(path.join(root, 'node_modules/sqlite3')), {
|
||||
code: 'ENOENT',
|
||||
});
|
||||
await assert.rejects(fs.access(path.join(root, 'node_modules/sequelize')), {
|
||||
code: 'ENOENT',
|
||||
});
|
||||
await fs.access(path.join(root, 'scripts/build-upgrade-bundle.cjs'));
|
||||
await artifacts.verifyUpgrade(root, path.join(root, 'static'));
|
||||
assert.match(await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()), /<html/i);
|
||||
assert.match(
|
||||
await fetch(`http://127.0.0.1:${port}/`).then((r) => r.text()),
|
||||
/<html/i,
|
||||
);
|
||||
await preserved();
|
||||
console.log('Testing offline installer and Bash reload from the installed npm package');
|
||||
console.log(
|
||||
'Testing offline installer and Bash reload from the installed npm package',
|
||||
);
|
||||
const requestsBeforeOffline = requests;
|
||||
await run(process.execPath, [path.join(root, 'shell/install-bundle.cjs'), '--root', root, '--file', fixture.archive, '--download-only'], { cwd: temp, env, timeout: 240000 });
|
||||
assert.equal(requests, requestsBeforeOffline, 'Offline preparation contacted the index');
|
||||
assert.equal((await healthy()).build.instanceId, afterOnline.build.instanceId);
|
||||
await run(path.join(prefix, 'bin/ql'), ['reload', 'system'], { cwd: temp, env, timeout: 240000 });
|
||||
await run(
|
||||
process.execPath,
|
||||
[
|
||||
path.join(root, 'shell/install-bundle.cjs'),
|
||||
'--root',
|
||||
root,
|
||||
'--file',
|
||||
fixture.archive,
|
||||
'--download-only',
|
||||
],
|
||||
{ cwd: temp, env, timeout: 240000 },
|
||||
);
|
||||
assert.equal(
|
||||
requests,
|
||||
requestsBeforeOffline,
|
||||
'Offline preparation contacted the index',
|
||||
);
|
||||
assert.equal(
|
||||
(await healthy()).build.instanceId,
|
||||
afterOnline.build.instanceId,
|
||||
);
|
||||
await run(path.join(prefix, 'bin/ql'), ['reload', 'system'], {
|
||||
cwd: temp,
|
||||
env,
|
||||
timeout: 240000,
|
||||
});
|
||||
const afterOffline = await healthy();
|
||||
assert.notEqual(afterOffline.build.instanceId, afterOnline.build.instanceId);
|
||||
await checkRuntime();
|
||||
assert.notEqual(
|
||||
afterOffline.build.instanceId,
|
||||
afterOnline.build.instanceId,
|
||||
);
|
||||
assert.equal(afterOffline.build.sourceCommit, fixture.sourceCommit);
|
||||
await fs.access(path.join(root, 'node_modules/.pnpm'));
|
||||
assert.equal(hash(await fs.readFile(process.execPath)), nodeSha);
|
||||
@@ -312,36 +643,66 @@ async function verifyPanel(input, output) {
|
||||
let taskExecuted = false;
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const log = String(await api(`crons/${cron.id}/log`));
|
||||
if (log.includes(nonce) && /完成|执行结束|finished|completed/i.test(log)) { taskExecuted = true; break; }
|
||||
if (
|
||||
log.includes(nonce) &&
|
||||
/完成|执行结束|finished|completed/i.test(log)
|
||||
) {
|
||||
taskExecuted = true;
|
||||
break;
|
||||
}
|
||||
await delay(300);
|
||||
}
|
||||
assert.ok(taskExecuted, 'Retained task did not execute after npm package upgrades');
|
||||
assert.ok(
|
||||
taskExecuted,
|
||||
'Retained task did not execute after npm package upgrades',
|
||||
);
|
||||
const result = {
|
||||
node: process.version, platform: fixture.target,
|
||||
archiveSha256: hash(await fs.readFile(input)), sourceCommit: build.sourceCommit,
|
||||
installedProductionDependencies: true, restoredLockVerified: true,
|
||||
legacyNpmVersion: legacy.version, legacyNpmIntegrity: legacy.integrity,
|
||||
legacyNpmOverwriteInstalled: true, legacyExternalDataPreserved: true,
|
||||
node: process.version,
|
||||
platform: fixture.target,
|
||||
archiveSha256: hash(await fs.readFile(input)),
|
||||
sourceCommit: build.sourceCommit,
|
||||
installedProductionDependencies: true,
|
||||
restoredLockVerified: true,
|
||||
legacyNpmVersion: legacy.version,
|
||||
legacyNpmIntegrity: legacy.integrity,
|
||||
legacyNpmOverwriteInstalled: true,
|
||||
legacyExternalDataPreserved: true,
|
||||
customRootEnvironmentPreservedAutomatically: true,
|
||||
officialPlatformBundle: { baseImage: fixture.baseImage, sourceCommit: fixture.sourceCommit, sha256: fixture.sha256 },
|
||||
officialPlatformBundle: {
|
||||
baseImage: fixture.baseImage,
|
||||
sourceCommit: fixture.sourceCommit,
|
||||
sha256: fixture.sha256,
|
||||
},
|
||||
migratedFromNpmToOfficialPnpmLayout: true,
|
||||
existingNodePreserved: true,
|
||||
frontendAndBackendStarted: true, checksumFailureKeptOldService: true,
|
||||
prestartBeforeReady: true, onlineActualRestart: true,
|
||||
offlinePreparationDidNotContactIndex: true, offlineBashActualRestart: true,
|
||||
accountEnvironmentConfigurationAndScriptsPreserved: true, retainedTaskExecuted: true,
|
||||
initialBuild: original.build, afterOnline: afterOnline.build, afterOffline: afterOffline.build,
|
||||
nativeUrlRuntimeProbes: runtimeProbes,
|
||||
frontendAndBackendStarted: true,
|
||||
checksumFailureKeptOldService: true,
|
||||
prestartBeforeReady: true,
|
||||
onlineActualRestart: true,
|
||||
offlinePreparationDidNotContactIndex: true,
|
||||
offlineBashActualRestart: true,
|
||||
accountEnvironmentConfigurationAndScriptsPreserved: true,
|
||||
retainedTaskExecuted: true,
|
||||
initialBuild: original.build,
|
||||
afterOnline: afterOnline.build,
|
||||
afterOffline: afterOffline.build,
|
||||
};
|
||||
if (output) {
|
||||
await fs.mkdir(output, { recursive: true });
|
||||
await fs.writeFile(path.join(output, `verification-${process.version}.json`), JSON.stringify(result, null, 2));
|
||||
await fs.writeFile(
|
||||
path.join(output, `verification-${process.version}.json`),
|
||||
JSON.stringify(result, null, 2),
|
||||
);
|
||||
}
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (root) {
|
||||
for (const file of ['qinglong-error.log', 'qinglong-out.log']) {
|
||||
const text = await fs.readFile(path.join(env.PM2_HOME, 'logs', file), 'utf8').catch(() => '');
|
||||
const text = await fs
|
||||
.readFile(path.join(env.PM2_HOME, 'logs', file), 'utf8')
|
||||
.catch(() => '');
|
||||
if (text) console.error(text.slice(-6000));
|
||||
}
|
||||
}
|
||||
@@ -351,19 +712,71 @@ async function verifyPanel(input, output) {
|
||||
fixtureServer.closeAllConnections();
|
||||
await new Promise((resolve) => fixtureServer.close(resolve));
|
||||
}
|
||||
// A failed assertion must not remove a prefix that its detached updater is
|
||||
// still extracting into. Identify only this private installation's job.
|
||||
if (root) {
|
||||
const state = await read(
|
||||
path.join(root, '.tmp/panel-upgrade.json'),
|
||||
).catch(() => undefined);
|
||||
if (
|
||||
Number.isSafeInteger(state?.pid) &&
|
||||
state.pid > 1 &&
|
||||
typeof state.id === 'string'
|
||||
) {
|
||||
const command = (
|
||||
await fs
|
||||
.readFile(`/proc/${state.pid}/cmdline`, 'utf8')
|
||||
.catch(() => '')
|
||||
).split('\0');
|
||||
if (
|
||||
command.includes(
|
||||
path.join(root, 'cli/src/internal/maintenance/panelUpgrade.cjs'),
|
||||
) &&
|
||||
command.includes('worker') &&
|
||||
command.includes(state.id)
|
||||
) {
|
||||
const signal = (name) => {
|
||||
try {
|
||||
process.kill(-state.pid, name);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error.code !== 'ESRCH') throw error;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
signal('SIGTERM');
|
||||
const deadline = Date.now() + 30000;
|
||||
while (signal(0) && Date.now() < deadline) await delay(100);
|
||||
if (signal(0)) signal('SIGKILL');
|
||||
}
|
||||
}
|
||||
}
|
||||
await run(pm2, ['kill'], { env, cwd: temp }).catch(() => {});
|
||||
await run('nginx', ['-s', 'quit'], { env, cwd: temp }).catch(() => {});
|
||||
await fs.rm(temp, { recursive: true, force: true });
|
||||
await fs.rm(temp, {
|
||||
recursive: true,
|
||||
force: true,
|
||||
maxRetries: 3,
|
||||
retryDelay: 100,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { checkPackedFiles, packPanel, verifyPanel };
|
||||
if (require.main === module) {
|
||||
const [action, input, output] = process.argv.slice(2);
|
||||
const operation = action === 'pack' && input
|
||||
? packPanel(path.resolve(input))
|
||||
: action === 'verify' && input
|
||||
const operation =
|
||||
action === 'pack' && input
|
||||
? packPanel(path.resolve(input))
|
||||
: action === 'verify' && input
|
||||
? verifyPanel(path.resolve(input), output && path.resolve(output))
|
||||
: Promise.reject(new Error('Usage: node scripts/verify-panel-package.cjs pack OUTPUT | verify ARCHIVE_OR_DIRECTORY [RESULT_DIRECTORY]'));
|
||||
operation.catch((error) => { console.error(error); process.exitCode = 1; });
|
||||
: Promise.reject(
|
||||
new Error(
|
||||
'Usage: node scripts/verify-panel-package.cjs pack OUTPUT | verify ARCHIVE_OR_DIRECTORY [RESULT_DIRECTORY]',
|
||||
),
|
||||
);
|
||||
operation.catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
// Verify the actual archive under an unsupported Node without touching any
|
||||
// existing installation. This is also the Node 20 negative CI matrix entry.
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const os = require('node:os');
|
||||
const assert = require('node:assert/strict');
|
||||
const { execFile, spawn } = require('node:child_process');
|
||||
const { promisify } = require('node:util');
|
||||
const execute = promisify(execFile);
|
||||
|
||||
async function verifyRuntimeFloor(input, output) {
|
||||
assert.equal(
|
||||
Number(process.versions.node.split('.')[0]),
|
||||
20,
|
||||
'Run this rejection test with Node 20',
|
||||
);
|
||||
const names = (await fs.readdir(input)).filter((name) =>
|
||||
name.endsWith('.tgz'),
|
||||
);
|
||||
assert.equal(names.length, 1, 'Expected one actual npm archive');
|
||||
const archive = path.resolve(input, names[0]);
|
||||
const temp = await fs.realpath(
|
||||
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-node-floor-')),
|
||||
);
|
||||
const env = {
|
||||
PATH: process.env.PATH,
|
||||
HOME: temp,
|
||||
TMPDIR: temp,
|
||||
npm_config_cache: path.join(temp, 'cache'),
|
||||
npm_config_userconfig: path.join(temp, 'npmrc'),
|
||||
npm_config_globalconfig: path.join(temp, 'global-npmrc'),
|
||||
};
|
||||
const npm = (args) =>
|
||||
execute('npm', args, {
|
||||
cwd: temp,
|
||||
env,
|
||||
encoding: 'utf8',
|
||||
timeout: 600000,
|
||||
maxBuffer: 4 * 1024 * 1024,
|
||||
});
|
||||
try {
|
||||
await fs.writeFile(env.npm_config_userconfig, '');
|
||||
await fs.writeFile(env.npm_config_globalconfig, '');
|
||||
const legacy = path.join(temp, 'legacy');
|
||||
await fs.mkdir(legacy);
|
||||
await fs.writeFile(
|
||||
path.join(legacy, 'package.json'),
|
||||
JSON.stringify({ name: '@whyour/qinglong', version: '2.22.0' }),
|
||||
);
|
||||
await fs.writeFile(path.join(legacy, 'legacy-marker'), 'old application');
|
||||
const [old] = JSON.parse(
|
||||
(
|
||||
await execute('npm', ['pack', '--json', '--ignore-scripts'], {
|
||||
cwd: legacy,
|
||||
env,
|
||||
})
|
||||
).stdout,
|
||||
);
|
||||
for (const global of [false, true]) {
|
||||
const prefix = path.join(temp, global ? 'global' : 'local');
|
||||
const installed = path.join(
|
||||
prefix,
|
||||
global
|
||||
? 'lib/node_modules/@whyour/qinglong'
|
||||
: 'node_modules/@whyour/qinglong',
|
||||
);
|
||||
const args = [
|
||||
'install',
|
||||
...(global ? ['--global'] : []),
|
||||
'--prefix',
|
||||
prefix,
|
||||
'--omit=dev',
|
||||
'--no-audit',
|
||||
'--no-fund',
|
||||
];
|
||||
await npm([...args, path.join(legacy, old.filename)]);
|
||||
const environment = Buffer.from(
|
||||
'QL_DATA_DIR=/external/data\r\nSYNTHETIC_SECRET=keep\n',
|
||||
);
|
||||
const data = path.join(temp, global ? 'global-data' : 'local-data');
|
||||
await fs.writeFile(path.join(installed, '.env'), environment);
|
||||
await fs.writeFile(data, 'synthetic old data');
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
'-e',
|
||||
`
|
||||
const fs=require('node:fs'),http=require('node:http');
|
||||
const value=fs.readFileSync(process.argv[1],'utf8')+fs.readFileSync(process.argv[2],'utf8');
|
||||
const server=http.createServer((q,r)=>r.end(value));
|
||||
server.listen(0,'127.0.0.1',()=>process.send(server.address().port));
|
||||
`,
|
||||
path.join(installed, '.env'),
|
||||
data,
|
||||
],
|
||||
{ env, stdio: ['ignore', 'ignore', 'pipe', 'ipc'] },
|
||||
);
|
||||
const stopped = new Promise((resolve) => child.once('exit', resolve));
|
||||
try {
|
||||
const port = await new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
() => reject(new Error('Old service startup timed out')),
|
||||
10000,
|
||||
);
|
||||
child.once('message', (value) => {
|
||||
clearTimeout(timer);
|
||||
resolve(value);
|
||||
});
|
||||
child.once('error', (error) => {
|
||||
clearTimeout(timer);
|
||||
reject(error);
|
||||
});
|
||||
child.once('exit', () => {
|
||||
clearTimeout(timer);
|
||||
reject(new Error('Old service exited'));
|
||||
});
|
||||
});
|
||||
const url = `http://127.0.0.1:${port}`;
|
||||
const before = await (await fetch(url)).text();
|
||||
await assert.rejects(npm([...args, archive]), (error) => {
|
||||
assert.equal(
|
||||
error.killed,
|
||||
false,
|
||||
`npm did not reach the runtime guard: ${error.message}`,
|
||||
);
|
||||
assert.match(error.stderr + error.stdout, /requires Node >=22\.17/);
|
||||
return true;
|
||||
});
|
||||
assert.equal(
|
||||
JSON.parse(await fs.readFile(path.join(installed, 'package.json')))
|
||||
.version,
|
||||
'2.22.0',
|
||||
);
|
||||
assert.deepEqual(
|
||||
await fs.readFile(path.join(installed, '.env')),
|
||||
environment,
|
||||
);
|
||||
assert.equal(await fs.readFile(data, 'utf8'), 'synthetic old data');
|
||||
assert.equal(child.exitCode, null);
|
||||
assert.equal(await (await fetch(url)).text(), before);
|
||||
} finally {
|
||||
child.kill('SIGKILL');
|
||||
await stopped;
|
||||
}
|
||||
}
|
||||
const result = {
|
||||
node: process.version,
|
||||
unsupportedRuntimeRejected: true,
|
||||
localAndGlobalOldPackagePreserved: true,
|
||||
environmentAndDataPreserved: true,
|
||||
oldServiceStayedHealthy: true,
|
||||
};
|
||||
if (output) {
|
||||
await fs.mkdir(output, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(output, `verification-${process.version}.json`),
|
||||
JSON.stringify(result, null, 2),
|
||||
);
|
||||
}
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
return result;
|
||||
} finally {
|
||||
await fs.rm(temp, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { verifyRuntimeFloor };
|
||||
if (require.main === module) {
|
||||
const [input, output] = process.argv.slice(2);
|
||||
if (!input)
|
||||
throw new Error(
|
||||
'Usage: node scripts/verify-panel-runtime-floor.cjs ARCHIVE_DIRECTORY [RESULT_DIRECTORY]',
|
||||
);
|
||||
verifyRuntimeFloor(path.resolve(input), output && path.resolve(output)).catch(
|
||||
(error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
|
||||
// Run in new processes: a healthy parent can outlive a broken native runtime.
|
||||
function probeNodeRuntime(executable = process.execPath) {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-node-url-'));
|
||||
const source = `
|
||||
const assert = require('node:assert/strict');
|
||||
const { pathToFileURL, fileURLToPath } = require('node:url');
|
||||
for (let i = 0; i < 1000; i++) {
|
||||
assert.equal(new URL('/任务?q=' + i, 'https://example.com').hostname, 'example.com');
|
||||
assert.equal(new URL('https://例子.测试').hostname, 'xn--fsqu00a.xn--0zwm56d');
|
||||
assert.equal(new URL('http://[::1]:5700').port, '5700');
|
||||
const filename = '/tmp/中文 path/' + i + '.cjs';
|
||||
assert.equal(fileURLToPath(pathToFileURL(filename)), filename);
|
||||
}
|
||||
console.log('QINGLONG_NODE_URL_OK');
|
||||
`;
|
||||
const file = path.join(directory, 'probe.cjs');
|
||||
fs.writeFileSync(file, source);
|
||||
try {
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const args = process.execArgv.includes('--jitless') ? ['--jitless'] : [];
|
||||
args.push(...(i % 2 ? [file] : ['-e', source]));
|
||||
const child = spawnSync(executable, args, {
|
||||
encoding: 'utf8',
|
||||
timeout: 20000,
|
||||
maxBuffer: 65536,
|
||||
env: { ...process.env, NODE_OPTIONS: '' },
|
||||
});
|
||||
assert.equal(
|
||||
child.status,
|
||||
0,
|
||||
`Node URL runtime probe failed (${i % 2 ? 'file' : 'eval'}): signal=${
|
||||
child.signal
|
||||
}; status=${child.status}; ${child.error?.message || child.stderr}`,
|
||||
);
|
||||
assert.match(child.stdout, /QINGLONG_NODE_URL_OK/);
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
node: process.version,
|
||||
architecture: process.arch,
|
||||
ada: process.versions.ada,
|
||||
processes: 4,
|
||||
urlIterationsPerProcess: 1000,
|
||||
nodeSha256: createHash('sha256')
|
||||
.update(fs.readFileSync(executable))
|
||||
.digest('hex'),
|
||||
};
|
||||
} finally {
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { probeNodeRuntime };
|
||||
if (require.main === module) console.log(JSON.stringify(probeNodeRuntime()));
|
||||
@@ -0,0 +1,17 @@
|
||||
const minimum = '22.17.0';
|
||||
|
||||
function assertNodeRuntime(version = process.versions.node) {
|
||||
const [major, minor] = version.split('.').map(Number);
|
||||
if (!Number.isInteger(major) || !Number.isInteger(minor) || major < 22 || (major === 22 && minor < 17)) {
|
||||
throw Object.assign(new Error(`Qinglong requires Node >=${minimum}; upgrade Node before installing. Application updates retain the existing Node.`), { code: 'UNSUPPORTED_NODE' });
|
||||
}
|
||||
if (version === process.versions.node) {
|
||||
try {
|
||||
if (typeof require('node:sqlite').DatabaseSync !== 'function') throw new Error();
|
||||
} catch {
|
||||
throw Object.assign(new Error('This Node build does not provide node:sqlite; use a Node build with SQLite support.'), { code: 'UNSUPPORTED_NODE_SQLITE' });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { minimum, assertNodeRuntime };
|
||||
@@ -14,7 +14,6 @@ function temporary(t) {
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
test('environment values remain literal when the generated JavaScript loads', async () => {
|
||||
const output = {};
|
||||
const Env = load(path.resolve('back/services/env.ts'), {
|
||||
@@ -42,7 +41,6 @@ test('environment values remain literal when the generated JavaScript loads', as
|
||||
assert.equal(context.process.env[`VALUE_${i}`], value),
|
||||
);
|
||||
});
|
||||
|
||||
test('subscription shell parameters round trip without expansion or command substitution', () => {
|
||||
const { formatCommand } = load(path.resolve('back/config/subscription.ts'));
|
||||
const payload =
|
||||
@@ -83,7 +81,6 @@ test('subscription shell parameters round trip without expansion or command subs
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('SSH alias traversal and symlinks cannot write or remove outside files', async (t) => {
|
||||
const base = temporary(t);
|
||||
const root = path.join(base, 'ssh');
|
||||
@@ -141,14 +138,17 @@ test('SSH alias traversal and symlinks cannot write or remove outside files', as
|
||||
service.generateSingleSshConfig('safe', 'host', '$(printf BAD)'),
|
||||
);
|
||||
});
|
||||
|
||||
function cronService(config, updates = []) {
|
||||
const Cron = load(path.resolve('back/services/cron.ts'), {
|
||||
typedi: decorators,
|
||||
'../config': config,
|
||||
'../data/cron': {
|
||||
CrontabStatus: { running: 0, idle: 1, queued: 3 },
|
||||
CrontabModel: { update: async (v) => updates.push(v) },
|
||||
CrontabModel: {
|
||||
nativeUpdate: async (where, v) => {
|
||||
return updates.push(v);
|
||||
},
|
||||
},
|
||||
},
|
||||
'../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} },
|
||||
'../config/util': { fileExist: async (p) => fs.existsSync(p) },
|
||||
@@ -183,7 +183,6 @@ function subscriptionService(config, model = {}) {
|
||||
}).default;
|
||||
return new Subscription(logger, {}, {}, {}, {});
|
||||
}
|
||||
|
||||
test('stored cron/subscription paths cannot read or enumerate outside the log root', async (t) => {
|
||||
const base = temporary(t);
|
||||
const root = path.join(base, 'log');
|
||||
@@ -224,7 +223,6 @@ test('stored cron/subscription paths cannot read or enumerate outside the log ro
|
||||
(await subscription.logs(1)).some((x) => x.filename === 'normal.log'),
|
||||
);
|
||||
});
|
||||
|
||||
test('subscription aliases are checked before persistence and forced deletion', async (t) => {
|
||||
const base = temporary(t);
|
||||
for (const dir of ['scripts', 'repo']) fs.mkdirSync(path.join(base, dir));
|
||||
@@ -235,8 +233,12 @@ test('subscription aliases are checked before persistence and forced deletion',
|
||||
repoPath: path.join(base, 'repo'),
|
||||
},
|
||||
{
|
||||
findAll: async () => [{ alias: '../victim' }],
|
||||
destroy: async () => writes++,
|
||||
find: async (where, read = {}) => {
|
||||
return [{ alias: '../victim' }];
|
||||
},
|
||||
nativeDelete: async (where, read = {}) => {
|
||||
return writes++;
|
||||
},
|
||||
},
|
||||
);
|
||||
service.insert = service.updateDb = async () => writes++;
|
||||
@@ -254,14 +256,17 @@ test('subscription aliases are checked before persistence and forced deletion',
|
||||
);
|
||||
assert.equal(writes, 0);
|
||||
});
|
||||
|
||||
function systemService(config, overrides = {}) {
|
||||
const System = load(path.resolve('back/services/system.ts'), {
|
||||
typedi: decorators,
|
||||
'../config': config,
|
||||
'../config/const': {},
|
||||
'../data/dependence': {
|
||||
DependenceModel: { findAll: async () => [] },
|
||||
DependenceModel: {
|
||||
find: async (where, read = {}) => {
|
||||
return [];
|
||||
},
|
||||
},
|
||||
DependenceTypes: {},
|
||||
DependenceStatus: {},
|
||||
},
|
||||
@@ -277,7 +282,6 @@ function systemService(config, overrides = {}) {
|
||||
}).default;
|
||||
return new System(logger, {}, {});
|
||||
}
|
||||
|
||||
test('backup rejects unrecognized paths before spawning; valid export uses argv without a shell', async (t) => {
|
||||
const base = temporary(t);
|
||||
const calls = [];
|
||||
@@ -333,7 +337,6 @@ test('backup rejects unrecognized paths before spawning; valid export uses argv
|
||||
assert.equal(calls[0][2].shell, undefined);
|
||||
assert.equal(downloaded, true);
|
||||
});
|
||||
|
||||
test('proxy, language and mirror settings stay literal at their shell boundaries', async (t) => {
|
||||
const base = temporary(t);
|
||||
const commands = [];
|
||||
@@ -373,7 +376,6 @@ test('proxy, language and mirror settings stay literal at their shell boundaries
|
||||
}
|
||||
assert.equal((await service.reloadSystem('$(printf BAD)')).code, 400);
|
||||
});
|
||||
|
||||
test('the ignored-log sentinel remains supported without exposing device files', async (t) => {
|
||||
const root = temporary(t);
|
||||
const cron = cronService({ logPath: root });
|
||||
@@ -385,7 +387,6 @@ test('the ignored-log sentinel remains supported without exposing device files',
|
||||
cron.status({ ids: [], status: 0, log_path: '/dev/zero' }),
|
||||
);
|
||||
});
|
||||
|
||||
test('normal backup creates an archive containing the selected data directories', async (t) => {
|
||||
const base = temporary(t);
|
||||
const dataPath = path.join(base, 'data');
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const path = require('node:path');
|
||||
const os = require('node:os');
|
||||
const fs = require('node:fs/promises');
|
||||
const express = require('express');
|
||||
const jwt = require('jsonwebtoken');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
|
||||
test('authentication cache propagates database read and write failures', async () => {
|
||||
const failure = new Error('synthetic unavailable database');
|
||||
const CacheStore = load(path.resolve('back/data/cacheStore.ts'), {
|
||||
'.': {
|
||||
database: {
|
||||
request: async () => {
|
||||
throw failure;
|
||||
},
|
||||
},
|
||||
},
|
||||
}).CacheStore;
|
||||
const { shareStore } = load(path.resolve('back/shared/store.ts'), {
|
||||
'../data/cacheStore': { CacheStore },
|
||||
});
|
||||
await assert.rejects(shareStore.getAuthInfo(), (error) => error === failure);
|
||||
await assert.rejects(
|
||||
shareStore.updateAuthInfo({ username: 'qa' }),
|
||||
(error) => error === failure,
|
||||
);
|
||||
});
|
||||
|
||||
test('HTTP authentication distinguishes unavailable storage from an invalid session', async (t) => {
|
||||
const root = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), 'ql-auth-availability-'),
|
||||
);
|
||||
const secret = 'synthetic-auth-availability';
|
||||
const token = jwt.sign({}, secret, { algorithm: 'HS384', expiresIn: '1h' });
|
||||
const auth = { username: 'qa', password: 'configured', token };
|
||||
let unavailable = false;
|
||||
let initialized = 0;
|
||||
const read = async (value) => {
|
||||
if (unavailable) throw new Error(`SQL failure must not expose ${token}`);
|
||||
return value;
|
||||
};
|
||||
const router = express.Router();
|
||||
router.get('/system/update-status', (_req, res) => res.json({ code: 200 }));
|
||||
router.get('/envs', (_req, res) => res.json({ code: 200 }));
|
||||
router.get('/health', (_req, res) => res.json({ code: 200 }));
|
||||
router.put('/user/init', (_req, res) => {
|
||||
initialized++;
|
||||
res.json({ code: 200 });
|
||||
});
|
||||
const app = express();
|
||||
load(path.resolve('back/loaders/express.ts'), {
|
||||
'../config': {
|
||||
rootPath: root,
|
||||
uploadPath: path.join(root, 'upload'),
|
||||
configPath: path.join(root, 'config'),
|
||||
api: { prefix: '/api' },
|
||||
apiWhiteList: ['/api/health', '/api/user/init'],
|
||||
jwt: { secret },
|
||||
},
|
||||
'../api': () => router,
|
||||
'../config/util': {
|
||||
getToken: (req) =>
|
||||
(req.headers.authorization || '').replace(/^Bearer /, ''),
|
||||
getPlatform: () => 'desktop',
|
||||
},
|
||||
'../shared/store': {
|
||||
shareStore: {
|
||||
getAuthInfo: () => read(auth),
|
||||
getApps: () =>
|
||||
read([
|
||||
{
|
||||
scopes: ['envs'],
|
||||
tokens: [
|
||||
{ value: 'app-token', expiration: Date.now() / 1000 + 3600 },
|
||||
],
|
||||
},
|
||||
]),
|
||||
},
|
||||
},
|
||||
'../shared/i18n': { t: (value) => value },
|
||||
'../config/serverEnv': { serveEnv: (_req, res) => res.end() },
|
||||
'../data/open': {},
|
||||
'../data/system': {},
|
||||
}).default({ app });
|
||||
const server = app.listen(0, '127.0.0.1');
|
||||
t.after(async () => {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
await new Promise((resolve) => server.once('listening', resolve));
|
||||
const request = async (route, value, method = 'GET') => {
|
||||
const response = await fetch(
|
||||
`http://127.0.0.1:${server.address().port}${route}`,
|
||||
{
|
||||
method,
|
||||
headers: value ? { Authorization: `Bearer ${value}` } : {},
|
||||
signal: AbortSignal.timeout(3000),
|
||||
},
|
||||
);
|
||||
return { status: response.status, body: await response.text() };
|
||||
};
|
||||
assert.equal((await request('/api/system/update-status', token)).status, 200);
|
||||
unavailable = true;
|
||||
for (const [route, value, method] of [
|
||||
['/api/system/update-status', token],
|
||||
['/open/envs', 'app-token'],
|
||||
['/api/user/init', undefined, 'PUT'],
|
||||
]) {
|
||||
const response = await request(route, value, method);
|
||||
assert.equal(response.status, 503);
|
||||
assert.equal(JSON.parse(response.body).code, 503);
|
||||
assert.ok(!response.body.includes(token));
|
||||
assert.ok(!response.body.includes('SQL failure'));
|
||||
}
|
||||
assert.equal(initialized, 0);
|
||||
assert.equal((await request('/api/health')).status, 200);
|
||||
unavailable = false;
|
||||
assert.equal((await request('/api/system/update-status', token)).status, 200);
|
||||
const invalid = jwt.sign({ unrecognized: true }, secret, {
|
||||
algorithm: 'HS384',
|
||||
expiresIn: '1h',
|
||||
});
|
||||
assert.equal(
|
||||
(await request('/api/system/update-status', invalid)).status,
|
||||
401,
|
||||
);
|
||||
});
|
||||
@@ -10,15 +10,40 @@ test('system and node modes route extended cron once and keep portable rules in
|
||||
const end = source.indexOf(' private async getLogName', begin);
|
||||
const saveBegin = source.indexOf(' private async setCrontab(');
|
||||
const saveEnd = source.indexOf(' public importCrontab(', saveBegin);
|
||||
const js = ts.transpileModule(`class Fixture {${source.slice(begin, end)}${source.slice(saveBegin, saveEnd)}}; module.exports = Fixture;`, {
|
||||
compilerOptions: { target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText;
|
||||
const js = ts.transpileModule(
|
||||
`class Fixture {${source.slice(begin, end)}${source.slice(
|
||||
saveBegin,
|
||||
saveEnd,
|
||||
)}}; module.exports = Fixture;`,
|
||||
{
|
||||
compilerOptions: { target: ts.ScriptTarget.ES2020 },
|
||||
},
|
||||
).outputText;
|
||||
for (const mode of ['system', 'node']) {
|
||||
const module = { exports: {} };
|
||||
let file = '', installs = 0;
|
||||
new Function('module', 'process', 'execSync', 'ScheduleType', 'config', 'writeFileWithLock', 'CrontabModel', js)(
|
||||
module, { env: { QL_SCHEDULER: mode } }, () => { installs++; }, ScheduleType,
|
||||
{ crontabFile: '/unused' }, async (_, data) => { file = data; }, { update: async () => {} },
|
||||
let file = '',
|
||||
installs = 0;
|
||||
new Function(
|
||||
'module',
|
||||
'process',
|
||||
'execSync',
|
||||
'ScheduleType',
|
||||
'config',
|
||||
'writeFileWithLock',
|
||||
'CrontabModel',
|
||||
js,
|
||||
)(
|
||||
module,
|
||||
{ env: { QL_SCHEDULER: mode } },
|
||||
() => {
|
||||
installs++;
|
||||
},
|
||||
ScheduleType,
|
||||
{ crontabFile: '/unused' },
|
||||
async (_, data) => {
|
||||
file = data;
|
||||
},
|
||||
{ nativeUpdate: async () => {} },
|
||||
);
|
||||
const service = new module.exports();
|
||||
service.makeCommand = (row) => `task ${row.id}.js`;
|
||||
@@ -40,20 +65,52 @@ test('system and node modes route extended cron once and keep portable rules in
|
||||
{ id: 'named', schedule: '0 0 * JAN MON' },
|
||||
{ id: 'macro', schedule: '@daily' },
|
||||
{ id: 'short', schedule: '*' },
|
||||
{ id: 'extra', schedule: '* * * * *', extra_schedules: [{ schedule: '0 * * * *' }] },
|
||||
{
|
||||
id: 'extra',
|
||||
schedule: '* * * * *',
|
||||
extra_schedules: [{ schedule: '0 * * * *' }],
|
||||
},
|
||||
{ id: 'once', schedule: '@once' },
|
||||
{ id: 'boot', schedule: '@boot' },
|
||||
];
|
||||
for (const row of rows) {
|
||||
const special = ['once', 'boot'].includes(row.id);
|
||||
const portable = ['plain', 'spaces'].includes(row.id);
|
||||
assert.equal(service.shouldUseCronClient(row), !special && (mode === 'node' || !portable), `${mode}: ${row.id}`);
|
||||
assert.equal(
|
||||
service.shouldUseCronClient(row),
|
||||
!special && (mode === 'node' || !portable),
|
||||
`${mode}: ${row.id}`,
|
||||
);
|
||||
}
|
||||
await service.setCrontab({ data: rows, total: rows.length });
|
||||
assert.equal(installs, mode === 'system' ? 1 : 0);
|
||||
for (const id of ['numeric-step', 'sunday-seven', 'day-step', 'restricted-days', 'restricted-week', 'annually', 'midnight', 'minutely', 'seconds', 'question', 'last', 'nth', 'named', 'macro', 'short', 'extra', 'once', 'boot']) {
|
||||
const line = file.split('\n').find((line) => line.endsWith(`task ${id}.js`));
|
||||
assert.ok(line.startsWith('# '), `${mode}: ${id} cannot also run from system crontab`);
|
||||
for (const id of [
|
||||
'numeric-step',
|
||||
'sunday-seven',
|
||||
'day-step',
|
||||
'restricted-days',
|
||||
'restricted-week',
|
||||
'annually',
|
||||
'midnight',
|
||||
'minutely',
|
||||
'seconds',
|
||||
'question',
|
||||
'last',
|
||||
'nth',
|
||||
'named',
|
||||
'macro',
|
||||
'short',
|
||||
'extra',
|
||||
'once',
|
||||
'boot',
|
||||
]) {
|
||||
const line = file
|
||||
.split('\n')
|
||||
.find((line) => line.endsWith(`task ${id}.js`));
|
||||
assert.ok(
|
||||
line.startsWith('# '),
|
||||
`${mode}: ${id} cannot also run from system crontab`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,28 +1,19 @@
|
||||
const { DatabaseClient } = require('../../back/data/databaseClient');
|
||||
const { repository, initialize } = require('../helpers/repository-fixture.cjs');
|
||||
const { orderBy } = require('../../back/data/queries');
|
||||
require('ts-node/register/transpile-only');
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const { Sequelize, DataTypes } = require('sequelize');
|
||||
const dayjs = require('dayjs');
|
||||
const express = require('express');
|
||||
|
||||
test('today result lists include mixed and deleted tasks and exclude other dates', async (t) => {
|
||||
const db = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const db = new DatabaseClient(':memory:');
|
||||
t.after(() => db.close());
|
||||
const crons = db.define('Cron', {
|
||||
name: DataTypes.STRING,
|
||||
command: DataTypes.STRING,
|
||||
});
|
||||
const stats = db.define('Stat', {
|
||||
ref_id: DataTypes.INTEGER,
|
||||
date: DataTypes.STRING,
|
||||
fail_count: DataTypes.INTEGER,
|
||||
success_count: DataTypes.INTEGER,
|
||||
});
|
||||
const crons = repository(db, 'Crontab');
|
||||
const stats = repository(db, 'CrontabStat');
|
||||
const replacements = {
|
||||
'../../back/data': { database: db },
|
||||
'../../back/data/cron': { CrontabModel: crons },
|
||||
'../../back/data/cronStats': { CrontabStatModel: stats },
|
||||
'../../back/data/runningInstance': {},
|
||||
@@ -42,14 +33,14 @@ test('today result lists include mixed and deleted tasks and exclude other dates
|
||||
else delete require.cache[resolved];
|
||||
});
|
||||
}
|
||||
await db.sync();
|
||||
await crons.bulkCreate([
|
||||
await initialize(db);
|
||||
await crons.insertMany([
|
||||
{ id: 1, name: 'Recovered task', command: 'task recovered.js' },
|
||||
{ id: 2, name: '', command: 'task unnamed.js' },
|
||||
{ id: 3, name: 'Successful task', command: 'task success.js' },
|
||||
]);
|
||||
const today = dayjs().format('YYYY-MM-DD');
|
||||
await stats.bulkCreate([
|
||||
await stats.insertMany([
|
||||
{ ref_id: 1, date: today, fail_count: 2, success_count: 1 },
|
||||
{ ref_id: 2, date: today, fail_count: 1, success_count: 0 },
|
||||
{ ref_id: 3, date: today, fail_count: 0, success_count: 4 },
|
||||
@@ -135,7 +126,7 @@ test('today result lists include mixed and deleted tasks and exclude other dates
|
||||
},
|
||||
],
|
||||
});
|
||||
await stats.destroy({ where: {} });
|
||||
await stats.nativeDelete({});
|
||||
await successHandler(
|
||||
{},
|
||||
{
|
||||
@@ -160,7 +151,7 @@ test('today result lists include mixed and deleted tasks and exclude other dates
|
||||
},
|
||||
);
|
||||
assert.deepEqual(response, { code: 200, data: [] });
|
||||
await stats.drop();
|
||||
await db.query('DROP TABLE CrontabStats');
|
||||
let forwarded;
|
||||
await handler(
|
||||
{},
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
const test = require('node:test'),
|
||||
assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises'),
|
||||
path = require('node:path'),
|
||||
os = require('node:os');
|
||||
const {
|
||||
Sequelize,
|
||||
DataTypes,
|
||||
Op,
|
||||
QueryTypes,
|
||||
where,
|
||||
col,
|
||||
} = require('sequelize');
|
||||
const { DatabaseClient } = require('../../back/data/databaseClient');
|
||||
const { Repository } = require('../../back/data/repository');
|
||||
const { initialize } = require('../helpers/repository-fixture.cjs');
|
||||
const names = {
|
||||
App: 'Apps',
|
||||
Auth: 'Auths',
|
||||
Crontab: 'Crontabs',
|
||||
CrontabStat: 'CrontabStats',
|
||||
CrontabView: 'CrontabViews',
|
||||
Dependence: 'Dependences',
|
||||
Env: 'Envs',
|
||||
RunningInstance: 'RunningInstances',
|
||||
Subscription: 'Subscriptions',
|
||||
};
|
||||
async function legacy(file) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: file,
|
||||
logging: false,
|
||||
});
|
||||
const models = {};
|
||||
for (const [name, tableName] of Object.entries(names)) {
|
||||
const columns = await database.query(`PRAGMA table_info("${tableName}")`, {
|
||||
type: QueryTypes.SELECT,
|
||||
});
|
||||
const attrs = Object.fromEntries(
|
||||
columns.map((c) => [
|
||||
c.name,
|
||||
{
|
||||
type:
|
||||
c.type === 'JSON'
|
||||
? DataTypes.JSON
|
||||
: c.type === 'DATETIME'
|
||||
? DataTypes.DATE
|
||||
: c.type === 'TINYINT(1)'
|
||||
? DataTypes.BOOLEAN
|
||||
: /INT|NUMBER/.test(c.type)
|
||||
? DataTypes.NUMBER
|
||||
: DataTypes.STRING,
|
||||
...(c.pk ? { primaryKey: true, autoIncrement: true } : {}),
|
||||
},
|
||||
]),
|
||||
);
|
||||
models[name] = database.define(name, attrs, {
|
||||
tableName,
|
||||
freezeTableName: true,
|
||||
});
|
||||
}
|
||||
return { database, models };
|
||||
}
|
||||
|
||||
test('all nine legacy tables remain readable in both versions without changing schema, IDs, dates or .env', async (t) => {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-orm-legacy-'));
|
||||
t.after(() => fs.rm(dir, { recursive: true, force: true }));
|
||||
const file = path.join(dir, 'database.sqlite'),
|
||||
envFile = path.join(dir, '.env');
|
||||
const environment = 'QL_DATA_DIR=/synthetic/old-data\nSYNTHETIC_ENV=keep\n';
|
||||
await fs.writeFile(envFile, environment);
|
||||
const seed = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: file,
|
||||
logging: false,
|
||||
});
|
||||
try {
|
||||
const sql = await fs.readFile(
|
||||
path.resolve('test/fixtures/sequelize-2.23-schema.sql'),
|
||||
'utf8',
|
||||
);
|
||||
for (const statement of sql
|
||||
.replace(/^--.*$/gm, '')
|
||||
.split(';')
|
||||
.map((x) => x.trim())
|
||||
.filter(Boolean))
|
||||
await seed.query(statement);
|
||||
} finally {
|
||||
await seed.close();
|
||||
}
|
||||
let old = await legacy(file);
|
||||
const payloads = {
|
||||
App: { name: 'legacy', scopes: ['envs'], client_id: 'synthetic-client' },
|
||||
Auth: {
|
||||
type: 'authConfig',
|
||||
info: {
|
||||
username: 'synthetic-user',
|
||||
password: 'synthetic-hash',
|
||||
tokens: { desktop: ['keep'] },
|
||||
},
|
||||
},
|
||||
Crontab: {
|
||||
name: 'legacy',
|
||||
command: 'echo synthetic',
|
||||
schedule: '0 0 * * *',
|
||||
saved: true,
|
||||
labels: ['中文'],
|
||||
},
|
||||
CrontabStat: { ref_id: 7, date: '2026-10-21', run_count: 2 },
|
||||
CrontabView: {
|
||||
name: 'legacy',
|
||||
filters: [{ property: 'status', operation: 'In', value: [1] }],
|
||||
},
|
||||
Dependence: { name: 'legacy', type: 0, log: ['旧日志'] },
|
||||
Env: {
|
||||
name: 'legacy',
|
||||
value: '旧值',
|
||||
labels: ['中文'],
|
||||
position: 4500000000000000,
|
||||
},
|
||||
RunningInstance: { cron_id: 7, started_at: 100, status: 2, exit_code: 143 },
|
||||
Subscription: {
|
||||
name: 'legacy',
|
||||
alias: 'legacy',
|
||||
pull_option: { username: 'synthetic', password: 'synthetic-only' },
|
||||
},
|
||||
};
|
||||
const date = new Date('2020-01-02T03:04:05.006Z');
|
||||
for (const [name, payload] of Object.entries(payloads))
|
||||
await old.models[name].create({
|
||||
id: 7,
|
||||
...payload,
|
||||
createdAt: date,
|
||||
updatedAt: date,
|
||||
});
|
||||
const schema = await old.database.query(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type,name",
|
||||
{ type: QueryTypes.SELECT },
|
||||
);
|
||||
await old.database.close();
|
||||
const client = new DatabaseClient(file);
|
||||
t.after(() => client.close());
|
||||
await initialize(client);
|
||||
for (const [name, payload] of Object.entries(payloads)) {
|
||||
const repository = new Repository(name, () => client),
|
||||
row = await repository.findOne({ id: 7 });
|
||||
for (const [field, value] of Object.entries(payload))
|
||||
assert.deepEqual(row[field], value, `${name}.${field}`);
|
||||
assert.equal(row.createdAt.toISOString(), date.toISOString());
|
||||
const added = await repository.insert({
|
||||
...payload,
|
||||
id: undefined,
|
||||
...(payload.name ? { name: 'new' } : {}),
|
||||
...(payload.alias ? { alias: 'new' } : {}),
|
||||
...(payload.date ? { date: '2026-10-22' } : {}),
|
||||
});
|
||||
assert(added.id > 7, name);
|
||||
assert(added.createdAt instanceof Date);
|
||||
}
|
||||
assert.deepEqual(
|
||||
await client.query(
|
||||
"SELECT type,name,sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type,name",
|
||||
),
|
||||
schema,
|
||||
);
|
||||
await client.close();
|
||||
old = await legacy(file);
|
||||
try {
|
||||
for (const name of Object.keys(payloads)) {
|
||||
const rows = await old.models[name].findAll({ order: [['id', 'ASC']] });
|
||||
assert.equal(rows.length, 2);
|
||||
assert.equal(rows[0].createdAt.toISOString(), date.toISOString());
|
||||
assert(rows[1].createdAt instanceof Date);
|
||||
assert(!Number.isNaN(+rows[1].createdAt));
|
||||
}
|
||||
assert.equal(await fs.readFile(envFile, 'utf8'), environment);
|
||||
} finally {
|
||||
await old.database.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('native filters retain legacy NULL, IN, nested groups and JSON text matching', async (t) => {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-orm-filters-'));
|
||||
t.after(() => fs.rm(dir, { recursive: true, force: true }));
|
||||
const file = path.join(dir, 'database.sqlite');
|
||||
const client = new DatabaseClient(file);
|
||||
t.after(() => client.close());
|
||||
await initialize(client);
|
||||
const repository = new Repository('Env', () => client);
|
||||
for (const row of [
|
||||
{ name: 'one', value: 'a', status: 0, labels: ['中文', 'keep'] },
|
||||
{ name: 'two', value: 'b', status: 1, labels: [] },
|
||||
{ name: null, value: 'c', status: 0, labels: null },
|
||||
])
|
||||
await repository.insert(row);
|
||||
const old = await legacy(file);
|
||||
t.after(() => old.database.close());
|
||||
const cases = [
|
||||
[{ status: [0, 1] }, { status: [0, 1] }],
|
||||
[{ name: { [Op.not]: null } }, { name: { $ne: null } }],
|
||||
[
|
||||
{ [Op.or]: [{ name: 'missing' }, {}] },
|
||||
{ $or: [{ name: 'missing' }, {}] },
|
||||
],
|
||||
[{ [Op.and]: [{ status: 0 }, {}] }, { $and: [{ status: 0 }, {}] }],
|
||||
[{ [Op.or]: [] }, { $or: [] }],
|
||||
[
|
||||
where(col('labels'), { [Op.like]: '%keep%' }),
|
||||
{ labels: { $like: '%keep%' } },
|
||||
],
|
||||
[{ name: { [Op.notIn]: ['one'] } }, { name: { $nin: ['one'] } }],
|
||||
];
|
||||
for (const [before, after] of cases) {
|
||||
const expected = (
|
||||
await old.models.Env.findAll({ where: before, order: [['id', 'ASC']] })
|
||||
).map((x) => x.id);
|
||||
const actual = (
|
||||
await repository.find(after, { orderBy: [{ id: 'asc' }] })
|
||||
).map((x) => x.id);
|
||||
assert.deepEqual(actual, expected, JSON.stringify(after));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
const test = require('node:test'),
|
||||
assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises'),
|
||||
path = require('node:path'),
|
||||
os = require('node:os'),
|
||||
{ spawnSync } = require('node:child_process');
|
||||
const { minimum, assertNodeRuntime } = require('../../shell/runtime-guard.cjs');
|
||||
const { databaseStartupFailure } = require('../../back/data/databaseErrors');
|
||||
const {
|
||||
dependencyFailure,
|
||||
} = require('../../cli/src/internal/maintenance/upgradePrestart.cjs');
|
||||
|
||||
test('runtime policy matches package metadata and accepts the exact Node minimum', () => {
|
||||
assert.equal(minimum, require('../../package.json').qinglongRuntimeMinimum);
|
||||
for (const version of ['22.17.0', '22.23.2', '24.18.1', '26.0.0'])
|
||||
assert.doesNotThrow(() => assertNodeRuntime(version));
|
||||
for (const version of ['20.20.2', '22.16.9', 'invalid'])
|
||||
assert.throws(() => assertNodeRuntime(version), {
|
||||
code: 'UNSUPPORTED_NODE',
|
||||
});
|
||||
});
|
||||
|
||||
test('database startup repairs only third-party dependency import failures', () => {
|
||||
for (const message of [
|
||||
"Cannot find module '@mikro-orm/sql'",
|
||||
"Cannot find module '/candidate/node_modules/@mikro-orm/core/missing.js'",
|
||||
"Cannot find module './missing.js'\nRequire stack:\n- /candidate/node_modules/@mikro-orm/sql/index.js",
|
||||
]) {
|
||||
const result = databaseStartupFailure(
|
||||
Object.assign(new Error(message), { code: 'MODULE_NOT_FOUND' }),
|
||||
);
|
||||
assert.equal(dependencyFailure(result), true);
|
||||
}
|
||||
for (const value of [
|
||||
new Error('heap out of memory'),
|
||||
undefined,
|
||||
Object.assign(
|
||||
new Error(
|
||||
"Cannot find module './schema'\nRequire stack:\n- /candidate/static/build/data/databaseWorker.js",
|
||||
),
|
||||
{ code: 'MODULE_NOT_FOUND' },
|
||||
),
|
||||
Object.assign(new Error("Cannot find module 'node:sqlite'"), {
|
||||
code: 'MODULE_NOT_FOUND',
|
||||
}),
|
||||
])
|
||||
assert.equal(dependencyFailure(databaseStartupFailure(value)), false);
|
||||
});
|
||||
|
||||
test('Node 20 rejects backend initialization before changing its old database or .env', async (t) => {
|
||||
const node20 = process.env.QL_TEST_NODE20;
|
||||
if (!node20) {
|
||||
t.skip('Set QL_TEST_NODE20 to verify an existing Node 20 runtime');
|
||||
return;
|
||||
}
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-runtime-floor-'));
|
||||
t.after(() => fs.rm(dir, { recursive: true, force: true }));
|
||||
const data = path.join(dir, 'data'),
|
||||
db = path.join(data, 'db/database.sqlite'),
|
||||
env = path.join(dir, '.env');
|
||||
await fs.mkdir(path.dirname(db), { recursive: true });
|
||||
await fs.writeFile(db, 'old synthetic database marker');
|
||||
await fs.writeFile(env, 'SYNTHETIC_ENV=keep\n');
|
||||
const result = spawnSync(
|
||||
node20,
|
||||
[path.resolve('static/build/bootstrap/database.js')],
|
||||
{
|
||||
env: { ...process.env, QL_DIR: dir, QL_DATA_DIR: data },
|
||||
encoding: 'utf8',
|
||||
timeout: 10000,
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 1, result.stderr);
|
||||
assert.match(result.stderr, /Node >=22\.17/);
|
||||
assert.equal(await fs.readFile(db, 'utf8'), 'old synthetic database marker');
|
||||
assert.equal(await fs.readFile(env, 'utf8'), 'SYNTHETIC_ENV=keep\n');
|
||||
});
|
||||
@@ -0,0 +1,515 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const os = require('node:os');
|
||||
const { DatabaseClient } = require('../../back/data/databaseClient');
|
||||
const { Repository } = require('../../back/data/repository');
|
||||
const { initialSchema } = require('../../back/data/schema');
|
||||
const { migrateSchema } = require('../../back/shared/schemaMigrations');
|
||||
|
||||
test('requests buffered before bootstrap and during owner-monitor startup retain their order', async (t) => {
|
||||
const { fork } = require('node:child_process');
|
||||
const directory = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), 'ql-db-bootstrap-queue-'),
|
||||
);
|
||||
const script = path.join(directory, 'delayed.ts');
|
||||
await fs.writeFile(
|
||||
script,
|
||||
`setTimeout(()=>require(${JSON.stringify(
|
||||
require.resolve('../../back/data/databaseProcess'),
|
||||
)}),100)`,
|
||||
);
|
||||
const child = fork(script, [], {
|
||||
execArgv: [
|
||||
'--require',
|
||||
require.resolve('ts-node/register/transpile-only'),
|
||||
'--max-old-space-size=128',
|
||||
],
|
||||
env: {
|
||||
...process.env,
|
||||
TS_NODE_PROJECT: path.resolve('back/data/tsconfig.worker.json'),
|
||||
},
|
||||
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
||||
});
|
||||
const stopped = new Promise((resolve) => child.once('exit', resolve));
|
||||
let stderr = '';
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
t.after(async () => {
|
||||
child.kill('SIGKILL');
|
||||
await stopped;
|
||||
await fs.rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
const responses = new Promise((resolve, reject) => {
|
||||
const replies = [];
|
||||
const timer = setTimeout(
|
||||
() => reject(new Error('Buffered startup requests were lost')),
|
||||
30000,
|
||||
);
|
||||
child.on('message', (reply) => {
|
||||
if (reply.error) {
|
||||
clearTimeout(timer);
|
||||
reject(new Error(reply.error.message));
|
||||
return;
|
||||
}
|
||||
replies.push(reply);
|
||||
if (replies.length === 2) {
|
||||
clearTimeout(timer);
|
||||
resolve(replies);
|
||||
}
|
||||
});
|
||||
child.once('exit', () => {
|
||||
clearTimeout(timer);
|
||||
reject(new Error('Database exited during bootstrap: ' + stderr));
|
||||
});
|
||||
child.once('error', reject);
|
||||
});
|
||||
child.send({ storage: ':memory:', ownerPid: process.pid });
|
||||
child.send({ id: 1, operation: 'query', args: ['SELECT 42 AS value'] });
|
||||
child.send({ id: 2, operation: 'query', args: ['SELECT 43 AS value'] });
|
||||
assert.deepEqual(await responses, [
|
||||
{ id: 1, value: [{ value: 42 }] },
|
||||
{ id: 2, value: [{ value: 43 }] },
|
||||
]);
|
||||
});
|
||||
|
||||
test('killing the owner during synchronous SQL releases the write lock and rolls back before restart', async (t) => {
|
||||
const { fork } = require('node:child_process');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const directory = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), 'ql-db-owner-exit-'),
|
||||
);
|
||||
const file = path.join(directory, 'data.sqlite');
|
||||
const script = path.join(directory, 'owner.cjs');
|
||||
await fs.writeFile(
|
||||
script,
|
||||
`
|
||||
const {DatabaseClient}=require(${JSON.stringify(
|
||||
require.resolve('../../back/data/databaseClient'),
|
||||
)});
|
||||
const db=new DatabaseClient(process.argv[2]);
|
||||
(async()=>{
|
||||
await db.query('CREATE TABLE probe(value INTEGER)');
|
||||
await db.query('INSERT INTO probe VALUES(1)');
|
||||
await db.transaction(async()=>{
|
||||
await db.query('UPDATE probe SET value=2');
|
||||
const running=db.query('WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<200000000000) SELECT SUM(x) FROM n');
|
||||
setTimeout(()=>process.send({databasePid:db.worker.pid}),300);
|
||||
await running;
|
||||
});
|
||||
})().catch(error=>{console.error(error);process.exitCode=1});
|
||||
`,
|
||||
);
|
||||
const owner = fork(script, [file], {
|
||||
execArgv: ['--require', require.resolve('ts-node/register/transpile-only')],
|
||||
env: {
|
||||
...process.env,
|
||||
TS_NODE_PROJECT: path.resolve('back/tsconfig.json'),
|
||||
},
|
||||
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
||||
});
|
||||
const exited = new Promise((resolve) => owner.once('exit', resolve));
|
||||
let databasePid,
|
||||
probe,
|
||||
stderr = '',
|
||||
unlocked = false;
|
||||
owner.stderr.on('data', (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
t.after(async () => {
|
||||
probe?.close();
|
||||
owner.kill('SIGKILL');
|
||||
await exited;
|
||||
if (databasePid && !unlocked) {
|
||||
try {
|
||||
process.kill(databasePid, 'SIGKILL');
|
||||
} catch (error) {
|
||||
if (error.code !== 'ESRCH') throw error;
|
||||
}
|
||||
}
|
||||
await fs.rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
databasePid = await new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
() => reject(new Error('Owner startup timed out: ' + stderr)),
|
||||
30000,
|
||||
);
|
||||
owner.once('message', (message) => {
|
||||
clearTimeout(timer);
|
||||
resolve(message.databasePid);
|
||||
});
|
||||
owner.once('exit', () => {
|
||||
clearTimeout(timer);
|
||||
reject(new Error('Owner exited before SQL: ' + stderr));
|
||||
});
|
||||
owner.once('error', reject);
|
||||
});
|
||||
probe = new DatabaseSync(file);
|
||||
probe.exec('PRAGMA busy_timeout=50');
|
||||
assert.throws(() => probe.exec('BEGIN IMMEDIATE'), /locked/);
|
||||
owner.kill('SIGKILL');
|
||||
await exited;
|
||||
const deadline = Date.now() + 5000;
|
||||
do {
|
||||
try {
|
||||
probe.exec('BEGIN IMMEDIATE;ROLLBACK');
|
||||
unlocked = true;
|
||||
break;
|
||||
} catch (error) {
|
||||
if (!/locked/.test(error.message)) throw error;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
} while (Date.now() < deadline);
|
||||
assert.equal(
|
||||
unlocked,
|
||||
true,
|
||||
'The orphan database process still blocks the old service',
|
||||
);
|
||||
assert.equal(probe.prepare('SELECT value FROM probe').get().value, 1);
|
||||
});
|
||||
|
||||
async function fixture(t, file = ':memory:') {
|
||||
const db = new DatabaseClient(file);
|
||||
t.after(() => db.close());
|
||||
await db.transaction(async () => {
|
||||
for (const sql of initialSchema) await db.query(sql);
|
||||
await migrateSchema(db);
|
||||
});
|
||||
return {
|
||||
db,
|
||||
env: new Repository('Env', () => db),
|
||||
auth: new Repository('Auth', () => db),
|
||||
};
|
||||
}
|
||||
|
||||
test('direct ts-node CLI startup registers the isolated database worker compiler', async () => {
|
||||
const execute = require('node:util').promisify(
|
||||
require('node:child_process').execFile,
|
||||
);
|
||||
const result = await execute(
|
||||
process.execPath,
|
||||
[
|
||||
path.resolve('node_modules/ts-node/dist/bin.js'),
|
||||
'--transpile-only',
|
||||
'--project',
|
||||
'back/tsconfig.json',
|
||||
'-e',
|
||||
`const {DatabaseClient}=require('./back/data/databaseClient');const db=new DatabaseClient(':memory:');db.query('SELECT 42 AS value').then(async rows=>{console.log(JSON.stringify(rows));await db.close()}).catch(async error=>{console.error(error);process.exitCode=1;await db.close().catch(()=>{})})`,
|
||||
],
|
||||
{ cwd: path.resolve('.'), timeout: 30000 },
|
||||
);
|
||||
assert.deepEqual(JSON.parse(result.stdout), [{ value: 42 }]);
|
||||
});
|
||||
|
||||
test('worker round-trips legacy field names, JSON, dates, null and precise positions', async (t) => {
|
||||
const { db, env } = await fixture(t);
|
||||
const row = await env.insert({
|
||||
name: '中文',
|
||||
value: 'keep',
|
||||
position: 4500000000000000,
|
||||
labels: ['old', '中文'],
|
||||
remarks: null,
|
||||
});
|
||||
assert.deepEqual((await env.findOne({ id: row.id })).labels, row.labels);
|
||||
assert.equal(row.position, 4500000000000000);
|
||||
assert.equal(row.remarks, null);
|
||||
assert(row.createdAt instanceof Date);
|
||||
const stored = (
|
||||
await db.query('SELECT typeof(createdAt) AS type,createdAt FROM Envs')
|
||||
)[0];
|
||||
assert.equal(stored.type, 'text');
|
||||
assert.match(
|
||||
stored.createdAt,
|
||||
/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{3} \+00:00$/,
|
||||
);
|
||||
assert.equal(
|
||||
(await db.query('SELECT count(*) AS n FROM SchemaMigrations'))[0].n,
|
||||
15,
|
||||
);
|
||||
});
|
||||
|
||||
test('JSON LIKE uses serialized text without quoting the pattern', async (t) => {
|
||||
const { env } = await fixture(t);
|
||||
await env.insert({ name: 'a', value: 'b', labels: ['中文', 'old'] });
|
||||
assert.equal((await env.find({ labels: { $like: '%old%' } })).length, 1);
|
||||
assert.equal(
|
||||
(await env.find({ $not: { labels: { $like: '%old%' } } })).length,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('transaction commit, failure and nested savepoints preserve ownership', async (t) => {
|
||||
const { db, env } = await fixture(t);
|
||||
await env.insert({ name: 'a', value: 'old' });
|
||||
await db.transaction(async () => {
|
||||
await env.nativeUpdate({ name: 'a' }, { value: 'committed' });
|
||||
await assert.rejects(
|
||||
db.transaction(async () => {
|
||||
await env.nativeUpdate({ name: 'a' }, { value: 'rollback' });
|
||||
throw Error('injected');
|
||||
}),
|
||||
/injected/,
|
||||
);
|
||||
assert.equal((await env.findOne({ name: 'a' })).value, 'committed');
|
||||
});
|
||||
await assert.rejects(
|
||||
db.transaction(async () => {
|
||||
await env.nativeUpdate({ name: 'a' }, { value: 'rollback all' });
|
||||
throw Error('outer failure');
|
||||
}),
|
||||
/outer failure/,
|
||||
);
|
||||
assert.equal((await env.findOne({ name: 'a' })).value, 'committed');
|
||||
});
|
||||
|
||||
test('unrelated reads wait for rollback and cannot see uncommitted data', async (t) => {
|
||||
const { db, env } = await fixture(t);
|
||||
await env.insert({ name: 'a', value: 'old' });
|
||||
let release, writing;
|
||||
const started = new Promise((r) => {
|
||||
writing = r;
|
||||
});
|
||||
const gate = new Promise((r) => {
|
||||
release = r;
|
||||
});
|
||||
const transaction = db.transaction(async () => {
|
||||
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
|
||||
writing();
|
||||
await gate;
|
||||
throw Error('rollback');
|
||||
});
|
||||
const rejected = assert.rejects(transaction, /rollback/);
|
||||
await started;
|
||||
const read = env.findOne({ name: 'a' });
|
||||
release();
|
||||
await rejected;
|
||||
assert.equal((await read).value, 'old');
|
||||
});
|
||||
|
||||
test('expired async transaction scopes cannot write after commit', async (t) => {
|
||||
const { db, env } = await fixture(t);
|
||||
// Capture an asynchronous resource inside the scope, then release it afterwards.
|
||||
let release, result;
|
||||
const gate = new Promise((r) => {
|
||||
release = r;
|
||||
});
|
||||
await db.transaction(async () => {
|
||||
result = gate.then(() => env.insert({ name: 'late' }));
|
||||
});
|
||||
const rejected = assert.rejects(result, { code: 'TRANSACTION_SCOPE' });
|
||||
release();
|
||||
await rejected;
|
||||
assert.equal(await env.count(), 0);
|
||||
});
|
||||
|
||||
test('conditional updates keep compare-and-swap counts and do not clear undefined data', async (t) => {
|
||||
const { env } = await fixture(t);
|
||||
const row = await env.insert({ name: 'a', value: 'old' });
|
||||
assert.equal(
|
||||
await env.nativeUpdate({ id: row.id, value: 'newer' }, { value: 'bad' }),
|
||||
0,
|
||||
);
|
||||
assert.equal(
|
||||
await env.nativeUpdate(
|
||||
{ id: row.id, value: 'old' },
|
||||
{ value: 'new', remarks: undefined },
|
||||
),
|
||||
1,
|
||||
);
|
||||
assert.equal((await env.findOne({ id: row.id })).value, 'new');
|
||||
await assert.rejects(env.nativeUpdate({ id: undefined }, { value: 'bad' }));
|
||||
});
|
||||
|
||||
test('missing predicates cannot become unrestricted writes or disappear inside an IN filter', async (t) => {
|
||||
const { env } = await fixture(t);
|
||||
const row = await env.insert({ name: 'retained', value: 'old' });
|
||||
for (const where of [undefined, null, { id: { $in: [row.id, undefined] } }]) {
|
||||
await assert.rejects(env.nativeUpdate(where, { value: 'must not write' }));
|
||||
await assert.rejects(env.nativeDelete(where));
|
||||
}
|
||||
assert.equal(await env.count(), 1);
|
||||
assert.equal((await env.findOne({ id: row.id })).value, 'old');
|
||||
});
|
||||
|
||||
test('worker failures reject writes, never replay them, and roll back SQLite', async (t) => {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-db-worker-'));
|
||||
t.after(() => fs.rm(dir, { recursive: true, force: true }));
|
||||
const file = path.join(dir, 'database.sqlite');
|
||||
const { db, env } = await fixture(t, file);
|
||||
await env.insert({ name: 'a', value: 'old' });
|
||||
await assert.rejects(
|
||||
db.transaction(async () => {
|
||||
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
|
||||
db.worker.kill('SIGKILL');
|
||||
await db.stopped;
|
||||
await env.count();
|
||||
}),
|
||||
);
|
||||
assert.equal(db.healthy, false);
|
||||
await assert.rejects(env.insert({ name: 'replayed' }));
|
||||
const reopened = new DatabaseClient(file);
|
||||
t.after(() => reopened.close());
|
||||
const records = new Repository('Env', () => reopened);
|
||||
assert.equal((await records.findOne({ name: 'a' })).value, 'old');
|
||||
assert.equal(await records.count(), 1);
|
||||
});
|
||||
|
||||
test('a long database query leaves main-thread timers responsive', async (t) => {
|
||||
const { db } = await fixture(t);
|
||||
const started = performance.now();
|
||||
const expensive = db.query(
|
||||
'WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<2000000) SELECT SUM(x) AS n FROM n',
|
||||
);
|
||||
const timer = await new Promise((r) =>
|
||||
setTimeout(() => r(performance.now() - started), 10),
|
||||
);
|
||||
assert(timer < 150, `main-thread timer delayed ${timer}ms`);
|
||||
assert.equal((await expensive)[0].n, 2000001000000);
|
||||
});
|
||||
|
||||
test('existing Keyv authentication cache stays byte-compatible with legacy readers', async (t) => {
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const Keyv = require('keyv').default;
|
||||
const { CacheStore } = require('../../back/data/cacheStore');
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-cache-compat-'));
|
||||
t.after(() => fs.rm(dir, { recursive: true, force: true }));
|
||||
const cacheFile = path.join(dir, 'keyv.sqlite');
|
||||
const old = new DatabaseSync(cacheFile);
|
||||
old.exec('CREATE TABLE keyv(key VARCHAR(255) PRIMARY KEY,value TEXT)');
|
||||
const value = {
|
||||
username: 'synthetic-user',
|
||||
password: 'synthetic-hash',
|
||||
tokens: { desktop: ['keep'] },
|
||||
};
|
||||
old
|
||||
.prepare('INSERT INTO keyv VALUES(?,?)')
|
||||
.run('keyv:authInfo', JSON.stringify({ value, expires: null }));
|
||||
old.close();
|
||||
const db = new DatabaseClient(':memory:', cacheFile);
|
||||
t.after(() => db.close());
|
||||
const cache = new Keyv({ store: new CacheStore(() => db) });
|
||||
assert.deepEqual(await cache.get('authInfo'), value);
|
||||
const next = { ...value, username: 'new-synthetic-user' };
|
||||
await cache.set('authInfo', next);
|
||||
const legacy = new DatabaseSync(cacheFile, { readOnly: true });
|
||||
try {
|
||||
assert.deepEqual(
|
||||
JSON.parse(
|
||||
legacy
|
||||
.prepare('SELECT value FROM keyv WHERE key=?')
|
||||
.get('keyv:authInfo').value,
|
||||
).value,
|
||||
next,
|
||||
);
|
||||
} finally {
|
||||
legacy.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('optional DTO fields and nested JSON values may be undefined on insert', async (t) => {
|
||||
const { env, auth } = await fixture(t);
|
||||
const created = await env.insert({
|
||||
name: 'optional',
|
||||
value: 'keep',
|
||||
id: undefined,
|
||||
remarks: undefined,
|
||||
});
|
||||
assert.equal(created.value, 'keep');
|
||||
const row = await auth.insert({
|
||||
type: 'notifyLog',
|
||||
info: { title: 'keep', taskId: undefined },
|
||||
});
|
||||
assert.deepEqual(row.info, { title: 'keep' });
|
||||
});
|
||||
|
||||
test('legacy null pin flags sort like zero flags before comparing positions', async (t) => {
|
||||
const { env } = await fixture(t);
|
||||
await env.insert({ name: 'null', position: 20, isPinned: null });
|
||||
await env.insert({ name: 'zero', position: 10, isPinned: 0 });
|
||||
await env.insert({ name: 'pinned', position: 1, isPinned: 1 });
|
||||
assert.deepEqual(
|
||||
(
|
||||
await env.find(
|
||||
{},
|
||||
{ orderBy: [{ isPinned: 'desc' }, { position: 'desc' }] },
|
||||
)
|
||||
).map((x) => x.name),
|
||||
['pinned', 'null', 'zero'],
|
||||
);
|
||||
});
|
||||
|
||||
test('a stalled transaction callback times out and rolls back before unrelated reads', async (t) => {
|
||||
const { db, env } = await fixture(t);
|
||||
db.timeoutMs = 100;
|
||||
await env.insert({ name: 'a', value: 'old' });
|
||||
let release;
|
||||
const gate = new Promise((r) => {
|
||||
release = r;
|
||||
});
|
||||
await assert.rejects(
|
||||
db.transaction(async () => {
|
||||
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
|
||||
await gate;
|
||||
await env.insert({ name: 'too-late' });
|
||||
}),
|
||||
{ code: 'DATABASE_TRANSACTION_TIMEOUT' },
|
||||
);
|
||||
release();
|
||||
assert.equal((await env.findOne({ name: 'a' })).value, 'old');
|
||||
assert.equal(await env.count(), 1);
|
||||
assert.equal(db.healthy, true);
|
||||
});
|
||||
|
||||
test('a stuck database worker is terminated, rejects its queue and never replays writes', async (t) => {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-db-timeout-'));
|
||||
t.after(() => fs.rm(dir, { recursive: true, force: true }));
|
||||
const file = path.join(dir, 'database.sqlite');
|
||||
const { db, env } = await fixture(t, file);
|
||||
await env.insert({ name: 'a', value: 'old' });
|
||||
db.timeoutMs = 100;
|
||||
await assert.rejects(
|
||||
db.transaction(async () => {
|
||||
await env.nativeUpdate({ name: 'a' }, { value: 'uncommitted' });
|
||||
await db.query(
|
||||
'WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<200000000) SELECT SUM(x) FROM n',
|
||||
);
|
||||
}),
|
||||
(e) =>
|
||||
['DATABASE_TRANSACTION_TIMEOUT', 'DATABASE_WORKER_TIMEOUT'].includes(
|
||||
e.code,
|
||||
),
|
||||
);
|
||||
assert.equal(db.healthy, false);
|
||||
await assert.rejects(env.insert({ name: 'replayed' }));
|
||||
await db.close();
|
||||
const next = new DatabaseClient(file);
|
||||
t.after(() => next.close());
|
||||
const records = new Repository('Env', () => next);
|
||||
assert.equal((await records.findOne({ name: 'a' })).value, 'old');
|
||||
assert.equal(await records.count(), 1);
|
||||
});
|
||||
|
||||
test('the database queue is bounded while a transaction owns its connection', async (t) => {
|
||||
const { db, env } = await fixture(t);
|
||||
db.queueLimit = 2;
|
||||
db.timeoutMs = 3000;
|
||||
let entered, release;
|
||||
const started = new Promise((r) => {
|
||||
entered = r;
|
||||
});
|
||||
const gate = new Promise((r) => {
|
||||
release = r;
|
||||
});
|
||||
const transaction = db.transaction(async () => {
|
||||
entered();
|
||||
await gate;
|
||||
});
|
||||
await started;
|
||||
const waiting = env.count();
|
||||
await assert.rejects(env.count(), { code: 'DATABASE_BUSY' });
|
||||
release();
|
||||
await transaction;
|
||||
assert.equal(await waiting, 0);
|
||||
});
|
||||
@@ -1,6 +1,5 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const test = require('node:test');
|
||||
|
||||
test('dependency listing marks cache entries missing from disk for reinstall', async (t) => {
|
||||
const moduleStubs = new Map();
|
||||
const stubModule = (modulePath, exports) => {
|
||||
@@ -15,7 +14,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
|
||||
paths: [],
|
||||
};
|
||||
};
|
||||
|
||||
const DependenceStatus = {
|
||||
installing: 0,
|
||||
installed: 1,
|
||||
@@ -28,7 +26,7 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
|
||||
];
|
||||
const updates = [];
|
||||
const DependenceModel = {
|
||||
findAll: async ({ where }) => {
|
||||
find: async (where, read = {}) => {
|
||||
if (where.status === DependenceStatus.installed) {
|
||||
return docs;
|
||||
}
|
||||
@@ -39,11 +37,10 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
|
||||
: doc.status,
|
||||
}));
|
||||
},
|
||||
update: async ({ status }, { where }) => {
|
||||
nativeUpdate: async (where, { status }) => {
|
||||
updates.push({ status, ids: where.id });
|
||||
},
|
||||
};
|
||||
|
||||
stubModule('../../back/config', {
|
||||
__esModule: true,
|
||||
default: {},
|
||||
@@ -83,7 +80,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
|
||||
__esModule: true,
|
||||
default: class SockService {},
|
||||
});
|
||||
|
||||
const servicePath = require.resolve('../../back/services/dependence');
|
||||
const originalService = require.cache[servicePath];
|
||||
delete require.cache[servicePath];
|
||||
@@ -101,7 +97,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const DependenceService = require('../../back/services/dependence').default;
|
||||
const service = new DependenceService({}, {});
|
||||
const result = await service.dependencies({
|
||||
@@ -109,7 +104,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
|
||||
type: 'nodejs',
|
||||
status: '',
|
||||
});
|
||||
|
||||
assert.deepEqual(updates, [
|
||||
{ status: DependenceStatus.installFailed, ids: [1] },
|
||||
]);
|
||||
@@ -120,7 +114,6 @@ test('dependency listing marks cache entries missing from disk for reinstall', a
|
||||
{ id: 2, status: DependenceStatus.installed },
|
||||
],
|
||||
);
|
||||
|
||||
await service.dependencies({ searchValue: '', type: 'linux', status: '' });
|
||||
assert.equal(updates.length, 1);
|
||||
});
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const test = require('node:test');
|
||||
const express = require('express');
|
||||
|
||||
function mockModule(modulePath, exports) {
|
||||
const filename = require.resolve(modulePath);
|
||||
require.cache[filename] = {
|
||||
@@ -13,7 +12,6 @@ function mockModule(modulePath, exports) {
|
||||
paths: [],
|
||||
};
|
||||
}
|
||||
|
||||
mockModule('../../back/config', {
|
||||
__esModule: true,
|
||||
default: {
|
||||
@@ -49,15 +47,17 @@ for (const service of ['config', 'script', 'log']) {
|
||||
}
|
||||
mockModule('../../back/data/runningInstance', {
|
||||
InstanceStatus: { running: 'running' },
|
||||
RunningInstanceModel: { findOne: async () => null },
|
||||
RunningInstanceModel: {
|
||||
findOne: async (where, read = {}) => {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const deprecatedRoutes = [
|
||||
['config', '/configs/detail'],
|
||||
['script', '/scripts/detail'],
|
||||
['log', '/logs/detail'],
|
||||
];
|
||||
|
||||
for (const [moduleName, replacement] of deprecatedRoutes) {
|
||||
test(`${moduleName} filename route points callers to its detail API`, () => {
|
||||
const app = express.Router();
|
||||
@@ -68,12 +68,10 @@ for (const [moduleName, replacement] of deprecatedRoutes) {
|
||||
);
|
||||
const handler = deprecatedRoute.route.stack.at(-1).handle;
|
||||
let responseBody;
|
||||
|
||||
handler(
|
||||
{ params: { file: 'Example.js' }, query: {} },
|
||||
{ send: (body) => (responseBody = body) },
|
||||
);
|
||||
|
||||
assert.deepEqual(responseBody, {
|
||||
code: 410,
|
||||
message: `接口已下线,请使用 ${replacement} 接口`,
|
||||
|
||||
@@ -5,7 +5,6 @@ const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { execFileSync } = require('node:child_process');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
|
||||
const types = { nodejs: 0, python3: 1, linux: 2 };
|
||||
const statuses = { installed: 1, installFailed: 2, queued: 6, cancelled: 7 };
|
||||
const util = load('back/config/util.ts', {
|
||||
@@ -15,7 +14,6 @@ const util = load('back/config/util.ts', {
|
||||
'../shared/utils': {},
|
||||
'../data/dependence': { DependenceTypes: types },
|
||||
});
|
||||
|
||||
function serviceFor(docs) {
|
||||
const probes = [],
|
||||
removed = [],
|
||||
@@ -28,18 +26,21 @@ function serviceFor(docs) {
|
||||
DependenceStatus: statuses,
|
||||
versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' },
|
||||
DependenceModel: {
|
||||
findAll: async ({ where }) =>
|
||||
docs.filter(
|
||||
find: async (where, read = {}) => {
|
||||
return docs.filter(
|
||||
(d) =>
|
||||
(where.status === undefined || d.status === where.status) &&
|
||||
(!where.id || where.id.includes(d.id)),
|
||||
),
|
||||
update: async (values, { where }) => {
|
||||
);
|
||||
},
|
||||
nativeUpdate: async (where, values) => {
|
||||
docs
|
||||
.filter((d) => where.id.includes(d.id))
|
||||
.forEach((d) => Object.assign(d, values));
|
||||
},
|
||||
destroy: async ({ where }) => removed.push(...where.id),
|
||||
nativeDelete: async (where, read = {}) => {
|
||||
return removed.push(...where.id);
|
||||
},
|
||||
},
|
||||
},
|
||||
'../config/util': {
|
||||
@@ -62,7 +63,6 @@ function serviceFor(docs) {
|
||||
};
|
||||
return { service, probes, removed, uninstalls };
|
||||
}
|
||||
|
||||
test('listing preserves Python source results and still detects missing named packages', async () => {
|
||||
const docs = [
|
||||
' requirements with spaces/requirements.txt ',
|
||||
@@ -88,7 +88,6 @@ test('listing preserves Python source results and still detects missing named pa
|
||||
// Source installs must continue to execute pip rather than treating the source as a package name.
|
||||
assert.equal(util.getGetCommand(1, docs[0].name), 'echo ""');
|
||||
});
|
||||
|
||||
test('deletion cleans previously misclassified Python sources and only queues intended records', async () => {
|
||||
const docs = [
|
||||
{ id: 1, name: 'requirements.txt', type: 1, status: 2 },
|
||||
@@ -106,7 +105,6 @@ test('deletion cleans previously misclassified Python sources and only queues in
|
||||
]);
|
||||
assert.equal(docs[2].status, 2);
|
||||
});
|
||||
|
||||
test('Alpine container mirror command runs without sudo and writes valid repositories', async (t) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-alpine-mirror-'));
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
@@ -151,7 +149,6 @@ test('Alpine container mirror command runs without sudo and writes valid reposit
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('force deletion removes a source record even when its requirements file is gone', async () => {
|
||||
const removed = [],
|
||||
updates = [];
|
||||
@@ -167,8 +164,12 @@ test('force deletion removes a source record even when its requirements file is
|
||||
DependenceTypes: types,
|
||||
DependenceStatus: { ...statuses, removeFailed: 5 },
|
||||
DependenceModel: {
|
||||
destroy: async ({ where }) => removed.push(...where.id),
|
||||
update: async (values) => updates.push(values),
|
||||
nativeDelete: async (where, read = {}) => {
|
||||
return removed.push(...where.id);
|
||||
},
|
||||
nativeUpdate: async (where, values) => {
|
||||
return updates.push(values);
|
||||
},
|
||||
},
|
||||
},
|
||||
'../shared/pLimit': {
|
||||
@@ -186,7 +187,6 @@ test('force deletion removes a source record even when its requirements file is
|
||||
await service.installOrUninstallDependency(dep, false, true);
|
||||
assert.deepEqual(removed, [1]);
|
||||
});
|
||||
|
||||
test('Python extras run pip even when the base package version is already installed', async (t) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-extra-install-'));
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
@@ -217,8 +217,12 @@ test('Python extras run pip even when the base package version is already instal
|
||||
DependenceStatus: { ...statuses, installing: 0, cancelled: 7 },
|
||||
versionDependenceCommandTypes: { 0: '@', 1: '==', 2: '=' },
|
||||
DependenceModel: {
|
||||
findAll: async () => [dep],
|
||||
update: async (values) => Object.assign(dep, values),
|
||||
find: async (where, read = {}) => {
|
||||
return [dep];
|
||||
},
|
||||
nativeUpdate: async (where, values) => {
|
||||
return Object.assign(dep, values);
|
||||
},
|
||||
},
|
||||
},
|
||||
'../shared/pLimit': {
|
||||
|
||||
@@ -9,10 +9,8 @@ const path = require('node:path');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
const { observeChildProcess } = require('../../back/shared/childProcess');
|
||||
const { LogStreamManager } = require('../../back/shared/logStreamManager');
|
||||
|
||||
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const logger = { info() {}, error() {} };
|
||||
|
||||
test(
|
||||
'spawn failure settles without exit, and the next scheduled run can proceed',
|
||||
{ timeout: 3000 },
|
||||
@@ -36,7 +34,11 @@ test(
|
||||
},
|
||||
'../loaders/logger': logger,
|
||||
'../data/cron': {
|
||||
CrontabModel: { findOne: async () => ({ isDisabled: 0 }) },
|
||||
CrontabModel: {
|
||||
findOne: async (where, read = {}) => {
|
||||
return { isDisabled: 0 };
|
||||
},
|
||||
},
|
||||
CrontabStatus: {},
|
||||
},
|
||||
'../data/runningInstance': {
|
||||
@@ -51,7 +53,6 @@ test(
|
||||
assert.equal(releases, 2);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'completion waits for slow log consumers and preserves the final output',
|
||||
{ timeout: 5000 },
|
||||
@@ -79,7 +80,6 @@ test(
|
||||
assert.match(chunks.join(''), /错误/);
|
||||
},
|
||||
);
|
||||
|
||||
test('exit is not completion, and UTF-8 split across writes remains intact', async () => {
|
||||
const child = new EventEmitter();
|
||||
child.stdout = new PassThrough();
|
||||
@@ -105,7 +105,6 @@ test('exit is not completion, and UTF-8 split across writes remains intact', asy
|
||||
await observed.completed;
|
||||
assert.equal(chunks.join(''), '末尾中文');
|
||||
});
|
||||
|
||||
test(
|
||||
'failed log sink drains large output instead of blocking the child',
|
||||
{ timeout: 5000 },
|
||||
@@ -124,7 +123,6 @@ test(
|
||||
assert.equal(result.error.message, 'ENOSPC');
|
||||
},
|
||||
);
|
||||
|
||||
function scheduleFixture() {
|
||||
let active = 0;
|
||||
const limit = async (_params, fn) => {
|
||||
@@ -150,7 +148,6 @@ function scheduleFixture() {
|
||||
}).default;
|
||||
return { service: new Schedule(logger), active: () => active };
|
||||
}
|
||||
|
||||
test(
|
||||
'PID response keeps queue capacity occupied until cleanup completes',
|
||||
{ timeout: 4000 },
|
||||
@@ -178,7 +175,6 @@ test(
|
||||
assert.equal(fixture.active(), 0);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'before/error/end callback failures settle and release capacity',
|
||||
{ timeout: 3000 },
|
||||
@@ -207,7 +203,6 @@ test(
|
||||
assert.equal(fixture.active(), 0);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'concurrent writes and closes preserve all bytes, and failed files can be closed',
|
||||
{ timeout: 5000 },
|
||||
|
||||
@@ -5,7 +5,6 @@ const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const express = require('express');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
|
||||
function fixture(t, customData = false) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-access-scope-'));
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
@@ -46,12 +45,15 @@ function fixture(t, customData = false) {
|
||||
'../config/const': {},
|
||||
'../data/runningInstance': {
|
||||
InstanceStatus: { running: 'running' },
|
||||
RunningInstanceModel: { findOne: async () => null },
|
||||
RunningInstanceModel: {
|
||||
findOne: async (where, read = {}) => {
|
||||
return null;
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
return { config, mocks, secret };
|
||||
}
|
||||
|
||||
function api(name, service, mocks) {
|
||||
mocks.typedi = { Container: { get: () => service } };
|
||||
const app = express.Router();
|
||||
@@ -79,7 +81,6 @@ function api(name, service, mocks) {
|
||||
return result;
|
||||
};
|
||||
}
|
||||
|
||||
for (const customData of [false, true]) {
|
||||
test(`config APIs reject script compatibility paths (custom=${customData})`, async (t) => {
|
||||
const { config, mocks, secret } = fixture(t, customData);
|
||||
@@ -144,7 +145,6 @@ for (const customData of [false, true]) {
|
||||
assert.equal(fs.readFileSync(secret, 'utf8'), 'PANEL-SECRET');
|
||||
});
|
||||
}
|
||||
|
||||
test('log APIs allow configuration-like names while retaining directory boundaries', async (t) => {
|
||||
const { config, mocks, secret } = fixture(t);
|
||||
const Log = load(path.resolve('back/services/log.ts'), mocks).default;
|
||||
|
||||
@@ -2,7 +2,7 @@ const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const ts = require('typescript');
|
||||
const { Sequelize, DataTypes, Model } = require('sequelize');
|
||||
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
const { CronResponse } = load('back/protos/api.ts');
|
||||
const source = fs.readFileSync('back/schedule/api.ts', 'utf8');
|
||||
@@ -11,31 +11,16 @@ const end = source.indexOf('export const getCronDetail', start);
|
||||
const code = ts.transpileModule(source.slice(start, end), {
|
||||
compilerOptions: { target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText;
|
||||
const normalize = new Function('Model', `${code}\nreturn normalizeCronData;`)(
|
||||
Model,
|
||||
);
|
||||
const normalize = new Function(`${code}\nreturn normalizeCronData;`)();
|
||||
|
||||
test('gRPC cron response preserves Sequelize model fields through protobuf serialization', async (t) => {
|
||||
const db = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
t.after(() => db.close());
|
||||
const Cron = db.define('Cron', {
|
||||
name: DataTypes.STRING,
|
||||
command: DataTypes.STRING,
|
||||
schedule: DataTypes.STRING,
|
||||
labels: DataTypes.JSON,
|
||||
extra_schedules: DataTypes.JSON,
|
||||
});
|
||||
const model = Cron.build({
|
||||
test('gRPC cron response preserves plain repository fields through protobuf serialization', async (t) => {
|
||||
const model = {
|
||||
id: 7,
|
||||
name: 'smoke',
|
||||
command: 'true',
|
||||
schedule: '* * * * *',
|
||||
labels: ['test'],
|
||||
});
|
||||
};
|
||||
const decoded = CronResponse.decode(
|
||||
CronResponse.encode({ code: 200, data: normalize(model) }).finish(),
|
||||
);
|
||||
|
||||
@@ -5,7 +5,6 @@ const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
const { LogStreamManager } = require('../../back/shared/logStreamManager');
|
||||
|
||||
async function fixture(t) {
|
||||
const base = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-log-boundary-'));
|
||||
t.after(() => fs.rm(base, { recursive: true, force: true }));
|
||||
@@ -31,7 +30,6 @@ async function fixture(t) {
|
||||
];
|
||||
return { root, outside, victim, invalid };
|
||||
}
|
||||
|
||||
test('log streams reject traversal, sibling prefixes and escaping symlinks before writing', async (t) => {
|
||||
const { root, outside, victim, invalid } = await fixture(t);
|
||||
const manager = new LogStreamManager(root);
|
||||
@@ -58,7 +56,6 @@ test('log streams reject traversal, sibling prefixes and escaping symlinks befor
|
||||
await manager.closeAll();
|
||||
assert.equal(await fs.readFile(log, 'utf8'), '开始\n结束\n');
|
||||
});
|
||||
|
||||
test('log initialization rejects unsafe paths before mkdir or file writes', async (t) => {
|
||||
const { root, outside, victim, invalid } = await fixture(t);
|
||||
const { handleLogPath } = load(path.resolve('back/config/util.ts'), {
|
||||
@@ -87,7 +84,6 @@ test('log initialization rejects unsafe paths before mkdir or file writes', asyn
|
||||
assert.equal(await handleLogPath(log, 'ignored'), log);
|
||||
assert.equal(await fs.readFile(log, 'utf8'), 'initial');
|
||||
});
|
||||
|
||||
test('manual execution rejects escaping log names before creating directories or spawning', async (t) => {
|
||||
const { root, outside, victim } = await fixture(t);
|
||||
let spawned = 0;
|
||||
@@ -97,7 +93,7 @@ test('manual execution rejects escaping log names before creating directories or
|
||||
'../config': { logPath: root },
|
||||
'../data/cron': {
|
||||
CrontabStatus: { queued: 3, idle: 1 },
|
||||
CrontabModel: { update: async () => {} },
|
||||
CrontabModel: { nativeUpdate: async (where, data) => {} },
|
||||
},
|
||||
'../data/runningInstance': { RunningInstanceModel: {}, InstanceStatus: {} },
|
||||
'../config/util': {},
|
||||
|
||||
@@ -6,7 +6,6 @@ const os = require('node:os');
|
||||
const { spawn } = require('node:child_process');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
const { LogStreamManager } = require('../../back/shared/logStreamManager');
|
||||
|
||||
test(
|
||||
'manual execution captures a short child before slow status storage and flushes before release',
|
||||
{ timeout: 5000 },
|
||||
@@ -22,10 +21,13 @@ test(
|
||||
'../data/cron': {
|
||||
CrontabStatus: { queued: 3, running: 0, idle: 1 },
|
||||
CrontabModel: {
|
||||
update: async (data, options) => {
|
||||
nativeUpdate: async (where, data) => {
|
||||
const options = {
|
||||
where: where,
|
||||
};
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
updates.push({ data, options });
|
||||
return [1];
|
||||
return 1;
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
const { DatabaseClient } = require('../../back/data/databaseClient');
|
||||
const { repository, initialize } = require('../helpers/repository-fixture.cjs');
|
||||
const { orderBy } = require('../../back/data/queries');
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
@@ -9,36 +12,21 @@ const { Sequelize, DataTypes } = require('sequelize');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
const { killTask } = require('../../back/config/util');
|
||||
const { LogStreamManager } = require('../../back/shared/logStreamManager');
|
||||
|
||||
for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
|
||||
test(
|
||||
`manual startup loses its conditional claim after ${conflict} and terminates the late child`,
|
||||
{ timeout: 10000 },
|
||||
async (t) => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-manual-claim-'));
|
||||
const db = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const db = new DatabaseClient(':memory:');
|
||||
t.after(async () => {
|
||||
await db.close();
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
const crons = db.define('Cron', {
|
||||
status: DataTypes.INTEGER,
|
||||
pid: DataTypes.INTEGER,
|
||||
log_path: DataTypes.STRING,
|
||||
queued_token: DataTypes.STRING,
|
||||
command: DataTypes.STRING,
|
||||
});
|
||||
const instances = db.define('Instance', {
|
||||
cron_id: DataTypes.INTEGER,
|
||||
status: DataTypes.INTEGER,
|
||||
pid: DataTypes.INTEGER,
|
||||
});
|
||||
await db.sync();
|
||||
await crons.create({
|
||||
const crons = repository(db, 'Crontab');
|
||||
const instances = repository(db, 'RunningInstance');
|
||||
await initialize(db);
|
||||
await crons.insert({
|
||||
id: 1,
|
||||
status: 3,
|
||||
pid: null,
|
||||
@@ -102,7 +90,9 @@ for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
|
||||
}).default;
|
||||
const service = new CronService({ info() {}, error() {} });
|
||||
service.getDb = async () =>
|
||||
(await crons.findByPk(1)).get({ plain: true });
|
||||
await crons.findOne({
|
||||
id: 1,
|
||||
});
|
||||
service.makeCommand = () => 'ignored';
|
||||
const running = service.runSingle(1);
|
||||
await pending;
|
||||
@@ -115,16 +105,22 @@ for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
|
||||
assert.equal(pendingRuns.length, 1);
|
||||
assert.equal(
|
||||
pendingRuns[0].token,
|
||||
(await crons.findByPk(1)).queued_token,
|
||||
(
|
||||
await crons.findOne({
|
||||
id: 1,
|
||||
})
|
||||
).queued_token,
|
||||
);
|
||||
} else if (conflict === 'stop') await service.stop([1]);
|
||||
else await crons.update({ log_path: 'newer.log' }, { where: { id: 1 } });
|
||||
else await crons.nativeUpdate({ id: 1 }, { log_path: 'newer.log' });
|
||||
release();
|
||||
const result = await running;
|
||||
assert.match(result.error.message, /stopped or superseded/);
|
||||
assert.ok(child.signalCode || child.exitCode !== null);
|
||||
assert.throws(() => process.kill(child.pid, 0), { code: 'ESRCH' });
|
||||
const row = await crons.findByPk(1);
|
||||
const row = await crons.findOne({
|
||||
id: 1,
|
||||
});
|
||||
assert.equal(row.status, conflict === 'stop' ? 1 : 3);
|
||||
assert.equal(
|
||||
row.log_path,
|
||||
@@ -135,7 +131,6 @@ for (const conflict of ['stop', 'newer-queue', 'stop-requeue']) {
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test(
|
||||
'verified termination waits for a SIGTERM-resistant target to exit',
|
||||
{ timeout: 5000 },
|
||||
@@ -158,22 +153,12 @@ test(
|
||||
assert.equal(child.signalCode, 'SIGKILL');
|
||||
},
|
||||
);
|
||||
|
||||
test('a runner waiting for a concurrency slot cannot adopt a newer queued generation', async (t) => {
|
||||
const db = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const db = new DatabaseClient(':memory:');
|
||||
t.after(() => db.close());
|
||||
const crons = db.define('Cron', {
|
||||
status: DataTypes.INTEGER,
|
||||
pid: DataTypes.INTEGER,
|
||||
log_path: DataTypes.STRING,
|
||||
queued_token: DataTypes.STRING,
|
||||
});
|
||||
await db.sync();
|
||||
await crons.create({ id: 1, status: 1, log_path: 'same.log' });
|
||||
const crons = repository(db, 'Crontab');
|
||||
await initialize(db);
|
||||
await crons.insert({ id: 1, status: 1, log_path: 'same.log' });
|
||||
const waiting = [];
|
||||
const CronService = load('back/services/cron.ts', {
|
||||
'../config': {},
|
||||
@@ -182,7 +167,7 @@ test('a runner waiting for a concurrency slot cannot adopt a newer queued genera
|
||||
CrontabStatus: { queued: 3, idle: 1, running: 0 },
|
||||
},
|
||||
'../data/runningInstance': {
|
||||
RunningInstanceModel: { findAll: async () => [] },
|
||||
RunningInstanceModel: { find: async () => [] },
|
||||
InstanceStatus: { running: 0 },
|
||||
},
|
||||
'../config/util': {
|
||||
@@ -205,14 +190,24 @@ test('a runner waiting for a concurrency slot cannot adopt a newer queued genera
|
||||
const errors = [];
|
||||
const service = new CronService({ error: (...args) => errors.push(args) });
|
||||
await service.run([1]);
|
||||
const first = (await crons.findByPk(1)).queued_token;
|
||||
const first = (
|
||||
await crons.findOne({
|
||||
id: 1,
|
||||
})
|
||||
).queued_token;
|
||||
await service.stop([1]);
|
||||
await service.run([1]);
|
||||
const second = (await crons.findByPk(1)).queued_token;
|
||||
const second = (
|
||||
await crons.findOne({
|
||||
id: 1,
|
||||
})
|
||||
).queued_token;
|
||||
assert.notEqual(first, second);
|
||||
assert.equal(waiting.length, 2);
|
||||
await waiting[0]();
|
||||
const row = await crons.findByPk(1);
|
||||
const row = await crons.findOne({
|
||||
id: 1,
|
||||
});
|
||||
assert.equal(row.status, 3);
|
||||
assert.equal(row.queued_token, second);
|
||||
assert.equal(row.log_path, 'same.log');
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { probeNodeRuntime } = require('../../shell/node-runtime-probe.cjs');
|
||||
|
||||
test('the retained Node handles URL, IDNA and file paths in fresh eval and file processes', () => {
|
||||
const result = probeNodeRuntime();
|
||||
assert.equal(result.ok, true);
|
||||
assert.equal(result.processes, 4);
|
||||
assert.match(result.nodeSha256, /^[0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
test('a native child crash fails the runtime probe instead of accepting its healthy parent', (t) => {
|
||||
if (process.platform === 'win32') return t.skip('POSIX crash fixture');
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-node-crash-'));
|
||||
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
|
||||
const executable = path.join(directory, 'node');
|
||||
fs.writeFileSync(executable, '#!/bin/sh\nkill -SEGV $$\n', { mode: 0o700 });
|
||||
assert.throws(
|
||||
() => probeNodeRuntime(executable),
|
||||
/Node URL runtime probe failed.*SIGSEGV/,
|
||||
);
|
||||
});
|
||||
@@ -1,20 +1,16 @@
|
||||
const { DatabaseClient } = require('../../back/data/databaseClient');
|
||||
const { repository, initialize } = require('../helpers/repository-fixture.cjs');
|
||||
const { orderBy } = require('../../back/data/queries');
|
||||
const test = require('node:test'),
|
||||
assert = require('node:assert/strict'),
|
||||
{ Sequelize, DataTypes } = require('sequelize');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
const decorators = { Service: () => (x) => x, Inject: () => () => {} };
|
||||
async function fixture(t) {
|
||||
const db = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const db = new DatabaseClient(':memory:');
|
||||
t.after(() => db.close());
|
||||
const rows = db.define('System', {
|
||||
type: DataTypes.STRING,
|
||||
info: DataTypes.JSON,
|
||||
});
|
||||
await db.sync();
|
||||
const rows = repository(db, 'Auth');
|
||||
await initialize(db);
|
||||
const logger = { error() {} };
|
||||
const S = load('back/services/system.ts', {
|
||||
typedi: decorators,
|
||||
@@ -59,20 +55,20 @@ async function fixture(t) {
|
||||
}
|
||||
test('writes cap logs without reading the page and preserve other system data', async (t) => {
|
||||
const { rows, service } = await fixture(t);
|
||||
await rows.create({ type: 'authConfig', info: { keep: true } });
|
||||
await rows.insert({ type: 'authConfig', info: { keep: true } });
|
||||
for (let i = 0; i < 205; i++)
|
||||
await service.notify({ title: String(i), content: 'body' });
|
||||
assert.equal(await rows.count({ where: { type: 'notifyLog' } }), 200);
|
||||
assert.equal(await rows.count({ type: 'notifyLog' }), 200);
|
||||
const logs = await service.getNotifyLog();
|
||||
assert.equal(logs[0].title, '204');
|
||||
assert.equal(logs.at(-1).title, '5');
|
||||
assert.equal(logs[0].notifyType, 'email');
|
||||
assert.equal(await rows.count({ where: { type: 'authConfig' } }), 1);
|
||||
assert.equal(await rows.count({ type: 'authConfig' }), 1);
|
||||
});
|
||||
test('audit DB failure does not turn a successful notification into failure', async (t) => {
|
||||
const { rows, service } = await fixture(t);
|
||||
const create = rows.create;
|
||||
rows.create = async () => {
|
||||
const create = rows.insert;
|
||||
rows.insert = async () => {
|
||||
throw Error('disk full');
|
||||
};
|
||||
try {
|
||||
@@ -81,7 +77,7 @@ test('audit DB failure does not turn a successful notification into failure', as
|
||||
200,
|
||||
);
|
||||
} finally {
|
||||
rows.create = create;
|
||||
rows.insert = create;
|
||||
}
|
||||
});
|
||||
test('failed sends are recorded and message size is bounded', async (t) => {
|
||||
@@ -103,7 +99,6 @@ test('failed sends are recorded and message size is bounded', async (t) => {
|
||||
assert.equal(row.content.length, 8192);
|
||||
assert.equal(row.notifyType, 'gotify');
|
||||
});
|
||||
|
||||
test('script reports share retention and preserve source without sending twice', async (t) => {
|
||||
const { service } = await fixture(t);
|
||||
await service.recordNotifyLog({
|
||||
|
||||
+516
-106
@@ -12,20 +12,38 @@ const { restorePackageLock } = require('../../shell/install-package.cjs');
|
||||
const { restoreEnvironment } = require('../../shell/install-package.cjs');
|
||||
|
||||
async function packageFixture(t) {
|
||||
const temp = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-contract-')));
|
||||
const temp = await fs.realpath(
|
||||
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-contract-')),
|
||||
);
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const root = path.join(temp, 'source'), output = path.join(temp, 'archives');
|
||||
const root = path.join(temp, 'source'),
|
||||
output = path.join(temp, 'archives');
|
||||
await fs.mkdir(root);
|
||||
await fs.mkdir(output);
|
||||
const copy = async (name) => {
|
||||
await fs.mkdir(path.dirname(path.join(root, name)), { recursive: true });
|
||||
await fs.copyFile(path.resolve(name), path.join(root, name));
|
||||
};
|
||||
for (const name of ['.gitignore', 'docker/verify-build.cjs', 'docker/build-manifest.cjs', '.env.example', 'ecosystem.config.js', 'version.yaml', 'README.md', 'LICENSE', 'scripts/prepare-npm-package.cjs', 'back/protos/api.proto']) await copy(name);
|
||||
for (const name of [
|
||||
'.gitignore',
|
||||
'docker/verify-build.cjs',
|
||||
'docker/build-manifest.cjs',
|
||||
'.env.example',
|
||||
'ecosystem.config.js',
|
||||
'version.yaml',
|
||||
'README.md',
|
||||
'LICENSE',
|
||||
'scripts/prepare-npm-package.cjs',
|
||||
'back/protos/api.proto',
|
||||
])
|
||||
await copy(name);
|
||||
for (const directory of ['shell', 'sample', 'cli/src/internal/maintenance']) {
|
||||
await fs.cp(directory, path.join(root, directory), {
|
||||
recursive: true,
|
||||
filter: (source) => !/[/\\](?:__pycache__|node_modules)(?:[/\\]|$)|[/\\]preload[/\\](?:env\.|notify\.|__ql_notify__\.|lang_env\.sh)/.test(source),
|
||||
filter: (source) =>
|
||||
!/[/\\](?:__pycache__|node_modules)(?:[/\\]|$)|[/\\]preload[/\\](?:env\.|notify\.|__ql_notify__\.|lang_env\.sh)/.test(
|
||||
source,
|
||||
),
|
||||
});
|
||||
}
|
||||
const metadata = JSON.parse(await fs.readFile('package.json', 'utf8'));
|
||||
@@ -37,13 +55,59 @@ async function packageFixture(t) {
|
||||
const lock = 'lockfileVersion: 6.0\n';
|
||||
await fs.writeFile(path.join(root, 'pnpm-lock.yaml'), lock);
|
||||
const files = {};
|
||||
for (const name of ['build/app.js', 'build/token.js', 'build/bootstrap/upgradeProbe.js', 'dist/index.html', 'build/app.js.map', 'dist/app.js.gz']) {
|
||||
for (const name of [
|
||||
'build/app.js',
|
||||
'build/token.js',
|
||||
'build/bootstrap/upgradeProbe.js',
|
||||
'dist/index.html',
|
||||
'build/app.js.map',
|
||||
'dist/app.js.gz',
|
||||
...[
|
||||
'databaseClient',
|
||||
'databaseErrors',
|
||||
'databaseProcess',
|
||||
'databaseWorker',
|
||||
'repository',
|
||||
'schema',
|
||||
'queries',
|
||||
'cacheStore',
|
||||
'statistics',
|
||||
].map((name) => `build/data/${name}.js`),
|
||||
]) {
|
||||
const bytes = name.endsWith('.html') ? '<html>pack fixture</html>' : '{}';
|
||||
await fs.mkdir(path.dirname(path.join(root, 'static', name)), { recursive: true });
|
||||
await fs.mkdir(path.dirname(path.join(root, 'static', name)), {
|
||||
recursive: true,
|
||||
});
|
||||
await fs.writeFile(path.join(root, 'static', name), bytes);
|
||||
files[name] = createHash('sha256').update(bytes).digest('hex');
|
||||
}
|
||||
for (const name of ['.env', '.env.local', '.env.production', '.aws/credentials', '.ssh/id_rsa', '.vscode/settings.json', '.github/workflows/private.yml', '.tmp/private.json', '.DS_Store', 'output/private.json', 'data/config/auth.json', 'back/private.json', 'back/private.ts', 'sample/tool.ts', 'sample/private.json', 'sample/__pycache__/notify.pyc', 'shell/preload/env.js', 'shell/preload/env.py', 'shell/preload/notify.js', 'shell/preload/__ql_notify__.js', 'shell/preload/lang_env.sh', 'shell/preload/task-notify.json', 'shell/preload/__pycache__/env.pyc', 'cli/dist/npm/ql.js', 'cli/src/internal/private.cjs']) {
|
||||
for (const name of [
|
||||
'.env',
|
||||
'.env.local',
|
||||
'.env.production',
|
||||
'.aws/credentials',
|
||||
'.ssh/id_rsa',
|
||||
'.vscode/settings.json',
|
||||
'.github/workflows/private.yml',
|
||||
'.tmp/private.json',
|
||||
'.DS_Store',
|
||||
'output/private.json',
|
||||
'data/config/auth.json',
|
||||
'back/private.json',
|
||||
'back/private.ts',
|
||||
'sample/tool.ts',
|
||||
'sample/private.json',
|
||||
'sample/__pycache__/notify.pyc',
|
||||
'shell/preload/env.js',
|
||||
'shell/preload/env.py',
|
||||
'shell/preload/notify.js',
|
||||
'shell/preload/__ql_notify__.js',
|
||||
'shell/preload/lang_env.sh',
|
||||
'shell/preload/task-notify.json',
|
||||
'shell/preload/__pycache__/env.pyc',
|
||||
'cli/dist/npm/ql.js',
|
||||
'cli/src/internal/private.cjs',
|
||||
]) {
|
||||
await fs.mkdir(path.dirname(path.join(root, name)), { recursive: true });
|
||||
await fs.writeFile(path.join(root, name), 'DO_NOT_PUBLISH');
|
||||
}
|
||||
@@ -51,58 +115,152 @@ async function packageFixture(t) {
|
||||
const env = { ...process.env, HOME: temp, NODE_ENV: 'production' };
|
||||
await execute('git', ['init', '-q'], { cwd: root, env });
|
||||
await execute('git', ['add', '-A'], { cwd: root, env });
|
||||
await execute('git', ['-c', 'user.name=Package Test', '-c', 'user.email=test@example.invalid', '-c', 'core.hooksPath=/dev/null', 'commit', '-qm', 'fixture'], { cwd: root, env });
|
||||
await fs.writeFile(path.join(root, 'static/build-info.json'), JSON.stringify({
|
||||
version: 1, dirty: false, sourceCommit: (await execute('git', ['rev-parse', 'HEAD'], { cwd: root })).stdout.trim(), files,
|
||||
lockfileSha256: createHash('sha256').update(lock).digest('hex'),
|
||||
}));
|
||||
const pack = () => execute('npm', ['pack', '--json', '--cache', path.join(temp, 'cache'), '--pack-destination', output], { cwd: root, env, timeout: 60000, maxBuffer: 4 * 1024 * 1024 });
|
||||
await execute(
|
||||
'git',
|
||||
[
|
||||
'-c',
|
||||
'user.name=Package Test',
|
||||
'-c',
|
||||
'user.email=test@example.invalid',
|
||||
'-c',
|
||||
'core.hooksPath=/dev/null',
|
||||
'commit',
|
||||
'-qm',
|
||||
'fixture',
|
||||
],
|
||||
{ cwd: root, env },
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(root, 'static/build-info.json'),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
dirty: false,
|
||||
sourceCommit: (
|
||||
await execute('git', ['rev-parse', 'HEAD'], { cwd: root })
|
||||
).stdout.trim(),
|
||||
files,
|
||||
lockfileSha256: createHash('sha256').update(lock).digest('hex'),
|
||||
}),
|
||||
);
|
||||
const pack = () =>
|
||||
execute(
|
||||
'npm',
|
||||
[
|
||||
'pack',
|
||||
'--json',
|
||||
'--cache',
|
||||
path.join(temp, 'cache'),
|
||||
'--pack-destination',
|
||||
output,
|
||||
],
|
||||
{ cwd: root, env, timeout: 60000, maxBuffer: 4 * 1024 * 1024 },
|
||||
);
|
||||
return { root, output, temp, env, lock, files, pack };
|
||||
}
|
||||
|
||||
test('real npm pack excludes local state and its installed lifecycle restores verified runtime files', async (t) => {
|
||||
const { root, output, temp, env, lock, files, pack } = await packageFixture(t);
|
||||
const { root, output, temp, env, lock, files, pack } = await packageFixture(
|
||||
t,
|
||||
);
|
||||
// The whitelist must work by itself, even for artifacts ignored by Git.
|
||||
await assert.rejects(fs.access(path.join(root, '.npmignore')), { code: 'ENOENT' });
|
||||
await assert.rejects(fs.access(path.join(root, '.npmignore')), {
|
||||
code: 'ENOENT',
|
||||
});
|
||||
const packed = await pack();
|
||||
const [archive] = JSON.parse(packed.stdout);
|
||||
checkPackedFiles(archive.files);
|
||||
const names = new Set(archive.files.map((file) => file.path));
|
||||
assert.ok(names.has('shell/upgrade-lock.yaml'));
|
||||
for (const name of Object.keys(files)) assert.ok(names.has('static/' + name));
|
||||
await assert.rejects(fs.access(path.join(root, 'shell/upgrade-lock.yaml')), { code: 'ENOENT' });
|
||||
await assert.rejects(fs.access(path.join(root, 'shell/upgrade-lock.yaml')), {
|
||||
code: 'ENOENT',
|
||||
});
|
||||
const prefix = path.join(temp, 'installed');
|
||||
await execute('npm', ['install', '--offline', '--omit=dev', '--no-audit', '--no-fund', '--cache', path.join(temp, 'cache'), '--prefix', prefix, path.join(output, archive.filename)], { cwd: temp, env, timeout: 60000 });
|
||||
await execute(
|
||||
'npm',
|
||||
[
|
||||
'install',
|
||||
'--offline',
|
||||
'--omit=dev',
|
||||
'--no-audit',
|
||||
'--no-fund',
|
||||
'--cache',
|
||||
path.join(temp, 'cache'),
|
||||
'--prefix',
|
||||
prefix,
|
||||
path.join(output, archive.filename),
|
||||
],
|
||||
{ cwd: temp, env, timeout: 60000 },
|
||||
);
|
||||
const installed = path.join(prefix, 'node_modules/@whyour/qinglong');
|
||||
for (const { path: name } of archive.files)
|
||||
assert.equal((await fs.readFile(path.join(installed, name))).includes(Buffer.from('DO_NOT_PUBLISH')), false, `Private fixture content in npm package: ${name}`);
|
||||
await assert.rejects(fs.access(path.join(installed, '.env')), { code: 'ENOENT' });
|
||||
assert.equal(await fs.readFile(path.join(installed, 'pnpm-lock.yaml'), 'utf8'), lock);
|
||||
assert.equal(typeof require(path.join(installed, 'shell/upgrade.cjs')).stageUpgrade, 'function');
|
||||
assert.ok((await fs.stat(path.join(installed, 'shell/update.sh'))).mode & 0o111);
|
||||
for (const [name, file] of Object.entries(JSON.parse(await fs.readFile(path.join(installed, 'package.json'), 'utf8')).bin)) {
|
||||
assert.equal(
|
||||
(await fs.readFile(path.join(installed, name))).includes(
|
||||
Buffer.from('DO_NOT_PUBLISH'),
|
||||
),
|
||||
false,
|
||||
`Private fixture content in npm package: ${name}`,
|
||||
);
|
||||
await assert.rejects(fs.access(path.join(installed, '.env')), {
|
||||
code: 'ENOENT',
|
||||
});
|
||||
assert.equal(
|
||||
await fs.readFile(path.join(installed, 'pnpm-lock.yaml'), 'utf8'),
|
||||
lock,
|
||||
);
|
||||
assert.equal(
|
||||
typeof require(path.join(installed, 'shell/upgrade.cjs')).stageUpgrade,
|
||||
'function',
|
||||
);
|
||||
assert.ok(
|
||||
(await fs.stat(path.join(installed, 'shell/update.sh'))).mode & 0o111,
|
||||
);
|
||||
for (const [name, file] of Object.entries(
|
||||
JSON.parse(await fs.readFile(path.join(installed, 'package.json'), 'utf8'))
|
||||
.bin,
|
||||
)) {
|
||||
await fs.access(path.join(prefix, 'node_modules/.bin', name));
|
||||
assert.ok((await fs.stat(path.join(installed, file))).mode & 0o111, file);
|
||||
}
|
||||
await fs.writeFile(path.join(installed, 'shell/upgrade-lock.yaml'), 'tampered');
|
||||
await fs.writeFile(
|
||||
path.join(installed, 'shell/upgrade-lock.yaml'),
|
||||
'tampered',
|
||||
);
|
||||
assert.throws(() => restorePackageLock(installed), /does not match/);
|
||||
assert.equal(await fs.readFile(path.join(installed, 'pnpm-lock.yaml'), 'utf8'), lock);
|
||||
assert.ok(!(await fs.readdir(installed)).some((name) => name.startsWith('.upgrade-lock-')));
|
||||
assert.equal(
|
||||
await fs.readFile(path.join(installed, 'pnpm-lock.yaml'), 'utf8'),
|
||||
lock,
|
||||
);
|
||||
assert.ok(
|
||||
!(await fs.readdir(installed)).some((name) =>
|
||||
name.startsWith('.upgrade-lock-'),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('postinstall still supports minimal Docker dependency stages before shell files are copied', async (t) => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-builder-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const metadata = JSON.parse(await fs.readFile('package.json', 'utf8'));
|
||||
await fs.writeFile(path.join(temp, 'package.json'), JSON.stringify({ scripts: { postinstall: metadata.scripts.postinstall } }));
|
||||
await fs.writeFile(
|
||||
path.join(temp, 'package.json'),
|
||||
JSON.stringify({ scripts: { postinstall: metadata.scripts.postinstall } }),
|
||||
);
|
||||
await fs.writeFile(path.join(temp, 'pnpm-lock.yaml'), 'source lock');
|
||||
await execute('npm', ['run', 'postinstall', '--cache', path.join(temp, 'cache')], { cwd: temp, timeout: 30000 });
|
||||
await execute(
|
||||
'npm',
|
||||
['run', 'postinstall', '--cache', path.join(temp, 'cache')],
|
||||
{ cwd: temp, timeout: 30000 },
|
||||
);
|
||||
});
|
||||
|
||||
test('a missing npm runtime lockfile fails instead of claiming an installed package is ready', async (t) => {
|
||||
const temp = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-lock-missing-'));
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
assert.throws(() => restorePackageLock(temp), /Missing npm package upgrade lockfile/);
|
||||
assert.throws(
|
||||
() => restorePackageLock(temp),
|
||||
/Missing npm package upgrade lockfile/,
|
||||
);
|
||||
});
|
||||
|
||||
test('real npm overwrites preserve .env automatically and failed installs restore the old package', async (t) => {
|
||||
@@ -111,62 +269,165 @@ test('real npm overwrites preserve .env automatically and failed installs restor
|
||||
const archive = path.join(output, packed.filename);
|
||||
const legacy = path.join(temp, 'legacy');
|
||||
await fs.mkdir(legacy);
|
||||
await fs.writeFile(path.join(legacy, 'package.json'), JSON.stringify({ name: '@whyour/qinglong', version: '2.22.0' }));
|
||||
await fs.writeFile(
|
||||
path.join(legacy, 'package.json'),
|
||||
JSON.stringify({ name: '@whyour/qinglong', version: '2.22.0' }),
|
||||
);
|
||||
await fs.writeFile(path.join(legacy, 'old-package-marker'), 'legacy');
|
||||
const [old] = JSON.parse((await execute('npm', ['pack', '--ignore-scripts', '--json', '--cache', path.join(temp, 'cache')], { cwd: legacy, env })).stdout);
|
||||
const [old] = JSON.parse(
|
||||
(
|
||||
await execute(
|
||||
'npm',
|
||||
[
|
||||
'pack',
|
||||
'--ignore-scripts',
|
||||
'--json',
|
||||
'--cache',
|
||||
path.join(temp, 'cache'),
|
||||
],
|
||||
{ cwd: legacy, env },
|
||||
)
|
||||
).stdout,
|
||||
);
|
||||
const broken = path.join(temp, 'broken');
|
||||
await fs.mkdir(broken);
|
||||
await fs.cp(root, broken, { recursive: true });
|
||||
await fs.writeFile(path.join(broken, 'shell/upgrade-lock.yaml'), 'invalid lock');
|
||||
const [bad] = JSON.parse((await execute('npm', ['pack', '--ignore-scripts', '--json', '--cache', path.join(temp, 'cache')], { cwd: broken, env })).stdout);
|
||||
await fs.writeFile(
|
||||
path.join(broken, 'shell/upgrade-lock.yaml'),
|
||||
'invalid lock',
|
||||
);
|
||||
const [bad] = JSON.parse(
|
||||
(
|
||||
await execute(
|
||||
'npm',
|
||||
[
|
||||
'pack',
|
||||
'--ignore-scripts',
|
||||
'--json',
|
||||
'--cache',
|
||||
path.join(temp, 'cache'),
|
||||
],
|
||||
{ cwd: broken, env },
|
||||
)
|
||||
).stdout,
|
||||
);
|
||||
|
||||
for (const global of [false, true]) {
|
||||
await t.test(global ? 'global npm install' : 'local npm install', async () => {
|
||||
const prefix = path.join(temp, global ? 'global' : 'local');
|
||||
const linkedPrefix = prefix + '-link';
|
||||
await fs.mkdir(prefix);
|
||||
await fs.symlink(prefix, linkedPrefix);
|
||||
const installed = path.join(prefix, global ? 'lib/node_modules/@whyour/qinglong' : 'node_modules/@whyour/qinglong');
|
||||
const install = (file) => execute('npm', ['install', ...(global ? ['--global'] : []), '--prefix', linkedPrefix, '--force', '--offline', '--omit=dev', '--no-audit', '--no-fund', '--cache', path.join(temp, 'cache'), file], { cwd: temp, env, timeout: 60000 });
|
||||
await install(path.join(legacy, old.filename));
|
||||
const bytes = Buffer.from('QL_DATA_DIR=/external/data\r\nSECRET="配置 test"\r\n# no trailing newline');
|
||||
const external = path.join(prefix, 'external.env');
|
||||
await fs.writeFile(external, bytes);
|
||||
await fs.symlink(external, path.join(installed, '.env'));
|
||||
await assert.rejects(install(archive), (error) => {
|
||||
assert.match(error.stderr, /must be a regular file/);
|
||||
return true;
|
||||
});
|
||||
assert.equal(JSON.parse(await fs.readFile(path.join(installed, 'package.json'))).version, '2.22.0');
|
||||
assert.equal(await fs.readlink(path.join(installed, '.env')), external);
|
||||
assert.deepEqual(await fs.readFile(external), bytes);
|
||||
await fs.rm(path.join(installed, '.env'));
|
||||
await fs.writeFile(path.join(installed, '.env'), bytes, { mode: 0o644 });
|
||||
await assert.rejects(install(path.join(broken, bad.filename)), (error) => {
|
||||
assert.match(error.stderr, /does not match its build manifest/);
|
||||
return true;
|
||||
});
|
||||
assert.equal(JSON.parse(await fs.readFile(path.join(installed, 'package.json'))).version, '2.22.0');
|
||||
assert.equal(await fs.readFile(path.join(installed, 'old-package-marker'), 'utf8'), 'legacy');
|
||||
assert.deepEqual(await fs.readFile(path.join(installed, '.env')), bytes);
|
||||
await install(archive);
|
||||
assert.deepEqual(await fs.readFile(path.join(installed, '.env')), bytes);
|
||||
assert.equal((await fs.stat(path.join(installed, '.env'))).mode & 0o777, 0o600);
|
||||
await assert.rejects(fs.access(path.join(installed, 'old-package-marker')), { code: 'ENOENT' });
|
||||
// Same-version forced reinstall must also retain custom configuration.
|
||||
await install(archive);
|
||||
assert.deepEqual(await fs.readFile(path.join(installed, '.env')), bytes);
|
||||
assert.ok(!(await fs.readdir(installed)).some((name) => name.startsWith('.upgrade-env-')));
|
||||
});
|
||||
await t.test(
|
||||
global ? 'global npm install' : 'local npm install',
|
||||
async () => {
|
||||
const prefix = path.join(temp, global ? 'global' : 'local');
|
||||
const linkedPrefix = prefix + '-link';
|
||||
await fs.mkdir(prefix);
|
||||
await fs.symlink(prefix, linkedPrefix);
|
||||
const installed = path.join(
|
||||
prefix,
|
||||
global
|
||||
? 'lib/node_modules/@whyour/qinglong'
|
||||
: 'node_modules/@whyour/qinglong',
|
||||
);
|
||||
const install = (file) =>
|
||||
execute(
|
||||
'npm',
|
||||
[
|
||||
'install',
|
||||
...(global ? ['--global'] : []),
|
||||
'--prefix',
|
||||
linkedPrefix,
|
||||
'--force',
|
||||
'--offline',
|
||||
'--omit=dev',
|
||||
'--no-audit',
|
||||
'--no-fund',
|
||||
'--cache',
|
||||
path.join(temp, 'cache'),
|
||||
file,
|
||||
],
|
||||
{ cwd: temp, env, timeout: 60000 },
|
||||
);
|
||||
await install(path.join(legacy, old.filename));
|
||||
const bytes = Buffer.from(
|
||||
'QL_DATA_DIR=/external/data\r\nSECRET="配置 test"\r\n# no trailing newline',
|
||||
);
|
||||
const external = path.join(prefix, 'external.env');
|
||||
await fs.writeFile(external, bytes);
|
||||
await fs.symlink(external, path.join(installed, '.env'));
|
||||
await assert.rejects(install(archive), (error) => {
|
||||
assert.match(error.stderr, /must be a regular file/);
|
||||
return true;
|
||||
});
|
||||
assert.equal(
|
||||
JSON.parse(await fs.readFile(path.join(installed, 'package.json')))
|
||||
.version,
|
||||
'2.22.0',
|
||||
);
|
||||
assert.equal(await fs.readlink(path.join(installed, '.env')), external);
|
||||
assert.deepEqual(await fs.readFile(external), bytes);
|
||||
await fs.rm(path.join(installed, '.env'));
|
||||
await fs.writeFile(path.join(installed, '.env'), bytes, {
|
||||
mode: 0o644,
|
||||
});
|
||||
await assert.rejects(
|
||||
install(path.join(broken, bad.filename)),
|
||||
(error) => {
|
||||
assert.match(error.stderr, /does not match its build manifest/);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
JSON.parse(await fs.readFile(path.join(installed, 'package.json')))
|
||||
.version,
|
||||
'2.22.0',
|
||||
);
|
||||
assert.equal(
|
||||
await fs.readFile(path.join(installed, 'old-package-marker'), 'utf8'),
|
||||
'legacy',
|
||||
);
|
||||
assert.deepEqual(
|
||||
await fs.readFile(path.join(installed, '.env')),
|
||||
bytes,
|
||||
);
|
||||
await install(archive);
|
||||
assert.deepEqual(
|
||||
await fs.readFile(path.join(installed, '.env')),
|
||||
bytes,
|
||||
);
|
||||
assert.equal(
|
||||
(await fs.stat(path.join(installed, '.env'))).mode & 0o777,
|
||||
0o600,
|
||||
);
|
||||
await assert.rejects(
|
||||
fs.access(path.join(installed, 'old-package-marker')),
|
||||
{ code: 'ENOENT' },
|
||||
);
|
||||
// Same-version forced reinstall must also retain custom configuration.
|
||||
await install(archive);
|
||||
assert.deepEqual(
|
||||
await fs.readFile(path.join(installed, '.env')),
|
||||
bytes,
|
||||
);
|
||||
assert.ok(
|
||||
!(await fs.readdir(installed)).some((name) =>
|
||||
name.startsWith('.upgrade-env-'),
|
||||
),
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('environment recovery uses only the exact retired package and never overwrites existing configuration', async (t) => {
|
||||
const temp = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-env-')));
|
||||
const temp = await fs.realpath(
|
||||
await fs.mkdtemp(path.join(os.tmpdir(), 'ql-npm-env-')),
|
||||
);
|
||||
t.after(() => fs.rm(temp, { recursive: true, force: true }));
|
||||
const root = path.join(temp, 'node_modules/@whyour/qinglong');
|
||||
await fs.mkdir(root, { recursive: true });
|
||||
const suffix = createHash('sha1').update(root).digest('base64').replace(/[^a-zA-Z0-9]+/g, '').slice(0, 8);
|
||||
const suffix = createHash('sha1')
|
||||
.update(root)
|
||||
.digest('base64')
|
||||
.replace(/[^a-zA-Z0-9]+/g, '')
|
||||
.slice(0, 8);
|
||||
const retired = path.join(path.dirname(root), `.qinglong-${suffix}`);
|
||||
const destination = path.join(root, '.env');
|
||||
const other = path.join(path.dirname(root), '.qinglong-unrelated');
|
||||
@@ -175,10 +436,19 @@ test('environment recovery uses only the exact retired package and never overwri
|
||||
restoreEnvironment(root);
|
||||
await assert.rejects(fs.access(destination), { code: 'ENOENT' });
|
||||
await fs.mkdir(retired);
|
||||
await fs.writeFile(path.join(retired, 'package.json'), JSON.stringify({ name: 'another-package' }));
|
||||
await fs.writeFile(
|
||||
path.join(retired, 'package.json'),
|
||||
JSON.stringify({ name: 'another-package' }),
|
||||
);
|
||||
await fs.writeFile(path.join(retired, '.env'), 'private');
|
||||
assert.throws(() => restoreEnvironment(root), /Unexpected retired npm package/);
|
||||
await fs.writeFile(path.join(retired, 'package.json'), JSON.stringify({ name: '@whyour/qinglong' }));
|
||||
assert.throws(
|
||||
() => restoreEnvironment(root),
|
||||
/Unexpected retired npm package/,
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(retired, 'package.json'),
|
||||
JSON.stringify({ name: '@whyour/qinglong' }),
|
||||
);
|
||||
await fs.rm(path.join(retired, '.env'));
|
||||
restoreEnvironment(root);
|
||||
await assert.rejects(fs.access(destination), { code: 'ENOENT' });
|
||||
@@ -195,62 +465,202 @@ test('environment recovery uses only the exact retired package and never overwri
|
||||
await fs.writeFile(destination, 'new custom configuration');
|
||||
await fs.writeFile(path.join(retired, '.env'), 'old configuration');
|
||||
restoreEnvironment(root);
|
||||
assert.equal(await fs.readFile(destination, 'utf8'), 'new custom configuration');
|
||||
assert.ok(!(await fs.readdir(root)).some((name) => name.startsWith('.upgrade-env-')));
|
||||
assert.equal(
|
||||
await fs.readFile(destination, 'utf8'),
|
||||
'new custom configuration',
|
||||
);
|
||||
assert.ok(
|
||||
!(await fs.readdir(root)).some((name) => name.startsWith('.upgrade-env-')),
|
||||
);
|
||||
});
|
||||
|
||||
test('npm publication consumes the exact archive whose isolated installation passed', () => {
|
||||
const yaml = require('js-yaml');
|
||||
const sync = require('node:fs');
|
||||
const workflow = yaml.load(sync.readFileSync('.github/workflows/build-docker-image.yml', 'utf8'));
|
||||
const workflow = yaml.load(
|
||||
sync.readFileSync('.github/workflows/build-docker-image.yml', 'utf8'),
|
||||
);
|
||||
const verify = workflow.jobs['verify-npm-package'];
|
||||
assert.deepEqual(verify.needs, ['build-static', 'build-npm-migration-bundle']);
|
||||
assert.ok(verify.steps.some((step) => step.with?.name === 'npm-migration-bundle-${{ github.sha }}'));
|
||||
assert.deepEqual(verify.needs, [
|
||||
'build-static',
|
||||
'build-npm-migration-bundle',
|
||||
'build-alpine',
|
||||
]);
|
||||
assert.ok(
|
||||
verify.steps.some(
|
||||
(step) =>
|
||||
step.with?.name ===
|
||||
"npm-migration-bundle-${{ matrix.node == 'alpine' && 'musl' || 'glibc' }}-${{ github.sha }}",
|
||||
),
|
||||
);
|
||||
const migration = workflow.jobs['build-npm-migration-bundle'];
|
||||
assert.equal(migration.needs, 'build-debian');
|
||||
assert.ok(migration.steps.some((step) => /Dockerfile.upgrade-bundle/.test(step.run || '')));
|
||||
assert.ok(!migration.steps.some((step) => /publish-upgrade-bundles/.test(step.run || '')));
|
||||
assert.deepEqual(verify.strategy.matrix.node, ['20.18.1', '24']);
|
||||
assert.match(verify.container, /node:.*bookworm/);
|
||||
const tools = verify.steps.find((step) => step.name === 'Install isolated service test tools');
|
||||
assert.match(tools.run, /npm install --global --prefix \/usr\/local.*pm2@6\.0\.14/);
|
||||
const service = verify.steps.find((step) => step.name === 'Install and upgrade the exact npm archive');
|
||||
assert.deepEqual(migration.needs, ['build-debian', 'build-alpine']);
|
||||
assert.deepEqual(migration.strategy.matrix.libc, ['glibc', 'musl']);
|
||||
assert.ok(
|
||||
migration.steps.some((step) =>
|
||||
/Dockerfile.upgrade-bundle/.test(step.run || ''),
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
!migration.steps.some((step) =>
|
||||
/publish-upgrade-bundles/.test(step.run || ''),
|
||||
),
|
||||
);
|
||||
assert.deepEqual(verify.strategy.matrix.node, [
|
||||
'20.18.1',
|
||||
'22.17.0',
|
||||
'24',
|
||||
'alpine',
|
||||
]);
|
||||
assert.match(
|
||||
verify.container.image,
|
||||
/needs\.build-alpine\.outputs\.image-digest/,
|
||||
);
|
||||
assert.match(verify.container.image, /node:.*bookworm/);
|
||||
const tools = verify.steps.find(
|
||||
(step) => step.name === 'Install isolated service test tools',
|
||||
);
|
||||
assert.match(
|
||||
tools.run,
|
||||
/npm install --global --prefix \/usr\/local.*pm2@6\.0\.14/,
|
||||
);
|
||||
const service = verify.steps.find(
|
||||
(step) => step.name === 'Install and upgrade the exact npm archive',
|
||||
);
|
||||
assert.equal(service.if, "matrix.node != '20.18.1'");
|
||||
assert.ok(
|
||||
verify.steps.some(
|
||||
(step) =>
|
||||
step.if === "matrix.node == '20.18.1'" &&
|
||||
/verify-panel-runtime-floor/.test(step.run || ''),
|
||||
),
|
||||
);
|
||||
assert.equal(service.env.QL_PANEL_TEST_PM2, '/usr/local/bin/pm2');
|
||||
assert.ok(workflow.jobs.publish.needs.includes('verify-npm-package'));
|
||||
const download = workflow.jobs.publish.steps.find((step) => step.uses?.startsWith('actions/download-artifact@'));
|
||||
const download = workflow.jobs.publish.steps.find((step) =>
|
||||
step.uses?.startsWith('actions/download-artifact@'),
|
||||
);
|
||||
assert.equal(download.with.name, 'qinglong-npm-${{ github.sha }}');
|
||||
const publication = workflow.jobs.publish.steps.find((step) => step.name === 'Publish npm package with OIDC');
|
||||
const publication = workflow.jobs.publish.steps.find(
|
||||
(step) => step.name === 'Publish npm package with OIDC',
|
||||
);
|
||||
assert.match(publication.run, /npm publish "\$archive" --ignore-scripts/);
|
||||
assert.ok(!workflow.jobs.publish.steps.some((step) => /pnpm build/.test(step.run || '')));
|
||||
assert.ok(
|
||||
!workflow.jobs.publish.steps.some((step) =>
|
||||
/pnpm build/.test(step.run || ''),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
for (const scenario of ['dirty-source', 'stale-commit', 'dirty-build', 'changed-lock', 'changed-output', 'missing-output', 'extra-output']) {
|
||||
test('an actual npm lifecycle failure on Node 20 preserves local and global old installs', async (t) => {
|
||||
const node20 = process.env.QL_TEST_NODE20;
|
||||
if (!node20)
|
||||
return t.skip(
|
||||
'Set QL_TEST_NODE20 to exercise actual rejected npm installs',
|
||||
);
|
||||
const { output, env, pack } = await packageFixture(t);
|
||||
await pack();
|
||||
const result = await execute(
|
||||
node20,
|
||||
[path.resolve('scripts/verify-panel-runtime-floor.cjs'), output],
|
||||
{
|
||||
env: {
|
||||
...env,
|
||||
PATH: path.dirname(node20) + path.delimiter + process.env.PATH,
|
||||
},
|
||||
timeout: 60000,
|
||||
},
|
||||
);
|
||||
assert.equal(JSON.parse(result.stdout).oldServiceStayedHealthy, true);
|
||||
});
|
||||
|
||||
for (const scenario of [
|
||||
'dirty-source',
|
||||
'stale-commit',
|
||||
'dirty-build',
|
||||
'changed-lock',
|
||||
'changed-output',
|
||||
'missing-output',
|
||||
'extra-output',
|
||||
]) {
|
||||
test(`native npm pack rejects ${scenario} before producing an archive`, async (t) => {
|
||||
const { root, output, pack } = await packageFixture(t);
|
||||
const manifestFile = path.join(root, 'static/build-info.json');
|
||||
const manifest = JSON.parse(await fs.readFile(manifestFile, 'utf8'));
|
||||
if (scenario === 'dirty-source') await fs.appendFile(path.join(root, 'shell/start.sh'), '\n# uncommitted change\n');
|
||||
if (scenario === 'dirty-source')
|
||||
await fs.appendFile(
|
||||
path.join(root, 'shell/start.sh'),
|
||||
'\n# uncommitted change\n',
|
||||
);
|
||||
if (scenario === 'stale-commit') {
|
||||
await fs.appendFile(path.join(root, 'shell/start.sh'), '\n# committed change after build\n');
|
||||
await fs.appendFile(
|
||||
path.join(root, 'shell/start.sh'),
|
||||
'\n# committed change after build\n',
|
||||
);
|
||||
await execute('git', ['add', '-A'], { cwd: root });
|
||||
await execute('git', ['-c', 'user.name=Package Test', '-c', 'user.email=test@example.invalid', '-c', 'core.hooksPath=/dev/null', 'commit', '-qm', 'new source'], { cwd: root });
|
||||
await execute(
|
||||
'git',
|
||||
[
|
||||
'-c',
|
||||
'user.name=Package Test',
|
||||
'-c',
|
||||
'user.email=test@example.invalid',
|
||||
'-c',
|
||||
'core.hooksPath=/dev/null',
|
||||
'commit',
|
||||
'-qm',
|
||||
'new source',
|
||||
],
|
||||
{ cwd: root },
|
||||
);
|
||||
}
|
||||
if (scenario === 'dirty-build') { manifest.dirty = true; await fs.writeFile(manifestFile, JSON.stringify(manifest)); }
|
||||
if (scenario === 'changed-lock') {
|
||||
await fs.appendFile(path.join(root, 'pnpm-lock.yaml'), '# changed lock\n');
|
||||
await execute('git', ['add', '-A'], { cwd: root });
|
||||
await execute('git', ['-c', 'user.name=Package Test', '-c', 'user.email=test@example.invalid', '-c', 'core.hooksPath=/dev/null', 'commit', '-qm', 'new lock'], { cwd: root });
|
||||
manifest.sourceCommit = (await execute('git', ['rev-parse', 'HEAD'], { cwd: root })).stdout.trim();
|
||||
if (scenario === 'dirty-build') {
|
||||
manifest.dirty = true;
|
||||
await fs.writeFile(manifestFile, JSON.stringify(manifest));
|
||||
}
|
||||
if (scenario === 'changed-output') await fs.appendFile(path.join(root, 'static/build/app.js'), 'changed');
|
||||
if (scenario === 'missing-output') await fs.rm(path.join(root, 'static/build/app.js'));
|
||||
if (scenario === 'extra-output') await fs.writeFile(path.join(root, 'static/build/leftover.js'), 'old');
|
||||
if (scenario === 'changed-lock') {
|
||||
await fs.appendFile(
|
||||
path.join(root, 'pnpm-lock.yaml'),
|
||||
'# changed lock\n',
|
||||
);
|
||||
await execute('git', ['add', '-A'], { cwd: root });
|
||||
await execute(
|
||||
'git',
|
||||
[
|
||||
'-c',
|
||||
'user.name=Package Test',
|
||||
'-c',
|
||||
'user.email=test@example.invalid',
|
||||
'-c',
|
||||
'core.hooksPath=/dev/null',
|
||||
'commit',
|
||||
'-qm',
|
||||
'new lock',
|
||||
],
|
||||
{ cwd: root },
|
||||
);
|
||||
manifest.sourceCommit = (
|
||||
await execute('git', ['rev-parse', 'HEAD'], { cwd: root })
|
||||
).stdout.trim();
|
||||
await fs.writeFile(manifestFile, JSON.stringify(manifest));
|
||||
}
|
||||
if (scenario === 'changed-output')
|
||||
await fs.appendFile(path.join(root, 'static/build/app.js'), 'changed');
|
||||
if (scenario === 'missing-output')
|
||||
await fs.rm(path.join(root, 'static/build/app.js'));
|
||||
if (scenario === 'extra-output')
|
||||
await fs.writeFile(path.join(root, 'static/build/leftover.js'), 'old');
|
||||
await assert.rejects(pack(), (error) => {
|
||||
assert.match(error.stderr, /clean source|do not match|checksum mismatch|file set mismatch/i);
|
||||
assert.match(
|
||||
error.stderr,
|
||||
/clean source|do not match|checksum mismatch|file set mismatch/i,
|
||||
);
|
||||
return true;
|
||||
});
|
||||
assert.deepEqual(await fs.readdir(output), []);
|
||||
await assert.rejects(fs.access(path.join(root, 'shell/upgrade-lock.yaml')), { code: 'ENOENT' });
|
||||
await assert.rejects(
|
||||
fs.access(path.join(root, 'shell/upgrade-lock.yaml')),
|
||||
{ code: 'ENOENT' },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -7,6 +7,295 @@ const tar = require('tar');
|
||||
const bundle = require('../../cli/src/internal/maintenance/platformBundle.cjs');
|
||||
const artifacts = require('../../shell/upgrade.cjs');
|
||||
|
||||
for (const fault of ['writeBuffer', 'writeBuffers'])
|
||||
test(`archive creation completes a native partial ${fault} result`, async (t) => {
|
||||
const directory = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), 'ql-archive-create-'),
|
||||
);
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
await fs.mkdir(path.join(directory, 'qinglong'));
|
||||
const bytes = require('node:crypto').randomBytes(512 * 1024 + 137);
|
||||
await fs.writeFile(path.join(directory, 'qinglong/probe'), bytes);
|
||||
const archive = path.join(directory, 'archive.tar.gz'),
|
||||
destination = path.join(directory, 'extracted');
|
||||
await fs.mkdir(destination);
|
||||
const execute = require('node:util').promisify(
|
||||
require('node:child_process').execFile,
|
||||
);
|
||||
const result = await execute(
|
||||
process.execPath,
|
||||
[
|
||||
'-e',
|
||||
`
|
||||
const fs=require('node:fs/promises'),path=require('node:path');
|
||||
const {writeBundleArchive}=require(process.argv[1]),bundle=require(process.argv[2]);
|
||||
const binding=process.binding('fs'),fault=process.argv[6],original=binding[fault];
|
||||
let injected=0;
|
||||
binding[fault]=function(...args) {
|
||||
if(!injected) {
|
||||
if(fault==='writeBuffer' && args[3]>1) {args[3]=Math.floor(args[3]/2);injected++;}
|
||||
else if(fault==='writeBuffers') {
|
||||
let remaining=Math.floor(args[1].reduce((n,b)=>n+b.byteLength,0)/2);
|
||||
if(remaining) {
|
||||
const short=[];for(const b of args[1]) {const part=b.subarray(0,remaining);if(part.length)short.push(part);remaining-=part.length;if(!remaining)break;}
|
||||
args[1]=short;injected++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return original.apply(this,args);
|
||||
};
|
||||
(async()=>{
|
||||
await writeBundleArchive(process.argv[3],process.argv[4]);
|
||||
binding[fault]=original;
|
||||
await bundle.extractArchive(process.argv[4],process.argv[5]);
|
||||
const bytes=await fs.readFile(path.join(process.argv[5],'qinglong/probe'));
|
||||
console.log(JSON.stringify({injected,length:bytes.length,sha256:require('node:crypto').createHash('sha256').update(bytes).digest('hex')}));
|
||||
})().catch(error=>{binding[fault]=original;console.error(error);process.exitCode=1});
|
||||
`,
|
||||
require.resolve('../../scripts/build-upgrade-bundle.cjs'),
|
||||
require.resolve(
|
||||
'../../cli/src/internal/maintenance/platformBundle.cjs',
|
||||
),
|
||||
directory,
|
||||
archive,
|
||||
destination,
|
||||
fault,
|
||||
],
|
||||
{ timeout: 15000 },
|
||||
);
|
||||
const observed = JSON.parse(result.stdout);
|
||||
assert.equal(observed.injected, 1);
|
||||
assert.equal(observed.length, bytes.length);
|
||||
assert.equal(observed.sha256, bundle.sha256(bytes));
|
||||
});
|
||||
|
||||
test('archive creation rejects an output filesystem error', async (t) => {
|
||||
const f = await fixture(t);
|
||||
await assert.rejects(
|
||||
require('../../scripts/build-upgrade-bundle.cjs').writeBundleArchive(
|
||||
f.directory,
|
||||
path.join(f.directory, 'qinglong'),
|
||||
),
|
||||
{ code: 'EISDIR' },
|
||||
);
|
||||
});
|
||||
|
||||
test('archive extraction completes a native partial write before publishing the file', async (t) => {
|
||||
const directory = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), 'ql-archive-short-write-'),
|
||||
);
|
||||
t.after(() => fs.rm(directory, { recursive: true, force: true }));
|
||||
await fs.mkdir(path.join(directory, 'qinglong'));
|
||||
const bytes = require('node:crypto').randomBytes(512 * 1024 + 137);
|
||||
await fs.writeFile(path.join(directory, 'qinglong/probe'), bytes);
|
||||
const archive = path.join(directory, 'archive.tar.gz');
|
||||
await tar.c({ file: archive, gzip: true, cwd: directory }, ['qinglong']);
|
||||
const destination = path.join(directory, 'extracted');
|
||||
await fs.mkdir(destination);
|
||||
const execute = require('node:util').promisify(
|
||||
require('node:child_process').execFile,
|
||||
);
|
||||
const result = await execute(
|
||||
process.execPath,
|
||||
[
|
||||
'-e',
|
||||
`
|
||||
const fs=require('node:fs/promises'),path=require('node:path');
|
||||
const bundle=require(process.argv[1]);
|
||||
// Isolate native I/O fault injection in this child. Both callback writers
|
||||
// and FileHandle.writeFile receive the real shortened write result.
|
||||
const binding=process.binding('fs'),write=binding.writeBuffer;
|
||||
let injected=0,calls=0;
|
||||
binding.writeBuffer=function(...args) {
|
||||
calls++;
|
||||
if(!injected && args[3]>1) {injected++;args[3]=Math.floor(args[3]/2);}
|
||||
return write.apply(this,args);
|
||||
};
|
||||
(async()=>{
|
||||
await bundle.extractArchive(process.argv[2],process.argv[3]);
|
||||
binding.writeBuffer=write;
|
||||
const bytes=await fs.readFile(path.join(process.argv[3],'qinglong/probe'));
|
||||
console.log(JSON.stringify({injected,calls,length:bytes.length,sha256:require('node:crypto').createHash('sha256').update(bytes).digest('hex')}));
|
||||
})().catch(error=>{binding.writeBuffer=write;console.error(error);process.exitCode=1});
|
||||
`,
|
||||
require.resolve('../../cli/src/internal/maintenance/platformBundle.cjs'),
|
||||
archive,
|
||||
destination,
|
||||
],
|
||||
{ timeout: 15000 },
|
||||
);
|
||||
const observed = JSON.parse(result.stdout);
|
||||
assert.equal(observed.injected, 1);
|
||||
assert.ok(observed.calls >= 2);
|
||||
assert.equal(observed.length, bytes.length);
|
||||
assert.equal(observed.sha256, bundle.sha256(bytes));
|
||||
});
|
||||
|
||||
test('archive extraction refuses an existing payload root, including a link to another directory', async (t) => {
|
||||
const f = await fixture(t);
|
||||
const outside = path.join(f.directory, 'outside'),
|
||||
destination = path.join(f.directory, 'existing-extract');
|
||||
await fs.mkdir(outside);
|
||||
await fs.writeFile(path.join(outside, 'keep'), 'unchanged');
|
||||
await fs.mkdir(destination);
|
||||
await fs.symlink(outside, path.join(destination, 'qinglong'));
|
||||
await assert.rejects(bundle.extractArchive(f.archive, destination), {
|
||||
code: 'EEXIST',
|
||||
});
|
||||
assert.deepEqual(await fs.readdir(outside), ['keep']);
|
||||
assert.equal(
|
||||
await fs.readFile(path.join(outside, 'keep'), 'utf8'),
|
||||
'unchanged',
|
||||
);
|
||||
});
|
||||
|
||||
for (const fault of ['write', 'parse', 'read'])
|
||||
test(`archive ${fault} failure closes pending writes before cleanup`, async (t) => {
|
||||
const f = await fixture(t);
|
||||
await fs.writeFile(
|
||||
path.join(f.source, 'probe-large'),
|
||||
require('node:crypto').randomBytes(512 * 1024 + 137),
|
||||
);
|
||||
f.manifest.files = await bundle.inventory({
|
||||
source: f.source,
|
||||
static: f.staticRoot,
|
||||
});
|
||||
await f.pack();
|
||||
const execute = require('node:util').promisify(
|
||||
require('node:child_process').execFile,
|
||||
);
|
||||
const destination = path.join(f.directory, 'failed-extract');
|
||||
await fs.mkdir(destination);
|
||||
const result = await execute(
|
||||
process.execPath,
|
||||
[
|
||||
'-e',
|
||||
`
|
||||
const fs=require('node:fs'),p=fs.promises,path=require('node:path'),{Readable}=require('node:stream');
|
||||
const fault=process.argv[4];
|
||||
if(fault!=='write') {
|
||||
const bytes=fs.readFileSync(process.argv[2]);
|
||||
fs.createReadStream=()=>fault==='parse'
|
||||
? Readable.from([bytes.subarray(0,Math.floor(bytes.length/2))])
|
||||
: new Readable({read(){this.destroy(Object.assign(new Error('read failed'),{code:'EIO'}))}});
|
||||
}
|
||||
const bundle=require(process.argv[1]);
|
||||
const open=p.open;
|
||||
let opened=0,closed=0,faultFd;
|
||||
p.open=async(...args)=>{
|
||||
const handle=await open(...args),close=handle.close.bind(handle);opened++;
|
||||
if(String(args[0]).endsWith('/probe-large'))faultFd=handle.fd;
|
||||
handle.close=async()=>{try{return await close()}finally{closed++}};
|
||||
return handle;
|
||||
};
|
||||
if(fault==='write') {
|
||||
const binding=process.binding('fs'),write=binding.writeBuffer;
|
||||
binding.writeBuffer=function(...args){
|
||||
if(args[0]===faultFd)return Promise.reject(Object.assign(new Error('disk full'),{code:'ENOSPC'}));
|
||||
return write.apply(this,args);
|
||||
};
|
||||
}
|
||||
(async()=>{
|
||||
let error;
|
||||
try {await bundle.extractArchive(process.argv[2],process.argv[3]);}catch(e){error=e;}
|
||||
if(!error)throw new Error('Fault was not rejected');
|
||||
if(opened!==closed)throw new Error('Open write handles remain: '+opened+'/'+closed);
|
||||
await p.rm(process.argv[3],{recursive:true});
|
||||
await new Promise(r=>setTimeout(r,100));
|
||||
if(fs.existsSync(process.argv[3]))throw new Error('Extraction continued after failure');
|
||||
console.log(JSON.stringify({rejected:true,opened,closed}));
|
||||
})().catch(error=>{console.error(error);process.exitCode=1});
|
||||
`,
|
||||
require.resolve(
|
||||
'../../cli/src/internal/maintenance/platformBundle.cjs',
|
||||
),
|
||||
f.archive,
|
||||
destination,
|
||||
fault,
|
||||
],
|
||||
{ timeout: 15000 },
|
||||
);
|
||||
assert.equal(JSON.parse(result.stdout).rejected, true);
|
||||
});
|
||||
|
||||
test('payload rejection identifies missing, extra, changed bytes and modes without logging contents', async (t) => {
|
||||
const f = await fixture(t);
|
||||
const changed = path.join(f.source, 'sample/config.sample.sh');
|
||||
await fs.writeFile(changed, 'private contents never printed');
|
||||
await fs.chmod(changed, 0o700);
|
||||
await fs.rm(path.join(f.source, 'version.yaml'));
|
||||
await fs.writeFile(path.join(f.source, 'unexpected'), 'extra contents');
|
||||
await assert.rejects(
|
||||
bundle.verifyPayload(
|
||||
{ source: f.source, static: f.staticRoot },
|
||||
f.manifest,
|
||||
f.target,
|
||||
),
|
||||
(error) => {
|
||||
assert.doesNotMatch(error.message, /private contents|extra contents/);
|
||||
const detail = JSON.parse(error.message.split('match: ')[1]);
|
||||
assert.equal(detail.count, 3);
|
||||
assert.equal(
|
||||
detail.files.find((x) => x.name === 'source/version.yaml').actual,
|
||||
null,
|
||||
);
|
||||
assert.equal(
|
||||
detail.files.find((x) => x.name === 'source/unexpected').expected,
|
||||
null,
|
||||
);
|
||||
const item = detail.files.find(
|
||||
(x) => x.name === 'source/sample/config.sample.sh',
|
||||
);
|
||||
assert.equal(item.actual.mode, 0o700);
|
||||
assert.notEqual(item.actual.sha256, item.expected.sha256);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('archive extraction retains exact artifact modes under a private umask without changing it', async (t) => {
|
||||
const f = await fixture(t);
|
||||
await fs.chmod(path.join(f.source, 'sample/config.sample.sh'), 0o755);
|
||||
await fs.chmod(path.join(f.source, 'version.yaml'), 0o644);
|
||||
f.manifest.files = await bundle.inventory({
|
||||
source: f.source,
|
||||
static: f.staticRoot,
|
||||
});
|
||||
await f.pack();
|
||||
const destination = path.join(f.directory, 'private-mask-extraction');
|
||||
await fs.mkdir(destination);
|
||||
const execute = require('node:util').promisify(
|
||||
require('node:child_process').execFile,
|
||||
);
|
||||
const result = await execute(process.execPath, [
|
||||
'-e',
|
||||
`
|
||||
const fs=require('node:fs/promises'),path=require('node:path');
|
||||
const bundle=require(process.argv[1]);
|
||||
(async()=>{
|
||||
process.umask(0o077);
|
||||
const destination=process.argv[3];
|
||||
await bundle.extractArchive(process.argv[2],destination);
|
||||
const selected={source:path.join(destination,'qinglong/source'),static:path.join(destination,'qinglong/static')};
|
||||
const manifest=JSON.parse(await fs.readFile(path.join(destination,'qinglong/bundle.json')));
|
||||
await bundle.verifyPayload(selected,manifest,JSON.parse(process.argv[4]));
|
||||
console.log(JSON.stringify({umask:process.umask(),script:(await fs.stat(path.join(selected.source,'sample/config.sample.sh'))).mode&0o777,metadata:(await fs.stat(path.join(selected.source,'version.yaml'))).mode&0o777,directory:(await fs.stat(selected.source)).mode&0o777,link:(await fs.lstat(path.join(selected.source,'node_modules/sqlite3'))).isSymbolicLink()}));
|
||||
})().catch(error=>{console.error(error);process.exitCode=1});
|
||||
`,
|
||||
require.resolve('../../cli/src/internal/maintenance/platformBundle.cjs'),
|
||||
f.archive,
|
||||
destination,
|
||||
JSON.stringify(f.target),
|
||||
]);
|
||||
assert.deepEqual(JSON.parse(result.stdout), {
|
||||
umask: 0o077,
|
||||
script: 0o755,
|
||||
metadata: 0o644,
|
||||
directory: 0o755,
|
||||
link: true,
|
||||
});
|
||||
});
|
||||
|
||||
test('dependency probes allow slow ARMv6 startup while preserving cancellation and normal Node execution', async () => {
|
||||
const vm = require('node:vm');
|
||||
const filename = require.resolve(
|
||||
@@ -874,3 +1163,51 @@ test('full dependency bundles get a bounded slow-link download budget instead of
|
||||
);
|
||||
assert.ok(observed);
|
||||
});
|
||||
|
||||
test('MikroORM bundles load the official dialect and built-in SQLite while ignoring unusable legacy addons', async (t) => {
|
||||
const { createRequire } = require('node:module');
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'ql-mikro-smoke-'));
|
||||
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
||||
await fs.writeFile(
|
||||
path.join(root, 'package.json'),
|
||||
JSON.stringify({
|
||||
dependencies: { '@mikro-orm/core': '7.2.5', '@mikro-orm/sql': '7.2.5' },
|
||||
}),
|
||||
);
|
||||
const resolver = createRequire(require.resolve('@mikro-orm/sql'));
|
||||
for (const name of ['@mikro-orm/core', '@mikro-orm/sql', 'kysely']) {
|
||||
let source = path.dirname(resolver.resolve(name));
|
||||
while (
|
||||
JSON.parse(
|
||||
await fs
|
||||
.readFile(path.join(source, 'package.json'), 'utf8')
|
||||
.catch(() => '{"name":""}'),
|
||||
).name !== name
|
||||
)
|
||||
source = path.dirname(source);
|
||||
await fs.cp(source, path.join(root, 'node_modules', name), {
|
||||
recursive: true,
|
||||
});
|
||||
}
|
||||
for (const name of [
|
||||
'sqlite3',
|
||||
'sequelize',
|
||||
'cron-parser',
|
||||
'@grpc/grpc-js',
|
||||
'undici',
|
||||
]) {
|
||||
const directory = path.join(root, 'node_modules', name);
|
||||
await fs.mkdir(directory, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(directory, 'package.json'),
|
||||
JSON.stringify({ name, main: 'index.js' }),
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(directory, 'index.js'),
|
||||
['sqlite3', 'sequelize'].includes(name)
|
||||
? 'throw Error("legacy addon must not load")'
|
||||
: 'module.exports={}',
|
||||
);
|
||||
}
|
||||
await bundle.smokeDependencies(root);
|
||||
});
|
||||
|
||||
@@ -2,97 +2,234 @@ const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
const tick = () => new Promise(setImmediate);
|
||||
function gate() { let release; const promise = new Promise(r => release = r); return { promise, release }; }
|
||||
|
||||
function gate() {
|
||||
let release;
|
||||
const promise = new Promise((r) => (release = r));
|
||||
return { promise, release };
|
||||
}
|
||||
async function fixture() {
|
||||
const logger = { info() {}, error() {}, warn() {} }, notices = [];
|
||||
const logger = { info() {}, error() {}, warn() {} },
|
||||
notices = [];
|
||||
const limit = load('back/shared/pLimit.ts', {
|
||||
'../data/system': { AuthDataType: { systemConfig: 'systemConfig' }, SystemModel: { sync: async () => {}, findOne: async () => ({ info: { cronConcurrency: 1 } }) } },
|
||||
'../data/system': {
|
||||
AuthDataType: { systemConfig: 'systemConfig' },
|
||||
SystemModel: {
|
||||
sync: async () => {},
|
||||
findOne: async (where, read = {}) => {
|
||||
return { info: { cronConcurrency: 1 } };
|
||||
},
|
||||
},
|
||||
},
|
||||
'../loaders/logger': logger,
|
||||
'../services/notify': class {},
|
||||
'../shared/i18n': { t: s => s, tf: s => s },
|
||||
'../shared/i18n': { t: (s) => s, tf: (s) => s },
|
||||
'../config': { grpcPort: 1 },
|
||||
'../protos/api': { ApiClient: class { systemNotify(value, cb) { notices.push(value); cb(); } } },
|
||||
'../protos/api': {
|
||||
ApiClient: class {
|
||||
systemNotify(value, cb) {
|
||||
notices.push(value);
|
||||
cb();
|
||||
}
|
||||
},
|
||||
},
|
||||
'../config/grpcCerts': { getGrpcCerts: () => null },
|
||||
}).default;
|
||||
await tick();
|
||||
const spawned = [], records = new Map(), jobs = new Map();
|
||||
const spawned = [],
|
||||
records = new Map(),
|
||||
jobs = new Map();
|
||||
let readGate, killGate;
|
||||
const { runCron } = load('back/shared/runCron.ts', {
|
||||
'cross-spawn': { spawn: command => { spawned.push(command); return {}; } },
|
||||
'cross-spawn': {
|
||||
spawn: (command) => {
|
||||
spawned.push(command);
|
||||
return {};
|
||||
},
|
||||
},
|
||||
'./pLimit': limit,
|
||||
'../loaders/logger': logger,
|
||||
'../data/cron': { CrontabModel: { findOne: async ({ where }) => { const row = records.get(String(where.id)); if (readGate) await readGate.promise; return row; }, update: async () => {} }, CrontabStatus: { running: 0, queued: 3, idle: 1 } },
|
||||
'../data/runningInstance': { RunningInstanceModel: { update: async () => {} }, InstanceStatus: { running: 0, stopped: 2 } },
|
||||
'../config/util': { killTask: async () => { if (killGate) await killGate.promise; } },
|
||||
'./childProcess': { observeChildProcess: () => ({ completed: Promise.resolve({ code: 0 }) }), asError: e => e },
|
||||
'../data/cron': {
|
||||
CrontabModel: {
|
||||
findOne: async (where, read = {}) => {
|
||||
const row = records.get(String(where.id));
|
||||
if (readGate) await readGate.promise;
|
||||
return row;
|
||||
},
|
||||
nativeUpdate: async (where, data) => {},
|
||||
},
|
||||
CrontabStatus: { running: 0, queued: 3, idle: 1 },
|
||||
},
|
||||
'../data/runningInstance': {
|
||||
RunningInstanceModel: { nativeUpdate: async (where, data) => {} },
|
||||
InstanceStatus: { running: 0, stopped: 2 },
|
||||
},
|
||||
'../config/util': {
|
||||
killTask: async () => {
|
||||
if (killGate) await killGate.promise;
|
||||
},
|
||||
},
|
||||
'./childProcess': {
|
||||
observeChildProcess: () => ({ completed: Promise.resolve({ code: 0 }) }),
|
||||
asError: (e) => e,
|
||||
},
|
||||
});
|
||||
const { addCron } = load('back/schedule/addCron.ts', {
|
||||
'../shared/cronScheduler': { createCronJob: (schedule, callback) => ({ start() {}, cancel() {}, fire: callback }) },
|
||||
'../shared/cronSchedule': { isValidCronSchedule: x => x !== 'invalid' },
|
||||
'./data': { scheduleStacks: jobs }, '../shared/runCron': { runCron }, '../loaders/logger': logger,
|
||||
'../shared/i18n': { t: (s) => s, tf: s => s },
|
||||
'../shared/cronScheduler': {
|
||||
createCronJob: (schedule, callback) => ({
|
||||
start() {},
|
||||
cancel() {},
|
||||
fire: callback,
|
||||
}),
|
||||
},
|
||||
'../shared/cronSchedule': { isValidCronSchedule: (x) => x !== 'invalid' },
|
||||
'./data': { scheduleStacks: jobs },
|
||||
'../shared/runCron': { runCron },
|
||||
'../loaders/logger': logger,
|
||||
'../shared/i18n': { t: (s) => s, tf: (s) => s },
|
||||
});
|
||||
const { delCron } = load('back/schedule/delCron.ts', {
|
||||
'./data': { scheduleStacks: jobs },
|
||||
'../loaders/logger': logger,
|
||||
});
|
||||
const { delCron } = load('back/schedule/delCron.ts', { './data': { scheduleStacks: jobs }, '../loaders/logger': logger });
|
||||
function add(id = '1', command = 'old', replace = false) {
|
||||
records.set(id, { isDisabled: 0, allow_multiple_instances: 1 });
|
||||
addCron({ request: { crons: [{ id, command, schedule: '* * * * * *' }], replace } }, err => assert.ifError(err));
|
||||
addCron(
|
||||
{
|
||||
request: { crons: [{ id, command, schedule: '* * * * * *' }], replace },
|
||||
},
|
||||
(err) => assert.ifError(err),
|
||||
);
|
||||
return jobs.get(id)[0];
|
||||
}
|
||||
function remove(id = '1') { delCron({ request: { ids: [id] } }, err => assert.ifError(err)); }
|
||||
async function block() { const g = gate(); const done = limit.runWithCronLimit({ id: 'blocker' }, async () => { await g.promise; }); await tick(); return { release: async () => { g.release(); await done; } }; }
|
||||
return { limit, spawned, records, jobs, add, remove, block, notices, setReadGate: g => readGate = g, setKillGate: g => killGate = g };
|
||||
function remove(id = '1') {
|
||||
delCron({ request: { ids: [id] } }, (err) => assert.ifError(err));
|
||||
}
|
||||
async function block() {
|
||||
const g = gate();
|
||||
const done = limit.runWithCronLimit({ id: 'blocker' }, async () => {
|
||||
await g.promise;
|
||||
});
|
||||
await tick();
|
||||
return {
|
||||
release: async () => {
|
||||
g.release();
|
||||
await done;
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
limit,
|
||||
spawned,
|
||||
records,
|
||||
jobs,
|
||||
add,
|
||||
remove,
|
||||
block,
|
||||
notices,
|
||||
setReadGate: (g) => (readGate = g),
|
||||
setKillGate: (g) => (killGate = g),
|
||||
};
|
||||
}
|
||||
|
||||
for (const operation of ['disable', 'delete', 'update', 'replace', 'disable-enable']) {
|
||||
for (const operation of [
|
||||
'disable',
|
||||
'delete',
|
||||
'update',
|
||||
'replace',
|
||||
'disable-enable',
|
||||
]) {
|
||||
test(`queued scheduled execution is invalidated by ${operation}`, async () => {
|
||||
const f = await fixture(), old = f.add(), blocker = await f.block();
|
||||
const f = await fixture(),
|
||||
old = f.add(),
|
||||
blocker = await f.block();
|
||||
const queued = old.fire();
|
||||
if (operation === 'disable' || operation === 'disable-enable') { f.records.get('1').isDisabled = 1; f.remove(); }
|
||||
if (operation === 'delete') { f.records.delete('1'); f.remove(); }
|
||||
if (operation === 'disable' || operation === 'disable-enable') {
|
||||
f.records.get('1').isDisabled = 1;
|
||||
f.remove();
|
||||
}
|
||||
if (operation === 'delete') {
|
||||
f.records.delete('1');
|
||||
f.remove();
|
||||
}
|
||||
if (operation === 'update') f.add('1', 'new');
|
||||
if (operation === 'replace') f.add('1', 'new', true);
|
||||
if (operation === 'disable-enable') f.add('1', 'old');
|
||||
const next = f.jobs.get('1')?.[0].fire();
|
||||
await blocker.release(); await queued; await next;
|
||||
assert.deepEqual(f.spawned, ['update', 'replace'].includes(operation) ? ['new'] : operation === 'disable-enable' ? ['old'] : []);
|
||||
await blocker.release();
|
||||
await queued;
|
||||
await next;
|
||||
assert.deepEqual(
|
||||
f.spawned,
|
||||
['update', 'replace'].includes(operation)
|
||||
? ['new']
|
||||
: operation === 'disable-enable'
|
||||
? ['old']
|
||||
: [],
|
||||
);
|
||||
assert.equal(f.limit.cronLimitPendingCount, 0);
|
||||
assert.equal(f.limit.cronLimitActiveCount, 0);
|
||||
});
|
||||
}
|
||||
|
||||
test('stale callbacks cannot re-enqueue after cancellation', async () => {
|
||||
const f = await fixture(), old = f.add(); f.remove(); await old.fire();
|
||||
const f = await fixture(),
|
||||
old = f.add();
|
||||
f.remove();
|
||||
await old.fire();
|
||||
assert.deepEqual(f.spawned, []);
|
||||
});
|
||||
|
||||
test('invalidation during an asynchronous database read prevents spawning', async () => {
|
||||
const f = await fixture(), old = f.add(), pending = gate(); f.setReadGate(pending);
|
||||
const run = old.fire(); await tick(); f.remove(); pending.release(); await run;
|
||||
const f = await fixture(),
|
||||
old = f.add(),
|
||||
pending = gate();
|
||||
f.setReadGate(pending);
|
||||
const run = old.fire();
|
||||
await tick();
|
||||
f.remove();
|
||||
pending.release();
|
||||
await run;
|
||||
assert.deepEqual(f.spawned, []);
|
||||
});
|
||||
|
||||
test('invalidation while replacing a running instance prevents the old spawn', async () => {
|
||||
const f = await fixture(), old = f.add(), pending = gate();
|
||||
f.records.set('1', { isDisabled: 0, allow_multiple_instances: 0, pid: 10, status: 0 }); f.setKillGate(pending);
|
||||
const run = old.fire(); await tick(); f.remove(); pending.release(); await run;
|
||||
const f = await fixture(),
|
||||
old = f.add(),
|
||||
pending = gate();
|
||||
f.records.set('1', {
|
||||
isDisabled: 0,
|
||||
allow_multiple_instances: 0,
|
||||
pid: 10,
|
||||
status: 0,
|
||||
});
|
||||
f.setKillGate(pending);
|
||||
const run = old.fire();
|
||||
await tick();
|
||||
f.remove();
|
||||
pending.release();
|
||||
await run;
|
||||
assert.deepEqual(f.spawned, []);
|
||||
});
|
||||
|
||||
test('database removal or disable is respected before scheduler RPC arrives', async () => {
|
||||
for (const disabled of [true, false]) {
|
||||
const f = await fixture(), old = f.add(), blocker = await f.block(); const run = old.fire();
|
||||
if (disabled) f.records.get('1').isDisabled = 1; else f.records.delete('1');
|
||||
await blocker.release(); await run; assert.deepEqual(f.spawned, []);
|
||||
const f = await fixture(),
|
||||
old = f.add(),
|
||||
blocker = await f.block();
|
||||
const run = old.fire();
|
||||
if (disabled) f.records.get('1').isDisabled = 1;
|
||||
else f.records.delete('1');
|
||||
await blocker.release();
|
||||
await run;
|
||||
assert.deepEqual(f.spawned, []);
|
||||
}
|
||||
});
|
||||
|
||||
test('five stale runs do not suppress the new revision or erase its repeat accounting', async () => {
|
||||
const f = await fixture(), old = f.add(), blocker = await f.block();
|
||||
const f = await fixture(),
|
||||
old = f.add(),
|
||||
blocker = await f.block();
|
||||
const pending = Array.from({ length: 5 }, () => old.fire());
|
||||
const current = f.add('1', 'new'); pending.push(...Array.from({ length: 5 }, () => current.fire()));
|
||||
const current = f.add('1', 'new');
|
||||
pending.push(...Array.from({ length: 5 }, () => current.fire()));
|
||||
assert.equal(f.notices.length, 0);
|
||||
await blocker.release(); await Promise.all(pending);
|
||||
await blocker.release();
|
||||
await Promise.all(pending);
|
||||
assert.deepEqual(f.spawned, Array(5).fill('new'));
|
||||
await current.fire(); assert.equal(f.spawned.length, 6);
|
||||
await current.fire();
|
||||
assert.equal(f.spawned.length, 6);
|
||||
});
|
||||
|
||||
@@ -1,23 +1,16 @@
|
||||
const { DatabaseClient } = require('../../back/data/databaseClient');
|
||||
const { repository, initialize } = require('../helpers/repository-fixture.cjs');
|
||||
const { orderBy } = require('../../back/data/queries');
|
||||
const assert = require('node:assert/strict');
|
||||
const test = require('node:test');
|
||||
const { Sequelize, DataTypes } = require('sequelize');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
|
||||
test('startup closes interrupted instances without inventing exit codes or rewriting history', async (t) => {
|
||||
const db = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const db = new DatabaseClient(':memory:');
|
||||
t.after(() => db.close());
|
||||
const instances = db.define('Instance', {
|
||||
status: DataTypes.INTEGER,
|
||||
started_at: DataTypes.INTEGER,
|
||||
finished_at: DataTypes.INTEGER,
|
||||
exit_code: DataTypes.INTEGER,
|
||||
});
|
||||
await db.sync();
|
||||
await instances.bulkCreate([
|
||||
const instances = repository(db, 'RunningInstance');
|
||||
await initialize(db);
|
||||
await instances.insertMany([
|
||||
{ id: 1, status: 0, started_at: 100 },
|
||||
{ id: 2, status: 1, started_at: 100, finished_at: 110, exit_code: 0 },
|
||||
{ id: 3, status: 2, started_at: 100, finished_at: 120, exit_code: 143 },
|
||||
@@ -39,25 +32,25 @@ test('startup closes interrupted instances without inventing exit codes or rewri
|
||||
'../services/user': class {},
|
||||
'../services/open': class {},
|
||||
'../data/cron': {
|
||||
CrontabModel: { update: async () => {}, findAll: async () => [] },
|
||||
CrontabModel: { nativeUpdate: async () => {}, find: async () => [] },
|
||||
CrontabStatus: { idle: 1 },
|
||||
},
|
||||
'../data/dependence': {
|
||||
DependenceModel: { findAll: async () => [], update: async () => {} },
|
||||
DependenceModel: { find: async () => [], nativeUpdate: async () => {} },
|
||||
DependenceStatus: { queued: 0 },
|
||||
},
|
||||
'../data/cronView': {
|
||||
CrontabViewModel: { findAll: async () => [{}] },
|
||||
CrontabViewModel: { find: async () => [{}] },
|
||||
CronViewType: { 系统: 0 },
|
||||
},
|
||||
'../data/env': {},
|
||||
'../data/system': {
|
||||
SystemModel: { findOrCreate: async () => [systemConfig] },
|
||||
SystemModel: { getOrCreate: async () => systemConfig },
|
||||
AuthDataType: {},
|
||||
},
|
||||
'../data/open': {
|
||||
AppModel: {
|
||||
findOne: async () => ({ get: () => ({ scopes: ['dashboard'] }) }),
|
||||
findOne: async () => ({ scopes: ['dashboard'] }),
|
||||
},
|
||||
},
|
||||
'../data/runningInstance': {
|
||||
@@ -82,7 +75,12 @@ test('startup closes interrupted instances without inventing exit codes or rewri
|
||||
} finally {
|
||||
global.setTimeout = originalTimeout;
|
||||
}
|
||||
const rows = await instances.findAll({ order: [['id', 'ASC']], raw: true });
|
||||
const rows = await instances.find(
|
||||
{},
|
||||
{
|
||||
orderBy: orderBy([['id', 'ASC']]),
|
||||
},
|
||||
);
|
||||
assert.equal(rows[0].status, 2);
|
||||
assert.ok(
|
||||
rows[0].finished_at >= before &&
|
||||
@@ -110,7 +108,12 @@ test('startup closes interrupted instances without inventing exit codes or rewri
|
||||
global.setTimeout = originalTimeout;
|
||||
}
|
||||
assert.deepEqual(
|
||||
await instances.findAll({ order: [['id', 'ASC']], raw: true }),
|
||||
await instances.find(
|
||||
{},
|
||||
{
|
||||
orderBy: orderBy([['id', 'ASC']]),
|
||||
},
|
||||
),
|
||||
rows,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -3,7 +3,6 @@ const fs = require('node:fs/promises');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const test = require('node:test');
|
||||
const { Op } = require('sequelize');
|
||||
|
||||
const {
|
||||
getDirectorySize,
|
||||
@@ -62,7 +61,7 @@ test('cleanup previews first, protects running instances, and uses explicit opti
|
||||
paths: [],
|
||||
};
|
||||
};
|
||||
stubModule('../../back/data', { sequelize });
|
||||
stubModule('../../back/data', { database: sequelize });
|
||||
stubModule('../../back/data/cronStats', { CrontabStatModel });
|
||||
stubModule('../../back/data/runningInstance', {
|
||||
InstanceStatus,
|
||||
@@ -85,12 +84,12 @@ test('cleanup previews first, protects running instances, and uses explicit opti
|
||||
let instanceCountCalls = 0;
|
||||
let statCountCalls = 0;
|
||||
config.dependenceCachePath = directory;
|
||||
RunningInstanceModel.count = async ({ where }) => {
|
||||
RunningInstanceModel.count = async (where) => {
|
||||
instanceCountCalls++;
|
||||
instanceWhere = where;
|
||||
return 2;
|
||||
};
|
||||
RunningInstanceModel.destroy = async ({ where }) => {
|
||||
RunningInstanceModel.nativeDelete = async (where) => {
|
||||
instanceWhere = where;
|
||||
return 2;
|
||||
};
|
||||
@@ -98,7 +97,7 @@ test('cleanup previews first, protects running instances, and uses explicit opti
|
||||
statCountCalls++;
|
||||
return 3;
|
||||
};
|
||||
CrontabStatModel.destroy = async () => 3;
|
||||
CrontabStatModel.nativeDelete = async () => 3;
|
||||
sequelize.transaction = async (callback) => callback({});
|
||||
sequelize.query = async (query) => {
|
||||
vacuumed = query === 'VACUUM';
|
||||
@@ -122,7 +121,7 @@ test('cleanup previews first, protects running instances, and uses explicit opti
|
||||
compactDatabase: true,
|
||||
});
|
||||
|
||||
assert.equal(instanceWhere.status[Op.ne], InstanceStatus.running);
|
||||
assert.equal(instanceWhere.status.$ne, InstanceStatus.running);
|
||||
assert.equal(result.deleted.runningInstances, 2);
|
||||
assert.equal(result.deleted.cronStats, 3);
|
||||
assert.equal(result.deleted.dependenceCacheBytes, 13);
|
||||
|
||||
@@ -84,3 +84,17 @@ test('schedule health check uses the HTTP health endpoint and system logs', asyn
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('scheduler liveness reports a failed database process without calling HTTP', async () => {
|
||||
const load=require('../helpers/load-security-module.cjs');
|
||||
const database={healthy:true};
|
||||
const {check}=load('back/schedule/health.ts',{
|
||||
'../data':{database},
|
||||
'../config':{},
|
||||
undici:{request:()=>assert.fail('local scheduler checks must not call HTTP')},
|
||||
});
|
||||
const probe=()=>new Promise(resolve=>check({request:{service:'scheduler'}},(error,response)=>resolve({error,response})));
|
||||
assert.deepEqual((await probe()).response,{status:1});
|
||||
database.healthy=false;
|
||||
assert.deepEqual((await probe()).response,{status:2});
|
||||
});
|
||||
|
||||
@@ -13,7 +13,9 @@ function createLimit() {
|
||||
AuthDataType: { systemConfig: 'systemConfig' },
|
||||
SystemModel: {
|
||||
sync: async () => {},
|
||||
findOne: async () => ({ info: { cronConcurrency: 1 } }),
|
||||
findOne: async (where, read = {}) => {
|
||||
return { info: { cronConcurrency: 1 } };
|
||||
},
|
||||
},
|
||||
},
|
||||
'../loaders/logger': logger,
|
||||
@@ -47,11 +49,13 @@ function createSystem(client, limit, options = {}) {
|
||||
'../data/system': {
|
||||
AuthDataType: { systemConfig: 'systemConfig' },
|
||||
SystemModel: {
|
||||
findOne: async () => ({
|
||||
...store.state,
|
||||
get: () => ({ ...store.state }),
|
||||
}),
|
||||
update: async (values) => {
|
||||
findOne: async (where, read = {}) => {
|
||||
return {
|
||||
...{ ...store.state },
|
||||
...store.state,
|
||||
};
|
||||
},
|
||||
nativeUpdate: async (where, values) => {
|
||||
writes.push(values);
|
||||
store.state = { ...store.state, ...values };
|
||||
},
|
||||
@@ -141,7 +145,6 @@ function enqueue(t, limit, count) {
|
||||
});
|
||||
return { releases, runs };
|
||||
}
|
||||
|
||||
test('concurrency changes reach the independent scheduler queue over mTLS immediately and preserve running jobs', async (t) => {
|
||||
const f = await rpcFixture(t);
|
||||
const { releases, runs } = enqueue(t, f.schedulerLimit, 3);
|
||||
@@ -175,7 +178,6 @@ test('concurrency changes reach the independent scheduler queue over mTLS immedi
|
||||
await runs[2];
|
||||
assert.equal(f.state().info.cronConcurrency, 1);
|
||||
});
|
||||
|
||||
test('zero, null and omitted concurrency restore the default in both API and scheduler queues without a restart', async (t) => {
|
||||
const f = await rpcFixture(t);
|
||||
for (const value of [0, null, undefined]) {
|
||||
@@ -200,7 +202,6 @@ test('zero, null and omitted concurrency restore the default in both API and sch
|
||||
await Promise.all([...scheduler.runs, ...manual.runs]);
|
||||
}
|
||||
});
|
||||
|
||||
test('invalid concurrency is rejected before configuration writes or queue changes', async () => {
|
||||
const mutations = [];
|
||||
const f = createSystem(
|
||||
@@ -225,7 +226,6 @@ test('invalid concurrency is rejected before configuration writes or queue chang
|
||||
);
|
||||
assert.deepEqual(mutations, []);
|
||||
});
|
||||
|
||||
for (const code of [grpc.status.UNAVAILABLE, grpc.status.DEADLINE_EXCEEDED]) {
|
||||
test(`concurrency RPC failure ${code} invalidates readiness and is not reported as success or retried`, async () => {
|
||||
let writes = 0,
|
||||
@@ -266,13 +266,11 @@ for (const code of [grpc.status.UNAVAILABLE, grpc.status.DEADLINE_EXCEEDED]) {
|
||||
assert.equal(invalidations, 1);
|
||||
});
|
||||
}
|
||||
|
||||
function mutationConfig(t) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-concurrency-lock-'));
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
return { crontabFile: path.join(root, 'crontab.list') };
|
||||
}
|
||||
|
||||
test('independent API service instances serialize database and scheduler updates with the shared filesystem lock', async (t) => {
|
||||
const config = mutationConfig(t);
|
||||
const store = { state: { id: 1, info: { cronConcurrency: 1 } } };
|
||||
@@ -310,7 +308,6 @@ test('independent API service instances serialize database and scheduler updates
|
||||
[1, 1, 1],
|
||||
);
|
||||
});
|
||||
|
||||
for (const newerLimit of [undefined, 2]) {
|
||||
test(`production recovery restores persisted concurrency before health becomes ready (newer limit: ${newerLimit})`, async (t) => {
|
||||
const config = mutationConfig(t);
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
const { DatabaseClient } = require('../../back/data/databaseClient');
|
||||
const { repository, initialize } = require('../helpers/repository-fixture.cjs');
|
||||
const { orderBy } = require('../../back/data/queries');
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs/promises');
|
||||
@@ -9,7 +12,6 @@ const { setTimeout: delay } = require('node:timers/promises');
|
||||
const { Sequelize, DataTypes } = require('sequelize');
|
||||
const { status } = require('@grpc/grpc-js');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
|
||||
function gate() {
|
||||
let release;
|
||||
const promise = new Promise((resolve) => {
|
||||
@@ -17,25 +19,13 @@ function gate() {
|
||||
});
|
||||
return { promise, release };
|
||||
}
|
||||
|
||||
async function fixture(t) {
|
||||
const root = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), 'ql-scheduler-mutation-'),
|
||||
);
|
||||
const db = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const crons = db.define('Cron', {
|
||||
name: DataTypes.STRING,
|
||||
command: DataTypes.STRING,
|
||||
schedule: DataTypes.STRING,
|
||||
isDisabled: DataTypes.INTEGER,
|
||||
saved: DataTypes.BOOLEAN,
|
||||
queued_token: DataTypes.STRING,
|
||||
});
|
||||
await db.sync();
|
||||
const db = new DatabaseClient(':memory:');
|
||||
const crons = repository(db, 'Crontab');
|
||||
await initialize(db);
|
||||
t.after(async () => {
|
||||
await db.close();
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
@@ -75,15 +65,14 @@ async function fixture(t) {
|
||||
service.getLogName = async () => 'task';
|
||||
service.shouldUseCronClient = () => true;
|
||||
service.makeCommand = (doc) => doc.command;
|
||||
service.crontabs = async () => ({ data: await crons.findAll({ raw: true }) });
|
||||
service.crontabs = async () => ({ data: await crons.find({}, {}) });
|
||||
service.setCrontab = async () => {};
|
||||
return { service, crons, jobs, client, root };
|
||||
}
|
||||
|
||||
for (const operation of ['create', 'update', 'remove', 'disabled', 'enabled']) {
|
||||
test(`recovery snapshot cannot overwrite concurrent ${operation}`, async (t) => {
|
||||
const { service, crons, jobs } = await fixture(t);
|
||||
await crons.create({
|
||||
await crons.insert({
|
||||
id: 1,
|
||||
command: 'old',
|
||||
schedule: '* * * * *',
|
||||
@@ -101,7 +90,7 @@ for (const operation of ['create', 'update', 'remove', 'disabled', 'enabled']) {
|
||||
};
|
||||
const recovery = service.autosave_crontab(true);
|
||||
await entered.promise;
|
||||
const before = await crons.findAll({ raw: true });
|
||||
const before = await crons.find({}, {});
|
||||
const mutation =
|
||||
operation === 'create'
|
||||
? service.create({ command: 'new', schedule: '* * * * *' })
|
||||
@@ -110,13 +99,13 @@ for (const operation of ['create', 'update', 'remove', 'disabled', 'enabled']) {
|
||||
: service[operation]([1]);
|
||||
await delay(50);
|
||||
assert.deepEqual(
|
||||
await crons.findAll({ raw: true }),
|
||||
await crons.find({}, {}),
|
||||
before,
|
||||
'DB writes must also wait for recovery',
|
||||
);
|
||||
release.release();
|
||||
await Promise.all([recovery, mutation]);
|
||||
const rows = await crons.findAll({ raw: true });
|
||||
const rows = await crons.find({}, {});
|
||||
assert.deepEqual(
|
||||
[...jobs.entries()].sort(),
|
||||
rows
|
||||
@@ -126,7 +115,6 @@ for (const operation of ['create', 'update', 'remove', 'disabled', 'enabled']) {
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
test('recovery waits for an admitted mutation to finish registration before reading the DB', async (t) => {
|
||||
const { service, client, jobs } = await fixture(t);
|
||||
const entered = gate(),
|
||||
@@ -154,12 +142,11 @@ test('recovery waits for an admitted mutation to finish registration before read
|
||||
await Promise.all([mutation, recovery]);
|
||||
assert.deepEqual([...jobs.values()], ['new']);
|
||||
});
|
||||
|
||||
for (const operation of ['create', 'update', 'enabled']) {
|
||||
test(`${operation} preserves scheduler 503 after rollback and releases its lock`, async (t) => {
|
||||
const { service, crons, client } = await fixture(t);
|
||||
if (operation !== 'create')
|
||||
await crons.create({
|
||||
await crons.insert({
|
||||
id: 1,
|
||||
command: 'old',
|
||||
schedule: '* * * * *',
|
||||
@@ -180,9 +167,11 @@ for (const operation of ['create', 'update', 'enabled']) {
|
||||
(err) =>
|
||||
err.status === 503 && err.cause === cause && /回滚/.test(err.message),
|
||||
);
|
||||
if (operation === 'create') assert.equal(await crons.count(), 0);
|
||||
if (operation === 'create') assert.equal(await crons.count({}), 0);
|
||||
else {
|
||||
const row = await crons.findByPk(1);
|
||||
const row = await crons.findOne({
|
||||
id: 1,
|
||||
});
|
||||
assert.equal(row.command, 'old');
|
||||
assert.equal(row.isDisabled, operation === 'enabled' ? 1 : 0);
|
||||
}
|
||||
@@ -190,7 +179,6 @@ for (const operation of ['create', 'update', 'enabled']) {
|
||||
await service.autosave_crontab(true);
|
||||
});
|
||||
}
|
||||
|
||||
test(
|
||||
'scheduler lock excludes a second OS process and leaves no lock after completion',
|
||||
{ timeout: 10000 },
|
||||
@@ -247,18 +235,23 @@ test(
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
for (const operation of ['update', 'remove', 'disabled']) {
|
||||
for (const code of [status.DEADLINE_EXCEEDED, status.UNAVAILABLE]) {
|
||||
for (const applied of [false, true]) {
|
||||
test(`${operation} aborts on RPC ${code} with deletion applied=${applied} and reconciles the original DB`, async (t) => {
|
||||
const { service, crons, client, jobs } = await fixture(t);
|
||||
await crons.create({
|
||||
id: 1, command: 'old', schedule: '* * * * *', isDisabled: 0,
|
||||
await crons.insert({
|
||||
id: 1,
|
||||
command: 'old',
|
||||
schedule: '* * * * *',
|
||||
isDisabled: 0,
|
||||
});
|
||||
jobs.set('1', 'old');
|
||||
const before = await crons.findAll({ raw: true });
|
||||
const failure = Object.assign(Error('uncertain deletion'), { code, status: 503 });
|
||||
const before = await crons.find({}, {});
|
||||
const failure = Object.assign(Error('uncertain deletion'), {
|
||||
code,
|
||||
status: 503,
|
||||
});
|
||||
let deletes = 0;
|
||||
client.delCron = async (ids) => {
|
||||
deletes++;
|
||||
@@ -266,8 +259,10 @@ for (const operation of ['update', 'remove', 'disabled']) {
|
||||
throw failure;
|
||||
};
|
||||
const add = client.addCron;
|
||||
client.addCron = async () => assert.fail('must not register after failed deletion');
|
||||
service.setCrontab = async () => assert.fail('must not publish after failed deletion');
|
||||
client.addCron = async () =>
|
||||
assert.fail('must not register after failed deletion');
|
||||
service.setCrontab = async () =>
|
||||
assert.fail('must not publish after failed deletion');
|
||||
await assert.rejects(
|
||||
operation === 'update'
|
||||
? service.update({ id: 1, command: 'new' })
|
||||
@@ -275,7 +270,7 @@ for (const operation of ['update', 'remove', 'disabled']) {
|
||||
(error) => error === failure && error.status === 503,
|
||||
);
|
||||
assert.equal(deletes, 1, 'must not replay an uncertain deletion');
|
||||
assert.deepEqual(await crons.findAll({ raw: true }), before);
|
||||
assert.deepEqual(await crons.find({}, {}), before);
|
||||
client.addCron = add;
|
||||
service.setCrontab = async () => {};
|
||||
await service.autosave_crontab(true);
|
||||
@@ -284,10 +279,9 @@ for (const operation of ['update', 'remove', 'disabled']) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test('configuration rollback cannot restore an obsolete manual queue token', async (t) => {
|
||||
const { service, crons, client } = await fixture(t);
|
||||
await crons.create({
|
||||
await crons.insert({
|
||||
id: 1,
|
||||
command: 'old',
|
||||
schedule: '* * * * *',
|
||||
@@ -298,7 +292,7 @@ test('configuration rollback cannot restore an obsolete manual queue token', asy
|
||||
client.addCron = async () => {
|
||||
if (first) {
|
||||
first = false;
|
||||
await crons.update({ queued_token: 'new-token' }, { where: { id: 1 } });
|
||||
await crons.nativeUpdate({ id: 1 }, { queued_token: 'new-token' });
|
||||
throw Object.assign(Error('unavailable'), { status: 503 });
|
||||
}
|
||||
};
|
||||
@@ -306,7 +300,9 @@ test('configuration rollback cannot restore an obsolete manual queue token', asy
|
||||
service.update({ id: 1, command: 'new' }),
|
||||
(err) => err.status === 503,
|
||||
);
|
||||
const row = await crons.findByPk(1);
|
||||
const row = await crons.findOne({
|
||||
id: 1,
|
||||
});
|
||||
assert.equal(row.command, 'old');
|
||||
assert.equal(row.queued_token, 'new-token');
|
||||
});
|
||||
|
||||
@@ -3,16 +3,12 @@ const test = require('node:test');
|
||||
const fs = require('node:fs/promises');
|
||||
const path = require('node:path');
|
||||
const os = require('node:os');
|
||||
const { Sequelize, QueryTypes } = require('sequelize');
|
||||
const { DatabaseClient } = require('../../back/data/databaseClient');
|
||||
const { migrateSchema } = require('../../back/shared/schemaMigrations');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
|
||||
async function createLegacy(storage = ':memory:', omitEnvs = false) {
|
||||
const database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage,
|
||||
logging: false,
|
||||
});
|
||||
const database = new DatabaseClient(storage);
|
||||
for (const table of [
|
||||
'CrontabViews',
|
||||
'Subscriptions',
|
||||
@@ -34,15 +30,11 @@ test('legacy database upgrades without losing rows and repeated migration is ide
|
||||
t.after(() => database.close());
|
||||
await migrateSchema(database);
|
||||
await migrateSchema(database);
|
||||
const rows = await database.query('SELECT * FROM "Crontabs"', {
|
||||
type: QueryTypes.SELECT,
|
||||
});
|
||||
const rows = await database.query('SELECT * FROM "Crontabs"');
|
||||
assert.equal(rows[0].name, 'keep-me');
|
||||
assert.equal(rows[0].queued_token, null);
|
||||
assert.ok(Object.hasOwn(rows[0], 'allow_multiple_instances'));
|
||||
const applied = await database.query('SELECT id FROM "SchemaMigrations"', {
|
||||
type: QueryTypes.SELECT,
|
||||
});
|
||||
const applied = await database.query('SELECT id FROM "SchemaMigrations"');
|
||||
assert.equal(applied.length, 15);
|
||||
});
|
||||
|
||||
@@ -50,7 +42,12 @@ test('migration failure rolls back added columns and can be retried after repair
|
||||
const database = await createLegacy(':memory:', true);
|
||||
t.after(() => database.close());
|
||||
await assert.rejects(migrateSchema(database));
|
||||
const columns = await database.getQueryInterface().describeTable('Crontabs');
|
||||
const columns = Object.fromEntries(
|
||||
(await database.query('PRAGMA table_info(Crontabs)')).map((row) => [
|
||||
row.name,
|
||||
row,
|
||||
]),
|
||||
);
|
||||
assert.equal(Object.hasOwn(columns, 'work_dir'), false);
|
||||
await database.query(
|
||||
'CREATE TABLE "Envs" (id INTEGER PRIMARY KEY, name TEXT)',
|
||||
@@ -58,7 +55,12 @@ test('migration failure rolls back added columns and can be retried after repair
|
||||
await migrateSchema(database);
|
||||
assert.ok(
|
||||
Object.hasOwn(
|
||||
await database.getQueryInterface().describeTable('Crontabs'),
|
||||
Object.fromEntries(
|
||||
(await database.query('PRAGMA table_info(Crontabs)')).map((row) => [
|
||||
row.name,
|
||||
row,
|
||||
]),
|
||||
),
|
||||
'work_dir',
|
||||
),
|
||||
);
|
||||
@@ -72,30 +74,25 @@ test('offline pre-upgrade backup restores the legacy schema and data', async (t)
|
||||
let database = await createLegacy(file);
|
||||
await database.close();
|
||||
await fs.copyFile(file, backup);
|
||||
database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: file,
|
||||
logging: false,
|
||||
});
|
||||
database = new DatabaseClient(file);
|
||||
await migrateSchema(database);
|
||||
await database.close();
|
||||
await fs.copyFile(backup, file);
|
||||
database = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: file,
|
||||
logging: false,
|
||||
});
|
||||
database = new DatabaseClient(file);
|
||||
try {
|
||||
assert.equal(
|
||||
Object.hasOwn(
|
||||
await database.getQueryInterface().describeTable('Crontabs'),
|
||||
Object.fromEntries(
|
||||
(await database.query('PRAGMA table_info(Crontabs)')).map((row) => [
|
||||
row.name,
|
||||
row,
|
||||
]),
|
||||
),
|
||||
'work_dir',
|
||||
),
|
||||
false,
|
||||
);
|
||||
const rows = await database.query('SELECT name FROM "Crontabs"', {
|
||||
type: QueryTypes.SELECT,
|
||||
});
|
||||
const rows = await database.query('SELECT name FROM "Crontabs"');
|
||||
assert.equal(rows[0].name, 'keep-me');
|
||||
} finally {
|
||||
await database.close();
|
||||
@@ -103,23 +100,15 @@ test('offline pre-upgrade backup restores the legacy schema and data', async (t)
|
||||
});
|
||||
|
||||
test('database loader rejects initialization failure rather than allowing workers to start', async () => {
|
||||
const mocks = { './logger': { error() {}, info() {} } };
|
||||
for (const [file, model] of [
|
||||
['env', 'EnvModel'],
|
||||
['cron', 'CrontabModel'],
|
||||
['dependence', 'DependenceModel'],
|
||||
['open', 'AppModel'],
|
||||
['system', 'SystemModel'],
|
||||
['subscription', 'SubscriptionModel'],
|
||||
['cronView', 'CrontabViewModel'],
|
||||
['cronStats', 'CrontabStatModel'],
|
||||
['runningInstance', 'RunningInstanceModel'],
|
||||
])
|
||||
mocks[`../data/${file}`] = { [model]: { sync: async () => {} } };
|
||||
mocks['../data'] = { sequelize: {} };
|
||||
mocks['../shared/schemaMigrations'] = {
|
||||
migrateSchema: async () => {
|
||||
throw new Error('SQLITE_FULL');
|
||||
const mocks = {
|
||||
'./logger': { error() {}, info() {} },
|
||||
'../data': {
|
||||
database: { transaction: (action) => action(), query: async () => [] },
|
||||
},
|
||||
'../shared/schemaMigrations': {
|
||||
migrateSchema: async () => {
|
||||
throw new Error('SQLITE_FULL');
|
||||
},
|
||||
},
|
||||
};
|
||||
const initialize = load(path.resolve('back/loaders/db.ts'), mocks).default;
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const load = require('../helpers/load-security-module.cjs');
|
||||
|
||||
const logger = { info() {}, error() {} };
|
||||
|
||||
function cronFixture(killTask) {
|
||||
const events = [];
|
||||
const { runCron } = load('back/shared/runCron.ts', {
|
||||
@@ -25,23 +23,30 @@ function cronFixture(killTask) {
|
||||
'../config/util': { killTask },
|
||||
'../data/cron': {
|
||||
CrontabModel: {
|
||||
findOne: async () => ({
|
||||
pid: 100,
|
||||
status: 0,
|
||||
allow_multiple_instances: 0,
|
||||
}),
|
||||
update: async () => events.push('idle'),
|
||||
findOne: async (where, read = {}) => {
|
||||
return {
|
||||
pid: 100,
|
||||
status: 0,
|
||||
allow_multiple_instances: 0,
|
||||
};
|
||||
},
|
||||
nativeUpdate: async (where, data) => {
|
||||
return events.push('idle');
|
||||
},
|
||||
},
|
||||
CrontabStatus: { running: 0, idle: 1, queued: 3 },
|
||||
},
|
||||
'../data/runningInstance': {
|
||||
RunningInstanceModel: { update: async () => events.push('stopped') },
|
||||
RunningInstanceModel: {
|
||||
nativeUpdate: async (where, data) => {
|
||||
return events.push('stopped');
|
||||
},
|
||||
},
|
||||
InstanceStatus: { running: 0, stopped: 2 },
|
||||
},
|
||||
});
|
||||
return { run: () => runCron('ignored', { id: '1' }), events };
|
||||
}
|
||||
|
||||
test('single-instance replacement does not spawn or finalize when termination fails', async () => {
|
||||
const f = cronFixture(async () => {
|
||||
throw Error('EPERM');
|
||||
@@ -49,7 +54,6 @@ test('single-instance replacement does not spawn or finalize when termination fa
|
||||
await f.run();
|
||||
assert.deepEqual(f.events, ['release']);
|
||||
});
|
||||
|
||||
test('single-instance replacement waits for verified exit before spawning', async () => {
|
||||
let finish, entered, verified;
|
||||
const pending = new Promise((resolve) => {
|
||||
@@ -72,7 +76,6 @@ test('single-instance replacement waits for verified exit before spawning', asyn
|
||||
assert.equal(verified, true);
|
||||
assert.deepEqual(f.events, ['stopped', 'idle', 'spawn', 'release']);
|
||||
});
|
||||
|
||||
function subscriptionFixture(killTask) {
|
||||
const updates = [];
|
||||
const Service = load('back/services/subscription.ts', {
|
||||
@@ -81,11 +84,18 @@ function subscriptionFixture(killTask) {
|
||||
'../config/const': {},
|
||||
'../data/subscription': {
|
||||
SubscriptionModel: {
|
||||
findAll: async () => [
|
||||
{ id: 1, pid: 101 },
|
||||
{ id: 2, pid: 102 },
|
||||
],
|
||||
update: async (values, query) => updates.push({ values, query }),
|
||||
find: async (where, read = {}) => {
|
||||
return [
|
||||
{ id: 1, pid: 101 },
|
||||
{ id: 2, pid: 102 },
|
||||
];
|
||||
},
|
||||
nativeUpdate: async (where, values) => {
|
||||
const query = {
|
||||
where: where,
|
||||
};
|
||||
return updates.push({ values, query });
|
||||
},
|
||||
},
|
||||
SubscriptionStatus: { idle: 1 },
|
||||
},
|
||||
@@ -102,7 +112,6 @@ function subscriptionFixture(killTask) {
|
||||
}).default;
|
||||
return { service: new Service(logger, {}, {}, {}, {}), updates };
|
||||
}
|
||||
|
||||
test('batch subscription stop preserves failed items and reports failure while stopping others', async () => {
|
||||
const calls = [];
|
||||
const denied = Error('EPERM');
|
||||
@@ -119,7 +128,6 @@ test('batch subscription stop preserves failed items and reports failure while s
|
||||
assert.equal(f.updates[0].query.where.id, 2);
|
||||
assert.equal(f.updates[0].values.status, 1);
|
||||
});
|
||||
|
||||
function scriptFixture(killTask, pid = 100) {
|
||||
const Service = load('back/services/script.ts', {
|
||||
'../config': { scriptPath: '/scripts' },
|
||||
@@ -133,7 +141,6 @@ function scriptFixture(killTask, pid = 100) {
|
||||
}).default;
|
||||
return new Service(logger, {}, {}, {});
|
||||
}
|
||||
|
||||
test('script stop propagates failure instead of returning success', async () => {
|
||||
const denied = Error('EPERM');
|
||||
const service = scriptFixture(async () => {
|
||||
@@ -144,7 +151,6 @@ test('script stop propagates failure instead of returning success', async () =>
|
||||
(e) => e === denied,
|
||||
);
|
||||
});
|
||||
|
||||
test('script stop verifies exit and treats an absent queued process as already stopped', async () => {
|
||||
const calls = [];
|
||||
const service = scriptFixture(async (...args) => calls.push(args), undefined);
|
||||
|
||||
+141
-51
@@ -1,3 +1,6 @@
|
||||
const { DatabaseClient } = require('../../back/data/databaseClient');
|
||||
const { repository, initialize } = require('../helpers/repository-fixture.cjs');
|
||||
const { orderBy } = require('../../back/data/queries');
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const path = require('node:path');
|
||||
@@ -8,34 +11,16 @@ const load = require('../helpers/load-security-module.cjs');
|
||||
async function fixture(t, onKill = async () => {}) {
|
||||
const logPath = fs.mkdtempSync(path.join(os.tmpdir(), 'ql-stop-log-'));
|
||||
t.after(() => fs.rmSync(logPath, { recursive: true, force: true }));
|
||||
const db = new Sequelize({
|
||||
dialect: 'sqlite',
|
||||
storage: ':memory:',
|
||||
logging: false,
|
||||
});
|
||||
const db = new DatabaseClient(':memory:');
|
||||
t.after(() => db.close());
|
||||
const crons = db.define('Cron', {
|
||||
command: DataTypes.STRING,
|
||||
status: DataTypes.INTEGER,
|
||||
pid: DataTypes.INTEGER,
|
||||
log_path: DataTypes.STRING,
|
||||
queued_token: DataTypes.STRING,
|
||||
last_execution_time: DataTypes.INTEGER,
|
||||
last_running_time: DataTypes.INTEGER,
|
||||
});
|
||||
const instances = db.define('Instance', {
|
||||
cron_id: DataTypes.INTEGER,
|
||||
pid: DataTypes.INTEGER,
|
||||
status: DataTypes.INTEGER,
|
||||
exit_code: DataTypes.INTEGER,
|
||||
finished_at: DataTypes.INTEGER,
|
||||
});
|
||||
await db.sync();
|
||||
await crons.bulkCreate([
|
||||
const crons = repository(db, 'Crontab');
|
||||
const instances = repository(db, 'RunningInstance');
|
||||
await initialize(db);
|
||||
await crons.insertMany([
|
||||
{ id: 1, command: 'task one.js', status: 0, pid: 101, log_path: 'one.log' },
|
||||
{ id: 2, command: 'task two.js', status: 0, pid: 201, log_path: 'two.log' },
|
||||
]);
|
||||
await instances.bulkCreate([
|
||||
await instances.insertMany([
|
||||
{ id: 1, cron_id: 1, pid: 101, status: 0 },
|
||||
{ id: 2, cron_id: 1, pid: 102, status: 0 },
|
||||
{ id: 3, cron_id: 1, pid: 99, status: 1, exit_code: 0 },
|
||||
@@ -74,7 +59,10 @@ async function fixture(t, onKill = async () => {}) {
|
||||
'../shared/childProcess': require('../../back/shared/childProcess'),
|
||||
}).default;
|
||||
service = new CronService({ info() {}, error() {} });
|
||||
service.getDb = ({ id }) => crons.findByPk(id);
|
||||
service.getDb = ({ id }) =>
|
||||
crons.findOne({
|
||||
id: id,
|
||||
});
|
||||
return { service, instances, crons, killed };
|
||||
}
|
||||
async function report(service, pid, code) {
|
||||
@@ -94,7 +82,12 @@ test('stop wins when shell exit writes error or success during termination, pres
|
||||
await report(service, 102, 0);
|
||||
});
|
||||
await f.service.stop([1]);
|
||||
const rows = await f.instances.findAll({ order: [['id', 'ASC']], raw: true });
|
||||
const rows = await f.instances.find(
|
||||
{},
|
||||
{
|
||||
orderBy: orderBy([['id', 'ASC']]),
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
rows.map((r) => [r.id, r.status, r.exit_code]),
|
||||
[
|
||||
@@ -110,62 +103,159 @@ test('late shell completion cannot overwrite stopped instances', async (t) => {
|
||||
const f = await fixture(t);
|
||||
await f.service.stop([1]);
|
||||
await report(f.service, 101, 143);
|
||||
assert.equal((await f.instances.findByPk(1)).status, 2);
|
||||
assert.equal((await f.instances.findByPk(2)).status, 2);
|
||||
assert.equal(
|
||||
(
|
||||
await f.instances.findOne({
|
||||
id: 1,
|
||||
})
|
||||
).status,
|
||||
2,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await f.instances.findOne({
|
||||
id: 2,
|
||||
})
|
||||
).status,
|
||||
2,
|
||||
);
|
||||
});
|
||||
test('rows created after the stop snapshot are not relabelled', async (t) => {
|
||||
const f = await fixture(t, async ({ instances }) => {
|
||||
if (!(await instances.findByPk(6)))
|
||||
await instances.create({ id: 6, cron_id: 1, pid: 103, status: 0 });
|
||||
if (
|
||||
!(await instances.findOne({
|
||||
id: 6,
|
||||
}))
|
||||
)
|
||||
await instances.insert({ id: 6, cron_id: 1, pid: 103, status: 0 });
|
||||
});
|
||||
await f.service.stop([1]);
|
||||
assert.equal((await f.instances.findByPk(6)).status, 0);
|
||||
assert.equal((await f.instances.findByPk(1)).status, 2);
|
||||
assert.equal(
|
||||
(
|
||||
await f.instances.findOne({
|
||||
id: 6,
|
||||
})
|
||||
).status,
|
||||
0,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await f.instances.findOne({
|
||||
id: 1,
|
||||
})
|
||||
).status,
|
||||
2,
|
||||
);
|
||||
});
|
||||
test('repeated stop does not rewrite historical rows or their finished timestamps', async (t) => {
|
||||
const f = await fixture(t);
|
||||
await f.service.stop([1]);
|
||||
const before = await f.instances.findAll({
|
||||
raw: true,
|
||||
order: [['id', 'ASC']],
|
||||
});
|
||||
const before = await f.instances.find(
|
||||
{},
|
||||
{
|
||||
orderBy: orderBy([['id', 'ASC']]),
|
||||
},
|
||||
);
|
||||
await f.service.stop([1]);
|
||||
assert.deepEqual(
|
||||
await f.instances.findAll({ raw: true, order: [['id', 'ASC']] }),
|
||||
await f.instances.find(
|
||||
{},
|
||||
{
|
||||
orderBy: orderBy([['id', 'ASC']]),
|
||||
},
|
||||
),
|
||||
before,
|
||||
);
|
||||
});
|
||||
test('batch stop captures running instances from every requested cron', async (t) => {
|
||||
const f = await fixture(t);
|
||||
await f.service.stop([1, 2]);
|
||||
assert.equal((await f.instances.findByPk(5)).status, 2);
|
||||
assert.equal((await f.instances.findByPk(3)).status, 1);
|
||||
assert.equal(
|
||||
(
|
||||
await f.instances.findOne({
|
||||
id: 5,
|
||||
})
|
||||
).status,
|
||||
2,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await f.instances.findOne({
|
||||
id: 3,
|
||||
})
|
||||
).status,
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
test('stop signals every snapshotted PID exactly once and preserves a later running instance', async (t) => {
|
||||
const f = await fixture(t, async ({ instances, crons }) => {
|
||||
if (!(await instances.findByPk(6))) {
|
||||
await instances.create({ id: 6, cron_id: 1, pid: 103, status: 0 });
|
||||
await crons.update(
|
||||
if (
|
||||
!(await instances.findOne({
|
||||
id: 6,
|
||||
}))
|
||||
) {
|
||||
await instances.insert({ id: 6, cron_id: 1, pid: 103, status: 0 });
|
||||
await crons.nativeUpdate(
|
||||
{ id: 1 },
|
||||
{ pid: 103, log_path: 'later.log', status: 0 },
|
||||
{ where: { id: 1 } },
|
||||
);
|
||||
}
|
||||
});
|
||||
await f.service.stop([1]);
|
||||
assert.deepEqual(f.killed, [101, 102]);
|
||||
assert.equal((await f.instances.findByPk(6)).status, 0);
|
||||
assert.equal((await f.crons.findByPk(1)).pid, 103);
|
||||
assert.equal((await f.crons.findByPk(1)).status, 0);
|
||||
assert.equal(
|
||||
(
|
||||
await f.instances.findOne({
|
||||
id: 6,
|
||||
})
|
||||
).status,
|
||||
0,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await f.crons.findOne({
|
||||
id: 1,
|
||||
})
|
||||
).pid,
|
||||
103,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await f.crons.findOne({
|
||||
id: 1,
|
||||
})
|
||||
).status,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('failed termination is not finalized as stopped', async (t) => {
|
||||
const f = await fixture(t, async ({ pid }) => {
|
||||
if (pid === 102) throw Error('still alive');
|
||||
});
|
||||
await f.service.stop([1]);
|
||||
assert.deepEqual(f.killed, [101, 102]);
|
||||
assert.equal((await f.instances.findByPk(1)).status, 2);
|
||||
assert.equal((await f.instances.findByPk(2)).status, 0);
|
||||
assert.equal((await f.crons.findByPk(1)).status, 0);
|
||||
assert.equal(
|
||||
(
|
||||
await f.instances.findOne({
|
||||
id: 1,
|
||||
})
|
||||
).status,
|
||||
2,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await f.instances.findOne({
|
||||
id: 2,
|
||||
})
|
||||
).status,
|
||||
0,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await f.crons.findOne({
|
||||
id: 1,
|
||||
})
|
||||
).status,
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user